Browse code

package builder: Move SpecDependencyGenerator class to SpecDeps.py

It is confusing to have the SpecDependencyGenerator class in
SpecData.py, while having just a main() function inside SpecDeps.py.
Fix this by moving SpecDependencyGenerator to SpecDeps.py

Change-Id: I6cc56c89a7762f8a9cd57cdda64a234db832b0f6
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/6039
Tested-by: gerrit-photon <photon-checkins@vmware.com>
Reviewed-by: Alexey Makhalov <amakhalov@vmware.com>
Reviewed-by: Ajay Kaher <akaher@vmware.com>
Reviewed-by: Srivatsa S. Bhat <srivatsab@vmware.com>

Srivatsa S. Bhat (VMware) authored on 2018/10/30 07:58:30
Showing 2 changed files
... ...
@@ -373,162 +373,3 @@ class SPECS(object):
373 373
         self.specData = SpecObjectsUtils(constants.logPath)
374 374
         self.specData.readSpecsAndConvertToSerializableObjects(constants.specPath)
375 375
 
376
-
377
-class SpecDependencyGenerator(object):
378
-
379
-    def __init__(self, logPath, logLevel):
380
-        self.logger = Logger.getLogger("Serializable Spec objects", logPath, logLevel)
381
-
382
-    def findTotalRequires(self, mapDependencies, depQue, parent):
383
-        while not depQue.empty():
384
-            specPkg = depQue.get()
385
-            try:
386
-                listRequiredPackages = SPECS.getData().getRequiresForPkg(specPkg)
387
-            except Exception as e:
388
-                self.logger.info("Caught Exception:"+str(e))
389
-                self.logger.info(specPkg + " is missing")
390
-                raise e
391
-
392
-            for depPkg in listRequiredPackages:
393
-                if depPkg in mapDependencies:
394
-                    if mapDependencies[depPkg] < mapDependencies[specPkg] + 1:
395
-                        mapDependencies[depPkg] = mapDependencies[specPkg] + 1
396
-                        parent[depPkg] = specPkg
397
-                        self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
398
-                else:
399
-                    mapDependencies[depPkg] = mapDependencies[specPkg] + 1
400
-                    parent[depPkg] = specPkg
401
-                    depQue.put(depPkg)
402
-
403
-    def getBasePackagesRequired(self, pkg):
404
-        listBasePackagesRequired=[]
405
-        listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg)
406
-        listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg))
407
-        for p in listPackagesRequired:
408
-            basePkg = SPECS.getData().getBasePkg(p)
409
-            if basePkg not in listBasePackagesRequired:
410
-                listBasePackagesRequired.append(basePkg)
411
-        return listBasePackagesRequired
412
-
413
-
414
-    def findTotalWhoNeeds(self, depList, whoNeeds):
415
-        while depList:
416
-            pkg = depList.pop(0)
417
-            for depPackage in SPECS.getData().getListPackages():
418
-                for version in SPECS.getData().getVersions(depPackage):
419
-                    depBasePkg = depPackage+"-"+version
420
-                    if depBasePkg in whoNeeds:
421
-                        continue
422
-                    if pkg in self.getBasePackagesRequired(depBasePkg):
423
-                        whoNeeds.append(depBasePkg)
424
-                        if depBasePkg not in depList:
425
-                            depList.append(depBasePkg)
426
-
427
-    def printTree(self, children, curParent, depth):
428
-        if curParent in children:
429
-            for child in children[curParent]:
430
-                self.logger.info("\t" * depth + child)
431
-                self.printTree(children, child, depth + 1)
432
-
433
-    def getAllPackageNames(self, jsonFilePath):
434
-        with open(jsonFilePath) as jsonData:
435
-            option_list_json = json.load(jsonData)
436
-            packages = option_list_json["packages"]
437
-            return packages
438
-
439
-    def updateLevels(self, mapDependencies, inPkg, parent, level):
440
-        listPackages = SPECS.getData().getPackagesForPkg(inPkg)
441
-        for depPkg in SPECS.getData().getRequiresForPkg(inPkg):
442
-            if depPkg in listPackages:
443
-                continue
444
-            if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:
445
-                mapDependencies[depPkg] = level + 1
446
-                parent[depPkg] = inPkg
447
-                self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
448
-
449
-    def calculateSpecDependency(self, inputPackages, mapDependencies, parent):
450
-        depQue = queue.Queue()
451
-        for package in inputPackages:
452
-            if SPECS.getData().isRPMPackage(package):
453
-                for version in SPECS.getData().getVersions(package):
454
-                    pkg = package+"-"+version
455
-                    if pkg not in mapDependencies:
456
-                        mapDependencies[pkg] = 0
457
-                        parent[pkg] = ""
458
-                        depQue.put(pkg)
459
-                        self.findTotalRequires(mapDependencies, depQue, parent)
460
-            else:
461
-                self.logger.info("Could not find spec for " + package)
462
-
463
-    def displayDependencies(self, displayOption, inputType, inputValue, allDeps, parent):
464
-        children = {}
465
-        sortedList = []
466
-        for elem in sorted(allDeps.items(), key=operator.itemgetter(1), reverse=True):
467
-            sortedList.append(elem[0])
468
-        # construct all children nodes
469
-        if displayOption == "tree":
470
-            for k, v in parent.iteritems():
471
-                children.setdefault(v, []).append(k)
472
-            if inputType == "json":
473
-                self.logger.info("Dependency Mappings for {}".format(inputValue) + " :")
474
-                self.logger.info("-" * 52 + " {}".format(children))
475
-                self.logger.info("-" * 52)
476
-            if "" in children:
477
-                for child in children[""]:
478
-                    self.logger.info(child)
479
-                    self.printTree(children, child, 1)
480
-                self.logger.info("*" * 18 + " {} ".format(len(sortedList)) +
481
-                      "packages in total " + "*" * 18)
482
-            else:
483
-                if inputType == "pkg" and len(children) > 0:
484
-                    self.logger.info("cyclic dependency detected, mappings: \n", children)
485
-
486
-        # To display a flat list of all packages
487
-        elif displayOption == "list":
488
-            self.logger.info(sortedList)
489
-
490
-        # To generate a new JSON file based on given input json file
491
-        elif displayOption == "json" and inputType == "json":
492
-            d = {'packages': sortedList}
493
-            with open(inputValue, 'w') as outfile:
494
-                json.dump(d, outfile)
495
-
496
-        return sortedList
497
-
498
-    def process(self, inputType, inputValue, displayOption, outputFile=None):
499
-        whoNeedsList = []
500
-        inputPackages = []
501
-        whatNeedsBuild = []
502
-        mapDependencies = {}
503
-        parent = {}
504
-        if inputType == "pkg" or inputType == "json":
505
-            if inputType == "pkg":
506
-                inputPackages.append(inputValue)
507
-            else:
508
-                inputPackages = self.getAllPackageNames(inputValue)
509
-
510
-            self.calculateSpecDependency(inputPackages, mapDependencies, parent)
511
-            if outputFile is not None:
512
-                return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent)
513
-            else:
514
-                return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent)
515
-        elif inputType == "get-upward-deps":
516
-            depList = []
517
-            for specFile in inputValue.split(":"):
518
-                if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
519
-                    specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
520
-                    whoNeedsList.append(specObj.name+"-"+specObj.version)
521
-                    depList.append(specObj.name+"-"+specObj.version)
522
-            self.findTotalWhoNeeds(depList, whoNeedsList)
523
-            return whoNeedsList
524
-
525
-        elif inputType == "who-needs":
526
-            for depPackage in SPECS.getData().mapPackageToSpec:
527
-                pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue)
528
-                for version in SPECS.getData().getVersions(depPackage):
529
-                    depPkg = depPackage+"-"+version
530
-                    self.logger.info(depPkg)
531
-                    if pkg in SPECS.getData().getRequiresForPkg(depPkg):
532
-                        whoNeedsList.append(depPkg)
533
-            self.logger.info(whoNeedsList)
534
-            return whoNeedsList
... ...
@@ -5,10 +5,13 @@
5 5
 #    Author: Harish Udaiya Kumar <hudaiyakumar@vmware.com>
6 6
 import sys
7 7
 import os
8
+import json
9
+import queue
10
+import operator
8 11
 from argparse import ArgumentParser
9 12
 import shutil
10 13
 import traceback
11
-from SpecData import SpecDependencyGenerator, SPECS
14
+from SpecData import SPECS
12 15
 from jsonwrapper import JsonWrapper
13 16
 from constants import constants
14 17
 from CommandUtils import CommandUtils
... ...
@@ -21,6 +24,166 @@ SPEC_FILE_DIR = "../../SPECS"
21 21
 LOG_FILE_DIR = "../../stage/LOGS"
22 22
 
23 23
 
24
+class SpecDependencyGenerator(object):
25
+
26
+    def __init__(self, logPath, logLevel):
27
+        self.logger = Logger.getLogger("Serializable Spec objects", logPath, logLevel)
28
+
29
+    def findTotalRequires(self, mapDependencies, depQue, parent):
30
+        while not depQue.empty():
31
+            specPkg = depQue.get()
32
+            try:
33
+                listRequiredPackages = SPECS.getData().getRequiresForPkg(specPkg)
34
+            except Exception as e:
35
+                self.logger.info("Caught Exception:"+str(e))
36
+                self.logger.info(specPkg + " is missing")
37
+                raise e
38
+
39
+            for depPkg in listRequiredPackages:
40
+                if depPkg in mapDependencies:
41
+                    if mapDependencies[depPkg] < mapDependencies[specPkg] + 1:
42
+                        mapDependencies[depPkg] = mapDependencies[specPkg] + 1
43
+                        parent[depPkg] = specPkg
44
+                        self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
45
+                else:
46
+                    mapDependencies[depPkg] = mapDependencies[specPkg] + 1
47
+                    parent[depPkg] = specPkg
48
+                    depQue.put(depPkg)
49
+
50
+    def getBasePackagesRequired(self, pkg):
51
+        listBasePackagesRequired=[]
52
+        listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg)
53
+        listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg))
54
+        for p in listPackagesRequired:
55
+            basePkg = SPECS.getData().getBasePkg(p)
56
+            if basePkg not in listBasePackagesRequired:
57
+                listBasePackagesRequired.append(basePkg)
58
+        return listBasePackagesRequired
59
+
60
+
61
+    def findTotalWhoNeeds(self, depList, whoNeeds):
62
+        while depList:
63
+            pkg = depList.pop(0)
64
+            for depPackage in SPECS.getData().getListPackages():
65
+                for version in SPECS.getData().getVersions(depPackage):
66
+                    depBasePkg = depPackage+"-"+version
67
+                    if depBasePkg in whoNeeds:
68
+                        continue
69
+                    if pkg in self.getBasePackagesRequired(depBasePkg):
70
+                        whoNeeds.append(depBasePkg)
71
+                        if depBasePkg not in depList:
72
+                            depList.append(depBasePkg)
73
+
74
+    def printTree(self, children, curParent, depth):
75
+        if curParent in children:
76
+            for child in children[curParent]:
77
+                self.logger.info("\t" * depth + child)
78
+                self.printTree(children, child, depth + 1)
79
+
80
+    def getAllPackageNames(self, jsonFilePath):
81
+        with open(jsonFilePath) as jsonData:
82
+            option_list_json = json.load(jsonData)
83
+            packages = option_list_json["packages"]
84
+            return packages
85
+
86
+    def updateLevels(self, mapDependencies, inPkg, parent, level):
87
+        listPackages = SPECS.getData().getPackagesForPkg(inPkg)
88
+        for depPkg in SPECS.getData().getRequiresForPkg(inPkg):
89
+            if depPkg in listPackages:
90
+                continue
91
+            if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:
92
+                mapDependencies[depPkg] = level + 1
93
+                parent[depPkg] = inPkg
94
+                self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
95
+
96
+    def calculateSpecDependency(self, inputPackages, mapDependencies, parent):
97
+        depQue = queue.Queue()
98
+        for package in inputPackages:
99
+            if SPECS.getData().isRPMPackage(package):
100
+                for version in SPECS.getData().getVersions(package):
101
+                    pkg = package+"-"+version
102
+                    if pkg not in mapDependencies:
103
+                        mapDependencies[pkg] = 0
104
+                        parent[pkg] = ""
105
+                        depQue.put(pkg)
106
+                        self.findTotalRequires(mapDependencies, depQue, parent)
107
+            else:
108
+                self.logger.info("Could not find spec for " + package)
109
+
110
+    def displayDependencies(self, displayOption, inputType, inputValue, allDeps, parent):
111
+        children = {}
112
+        sortedList = []
113
+        for elem in sorted(allDeps.items(), key=operator.itemgetter(1), reverse=True):
114
+            sortedList.append(elem[0])
115
+        # construct all children nodes
116
+        if displayOption == "tree":
117
+            for k, v in parent.iteritems():
118
+                children.setdefault(v, []).append(k)
119
+            if inputType == "json":
120
+                self.logger.info("Dependency Mappings for {}".format(inputValue) + " :")
121
+                self.logger.info("-" * 52 + " {}".format(children))
122
+                self.logger.info("-" * 52)
123
+            if "" in children:
124
+                for child in children[""]:
125
+                    self.logger.info(child)
126
+                    self.printTree(children, child, 1)
127
+                self.logger.info("*" * 18 + " {} ".format(len(sortedList)) +
128
+                      "packages in total " + "*" * 18)
129
+            else:
130
+                if inputType == "pkg" and len(children) > 0:
131
+                    self.logger.info("cyclic dependency detected, mappings: \n", children)
132
+
133
+        # To display a flat list of all packages
134
+        elif displayOption == "list":
135
+            self.logger.info(sortedList)
136
+
137
+        # To generate a new JSON file based on given input json file
138
+        elif displayOption == "json" and inputType == "json":
139
+            d = {'packages': sortedList}
140
+            with open(inputValue, 'w') as outfile:
141
+                json.dump(d, outfile)
142
+
143
+        return sortedList
144
+
145
+    def process(self, inputType, inputValue, displayOption, outputFile=None):
146
+        whoNeedsList = []
147
+        inputPackages = []
148
+        whatNeedsBuild = []
149
+        mapDependencies = {}
150
+        parent = {}
151
+        if inputType == "pkg" or inputType == "json":
152
+            if inputType == "pkg":
153
+                inputPackages.append(inputValue)
154
+            else:
155
+                inputPackages = self.getAllPackageNames(inputValue)
156
+
157
+            self.calculateSpecDependency(inputPackages, mapDependencies, parent)
158
+            if outputFile is not None:
159
+                return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent)
160
+            else:
161
+                return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent)
162
+        elif inputType == "get-upward-deps":
163
+            depList = []
164
+            for specFile in inputValue.split(":"):
165
+                if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
166
+                    specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
167
+                    whoNeedsList.append(specObj.name+"-"+specObj.version)
168
+                    depList.append(specObj.name+"-"+specObj.version)
169
+            self.findTotalWhoNeeds(depList, whoNeedsList)
170
+            return whoNeedsList
171
+
172
+        elif inputType == "who-needs":
173
+            for depPackage in SPECS.getData().mapPackageToSpec:
174
+                pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue)
175
+                for version in SPECS.getData().getVersions(depPackage):
176
+                    depPkg = depPackage+"-"+version
177
+                    self.logger.info(depPkg)
178
+                    if pkg in SPECS.getData().getRequiresForPkg(depPkg):
179
+                        whoNeedsList.append(depPkg)
180
+            self.logger.info(whoNeedsList)
181
+            return whoNeedsList
182
+
183
+
24 184
 def main():
25 185
     usage = "Usage: %prog [options]"
26 186
     parser = ArgumentParser(usage)