It is confusing to have the SpecDependencyGenerator class in
SpecData.py, while having just a main() function inside SpecDeps.py.
Fix this by moving SpecDependencyGenerator to SpecDeps.py
Change-Id: I6cc56c89a7762f8a9cd57cdda64a234db832b0f6
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/6039
Tested-by: gerrit-photon <photon-checkins@vmware.com>
Reviewed-by: Alexey Makhalov <amakhalov@vmware.com>
Reviewed-by: Ajay Kaher <akaher@vmware.com>
Reviewed-by: Srivatsa S. Bhat <srivatsab@vmware.com>
... | ... |
@@ -373,162 +373,3 @@ class SPECS(object): |
373 | 373 |
self.specData = SpecObjectsUtils(constants.logPath) |
374 | 374 |
self.specData.readSpecsAndConvertToSerializableObjects(constants.specPath) |
375 | 375 |
|
376 |
- |
|
377 |
-class SpecDependencyGenerator(object): |
|
378 |
- |
|
379 |
- def __init__(self, logPath, logLevel): |
|
380 |
- self.logger = Logger.getLogger("Serializable Spec objects", logPath, logLevel) |
|
381 |
- |
|
382 |
- def findTotalRequires(self, mapDependencies, depQue, parent): |
|
383 |
- while not depQue.empty(): |
|
384 |
- specPkg = depQue.get() |
|
385 |
- try: |
|
386 |
- listRequiredPackages = SPECS.getData().getRequiresForPkg(specPkg) |
|
387 |
- except Exception as e: |
|
388 |
- self.logger.info("Caught Exception:"+str(e)) |
|
389 |
- self.logger.info(specPkg + " is missing") |
|
390 |
- raise e |
|
391 |
- |
|
392 |
- for depPkg in listRequiredPackages: |
|
393 |
- if depPkg in mapDependencies: |
|
394 |
- if mapDependencies[depPkg] < mapDependencies[specPkg] + 1: |
|
395 |
- mapDependencies[depPkg] = mapDependencies[specPkg] + 1 |
|
396 |
- parent[depPkg] = specPkg |
|
397 |
- self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg]) |
|
398 |
- else: |
|
399 |
- mapDependencies[depPkg] = mapDependencies[specPkg] + 1 |
|
400 |
- parent[depPkg] = specPkg |
|
401 |
- depQue.put(depPkg) |
|
402 |
- |
|
403 |
- def getBasePackagesRequired(self, pkg): |
|
404 |
- listBasePackagesRequired=[] |
|
405 |
- listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg) |
|
406 |
- listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg)) |
|
407 |
- for p in listPackagesRequired: |
|
408 |
- basePkg = SPECS.getData().getBasePkg(p) |
|
409 |
- if basePkg not in listBasePackagesRequired: |
|
410 |
- listBasePackagesRequired.append(basePkg) |
|
411 |
- return listBasePackagesRequired |
|
412 |
- |
|
413 |
- |
|
414 |
- def findTotalWhoNeeds(self, depList, whoNeeds): |
|
415 |
- while depList: |
|
416 |
- pkg = depList.pop(0) |
|
417 |
- for depPackage in SPECS.getData().getListPackages(): |
|
418 |
- for version in SPECS.getData().getVersions(depPackage): |
|
419 |
- depBasePkg = depPackage+"-"+version |
|
420 |
- if depBasePkg in whoNeeds: |
|
421 |
- continue |
|
422 |
- if pkg in self.getBasePackagesRequired(depBasePkg): |
|
423 |
- whoNeeds.append(depBasePkg) |
|
424 |
- if depBasePkg not in depList: |
|
425 |
- depList.append(depBasePkg) |
|
426 |
- |
|
427 |
- def printTree(self, children, curParent, depth): |
|
428 |
- if curParent in children: |
|
429 |
- for child in children[curParent]: |
|
430 |
- self.logger.info("\t" * depth + child) |
|
431 |
- self.printTree(children, child, depth + 1) |
|
432 |
- |
|
433 |
- def getAllPackageNames(self, jsonFilePath): |
|
434 |
- with open(jsonFilePath) as jsonData: |
|
435 |
- option_list_json = json.load(jsonData) |
|
436 |
- packages = option_list_json["packages"] |
|
437 |
- return packages |
|
438 |
- |
|
439 |
- def updateLevels(self, mapDependencies, inPkg, parent, level): |
|
440 |
- listPackages = SPECS.getData().getPackagesForPkg(inPkg) |
|
441 |
- for depPkg in SPECS.getData().getRequiresForPkg(inPkg): |
|
442 |
- if depPkg in listPackages: |
|
443 |
- continue |
|
444 |
- if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1: |
|
445 |
- mapDependencies[depPkg] = level + 1 |
|
446 |
- parent[depPkg] = inPkg |
|
447 |
- self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg]) |
|
448 |
- |
|
449 |
- def calculateSpecDependency(self, inputPackages, mapDependencies, parent): |
|
450 |
- depQue = queue.Queue() |
|
451 |
- for package in inputPackages: |
|
452 |
- if SPECS.getData().isRPMPackage(package): |
|
453 |
- for version in SPECS.getData().getVersions(package): |
|
454 |
- pkg = package+"-"+version |
|
455 |
- if pkg not in mapDependencies: |
|
456 |
- mapDependencies[pkg] = 0 |
|
457 |
- parent[pkg] = "" |
|
458 |
- depQue.put(pkg) |
|
459 |
- self.findTotalRequires(mapDependencies, depQue, parent) |
|
460 |
- else: |
|
461 |
- self.logger.info("Could not find spec for " + package) |
|
462 |
- |
|
463 |
- def displayDependencies(self, displayOption, inputType, inputValue, allDeps, parent): |
|
464 |
- children = {} |
|
465 |
- sortedList = [] |
|
466 |
- for elem in sorted(allDeps.items(), key=operator.itemgetter(1), reverse=True): |
|
467 |
- sortedList.append(elem[0]) |
|
468 |
- # construct all children nodes |
|
469 |
- if displayOption == "tree": |
|
470 |
- for k, v in parent.iteritems(): |
|
471 |
- children.setdefault(v, []).append(k) |
|
472 |
- if inputType == "json": |
|
473 |
- self.logger.info("Dependency Mappings for {}".format(inputValue) + " :") |
|
474 |
- self.logger.info("-" * 52 + " {}".format(children)) |
|
475 |
- self.logger.info("-" * 52) |
|
476 |
- if "" in children: |
|
477 |
- for child in children[""]: |
|
478 |
- self.logger.info(child) |
|
479 |
- self.printTree(children, child, 1) |
|
480 |
- self.logger.info("*" * 18 + " {} ".format(len(sortedList)) + |
|
481 |
- "packages in total " + "*" * 18) |
|
482 |
- else: |
|
483 |
- if inputType == "pkg" and len(children) > 0: |
|
484 |
- self.logger.info("cyclic dependency detected, mappings: \n", children) |
|
485 |
- |
|
486 |
- # To display a flat list of all packages |
|
487 |
- elif displayOption == "list": |
|
488 |
- self.logger.info(sortedList) |
|
489 |
- |
|
490 |
- # To generate a new JSON file based on given input json file |
|
491 |
- elif displayOption == "json" and inputType == "json": |
|
492 |
- d = {'packages': sortedList} |
|
493 |
- with open(inputValue, 'w') as outfile: |
|
494 |
- json.dump(d, outfile) |
|
495 |
- |
|
496 |
- return sortedList |
|
497 |
- |
|
498 |
- def process(self, inputType, inputValue, displayOption, outputFile=None): |
|
499 |
- whoNeedsList = [] |
|
500 |
- inputPackages = [] |
|
501 |
- whatNeedsBuild = [] |
|
502 |
- mapDependencies = {} |
|
503 |
- parent = {} |
|
504 |
- if inputType == "pkg" or inputType == "json": |
|
505 |
- if inputType == "pkg": |
|
506 |
- inputPackages.append(inputValue) |
|
507 |
- else: |
|
508 |
- inputPackages = self.getAllPackageNames(inputValue) |
|
509 |
- |
|
510 |
- self.calculateSpecDependency(inputPackages, mapDependencies, parent) |
|
511 |
- if outputFile is not None: |
|
512 |
- return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent) |
|
513 |
- else: |
|
514 |
- return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent) |
|
515 |
- elif inputType == "get-upward-deps": |
|
516 |
- depList = [] |
|
517 |
- for specFile in inputValue.split(":"): |
|
518 |
- if specFile in SPECS.getData().mapSpecFileNameToSpecObj: |
|
519 |
- specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile] |
|
520 |
- whoNeedsList.append(specObj.name+"-"+specObj.version) |
|
521 |
- depList.append(specObj.name+"-"+specObj.version) |
|
522 |
- self.findTotalWhoNeeds(depList, whoNeedsList) |
|
523 |
- return whoNeedsList |
|
524 |
- |
|
525 |
- elif inputType == "who-needs": |
|
526 |
- for depPackage in SPECS.getData().mapPackageToSpec: |
|
527 |
- pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue) |
|
528 |
- for version in SPECS.getData().getVersions(depPackage): |
|
529 |
- depPkg = depPackage+"-"+version |
|
530 |
- self.logger.info(depPkg) |
|
531 |
- if pkg in SPECS.getData().getRequiresForPkg(depPkg): |
|
532 |
- whoNeedsList.append(depPkg) |
|
533 |
- self.logger.info(whoNeedsList) |
|
534 |
- return whoNeedsList |
... | ... |
@@ -5,10 +5,13 @@ |
5 | 5 |
# Author: Harish Udaiya Kumar <hudaiyakumar@vmware.com> |
6 | 6 |
import sys |
7 | 7 |
import os |
8 |
+import json |
|
9 |
+import queue |
|
10 |
+import operator |
|
8 | 11 |
from argparse import ArgumentParser |
9 | 12 |
import shutil |
10 | 13 |
import traceback |
11 |
-from SpecData import SpecDependencyGenerator, SPECS |
|
14 |
+from SpecData import SPECS |
|
12 | 15 |
from jsonwrapper import JsonWrapper |
13 | 16 |
from constants import constants |
14 | 17 |
from CommandUtils import CommandUtils |
... | ... |
@@ -21,6 +24,166 @@ SPEC_FILE_DIR = "../../SPECS" |
21 | 21 |
LOG_FILE_DIR = "../../stage/LOGS" |
22 | 22 |
|
23 | 23 |
|
24 |
+class SpecDependencyGenerator(object): |
|
25 |
+ |
|
26 |
+ def __init__(self, logPath, logLevel): |
|
27 |
+ self.logger = Logger.getLogger("Serializable Spec objects", logPath, logLevel) |
|
28 |
+ |
|
29 |
+ def findTotalRequires(self, mapDependencies, depQue, parent): |
|
30 |
+ while not depQue.empty(): |
|
31 |
+ specPkg = depQue.get() |
|
32 |
+ try: |
|
33 |
+ listRequiredPackages = SPECS.getData().getRequiresForPkg(specPkg) |
|
34 |
+ except Exception as e: |
|
35 |
+ self.logger.info("Caught Exception:"+str(e)) |
|
36 |
+ self.logger.info(specPkg + " is missing") |
|
37 |
+ raise e |
|
38 |
+ |
|
39 |
+ for depPkg in listRequiredPackages: |
|
40 |
+ if depPkg in mapDependencies: |
|
41 |
+ if mapDependencies[depPkg] < mapDependencies[specPkg] + 1: |
|
42 |
+ mapDependencies[depPkg] = mapDependencies[specPkg] + 1 |
|
43 |
+ parent[depPkg] = specPkg |
|
44 |
+ self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg]) |
|
45 |
+ else: |
|
46 |
+ mapDependencies[depPkg] = mapDependencies[specPkg] + 1 |
|
47 |
+ parent[depPkg] = specPkg |
|
48 |
+ depQue.put(depPkg) |
|
49 |
+ |
|
50 |
+ def getBasePackagesRequired(self, pkg): |
|
51 |
+ listBasePackagesRequired=[] |
|
52 |
+ listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg) |
|
53 |
+ listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg)) |
|
54 |
+ for p in listPackagesRequired: |
|
55 |
+ basePkg = SPECS.getData().getBasePkg(p) |
|
56 |
+ if basePkg not in listBasePackagesRequired: |
|
57 |
+ listBasePackagesRequired.append(basePkg) |
|
58 |
+ return listBasePackagesRequired |
|
59 |
+ |
|
60 |
+ |
|
61 |
+ def findTotalWhoNeeds(self, depList, whoNeeds): |
|
62 |
+ while depList: |
|
63 |
+ pkg = depList.pop(0) |
|
64 |
+ for depPackage in SPECS.getData().getListPackages(): |
|
65 |
+ for version in SPECS.getData().getVersions(depPackage): |
|
66 |
+ depBasePkg = depPackage+"-"+version |
|
67 |
+ if depBasePkg in whoNeeds: |
|
68 |
+ continue |
|
69 |
+ if pkg in self.getBasePackagesRequired(depBasePkg): |
|
70 |
+ whoNeeds.append(depBasePkg) |
|
71 |
+ if depBasePkg not in depList: |
|
72 |
+ depList.append(depBasePkg) |
|
73 |
+ |
|
74 |
+ def printTree(self, children, curParent, depth): |
|
75 |
+ if curParent in children: |
|
76 |
+ for child in children[curParent]: |
|
77 |
+ self.logger.info("\t" * depth + child) |
|
78 |
+ self.printTree(children, child, depth + 1) |
|
79 |
+ |
|
80 |
+ def getAllPackageNames(self, jsonFilePath): |
|
81 |
+ with open(jsonFilePath) as jsonData: |
|
82 |
+ option_list_json = json.load(jsonData) |
|
83 |
+ packages = option_list_json["packages"] |
|
84 |
+ return packages |
|
85 |
+ |
|
86 |
+ def updateLevels(self, mapDependencies, inPkg, parent, level): |
|
87 |
+ listPackages = SPECS.getData().getPackagesForPkg(inPkg) |
|
88 |
+ for depPkg in SPECS.getData().getRequiresForPkg(inPkg): |
|
89 |
+ if depPkg in listPackages: |
|
90 |
+ continue |
|
91 |
+ if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1: |
|
92 |
+ mapDependencies[depPkg] = level + 1 |
|
93 |
+ parent[depPkg] = inPkg |
|
94 |
+ self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg]) |
|
95 |
+ |
|
96 |
+ def calculateSpecDependency(self, inputPackages, mapDependencies, parent): |
|
97 |
+ depQue = queue.Queue() |
|
98 |
+ for package in inputPackages: |
|
99 |
+ if SPECS.getData().isRPMPackage(package): |
|
100 |
+ for version in SPECS.getData().getVersions(package): |
|
101 |
+ pkg = package+"-"+version |
|
102 |
+ if pkg not in mapDependencies: |
|
103 |
+ mapDependencies[pkg] = 0 |
|
104 |
+ parent[pkg] = "" |
|
105 |
+ depQue.put(pkg) |
|
106 |
+ self.findTotalRequires(mapDependencies, depQue, parent) |
|
107 |
+ else: |
|
108 |
+ self.logger.info("Could not find spec for " + package) |
|
109 |
+ |
|
110 |
+ def displayDependencies(self, displayOption, inputType, inputValue, allDeps, parent): |
|
111 |
+ children = {} |
|
112 |
+ sortedList = [] |
|
113 |
+ for elem in sorted(allDeps.items(), key=operator.itemgetter(1), reverse=True): |
|
114 |
+ sortedList.append(elem[0]) |
|
115 |
+ # construct all children nodes |
|
116 |
+ if displayOption == "tree": |
|
117 |
+ for k, v in parent.iteritems(): |
|
118 |
+ children.setdefault(v, []).append(k) |
|
119 |
+ if inputType == "json": |
|
120 |
+ self.logger.info("Dependency Mappings for {}".format(inputValue) + " :") |
|
121 |
+ self.logger.info("-" * 52 + " {}".format(children)) |
|
122 |
+ self.logger.info("-" * 52) |
|
123 |
+ if "" in children: |
|
124 |
+ for child in children[""]: |
|
125 |
+ self.logger.info(child) |
|
126 |
+ self.printTree(children, child, 1) |
|
127 |
+ self.logger.info("*" * 18 + " {} ".format(len(sortedList)) + |
|
128 |
+ "packages in total " + "*" * 18) |
|
129 |
+ else: |
|
130 |
+ if inputType == "pkg" and len(children) > 0: |
|
131 |
+ self.logger.info("cyclic dependency detected, mappings: \n", children) |
|
132 |
+ |
|
133 |
+ # To display a flat list of all packages |
|
134 |
+ elif displayOption == "list": |
|
135 |
+ self.logger.info(sortedList) |
|
136 |
+ |
|
137 |
+ # To generate a new JSON file based on given input json file |
|
138 |
+ elif displayOption == "json" and inputType == "json": |
|
139 |
+ d = {'packages': sortedList} |
|
140 |
+ with open(inputValue, 'w') as outfile: |
|
141 |
+ json.dump(d, outfile) |
|
142 |
+ |
|
143 |
+ return sortedList |
|
144 |
+ |
|
145 |
+ def process(self, inputType, inputValue, displayOption, outputFile=None): |
|
146 |
+ whoNeedsList = [] |
|
147 |
+ inputPackages = [] |
|
148 |
+ whatNeedsBuild = [] |
|
149 |
+ mapDependencies = {} |
|
150 |
+ parent = {} |
|
151 |
+ if inputType == "pkg" or inputType == "json": |
|
152 |
+ if inputType == "pkg": |
|
153 |
+ inputPackages.append(inputValue) |
|
154 |
+ else: |
|
155 |
+ inputPackages = self.getAllPackageNames(inputValue) |
|
156 |
+ |
|
157 |
+ self.calculateSpecDependency(inputPackages, mapDependencies, parent) |
|
158 |
+ if outputFile is not None: |
|
159 |
+ return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent) |
|
160 |
+ else: |
|
161 |
+ return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent) |
|
162 |
+ elif inputType == "get-upward-deps": |
|
163 |
+ depList = [] |
|
164 |
+ for specFile in inputValue.split(":"): |
|
165 |
+ if specFile in SPECS.getData().mapSpecFileNameToSpecObj: |
|
166 |
+ specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile] |
|
167 |
+ whoNeedsList.append(specObj.name+"-"+specObj.version) |
|
168 |
+ depList.append(specObj.name+"-"+specObj.version) |
|
169 |
+ self.findTotalWhoNeeds(depList, whoNeedsList) |
|
170 |
+ return whoNeedsList |
|
171 |
+ |
|
172 |
+ elif inputType == "who-needs": |
|
173 |
+ for depPackage in SPECS.getData().mapPackageToSpec: |
|
174 |
+ pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue) |
|
175 |
+ for version in SPECS.getData().getVersions(depPackage): |
|
176 |
+ depPkg = depPackage+"-"+version |
|
177 |
+ self.logger.info(depPkg) |
|
178 |
+ if pkg in SPECS.getData().getRequiresForPkg(depPkg): |
|
179 |
+ whoNeedsList.append(depPkg) |
|
180 |
+ self.logger.info(whoNeedsList) |
|
181 |
+ return whoNeedsList |
|
182 |
+ |
|
183 |
+ |
|
24 | 184 |
def main(): |
25 | 185 |
usage = "Usage: %prog [options]" |
26 | 186 |
parser = ArgumentParser(usage) |