#! /usr/bin/python3 # # Copyright (C) 2015 vmware inc. # # Author: Harish Udaiya Kumar import sys import os import json import queue import operator from argparse import ArgumentParser import shutil import traceback from SpecData import SPECS from jsonwrapper import JsonWrapper from constants import constants from CommandUtils import CommandUtils from StringUtils import StringUtils from Logger import Logger DEFAULT_INPUT_TYPE = "pkg" DEFAULT_DISPLAY_OPTION = "tree" SPEC_FILE_DIR = "../../SPECS" LOG_FILE_DIR = "../../stage/LOGS" class SpecDependencyGenerator(object): def __init__(self, logPath, logLevel): self.logger = Logger.getLogger("Serializable Spec objects", logPath, logLevel) def findTotalRequires(self, mapDependencies, depQue, parent): while not depQue.empty(): specPkg = depQue.get() try: listRequiredPackages = SPECS.getData().getRequiresForPkg(specPkg) except Exception as e: self.logger.info("Caught Exception:"+str(e)) self.logger.info(specPkg + " is missing") raise e for depPkg in listRequiredPackages: if depPkg in mapDependencies: if mapDependencies[depPkg] < mapDependencies[specPkg] + 1: mapDependencies[depPkg] = mapDependencies[specPkg] + 1 parent[depPkg] = specPkg self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg]) else: mapDependencies[depPkg] = mapDependencies[specPkg] + 1 parent[depPkg] = specPkg depQue.put(depPkg) def getBasePackagesRequired(self, pkg): listBasePackagesRequired=[] listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg) listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg)) for p in listPackagesRequired: basePkg = SPECS.getData().getBasePkg(p) if basePkg not in listBasePackagesRequired: listBasePackagesRequired.append(basePkg) return listBasePackagesRequired def findTotalWhoNeeds(self, depList, whoNeeds): while depList: pkg = depList.pop(0) for depPackage in SPECS.getData().getListPackages(): for version in SPECS.getData().getVersions(depPackage): depBasePkg = depPackage+"-"+version if depBasePkg in whoNeeds: continue if pkg in self.getBasePackagesRequired(depBasePkg): whoNeeds.append(depBasePkg) if depBasePkg not in depList: depList.append(depBasePkg) def printTree(self, children, curParent, depth): if curParent in children: for child in children[curParent]: self.logger.info("\t" * depth + child) self.printTree(children, child, depth + 1) def getAllPackageNames(self, jsonFilePath): with open(jsonFilePath) as jsonData: option_list_json = json.load(jsonData) packages = option_list_json["packages"] return packages def updateLevels(self, mapDependencies, inPkg, parent, level): listPackages = SPECS.getData().getPackagesForPkg(inPkg) for depPkg in SPECS.getData().getRequiresForPkg(inPkg): if depPkg in listPackages: continue if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1: mapDependencies[depPkg] = level + 1 parent[depPkg] = inPkg self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg]) def calculateSpecDependency(self, inputPackages, mapDependencies, parent): depQue = queue.Queue() for package in inputPackages: if SPECS.getData().isRPMPackage(package): for version in SPECS.getData().getVersions(package): pkg = package+"-"+version if pkg not in mapDependencies: mapDependencies[pkg] = 0 parent[pkg] = "" depQue.put(pkg) self.findTotalRequires(mapDependencies, depQue, parent) else: self.logger.info("Could not find spec for " + package) def displayDependencies(self, displayOption, inputType, inputValue, allDeps, parent): children = {} sortedList = [] for elem in sorted(allDeps.items(), key=operator.itemgetter(1), reverse=True): sortedList.append(elem[0]) # construct all children nodes if displayOption == "tree": for k, v in parent.iteritems(): children.setdefault(v, []).append(k) if inputType == "json": self.logger.info("Dependency Mappings for {}".format(inputValue) + " :") self.logger.info("-" * 52 + " {}".format(children)) self.logger.info("-" * 52) if "" in children: for child in children[""]: self.logger.info(child) self.printTree(children, child, 1) self.logger.info("*" * 18 + " {} ".format(len(sortedList)) + "packages in total " + "*" * 18) else: if inputType == "pkg" and len(children) > 0: self.logger.info("cyclic dependency detected, mappings: \n", children) # To display a flat list of all packages elif displayOption == "list": self.logger.info(sortedList) # To generate a new JSON file based on given input json file elif displayOption == "json" and inputType == "json": d = {'packages': sortedList} with open(inputValue, 'w') as outfile: json.dump(d, outfile) return sortedList def process(self, inputType, inputValue, displayOption, outputFile=None): whoNeedsList = [] inputPackages = [] whatNeedsBuild = [] mapDependencies = {} parent = {} if inputType == "pkg" or inputType == "json": if inputType == "pkg": inputPackages.append(inputValue) else: inputPackages = self.getAllPackageNames(inputValue) self.calculateSpecDependency(inputPackages, mapDependencies, parent) if outputFile is not None: return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent) else: return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent) elif inputType == "get-upward-deps": depList = [] for specFile in inputValue.split(":"): if specFile in SPECS.getData().mapSpecFileNameToSpecObj: specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile] whoNeedsList.append(specObj.name+"-"+specObj.version) depList.append(specObj.name+"-"+specObj.version) self.findTotalWhoNeeds(depList, whoNeedsList) return whoNeedsList elif inputType == "who-needs": for depPackage in SPECS.getData().mapPackageToSpec: pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue) for version in SPECS.getData().getVersions(depPackage): depPkg = depPackage+"-"+version self.logger.info(depPkg) if pkg in SPECS.getData().getRequiresForPkg(depPkg): whoNeedsList.append(depPkg) self.logger.info(whoNeedsList) return whoNeedsList def main(): usage = "Usage: %prog [options]" parser = ArgumentParser(usage) parser.add_argument("-i", "--input-type", dest="input_type", default=DEFAULT_INPUT_TYPE) parser.add_argument("-p", "--pkg", dest="pkg") parser.add_argument("-f", "--file", dest="json_file", default="packages_minimal.json") parser.add_argument("-d", "--display-option", dest="display_option", default=DEFAULT_DISPLAY_OPTION) parser.add_argument("-s", "--spec-path", dest="spec_path", default=SPEC_FILE_DIR) parser.add_argument("-l", "--log-path", dest="log_path", default=LOG_FILE_DIR) parser.add_argument("-y", "--log-level", dest="log_level", default="info") parser.add_argument("-t", "--stage-dir", dest="stage_dir", default="../../stage") parser.add_argument("-a", "--input-data-dir", dest="input_data_dir", default="../../common/data/") parser.add_argument("-o", "--output-dir", dest="output_dir", default="../../stage/common/data") options = parser.parse_args() constants.setSpecPath(options.spec_path) constants.setLogPath(options.log_path) constants.setLogLevel(options.log_level) constants.initialize() cmdUtils = CommandUtils() logger = Logger.getLogger("SpecDeps", options.log_path, options.log_level) if not os.path.isdir(options.output_dir): cmdUtils.runCommandInShell("mkdir -p "+options.output_dir) if not options.input_data_dir.endswith('/'): options.input_data_dir += '/' try: specDeps = SpecDependencyGenerator(options.log_path, options.log_level) if options.input_type == "remove-upward-deps": whoNeedsList = specDeps.process("get-upward-deps", options.pkg, options.display_option) logger.info("Removing upward dependencies: " + str(whoNeedsList)) for pkg in whoNeedsList: package, version = StringUtils.splitPackageNameAndVersion(pkg) release = SPECS.getData().getRelease(package, version) for p in SPECS.getData().getPackages(package,version): buildarch=SPECS.getData().getBuildArch(p, version) rpmFile = "stage/RPMS/" + buildarch + "/" + p + "-" + version + "-" + release + ".*" + buildarch+".rpm" cmdUtils.runCommandInShell("rm -f "+rpmFile) elif options.input_type == "print-upward-deps": whoNeedsList = specDeps.process("get-upward-deps", options.pkg, options.display_option) logger.info("Upward dependencies: " + str(whoNeedsList)) # To display/print package dependencies on console elif (options.input_type == "pkg" or options.input_type == "who-needs"): specDeps.process(options.input_type, options.pkg, options.display_option) elif options.input_type == "json": list_json_files = options.json_file.split("\n") # Generate the expanded package dependencies json file based on package_list_file logger.info("Generating the install time dependency list for all json files") for json_file in list_json_files: shutil.copy2(json_file, options.output_dir) json_wrapper_option_list = JsonWrapper(json_file) option_list_json = json_wrapper_option_list.read() options_sorted = option_list_json.items() for install_option in options_sorted: output_file = None input_value = os.path.join(os.path.dirname(json_file), install_option[1]["file"]) if options.display_option == "tree" and install_option[1]["title"] == "ISO Packages": continue if options.display_option == "json": output_file = os.path.join(options.output_dir, install_option[1]["file"]) specDeps.process(options.input_type, input_value, options.display_option, output_file) except Exception as e: traceback.print_exc() sys.stderr.write(str(e)) sys.stderr.write("Failed to generate dependency lists from spec files\n") sys.exit(1) sys.exit(0) if __name__ == "__main__": main()