support/package-builder/PackageBuildDataGenerator.py
87815216
 import copy
2820c61a
 from Logger import Logger
 from constants import constants
45c9260c
 from SpecData import SPECS
2820c61a
 
518d6a6f
 def removeDuplicateEntriesInList(myList):
87815216
     myListCopy = []
518d6a6f
     for p in myList:
         if p not in myListCopy:
             myListCopy.append(p)
     return myListCopy
87815216
 
2820c61a
 class PackageBuildDataGenerator(object):
87815216
 
     cycleCount = 0
 
     def __init__(self, logName=None, logPath=None):
2820c61a
         if logName is None:
             logName = "PackageBuildDataGenerator"
         if logPath is None:
             logPath = constants.logPath
87815216
         self.logName = logName
         self.logPath = logPath
         self.logger = Logger.getLogger(logName, logPath)
         self.__mapCyclesToPackageList = {}
         self.__mapPackageToCycle = {}
         self.__buildDependencyGraph = {}
         self.__runTimeDependencyGraph = {}
         self.__sortedPackageList = []
         self.__sortedBuildDependencyGraph = {}
 
     def getPackageBuildData(self, listPackages):
518d6a6f
         self.__readDependencyGraphAndCyclesForGivenPackages(listPackages)
         self.__getSortedBuildOrderListForGivenPackages(listPackages)
         return self.__mapCyclesToPackageList, self.__mapPackageToCycle, self.__sortedPackageList
2820c61a
 
87815216
     #todo
     def findCompleteListOfPackagesRequiredToBuildGivenPackages(self, listPackages):
         return list(self.__buildDependencyGraph.keys())
 
     def createSortListForPkg(self, pkg):
         runTimeDepPkgList = self.__runTimeDependencyGraph[pkg]
518d6a6f
         runTimeDepPkgList.append(pkg)
87815216
         sortListForPkg = []
 
518d6a6f
         for p in runTimeDepPkgList:
87815216
             basePkg = SPECS.getData().getSpecName(p)
518d6a6f
             for bPkg in self.__sortedBuildDependencyGraph[basePkg]:
                 if bPkg not in sortListForPkg:
                     sortListForPkg.append(bPkg)
87815216
 
518d6a6f
         return sortListForPkg
87815216
 
     def getCircularDependentPackages(self, pkg):
         circularDependentPackages = []
         if pkg in self.__mapPackageToCycle:
             circularDependentPackages.extend(
                 self.__mapCyclesToPackageList[self.__mapPackageToCycle[pkg]])
518d6a6f
             circularDependentPackages.remove(pkg)
         return circularDependentPackages
87815216
 
     def __getSortedBuildOrderListForGivenPackages(self, listPackages):
 
         alreadyProcessedPackages = []
         sortedList = []
         completeListPackagesToBuild = self.findCompleteListOfPackagesRequiredToBuildGivenPackages(
             listPackages)
518d6a6f
         packageIndexInSortedList = 0
87815216
         prevSortListLen = 0
 
518d6a6f
         while completeListPackagesToBuild:
87815216
 
518d6a6f
             # find next package to process
87815216
             pkg = None
             index = -1
             lenList = len(sortedList)
518d6a6f
             for  i in range(lenList):
                 if sortedList[i] in alreadyProcessedPackages:
2820c61a
                     continue
518d6a6f
                 pkg = sortedList[i]
                 packageIndexInSortedList = i
2820c61a
                 break
87815216
 
2820c61a
             if pkg is None:
518d6a6f
                 pkg = completeListPackagesToBuild.pop()
                 packageIndexInSortedList = len(sortedList)
 
             #creating sort list for package
87815216
             sortListForPkg = self.createSortListForPkg(pkg)
 
518d6a6f
             #remove any cyclic packages in sortListForPkg if they already exists in sortedList
87815216
             circularDependentPackages = self.getCircularDependentPackages(pkg)
2820c61a
             for p in circularDependentPackages:
518d6a6f
                 if p in sortedList and p in sortListForPkg:
                     sortListForPkg.remove(p)
87815216
 
518d6a6f
             # insert sort list of package in global sorted list
87815216
             index = packageIndexInSortedList
             subList = []
518d6a6f
             if packageIndexInSortedList > 0:
87815216
                 subList = sortedList[:packageIndexInSortedList]
518d6a6f
             for p in sortListForPkg:
2820c61a
                 if  p not in subList:
518d6a6f
                     sortedList.insert(index, p)
2820c61a
                     index = index + 1
87815216
 
2820c61a
             alreadyProcessedPackages.append(p)
87815216
 
518d6a6f
             # Remove duplicate entries in sorted list in intervals
87815216
             if (len(sortedList) - prevSortListLen) > 100:
518d6a6f
                 self.logger.info("Removing duplicates in sortedList")
                 sortedList = removeDuplicateEntriesInList(sortedList)
2820c61a
             else:
87815216
                 prevSortListLen = len(sortedList)
 
518d6a6f
         self.logger.info("Removing duplicates in sorted list")
         sortedList = removeDuplicateEntriesInList(sortedList)
87815216
 
518d6a6f
         self.logger.info("Sorted list:")
         self.logger.info(sortedList)
87815216
         self.__sortedPackageList = sortedList
 
     def __constructBuildAndRunTimeDependencyGraph(self, package):
         basePackage = SPECS.getData().getSpecName(package)
 
         addBuildTimeGraph = True
         addRunTimeGraph = True
         if basePackage in self.__buildDependencyGraph:
518d6a6f
             addBuildTimeGraph = False
87815216
         if basePackage in self.__runTimeDependencyGraph:
             addRunTimeGraph = False
 
         nextPackagesToConstructGraph = []
518d6a6f
         if addBuildTimeGraph:
87815216
             listDependentRpmPackages = SPECS.getData().getBuildRequiresForPackage(basePackage)
             listDependentPackages = []
518d6a6f
             for rpmPkg in listDependentRpmPackages:
87815216
                 basePkg = SPECS.getData().getSpecName(rpmPkg)
518d6a6f
                 if basePkg not in listDependentPackages:
                     listDependentPackages.append(basePkg)
87815216
             self.__buildDependencyGraph[basePackage] = listDependentPackages
518d6a6f
             nextPackagesToConstructGraph.extend(listDependentPackages)
87815216
 
518d6a6f
         if addRunTimeGraph:
87815216
             listRpmPackages = SPECS.getData().getPackages(basePackage)
518d6a6f
             for rpmPkg in listRpmPackages:
87815216
                 listDependentRpmPackages = SPECS.getData().getRequiresAllForPackage(rpmPkg)
                 self.__runTimeDependencyGraph[rpmPkg] = listDependentRpmPackages[:]
518d6a6f
                 nextPackagesToConstructGraph.extend(listDependentRpmPackages)
2820c61a
 
518d6a6f
         for pkg in nextPackagesToConstructGraph:
             self.__constructBuildAndRunTimeDependencyGraph(pkg)
87815216
 
     def __readDependencyGraphAndCyclesForGivenPackages(self, listPackages):
b650a058
         self.logger.info("Reading dependency graph to check for cycles")
2820c61a
         for pkg in listPackages:
518d6a6f
             self.__constructBuildAndRunTimeDependencyGraph(pkg)
87815216
 
         for pkg in self.__buildDependencyGraph.keys():
             sortedPackageList, circularDependentPackages = self.topologicalSortPackages(
                 self.__buildDependencyGraph, pkg)
             if len(circularDependentPackages) > 0:
2820c61a
                 self.logger.error("Found circular dependency")
                 self.logger.error(circularDependentPackages)
518d6a6f
                 raise Exception("Build Time Circular Dependency")
87815216
             self.__sortedBuildDependencyGraph[pkg] = sortedPackageList
         sortedPackageList, circularDependentPackages = self.topologicalSortPackages(
             self.__runTimeDependencyGraph)
         if len(circularDependentPackages) > 0:
2820c61a
             self.__findCircularDependencies(circularDependentPackages)
87815216
 
518d6a6f
     def topologicalSortPackages(self, dependencyGraph, package=None):
87815216
         noDepPackages = set()
2820c61a
         sortedPackageList = []
         dependentOfPackage = dict()
87815216
 
         dependentPackages = {}
2820c61a
         if package is None:
87815216
             dependentPackages = copy.deepcopy(dependencyGraph)
2820c61a
         else:
87815216
             listDepPkgs = set()
2820c61a
             listDepPkgs.add(package)
             while listDepPkgs:
                 pkg = listDepPkgs.pop()
87815216
                 if pkg in dependentPackages:
2820c61a
                     continue
87815216
                 dependentPackages[pkg] = dependencyGraph[pkg][:]
518d6a6f
                 for depPkg in dependencyGraph[pkg]:
2820c61a
                     listDepPkgs.add(depPkg)
87815216
 
2820c61a
         #Find packages with no dependencies and generate dependentof_package edge list
         for pkg in dependentPackages:
             if len(dependentPackages[pkg]) == 0:
                 noDepPackages.add(pkg)
             else:
                 for depPkg in dependentPackages[pkg]:
87815216
                     if depPkg not in dependentOfPackage:
                         dependentOfPackage[depPkg] = [pkg]
2820c61a
                     else:
                         if pkg not in dependentOfPackage[depPkg]:
                             dependentOfPackage[depPkg].append(pkg)
87815216
 
2820c61a
         while noDepPackages:
             pkg = noDepPackages.pop()
             sortedPackageList.append(pkg)
             if dependentOfPackage.get(pkg) is not None:
                 for childPkg in list(dependentOfPackage.get(pkg)):
                     dependentOfPackage.get(pkg).remove(childPkg)
                     dependentPackages[childPkg].remove(pkg)
87815216
                     if len(dependentPackages[childPkg]) == 0:
2820c61a
                         noDepPackages.add(childPkg)
87815216
 
518d6a6f
         # creating circular dependency graph for given dependency graph
87815216
         circularDependencyGraph = {}
         for pkg in dependentPackages.keys():
2820c61a
             if len(dependentPackages[pkg]) != 0:
87815216
                 circularDependencyGraph[pkg] = dependentPackages[pkg]
 
         #return (non-circular dependent package in sorted order and circular dependent
         #package list in a dependencyGraph)
         return sortedPackageList, circularDependencyGraph
 
     def __constructDependencyMap(self, yclicDependencyGraph):
518d6a6f
         self.logger.info("Constructing dependency map from circular dependency graph.....")
87815216
         constructDependencyMap = {}
         for node in cyclicDependencyGraph.keys():
             tmpDepNodeList = []
2820c61a
             tmpDepNodeList.append(node)
87815216
             depNodeList = []
             while len(tmpDepNodeList) != 0:
2820c61a
                 currentNode = tmpDepNodeList.pop()
518d6a6f
                 addDepNodeList = cyclicDependencyGraph[currentNode]
2820c61a
                 depNodeList.append(currentNode)
                 for depNode in addDepNodeList:
                     if depNode in depNodeList:
                         continue
                     else:
                         if depNode not in tmpDepNodeList:
                             tmpDepNodeList.append(depNode)
             depNodeList.remove(node)
87815216
             constructDependencyMap[node] = depNodeList
2820c61a
         self.logger.info("Dependency Map:")
         self.logger.info(constructDependencyMap)
518d6a6f
         return constructDependencyMap
87815216
 
     def __findCircularDependencies(self, cyclicDependencyGraph):
b650a058
         self.logger.info("Looking for circular dependencies")
518d6a6f
         if len(cyclicDependencyGraph) == 0:
             return
         #step1: construct dependency map from dependency graph
87815216
         constructDependencyMap = self.__constructDependencyMap(cyclicDependencyGraph)
 
518d6a6f
         #step2: find cycles in dependency map
         self.logger.info("Finding and adding cycles using constructed dependency map......")
87815216
         cycleCount = 0
         for node in cyclicDependencyGraph.keys():
             listDepPkg = constructDependencyMap[node]
             cycPkgs = []
             if node not in self.__mapPackageToCycle:
2820c61a
                 for depPkg in listDepPkg:
                     x = constructDependencyMap[depPkg]
                     if node in x:
                         cycPkgs.append(depPkg)
87815216
 
2820c61a
                 if len(cycPkgs) != 0:
                     cycPkgs.append(node)
87815216
                     cycleName = "cycle" + str(PackageBuildDataGenerator.cycleCount)
                     PackageBuildDataGenerator.cycleCount += 1
2820c61a
                     for x in cycPkgs:
87815216
                         self.__mapPackageToCycle[x] = cycleName
                     self.__mapCyclesToPackageList[cycleName] = cycPkgs
518d6a6f
                     self.logger.info("New circular dependency found:")
87815216
                     self.logger.info(cycleName + " " + ",".join(cycPkgs))
                     cycleCount += 1
 
         if cycleCount > 0:
             self.logger.info("Found " + str(cycleCount) + " cycles.")
518d6a6f
             self.logger.info("Successfully added all detected circular dependencies to list.")
         else:
             self.logger.info("No circular dependencies found.")