87815216 |
import copy |
2820c61a |
from Logger import Logger
from constants import constants |
45c9260c |
from SpecData import SPECS |
2820c61a |
|
518d6a6f |
def removeDuplicateEntriesInList(myList): |
87815216 |
myListCopy = [] |
518d6a6f |
for p in myList:
if p not in myListCopy:
myListCopy.append(p)
return myListCopy |
87815216 |
|
2820c61a |
class PackageBuildDataGenerator(object): |
87815216 |
cycleCount = 0
def __init__(self, logName=None, logPath=None): |
2820c61a |
if logName is None:
logName = "PackageBuildDataGenerator"
if logPath is None:
logPath = constants.logPath |
87815216 |
self.logName = logName
self.logPath = logPath
self.logger = Logger.getLogger(logName, logPath)
self.__mapCyclesToPackageList = {}
self.__mapPackageToCycle = {}
self.__buildDependencyGraph = {}
self.__runTimeDependencyGraph = {}
self.__sortedPackageList = []
self.__sortedBuildDependencyGraph = {}
def getPackageBuildData(self, listPackages): |
518d6a6f |
self.__readDependencyGraphAndCyclesForGivenPackages(listPackages)
self.__getSortedBuildOrderListForGivenPackages(listPackages)
return self.__mapCyclesToPackageList, self.__mapPackageToCycle, self.__sortedPackageList |
2820c61a |
|
87815216 |
#todo
def findCompleteListOfPackagesRequiredToBuildGivenPackages(self, listPackages):
return list(self.__buildDependencyGraph.keys())
def createSortListForPkg(self, pkg):
runTimeDepPkgList = self.__runTimeDependencyGraph[pkg] |
518d6a6f |
runTimeDepPkgList.append(pkg) |
87815216 |
sortListForPkg = []
|
518d6a6f |
for p in runTimeDepPkgList: |
87815216 |
basePkg = SPECS.getData().getSpecName(p) |
518d6a6f |
for bPkg in self.__sortedBuildDependencyGraph[basePkg]:
if bPkg not in sortListForPkg:
sortListForPkg.append(bPkg) |
87815216 |
|
518d6a6f |
return sortListForPkg |
87815216 |
def getCircularDependentPackages(self, pkg):
circularDependentPackages = []
if pkg in self.__mapPackageToCycle:
circularDependentPackages.extend(
self.__mapCyclesToPackageList[self.__mapPackageToCycle[pkg]]) |
518d6a6f |
circularDependentPackages.remove(pkg)
return circularDependentPackages |
87815216 |
def __getSortedBuildOrderListForGivenPackages(self, listPackages):
alreadyProcessedPackages = []
sortedList = []
completeListPackagesToBuild = self.findCompleteListOfPackagesRequiredToBuildGivenPackages(
listPackages) |
518d6a6f |
packageIndexInSortedList = 0 |
87815216 |
prevSortListLen = 0
|
518d6a6f |
while completeListPackagesToBuild: |
87815216 |
|
518d6a6f |
# find next package to process |
87815216 |
pkg = None
index = -1
lenList = len(sortedList) |
518d6a6f |
for i in range(lenList):
if sortedList[i] in alreadyProcessedPackages: |
2820c61a |
continue |
518d6a6f |
pkg = sortedList[i]
packageIndexInSortedList = i |
2820c61a |
break |
87815216 |
|
2820c61a |
if pkg is None: |
518d6a6f |
pkg = completeListPackagesToBuild.pop()
packageIndexInSortedList = len(sortedList)
#creating sort list for package |
87815216 |
sortListForPkg = self.createSortListForPkg(pkg)
|
518d6a6f |
#remove any cyclic packages in sortListForPkg if they already exists in sortedList |
87815216 |
circularDependentPackages = self.getCircularDependentPackages(pkg) |
2820c61a |
for p in circularDependentPackages: |
518d6a6f |
if p in sortedList and p in sortListForPkg:
sortListForPkg.remove(p) |
87815216 |
|
518d6a6f |
# insert sort list of package in global sorted list |
87815216 |
index = packageIndexInSortedList
subList = [] |
518d6a6f |
if packageIndexInSortedList > 0: |
87815216 |
subList = sortedList[:packageIndexInSortedList] |
518d6a6f |
for p in sortListForPkg: |
2820c61a |
if p not in subList: |
518d6a6f |
sortedList.insert(index, p) |
2820c61a |
index = index + 1 |
87815216 |
|
2820c61a |
alreadyProcessedPackages.append(p) |
87815216 |
|
518d6a6f |
# Remove duplicate entries in sorted list in intervals |
87815216 |
if (len(sortedList) - prevSortListLen) > 100: |
518d6a6f |
self.logger.info("Removing duplicates in sortedList")
sortedList = removeDuplicateEntriesInList(sortedList) |
2820c61a |
else: |
87815216 |
prevSortListLen = len(sortedList)
|
518d6a6f |
self.logger.info("Removing duplicates in sorted list")
sortedList = removeDuplicateEntriesInList(sortedList) |
87815216 |
|
518d6a6f |
self.logger.info("Sorted list:")
self.logger.info(sortedList) |
87815216 |
self.__sortedPackageList = sortedList
def __constructBuildAndRunTimeDependencyGraph(self, package):
basePackage = SPECS.getData().getSpecName(package)
addBuildTimeGraph = True
addRunTimeGraph = True
if basePackage in self.__buildDependencyGraph: |
518d6a6f |
addBuildTimeGraph = False |
87815216 |
if basePackage in self.__runTimeDependencyGraph:
addRunTimeGraph = False
nextPackagesToConstructGraph = [] |
518d6a6f |
if addBuildTimeGraph: |
87815216 |
listDependentRpmPackages = SPECS.getData().getBuildRequiresForPackage(basePackage)
listDependentPackages = [] |
518d6a6f |
for rpmPkg in listDependentRpmPackages: |
87815216 |
basePkg = SPECS.getData().getSpecName(rpmPkg) |
518d6a6f |
if basePkg not in listDependentPackages:
listDependentPackages.append(basePkg) |
87815216 |
self.__buildDependencyGraph[basePackage] = listDependentPackages |
518d6a6f |
nextPackagesToConstructGraph.extend(listDependentPackages) |
87815216 |
|
518d6a6f |
if addRunTimeGraph: |
87815216 |
listRpmPackages = SPECS.getData().getPackages(basePackage) |
518d6a6f |
for rpmPkg in listRpmPackages: |
87815216 |
listDependentRpmPackages = SPECS.getData().getRequiresAllForPackage(rpmPkg)
self.__runTimeDependencyGraph[rpmPkg] = listDependentRpmPackages[:] |
518d6a6f |
nextPackagesToConstructGraph.extend(listDependentRpmPackages) |
2820c61a |
|
518d6a6f |
for pkg in nextPackagesToConstructGraph:
self.__constructBuildAndRunTimeDependencyGraph(pkg) |
87815216 |
def __readDependencyGraphAndCyclesForGivenPackages(self, listPackages): |
b650a058 |
self.logger.info("Reading dependency graph to check for cycles") |
2820c61a |
for pkg in listPackages: |
518d6a6f |
self.__constructBuildAndRunTimeDependencyGraph(pkg) |
87815216 |
for pkg in self.__buildDependencyGraph.keys():
sortedPackageList, circularDependentPackages = self.topologicalSortPackages(
self.__buildDependencyGraph, pkg)
if len(circularDependentPackages) > 0: |
2820c61a |
self.logger.error("Found circular dependency")
self.logger.error(circularDependentPackages) |
518d6a6f |
raise Exception("Build Time Circular Dependency") |
87815216 |
self.__sortedBuildDependencyGraph[pkg] = sortedPackageList
sortedPackageList, circularDependentPackages = self.topologicalSortPackages(
self.__runTimeDependencyGraph)
if len(circularDependentPackages) > 0: |
2820c61a |
self.__findCircularDependencies(circularDependentPackages) |
87815216 |
|
518d6a6f |
def topologicalSortPackages(self, dependencyGraph, package=None): |
87815216 |
noDepPackages = set() |
2820c61a |
sortedPackageList = []
dependentOfPackage = dict() |
87815216 |
dependentPackages = {} |
2820c61a |
if package is None: |
87815216 |
dependentPackages = copy.deepcopy(dependencyGraph) |
2820c61a |
else: |
87815216 |
listDepPkgs = set() |
2820c61a |
listDepPkgs.add(package)
while listDepPkgs:
pkg = listDepPkgs.pop() |
87815216 |
if pkg in dependentPackages: |
2820c61a |
continue |
87815216 |
dependentPackages[pkg] = dependencyGraph[pkg][:] |
518d6a6f |
for depPkg in dependencyGraph[pkg]: |
2820c61a |
listDepPkgs.add(depPkg) |
87815216 |
|
2820c61a |
#Find packages with no dependencies and generate dependentof_package edge list
for pkg in dependentPackages:
if len(dependentPackages[pkg]) == 0:
noDepPackages.add(pkg)
else:
for depPkg in dependentPackages[pkg]: |
87815216 |
if depPkg not in dependentOfPackage:
dependentOfPackage[depPkg] = [pkg] |
2820c61a |
else:
if pkg not in dependentOfPackage[depPkg]:
dependentOfPackage[depPkg].append(pkg) |
87815216 |
|
2820c61a |
while noDepPackages:
pkg = noDepPackages.pop()
sortedPackageList.append(pkg)
if dependentOfPackage.get(pkg) is not None:
for childPkg in list(dependentOfPackage.get(pkg)):
dependentOfPackage.get(pkg).remove(childPkg)
dependentPackages[childPkg].remove(pkg) |
87815216 |
if len(dependentPackages[childPkg]) == 0: |
2820c61a |
noDepPackages.add(childPkg) |
87815216 |
|
518d6a6f |
# creating circular dependency graph for given dependency graph |
87815216 |
circularDependencyGraph = {}
for pkg in dependentPackages.keys(): |
2820c61a |
if len(dependentPackages[pkg]) != 0: |
87815216 |
circularDependencyGraph[pkg] = dependentPackages[pkg]
#return (non-circular dependent package in sorted order and circular dependent
#package list in a dependencyGraph)
return sortedPackageList, circularDependencyGraph
def __constructDependencyMap(self, yclicDependencyGraph): |
518d6a6f |
self.logger.info("Constructing dependency map from circular dependency graph.....") |
87815216 |
constructDependencyMap = {}
for node in cyclicDependencyGraph.keys():
tmpDepNodeList = [] |
2820c61a |
tmpDepNodeList.append(node) |
87815216 |
depNodeList = []
while len(tmpDepNodeList) != 0: |
2820c61a |
currentNode = tmpDepNodeList.pop() |
518d6a6f |
addDepNodeList = cyclicDependencyGraph[currentNode] |
2820c61a |
depNodeList.append(currentNode)
for depNode in addDepNodeList:
if depNode in depNodeList:
continue
else:
if depNode not in tmpDepNodeList:
tmpDepNodeList.append(depNode)
depNodeList.remove(node) |
87815216 |
constructDependencyMap[node] = depNodeList |
2820c61a |
self.logger.info("Dependency Map:")
self.logger.info(constructDependencyMap) |
518d6a6f |
return constructDependencyMap |
87815216 |
def __findCircularDependencies(self, cyclicDependencyGraph): |
b650a058 |
self.logger.info("Looking for circular dependencies") |
518d6a6f |
if len(cyclicDependencyGraph) == 0:
return
#step1: construct dependency map from dependency graph |
87815216 |
constructDependencyMap = self.__constructDependencyMap(cyclicDependencyGraph)
|
518d6a6f |
#step2: find cycles in dependency map
self.logger.info("Finding and adding cycles using constructed dependency map......") |
87815216 |
cycleCount = 0
for node in cyclicDependencyGraph.keys():
listDepPkg = constructDependencyMap[node]
cycPkgs = []
if node not in self.__mapPackageToCycle: |
2820c61a |
for depPkg in listDepPkg:
x = constructDependencyMap[depPkg]
if node in x:
cycPkgs.append(depPkg) |
87815216 |
|
2820c61a |
if len(cycPkgs) != 0:
cycPkgs.append(node) |
87815216 |
cycleName = "cycle" + str(PackageBuildDataGenerator.cycleCount)
PackageBuildDataGenerator.cycleCount += 1 |
2820c61a |
for x in cycPkgs: |
87815216 |
self.__mapPackageToCycle[x] = cycleName
self.__mapCyclesToPackageList[cycleName] = cycPkgs |
518d6a6f |
self.logger.info("New circular dependency found:") |
87815216 |
self.logger.info(cycleName + " " + ",".join(cycPkgs))
cycleCount += 1
if cycleCount > 0:
self.logger.info("Found " + str(cycleCount) + " cycles.") |
518d6a6f |
self.logger.info("Successfully added all detected circular dependencies to list.")
else:
self.logger.info("No circular dependencies found.") |