support/package-builder/Scheduler.py
87815216
 import threading
 from queue import PriorityQueue
620867de
 import json
326d5ca8
 from ThreadPool import ThreadPool
d024e640
 from constants import constants
 from Logger import Logger
45c9260c
 from SpecData import SPECS
d024e640
 
 class Scheduler(object):
87815216
 
     lock = threading.Lock()
326d5ca8
     listOfAlreadyBuiltPackages = set()
87815216
     listOfPackagesToBuild = []
326d5ca8
     listOfPackagesCurrentlyBuilding = set()
87815216
     sortedList = []
     listOfPackagesNextToBuild = PriorityQueue()
     listOfFailedPackages = []
620867de
     alldependencyGraph = {}
     dependencyGraph = {}
     priorityMap = {}
87815216
     pkgWeights = {}
     logger = None
     event = None
     stopScheduling = False
 
d024e640
     @staticmethod
     def setEvent(event):
87815216
         Scheduler.event = event
 
d024e640
     @staticmethod
26b55679
     def setLog(logName, logPath, logLevel):
         Scheduler.logger = Logger.getLogger(logName, logPath, logLevel)
620867de
 
     @staticmethod
326d5ca8
     def setParams(sortedList, listOfAlreadyBuiltPackages):
         Scheduler.sortedList = sortedList
         Scheduler.listOfAlreadyBuiltPackages = listOfAlreadyBuiltPackages
         for x in Scheduler.sortedList:
             if x not in Scheduler.listOfAlreadyBuiltPackages or x in constants.testForceRPMS:
                 Scheduler.listOfPackagesToBuild.append(x)
         Scheduler.listOfPackagesCurrentlyBuilding = set()
         Scheduler.listOfPackagesNextToBuild = PriorityQueue()
         Scheduler.listOfFailedPackages = []
         Scheduler._setPriorities()
 
     @staticmethod
     def notifyPackageBuildCompleted(package):
         with Scheduler.lock:
             if package in Scheduler.listOfPackagesCurrentlyBuilding:
                 Scheduler.listOfPackagesCurrentlyBuilding.remove(package)
                 Scheduler.listOfAlreadyBuiltPackages.add(package)
 
     @staticmethod
     def notifyPackageBuildFailed(package):
         with Scheduler.lock:
             if package in Scheduler.listOfPackagesCurrentlyBuilding:
                 Scheduler.listOfPackagesCurrentlyBuilding.remove(package)
                 Scheduler.listOfFailedPackages.append(package)
 
     @staticmethod
     def isAllPackagesBuilt():
         if Scheduler.listOfPackagesToBuild:
             return False
         return True
 
     @staticmethod
     def isAnyPackagesFailedToBuild():
         if Scheduler.listOfFailedPackages:
             return True
         return False
 
     @staticmethod
     def getNextPackageToBuild():
         with Scheduler.lock:
             if Scheduler.stopScheduling:
                 return None
 
             if not Scheduler.listOfPackagesToBuild:
                 if Scheduler.event is not None:
                     Scheduler.event.set()
 
             if Scheduler.listOfPackagesNextToBuild.empty():
                 Scheduler._getListNextPackagesReadyToBuild()
 
             if Scheduler.listOfPackagesNextToBuild.empty():
                 return None
 
             packageTup = Scheduler.listOfPackagesNextToBuild.get()
 
             package = packageTup[1]
             if Scheduler.listOfPackagesNextToBuild.qsize() > 0:
                 ThreadPool.activateWorkerThreads(
                     Scheduler.listOfPackagesNextToBuild.qsize())
             Scheduler.listOfPackagesCurrentlyBuilding.add(package)
             Scheduler.listOfPackagesToBuild.remove(package)
             return package
 
     @staticmethod
f93ef2b0
     def _getBuildRequiredPackages(pkg):
620867de
         listRequiredRPMPackages = []
f93ef2b0
         listRequiredRPMPackages.extend(SPECS.getData().getBuildRequiresForPkg(pkg))
620867de
 
         listRequiredPackages = []
 
f93ef2b0
         for reqPkg in listRequiredRPMPackages:
             basePkg = SPECS.getData().getBasePkg(reqPkg)
620867de
             if basePkg not in listRequiredPackages:
                 listRequiredPackages.append(basePkg)
 
         return listRequiredPackages
 
 
     @staticmethod
326d5ca8
     def _getDependencies(package, parentPackage, k):
620867de
 
87815216
         for node in list(Scheduler.alldependencyGraph[package].keys()):
326d5ca8
             Scheduler._getDependencies(node, package, k)
620867de
 
87815216
         if parentPackage is None:
620867de
             return
         else:
             for node in Scheduler.alldependencyGraph[package].keys():
                 try:
                     Scheduler.alldependencyGraph[parentPackage][node] = max(
                         Scheduler.alldependencyGraph[parentPackage][node],
                         Scheduler.alldependencyGraph[package][node] * k)
                 except KeyError:
326d5ca8
                     Scheduler.alldependencyGraph[parentPackage][node] = (
                         Scheduler.alldependencyGraph[package][node] * k)
620867de
 
     @staticmethod
326d5ca8
     def _makeGraph():
         k = 2
620867de
         for package in Scheduler.sortedList:
98200cc1
                 Scheduler.dependencyGraph[package] = {}
                 Scheduler.alldependencyGraph[package] = {}
                 for child_package in Scheduler._getBuildRequiredPackages(package):
                     Scheduler.dependencyGraph[package][child_package] = 1
                 for child_package in Scheduler._getRequiredPackages(package):
                     Scheduler.alldependencyGraph[package][child_package] = 1
         for package in Scheduler.sortedList:
87815216
             for child_pkg in list(Scheduler.dependencyGraph[package].keys()):
326d5ca8
                 Scheduler._getDependencies(child_pkg, package, k)
87815216
                 for node in list(Scheduler.alldependencyGraph[child_pkg].keys()):
620867de
                     try:
                         Scheduler.dependencyGraph[package][node] = max(
                             Scheduler.dependencyGraph[package][node],
                             Scheduler.alldependencyGraph[child_pkg][node] * k)
                     except KeyError:
326d5ca8
                         Scheduler.dependencyGraph[package][node] = (
                             Scheduler.alldependencyGraph[child_pkg][node] * k)
87815216
         if constants.publishBuildDependencies:
             dependencyLists = {}
             for package in list(Scheduler.dependencyGraph.keys()):
                 dependencyLists[package] = []
                 for dependency in list(Scheduler.dependencyGraph[package].keys()):
                     dependencyLists[package].append(dependency)
326d5ca8
             with open(str(constants.logPath) + "/BuildDependencies.json", 'w') as graphfile:
                 graphfile.write(json.dumps(dependencyLists, sort_keys=True, indent=4))
620867de
 
     @staticmethod
326d5ca8
     def _parseWeights():
87815216
         Scheduler.pkgWeights.clear()
326d5ca8
         with open(constants.packageWeightsPath, 'r') as weightFile:
             Scheduler.pkgWeights = json.load(weightFile)
620867de
 
     @staticmethod
326d5ca8
     def _getWeight(package):
620867de
         try:
             return float(Scheduler.pkgWeights[package])
         except KeyError:
             return 0
 
     @staticmethod
98200cc1
     def _getPriority(package):
         try:
             return float(Scheduler.priorityMap[package])
         except KeyError:
             return 0
 
 
     @staticmethod
326d5ca8
     def _setPriorities():
87815216
         if constants.packageWeightsPath is None:
26b55679
             Scheduler.logger.debug("Priority Scheduler disabled")
98200cc1
             if constants.publishBuildDependencies:
26b55679
                 Scheduler.logger.debug("Publishing Build dependencies")
98200cc1
                 Scheduler._makeGraph()
87815216
         else:
26b55679
             Scheduler.logger.debug("Priority Scheduler enabled")
326d5ca8
             Scheduler._parseWeights()
 
             Scheduler._makeGraph()
             for package in Scheduler.sortedList:
                 try:
                     Scheduler.priorityMap[package] = Scheduler._getWeight(package)
                 except KeyError:
                     Scheduler.priorityMap[package] = 0
                 for child_pkg in Scheduler.dependencyGraph[package].keys():
                     Scheduler.priorityMap[child_pkg] = (
                         Scheduler.priorityMap[child_pkg]
                         + (Scheduler.dependencyGraph[package][child_pkg]
                         * (Scheduler._getWeight(package))))
26b55679
             Scheduler.logger.debug("set Priorities: Priority of all packages")
             Scheduler.logger.debug(Scheduler.priorityMap)
620867de
 
510e37f4
 
d024e640
     @staticmethod
f93ef2b0
     def _getRequiredPackages(pkg):
87815216
         listRequiredRPMPackages = []
f93ef2b0
         listRequiredRPMPackages.extend(SPECS.getData().getBuildRequiresForPkg(pkg))
         listRequiredRPMPackages.extend(SPECS.getData().getRequiresAllForPkg(pkg))
87815216
 
         listRequiredPackages = []
b5e09fac
 
f93ef2b0
         for reqPkg in listRequiredRPMPackages:
             basePkg = SPECS.getData().getBasePkg(reqPkg)
d024e640
             if basePkg not in listRequiredPackages:
                 listRequiredPackages.append(basePkg)
87815216
 
d024e640
         return listRequiredPackages
87815216
 
d024e640
     @staticmethod
326d5ca8
     def _getListNextPackagesReadyToBuild():
d024e640
         for pkg in Scheduler.listOfPackagesToBuild:
             if pkg in Scheduler.listOfPackagesCurrentlyBuilding:
                 continue
326d5ca8
             listRequiredPackages = Scheduler._getRequiredPackages(pkg)
87815216
             canBuild = True
d024e640
             for reqPkg in listRequiredPackages:
                 if reqPkg not in Scheduler.listOfAlreadyBuiltPackages:
87815216
                     canBuild = False
d024e640
                     break
             if canBuild:
98200cc1
                 Scheduler.listOfPackagesNextToBuild.put((-Scheduler._getPriority(pkg), pkg))
26b55679
                 Scheduler.logger.debug("Adding " + pkg + " to the schedule list")