support/package-builder/SpecData.py
2820c61a
 import os
0f1fdc4b
 import platform
87815216
 import queue
45c9260c
 import json
 import operator
87815216
 from distutils.version import StrictVersion
 from SpecUtils import Specutils
 from Logger import Logger
45c9260c
 from constants import constants
2820c61a
 
accc8120
 
 
2820c61a
 class SerializableSpecObject(object):
     def __init__(self):
87815216
         self.listPackages = []
         self.listRPMPackages = []
         self.name = ""
         self.version = ""
         self.release = ""
         self.buildRequirePackages = []
         self.checkBuildRequirePackages = []
         self.installRequiresAllPackages = []
         self.installRequiresPackages = {}
         self.specFile = ""
         self.listSources = []
         self.checksums = {}
         self.listPatches = []
         self.securityHardening = ""
         self.url = ""
         self.sourceurl = ""
         self.license = ""
         self.specDefs = {}
2820c61a
 
accc8120
 
2820c61a
 class SerializableSpecObjectsUtils(object):
5062126c
 
87815216
     def __init__(self, logPath):
         self.mapSerializableSpecObjects = {}
         self.mapPackageToSpec = {}
         self.logger = Logger.getLogger("Serializable Spec objects", logPath)
5062126c
 
87815216
     def readSpecsAndConvertToSerializableObjects(self, specFilesPath):
         listSpecFiles = []
         self.getListSpecFiles(listSpecFiles, specFilesPath)
2820c61a
         for specFile in listSpecFiles:
37022ec3
             skipUpdating = False
87815216
             spec = Specutils(specFile)
             specName = spec.getBasePackageName()
             specObj = SerializableSpecObject()
             specObj.name = specName
             specObj.buildRequirePackages = spec.getBuildRequiresAllPackages()
             specObj.installRequiresAllPackages = spec.getRequiresAllPackages()
             specObj.checkBuildRequirePackages = spec.getCheckBuildRequiresAllPackages()
             specObj.listPackages = spec.getPackageNames()
             specObj.specFile = specFile
             specObj.version = spec.getVersion()
             specObj.release = spec.getRelease()
             specObj.listSources = spec.getSourceNames()
             specObj.checksums = spec.getChecksums()
             specObj.specDefs = spec.getDefinitions()
             specObj.listPatches = spec.getPatchNames()
             specObj.securityHardening = spec.getSecurityHardeningOption()
             specObj.isCheckAvailable = spec.isCheckAvailable()
             specObj.license = spec.getLicense()
             specObj.url = spec.getURL()
             specObj.sourceurl = spec.getSourceURL()
2820c61a
             for specPkg in specObj.listPackages:
97a9151c
                 if specPkg in self.mapPackageToSpec:
                     existingObj = self.mapSerializableSpecObjects[self.mapPackageToSpec[specPkg]]
87815216
                     if self.compareVersions(existingObj, specObj) == 1:
97a9151c
                         skipUpdating = True
87815216
                         break
                 specObj.installRequiresPackages[specPkg] = spec.getRequires(specPkg)
                 self.mapPackageToSpec[specPkg] = specName
adf248d5
                 if spec.getIsRPMPackage(specPkg):
                     specObj.listRPMPackages.append(specPkg)
accc8120
             if not skipUpdating:
87815216
                 self.mapSerializableSpecObjects[specName] = specObj
5062126c
 
87815216
     def getListSpecFiles(self, listSpecFiles, path):
2820c61a
         for dirEntry in os.listdir(path):
             dirEntryPath = os.path.join(path, dirEntry)
87815216
             if (os.path.isfile(dirEntryPath) and
                     dirEntryPath.endswith(".spec") and
                     os.path.basename(dirEntryPath) not in
                     constants.skipSpecsForArch.get(platform.machine(), [])):
2820c61a
                 listSpecFiles.append(dirEntryPath)
             elif os.path.isdir(dirEntryPath):
87815216
                 self.getListSpecFiles(listSpecFiles, dirEntryPath)
5062126c
 
2820c61a
     def getBuildRequiresForPackage(self, package):
87815216
         specName = self.getSpecName(package)
51e6babb
         return self.mapSerializableSpecObjects[specName].buildRequirePackages
5062126c
 
2820c61a
     def getRequiresAllForPackage(self, package):
87815216
         specName = self.getSpecName(package)
51e6babb
         return self.mapSerializableSpecObjects[specName].installRequiresAllPackages
5062126c
 
2820c61a
     def getRequiresForPackage(self, package):
87815216
         specName = self.getSpecName(package)
         if package in self.mapSerializableSpecObjects[specName].installRequiresPackages:
51e6babb
             return self.mapSerializableSpecObjects[specName].installRequiresPackages[package]
2820c61a
         return None
9b2f8b85
 
5f40784b
     def getCheckBuildRequiresForPackage(self, package):
87815216
         specName = self.getSpecName(package)
5f40784b
         return self.mapSerializableSpecObjects[specName].checkBuildRequirePackages
 
2820c61a
     def getRelease(self, package):
87815216
         specName = self.getSpecName(package)
343d89e8
         return self.mapSerializableSpecObjects[specName].release
5062126c
 
2820c61a
     def getVersion(self, package):
87815216
         specName = self.getSpecName(package)
51e6babb
         return self.mapSerializableSpecObjects[specName].version
5062126c
 
2820c61a
     def getSpecFile(self, package):
87815216
         specName = self.getSpecName(package)
51e6babb
         return self.mapSerializableSpecObjects[specName].specFile
5062126c
 
2820c61a
     def getPatches(self, package):
87815216
         specName = self.getSpecName(package)
51e6babb
         return self.mapSerializableSpecObjects[specName].listPatches
3cc43c92
 
2820c61a
     def getSources(self, package):
87815216
         specName = self.getSpecName(package)
51e6babb
         return self.mapSerializableSpecObjects[specName].listSources
5062126c
 
3cc43c92
     def getSHA1(self, package, source):
87815216
         specName = self.getSpecName(package)
d355ea7e
         return self.mapSerializableSpecObjects[specName].checksums.get(source)
3cc43c92
 
2820c61a
     def getPackages(self, package):
87815216
         specName = self.getSpecName(package)
51e6babb
         return self.mapSerializableSpecObjects[specName].listPackages
37022ec3
 
adf248d5
     def getRPMPackages(self, package):
87815216
         specName = self.getSpecName(package)
adf248d5
         return self.mapSerializableSpecObjects[specName].listRPMPackages
 
accc8120
     @staticmethod
     def getReleaseNum(releaseVal):
         releaseNum = releaseVal.find("%")
         if releaseNum != -1:
             return releaseVal[0:releaseNum]
87815216
         else:
             return releaseVal
37022ec3
 
     def compareVersions(self, existingObj, newObject):
87815216
         if StrictVersion(existingObj.version) > StrictVersion(newObject.version):
             return 1
         elif StrictVersion(existingObj.version) < StrictVersion(newObject.version):
             return -1
         else:
             if (int(self.getReleaseNum(existingObj.release)) >
                     int(self.getReleaseNum(newObject.release))):
                 return 1
             else:
                 return -1
 
     def getSpecName(self, package):
         if package in self.mapPackageToSpec:
             specName = self.mapPackageToSpec[package]
             if specName in self.mapSerializableSpecObjects:
51e6babb
                 return specName
87815216
         self.logger.error("Could not able to find " + package + " package from specs")
         raise Exception("Invalid package:" + package)
b5e09fac
 
87815216
     def isRPMPackage(self, package):
         if package in self.mapPackageToSpec:
             specName = self.mapPackageToSpec[package]
             if specName in self.mapSerializableSpecObjects:
af3575c9
                 return True
         return False
b5e09fac
 
cb4e8710
     def getSecurityHardeningOption(self, package):
87815216
         specName = self.getSpecName(package)
cb4e8710
         return self.mapSerializableSpecObjects[specName].securityHardening
 
b5e09fac
     def isCheckAvailable(self, package):
87815216
         specName = self.getSpecName(package)
b5e09fac
         return self.mapSerializableSpecObjects[specName].isCheckAvailable
 
97a9151c
     def getListPackages(self):
87815216
         return list(self.mapSerializableSpecObjects.keys())
5062126c
 
     def getURL(self, package):
87815216
         specName = self.getSpecName(package)
343d89e8
         return self.mapSerializableSpecObjects[specName].url
5062126c
 
     def getSourceURL(self, package):
87815216
         specName = self.getSpecName(package)
343d89e8
         return self.mapSerializableSpecObjects[specName].sourceurl
5062126c
 
     def getLicense(self, package):
87815216
         specName = self.getSpecName(package)
343d89e8
         return self.mapSerializableSpecObjects[specName].license
5062126c
 
2820c61a
     def printAllObjects(self):
87815216
         listSpecs = self.mapSerializableSpecObjects.keys()
2820c61a
         for spec in listSpecs:
87815216
             specObj = self.mapSerializableSpecObjects[spec]
2820c61a
             self.logger.info("-----------Spec:"+specObj.name+"--------------")
             self.logger.info("Version:"+specObj.version)
             self.logger.info("Release:"+specObj.release)
             self.logger.info("SpecFile:"+specObj.specFile)
             self.logger.info(" ")
             self.logger.info("Source Files")
             self.logger.info(specObj.listSources)
             self.logger.info(" ")
             self.logger.info("Patch Files")
             self.logger.info(specObj.listPatches)
             self.logger.info(" ")
             self.logger.info(" ")
             self.logger.info("List RPM packages")
             self.logger.info(specObj.listPackages)
             self.logger.info(" ")
             self.logger.info(" ")
             self.logger.info("Build require packages")
             self.logger.info(specObj.buildRequirePackages)
             self.logger.info(" ")
             self.logger.info(" ")
             self.logger.info("install require packages")
             self.logger.info(specObj.installRequiresAllPackages)
             self.logger.info(" ")
             self.logger.info(specObj.installRequiresPackages)
cb4e8710
             self.logger.info("security_hardening: " + specObj.securityHardening)
2820c61a
             self.logger.info("------------------------------------------------")
5e29a499
 
accc8120
 
45c9260c
 class SPECS(object):
     __instance = None
     specData = None
 
     @staticmethod
     def getData():
         """ Static access method. """
87815216
         if SPECS.__instance is None:
45c9260c
             SPECS()
         return SPECS.__instance.specData
 
     def __init__(self):
         """ Virtually private constructor. """
87815216
         if SPECS.__instance is not None:
45c9260c
             raise Exception("This class is a singleton!")
         else:
             SPECS.__instance = self
         self.initialize()
 
     def initialize(self):
343d89e8
         # Preparse some files
accc8120
         # adding openjre8 version rpm macro
87815216
         if platform.machine() == "x86_64":
343d89e8
             spec = Specutils(constants.specPath + "/openjdk8/openjdk8.spec")
7e05c54b
         else:
             spec = Specutils(constants.specPath + "/openjdk8/openjdk8_aarch64.spec")
         java8version = spec.getVersion()
87815216
         constants.addMacro("JAVA8_VERSION", java8version)
343d89e8
 
accc8120
         # adding kernelversion rpm macro
343d89e8
         spec = Specutils(constants.specPath + "/linux/linux.spec")
         kernelversion = spec.getVersion()
87815216
         constants.addMacro("KERNEL_VERSION", kernelversion)
45c9260c
 
accc8120
         # adding kernelrelease rpm macro
343d89e8
         kernelrelease = spec.getRelease()
87815216
         constants.addMacro("KERNEL_RELEASE", kernelrelease)
45c9260c
 
accc8120
         # adding kernelsubrelease rpm macro
87815216
         a, b, c = kernelversion.split(".")
         kernelsubrelease = ('%02d%02d%03d%03d' % (int(a),
                                                   int(b), int(c),
                                                   int(kernelrelease.split('.')[0])))
45c9260c
         if kernelsubrelease:
87815216
             kernelsubrelease = "." + kernelsubrelease
             constants.addMacro("kernelsubrelease", kernelsubrelease)
45c9260c
 
343d89e8
         # Full parsing
         self.specData = SerializableSpecObjectsUtils(constants.logPath)
         self.specData.readSpecsAndConvertToSerializableObjects(constants.specPath)
45c9260c
 
 
accc8120
 class SpecDependencyGenerator(object):
45c9260c
 
accc8120
     def findTotalRequires(self, mapDependencies, depQue, parent):
45c9260c
         while not depQue.empty():
             specPkg = depQue.get()
accc8120
             try:
                 listRequiredPackages = SPECS.getData().getRequiresForPackage(specPkg)
             except Exception as e:
                 print("Caught Exception:"+str(e))
87815216
                 print(specPkg + " is missing")
accc8120
 
             for depPkg in listRequiredPackages:
                 if depPkg in mapDependencies:
                     if mapDependencies[depPkg] < mapDependencies[specPkg] + 1:
                         mapDependencies[depPkg] = mapDependencies[specPkg] + 1
45c9260c
                         parent[depPkg] = specPkg
accc8120
                         self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
45c9260c
                 else:
accc8120
                     mapDependencies[depPkg] = mapDependencies[specPkg] + 1
45c9260c
                     parent[depPkg] = specPkg
                     depQue.put(depPkg)
 
accc8120
     def findTotalWhoNeedsToBuild(self, depQue, whoNeedsBuild):
45c9260c
         while not depQue.empty():
             specPkg = depQue.get()
accc8120
             listPackagesRequiredToBuild = SPECS.getData().getBuildRequiresForPackage(specPkg)
             for depPkg in listPackagesRequiredToBuild:
                 depSpecPkg = SPECS.getData().getSpecName(depPkg)
                 if depSpecPkg not in whoNeedsBuild:
                     whoNeedsBuild.append(depSpecPkg)
                     depQue.put(depSpecPkg)
 
     def printTree(self, children, curParent, depth):
87815216
         if curParent in children:
45c9260c
             for child in children[curParent]:
accc8120
                 print ("\t" * depth + child)
                 self.printTree(children, child, depth + 1)
45c9260c
 
accc8120
     def getAllPackageNames(self, jsonFilePath):
326d5ca8
         with open(jsonFilePath) as jsonData:
             option_list_json = json.load(jsonData)
             packages = option_list_json["packages"]
             return packages
45c9260c
 
accc8120
     def updateLevels(self, mapDependencies, inPkg, parent, level):
         listPackages = SPECS.getData().getPackages(inPkg)
         for depPkg in SPECS.getData().getRequiresForPackage(inPkg):
             if depPkg in listPackages:
75cb675a
                 continue
accc8120
             if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:
                 mapDependencies[depPkg] = level + 1
45c9260c
                 parent[depPkg] = inPkg
accc8120
                 self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
45c9260c
 
accc8120
     def calculateSpecDependency(self, inputPackages, mapDependencies, parent):
87815216
         depQue = queue.Queue()
accc8120
         for pkg in inputPackages:
             if SPECS.getData().isRPMPackage(pkg):
                 if pkg not in mapDependencies:
                     mapDependencies[pkg] = 0
                     parent[pkg] = ""
                     depQue.put(pkg)
                     self.findTotalRequires(mapDependencies, depQue, parent)
45c9260c
             else:
accc8120
                 print("Could not find spec for "+pkg)
45c9260c
 
accc8120
     def displayDependencies(self, displayOption, inputType, inputValue, allDeps, parent):
         children = {}
45c9260c
         sortedList = []
         for elem in sorted(allDeps.items(), key=operator.itemgetter(1), reverse=True):
             sortedList.append(elem[0])
         # construct all children nodes
87815216
         if displayOption == "tree":
45c9260c
             for k, v in parent.iteritems():
                 children.setdefault(v, []).append(k)
87815216
             if inputType == "json":
                 print("Dependency Mappings for {}".format(inputValue) + " :")
                 print("-" * 52 + " {}".format(children))
                 print("-" * 52)
             if "" in children:
45c9260c
                 for child in children[""]:
87815216
                     print(child)
accc8120
                     self.printTree(children, child, 1)
87815216
                 print("*" * 18 + " {} ".format(len(sortedList)) +
                       "packages in total " + "*" * 18)
45c9260c
             else:
87815216
                 if inputType == "pkg" and len(children) > 0:
accc8120
                     print ("cyclic dependency detected, mappings: \n", children)
45c9260c
 
         # To display a flat list of all packages
87815216
         elif displayOption == "list":
accc8120
             print (sortedList)
45c9260c
 
         # To generate a new JSON file based on given input json file
87815216
         elif displayOption == "json" and inputType == "json":
accc8120
             d = {'packages': sortedList}
             with open(inputValue, 'w') as outfile:
45c9260c
                 json.dump(d, outfile)
 
accc8120
         return sortedList
45c9260c
 
accc8120
     def process(self, inputType, inputValue, displayOption, outputFile=None):
         whoNeedsList = []
         inputPackages = []
         whoNeedsBuild = []
         mapDependencies = {}
         parent = {}
         if inputType == "pkg" or inputType == "json":
             if inputType == "pkg":
                 inputPackages.append(inputValue)
45c9260c
             else:
accc8120
                 inputPackages = self.getAllPackageNames(inputValue)
45c9260c
 
accc8120
             self.calculateSpecDependency(inputPackages, mapDependencies, parent)
             if outputFile is not None:
                 return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent)
             else:
                 return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent)
         elif inputType == "who-needs":
             for depPkg in SPECS.getData().mapPackageToSpec:
                 if inputValue in SPECS.getData().getRequiresForPackage(depPkg):
                     whoNeedsList.append(depPkg)
             print (whoNeedsList)
             return whoNeedsList
         elif inputType == "who-needs-build":
             depQue = queue.Queue()
             depQue.put(inputValue)
             self.findTotalWhoNeedsToBuild(depQue, whoNeedsBuild)
             print ("Following specs need to be build again")
             print (whoNeedsBuild)
             return whoNeedsBuild