support/package-builder/SpecData.py
2820c61a
 import os
0f1fdc4b
 import platform
87815216
 import queue
45c9260c
 import json
 import operator
87815216
 from distutils.version import StrictVersion
 from SpecUtils import Specutils
 from Logger import Logger
45c9260c
 from constants import constants
2820c61a
 
accc8120
 
 
5d822c2d
 class SpecObject(object):
2820c61a
     def __init__(self):
87815216
         self.listPackages = []
         self.listRPMPackages = []
         self.name = ""
         self.version = ""
         self.release = ""
         self.buildRequirePackages = []
         self.checkBuildRequirePackages = []
         self.installRequiresAllPackages = []
         self.installRequiresPackages = {}
         self.specFile = ""
         self.listSources = []
         self.checksums = {}
         self.listPatches = []
         self.securityHardening = ""
         self.url = ""
         self.sourceurl = ""
         self.license = ""
         self.specDefs = {}
2820c61a
 
accc8120
 
5d822c2d
 class SpecObjectsUtils(object):
5062126c
 
87815216
     def __init__(self, logPath):
5d822c2d
         self.mapSpecObjects = {}
87815216
         self.mapPackageToSpec = {}
         self.logger = Logger.getLogger("Serializable Spec objects", logPath)
5062126c
 
87815216
     def readSpecsAndConvertToSerializableObjects(self, specFilesPath):
         listSpecFiles = []
         self.getListSpecFiles(listSpecFiles, specFilesPath)
2820c61a
         for specFile in listSpecFiles:
87815216
             spec = Specutils(specFile)
             specName = spec.getBasePackageName()
5d822c2d
             specObj = SpecObject()
87815216
             specObj.name = specName
e45f5730
             specObj.buildRequirePackages = spec.getBuildRequiresAllPackages()
             specObj.installRequiresAllPackages = spec.getRequiresAllPackages()
             specObj.checkBuildRequirePackages = spec.getCheckBuildRequiresAllPackages()
87815216
             specObj.listPackages = spec.getPackageNames()
             specObj.specFile = specFile
             specObj.version = spec.getVersion()
             specObj.release = spec.getRelease()
             specObj.listSources = spec.getSourceNames()
             specObj.checksums = spec.getChecksums()
             specObj.specDefs = spec.getDefinitions()
             specObj.listPatches = spec.getPatchNames()
             specObj.securityHardening = spec.getSecurityHardeningOption()
             specObj.isCheckAvailable = spec.isCheckAvailable()
             specObj.license = spec.getLicense()
             specObj.url = spec.getURL()
             specObj.sourceurl = spec.getSourceURL()
2820c61a
             for specPkg in specObj.listPackages:
87815216
                 specObj.installRequiresPackages[specPkg] = spec.getRequires(specPkg)
                 self.mapPackageToSpec[specPkg] = specName
adf248d5
                 if spec.getIsRPMPackage(specPkg):
                     specObj.listRPMPackages.append(specPkg)
5d822c2d
             if specName in self.mapSpecObjects:
                 self.mapSpecObjects[specName].append(specObj)
             else:
                 self.mapSpecObjects[specName]=[specObj]
         for key, value in self.mapSpecObjects.items():
             if len(value) > 1:
                 self.mapSpecObjects[key] = sorted(value,
                                                   key=lambda x : self.compareVersions(x),
                                                   reverse=True)
5062126c
 
87815216
     def getListSpecFiles(self, listSpecFiles, path):
2820c61a
         for dirEntry in os.listdir(path):
             dirEntryPath = os.path.join(path, dirEntry)
87815216
             if (os.path.isfile(dirEntryPath) and
                     dirEntryPath.endswith(".spec") and
                     os.path.basename(dirEntryPath) not in
                     constants.skipSpecsForArch.get(platform.machine(), [])):
2820c61a
                 listSpecFiles.append(dirEntryPath)
             elif os.path.isdir(dirEntryPath):
87815216
                 self.getListSpecFiles(listSpecFiles, dirEntryPath)
5062126c
 
5d822c2d
     def getBuildRequiresForPackage(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].buildRequirePackages
5062126c
 
5d822c2d
     def getRequiresAllForPackage(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].installRequiresAllPackages
5062126c
 
5d822c2d
     def getRequiresForPackage(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         if package in self.mapSpecObjects[specName][index].installRequiresPackages:
             return self.mapSpecObjects[specName][index].installRequiresPackages[package]
2820c61a
         return None
9b2f8b85
 
5d822c2d
     def getCheckBuildRequiresForPackage(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].checkBuildRequirePackages
5f40784b
 
4a93976f
     def getSpecObj(self, package):
         specName=self.getSpecName(package)
         return self.mapSpecObjects[specName]
 
e45f5730
     def getPkgNamesFromObj(self, objlist):
         listPkgName=[]
         listPkgNames=list(set(objlist))
         for name in listPkgNames:
                 listPkgName.append(name.package)
         listPkgName=list(set(listPkgName))
         return listPkgName
 
5d822c2d
     def getRelease(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].release
5062126c
 
5d822c2d
     def getVersion(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].version
5062126c
 
5d822c2d
     def getSpecFile(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].specFile
5062126c
 
5d822c2d
     def getPatches(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].listPatches
3cc43c92
 
5d822c2d
     def getSources(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].listSources
5062126c
 
5d822c2d
     def getSHA1(self, package, source, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].checksums.get(source)
3cc43c92
 
5d822c2d
     def getPackages(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].listPackages
37022ec3
 
5d822c2d
     def getRPMPackages(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].listRPMPackages
adf248d5
 
accc8120
     @staticmethod
5d822c2d
     def compareVersions(p):
         return (StrictVersion(p.version))
87815216
 
     def getSpecName(self, package):
         if package in self.mapPackageToSpec:
             specName = self.mapPackageToSpec[package]
5d822c2d
             if specName in self.mapSpecObjects:
51e6babb
                 return specName
87815216
         self.logger.error("Could not able to find " + package + " package from specs")
         raise Exception("Invalid package:" + package)
b5e09fac
 
87815216
     def isRPMPackage(self, package):
         if package in self.mapPackageToSpec:
             specName = self.mapPackageToSpec[package]
5d822c2d
             if specName in self.mapSpecObjects:
af3575c9
                 return True
         return False
b5e09fac
 
5d822c2d
     def getSecurityHardeningOption(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].securityHardening
cb4e8710
 
5d822c2d
     def isCheckAvailable(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].isCheckAvailable
b5e09fac
 
97a9151c
     def getListPackages(self):
5d822c2d
         return list(self.mapSpecObjects.keys())
5062126c
 
5d822c2d
     def getURL(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].url
5062126c
 
5d822c2d
     def getSourceURL(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].sourceurl
5062126c
 
5d822c2d
     def getLicense(self, package, index=0):
87815216
         specName = self.getSpecName(package)
5d822c2d
         return self.mapSpecObjects[specName][index].license
 
     def getNumberOfVersions(self, package):
         specName=self.getSpecName(package)
         return len(self.mapSpecObjects[specName])
5062126c
 
2820c61a
     def printAllObjects(self):
5d822c2d
         listSpecs = self.mapSpecObjects.keys()
2820c61a
         for spec in listSpecs:
5d822c2d
             specObj = self.mapSpecObjects[spec]
2820c61a
             self.logger.info("-----------Spec:"+specObj.name+"--------------")
             self.logger.info("Version:"+specObj.version)
             self.logger.info("Release:"+specObj.release)
             self.logger.info("SpecFile:"+specObj.specFile)
             self.logger.info(" ")
             self.logger.info("Source Files")
             self.logger.info(specObj.listSources)
             self.logger.info(" ")
             self.logger.info("Patch Files")
             self.logger.info(specObj.listPatches)
             self.logger.info(" ")
             self.logger.info(" ")
             self.logger.info("List RPM packages")
             self.logger.info(specObj.listPackages)
             self.logger.info(" ")
             self.logger.info(" ")
             self.logger.info("Build require packages")
e45f5730
             self.logger.info(self.getPkgNamesFromObj(specObj.buildRequirePackages))
2820c61a
             self.logger.info(" ")
             self.logger.info(" ")
             self.logger.info("install require packages")
e45f5730
             self.logger.info(self.getPkgNamesFromObj(specObj.installRequiresAllPackages))
2820c61a
             self.logger.info(" ")
             self.logger.info(specObj.installRequiresPackages)
cb4e8710
             self.logger.info("security_hardening: " + specObj.securityHardening)
2820c61a
             self.logger.info("------------------------------------------------")
5e29a499
 
accc8120
 
45c9260c
 class SPECS(object):
     __instance = None
     specData = None
 
     @staticmethod
     def getData():
         """ Static access method. """
87815216
         if SPECS.__instance is None:
45c9260c
             SPECS()
         return SPECS.__instance.specData
 
     def __init__(self):
         """ Virtually private constructor. """
87815216
         if SPECS.__instance is not None:
45c9260c
             raise Exception("This class is a singleton!")
         else:
             SPECS.__instance = self
         self.initialize()
 
     def initialize(self):
343d89e8
         # Preparse some files
accc8120
         # adding openjre8 version rpm macro
87815216
         if platform.machine() == "x86_64":
343d89e8
             spec = Specutils(constants.specPath + "/openjdk8/openjdk8.spec")
7e05c54b
         else:
             spec = Specutils(constants.specPath + "/openjdk8/openjdk8_aarch64.spec")
         java8version = spec.getVersion()
87815216
         constants.addMacro("JAVA8_VERSION", java8version)
343d89e8
 
accc8120
         # adding kernelversion rpm macro
343d89e8
         spec = Specutils(constants.specPath + "/linux/linux.spec")
         kernelversion = spec.getVersion()
87815216
         constants.addMacro("KERNEL_VERSION", kernelversion)
45c9260c
 
accc8120
         # adding kernelrelease rpm macro
343d89e8
         kernelrelease = spec.getRelease()
87815216
         constants.addMacro("KERNEL_RELEASE", kernelrelease)
45c9260c
 
accc8120
         # adding kernelsubrelease rpm macro
87815216
         a, b, c = kernelversion.split(".")
         kernelsubrelease = ('%02d%02d%03d%03d' % (int(a),
                                                   int(b), int(c),
                                                   int(kernelrelease.split('.')[0])))
45c9260c
         if kernelsubrelease:
87815216
             kernelsubrelease = "." + kernelsubrelease
             constants.addMacro("kernelsubrelease", kernelsubrelease)
45c9260c
 
343d89e8
         # Full parsing
5d822c2d
         self.specData = SpecObjectsUtils(constants.logPath)
343d89e8
         self.specData.readSpecsAndConvertToSerializableObjects(constants.specPath)
45c9260c
 
 
accc8120
 class SpecDependencyGenerator(object):
45c9260c
 
accc8120
     def findTotalRequires(self, mapDependencies, depQue, parent):
45c9260c
         while not depQue.empty():
             specPkg = depQue.get()
accc8120
             try:
                 listRequiredPackages = SPECS.getData().getRequiresForPackage(specPkg)
             except Exception as e:
                 print("Caught Exception:"+str(e))
87815216
                 print(specPkg + " is missing")
accc8120
 
             for depPkg in listRequiredPackages:
                 if depPkg in mapDependencies:
                     if mapDependencies[depPkg] < mapDependencies[specPkg] + 1:
                         mapDependencies[depPkg] = mapDependencies[specPkg] + 1
45c9260c
                         parent[depPkg] = specPkg
accc8120
                         self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
45c9260c
                 else:
accc8120
                     mapDependencies[depPkg] = mapDependencies[specPkg] + 1
45c9260c
                     parent[depPkg] = specPkg
                     depQue.put(depPkg)
 
accc8120
     def findTotalWhoNeedsToBuild(self, depQue, whoNeedsBuild):
45c9260c
         while not depQue.empty():
             specPkg = depQue.get()
accc8120
             listPackagesRequiredToBuild = SPECS.getData().getBuildRequiresForPackage(specPkg)
             for depPkg in listPackagesRequiredToBuild:
                 depSpecPkg = SPECS.getData().getSpecName(depPkg)
                 if depSpecPkg not in whoNeedsBuild:
                     whoNeedsBuild.append(depSpecPkg)
                     depQue.put(depSpecPkg)
 
     def printTree(self, children, curParent, depth):
87815216
         if curParent in children:
45c9260c
             for child in children[curParent]:
accc8120
                 print ("\t" * depth + child)
                 self.printTree(children, child, depth + 1)
45c9260c
 
accc8120
     def getAllPackageNames(self, jsonFilePath):
326d5ca8
         with open(jsonFilePath) as jsonData:
             option_list_json = json.load(jsonData)
             packages = option_list_json["packages"]
             return packages
45c9260c
 
accc8120
     def updateLevels(self, mapDependencies, inPkg, parent, level):
         listPackages = SPECS.getData().getPackages(inPkg)
         for depPkg in SPECS.getData().getRequiresForPackage(inPkg):
             if depPkg in listPackages:
75cb675a
                 continue
accc8120
             if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:
                 mapDependencies[depPkg] = level + 1
45c9260c
                 parent[depPkg] = inPkg
accc8120
                 self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
45c9260c
 
accc8120
     def calculateSpecDependency(self, inputPackages, mapDependencies, parent):
87815216
         depQue = queue.Queue()
accc8120
         for pkg in inputPackages:
             if SPECS.getData().isRPMPackage(pkg):
                 if pkg not in mapDependencies:
                     mapDependencies[pkg] = 0
                     parent[pkg] = ""
                     depQue.put(pkg)
                     self.findTotalRequires(mapDependencies, depQue, parent)
45c9260c
             else:
accc8120
                 print("Could not find spec for "+pkg)
45c9260c
 
accc8120
     def displayDependencies(self, displayOption, inputType, inputValue, allDeps, parent):
         children = {}
45c9260c
         sortedList = []
         for elem in sorted(allDeps.items(), key=operator.itemgetter(1), reverse=True):
             sortedList.append(elem[0])
         # construct all children nodes
87815216
         if displayOption == "tree":
45c9260c
             for k, v in parent.iteritems():
                 children.setdefault(v, []).append(k)
87815216
             if inputType == "json":
                 print("Dependency Mappings for {}".format(inputValue) + " :")
                 print("-" * 52 + " {}".format(children))
                 print("-" * 52)
             if "" in children:
45c9260c
                 for child in children[""]:
87815216
                     print(child)
accc8120
                     self.printTree(children, child, 1)
87815216
                 print("*" * 18 + " {} ".format(len(sortedList)) +
                       "packages in total " + "*" * 18)
45c9260c
             else:
87815216
                 if inputType == "pkg" and len(children) > 0:
accc8120
                     print ("cyclic dependency detected, mappings: \n", children)
45c9260c
 
         # To display a flat list of all packages
87815216
         elif displayOption == "list":
accc8120
             print (sortedList)
45c9260c
 
         # To generate a new JSON file based on given input json file
87815216
         elif displayOption == "json" and inputType == "json":
accc8120
             d = {'packages': sortedList}
             with open(inputValue, 'w') as outfile:
45c9260c
                 json.dump(d, outfile)
 
accc8120
         return sortedList
45c9260c
 
accc8120
     def process(self, inputType, inputValue, displayOption, outputFile=None):
         whoNeedsList = []
         inputPackages = []
         whoNeedsBuild = []
         mapDependencies = {}
         parent = {}
         if inputType == "pkg" or inputType == "json":
             if inputType == "pkg":
                 inputPackages.append(inputValue)
45c9260c
             else:
accc8120
                 inputPackages = self.getAllPackageNames(inputValue)
45c9260c
 
accc8120
             self.calculateSpecDependency(inputPackages, mapDependencies, parent)
             if outputFile is not None:
                 return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent)
             else:
                 return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent)
         elif inputType == "who-needs":
             for depPkg in SPECS.getData().mapPackageToSpec:
                 if inputValue in SPECS.getData().getRequiresForPackage(depPkg):
                     whoNeedsList.append(depPkg)
             print (whoNeedsList)
             return whoNeedsList
         elif inputType == "who-needs-build":
             depQue = queue.Queue()
             depQue.put(inputValue)
             self.findTotalWhoNeedsToBuild(depQue, whoNeedsBuild)
             print ("Following specs need to be build again")
             print (whoNeedsBuild)
             return whoNeedsBuild