import os
import platform
import queue
import json
import operator
from distutils.version import StrictVersion
from SpecUtils import Specutils
from Logger import Logger
from constants import constants
class SpecObject(object):
def __init__(self):
self.listPackages = []
self.listRPMPackages = []
self.name = ""
self.version = ""
self.release = ""
self.buildRequirePackages = []
self.checkBuildRequirePackages = []
self.installRequiresAllPackages = []
self.installRequiresPackages = {}
self.specFile = ""
self.listSources = []
self.checksums = {}
self.listPatches = []
self.securityHardening = ""
self.url = ""
self.sourceurl = ""
self.license = ""
self.specDefs = {}
class SpecObjectsUtils(object):
def __init__(self, logPath):
self.mapSpecObjects = {}
self.mapPackageToSpec = {}
self.logger = Logger.getLogger("Serializable Spec objects", logPath)
def readSpecsAndConvertToSerializableObjects(self, specFilesPath):
listSpecFiles = []
self.getListSpecFiles(listSpecFiles, specFilesPath)
for specFile in listSpecFiles:
spec = Specutils(specFile)
specName = spec.getBasePackageName()
specObj = SpecObject()
specObj.name = specName
specObj.buildRequirePackages = spec.getBuildRequiresAllPackages()
specObj.installRequiresAllPackages = spec.getRequiresAllPackages()
specObj.checkBuildRequirePackages = spec.getCheckBuildRequiresAllPackages()
specObj.listPackages = spec.getPackageNames()
specObj.specFile = specFile
specObj.version = spec.getVersion()
specObj.release = spec.getRelease()
specObj.listSources = spec.getSourceNames()
specObj.checksums = spec.getChecksums()
specObj.specDefs = spec.getDefinitions()
specObj.listPatches = spec.getPatchNames()
specObj.securityHardening = spec.getSecurityHardeningOption()
specObj.isCheckAvailable = spec.isCheckAvailable()
specObj.license = spec.getLicense()
specObj.url = spec.getURL()
specObj.sourceurl = spec.getSourceURL()
for specPkg in specObj.listPackages:
specObj.installRequiresPackages[specPkg] = spec.getRequires(specPkg)
self.mapPackageToSpec[specPkg] = specName
if spec.getIsRPMPackage(specPkg):
specObj.listRPMPackages.append(specPkg)
if specName in self.mapSpecObjects:
self.mapSpecObjects[specName].append(specObj)
else:
self.mapSpecObjects[specName]=[specObj]
for key, value in self.mapSpecObjects.items():
if len(value) > 1:
self.mapSpecObjects[key] = sorted(value,
key=lambda x : self.compareVersions(x),
reverse=True)
def getListSpecFiles(self, listSpecFiles, path):
for dirEntry in os.listdir(path):
dirEntryPath = os.path.join(path, dirEntry)
if (os.path.isfile(dirEntryPath) and
dirEntryPath.endswith(".spec") and
os.path.basename(dirEntryPath) not in
constants.skipSpecsForArch.get(platform.machine(), [])):
listSpecFiles.append(dirEntryPath)
elif os.path.isdir(dirEntryPath):
self.getListSpecFiles(listSpecFiles, dirEntryPath)
def getBuildRequiresForPackage(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].buildRequirePackages
def getRequiresAllForPackage(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].installRequiresAllPackages
def getRequiresForPackage(self, package, index=0):
specName = self.getSpecName(package)
if package in self.mapSpecObjects[specName][index].installRequiresPackages:
return self.mapSpecObjects[specName][index].installRequiresPackages[package]
return None
def getCheckBuildRequiresForPackage(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].checkBuildRequirePackages
def getSpecObj(self, package):
specName=self.getSpecName(package)
return self.mapSpecObjects[specName]
def getPkgNamesFromObj(self, objlist):
listPkgName=[]
listPkgNames=list(set(objlist))
for name in listPkgNames:
listPkgName.append(name.package)
listPkgName=list(set(listPkgName))
return listPkgName
def getRelease(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].release
def getVersion(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].version
def getSpecFile(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].specFile
def getPatches(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].listPatches
def getSources(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].listSources
def getSHA1(self, package, source, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].checksums.get(source)
def getPackages(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].listPackages
def getRPMPackages(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].listRPMPackages
@staticmethod
def compareVersions(p):
return (StrictVersion(p.version))
def getSpecName(self, package):
if package in self.mapPackageToSpec:
specName = self.mapPackageToSpec[package]
if specName in self.mapSpecObjects:
return specName
self.logger.error("Could not able to find " + package + " package from specs")
raise Exception("Invalid package:" + package)
def isRPMPackage(self, package):
if package in self.mapPackageToSpec:
specName = self.mapPackageToSpec[package]
if specName in self.mapSpecObjects:
return True
return False
def getSecurityHardeningOption(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].securityHardening
def isCheckAvailable(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].isCheckAvailable
def getListPackages(self):
return list(self.mapSpecObjects.keys())
def getURL(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].url
def getSourceURL(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].sourceurl
def getLicense(self, package, index=0):
specName = self.getSpecName(package)
return self.mapSpecObjects[specName][index].license
def getNumberOfVersions(self, package):
specName=self.getSpecName(package)
return len(self.mapSpecObjects[specName])
def printAllObjects(self):
listSpecs = self.mapSpecObjects.keys()
for spec in listSpecs:
specObj = self.mapSpecObjects[spec]
self.logger.info("-----------Spec:"+specObj.name+"--------------")
self.logger.info("Version:"+specObj.version)
self.logger.info("Release:"+specObj.release)
self.logger.info("SpecFile:"+specObj.specFile)
self.logger.info(" ")
self.logger.info("Source Files")
self.logger.info(specObj.listSources)
self.logger.info(" ")
self.logger.info("Patch Files")
self.logger.info(specObj.listPatches)
self.logger.info(" ")
self.logger.info(" ")
self.logger.info("List RPM packages")
self.logger.info(specObj.listPackages)
self.logger.info(" ")
self.logger.info(" ")
self.logger.info("Build require packages")
self.logger.info(self.getPkgNamesFromObj(specObj.buildRequirePackages))
self.logger.info(" ")
self.logger.info(" ")
self.logger.info("install require packages")
self.logger.info(self.getPkgNamesFromObj(specObj.installRequiresAllPackages))
self.logger.info(" ")
self.logger.info(specObj.installRequiresPackages)
self.logger.info("security_hardening: " + specObj.securityHardening)
self.logger.info("------------------------------------------------")
class SPECS(object):
__instance = None
specData = None
@staticmethod
def getData():
""" Static access method. """
if SPECS.__instance is None:
SPECS()
return SPECS.__instance.specData
def __init__(self):
""" Virtually private constructor. """
if SPECS.__instance is not None:
raise Exception("This class is a singleton!")
else:
SPECS.__instance = self
self.initialize()
def initialize(self):
# Preparse some files
# adding openjre8 version rpm macro
if platform.machine() == "x86_64":
spec = Specutils(constants.specPath + "/openjdk8/openjdk8.spec")
else:
spec = Specutils(constants.specPath + "/openjdk8/openjdk8_aarch64.spec")
java8version = spec.getVersion()
constants.addMacro("JAVA8_VERSION", java8version)
# adding openjre9 version rpm macro
if platform.machine() == "x86_64":
spec = Specutils(constants.specPath + "/openjdk9/openjdk9.spec")
java9version = spec.getVersion()
constants.addMacro("JAVA9_VERSION", java9version)
# adding openjre10 version rpm macro
if platform.machine() == "x86_64":
spec = Specutils(constants.specPath + "/openjdk10/openjdk10.spec")
java10version = spec.getVersion()
constants.addMacro("JAVA10_VERSION", java10version)
# adding kernelversion rpm macro
spec = Specutils(constants.specPath + "/linux/linux.spec")
kernelversion = spec.getVersion()
constants.addMacro("KERNEL_VERSION", kernelversion)
# adding kernelrelease rpm macro
kernelrelease = spec.getRelease()
constants.addMacro("KERNEL_RELEASE", kernelrelease)
# adding kernelsubrelease rpm macro
a, b, c = kernelversion.split(".")
kernelsubrelease = ('%02d%02d%03d%03d' % (int(a),
int(b), int(c),
int(kernelrelease.split('.')[0])))
if kernelsubrelease:
kernelsubrelease = "." + kernelsubrelease
constants.addMacro("kernelsubrelease", kernelsubrelease)
# Full parsing
self.specData = SpecObjectsUtils(constants.logPath)
self.specData.readSpecsAndConvertToSerializableObjects(constants.specPath)
class SpecDependencyGenerator(object):
def findTotalRequires(self, mapDependencies, depQue, parent):
while not depQue.empty():
specPkg = depQue.get()
try:
listRequiredPackages = SPECS.getData().getRequiresForPackage(specPkg)
except Exception as e:
print("Caught Exception:"+str(e))
print(specPkg + " is missing")
for depPkg in listRequiredPackages:
if depPkg in mapDependencies:
if mapDependencies[depPkg] < mapDependencies[specPkg] + 1:
mapDependencies[depPkg] = mapDependencies[specPkg] + 1
parent[depPkg] = specPkg
self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
else:
mapDependencies[depPkg] = mapDependencies[specPkg] + 1
parent[depPkg] = specPkg
depQue.put(depPkg)
def findTotalWhoNeedsToBuild(self, depQue, whoNeedsBuild):
while not depQue.empty():
specPkg = depQue.get()
listPackagesRequiredToBuild = SPECS.getData().getBuildRequiresForPackage(specPkg)
for depPkg in listPackagesRequiredToBuild:
depSpecPkg = SPECS.getData().getSpecName(depPkg)
if depSpecPkg not in whoNeedsBuild:
whoNeedsBuild.append(depSpecPkg)
depQue.put(depSpecPkg)
def printTree(self, children, curParent, depth):
if curParent in children:
for child in children[curParent]:
print ("\t" * depth + child)
self.printTree(children, child, depth + 1)
def getAllPackageNames(self, jsonFilePath):
with open(jsonFilePath) as jsonData:
option_list_json = json.load(jsonData)
packages = option_list_json["packages"]
return packages
def updateLevels(self, mapDependencies, inPkg, parent, level):
listPackages = SPECS.getData().getPackages(inPkg)
for depPkg in SPECS.getData().getRequiresForPackage(inPkg):
if depPkg in listPackages:
continue
if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:
mapDependencies[depPkg] = level + 1
parent[depPkg] = inPkg
self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
def calculateSpecDependency(self, inputPackages, mapDependencies, parent):
depQue = queue.Queue()
for pkg in inputPackages:
if SPECS.getData().isRPMPackage(pkg):
if pkg not in mapDependencies:
mapDependencies[pkg] = 0
parent[pkg] = ""
depQue.put(pkg)
self.findTotalRequires(mapDependencies, depQue, parent)
else:
print("Could not find spec for "+pkg)
def displayDependencies(self, displayOption, inputType, inputValue, allDeps, parent):
children = {}
sortedList = []
for elem in sorted(allDeps.items(), key=operator.itemgetter(1), reverse=True):
sortedList.append(elem[0])
# construct all children nodes
if displayOption == "tree":
for k, v in parent.iteritems():
children.setdefault(v, []).append(k)
if inputType == "json":
print("Dependency Mappings for {}".format(inputValue) + " :")
print("-" * 52 + " {}".format(children))
print("-" * 52)
if "" in children:
for child in children[""]:
print(child)
self.printTree(children, child, 1)
print("*" * 18 + " {} ".format(len(sortedList)) +
"packages in total " + "*" * 18)
else:
if inputType == "pkg" and len(children) > 0:
print ("cyclic dependency detected, mappings: \n", children)
# To display a flat list of all packages
elif displayOption == "list":
print (sortedList)
# To generate a new JSON file based on given input json file
elif displayOption == "json" and inputType == "json":
d = {'packages': sortedList}
with open(inputValue, 'w') as outfile:
json.dump(d, outfile)
return sortedList
def process(self, inputType, inputValue, displayOption, outputFile=None):
whoNeedsList = []
inputPackages = []
whoNeedsBuild = []
mapDependencies = {}
parent = {}
if inputType == "pkg" or inputType == "json":
if inputType == "pkg":
inputPackages.append(inputValue)
else:
inputPackages = self.getAllPackageNames(inputValue)
self.calculateSpecDependency(inputPackages, mapDependencies, parent)
if outputFile is not None:
return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent)
else:
return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent)
elif inputType == "who-needs":
for depPkg in SPECS.getData().mapPackageToSpec:
if inputValue in SPECS.getData().getRequiresForPackage(depPkg):
whoNeedsList.append(depPkg)
print (whoNeedsList)
return whoNeedsList
elif inputType == "who-needs-build":
depQue = queue.Queue()
depQue.put(inputValue)
self.findTotalWhoNeedsToBuild(depQue, whoNeedsBuild)
print ("Following specs need to be build again")
print (whoNeedsBuild)
return whoNeedsBuild