from CommandUtils import CommandUtils
from Logger import Logger
import os
import shutil
from constants import constants
import re
from time import sleep
import PullSources
import json
import collections
class PackageUtils(object):
def __init__(self,logName=None,logPath=None):
if logName is None:
self.logName = "PackageUtils"
if logPath is None:
logPath = constants.logPath
self.logName=logName
self.logPath=logPath
self.logger=Logger.getLogger(logName,logPath)
self.runInChrootCommand="./run-in-chroot.sh " + constants.sourcePath + " " + constants.rpmPath;
self.rpmBinary = "rpm"
self.installRPMPackageOptions = "-Uvh"
self.nodepsRPMPackageOptions = "--nodeps"
self.rpmbuildBinary = "rpmbuild"
self.rpmbuildBuildallOption = "-ba --clean"
self.rpmbuildNocheckOption = "--nocheck"
self.rpmbuildCheckOption ="-bi --clean"
self.rpmbuildDistOption = '--define \\\"dist %s\\\"' % constants.dist
self.queryRpmPackageOptions = "-qa"
self.forceRpmPackageOptions = "--force"
self.adjustGCCSpecScript="adjust-gcc-specs.sh"
self.rpmFilesToInstallInAOneShot=""
self.packagesToInstallInAOneShot=""
self.noDepsRPMFilesToInstallInAOneShot=""
self.noDepsPackagesToInstallInAOneShot=""
def getRPMArch(self,rpmName):
arch=""
if rpmName.find("x86_64") != -1:
arch="x86_64"
elif rpmName.find("noarch") != -1:
arch="noarch"
return arch
def getRPMDestDir(self,rpmName,rpmDir):
arch = self.getRPMArch(rpmName)
rpmDestDir=rpmDir+"/"+arch
return rpmDestDir
def copyRPM(self,rpmFile,destDir):
cmdUtils = CommandUtils()
rpmName=os.path.basename(rpmFile)
rpmDestDir=self.getRPMDestDir(rpmName,destDir)
rpmDestPath=rpmDestDir+"/"+rpmName
if os.geteuid()==0:
if not os.path.isdir(rpmDestDir):
cmdUtils.runCommandInShell("mkdir -p "+rpmDestDir)
shutil.copyfile(rpmFile, rpmDestPath)
return rpmDestPath
def installRPM(self,package,chrootID,noDeps=False,destLogPath=None):
# self.logger.info("Installing rpm for package:"+package)
# self.logger.debug("No deps:"+str(noDeps))
rpmfile=self.findRPMFileForGivenPackage(package)
if rpmfile is None:
self.logger.error("No rpm file found for package:"+package)
raise Exception("Missing rpm file: "+package)
rpmDestFile = self.copyRPM(rpmfile, chrootID+constants.topDirPath+"/RPMS")
rpmFile=rpmDestFile.replace(chrootID,"")
if noDeps:
self.noDepsRPMFilesToInstallInAOneShot += " " + rpmFile
self.noDepsPackagesToInstallInAOneShot += " " + package
else:
self.rpmFilesToInstallInAOneShot += " " + rpmFile
self.packagesToInstallInAOneShot += " " + package
def installRPMSInAOneShot(self,chrootID,destLogPath):
chrootCmd=self.runInChrootCommand+" "+chrootID
rpmInstallcmd=self.rpmBinary+" "+ self.installRPMPackageOptions
if self.noDepsRPMFilesToInstallInAOneShot != "":
self.logger.info("Installing nodeps rpms: " + self.noDepsPackagesToInstallInAOneShot)
logFile=chrootID+constants.topDirPath+"/LOGS/install_rpms_nodeps.log"
cmdUtils = CommandUtils()
cmd = rpmInstallcmd+" "+self.nodepsRPMPackageOptions + " " + self.noDepsRPMFilesToInstallInAOneShot
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if destLogPath is not None:
shutil.copy2(logFile, destLogPath)
if not returnVal:
self.logger.error("Unable to install rpms")
raise Exception("RPM installation failed")
if self.rpmFilesToInstallInAOneShot != "":
self.logger.info("Installing rpms: " + self.packagesToInstallInAOneShot)
logFile=chrootID+constants.topDirPath+"/LOGS/install_rpms.log"
cmdUtils = CommandUtils()
cmd=rpmInstallcmd+" "+self.rpmFilesToInstallInAOneShot
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if destLogPath is not None:
shutil.copy2(logFile, destLogPath)
if not returnVal:
self.logger.error("Unable to install rpms")
raise Exception("RPM installation failed")
def copySourcesTobuildroot(self,listSourceFiles,package,destDir):
cmdUtils = CommandUtils()
for source in listSourceFiles:
# Fetch/verify sources if sha1 not None.
sha1 = constants.specData.getSHA1(package, source)
if sha1 is not None:
PullSources.get(source, sha1, constants.sourcePath, constants.pullsourcesConfig)
sourcePath = cmdUtils.findFile(source,constants.sourcePath)
if sourcePath is None or len(sourcePath) == 0:
sourcePath = cmdUtils.findFile(source,constants.specPath)
if sourcePath is None or len(sourcePath) == 0:
if sha1 is None:
self.logger.error("No sha1 found or missing source for "+source)
raise Exception("No sha1 found or missing source for "+source)
else:
self.logger.error("Missing source: "+source+". Cannot find sources for package: "+package)
raise Exception("Missing source")
else:
if sha1 is None:
self.logger.error("No sha1 found for "+source)
raise Exception("No sha1 found")
if len(sourcePath) > 1:
self.logger.error("Multiple sources found for source:"+source+"\n"+ ",".join(sourcePath) +"\nUnable to determine one.")
raise Exception("Multiple sources found")
self.logger.info("Copying... Source path :" + source + " Source filename: " + sourcePath[0])
shutil.copy2(sourcePath[0], destDir)
def copyAdditionalBuildFiles(self,listAdditionalFiles,chrootID):
cmdUtils = CommandUtils()
for additionalFile in listAdditionalFiles:
source = additionalFile["src"].encode('utf-8')
destDir = chrootID + additionalFile["dst"].encode('utf-8')
if os.path.exists(source):
if os.path.isfile(source):
shutil.copy(source, destDir)
else:
shutil.copytree(source, destDir)
def buildRPMSForGivenPackage(self,package,chrootID,listBuildOptionPackages,pkgBuildOptionFile,destLogPath=None):
self.logger.info("Building rpm's for package:"+package)
listSourcesFiles = constants.specData.getSources(package)
listPatchFiles = constants.specData.getPatches(package)
specFile = constants.specData.getSpecFile(package)
specName = constants.specData.getSpecName(package) + ".spec"
chrootSourcePath=chrootID+constants.topDirPath+"/SOURCES/"
chrootSpecPath=constants.topDirPath+"/SPECS/"
chrootLogsFilePath=chrootID+constants.topDirPath+"/LOGS/"+package+".log"
chrootCmd=self.runInChrootCommand+" "+chrootID
shutil.copyfile(specFile, chrootID+chrootSpecPath+specName )
# FIXME: some sources are located in SPECS/.. how to mount?
# if os.geteuid()==0:
self.copySourcesTobuildroot(listSourcesFiles,package,chrootSourcePath)
self.copySourcesTobuildroot(listPatchFiles,package,chrootSourcePath)
listAdditionalFiles = []
macros = []
if package in listBuildOptionPackages:
jsonData = open(pkgBuildOptionFile)
pkg_build_option_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict)
jsonData.close()
pkgs_sorted = pkg_build_option_json.items()
for pkg in pkgs_sorted:
p = str(pkg[0].encode('utf-8'))
if p == package:
filelist = pkg[1]["files"]
for f in filelist:
listAdditionalFiles.append(f)
macrolist = pkg[1]["macros"]
for macro in macrolist:
macros.append(str(macro.encode('utf-8')))
self.copyAdditionalBuildFiles(listAdditionalFiles,chrootID)
#Adding rpm macros
listRPMMacros = constants.specData.getRPMMacros()
for macroName in listRPMMacros.keys():
macros.append(macroName+" "+listRPMMacros[macroName])
listRPMFiles=[]
listSRPMFiles=[]
try:
listRPMFiles,listSRPMFiles = self.buildRPM(chrootSpecPath+"/"+specName,chrootLogsFilePath,chrootCmd,package,macros)
except Exception as e:
self.logger.error("Failed while building rpm:"+package)
raise e
finally:
if destLogPath is not None:
if constants.rpmCheck and package in constants.testForceRPMS and constants.specData.isCheckAvailable(package):
cmd="sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' "+ chrootLogsFilePath
logFile = destLogPath+"/adjustTestFile.log"
returnVal = CommandUtils().runCommandInShell(cmd, logFile)
testLogFile = destLogPath+"/"+package+"-test.log"
shutil.copyfile(chrootLogsFilePath, testLogFile)
else:
shutil.copy2(chrootLogsFilePath, destLogPath)
self.logger.info("RPM build is successful")
for rpmFile in listRPMFiles:
self.copyRPM(chrootID+"/"+rpmFile, constants.rpmPath)
for srpmFile in listSRPMFiles:
srpmDestFile = self.copyRPM(chrootID+"/"+srpmFile, constants.sourceRpmPath)
def buildRPM(self,specFile,logFile,chrootCmd,package,macros):
rpmBuildcmd=self.rpmbuildBinary+" "+self.rpmbuildBuildallOption+" "+self.rpmbuildDistOption
if constants.rpmCheck and package in constants.testForceRPMS:
self.logger.info("#"*(68+2*len(package)))
if not constants.specData.isCheckAvailable(package):
self.logger.info("####### "+package+" MakeCheck is not available. Skipping MakeCheck TEST for "+package+ " #######")
rpmBuildcmd=self.rpmbuildBinary+" --clean"
else:
self.logger.info("####### "+package+" MakeCheck is available. Running MakeCheck TEST for "+package+ " #######")
rpmBuildcmd=self.rpmbuildBinary+" "+self.rpmbuildCheckOption
self.logger.info("#"*(68+2*len(package)))
else:
rpmBuildcmd+=" "+self.rpmbuildNocheckOption
for macro in macros:
rpmBuildcmd+=' --define \\\"%s\\\"' % macro
rpmBuildcmd+=" "+specFile
cmdUtils = CommandUtils()
self.logger.info("Building rpm....")
self.logger.info(rpmBuildcmd)
returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd)
if constants.rpmCheck and package in constants.testForceRPMS:
if not constants.specData.isCheckAvailable(package):
constants.testLogger.info(package+" : N/A")
elif returnVal:
constants.testLogger.info(package+" : PASS")
else:
constants.testLogger.error(package+" : FAIL" )
if constants.rpmCheck:
if not returnVal and constants.rpmCheckStopOnError:
self.logger.error("Checking rpm is failed "+specFile)
raise Exception("RPM check failed")
else:
if not returnVal:
self.logger.error("Building rpm is failed "+specFile)
raise Exception("RPM build failed")
#Extracting rpms created from log file
logfile=open(logFile,'r')
fileContents=logfile.readlines()
logfile.close()
listRPMFiles=[]
listSRPMFiles=[]
for i in range(0,len(fileContents)):
if re.search("^Wrote:",fileContents[i]):
listcontents=fileContents[i].split()
if (len(listcontents) == 2) and listcontents[1].strip()[-4:] == ".rpm" and listcontents[1].find("/RPMS/") != -1:
listRPMFiles.append(listcontents[1])
if (len(listcontents) == 2) and listcontents[1].strip()[-8:] == ".src.rpm" and listcontents[1].find("/SRPMS/") != -1:
listSRPMFiles.append(listcontents[1])
return listRPMFiles,listSRPMFiles
def findRPMFileForGivenPackage(self,package):
cmdUtils = CommandUtils()
version = constants.specData.getVersion(package)
release = constants.specData.getRelease(package)
listFoundRPMFiles = sum([cmdUtils.findFile(package+"-"+version+"-"+release+".x86_64.rpm",constants.rpmPath),
cmdUtils.findFile(package+"-"+version+"-"+release+".noarch.rpm",constants.rpmPath)], [])
if constants.inputRPMSPath is not None:
listFoundRPMFiles = sum([cmdUtils.findFile(package+"-"+version+"-"+release+".x86_64.rpm",constants.inputRPMSPath),
cmdUtils.findFile(package+"-"+version+"-"+release+".noarch.rpm",constants.inputRPMSPath)], listFoundRPMFiles)
if len(listFoundRPMFiles) == 1 :
return listFoundRPMFiles[0]
if len(listFoundRPMFiles) == 0 :
return None
if len(listFoundRPMFiles) > 1 :
self.logger.error("Found multiple rpm files for given package in rpm directory.Unable to determine the rpm file for package:"+package)
raise Exception("Multiple rpm files found")
def findPackageNameFromRPMFile(self,rpmfile):
rpmfile=os.path.basename(rpmfile)
releaseindex=rpmfile.rfind("-")
if releaseindex == -1:
self.logger.error("Invalid rpm file:"+rpmfile)
raise Exception("Invalid RPM")
versionindex=rpmfile[0:releaseindex].rfind("-")
if versionindex == -1:
self.logger.error("Invalid rpm file:"+rpmfile)
raise Exception("Invalid RPM")
packageName=rpmfile[0:versionindex]
return packageName
def findPackageInfoFromRPMFile(self,rpmfile):
rpmfile=os.path.basename(rpmfile)
rpmfile=rpmfile.replace(".x86_64.rpm","")
rpmfile=rpmfile.replace(".noarch.rpm","")
releaseindex=rpmfile.rfind("-")
if releaseindex == -1:
self.logger.error("Invalid rpm file:"+rpmfile)
raise Exception("Invalid RPM")
versionindex=rpmfile[0:releaseindex].rfind("-")
if versionindex == -1:
self.logger.error("Invalid rpm file:"+rpmfile)
raise Exception("Invalid RPM")
packageName=rpmfile[0:versionindex]
version=rpmfile[versionindex+1:releaseindex]
release=rpmfile[releaseindex+1:]
return packageName,version,release
def findInstalledRPMPackages(self, chrootID):
cmd = self.rpmBinary+" "+self.queryRpmPackageOptions
chrootCmd=self.runInChrootCommand+" "+chrootID
cmdUtils=CommandUtils()
result=cmdUtils.runCommandInShell2(cmd, chrootCmd)
if result is not None:
return result.split()
return result
def adjustGCCSpecs(self, package, chrootID, logPath):
opt = " " + constants.specData.getSecurityHardeningOption(package)
cmdUtils=CommandUtils()
cpcmd="cp "+ self.adjustGCCSpecScript+" "+chrootID+"/tmp/"+self.adjustGCCSpecScript
cmd = "/tmp/"+self.adjustGCCSpecScript+opt
logFile = logPath+"/adjustGCCSpecScript.log"
chrootCmd=self.runInChrootCommand+" "+chrootID
returnVal = cmdUtils.runCommandInShell(cpcmd, logFile)
if not returnVal:
self.logger.error("Error during copying the file adjust gcc spec")
raise Exception("Failed while copying adjust gcc spec file")
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if returnVal:
return
self.logger.debug(cmdUtils.runCommandInShell2("ls -la " + chrootID + "/tmp/" + self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" + self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("ps ax"))
self.logger.error("Failed while adjusting gcc specs")
raise Exception("Failed while adjusting gcc specs")