2820c61a |
from CommandUtils import CommandUtils
from Logger import Logger
import os
import shutil
from constants import constants |
518d6a6f |
import re |
75a2daa5 |
from time import sleep |
3cc43c92 |
import PullSources |
90d8acae |
import json
import collections |
2820c61a |
class PackageUtils(object): |
7322111a |
|
2820c61a |
def __init__(self,logName=None,logPath=None):
if logName is None:
self.logName = "PackageUtils"
if logPath is None:
logPath = constants.logPath
self.logName=logName
self.logPath=logPath
self.logger=Logger.getLogger(logName,logPath) |
542dc63c |
self.runInChrootCommand="./run-in-chroot.sh " + constants.sourcePath + " " + constants.rpmPath; |
518d6a6f |
self.rpmBinary = "rpm"
self.installRPMPackageOptions = "-Uvh"
self.nodepsRPMPackageOptions = "--nodeps" |
7322111a |
|
518d6a6f |
self.rpmbuildBinary = "rpmbuild" |
45e37dfb |
self.rpmbuildBuildallOption = "-ba --clean" |
518d6a6f |
self.rpmbuildNocheckOption = "--nocheck" |
4c54f74b |
self.rpmbuildDistOption = '--define \\\"dist %s\\\"' % constants.dist |
518d6a6f |
self.queryRpmPackageOptions = "-qa"
self.forceRpmPackageOptions = "--force" |
75a2daa5 |
self.adjustGCCSpecScript="adjust-gcc-specs.sh" |
542dc63c |
self.rpmFilesToInstallInAOneShot=""
self.packagesToInstallInAOneShot=""
self.noDepsRPMFilesToInstallInAOneShot=""
self.noDepsPackagesToInstallInAOneShot="" |
7322111a |
|
2cfb758d |
def getRPMArch(self,rpmName): |
2820c61a |
arch=""
if rpmName.find("x86_64") != -1: |
2cfb758d |
arch="x86_64" |
2820c61a |
elif rpmName.find("noarch") != -1:
arch="noarch" |
2cfb758d |
return arch
def getRPMDestDir(self,rpmName,rpmDir):
arch = self.getRPMArch(rpmName) |
2820c61a |
rpmDestDir=rpmDir+"/"+arch
return rpmDestDir |
7322111a |
|
2820c61a |
def copyRPM(self,rpmFile,destDir):
cmdUtils = CommandUtils()
rpmName=os.path.basename(rpmFile)
rpmDestDir=self.getRPMDestDir(rpmName,destDir)
rpmDestPath=rpmDestDir+"/"+rpmName |
542dc63c |
if os.geteuid()==0:
if not os.path.isdir(rpmDestDir):
cmdUtils.runCommandInShell("mkdir -p "+rpmDestDir)
shutil.copyfile(rpmFile, rpmDestPath) |
2820c61a |
return rpmDestPath |
7322111a |
|
2820c61a |
def installRPM(self,package,chrootID,noDeps=False,destLogPath=None): |
542dc63c |
# self.logger.info("Installing rpm for package:"+package)
# self.logger.debug("No deps:"+str(noDeps)) |
7322111a |
|
2820c61a |
rpmfile=self.findRPMFileForGivenPackage(package)
if rpmfile is None: |
518d6a6f |
self.logger.error("No rpm file found for package:"+package)
raise Exception("Missing rpm file") |
2820c61a |
rpmDestFile = self.copyRPM(rpmfile, chrootID+constants.topDirPath+"/RPMS")
rpmFile=rpmDestFile.replace(chrootID,"") |
542dc63c |
if noDeps:
self.noDepsRPMFilesToInstallInAOneShot += " " + rpmFile
self.noDepsPackagesToInstallInAOneShot += " " + package
else:
self.rpmFilesToInstallInAOneShot += " " + rpmFile
self.packagesToInstallInAOneShot += " " + package |
7322111a |
|
542dc63c |
def installRPMSInAOneShot(self,chrootID,destLogPath): |
2820c61a |
chrootCmd=self.runInChrootCommand+" "+chrootID |
518d6a6f |
rpmInstallcmd=self.rpmBinary+" "+ self.installRPMPackageOptions |
542dc63c |
if self.noDepsRPMFilesToInstallInAOneShot != "":
self.logger.info("Installing nodeps rpms: " + self.noDepsPackagesToInstallInAOneShot)
logFile=chrootID+constants.topDirPath+"/LOGS/install_rpms_nodeps.log"
cmdUtils = CommandUtils()
cmd = rpmInstallcmd+" "+self.nodepsRPMPackageOptions + " " + self.noDepsRPMFilesToInstallInAOneShot
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if destLogPath is not None:
shutil.copy2(logFile, destLogPath)
if not returnVal:
self.logger.error("Unable to install rpms")
raise Exception("RPM installation failed")
if self.rpmFilesToInstallInAOneShot != "":
self.logger.info("Installing rpms: " + self.packagesToInstallInAOneShot)
logFile=chrootID+constants.topDirPath+"/LOGS/install_rpms.log"
cmdUtils = CommandUtils()
cmd=rpmInstallcmd+" "+self.rpmFilesToInstallInAOneShot
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if destLogPath is not None:
shutil.copy2(logFile, destLogPath)
if not returnVal:
self.logger.error("Unable to install rpms")
raise Exception("RPM installation failed") |
7322111a |
|
2820c61a |
def copySourcesTobuildroot(self,listSourceFiles,package,destDir):
cmdUtils = CommandUtils()
for source in listSourceFiles: |
3cc43c92 |
# Fetch/verify sources if sha1 not None.
sha1 = constants.specData.getSHA1(package, source)
if sha1 is not None: |
a59d15a5 |
PullSources.get(source, sha1, constants.sourcePath, constants.pullsourcesConfig, self.logger) |
3cc43c92 |
|
518d6a6f |
sourcePath = cmdUtils.findFile(source,constants.sourcePath) |
2820c61a |
if sourcePath is None or len(sourcePath) == 0: |
518d6a6f |
sourcePath = cmdUtils.findFile(source,constants.specPath) |
7fba6943 |
if sourcePath is None or len(sourcePath) == 0:
if sha1 is None:
self.logger.error("No sha1 found or missing source for "+source)
raise Exception("No sha1 found or missing source")
else:
self.logger.error("Missing source: "+source+". Cannot find sources for package: "+package)
raise Exception("Missing source")
else:
if sha1 is None:
self.logger.error("No sha1 found for "+source)
raise Exception("No sha1 found") |
2820c61a |
if len(sourcePath) > 1: |
518d6a6f |
self.logger.error("Multiple sources found for source:"+source+"\n"+ ",".join(sourcePath) +"\nUnable to determine one.")
raise Exception("Multiple sources found") |
2689a7f2 |
self.logger.info("Copying... Source path :" + source + " Source filename: " + sourcePath[0]) |
90d8acae |
shutil.copy2(sourcePath[0], destDir) |
7322111a |
|
1cf11562 |
def copyAdditionalBuildFiles(self,listAdditionalFiles,chrootID): |
90d8acae |
cmdUtils = CommandUtils() |
1cf11562 |
for additionalFile in listAdditionalFiles:
source = additionalFile["src"].encode('utf-8')
destDir = chrootID + additionalFile["dst"].encode('utf-8') |
90d8acae |
if os.path.exists(source): |
1cf11562 |
if os.path.isfile(source):
shutil.copy(source, destDir)
else:
shutil.copytree(source, destDir) |
7322111a |
|
90d8acae |
def buildRPMSForGivenPackage(self,package,chrootID,listBuildOptionPackages,pkgBuildOptionFile,destLogPath=None): |
518d6a6f |
self.logger.info("Building rpm's for package:"+package) |
2820c61a |
listSourcesFiles = constants.specData.getSources(package)
listPatchFiles = constants.specData.getPatches(package)
specFile = constants.specData.getSpecFile(package)
specName = constants.specData.getSpecName(package) + ".spec" |
7322111a |
|
2820c61a |
chrootSourcePath=chrootID+constants.topDirPath+"/SOURCES/"
chrootSpecPath=constants.topDirPath+"/SPECS/"
chrootLogsFilePath=chrootID+constants.topDirPath+"/LOGS/"+package+".log"
chrootCmd=self.runInChrootCommand+" "+chrootID
shutil.copyfile(specFile, chrootID+chrootSpecPath+specName ) |
7322111a |
|
e8435c63 |
# FIXME: some sources are located in SPECS/.. how to mount?
# if os.geteuid()==0:
self.copySourcesTobuildroot(listSourcesFiles,package,chrootSourcePath) |
2820c61a |
self.copySourcesTobuildroot(listPatchFiles,package,chrootSourcePath) |
90d8acae |
listAdditionalFiles = []
macros = []
if package in listBuildOptionPackages:
jsonData = open(pkgBuildOptionFile)
pkg_build_option_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict)
jsonData.close()
pkgs_sorted = pkg_build_option_json.items()
for pkg in pkgs_sorted:
p = str(pkg[0].encode('utf-8'))
if p == package:
filelist = pkg[1]["files"]
for f in filelist: |
1cf11562 |
listAdditionalFiles.append(f) |
90d8acae |
macrolist = pkg[1]["macros"]
for macro in macrolist:
macros.append(str(macro.encode('utf-8'))) |
7322111a |
|
1cf11562 |
self.copyAdditionalBuildFiles(listAdditionalFiles,chrootID) |
90d8acae |
|
522ac0c7 |
#Adding rpm macros
listRPMMacros = constants.specData.getRPMMacros()
for macroName in listRPMMacros.keys():
macros.append(macroName+" "+listRPMMacros[macroName])
|
3ad2cb4c |
listRPMFiles=[] |
d2526915 |
listSRPMFiles=[] |
3ad2cb4c |
try: |
90d8acae |
listRPMFiles,listSRPMFiles = self.buildRPM(chrootSpecPath+"/"+specName,chrootLogsFilePath,chrootCmd,package,macros) |
3ad2cb4c |
except Exception as e:
self.logger.error("Failed while building rpm:"+package)
raise e
finally:
if destLogPath is not None:
shutil.copy2(chrootLogsFilePath, destLogPath) |
2cfb758d |
self.logger.info("RPM build is successful")
arch = self.getRPMArch(listRPMFiles[0]) |
7322111a |
|
2820c61a |
for rpmFile in listRPMFiles: |
7322111a |
rpmDestFilePath = self.copyRPM(chrootID+"/"+rpmFile, constants.rpmPath)
|
d2526915 |
for srpmFile in listSRPMFiles: |
7322111a |
srpmDestFile = self.copyRPM(chrootID+"/"+srpmFile, constants.sourceRpmPath) |
d2526915 |
|
90d8acae |
def buildRPM(self,specFile,logFile,chrootCmd,package,macros): |
7322111a |
|
bc90d582 |
rpmBuildcmd= self.rpmbuildBinary+" "+self.rpmbuildBuildallOption+" "+self.rpmbuildDistOption
if not constants.rpmCheck:
rpmBuildcmd+=" "+self.rpmbuildNocheckOption |
90d8acae |
for macro in macros:
rpmBuildcmd+=' --define \\\"%s\\\"' % macro |
518d6a6f |
rpmBuildcmd+=" "+specFile |
7322111a |
|
2820c61a |
cmdUtils = CommandUtils() |
42ffccb5 |
self.logger.info("Building rpm....") |
2c153d29 |
self.logger.info(rpmBuildcmd) |
518d6a6f |
returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd) |
2820c61a |
if not returnVal:
self.logger.error("Building rpm is failed "+specFile) |
518d6a6f |
raise Exception("RPM Build failed") |
7322111a |
|
518d6a6f |
#Extracting rpms created from log file
logfile=open(logFile,'r')
fileContents=logfile.readlines()
logfile.close()
listRPMFiles=[] |
d2526915 |
listSRPMFiles=[] |
518d6a6f |
for i in range(0,len(fileContents)):
if re.search("^Wrote:",fileContents[i]):
listcontents=fileContents[i].split()
if (len(listcontents) == 2) and listcontents[1].strip()[-4:] == ".rpm" and listcontents[1].find("/RPMS/") != -1:
listRPMFiles.append(listcontents[1]) |
d2526915 |
if (len(listcontents) == 2) and listcontents[1].strip()[-8:] == ".src.rpm" and listcontents[1].find("/SRPMS/") != -1:
listSRPMFiles.append(listcontents[1]) |
7322111a |
return listRPMFiles,listSRPMFiles
|
246d16b8 |
def findRPMFileForGivenPackage(self, package, version = "*"): |
2820c61a |
cmdUtils = CommandUtils() |
246d16b8 |
release = "*"
# If no version is specified, use the latest from the source
# code.
if version == "*":
version = constants.specData.getVersion(package)
release = constants.specData.getRelease(package) |
cd022cc1 |
listFoundRPMFiles = sum([cmdUtils.findFile(package+"-"+version+"-"+release+".x86_64.rpm",constants.rpmPath),
cmdUtils.findFile(package+"-"+version+"-"+release+".noarch.rpm",constants.rpmPath)], []) |
42ffccb5 |
if constants.inputRPMSPath is not None:
listFoundRPMFiles = sum([cmdUtils.findFile(package+"-"+version+"-"+release+".x86_64.rpm",constants.inputRPMSPath), |
7322111a |
cmdUtils.findFile(package+"-"+version+"-"+release+".noarch.rpm",constants.inputRPMSPath)], listFoundRPMFiles) |
2820c61a |
if len(listFoundRPMFiles) == 1 :
return listFoundRPMFiles[0]
if len(listFoundRPMFiles) == 0 :
return None
if len(listFoundRPMFiles) > 1 : |
518d6a6f |
self.logger.error("Found multiple rpm files for given package in rpm directory.Unable to determine the rpm file for package:"+package)
raise Exception("Multiple rpm files found") |
7322111a |
|
2820c61a |
def findPackageNameFromRPMFile(self,rpmfile):
rpmfile=os.path.basename(rpmfile)
releaseindex=rpmfile.rfind("-")
if releaseindex == -1:
self.logger.error("Invalid rpm file:"+rpmfile) |
518d6a6f |
raise Exception("Invalid RPM") |
2820c61a |
versionindex=rpmfile[0:releaseindex].rfind("-")
if versionindex == -1:
self.logger.error("Invalid rpm file:"+rpmfile) |
518d6a6f |
raise Exception("Invalid RPM") |
2820c61a |
packageName=rpmfile[0:versionindex] |
7322111a |
return packageName |
af3575c9 |
def findPackageInfoFromRPMFile(self,rpmfile):
rpmfile=os.path.basename(rpmfile)
rpmfile=rpmfile.replace(".x86_64.rpm","")
rpmfile=rpmfile.replace(".noarch.rpm","")
releaseindex=rpmfile.rfind("-")
if releaseindex == -1:
self.logger.error("Invalid rpm file:"+rpmfile)
raise Exception("Invalid RPM")
versionindex=rpmfile[0:releaseindex].rfind("-")
if versionindex == -1:
self.logger.error("Invalid rpm file:"+rpmfile)
raise Exception("Invalid RPM")
packageName=rpmfile[0:versionindex]
version=rpmfile[versionindex+1:releaseindex]
release=rpmfile[releaseindex+1:]
return packageName,version,release |
2cfb758d |
|
518d6a6f |
def findInstalledRPMPackages(self, chrootID):
cmd = self.rpmBinary+" "+self.queryRpmPackageOptions
chrootCmd=self.runInChrootCommand+" "+chrootID
cmdUtils=CommandUtils()
result=cmdUtils.runCommandInShell2(cmd, chrootCmd) |
610ab83e |
if result is not None:
return result.split() |
518d6a6f |
return result |
75a2daa5 |
def adjustGCCSpecs(self, package, chrootID, logPath):
opt = " " + constants.specData.getSecurityHardeningOption(package)
cmdUtils=CommandUtils() |
8d0a7e93 |
cpcmd="cp "+ self.adjustGCCSpecScript+" "+chrootID+"/tmp/"+self.adjustGCCSpecScript |
75a2daa5 |
cmd = "/tmp/"+self.adjustGCCSpecScript+opt
logFile = logPath+"/adjustGCCSpecScript.log"
chrootCmd=self.runInChrootCommand+" "+chrootID |
8d0a7e93 |
returnVal = cmdUtils.runCommandInShell(cpcmd, logFile)
if not returnVal: |
42ffccb5 |
self.logger.error("Error during copying the file adjust gcc spec") |
36c2f510 |
raise Exception("Failed while copying adjust gcc spec file") |
610ab83e |
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if returnVal:
return
self.logger.debug(cmdUtils.runCommandInShell2("ls -la " + chrootID + "/tmp/" + self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" + self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("ps ax"))
self.logger.error("Failed while adjusting gcc specs") |
36c2f510 |
raise Exception("Failed while adjusting gcc specs") |