2820c61a |
import os |
0f1fdc4b |
import platform |
2820c61a |
import shutil |
518d6a6f |
import re |
b8ab7fb4 |
import random
import string |
87815216 |
from CommandUtils import CommandUtils
from Logger import Logger
from constants import constants
import PullSources |
45c9260c |
from SpecData import SPECS |
e45f5730 |
from distutils.version import LooseVersion |
2820c61a |
class PackageUtils(object): |
97a9151c |
|
87815216 |
def __init__(self, logName=None, logPath=None): |
2820c61a |
if logName is None: |
f93ef2b0 |
logName = "PackageUtils" |
2820c61a |
if logPath is None:
logPath = constants.logPath |
87815216 |
self.logName = logName
self.logPath = logPath |
26b55679 |
self.logger = Logger.getLogger(logName, logPath, constants.logLevel) |
518d6a6f |
self.rpmBinary = "rpm"
self.installRPMPackageOptions = "-Uvh"
self.nodepsRPMPackageOptions = "--nodeps" |
97a9151c |
|
518d6a6f |
self.rpmbuildBinary = "rpmbuild" |
45e37dfb |
self.rpmbuildBuildallOption = "-ba --clean" |
518d6a6f |
self.rpmbuildNocheckOption = "--nocheck" |
87815216 |
self.rpmbuildCheckOption = "-bi --clean" |
518d6a6f |
self.queryRpmPackageOptions = "-qa"
self.forceRpmPackageOptions = "--force" |
7418d2bf |
self.replaceRpmPackageOptions = "--replacepkgs" |
87815216 |
self.adjustGCCSpecScript = "adjust-gcc-specs.sh"
self.rpmFilesToInstallInAOneShot = ""
self.packagesToInstallInAOneShot = ""
self.noDepsRPMFilesToInstallInAOneShot = ""
self.noDepsPackagesToInstallInAOneShot = "" |
8f56b626 |
self.logfnvalue = None |
87815216 |
|
8f56b626 |
def prepRPMforInstall(self, package, version, noDeps=False, destLogPath=None):
rpmfile = self.findRPMFileForGivenPackage(package, version) |
2820c61a |
if rpmfile is None: |
8f56b626 |
self.logger.error("No rpm file found for package: " + package)
raise Exception("Missing rpm file")
rpmName = os.path.basename(rpmfile)
#TODO: path from constants
if "PUBLISHRPMS" in rpmfile:
rpmDestFile = "/publishrpms/"
elif "PUBLISHXRPMS" in rpmfile:
rpmDestFile = "/publishxrpms/"
else:
rpmDestFile = constants.topDirPath + "/RPMS/"
if "noarch" in rpmfile:
rpmDestFile += "noarch/"
else:
rpmDestFile += platform.machine()+"/"
rpmDestFile += rpmName |
2820c61a |
|
542dc63c |
if noDeps: |
8f56b626 |
self.noDepsRPMFilesToInstallInAOneShot += " " + rpmDestFile |
542dc63c |
self.noDepsPackagesToInstallInAOneShot += " " + package
else: |
8f56b626 |
self.rpmFilesToInstallInAOneShot += " " + rpmDestFile |
542dc63c |
self.packagesToInstallInAOneShot += " " + package |
97a9151c |
|
8f56b626 |
def installRPMSInOneShot(self, sandbox): |
87815216 |
rpmInstallcmd = self.rpmBinary + " " + self.installRPMPackageOptions |
8f56b626 |
# TODO: Container sandbox might need + self.forceRpmPackageOptions |
542dc63c |
if self.noDepsRPMFilesToInstallInAOneShot != "": |
26b55679 |
self.logger.debug("Installing nodeps rpms: " + |
87815216 |
self.noDepsPackagesToInstallInAOneShot)
cmd = (rpmInstallcmd+" "+self.nodepsRPMPackageOptions + " " +
self.noDepsRPMFilesToInstallInAOneShot) |
8f56b626 |
returnVal = sandbox.run(cmd, logfn=self.logger.debug)
if returnVal != 0: |
7f9d2e12 |
self.logger.debug("Command Executed:" + cmd) |
8f56b626 |
self.logger.error("Unable to install rpms. Error {}".format(returnVal)) |
542dc63c |
raise Exception("RPM installation failed")
if self.rpmFilesToInstallInAOneShot != "": |
26b55679 |
self.logger.debug("Installing rpms: " + self.packagesToInstallInAOneShot) |
87815216 |
cmd = rpmInstallcmd+" "+self.rpmFilesToInstallInAOneShot |
8f56b626 |
returnVal = sandbox.run(cmd, logfn=self.logger.debug)
if returnVal != 0: |
7f9d2e12 |
self.logger.debug("Command Executed:" + cmd) |
542dc63c |
self.logger.error("Unable to install rpms")
raise Exception("RPM installation failed") |
97a9151c |
|
8f56b626 |
def buildRPMSForGivenPackage(self, sandbox, package, version, destLogPath): |
26b55679 |
self.logger.info("Building package : " + package) |
2820c61a |
|
f93ef2b0 |
listSourcesFiles = SPECS.getData().getSources(package, version)
listPatchFiles = SPECS.getData().getPatches(package, version)
specFile = SPECS.getData().getSpecFile(package, version) |
45c9260c |
specName = SPECS.getData().getSpecName(package) + ".spec" |
8f56b626 |
sourcePath = constants.topDirPath + "/SOURCES/"
specPath = constants.topDirPath + "/SPECS/"
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package)):
logFilePath = destLogPath + "/" + package + "-test.log"
else:
logFilePath = destLogPath + "/" + package + ".log"
sandbox.put(specFile, specPath + specName) |
97a9151c |
|
aac331d9 |
sources_urls, macros = self._getAdditionalBuildOptions(package)
self.logger.debug("Extra macros for " + package + ": " + str(macros))
self.logger.debug("Extra source URLs for " + package + ": " + str(sources_urls))
constants.setExtraSourcesURLs(package, sources_urls)
|
8f56b626 |
self._copySources(sandbox, listSourcesFiles, package, version, sourcePath)
self._copySources(sandbox, listPatchFiles, package, version, sourcePath) |
90d8acae |
|
9b2f8b85 |
#Adding rpm macros |
343d89e8 |
listRPMMacros = constants.userDefinedMacros |
326d5ca8 |
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value) |
9b2f8b85 |
|
87815216 |
listRPMFiles = []
listSRPMFiles = [] |
3ad2cb4c |
try: |
8f56b626 |
listRPMFiles, listSRPMFiles = self._buildRPM(sandbox, specPath + specName,
logFilePath, package, macros) |
26b55679 |
logmsg = package + " build done - RPMs : [ "
for f in listRPMFiles:
logmsg += (os.path.basename(f) + " ")
logmsg += "]\n"
self.logger.info(logmsg) |
3ad2cb4c |
except Exception as e: |
87815216 |
self.logger.error("Failed while building rpm:" + package) |
3ad2cb4c |
raise e
finally: |
8f56b626 |
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package)):
cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + logFilePath)
CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug) |
26b55679 |
self.logger.debug("RPM build is successful") |
b5e09fac |
|
f93ef2b0 |
def findRPMFileForGivenPackage(self, package,version="*"): |
2820c61a |
cmdUtils = CommandUtils() |
4a93976f |
if version == "*": |
f93ef2b0 |
version = SPECS.getData().getHighestVersion(package)
release = SPECS.getData().getRelease(package, version)
buildarch=SPECS.getData().getBuildArch(package, version) |
87815216 |
listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." + |
f93ef2b0 |
buildarch+".rpm", |
87815216 |
constants.rpmPath)], []) |
42ffccb5 |
if constants.inputRPMSPath is not None: |
87815216 |
listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + |
f93ef2b0 |
"." + buildarch+".rpm",
constants.inputRPMSPath)], |
87815216 |
listFoundRPMFiles)
if len(listFoundRPMFiles) == 1: |
2820c61a |
return listFoundRPMFiles[0] |
87815216 |
if len(listFoundRPMFiles) == 0: |
2820c61a |
return None |
87815216 |
if len(listFoundRPMFiles) > 1:
self.logger.error("Found multiple rpm files for given package in rpm directory." + |
f93ef2b0 |
"Unable to determine the rpm file for package:" + package +
" : " + str(listFoundRPMFiles)) |
518d6a6f |
raise Exception("Multiple rpm files found") |
97a9151c |
|
8f56b626 |
def findInstalledRPMPackages(self, sandbox):
rpms = None
def setOutValue(data):
nonlocal rpms
rpms = data |
87815216 |
cmd = self.rpmBinary + " " + self.queryRpmPackageOptions |
8f56b626 |
sandbox.run(cmd, logfn=setOutValue)
return rpms.split() |
75a2daa5 |
|
8f56b626 |
def adjustGCCSpecs(self, sandbox, package, version): |
f93ef2b0 |
opt = " " + SPECS.getData().getSecurityHardeningOption(package, version) |
8f56b626 |
sandbox.put(self.adjustGCCSpecScript, "/tmp") |
87815216 |
cmd = "/tmp/" + self.adjustGCCSpecScript + opt |
8f56b626 |
returnVal = sandbox.run(cmd, logfn=self.logger.debug)
if returnVal == 0: |
610ab83e |
return
|
8f56b626 |
# in debugging ...
sandbox.run("ls -la /tmp/" + self.adjustGCCSpecScript,
logfn=self.logger.debug)
sandbox.run("lsof /tmp/" + self.adjustGCCSpecScript,
logfn=self.logger.debug)
sandbox.run("ps ax", logfn=self.logger.debug) |
7418d2bf |
self.logger.error("Failed while adjusting gcc specs")
raise Exception("Failed while adjusting gcc specs")
|
f93ef2b0 |
def _verifyShaAndGetSourcePath(self, source, package, version): |
326d5ca8 |
cmdUtils = CommandUtils()
# Fetch/verify sources if sha1 not None. |
f93ef2b0 |
sha1 = SPECS.getData().getSHA1(package, version, source) |
326d5ca8 |
if sha1 is not None: |
a00911af |
PullSources.get(package, source, sha1, constants.sourcePath, |
aac331d9 |
constants.getPullSourcesURLs(package), self.logger) |
326d5ca8 |
sourcePath = cmdUtils.findFile(source, constants.sourcePath)
if not sourcePath:
sourcePath = cmdUtils.findFile(source, constants.specPath)
if not sourcePath:
if sha1 is None:
self.logger.error("No sha1 found or missing source for " + source)
raise Exception("No sha1 found or missing source for " + source)
else:
self.logger.error("Missing source: " + source +
". Cannot find sources for package: " + package)
raise Exception("Missing source")
else:
if sha1 is None:
self.logger.error("No sha1 found for "+source)
raise Exception("No sha1 found")
if len(sourcePath) > 1:
self.logger.error("Multiple sources found for source:" + source + "\n" +
",".join(sourcePath) +"\nUnable to determine one.")
raise Exception("Multiple sources found")
return sourcePath
|
8f56b626 |
def _copySources(self, sandbox, listSourceFiles, package, version, destDir):
# Fetch and verify sha1 if missing |
326d5ca8 |
for source in listSourceFiles: |
f93ef2b0 |
sourcePath = self._verifyShaAndGetSourcePath(source, package, version) |
26b55679 |
self.logger.debug("Copying... Source path :" + source + |
326d5ca8 |
" Source filename: " + sourcePath[0]) |
8f56b626 |
sandbox.put(sourcePath[0], destDir) |
326d5ca8 |
|
aac331d9 |
def _getAdditionalBuildOptions(self, package):
pullsources_urls = [] |
326d5ca8 |
macros = []
if package in constants.buildOptions.keys():
pkg = constants.buildOptions[package] |
aac331d9 |
pullsources_urls.extend(pkg["pullsources"])
macros.extend(pkg["macros"])
return pullsources_urls, macros |
326d5ca8 |
|
8f56b626 |
def _buildRPM(self, sandbox, specFile, logFile, package, macros): |
326d5ca8 |
rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption
if constants.rpmCheck and package in constants.testForceRPMS: |
26b55679 |
self.logger.debug("#" * (68 + 2 * len(package))) |
326d5ca8 |
if not SPECS.getData().isCheckAvailable(package): |
26b55679 |
self.logger.debug("####### " + package + |
326d5ca8 |
" MakeCheck is not available. Skipping MakeCheck TEST for " +
package + " #######")
rpmBuildcmd = self.rpmbuildBinary + " --clean"
else: |
26b55679 |
self.logger.debug("####### " + package + |
326d5ca8 |
" MakeCheck is available. Running MakeCheck TEST for " +
package + " #######")
rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption |
26b55679 |
self.logger.debug("#" * (68 + 2 * len(package))) |
326d5ca8 |
else:
rpmBuildcmd += " " + self.rpmbuildNocheckOption
for macro in macros: |
8f56b626 |
rpmBuildcmd += ' --define \"%s\"' % macro |
326d5ca8 |
rpmBuildcmd += " " + specFile
|
26b55679 |
self.logger.debug("Building rpm....")
self.logger.debug(rpmBuildcmd) |
8f56b626 |
returnVal = sandbox.run(rpmBuildcmd, logfile = logFile)
|
326d5ca8 |
if constants.rpmCheck and package in constants.testForceRPMS:
if not SPECS.getData().isCheckAvailable(package): |
26b55679 |
constants.testLogger.debug(package + " : N/A") |
8f56b626 |
elif returnVal == 0: |
26b55679 |
constants.testLogger.debug(package + " : PASS") |
326d5ca8 |
else: |
26b55679 |
constants.testLogger.debug(package + " : FAIL") |
326d5ca8 |
if constants.rpmCheck: |
8f56b626 |
if returnVal != 0 and constants.rpmCheckStopOnError: |
326d5ca8 |
self.logger.error("Checking rpm is failed " + specFile)
raise Exception("RPM check failed")
else: |
8f56b626 |
if returnVal != 0: |
326d5ca8 |
self.logger.error("Building rpm is failed " + specFile)
raise Exception("RPM build failed")
#Extracting rpms created from log file
listRPMFiles = []
listSRPMFiles = [] |
8f56b626 |
stageLogFile = logFile.replace(constants.topDirPath + "/LOGS", constants.logPath )
with open(stageLogFile, 'r') as logfile: |
326d5ca8 |
fileContents = logfile.readlines()
for i in range(0, len(fileContents)):
if re.search("^Wrote:", fileContents[i]):
listcontents = fileContents[i].split()
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".rpm") and
"/RPMS/" in listcontents[1]):
listRPMFiles.append(listcontents[1])
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".src.rpm") and
"/SRPMS/" in listcontents[1]):
listSRPMFiles.append(listcontents[1])
return listRPMFiles, listSRPMFiles
|