support/package-builder/PackageUtils.py
2820c61a
 import os
0f1fdc4b
 import platform
2820c61a
 import shutil
518d6a6f
 import re
b8ab7fb4
 import random
 import string
87815216
 from CommandUtils import CommandUtils
 from Logger import Logger
 from constants import constants
 import PullSources
45c9260c
 from SpecData import SPECS
e45f5730
 from distutils.version import LooseVersion
2820c61a
 
 class PackageUtils(object):
97a9151c
 
87815216
     def __init__(self, logName=None, logPath=None):
2820c61a
         if logName is None:
f93ef2b0
             logName = "PackageUtils"
2820c61a
         if logPath is None:
             logPath = constants.logPath
87815216
         self.logName = logName
         self.logPath = logPath
26b55679
         self.logger = Logger.getLogger(logName, logPath, constants.logLevel)
518d6a6f
         self.rpmBinary = "rpm"
         self.installRPMPackageOptions = "-Uvh"
         self.nodepsRPMPackageOptions = "--nodeps"
97a9151c
 
518d6a6f
         self.rpmbuildBinary = "rpmbuild"
45e37dfb
         self.rpmbuildBuildallOption = "-ba --clean"
518d6a6f
         self.rpmbuildNocheckOption = "--nocheck"
87815216
         self.rpmbuildCheckOption = "-bi --clean"
518d6a6f
         self.queryRpmPackageOptions = "-qa"
         self.forceRpmPackageOptions = "--force"
7418d2bf
         self.replaceRpmPackageOptions = "--replacepkgs"
87815216
         self.adjustGCCSpecScript = "adjust-gcc-specs.sh"
         self.rpmFilesToInstallInAOneShot = ""
         self.packagesToInstallInAOneShot = ""
         self.noDepsRPMFilesToInstallInAOneShot = ""
         self.noDepsPackagesToInstallInAOneShot = ""
8f56b626
         self.logfnvalue = None
87815216
 
8f56b626
     def prepRPMforInstall(self, package, version, noDeps=False, destLogPath=None):
         rpmfile = self.findRPMFileForGivenPackage(package, version)
2820c61a
         if rpmfile is None:
8f56b626
             self.logger.error("No rpm file found for package: " + package)
             raise Exception("Missing rpm file")
 
         rpmName = os.path.basename(rpmfile)
         #TODO: path from constants
         if "PUBLISHRPMS" in rpmfile:
             rpmDestFile = "/publishrpms/"
         elif "PUBLISHXRPMS" in rpmfile:
             rpmDestFile = "/publishxrpms/"
         else:
             rpmDestFile = constants.topDirPath + "/RPMS/"
         if "noarch" in rpmfile:
             rpmDestFile += "noarch/"
         else:
             rpmDestFile += platform.machine()+"/"
         rpmDestFile += rpmName
2820c61a
 
542dc63c
         if noDeps:
8f56b626
             self.noDepsRPMFilesToInstallInAOneShot += " " + rpmDestFile
542dc63c
             self.noDepsPackagesToInstallInAOneShot += " " + package
         else:
8f56b626
             self.rpmFilesToInstallInAOneShot += " " + rpmDestFile
542dc63c
             self.packagesToInstallInAOneShot += " " + package
97a9151c
 
8f56b626
     def installRPMSInOneShot(self, sandbox):
87815216
         rpmInstallcmd = self.rpmBinary + " " + self.installRPMPackageOptions
8f56b626
         # TODO: Container sandbox might need  + self.forceRpmPackageOptions
542dc63c
         if self.noDepsRPMFilesToInstallInAOneShot != "":
26b55679
             self.logger.debug("Installing nodeps rpms: " +
87815216
                              self.noDepsPackagesToInstallInAOneShot)
             cmd = (rpmInstallcmd+" "+self.nodepsRPMPackageOptions + " " +
                    self.noDepsRPMFilesToInstallInAOneShot)
8f56b626
             returnVal = sandbox.run(cmd, logfn=self.logger.debug)
             if returnVal != 0:
7f9d2e12
                 self.logger.debug("Command Executed:" + cmd)
8f56b626
                 self.logger.error("Unable to install rpms. Error {}".format(returnVal))
542dc63c
                 raise Exception("RPM installation failed")
         if self.rpmFilesToInstallInAOneShot != "":
26b55679
             self.logger.debug("Installing rpms: " + self.packagesToInstallInAOneShot)
87815216
             cmd = rpmInstallcmd+" "+self.rpmFilesToInstallInAOneShot
8f56b626
             returnVal = sandbox.run(cmd, logfn=self.logger.debug)
             if returnVal != 0:
7f9d2e12
                 self.logger.debug("Command Executed:" + cmd)
542dc63c
                 self.logger.error("Unable to install rpms")
                 raise Exception("RPM installation failed")
97a9151c
 
8f56b626
     def buildRPMSForGivenPackage(self, sandbox, package, version, destLogPath):
26b55679
         self.logger.info("Building package : " + package)
2820c61a
 
f93ef2b0
         listSourcesFiles = SPECS.getData().getSources(package, version)
         listPatchFiles = SPECS.getData().getPatches(package, version)
         specFile = SPECS.getData().getSpecFile(package, version)
45c9260c
         specName = SPECS.getData().getSpecName(package) + ".spec"
8f56b626
         sourcePath = constants.topDirPath + "/SOURCES/"
         specPath = constants.topDirPath + "/SPECS/"
         if (constants.rpmCheck and
                 package in constants.testForceRPMS and
                 SPECS.getData().isCheckAvailable(package)):
             logFilePath = destLogPath + "/" + package + "-test.log"
         else:
             logFilePath = destLogPath + "/" + package + ".log"
         sandbox.put(specFile, specPath + specName)
97a9151c
 
aac331d9
         sources_urls, macros = self._getAdditionalBuildOptions(package)
         self.logger.debug("Extra macros for " + package + ": " + str(macros))
         self.logger.debug("Extra source URLs for " + package + ": " + str(sources_urls))
         constants.setExtraSourcesURLs(package, sources_urls)
 
8f56b626
         self._copySources(sandbox, listSourcesFiles, package, version, sourcePath)
         self._copySources(sandbox, listPatchFiles, package, version, sourcePath)
90d8acae
 
9b2f8b85
         #Adding rpm macros
343d89e8
         listRPMMacros = constants.userDefinedMacros
326d5ca8
         for macroName, value in listRPMMacros.items():
             macros.append(macroName + " " + value)
9b2f8b85
 
87815216
         listRPMFiles = []
         listSRPMFiles = []
3ad2cb4c
         try:
8f56b626
             listRPMFiles, listSRPMFiles = self._buildRPM(sandbox, specPath + specName,
                                                          logFilePath, package, macros)
26b55679
             logmsg = package + " build done - RPMs : [ "
             for f in listRPMFiles:
                 logmsg += (os.path.basename(f) + " ")
             logmsg += "]\n"
             self.logger.info(logmsg)
3ad2cb4c
         except Exception as e:
87815216
             self.logger.error("Failed while building rpm:" + package)
3ad2cb4c
             raise e
         finally:
8f56b626
             if (constants.rpmCheck and
                     package in constants.testForceRPMS and
                     SPECS.getData().isCheckAvailable(package)):
                 cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + logFilePath)
                 CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug)
26b55679
         self.logger.debug("RPM build is successful")
b5e09fac
 
f93ef2b0
     def findRPMFileForGivenPackage(self, package,version="*"):
2820c61a
         cmdUtils = CommandUtils()
4a93976f
         if version == "*":
f93ef2b0
                 version = SPECS.getData().getHighestVersion(package)
         release = SPECS.getData().getRelease(package, version)
         buildarch=SPECS.getData().getBuildArch(package, version)
87815216
         listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." +
f93ef2b0
                                                    buildarch+".rpm",
87815216
                                                    constants.rpmPath)], [])
42ffccb5
         if constants.inputRPMSPath is not None:
87815216
             listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release +
f93ef2b0
                                                        "." + buildarch+".rpm",
                                                        constants.inputRPMSPath)],
87815216
                                     listFoundRPMFiles)
         if len(listFoundRPMFiles) == 1:
2820c61a
             return listFoundRPMFiles[0]
87815216
         if len(listFoundRPMFiles) == 0:
2820c61a
             return None
87815216
         if len(listFoundRPMFiles) > 1:
             self.logger.error("Found multiple rpm files for given package in rpm directory." +
f93ef2b0
                               "Unable to determine the rpm file for package:" + package +
                               " : " + str(listFoundRPMFiles))
518d6a6f
             raise Exception("Multiple rpm files found")
97a9151c
 
8f56b626
     def findInstalledRPMPackages(self, sandbox):
         rpms = None
         def setOutValue(data):
             nonlocal rpms
             rpms = data
87815216
         cmd = self.rpmBinary + " " + self.queryRpmPackageOptions
8f56b626
         sandbox.run(cmd, logfn=setOutValue)
         return rpms.split()
75a2daa5
 
8f56b626
     def adjustGCCSpecs(self, sandbox, package, version):
f93ef2b0
         opt = " " + SPECS.getData().getSecurityHardeningOption(package, version)
8f56b626
         sandbox.put(self.adjustGCCSpecScript, "/tmp")
87815216
         cmd = "/tmp/" + self.adjustGCCSpecScript + opt
8f56b626
         returnVal = sandbox.run(cmd, logfn=self.logger.debug)
         if returnVal == 0:
610ab83e
             return
 
8f56b626
         # in debugging ...
         sandbox.run("ls -la /tmp/" + self.adjustGCCSpecScript,
                     logfn=self.logger.debug)
         sandbox.run("lsof /tmp/" + self.adjustGCCSpecScript,
                     logfn=self.logger.debug)
         sandbox.run("ps ax", logfn=self.logger.debug)
7418d2bf
 
         self.logger.error("Failed while adjusting gcc specs")
         raise Exception("Failed while adjusting gcc specs")
 
f93ef2b0
     def _verifyShaAndGetSourcePath(self, source, package, version):
326d5ca8
         cmdUtils = CommandUtils()
         # Fetch/verify sources if sha1 not None.
f93ef2b0
         sha1 = SPECS.getData().getSHA1(package, version, source)
326d5ca8
         if sha1 is not None:
a00911af
             PullSources.get(package, source, sha1, constants.sourcePath,
aac331d9
                             constants.getPullSourcesURLs(package), self.logger)
326d5ca8
 
         sourcePath = cmdUtils.findFile(source, constants.sourcePath)
         if not sourcePath:
             sourcePath = cmdUtils.findFile(source, constants.specPath)
             if not sourcePath:
                 if sha1 is None:
                     self.logger.error("No sha1 found or missing source for " + source)
                     raise Exception("No sha1 found or missing source for " + source)
                 else:
                     self.logger.error("Missing source: " + source +
                                       ". Cannot find sources for package: " + package)
                     raise Exception("Missing source")
         else:
             if sha1 is None:
                 self.logger.error("No sha1 found for "+source)
                 raise Exception("No sha1 found")
         if len(sourcePath) > 1:
             self.logger.error("Multiple sources found for source:" + source + "\n" +
                               ",".join(sourcePath) +"\nUnable to determine one.")
             raise Exception("Multiple sources found")
         return sourcePath
 
8f56b626
     def _copySources(self, sandbox, listSourceFiles, package, version, destDir):
         # Fetch and verify sha1 if missing
326d5ca8
         for source in listSourceFiles:
f93ef2b0
             sourcePath = self._verifyShaAndGetSourcePath(source, package, version)
26b55679
             self.logger.debug("Copying... Source path :" + source +
326d5ca8
                              " Source filename: " + sourcePath[0])
8f56b626
             sandbox.put(sourcePath[0], destDir)
326d5ca8
 
aac331d9
     def _getAdditionalBuildOptions(self, package):
         pullsources_urls = []
326d5ca8
         macros = []
         if package in constants.buildOptions.keys():
             pkg = constants.buildOptions[package]
aac331d9
             pullsources_urls.extend(pkg["pullsources"])
             macros.extend(pkg["macros"])
         return pullsources_urls, macros
326d5ca8
 
 
8f56b626
     def _buildRPM(self, sandbox, specFile, logFile, package, macros):
326d5ca8
         rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption
 
         if constants.rpmCheck and package in constants.testForceRPMS:
26b55679
             self.logger.debug("#" * (68 + 2 * len(package)))
326d5ca8
             if not SPECS.getData().isCheckAvailable(package):
26b55679
                 self.logger.debug("####### " + package +
326d5ca8
                                  " MakeCheck is not available. Skipping MakeCheck TEST for " +
                                  package + " #######")
                 rpmBuildcmd = self.rpmbuildBinary + " --clean"
             else:
26b55679
                 self.logger.debug("####### " + package +
326d5ca8
                                  " MakeCheck is available. Running MakeCheck TEST for " +
                                  package + " #######")
                 rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption
26b55679
             self.logger.debug("#" * (68 + 2 * len(package)))
326d5ca8
         else:
             rpmBuildcmd += " " + self.rpmbuildNocheckOption
 
         for macro in macros:
8f56b626
             rpmBuildcmd += ' --define \"%s\"' % macro
326d5ca8
         rpmBuildcmd += " " + specFile
 
26b55679
         self.logger.debug("Building rpm....")
         self.logger.debug(rpmBuildcmd)
8f56b626
 
         returnVal = sandbox.run(rpmBuildcmd, logfile = logFile)
 
326d5ca8
         if constants.rpmCheck and package in constants.testForceRPMS:
             if not SPECS.getData().isCheckAvailable(package):
26b55679
                 constants.testLogger.debug(package + " : N/A")
8f56b626
             elif returnVal == 0:
26b55679
                 constants.testLogger.debug(package + " : PASS")
326d5ca8
             else:
26b55679
                 constants.testLogger.debug(package + " : FAIL")
326d5ca8
 
         if constants.rpmCheck:
8f56b626
             if returnVal != 0 and constants.rpmCheckStopOnError:
326d5ca8
                 self.logger.error("Checking rpm is failed " + specFile)
                 raise Exception("RPM check failed")
         else:
8f56b626
             if returnVal != 0:
326d5ca8
                 self.logger.error("Building rpm is failed " + specFile)
                 raise Exception("RPM build failed")
 
         #Extracting rpms created from log file
         listRPMFiles = []
         listSRPMFiles = []
8f56b626
         stageLogFile = logFile.replace(constants.topDirPath + "/LOGS", constants.logPath )
         with open(stageLogFile, 'r') as logfile:
326d5ca8
             fileContents = logfile.readlines()
             for i in range(0, len(fileContents)):
                 if re.search("^Wrote:", fileContents[i]):
                     listcontents = fileContents[i].split()
                     if ((len(listcontents) == 2) and
                             listcontents[1].strip().endswith(".rpm") and
                             "/RPMS/" in listcontents[1]):
                         listRPMFiles.append(listcontents[1])
                     if ((len(listcontents) == 2) and
                             listcontents[1].strip().endswith(".src.rpm") and
                             "/SRPMS/" in listcontents[1]):
                         listSRPMFiles.append(listcontents[1])
         return listRPMFiles, listSRPMFiles