support/package-builder/PackageBuilder.py
87815216
 import sys
 import os.path
2820c61a
 from PackageUtils import PackageUtils
 from Logger import Logger
 from ToolChainUtils import ToolChainUtils
 from CommandUtils import CommandUtils
 from constants import constants
45c9260c
 from SpecData import SPECS
f93ef2b0
 from StringUtils import StringUtils
8f56b626
 from Sandbox import Chroot, Container
2820c61a
 
56c77555
 class PackageBuilderBase(object):
326d5ca8
     def __init__(self, mapPackageToCycles, pkgBuildType):
f3cc9fa8
         # will be initialized in buildPackageFunction()
87815216
         self.logName = None
         self.logPath = None
         self.logger = None
         self.package = None
f93ef2b0
         self.version = None
2820c61a
         self.mapPackageToCycles = mapPackageToCycles
87815216
         self.listNodepsPackages = ["glibc", "gmp", "zlib", "file", "binutils", "mpfr",
                                    "mpc", "gcc", "ncurses", "util-linux", "groff", "perl",
                                    "texinfo", "rpm", "openssl", "go"]
56c77555
         self.pkgBuildType = pkgBuildType
97a9151c
 
f93ef2b0
     def buildPackageFunction(self, pkg):
         packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
         #do not build if RPM is already built
         #test only if the package is in the testForceRPMS with rpmCheck
         #build only if the package is not in the testForceRPMS with rpmCheck
         if self._checkIfPackageIsAlreadyBuilt(packageName, packageVersion):
             if not constants.rpmCheck:
                 return
             elif constants.rpmCheck and self.package not in constants.testForceRPMS:
                 return
326d5ca8
 
f93ef2b0
         self._buildPackagePrepareFunction(packageName, packageVersion)
         try:
             self._buildPackage()
         except Exception as e:
             # TODO: self.logger might be None
             self.logger.exception(e)
             raise e
 
     def _buildPackagePrepareFunction(self, package, version):
87815216
         self.package = package
f93ef2b0
         self.version = version
         self.logName = "build-" + package + "-" + version
         self.logPath = constants.logPath + "/" + package + "-" + version
56c77555
         if not os.path.isdir(self.logPath):
             cmdUtils = CommandUtils()
87815216
             cmdUtils.runCommandInShell("mkdir -p " + self.logPath)
26b55679
         self.logger = Logger.getLogger(self.logName, self.logPath, constants.logLevel)
97a9151c
 
f93ef2b0
     def _findPackageNameAndVersionFromRPMFile(self, rpmfile):
87815216
         rpmfile = os.path.basename(rpmfile)
         releaseindex = rpmfile.rfind("-")
2820c61a
         if releaseindex == -1:
87815216
             self.logger.error("Invalid rpm file:" + rpmfile)
2820c61a
             return None
f93ef2b0
         pkg = rpmfile[0:releaseindex]
         return pkg
97a9151c
 
8f56b626
     def _findInstalledPackages(self, sandbox):
56c77555
         pkgUtils = PackageUtils(self.logName, self.logPath)
8f56b626
         listInstalledRPMs = pkgUtils.findInstalledRPMPackages(sandbox)
87815216
         listInstalledPackages = []
2820c61a
         for installedRPM in listInstalledRPMs:
f93ef2b0
             pkg = self._findPackageNameAndVersionFromRPMFile(installedRPM)
             if pkg is not None:
                 listInstalledPackages.append(pkg)
56c77555
         return listInstalledPackages, listInstalledRPMs
adf248d5
 
f93ef2b0
     def _checkIfPackageIsAlreadyBuilt(self, package, version):
         basePkg = SPECS.getData().getSpecName(package)
         listRPMPackages = SPECS.getData().getRPMPackages(basePkg, version)
87815216
         packageIsAlreadyBuilt = True
f93ef2b0
         pkgUtils = PackageUtils()
adf248d5
         for pkg in listRPMPackages:
f93ef2b0
             if pkgUtils.findRPMFileForGivenPackage(pkg, version) is None:
87815216
                 packageIsAlreadyBuilt = False
adf248d5
                 break
         return packageIsAlreadyBuilt
 
f93ef2b0
     def _findRunTimeRequiredRPMPackages(self, rpmPackage, version):
         return SPECS.getData().getRequiresForPackage(rpmPackage, version)
97a9151c
 
f93ef2b0
     def _findBuildTimeRequiredPackages(self):
         return SPECS.getData().getBuildRequiresForPackage(self.package, self.version)
97a9151c
 
f93ef2b0
     def _findBuildTimeCheckRequiredPackages(self):
         return SPECS.getData().getCheckBuildRequiresForPackage(self.package, self.version)
5f40784b
 
8f56b626
     def _installPackage(self, pkgUtils, package, packageVersion, sandbox, destLogPath,
326d5ca8
                         listInstalledPackages, listInstalledRPMs):
f93ef2b0
         rpmfile = pkgUtils.findRPMFileForGivenPackage(package,packageVersion);
         if rpmfile is None:
             self.logger.error("No rpm file found for package: " + package + "-" + packageVersion)
             raise Exception("Missing rpm file")
         specificRPM = os.path.basename(rpmfile.replace(".rpm", ""))
         pkg = package+"-"+packageVersion
         if pkg in listInstalledPackages:
e45f5730
                 return
75cb675a
         # mark it as installed -  to avoid cyclic recursion
f93ef2b0
         listInstalledPackages.append(pkg)
e45f5730
         listInstalledRPMs.append(specificRPM)
8f56b626
         self._installDependentRunTimePackages(pkgUtils, package, packageVersion, sandbox, destLogPath,
326d5ca8
                                               listInstalledPackages, listInstalledRPMs)
87815216
         noDeps = False
326d5ca8
         if (package in self.mapPackageToCycles or
                 package in self.listNodepsPackages or
                 package in constants.noDepsPackageList):
87815216
             noDeps = True
8f56b626
         pkgUtils.prepRPMforInstall(package,packageVersion, noDeps, destLogPath)
2820c61a
 
8f56b626
     def _installDependentRunTimePackages(self, pkgUtils, package, packageVersion, sandbox, destLogPath,
326d5ca8
                                          listInstalledPackages, listInstalledRPMs):
f93ef2b0
         listRunTimeDependentPackages = self._findRunTimeRequiredRPMPackages(package, packageVersion)
326d5ca8
         if listRunTimeDependentPackages:
2820c61a
             for pkg in listRunTimeDependentPackages:
326d5ca8
                 if pkg in self.mapPackageToCycles:
2820c61a
                     continue
f93ef2b0
                 packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
87815216
                 latestPkgRPM = os.path.basename(
f93ef2b0
                     pkgUtils.findRPMFileForGivenPackage(packageName, packageVersion)).replace(".rpm", "")
56c77555
                 if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs:
2820c61a
                     continue
8f56b626
                 self._installPackage(pkgUtils, packageName,packageVersion, sandbox, destLogPath,listInstalledPackages, listInstalledRPMs)
326d5ca8
 
8f56b626
     def _findDependentPackagesAndInstalledRPM(self, sandbox):
         listInstalledPackages, listInstalledRPMs = self._findInstalledPackages(sandbox)
26b55679
         self.logger.debug(listInstalledPackages)
f93ef2b0
         listDependentPackages = self._findBuildTimeRequiredPackages()
e45f5730
         listTestPackages=[]
326d5ca8
         if constants.rpmCheck and self.package in constants.testForceRPMS:
f93ef2b0
             # One time optimization
             if constants.listMakeCheckRPMPkgWithVersionstoInstall is None:
                 constants.listMakeCheckRPMPkgWithVersionstoInstalli=[]
                 for package in constants.listMakeCheckRPMPkgtoInstall:
                     version = SPECS.getData().getHighestVersion(package)
                     constants.listMakeCheckRPMPkgWithVersionstoInstall.append(package+"-"+version)
 
             listDependentPackages.extend(self._findBuildTimeCheckRequiredPackages())
             testPackages = (set(constants.listMakeCheckRPMPkgWithVersionstoInstall) -
326d5ca8
                             set(listInstalledPackages) -
f93ef2b0
                             set([self.package+"-"+self.version]))
e45f5730
             listTestPackages=list(set(testPackages))
326d5ca8
             listDependentPackages = list(set(listDependentPackages))
e45f5730
         return listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs
326d5ca8
 
 class PackageBuilderContainer(PackageBuilderBase):
     def __init__(self, mapPackageToCycles, pkgBuildType):
         PackageBuilderBase.__init__(self, mapPackageToCycles, pkgBuildType)
56c77555
 
f93ef2b0
     def _buildPackage(self):
56c77555
         #should initialize a logger based on package name
8f56b626
         containerTaskName = "build-" + self.package + "-" + self.version
         container = None
56c77555
         try:
8f56b626
             container = Container(self.logger)
             container.create(containerTaskName)
56c77555
 
326d5ca8
             tcUtils = ToolChainUtils(self.logName, self.logPath)
8f56b626
             tcUtils.installCustomToolChainRPMS(container, self.package, self.version)
326d5ca8
 
e45f5730
             listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = (
8f56b626
                 self._findDependentPackagesAndInstalledRPM(container))
326d5ca8
 
             pkgUtils = PackageUtils(self.logName, self.logPath)
8f56b626
 
326d5ca8
             if listDependentPackages:
8f56b626
                 self.logger.debug("Installing the build time dependent packages......")
56c77555
                 for pkg in listDependentPackages:
f93ef2b0
                     packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
8f56b626
                     self._installPackage(pkgUtils, packageName, packageVersion, container, self.logPath,listInstalledPackages, listInstalledRPMs)
e45f5730
                 for pkg in listTestPackages:
4a93976f
                     flag = False
8f56b626
                     packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
f93ef2b0
                     for depPkg in listDependentPackages:
                         depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(depPkg)
                         if depPackageName == packageName:
                             flag = True
                             break;
4a93976f
                     if flag == False:
8f56b626
                         self._installPackage(pkgUtils, packageName,packageVersion, container, self.logPath,listInstalledPackages, listInstalledRPMs)
                 pkgUtils.installRPMSInOneShot(container)
26b55679
                 self.logger.debug("Finished installing the build time dependent packages....")
56c77555
 
26b55679
             self.logger.debug("BuildContainer-buildPackage: Start building the package: " +
326d5ca8
                              self.package)
8f56b626
             pkgUtils.adjustGCCSpecs(container, self.package, self.version)
             pkgUtils.buildRPMSForGivenPackage(container, self.package, self.version, self.logPath)
26b55679
             self.logger.debug("BuildContainer-buildPackage: Successfully built the package: " +
326d5ca8
                              self.package)
56c77555
         except Exception as e:
326d5ca8
             self.logger.error("Failed while building package:" + self.package)
8f56b626
             if container is not None:
                 self.logger.debug("Container " + container.getID() +
326d5ca8
                                   " retained for debugging.")
8f56b626
             logFileName = os.path.join(self.logPath, self.package + ".log")
56c77555
             fileLog = os.popen('tail -n 20 ' + logFileName).read()
326d5ca8
             self.logger.debug(fileLog)
56c77555
             raise e
 
         # Remove the container
8f56b626
         if container:
             container.destroy()
56c77555
 
326d5ca8
 class PackageBuilderChroot(PackageBuilderBase):
     def __init__(self, mapPackageToCycles, pkgBuildType):
         PackageBuilderBase.__init__(self, mapPackageToCycles, pkgBuildType)
56c77555
 
8f56b626
     def _buildPackage(self):
         chroot = None
56c77555
         try:
8f56b626
             chroot = Chroot(self.logger)
             chroot.create(self.package + "-" + self.version)
 
326d5ca8
             tUtils = ToolChainUtils(self.logName, self.logPath)
8f56b626
             tUtils.installToolChainRPMS(chroot, self.package, self.version, self.logPath)
56c77555
 
e45f5730
             listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = (
8f56b626
                 self._findDependentPackagesAndInstalledRPM(chroot))
326d5ca8
 
             pkgUtils = PackageUtils(self.logName, self.logPath)
5d822c2d
 
326d5ca8
             if listDependentPackages:
26b55679
                 self.logger.debug("Installing the build time dependent packages......")
56c77555
                 for pkg in listDependentPackages:
f93ef2b0
                     packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
8f56b626
                     self._installPackage(pkgUtils, packageName, packageVersion, chroot, self.logPath,listInstalledPackages, listInstalledRPMs)
e45f5730
                 for pkg in listTestPackages:
4a93976f
                     flag = False
f93ef2b0
                     packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
                     for depPkg in listDependentPackages:
                         depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(depPkg)
                         if depPackageName == packageName:
8f56b626
                             flag = True
                             break;
4a93976f
                     if flag == False:
8f56b626
                         self._installPackage(pkgUtils, packageName,packageVersion, chroot, self.logPath,listInstalledPackages, listInstalledRPMs)
                 pkgUtils.installRPMSInOneShot(chroot)
26b55679
                 self.logger.debug("Finished installing the build time dependent packages....")
326d5ca8
 
8f56b626
             pkgUtils.adjustGCCSpecs(chroot, self.package, self.version)
             pkgUtils.buildRPMSForGivenPackage(chroot, self.package, self.version,
f93ef2b0
                                               self.logPath)
26b55679
             self.logger.debug("Successfully built the package:" + self.package)
56c77555
         except Exception as e:
326d5ca8
             self.logger.error("Failed while building package:" + self.package)
8f56b626
             self.logger.debug("Chroot: " + chroot.getPath() +
326d5ca8
                               " not deleted for debugging.")
             logFileName = os.path.join(self.logPath, self.package + ".log")
56c77555
             fileLog = os.popen('tail -n 100 ' + logFileName).read()
26b55679
             self.logger.info(fileLog)
56c77555
             raise e
8f56b626
         if chroot:
             chroot.destroy()