support/package-builder/ToolChainUtils.py
87815216
 import os.path
 import traceback
342710b6
 import re
2820c61a
 from CommandUtils import CommandUtils
 from Logger import Logger
 from PackageUtils import PackageUtils
 from constants import constants
e45f5730
 from SpecData import SPECS
f93ef2b0
 from StringUtils import StringUtils
8f56b626
 from Sandbox import Chroot, Container
2820c61a
 
 class ToolChainUtils(object):
5fc28b16
 
87815216
     def __init__(self, logName=None, logPath=None):
2820c61a
         if logName is None:
             logName = "Toolchain Utils"
         if logPath is None:
             logPath = constants.logPath
87815216
         self.logName = logName
         self.logPath = logPath
26b55679
         self.logger = Logger.getLogger(logName, logPath, constants.logLevel)
87815216
         if os.geteuid() == 0:
             self.rpmCommand = "rpm"
45e37dfb
         else:
87815216
             self.rpmCommand = "fakeroot-ng rpm"
45e37dfb
 
f67fb5e3
     def _findPublishedRPM(self, package, rpmdirPath):
8f56b626
         listFoundRPMFiles = CommandUtils.findFile(package + "-*.rpm", rpmdirPath)
87815216
         listFilterRPMFiles = []
a4a06c39
         for f in listFoundRPMFiles:
87815216
             rpmFileName = os.path.basename(f)
             checkRPMName = rpmFileName.replace(package, "")
7acb019e
             rpmNameSplit = checkRPMName.split("-")
             if len(rpmNameSplit) == 3:
a4a06c39
                 listFilterRPMFiles.append(f)
87815216
         if len(listFilterRPMFiles) == 1:
a4a06c39
             return listFilterRPMFiles[0]
87815216
         if len(listFilterRPMFiles) == 0:
a4a06c39
             return None
87815216
         if len(listFilterRPMFiles) > 1:
             self.logger.error("Found multiple rpm files for given package in rpm directory." +
                               "Unable to determine the rpm file for package:" + package)
a4a06c39
             return None
5fc28b16
 
085306ae
     def buildCoreToolChainPackages(self):
a007b861
         self.logger.info("Step 1 : Building the core toolchain packages for " + constants.currentArch)
9bc3518e
         self.logger.info(constants.listCoreToolChainPackages)
26b55679
         self.logger.info("")
8f56b626
         chroot = None
7418d2bf
         pkgCount = 0
518d6a6f
         try:
87815216
             pkgUtils = PackageUtils(self.logName, self.logPath)
26b55679
             coreToolChainYetToBuild = []
fe8a5375
             doneList = []
fb300e7c
             for package in constants.listCoreToolChainPackages:
f93ef2b0
                 version = SPECS.getData().getHighestVersion(package)
f67fb5e3
                 rpmPkg = pkgUtils.findRPMFile(package, version)
0fae9cdd
                 if rpmPkg is not None:
fe8a5375
                     doneList.append(package+'-'+version)
0fae9cdd
                     continue
26b55679
                 else:
                     coreToolChainYetToBuild.append(package)
             if coreToolChainYetToBuild:
                 self.logger.info("The following core toolchain packages need to be built :")
                 self.logger.info(coreToolChainYetToBuild)
             else:
                 self.logger.info("Core toolchain packages are already available")
 
             for package in coreToolChainYetToBuild:
                 self.logger.debug("Building core toolchain package : " + package)
8f56b626
                 version = SPECS.getData().getHighestVersion(package)
a007b861
                 destLogPath = constants.logPath + "/" + package + "-" + version + "." + constants.currentArch
0fae9cdd
                 if not os.path.isdir(destLogPath):
8f56b626
                     CommandUtils.runCommandInShell("mkdir -p " + destLogPath)
                 chroot = Chroot(self.logger)
                 chroot.create(package + "-" + version)
fe8a5375
                 self.installToolchainRPMS(chroot, package, version, availablePackages=doneList)
8f56b626
                 pkgUtils.adjustGCCSpecs(chroot, package, version)
                 pkgUtils.buildRPMSForGivenPackage(chroot, package, version, destLogPath)
7418d2bf
                 pkgCount += 1
8f56b626
                 chroot.destroy()
fe8a5375
                 doneList.append(package+'-'+version)
26b55679
             self.logger.debug("Successfully built toolchain")
             self.logger.info("-" * 45 + "\n")
518d6a6f
         except Exception as e:
22540e38
             self.logger.error("Unable to build toolchain.")
d355ea7e
             # print stacktrace
             traceback.print_exc()
518d6a6f
             raise e
7418d2bf
         return pkgCount
87815216
 
f93ef2b0
     def getListDependentPackages(self, package, version):
a007b861
         listBuildRequiresPkg=SPECS.getData(constants.buildArch).getBuildRequiresForPackage(package, version)
         listBuildRequiresPkg.extend(SPECS.getData(constants.buildArch).getCheckBuildRequiresForPackage(package, version))
f93ef2b0
         return listBuildRequiresPkg
e45f5730
 
22540e38
     def installToolchainRPMS(self, chroot, packageName=None, packageVersion=None, usePublishedRPMS=True, availablePackages=None):
         self.logger.debug("Installing toolchain RPMS.......")
1eb9191f
         rpmFiles = ""
         packages = ""
8f56b626
         listBuildRequiresPackages = []
a007b861
 
         listRPMsToInstall=list(constants.listToolChainRPMsToInstall)
         if constants.crossCompiling:
             targetPackageName = packageName
             packageName = None
             packageVersion = None
             listRPMsToInstall.extend(['binutils-'+constants.targetArch+'-linux-gnu',
                                       'gcc-'+constants.targetArch+'-linux-gnu'])
8f56b626
         if packageName:
             listBuildRequiresPackages = self.getListDependentPackages(packageName, packageVersion)
a007b861
         for package in listRPMsToInstall:
87815216
             pkgUtils = PackageUtils(self.logName, self.logPath)
5fc28b16
             rpmFile = None
f67fb5e3
             version = None
 
             # Get proper package version
f93ef2b0
             for depPkg in listBuildRequiresPackages:
                 depPkgName, depPkgVersion = StringUtils.splitPackageNameAndVersion(depPkg)
                 if depPkgName == package:
                         version=depPkgVersion
f67fb5e3
                         break
a007b861
 
f67fb5e3
             if not version:
a007b861
                 version = SPECS.getData(constants.buildArch).getHighestVersion(package)
f67fb5e3
 
fe8a5375
             if availablePackages is not None:
a007b861
                 basePkg = SPECS.getData(constants.buildArch).getSpecName(package)+"-"+version
fe8a5375
                 isAvailable = basePkg in availablePackages
             else:
                 # if availablePackages is not provided (rear case) it is safe
                 # to use findRPMFile()
                 isAvailable = True
f67fb5e3
 
6d17349b
             if constants.rpmCheck:
a007b861
                 rpmFile = pkgUtils.findRPMFile(package, version, constants.buildArch)
f67fb5e3
 
             if rpmFile is None:
                 # Honor the toolchain list order.
                 # if index of depended package ('package') is more
                 # then index of the current package that we are
                 # building ('packageName'), then we _must_ use published
                 # `package` rpm.
                 if (packageName and
a007b861
                     packageName in listRPMsToInstall and
                     listRPMsToInstall.index(packageName) <
                         listRPMsToInstall.index(package)):
f67fb5e3
                     isAvailable = False
fe8a5375
                 if isAvailable:
a007b861
                     rpmFile = pkgUtils.findRPMFile(package, version, constants.buildArch)
f67fb5e3
 
1ffbd987
             if rpmFile is None:
a007b861
                 if not usePublishedRPMS or isAvailable or constants.crossCompiling:
                     raise Exception("%s-%s.%s not found in available packages" % (package, version, constants.buildArch))
 
                 # Safe to use published RPM
f67fb5e3
 
9874e684
                 # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
a007b861
                 if (package == "sqlite") and (constants.buildArch == "x86_64"):
9874e684
                     package = "sqlite-autoconf"
f67fb5e3
                 rpmFile = self._findPublishedRPM(package, constants.prevPublishRPMRepo)
1ffbd987
                 if rpmFile is None:
fb300e7c
                     if package in constants.listOfRPMsProvidedAfterBuild:
26b55679
                         self.logger.debug("No old version of " + package +
87815216
                                          " exists, skip until the new version is built")
d77dd20b
                         continue
f67fb5e3
                     self.logger.error("Unable to find published rpm " + package)
36c2f510
                     raise Exception("Input Error")
1ffbd987
             rpmFiles += " " + rpmFile
f93ef2b0
             packages += " " + package+"-"+version
1eb9191f
 
a007b861
         self.logger.debug(rpmFiles)
f93ef2b0
         self.logger.debug(packages)
87815216
         cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
a458a5f9
                chroot.getID() +" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles)
8f56b626
         retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
         if retVal != 0:
7f9d2e12
             self.logger.debug("Command Executed:" + cmd)
22540e38
             self.logger.error("Installing toolchain RPMS failed")
36c2f510
             raise Exception("RPM installation failed")
a458a5f9
         self.logger.debug("Successfully installed default toolchain RPMS in Chroot:" + chroot.getID())
a007b861
 
8f56b626
         if packageName:
22540e38
             self.installExtraToolchainRPMS(chroot, packageName, packageVersion)
8f56b626
 
a007b861
         if constants.crossCompiling:
             self.installTargetToolchain(chroot, targetPackageName)
 
22540e38
     def installExtraToolchainRPMS(self, sandbox, packageName, packageVersion):
a007b861
         listOfToolChainPkgs = SPECS.getData(constants.buildArch).getExtraBuildRequiresForPackage(packageName, packageVersion)
8f56b626
         if not listOfToolChainPkgs:
             return
a007b861
         self.logger.debug("Installing package specific toolchain RPMs for " + packageName +
8f56b626
                          ": " + str(listOfToolChainPkgs))
062d56d1
         rpmFiles = ""
         packages = ""
27a75769
         for package in listOfToolChainPkgs:
87815216
             pkgUtils = PackageUtils(self.logName, self.logPath)
342710b6
             if re.match("openjre*", packageName) is not None or re.match("openjdk*", packageName):
8f56b626
                 path = constants.prevPublishXRPMRepo
                 sandboxPath = "/publishxrpms"
27a75769
             else:
8f56b626
                 path = constants.prevPublishRPMRepo
                 sandboxPath = "/publishrpms"
f67fb5e3
             rpmFile = self._findPublishedRPM(package, path)
062d56d1
             if rpmFile is None:
87815216
                 self.logger.error("Unable to find rpm "+ package +
                                   " in current and previous versions")
062d56d1
                 raise Exception("Input Error")
8f56b626
             rpmFiles += " " + rpmFile.replace(path, sandboxPath)
062d56d1
             packages += " " + package
 
87815216
         self.logger.debug("Installing custom rpms:" + packages)
8f56b626
         cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force " + rpmFiles)
         retVal = sandbox.run(cmd, logfn=self.logger.debug)
         if retVal != 0:
7f9d2e12
             self.logger.debug("Command Executed:" + cmd)
8f56b626
             self.logger.error("Installing custom toolchains failed")
062d56d1
             raise Exception("RPM installation failed")
a007b861
 
     # Install target's core toolchain packages up to 'stopAtPackage' package
     def installTargetToolchain(self, chroot, stopAtPackage=None):
         self.logger.debug("Installing target toolchain RPMS.......")
         pkgUtils = PackageUtils(self.logName, self.logPath)
         rpmFiles = ""
         packages = ""
         for package in constants.listCoreToolChainPackages:
             if stopAtPackage and package == stopAtPackage:
                 break
             version = SPECS.getData().getHighestVersion(package)
             basePkg = SPECS.getData().getSpecName(package)
             # install all subpackages of given package
             # for instance: for 'glibc' we want glibc-devel, glibc-tools,
             #               glibc-i18n, etc also to be installed
             subpackages = SPECS.getData().getRPMPackages(basePkg, version)
             for p in subpackages:
                 rpmFile = pkgUtils.findRPMFile(p, version, constants.targetArch)
                 rpmFiles += " " + rpmFile
                 packages += " " + package+"-"+version
 
         self.logger.debug(packages)
 
         cmd = "mkdir -p " + chroot.getID() +"/target-"+ constants.targetArch+"/var/lib/rpm"
         CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
 
         if rpmFiles != "":
             cmd = (self.rpmCommand+" -Uvh --nodeps --ignorearch --noscripts --root "+
                    chroot.getID() +"/target-"+ constants.targetArch+
                    " --define \'_dbpath /var/lib/rpm\' "+rpmFiles)
             retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
             if retVal != 0:
                 self.logger.debug("Command Executed:" + cmd)
                 self.logger.error("Installing toolchain failed")
                 raise Exception("RPM installation failed")
         self.logger.debug("Successfully installed target toolchain RPMS in chroot:" + chroot.getID())