import os.path import platform import traceback import re from CommandUtils import CommandUtils from ChrootUtils import ChrootUtils from Logger import Logger from PackageUtils import PackageUtils from constants import constants class ToolChainUtils(object): def __init__(self, logName=None, logPath=None): if logName is None: logName = "Toolchain Utils" if logPath is None: logPath = constants.logPath self.logName = logName self.logPath = logPath self.logger = Logger.getLogger(logName, logPath) self.adjustToolChainScript = "adjust-tool-chain.sh" self.localegenScript = "./locale-gen.sh" self.localegenConfig = "./locale-gen.conf" self.prepareBuildRootCmd = "./prepare-build-root.sh" self.rpmbuildCommand = "rpmbuild" if os.geteuid() == 0: self.rpmCommand = "rpm" else: self.rpmCommand = "fakeroot-ng rpm" def prepareBuildRoot(self, chrootID): self.logger.info("Preparing build environment") cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/dev") cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/etc") cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/proc") cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/run") cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/sys") cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/tmp") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath) cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/RPMS/" + platform.machine()) cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/RPMS/noarch") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/SOURCES") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/SPECS") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/LOGS") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/BUILD") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/BUILDROOT") prepareChrootCmd = self.prepareBuildRootCmd + " " + chrootID logFile = self.logPath + "/prepareBuildRoot.log" returnVal = cmdUtils.runCommandInShell(prepareChrootCmd, logFile) if not returnVal: self.logger.error("Prepare build root script failed.Unable to prepare chroot.") raise Exception("Prepare build root script failed") self.logger.info("Successfully prepared chroot:" + chrootID) def findRPMFileInGivenLocation(self, package, rpmdirPath): cmdUtils = CommandUtils() listFoundRPMFiles = cmdUtils.findFile(package + "-*.rpm", rpmdirPath) listFilterRPMFiles = [] for f in listFoundRPMFiles: rpmFileName = os.path.basename(f) checkRPMName = rpmFileName.replace(package, "") rpmNameSplit = checkRPMName.split("-") if len(rpmNameSplit) == 3: listFilterRPMFiles.append(f) if len(listFilterRPMFiles) == 1: return listFilterRPMFiles[0] if len(listFilterRPMFiles) == 0: return None if len(listFilterRPMFiles) > 1: self.logger.error("Found multiple rpm files for given package in rpm directory." + "Unable to determine the rpm file for package:" + package) return None def buildCoreToolChainPackages(self): self.logger.info("Building core toolchain packages.....") chrootID = None pkgCount = 0 try: pkgUtils = PackageUtils(self.logName, self.logPath) for package in constants.listCoreToolChainPackages: rpmPkg = pkgUtils.findRPMFileForGivenPackage(package) if rpmPkg is not None: continue self.logger.info("Building core toolchain package: " + package) chrUtils = ChrootUtils(self.logName, self.logPath) chrootName = "build-"+package destLogPath = constants.logPath + "/build-" + package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + destLogPath) returnVal, chrootID = chrUtils.createChroot(chrootName) if not returnVal: self.logger.error("Creating chroot failed") raise Exception("creating chroot failed") self.installToolChainRPMS(chrootID, package, destLogPath) pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath) pkgUtils.buildRPMSForGivenPackage(package, chrootID, destLogPath) pkgCount += 1 chrUtils.destroyChroot(chrootID) chrootID = None self.logger.info("Successfully built toolchain") if chrootID is not None: chrUtils.destroyChroot(chrootID) except Exception as e: self.logger.error("Unable to build tool chain.") # print stacktrace traceback.print_exc() raise e return pkgCount def installToolChainRPMS(self, chrootID, packageName, logPath=None): if logPath is None: logPath = self.logPath cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) self.logger.info("Installing Tool Chain RPMS.......") rpmFiles = "" packages = "" for package in constants.listToolChainRPMsToInstall: pkgUtils = PackageUtils(self.logName, self.logPath) rpmFile = None if constants.rpmCheck: rpmFile = pkgUtils.findRPMFileForGivenPackage(package) else: if (packageName not in constants.listToolChainRPMsToInstall or constants.listToolChainRPMsToInstall.index(packageName) > constants.listToolChainRPMsToInstall.index(package)): rpmFile = pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf if (package == "sqlite") and (platform.machine() == "x86_64"): package = "sqlite-autoconf" rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: if package in constants.listOfRPMsProvidedAfterBuild: self.logger.info("No old version of " + package + " exists, skip until the new version is built") continue self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing toolchain rpms:" + packages) cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " + chrootID +" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles) retVal = cmdUtils.runCommandInShell(cmd, logPath + "/install_toolchain_rpms.log") if not retVal: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info("Successfully installed default Tool Chain RPMS in Chroot:" + chrootID) print("Building Package: ".format(packageName)) print(constants.perPackageToolChain) if packageName in constants.perPackageToolChain: print(constants.perPackageToolChain[packageName]) self.installCustomToolChainRPMS(chrootID, constants.perPackageToolChain[packageName].get(platform.machine(), []), packageName) def installCustomToolChainRPMS(self, chrootID, listOfToolChainPkgs, packageName): self.logger.info("Installing package specific tool chain RPMs for " + packageName + ".......") rpmFiles = "" packages = "" cmdUtils = CommandUtils() for package in listOfToolChainPkgs: pkgUtils = PackageUtils(self.logName, self.logPath) print("DEBUG:" + package) if re.match("openjre*", packageName) is not None or re.match("openjdk*", packageName): # x86_64 has openjdk/jre as a published rpms but aarch64 has openjdk8/jre8 # Remove this condition after publishxrpms for x86_^4 got updated if ((package == "openjdk" or package == "openjre") and platform.machine() == "aarch64"): package = package + "8" rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishXRPMRepo) else: rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: self.logger.error("Unable to find rpm "+ package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing custom rpms:" + packages) cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles) retVal = cmdUtils.runCommandInShell(cmd, self.logPath + "/install_custom_toolchain_rpms.log") if not retVal: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info("Successfully installed all Tool Chain X RPMS") def installToolChainRPMSinContainer(self, containerID): self.logger.info("Installing tool-chain RPMS in container: " + containerID.short_id) rpmFiles = "" packages = "" pkgUtils = PackageUtils(self.logName, self.logPath) for package in constants.listToolChainRPMPkgsToInstall: rpmFile = pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf # if (package == "sqlite") and (platform.machine() == "x86_64"): # package = "sqlite-autoconf" rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: if package in constants.listOfRPMsProvidedAfterBuild: self.logger.info("No old version of " + package + " exists, skip until the new version is built") continue self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") if rpmFile.find("stage/PUBLISHRPMS"): rpmFile = rpmFile.replace(constants.prevPublishRPMRepo, "/publishrpms") if rpmFile.find("stage/PUBLISHXRPMS"): rpmFile = rpmFile.replace(constants.prevPublishXRPMRepo, "/publishxrpms") if rpmFile.find("stage/RPMS"): rpmFile = rpmFile.replace(constants.rpmPath, constants.topDirPath + "/RPMS") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing tool-chain rpms: " + packages) cmd = "/usr/bin/bash -l -c '/usr/bin/rpm -Uvh --force --nodeps " + rpmFiles + "'" self.logger.info("VDBG-TCU-installToolChainRPMSinContainer: Installing rpms cmd: " + cmd) tcInstallLog = containerID.exec_run(cmd) # TODO: Find a way to collect exit status of the command that was run. if not tcInstallLog: self.logger.error("Installing tool chain in container failed") raise Exception("RPM installation in container failed") self.logger.info(tcInstallLog) self.logger.info("Successfully installed default tool-chain RPMS in container: " + containerID.short_id) def installCustomToolChainRPMSinContainer(self, containerID, listOfToolChainPkgs, packageName): self.logger.info("Installing package specific tool chain RPMs for " + packageName) rpmFiles = "" packages = "" for package in listOfToolChainPkgs: if re.match("openjre*", packageName) is not None or re.match("openjdk*", packageName): rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishXRPMRepo) else: rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") if rpmFile.find("stage/PUBLISHRPMS"): rpmFile = rpmFile.replace(constants.prevPublishRPMRepo, "/publishrpms") if rpmFile.find("stage/PUBLISHXRPMS"): rpmFile = rpmFile.replace(constants.prevPublishXRPMRepo, "/publishxrpms") if rpmFile.find("stage/RPMS"): rpmFile = rpmFile.replace(constants.rpmPath, constants.topDirPath + "/RPMS") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing rpms: " + packages) cmd = "rpm -Uvh --nodeps --force " + rpmFiles self.logger.debug("VDBG-TCU-installCustomToolChainRPMSinContainer: Installing rpms cmd: " + cmd) tcInstallLog = containerID.exec_run(cmd) # TODO: Find a way to collect exit status of the command that was run. if not tcInstallLog: self.logger.error("Installing tool chain in container failed") raise Exception("RPM installation in container failed") self.logger.info(tcInstallLog) self.logger.info("Successfully installed all tool-chain XRPMS in container: " + containerID.short_id)