from CommandUtils import CommandUtils from ChrootUtils import ChrootUtils from Logger import Logger from PackageUtils import PackageUtils from constants import constants import subprocess import os.path import traceback import shutil import json import collections import re class ToolChainUtils(object): def __init__(self,logName=None,logPath=None): if logName is None: logName = "Toolchain Utils" if logPath is None: logPath = constants.logPath self.logName=logName self.logPath=logPath self.logger=Logger.getLogger(logName,logPath) self.adjustToolChainScript = "adjust-tool-chain.sh" self.localegenScript = "./locale-gen.sh" self.localegenConfig = "./locale-gen.conf" self.prepareBuildRootCmd="./prepare-build-root.sh" self.rpmbuildCommand = "rpmbuild" if os.geteuid()==0: self.rpmCommand="rpm" else: self.rpmCommand="fakeroot-ng rpm" def prepareBuildRoot(self,chrootID): self.logger.info("Preparing build environment") cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p "+chrootID+"/tmp") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath) cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/x86_64") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/noarch") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/SOURCES") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/SPECS") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/LOGS") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/BUILD") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/BUILDROOT") package="filesystem" pkgUtils=PackageUtils(self.logName,self.logPath) rpmFile=pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: specFile=constants.specData.getSpecFile(package) cmd=self.rpmbuildCommand+" -ba --nocheck --define \'_topdir "+chrootID+constants.topDirPath+"\' --define \'_dbpath "+chrootID+"/var/lib/rpm\' --define \'dist "+constants.dist+"\' "+specFile self.logger.info(cmd) cmdUtils.runCommandInShell(cmd,self.logPath+"/filesystem.log") filesystemrpmFile = cmdUtils.findFile(package+"-[0-9]*.rpm", chrootID+constants.topDirPath+"/RPMS") filesystemsrpmFile = cmdUtils.findFile(package+"-[0-9]*.src.rpm", chrootID+constants.topDirPath+"/SRPMS") if len(filesystemrpmFile) > 0: shutil.copy2(filesystemrpmFile[0],constants.rpmPath+"/x86_64/") if len(filesystemsrpmFile) > 0: shutil.copy2(filesystemsrpmFile[0],constants.sourceRpmPath+"/") rpmFile=pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: self.logger.error("Cannot find filesystem rpm") raise Exception("Cannot find filesystem rpm") self.logger.debug("Installing filesystem rpms:" + package) if os.geteuid()==0: cmd=self.rpmCommand + " -i --nodeps --root "+chrootID+" --define '_dbpath /var/lib/rpm' "+ rpmFile else: cmd=self.rpmCommand + " -i --nodeps --badreloc --relocate /="+chrootID+" --define '_dbpath "+chrootID+"/var/lib/rpm' "+ rpmFile process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing filesystem rpm failed") raise Exception("RPM installation failed") prepareChrootCmd=self.prepareBuildRootCmd+" "+chrootID logFile=constants.logPath+"/prepareBuildRoot.log" returnVal=cmdUtils.runCommandInShell(prepareChrootCmd,logFile) if not returnVal: self.logger.error("Prepare build root script failed.Unable to prepare chroot.") raise Exception("Prepare build root script failed") self.logger.info("Successfully prepared chroot:"+chrootID) def installToolChain(self, chrootID, packageName, listBuildOptionPackages, pkgBuildOptionFile): self.logger.info("Installing toolchain.....") self.prepareBuildRoot(chrootID) cmdUtils = CommandUtils() rpmFiles = "" packages = "" for package in constants.listToolChainRPMPkgsToInstall: pkgUtils=PackageUtils(self.logName,self.logPath) # Default to any version. version = "*" if packageName in listBuildOptionPackages: jsonData = open(pkgBuildOptionFile) pkg_build_option_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict) jsonData.close() pkgs_sorted = pkg_build_option_json.items() for pkg in pkgs_sorted: p = str(pkg[0].encode('utf-8')) if p == packageName: overridelist = pkg[1]["override_toolchain"] for override in overridelist: if package == str(override["package"].encode('utf-8')): version = str(override["version"].encode('utf-8')) rpmFile=pkgUtils.findRPMFileForGivenPackage(package, version) if rpmFile is None: rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo, version) if rpmFile is None: if package == "util-linux-devel": self.logger.info("No old version of util-linux-devel exists, skip until the new version is built") continue if package == "flex-devel": self.logger.info("No old version of flex-devel exists, skip until the new version is built") continue self.logger.error("Unable to find rpm "+ package + version +" in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing toolchain rpms:" + packages) cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing toolchain rpms failed") raise Exception("RPM installation failed") self.logger.info("Installed default toolchain successfully on chroot:"+chrootID) if re.match("openjdk*", packageName) is not None or re.match("openjdk*", packageName) is not None: self.installToolChainXRPMS(chrootID); def installCoreToolChainPackages(self,chrootID): self.logger.info("Installing toolchain.....") cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) rpmFiles = "" packages = "" for package in constants.listToolChainRPMPkgsToBuild: pkgUtils=PackageUtils(self.logName,self.logPath) rpmFile = None if package in constants.listCoreToolChainRPMPackages: rpmFile=pkgUtils.findRPMFileForGivenPackage(package) else: rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: self.logger.error("Unable to find rpm "+ package) raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing core toolchain rpms:" + packages) cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing toolchain rpms failed") raise Exception("RPM installation failed") self.logger.info("Installed core tool chain packages successfully on chroot:"+chrootID) def findRPMFileInGivenLocation(self, package, rpmdirPath, version = "*"): cmdUtils = CommandUtils() listFoundRPMFiles = cmdUtils.findFile(package+"-"+version+"-*.rpm",rpmdirPath) listFilterRPMFiles=[] for f in listFoundRPMFiles: rpmFileName=os.path.basename(f) checkRPMName=rpmFileName.replace(package,"") rpmNameSplit = checkRPMName.split("-") if len(rpmNameSplit) == 3: listFilterRPMFiles.append(f) if len(listFilterRPMFiles) == 1 : return listFilterRPMFiles[0] if len(listFilterRPMFiles) == 0 : return None if len(listFilterRPMFiles) > 1 : # We use findRPMFileInGivenLocation() only when bootstrapping; # so it is okay to use any version for the core toolchain packages. if package in constants.listCoreToolChainRPMPackages: return listFilterRPMFiles[0] else: self.logger.error("Found multiple rpm files for given package in rpm directory.Unable to determine the rpm file for package:"+package) return None def buildCoreToolChainPackages(self, listBuildOptionPackages, pkgBuildOptionFile): self.logger.info("Building core tool chain packages.....") chrootID=None try: pkgUtils=PackageUtils(self.logName,self.logPath) for package in constants.listCoreToolChainRPMPackages: rpmPkg=pkgUtils.findRPMFileForGivenPackage(package) if rpmPkg is not None: continue chrUtils = ChrootUtils(self.logName,self.logPath) chrootName="build-core-toolchain" destLogPath=constants.logPath+"/build-"+package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p "+destLogPath) returnVal,chrootID = chrUtils.createChroot(chrootName) if not returnVal: self.logger.error("Creating chroot failed") raise Exception("creating chroot failed") self.installToolChainRPMS(chrootID, package, listBuildOptionPackages, pkgBuildOptionFile) pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath) pkgUtils.buildRPMSForGivenPackage(package, chrootID, listBuildOptionPackages, pkgBuildOptionFile, destLogPath) chrUtils.destroyChroot(chrootID) chrootID=None self.logger.info("Successfully built toolchain") if chrootID is not None: chrUtils.destroyChroot(chrootID) except Exception as e: self.logger.error("Unable to build tool chain.") # print stacktrace traceback.print_exc() raise e def installToolChainRPMS(self, chrootID, packageName, listBuildOptionPackages, pkgBuildOptionFile): cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) self.logger.info("Installing Tool Chain RPMS.......") rpmFiles = "" packages = "" for package in constants.listToolChainRPMPkgsToBuild: pkgUtils=PackageUtils(self.logName,self.logPath) rpmFile = None # Default to any version. version = "*" if packageName in listBuildOptionPackages: jsonData = open(pkgBuildOptionFile) pkg_build_option_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict) jsonData.close() pkgs_sorted = pkg_build_option_json.items() for pkg in pkgs_sorted: p = str(pkg[0].encode('utf-8')) if p == packageName: overridelist = pkg[1]["override_toolchain"] for override in overridelist: if package == str(override["package"].encode('utf-8')): version = str(override["version"].encode('utf-8')) if ((packageName not in constants.listToolChainRPMPkgsToBuild) or (constants.listToolChainRPMPkgsToBuild.index(packageName) > constants.listToolChainRPMPkgsToBuild.index(package))): rpmFile=pkgUtils.findRPMFileForGivenPackage(package, version) if rpmFile is None: rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo, version) if rpmFile is None: if package == "util-linux-devel": self.logger.info("No old version of util-linux-devel exists, skip until the new version is built") continue if package == "flex-devel": self.logger.info("No old version of flex-devel exists, skip until the new version is built") continue self.logger.error("Unable to find rpm "+ package + version +" in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing rpms:"+packages) cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info("Successfully installed default Tool Chain RPMS in Chroot:"+chrootID) if re.match("openjdk*", packageName) is not None or re.match("openjdk*", packageName) is not None: self.installToolChainXRPMS(chrootID); def installToolChainXRPMS(self, chrootID): self.logger.info("Installing Tool Chain X package RPMS.......") rpmFiles = "" packages = "" for package in constants.listToolChainXRPMsToInstall: pkgUtils=PackageUtils(self.logName,self.logPath) print "DEBUG:" + package rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishXRPMRepo) if rpmFile is None: self.logger.error("Unable to find rpm "+ package +" in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing rpms:"+packages) cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles print "Command Executed:" + cmd process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info("Successfully installed all Tool Chain X RPMS")