from PackageUtils import PackageUtils from Logger import Logger from ChrootUtils import ChrootUtils from ToolChainUtils import ToolChainUtils from CommandUtils import CommandUtils import os.path from constants import constants import shutil from SpecData import SPECS from StringUtils import StringUtils class PackageBuilderBase(object): def __init__(self, mapPackageToCycles, pkgBuildType): # will be initialized in buildPackageFunction() self.logName = None self.logPath = None self.logger = None self.package = None self.version = None self.mapPackageToCycles = mapPackageToCycles self.listNodepsPackages = ["glibc", "gmp", "zlib", "file", "binutils", "mpfr", "mpc", "gcc", "ncurses", "util-linux", "groff", "perl", "texinfo", "rpm", "openssl", "go"] self.pkgBuildType = pkgBuildType def _findPackageNameAndVersionFromRPMFile(self, rpmfile): rpmfile = os.path.basename(rpmfile) releaseindex = rpmfile.rfind("-") if releaseindex == -1: self.logger.error("Invalid rpm file:" + rpmfile) return None pkg = rpmfile[0:releaseindex] return pkg def checkIfPackageIsAlreadyBuilt(self, package, version): basePkg=SPECS.getData().getSpecName(package) listRPMPackages=SPECS.getData().getRPMPackages(basePkg, version) packageIsAlreadyBuilt=True pkgUtils = PackageUtils(self.logName,self.logPath) for pkg in listRPMPackages: if pkgUtils.findRPMFileForGivenPackage(pkg, version) is None: packageIsAlreadyBuilt=False break return packageIsAlreadyBuilt def findRunTimeRequiredRPMPackages(self,rpmPackage, version): listRequiredPackages=SPECS.getData().getRequiresForPackage(rpmPackage, version) return listRequiredPackages def findBuildTimeRequiredPackages(self): listRequiredPackages=SPECS.getData().getBuildRequiresForPackage(self.package, self.version) return listRequiredPackages def findBuildTimeCheckRequiredPackages(self): listRequiredPackages=SPECS.getData().getCheckBuildRequiresForPackage(self.package, self.version) return listRequiredPackages class PackageBuilder(PackageBuilderBase): def __init__(self,mapPackageToCycles,listAvailableCyclicPackages,listBuildOptionPackages,pkgBuildOptionFile): PackageBuilderBase.__init__(self, mapPackageToCycles, "chroot") # will be initialized in buildPackageThreadAPI() self.listAvailableCyclicPackages = listAvailableCyclicPackages self.listBuildOptionPackages=listBuildOptionPackages self.pkgBuildOptionFile=pkgBuildOptionFile def prepareBuildRoot(self): chrootID=None chrootName="build-"+self.package + "-" + self.version try: chrUtils = ChrootUtils(self.logName,self.logPath) returnVal,chrootID = chrUtils.createChroot(chrootName) self.logger.debug("Created new chroot: " + chrootID) if not returnVal: raise Exception("Unable to prepare build root") tUtils=ToolChainUtils(self.logName,self.logPath) tUtils.installToolChainRPMS(chrootID, self.package,self.version, self.listBuildOptionPackages, self.pkgBuildOptionFile, self.logPath) except Exception as e: if chrootID is not None: self.logger.debug("Deleting chroot: " + chrootID) chrUtils.destroyChroot(chrootID) raise e return chrootID def findInstalledPackages(self, instanceID): pkgUtils = PackageUtils(self.logName, self.logPath) if self.pkgBuildType == "chroot": listInstalledRPMs = pkgUtils.findInstalledRPMPackages(instanceID) listInstalledPackages = [] for installedRPM in listInstalledRPMs: pkg = self._findPackageNameAndVersionFromRPMFile(installedRPM) if pkg is not None: listInstalledPackages.append(pkg) return listInstalledPackages, listInstalledRPMs def buildPackageThreadAPI(self,package,outputMap, threadName,): packageName, packageVersion = StringUtils.splitPackageNameAndVersion(package) #do not build if RPM is already built #test only if the package is in the testForceRPMS with rpmCheck #build only if the package is not in the testForceRPMS with rpmCheck if self.checkIfPackageIsAlreadyBuilt(packageName, packageVersion): if not constants.rpmCheck: outputMap[threadName]=True return elif constants.rpmCheck and packageName not in constants.testForceRPMS: outputMap[threadName]=True return self.package=packageName self.version=packageVersion self.logName="build-"+package self.logPath=constants.logPath+"/build-"+package if not os.path.isdir(self.logPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p "+self.logPath) self.logger=Logger.getLogger(self.logName,self.logPath) try: self.buildPackage() outputMap[threadName]=True except Exception as e: # TODO: self.logger might be None self.logger.exception(e) outputMap[threadName]=False raise e def buildPackage(self): chrUtils = ChrootUtils(self.logName,self.logPath) chrootID=None try: chrootID = self.prepareBuildRoot() listInstalledPackages, listInstalledRPMs=self.findInstalledPackages(chrootID) listDependentPackages=self.findBuildTimeRequiredPackages() listTestPackages=[] if constants.rpmCheck and self.package in constants.testForceRPMS: listDependentPackages.extend(self.findBuildTimeCheckRequiredPackages()) testPackages=set(constants.listMakeCheckRPMPkgtoInstall)-set(listInstalledPackages)-set([self.package]) listTestPackages = list(set(testPackages)) listDependentPackages=list(set(listDependentPackages)) pkgUtils = PackageUtils(self.logName,self.logPath) if len(listDependentPackages) != 0: self.logger.info("Installing the build time dependent packages......") for pkg in listDependentPackages: packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) self.installPackage(pkgUtils,packageName,packageVersion,chrootID,self.logPath,listInstalledPackages,listInstalledRPMs) for pkg in listTestPackages: flag=False packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) for depPkg in listDependentPackages: depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(depPkg) if depPackageName == packageName: flag = True break; if flag == False: self.installPackage(pkgUtils,packageName,packageVersion,chrootID,self.logPath,listInstalledPackages,listInstalledRPMs) pkgUtils.installRPMSInAOneShot(chrootID,self.logPath) self.logger.info("Finished installing the build time dependent packages......") pkgUtils.adjustGCCSpecs(self.package, chrootID, self.logPath, self.version) pkgUtils.buildRPMSForGivenPackage(self.package,self.version,chrootID,self.listBuildOptionPackages,self.pkgBuildOptionFile,self.logPath) self.logger.info("Successfully built the package:"+self.package) except Exception as e: self.logger.error("Failed while building package:" + self.package) self.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.") logFileName = os.path.join(self.logPath, self.package + ".log") fileLog = os.popen('tail -n 100 ' + logFileName).read() self.logger.debug(fileLog) raise e if chrootID is not None: chrUtils.destroyChroot(chrootID) def installPackage(self,pkgUtils,package,packageVersion,chrootID,destLogPath,listInstalledPackages,listInstalledRPMs): rpmfile = pkgUtils.findRPMFileForGivenPackage(package,packageVersion); if rpmfile is None: self.logger.error("No rpm file found for package: " + package + "-" + packageVersion) raise Exception("Missing rpm file") specificRPM = os.path.basename(rpmfile.replace(".rpm", "")) pkg = package+"-"+packageVersion if pkg in listInstalledPackages: return # For linux packages, install the gcc dependencies from publish rpms if self.package in self.listBuildOptionPackages: if SPECS.getData().getSpecName(package) == "gcc": tUtils=ToolChainUtils(self.logName,self.logPath) overridenPkgVer = tUtils.getOverridenPackageVersion(self.package, package, self.listBuildOptionPackages, self.pkgBuildOptionFile) overridenPkg = package+"-"+overridenPkgVer if overridenPkg in listInstalledPackages: return # mark it as installed - to avoid cyclic recursion listInstalledPackages.append(pkg) listInstalledRPMs.append(specificRPM) self.installDependentRunTimePackages(pkgUtils,package,packageVersion, chrootID,destLogPath,listInstalledPackages,listInstalledRPMs) noDeps=False if self.mapPackageToCycles.has_key(package): noDeps = True if package in self.listNodepsPackages: noDeps=True if package in constants.noDepsPackageList: noDeps=True pkgUtils.installRPM(package,packageVersion,chrootID,noDeps,destLogPath) def installDependentRunTimePackages(self,pkgUtils,package,packageVersion,chrootID,destLogPath,listInstalledPackages,listInstalledRPMs): listRunTimeDependentPackages=self.findRunTimeRequiredRPMPackages(package,packageVersion) if len(listRunTimeDependentPackages) != 0: for pkg in listRunTimeDependentPackages: if self.mapPackageToCycles.has_key(pkg): continue packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) latestPkgRPM = os.path.basename( pkgUtils.findRPMFileForGivenPackage(packageName, packageVersion)).replace(".rpm", "") if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs: continue self.installPackage(pkgUtils,packageName,packageVersion,chrootID,destLogPath,listInstalledPackages,listInstalledRPMs)