import sys import os.path from PackageUtils import PackageUtils from Logger import Logger from ToolChainUtils import ToolChainUtils from CommandUtils import CommandUtils from constants import constants from SpecData import SPECS from StringUtils import StringUtils from Sandbox import Chroot, Container class PackageBuilderBase(object): def __init__(self, mapPackageToCycles, pkgBuildType): # will be initialized in buildPackageFunction() self.logName = None self.logPath = None self.logger = None self.package = None self.version = None self.mapPackageToCycles = mapPackageToCycles self.listNodepsPackages = ["glibc", "gmp", "zlib", "file", "binutils", "mpfr", "mpc", "gcc", "ncurses", "util-linux", "groff", "perl", "texinfo", "rpm", "openssl", "go"] self.pkgBuildType = pkgBuildType def buildPackageFunction(self, pkg): packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) #do not build if RPM is already built #test only if the package is in the testForceRPMS with rpmCheck #build only if the package is not in the testForceRPMS with rpmCheck if self._checkIfPackageIsAlreadyBuilt(packageName, packageVersion): if not constants.rpmCheck: return elif constants.rpmCheck and self.package not in constants.testForceRPMS: return self._buildPackagePrepareFunction(packageName, packageVersion) try: self._buildPackage() except Exception as e: # TODO: self.logger might be None self.logger.exception(e) raise e def _buildPackagePrepareFunction(self, package, version): self.package = package self.version = version self.logName = "build-" + package + "-" + version self.logPath = constants.logPath + "/" + package + "-" + version if not os.path.isdir(self.logPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + self.logPath) self.logger = Logger.getLogger(self.logName, self.logPath, constants.logLevel) def _findPackageNameAndVersionFromRPMFile(self, rpmfile): rpmfile = os.path.basename(rpmfile) releaseindex = rpmfile.rfind("-") if releaseindex == -1: self.logger.error("Invalid rpm file:" + rpmfile) return None pkg = rpmfile[0:releaseindex] return pkg def _findInstalledPackages(self, sandbox): pkgUtils = PackageUtils(self.logName, self.logPath) listInstalledRPMs = pkgUtils.findInstalledRPMPackages(sandbox) listInstalledPackages = [] for installedRPM in listInstalledRPMs: pkg = self._findPackageNameAndVersionFromRPMFile(installedRPM) if pkg is not None: listInstalledPackages.append(pkg) return listInstalledPackages, listInstalledRPMs def _checkIfPackageIsAlreadyBuilt(self, package, version): basePkg = SPECS.getData().getSpecName(package) listRPMPackages = SPECS.getData().getRPMPackages(basePkg, version) packageIsAlreadyBuilt = True pkgUtils = PackageUtils() for pkg in listRPMPackages: if pkgUtils.findRPMFileForGivenPackage(pkg, version) is None: packageIsAlreadyBuilt = False break return packageIsAlreadyBuilt def _findRunTimeRequiredRPMPackages(self, rpmPackage, version): return SPECS.getData().getRequiresForPackage(rpmPackage, version) def _findBuildTimeRequiredPackages(self): return SPECS.getData().getBuildRequiresForPackage(self.package, self.version) def _findBuildTimeCheckRequiredPackages(self): return SPECS.getData().getCheckBuildRequiresForPackage(self.package, self.version) def _installPackage(self, pkgUtils, package, packageVersion, sandbox, destLogPath, listInstalledPackages, listInstalledRPMs): rpmfile = pkgUtils.findRPMFileForGivenPackage(package,packageVersion); if rpmfile is None: self.logger.error("No rpm file found for package: " + package + "-" + packageVersion) raise Exception("Missing rpm file") specificRPM = os.path.basename(rpmfile.replace(".rpm", "")) pkg = package+"-"+packageVersion if pkg in listInstalledPackages: return # mark it as installed - to avoid cyclic recursion listInstalledPackages.append(pkg) listInstalledRPMs.append(specificRPM) self._installDependentRunTimePackages(pkgUtils, package, packageVersion, sandbox, destLogPath, listInstalledPackages, listInstalledRPMs) noDeps = False if (package in self.mapPackageToCycles or package in self.listNodepsPackages or package in constants.noDepsPackageList): noDeps = True pkgUtils.prepRPMforInstall(package,packageVersion, noDeps, destLogPath) def _installDependentRunTimePackages(self, pkgUtils, package, packageVersion, sandbox, destLogPath, listInstalledPackages, listInstalledRPMs): listRunTimeDependentPackages = self._findRunTimeRequiredRPMPackages(package, packageVersion) if listRunTimeDependentPackages: for pkg in listRunTimeDependentPackages: if pkg in self.mapPackageToCycles: continue packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) latestPkgRPM = os.path.basename( pkgUtils.findRPMFileForGivenPackage(packageName, packageVersion)).replace(".rpm", "") if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs: continue self._installPackage(pkgUtils, packageName,packageVersion, sandbox, destLogPath,listInstalledPackages, listInstalledRPMs) def _findDependentPackagesAndInstalledRPM(self, sandbox): listInstalledPackages, listInstalledRPMs = self._findInstalledPackages(sandbox) self.logger.debug(listInstalledPackages) listDependentPackages = self._findBuildTimeRequiredPackages() listTestPackages=[] if constants.rpmCheck and self.package in constants.testForceRPMS: # One time optimization if constants.listMakeCheckRPMPkgWithVersionstoInstall is None: constants.listMakeCheckRPMPkgWithVersionstoInstalli=[] for package in constants.listMakeCheckRPMPkgtoInstall: version = SPECS.getData().getHighestVersion(package) constants.listMakeCheckRPMPkgWithVersionstoInstall.append(package+"-"+version) listDependentPackages.extend(self._findBuildTimeCheckRequiredPackages()) testPackages = (set(constants.listMakeCheckRPMPkgWithVersionstoInstall) - set(listInstalledPackages) - set([self.package+"-"+self.version])) listTestPackages=list(set(testPackages)) listDependentPackages = list(set(listDependentPackages)) return listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs class PackageBuilderContainer(PackageBuilderBase): def __init__(self, mapPackageToCycles, pkgBuildType): PackageBuilderBase.__init__(self, mapPackageToCycles, pkgBuildType) def _buildPackage(self): #should initialize a logger based on package name containerTaskName = "build-" + self.package + "-" + self.version container = None try: container = Container(self.logger) container.create(containerTaskName) tcUtils = ToolChainUtils(self.logName, self.logPath) tcUtils.installCustomToolChainRPMS(container, self.package, self.version) listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = ( self._findDependentPackagesAndInstalledRPM(container)) pkgUtils = PackageUtils(self.logName, self.logPath) if listDependentPackages: self.logger.debug("Installing the build time dependent packages......") for pkg in listDependentPackages: packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) self._installPackage(pkgUtils, packageName, packageVersion, container, self.logPath,listInstalledPackages, listInstalledRPMs) for pkg in listTestPackages: flag = False packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) for depPkg in listDependentPackages: depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(depPkg) if depPackageName == packageName: flag = True break; if flag == False: self._installPackage(pkgUtils, packageName,packageVersion, container, self.logPath,listInstalledPackages, listInstalledRPMs) pkgUtils.installRPMSInOneShot(container) self.logger.debug("Finished installing the build time dependent packages....") self.logger.debug("BuildContainer-buildPackage: Start building the package: " + self.package) pkgUtils.adjustGCCSpecs(container, self.package, self.version) pkgUtils.buildRPMSForGivenPackage(container, self.package, self.version, self.logPath) self.logger.debug("BuildContainer-buildPackage: Successfully built the package: " + self.package) except Exception as e: self.logger.error("Failed while building package:" + self.package) if container is not None: self.logger.debug("Container " + container.getID() + " retained for debugging.") logFileName = os.path.join(self.logPath, self.package + ".log") fileLog = os.popen('tail -n 20 ' + logFileName).read() self.logger.debug(fileLog) raise e # Remove the container if container: container.destroy() class PackageBuilderChroot(PackageBuilderBase): def __init__(self, mapPackageToCycles, pkgBuildType): PackageBuilderBase.__init__(self, mapPackageToCycles, pkgBuildType) def _buildPackage(self): chroot = None try: chroot = Chroot(self.logger) chroot.create(self.package + "-" + self.version) tUtils = ToolChainUtils(self.logName, self.logPath) tUtils.installToolChainRPMS(chroot, self.package, self.version, self.logPath) listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = ( self._findDependentPackagesAndInstalledRPM(chroot)) pkgUtils = PackageUtils(self.logName, self.logPath) if listDependentPackages: self.logger.debug("Installing the build time dependent packages......") for pkg in listDependentPackages: packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) self._installPackage(pkgUtils, packageName, packageVersion, chroot, self.logPath,listInstalledPackages, listInstalledRPMs) for pkg in listTestPackages: flag = False packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) for depPkg in listDependentPackages: depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(depPkg) if depPackageName == packageName: flag = True break; if flag == False: self._installPackage(pkgUtils, packageName,packageVersion, chroot, self.logPath,listInstalledPackages, listInstalledRPMs) pkgUtils.installRPMSInOneShot(chroot) self.logger.debug("Finished installing the build time dependent packages....") pkgUtils.adjustGCCSpecs(chroot, self.package, self.version) pkgUtils.buildRPMSForGivenPackage(chroot, self.package, self.version, self.logPath) self.logger.debug("Successfully built the package:" + self.package) except Exception as e: self.logger.error("Failed while building package:" + self.package) self.logger.debug("Chroot: " + chroot.getPath() + " not deleted for debugging.") logFileName = os.path.join(self.logPath, self.package + ".log") fileLog = os.popen('tail -n 100 ' + logFileName).read() self.logger.info(fileLog) raise e if chroot: chroot.destroy()