87815216 |
import sys
import os.path |
2820c61a |
from PackageUtils import PackageUtils
from Logger import Logger
from ToolChainUtils import ToolChainUtils
from CommandUtils import CommandUtils
from constants import constants |
45c9260c |
from SpecData import SPECS |
f93ef2b0 |
from StringUtils import StringUtils |
8f56b626 |
from Sandbox import Chroot, Container |
2820c61a |
|
56c77555 |
class PackageBuilderBase(object): |
326d5ca8 |
def __init__(self, mapPackageToCycles, pkgBuildType): |
f3cc9fa8 |
# will be initialized in buildPackageFunction() |
87815216 |
self.logName = None
self.logPath = None
self.logger = None
self.package = None |
f93ef2b0 |
self.version = None |
2820c61a |
self.mapPackageToCycles = mapPackageToCycles |
87815216 |
self.listNodepsPackages = ["glibc", "gmp", "zlib", "file", "binutils", "mpfr",
"mpc", "gcc", "ncurses", "util-linux", "groff", "perl",
"texinfo", "rpm", "openssl", "go"] |
56c77555 |
self.pkgBuildType = pkgBuildType |
97a9151c |
|
f93ef2b0 |
def buildPackageFunction(self, pkg):
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self._checkIfPackageIsAlreadyBuilt(packageName, packageVersion):
if not constants.rpmCheck:
return
elif constants.rpmCheck and self.package not in constants.testForceRPMS:
return |
326d5ca8 |
|
f93ef2b0 |
self._buildPackagePrepareFunction(packageName, packageVersion)
try:
self._buildPackage()
except Exception as e:
# TODO: self.logger might be None
self.logger.exception(e)
raise e
def _buildPackagePrepareFunction(self, package, version): |
87815216 |
self.package = package |
f93ef2b0 |
self.version = version
self.logName = "build-" + package + "-" + version
self.logPath = constants.logPath + "/" + package + "-" + version |
56c77555 |
if not os.path.isdir(self.logPath):
cmdUtils = CommandUtils() |
87815216 |
cmdUtils.runCommandInShell("mkdir -p " + self.logPath) |
26b55679 |
self.logger = Logger.getLogger(self.logName, self.logPath, constants.logLevel) |
97a9151c |
|
f93ef2b0 |
def _findPackageNameAndVersionFromRPMFile(self, rpmfile): |
87815216 |
rpmfile = os.path.basename(rpmfile)
releaseindex = rpmfile.rfind("-") |
2820c61a |
if releaseindex == -1: |
87815216 |
self.logger.error("Invalid rpm file:" + rpmfile) |
2820c61a |
return None |
f93ef2b0 |
pkg = rpmfile[0:releaseindex]
return pkg |
97a9151c |
|
8f56b626 |
def _findInstalledPackages(self, sandbox): |
56c77555 |
pkgUtils = PackageUtils(self.logName, self.logPath) |
8f56b626 |
listInstalledRPMs = pkgUtils.findInstalledRPMPackages(sandbox) |
87815216 |
listInstalledPackages = [] |
2820c61a |
for installedRPM in listInstalledRPMs: |
f93ef2b0 |
pkg = self._findPackageNameAndVersionFromRPMFile(installedRPM)
if pkg is not None:
listInstalledPackages.append(pkg) |
56c77555 |
return listInstalledPackages, listInstalledRPMs |
adf248d5 |
|
f93ef2b0 |
def _checkIfPackageIsAlreadyBuilt(self, package, version):
basePkg = SPECS.getData().getSpecName(package)
listRPMPackages = SPECS.getData().getRPMPackages(basePkg, version) |
87815216 |
packageIsAlreadyBuilt = True |
f93ef2b0 |
pkgUtils = PackageUtils() |
adf248d5 |
for pkg in listRPMPackages: |
f93ef2b0 |
if pkgUtils.findRPMFileForGivenPackage(pkg, version) is None: |
87815216 |
packageIsAlreadyBuilt = False |
adf248d5 |
break
return packageIsAlreadyBuilt
|
f93ef2b0 |
def _findRunTimeRequiredRPMPackages(self, rpmPackage, version):
return SPECS.getData().getRequiresForPackage(rpmPackage, version) |
97a9151c |
|
f93ef2b0 |
def _findBuildTimeRequiredPackages(self):
return SPECS.getData().getBuildRequiresForPackage(self.package, self.version) |
97a9151c |
|
f93ef2b0 |
def _findBuildTimeCheckRequiredPackages(self):
return SPECS.getData().getCheckBuildRequiresForPackage(self.package, self.version) |
5f40784b |
|
8f56b626 |
def _installPackage(self, pkgUtils, package, packageVersion, sandbox, destLogPath, |
326d5ca8 |
listInstalledPackages, listInstalledRPMs): |
f93ef2b0 |
rpmfile = pkgUtils.findRPMFileForGivenPackage(package,packageVersion);
if rpmfile is None:
self.logger.error("No rpm file found for package: " + package + "-" + packageVersion)
raise Exception("Missing rpm file")
specificRPM = os.path.basename(rpmfile.replace(".rpm", ""))
pkg = package+"-"+packageVersion
if pkg in listInstalledPackages: |
e45f5730 |
return |
75cb675a |
# mark it as installed - to avoid cyclic recursion |
f93ef2b0 |
listInstalledPackages.append(pkg) |
e45f5730 |
listInstalledRPMs.append(specificRPM) |
8f56b626 |
self._installDependentRunTimePackages(pkgUtils, package, packageVersion, sandbox, destLogPath, |
326d5ca8 |
listInstalledPackages, listInstalledRPMs) |
87815216 |
noDeps = False |
326d5ca8 |
if (package in self.mapPackageToCycles or
package in self.listNodepsPackages or
package in constants.noDepsPackageList): |
87815216 |
noDeps = True |
8f56b626 |
pkgUtils.prepRPMforInstall(package,packageVersion, noDeps, destLogPath) |
2820c61a |
|
8f56b626 |
def _installDependentRunTimePackages(self, pkgUtils, package, packageVersion, sandbox, destLogPath, |
326d5ca8 |
listInstalledPackages, listInstalledRPMs): |
f93ef2b0 |
listRunTimeDependentPackages = self._findRunTimeRequiredRPMPackages(package, packageVersion) |
326d5ca8 |
if listRunTimeDependentPackages: |
2820c61a |
for pkg in listRunTimeDependentPackages: |
326d5ca8 |
if pkg in self.mapPackageToCycles: |
2820c61a |
continue |
f93ef2b0 |
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) |
87815216 |
latestPkgRPM = os.path.basename( |
f93ef2b0 |
pkgUtils.findRPMFileForGivenPackage(packageName, packageVersion)).replace(".rpm", "") |
56c77555 |
if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs: |
2820c61a |
continue |
8f56b626 |
self._installPackage(pkgUtils, packageName,packageVersion, sandbox, destLogPath,listInstalledPackages, listInstalledRPMs) |
326d5ca8 |
|
8f56b626 |
def _findDependentPackagesAndInstalledRPM(self, sandbox):
listInstalledPackages, listInstalledRPMs = self._findInstalledPackages(sandbox) |
26b55679 |
self.logger.debug(listInstalledPackages) |
f93ef2b0 |
listDependentPackages = self._findBuildTimeRequiredPackages() |
e45f5730 |
listTestPackages=[] |
326d5ca8 |
if constants.rpmCheck and self.package in constants.testForceRPMS: |
f93ef2b0 |
# One time optimization
if constants.listMakeCheckRPMPkgWithVersionstoInstall is None:
constants.listMakeCheckRPMPkgWithVersionstoInstalli=[]
for package in constants.listMakeCheckRPMPkgtoInstall:
version = SPECS.getData().getHighestVersion(package)
constants.listMakeCheckRPMPkgWithVersionstoInstall.append(package+"-"+version)
listDependentPackages.extend(self._findBuildTimeCheckRequiredPackages())
testPackages = (set(constants.listMakeCheckRPMPkgWithVersionstoInstall) - |
326d5ca8 |
set(listInstalledPackages) - |
f93ef2b0 |
set([self.package+"-"+self.version])) |
e45f5730 |
listTestPackages=list(set(testPackages)) |
326d5ca8 |
listDependentPackages = list(set(listDependentPackages)) |
e45f5730 |
return listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs |
326d5ca8 |
class PackageBuilderContainer(PackageBuilderBase):
def __init__(self, mapPackageToCycles, pkgBuildType):
PackageBuilderBase.__init__(self, mapPackageToCycles, pkgBuildType) |
56c77555 |
|
f93ef2b0 |
def _buildPackage(self): |
56c77555 |
#should initialize a logger based on package name |
8f56b626 |
containerTaskName = "build-" + self.package + "-" + self.version
container = None |
56c77555 |
try: |
8f56b626 |
container = Container(self.logger)
container.create(containerTaskName) |
56c77555 |
|
326d5ca8 |
tcUtils = ToolChainUtils(self.logName, self.logPath) |
8f56b626 |
tcUtils.installCustomToolChainRPMS(container, self.package, self.version) |
326d5ca8 |
|
e45f5730 |
listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = ( |
8f56b626 |
self._findDependentPackagesAndInstalledRPM(container)) |
326d5ca8 |
pkgUtils = PackageUtils(self.logName, self.logPath) |
8f56b626 |
|
326d5ca8 |
if listDependentPackages: |
8f56b626 |
self.logger.debug("Installing the build time dependent packages......") |
56c77555 |
for pkg in listDependentPackages: |
f93ef2b0 |
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) |
8f56b626 |
self._installPackage(pkgUtils, packageName, packageVersion, container, self.logPath,listInstalledPackages, listInstalledRPMs) |
e45f5730 |
for pkg in listTestPackages: |
4a93976f |
flag = False |
8f56b626 |
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) |
f93ef2b0 |
for depPkg in listDependentPackages:
depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(depPkg)
if depPackageName == packageName:
flag = True
break; |
4a93976f |
if flag == False: |
8f56b626 |
self._installPackage(pkgUtils, packageName,packageVersion, container, self.logPath,listInstalledPackages, listInstalledRPMs)
pkgUtils.installRPMSInOneShot(container) |
26b55679 |
self.logger.debug("Finished installing the build time dependent packages....") |
56c77555 |
|
26b55679 |
self.logger.debug("BuildContainer-buildPackage: Start building the package: " + |
326d5ca8 |
self.package) |
8f56b626 |
pkgUtils.adjustGCCSpecs(container, self.package, self.version)
pkgUtils.buildRPMSForGivenPackage(container, self.package, self.version, self.logPath) |
26b55679 |
self.logger.debug("BuildContainer-buildPackage: Successfully built the package: " + |
326d5ca8 |
self.package) |
56c77555 |
except Exception as e: |
326d5ca8 |
self.logger.error("Failed while building package:" + self.package) |
8f56b626 |
if container is not None:
self.logger.debug("Container " + container.getID() + |
326d5ca8 |
" retained for debugging.") |
8f56b626 |
logFileName = os.path.join(self.logPath, self.package + ".log") |
56c77555 |
fileLog = os.popen('tail -n 20 ' + logFileName).read() |
326d5ca8 |
self.logger.debug(fileLog) |
56c77555 |
raise e
# Remove the container |
8f56b626 |
if container:
container.destroy() |
56c77555 |
|
326d5ca8 |
class PackageBuilderChroot(PackageBuilderBase):
def __init__(self, mapPackageToCycles, pkgBuildType):
PackageBuilderBase.__init__(self, mapPackageToCycles, pkgBuildType) |
56c77555 |
|
8f56b626 |
def _buildPackage(self):
chroot = None |
56c77555 |
try: |
8f56b626 |
chroot = Chroot(self.logger)
chroot.create(self.package + "-" + self.version)
|
326d5ca8 |
tUtils = ToolChainUtils(self.logName, self.logPath) |
8f56b626 |
tUtils.installToolChainRPMS(chroot, self.package, self.version, self.logPath) |
56c77555 |
|
e45f5730 |
listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = ( |
8f56b626 |
self._findDependentPackagesAndInstalledRPM(chroot)) |
326d5ca8 |
pkgUtils = PackageUtils(self.logName, self.logPath) |
5d822c2d |
|
326d5ca8 |
if listDependentPackages: |
26b55679 |
self.logger.debug("Installing the build time dependent packages......") |
56c77555 |
for pkg in listDependentPackages: |
f93ef2b0 |
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) |
8f56b626 |
self._installPackage(pkgUtils, packageName, packageVersion, chroot, self.logPath,listInstalledPackages, listInstalledRPMs) |
e45f5730 |
for pkg in listTestPackages: |
4a93976f |
flag = False |
f93ef2b0 |
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
for depPkg in listDependentPackages:
depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(depPkg)
if depPackageName == packageName: |
8f56b626 |
flag = True
break; |
4a93976f |
if flag == False: |
8f56b626 |
self._installPackage(pkgUtils, packageName,packageVersion, chroot, self.logPath,listInstalledPackages, listInstalledRPMs)
pkgUtils.installRPMSInOneShot(chroot) |
26b55679 |
self.logger.debug("Finished installing the build time dependent packages....") |
326d5ca8 |
|
8f56b626 |
pkgUtils.adjustGCCSpecs(chroot, self.package, self.version)
pkgUtils.buildRPMSForGivenPackage(chroot, self.package, self.version, |
f93ef2b0 |
self.logPath) |
26b55679 |
self.logger.debug("Successfully built the package:" + self.package) |
56c77555 |
except Exception as e: |
326d5ca8 |
self.logger.error("Failed while building package:" + self.package) |
8f56b626 |
self.logger.debug("Chroot: " + chroot.getPath() + |
326d5ca8 |
" not deleted for debugging.")
logFileName = os.path.join(self.logPath, self.package + ".log") |
56c77555 |
fileLog = os.popen('tail -n 100 ' + logFileName).read() |
26b55679 |
self.logger.info(fileLog) |
56c77555 |
raise e |
8f56b626 |
if chroot:
chroot.destroy() |