| 2820c61a |
from PackageUtils import PackageUtils
from Logger import Logger
from ChrootUtils import ChrootUtils
from ToolChainUtils import ToolChainUtils
from CommandUtils import CommandUtils
import os.path
from constants import constants |
| 387ea52d |
import shutil |
| 45c9260c |
from SpecData import SPECS |
| 56c77555 |
import docker
import sys |
| 2820c61a |
|
| 56c77555 |
class PackageBuilderBase(object):
def __init__(self,mapPackageToCycles,listAvailableCyclicPackages,listBuildOptionPackages,pkgBuildOptionFile, pkgBuildType): |
| 7f9d2e12 |
# will be initialized in buildPackageThreadAPI()
self.logName=None
self.logPath=None
self.logger=None
self.package=None |
| 2820c61a |
self.mapPackageToCycles = mapPackageToCycles
self.listAvailableCyclicPackages = listAvailableCyclicPackages
self.listNodepsPackages = ["glibc","gmp","zlib","file","binutils","mpfr","mpc","gcc","ncurses","util-linux","groff","perl","texinfo","rpm","openssl","go"] |
| 90d8acae |
self.listBuildOptionPackages=listBuildOptionPackages
self.pkgBuildOptionFile=pkgBuildOptionFile |
| 56c77555 |
self.pkgBuildType = pkgBuildType |
| 97a9151c |
|
| 56c77555 |
def buildPackageThreadAPIPrepare(self,package,outputMap, threadName):
self.package=package
self.logName="build-"+package
self.logPath=constants.logPath+"/build-"+package
if not os.path.isdir(self.logPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p "+self.logPath)
self.logger=Logger.getLogger(self.logName,self.logPath) |
| 97a9151c |
|
| 2820c61a |
def findPackageNameFromRPMFile(self,rpmfile):
rpmfile=os.path.basename(rpmfile)
releaseindex=rpmfile.rfind("-")
if releaseindex == -1:
self.logger.error("Invalid rpm file:"+rpmfile)
return None
versionindex=rpmfile[0:releaseindex].rfind("-")
if versionindex == -1:
self.logger.error("Invalid rpm file:"+rpmfile)
return None
packageName=rpmfile[0:versionindex]
return packageName |
| 97a9151c |
|
| 56c77555 |
def findInstalledPackages(self, instanceID):
pkgUtils = PackageUtils(self.logName, self.logPath)
if self.pkgBuildType == "chroot":
listInstalledRPMs = pkgUtils.findInstalledRPMPackages(instanceID)
elif self.pkgBuildType == "container":
listInstalledRPMs = pkgUtils.findInstalledRPMPackagesInContainer(instanceID) |
| 2820c61a |
listInstalledPackages=[]
for installedRPM in listInstalledRPMs:
packageName=self.findPackageNameFromRPMFile(installedRPM)
if packageName is not None:
listInstalledPackages.append(packageName) |
| 56c77555 |
return listInstalledPackages, listInstalledRPMs |
| adf248d5 |
|
| 7f9d2e12 |
def checkIfPackageIsAlreadyBuilt(self):
basePkg=SPECS.getData().getSpecName(self.package) |
| 45c9260c |
listRPMPackages=SPECS.getData().getRPMPackages(basePkg) |
| adf248d5 |
packageIsAlreadyBuilt=True
pkgUtils = PackageUtils(self.logName,self.logPath)
for pkg in listRPMPackages:
if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
packageIsAlreadyBuilt=False
break
return packageIsAlreadyBuilt
|
| 2820c61a |
def findRunTimeRequiredRPMPackages(self,rpmPackage): |
| 45c9260c |
listRequiredPackages=SPECS.getData().getRequiresForPackage(rpmPackage) |
| 518d6a6f |
return listRequiredPackages |
| 97a9151c |
|
| 7f9d2e12 |
def findBuildTimeRequiredPackages(self):
listRequiredPackages=SPECS.getData().getBuildRequiresForPackage(self.package) |
| 518d6a6f |
return listRequiredPackages |
| 97a9151c |
|
| 7f9d2e12 |
def findBuildTimeCheckRequiredPackages(self):
listRequiredPackages=SPECS.getData().getCheckBuildRequiresForPackage(self.package) |
| 5f40784b |
return listRequiredPackages
|
| 56c77555 |
def installPackage(self, pkgUtils, package, instanceID, destLogPath, listInstalledPackages, listInstalledRPMs):
latestRPM = os.path.basename(pkgUtils.findRPMFileForGivenPackage(package)).replace(".rpm", "")
if package in listInstalledPackages and latestRPM in listInstalledRPMs: |
| 518d6a6f |
return |
| 75cb675a |
# mark it as installed - to avoid cyclic recursion
listInstalledPackages.append(package) |
| 56c77555 |
listInstalledRPMs.append(latestRPM)
self.installDependentRunTimePackages(pkgUtils,package,instanceID,destLogPath,listInstalledPackages, listInstalledRPMs) |
| 2820c61a |
noDeps=False
if self.mapPackageToCycles.has_key(package):
noDeps = True
if package in self.listNodepsPackages:
noDeps=True |
| a5e4be9f |
if package in constants.noDepsPackageList:
noDeps=True |
| 56c77555 |
if self.pkgBuildType == "chroot":
pkgUtils.installRPM(package,instanceID,noDeps,destLogPath)
elif self.pkgBuildType == "container":
pkgUtils.prepRPMforInstallInContainer(package, instanceID, noDeps, destLogPath) |
| 2820c61a |
|
| 56c77555 |
def installDependentRunTimePackages(self,pkgUtils,package,instanceID,destLogPath,listInstalledPackages, listInstalledRPMs): |
| 518d6a6f |
listRunTimeDependentPackages=self.findRunTimeRequiredRPMPackages(package) |
| 2820c61a |
if len(listRunTimeDependentPackages) != 0:
for pkg in listRunTimeDependentPackages:
if self.mapPackageToCycles.has_key(pkg) and pkg not in self.listAvailableCyclicPackages:
continue |
| 56c77555 |
latestPkgRPM = os.path.basename(pkgUtils.findRPMFileForGivenPackage(pkg)).replace(".rpm", "")
if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs: |
| 2820c61a |
continue |
| 56c77555 |
self.installPackage(pkgUtils, pkg, instanceID, destLogPath, listInstalledPackages, listInstalledRPMs)
class PackageBuilderContainer(object):
def __init__(self, mapPackageToCycles, listAvailableCyclicPackages, listBuildOptionPackages, pkgBuildOptionFile, pkgBuildType):
self.buildContainerImage = "photon_build_container:latest"
self.dockerClient = docker.from_env(version="auto")
self.base = PackageBuilderBase(mapPackageToCycles, listAvailableCyclicPackages,
listBuildOptionPackages, pkgBuildOptionFile, pkgBuildType)
def buildPackageThreadAPI(self, package, outputMap, threadName):
self.base.buildPackageThreadAPIPrepare(package, outputMap, threadName)
try:
self.buildPackage()
outputMap[threadName]=True
except Exception as e:
# TODO: self.logger might be None |
| b8ab7fb4 |
self.base.logger.exception(e) |
| 56c77555 |
outputMap[threadName]=False
raise e
def prepareBuildContainer(self, containerTaskName, packageName, isToolChainPackage=False):
# Prepare an empty chroot environment to let docker use the BUILD folder.
# This avoids docker using overlayFS which will cause make check failure.
chrootName="build-"+packageName
chrUtils = ChrootUtils(self.base.logName, self.base.logPath)
returnVal,chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
raise Exception("Unable to prepare build root")
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath)
cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/BUILD")
containerID = None
mountVols = {
constants.prevPublishRPMRepo: {'bind': '/publishrpms', 'mode': 'ro'},
constants.prevPublishXRPMRepo: {'bind': '/publishxrpms', 'mode': 'ro'},
constants.tmpDirPath: {'bind': '/tmp', 'mode': 'rw'},
constants.rpmPath: {'bind': constants.topDirPath + "/RPMS", 'mode': 'rw'},
constants.sourceRpmPath: {'bind': constants.topDirPath + "/SRPMS", 'mode': 'rw'},
constants.logPath + "/" + self.base.logName: {'bind': constants.topDirPath + "/LOGS", 'mode': 'rw'}, |
| b572cf8e |
chrootID + constants.topDirPath + "/BUILD": {'bind': constants.topDirPath + "/BUILD", 'mode': 'rw'},
constants.dockerUnixSocket: {'bind': constants.dockerUnixSocket, 'mode': 'rw'} |
| 56c77555 |
}
containerName = containerTaskName
containerName = containerName.replace("+", "p")
try:
oldContainerID = self.dockerClient.containers.get(containerName)
if oldContainerID is not None:
oldContainerID.remove(force=True)
except docker.errors.NotFound:
sys.exc_clear()
try:
self.base.logger.info("BuildContainer-prepareBuildContainer: Starting build container: " + containerName)
#TODO: Is init=True equivalent of --sig-proxy?
privilegedDocker = False
cap_list = ['SYS_PTRACE']
if packageName in constants.listReqPrivilegedDockerForTest:
privilegedDocker = True
containerID = self.dockerClient.containers.run(self.buildContainerImage,
detach=True,
cap_add=cap_list,
privileged=privilegedDocker,
name=containerName,
network_mode="host",
volumes=mountVols,
command="/bin/bash -l -c /wait.sh")
self.base.logger.debug("Started Photon build container for task " + containerTaskName
+ " ID: " + containerID.short_id)
if not containerID:
raise Exception("Unable to start Photon build container for task " + containerTaskName)
except Exception as e:
self.base.logger.debug("Unable to start Photon build container for task " + containerTaskName)
raise e
return containerID, chrootID |
| b5e09fac |
|
| 56c77555 |
def buildPackage(self):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self.base.checkIfPackageIsAlreadyBuilt():
if not constants.rpmCheck:
self.base.logger.info("Skipping building the package:" + self.base.package)
return
elif constants.rpmCheck and self.base.package not in constants.testForceRPMS:
self.base.logger.info("Skipping testing the package:" + self.base.package)
return
#should initialize a logger based on package name
containerTaskName = "build-" + self.base.package
containerID = None
chrootID = None
isToolChainPackage = False
if self.base.package in constants.listToolChainPackages:
isToolChainPackage = True
destLogPath = constants.logPath + "/build-" + self.base.package
try:
containerID, chrootID = self.prepareBuildContainer(containerTaskName, self.base.package, isToolChainPackage)
tcUtils = ToolChainUtils(self.base.logName, self.base.logPath)
if self.base.package in constants.perPackageToolChain:
self.base.logger.debug(constants.perPackageToolChain[self.base.package])
tcUtils.installCustomToolChainRPMSinContainer(containerID,
constants.perPackageToolChain[self.base.package],
self.base.package);
listInstalledPackages, listInstalledRPMs = self.base.findInstalledPackages(containerID)
self.base.logger.info(listInstalledPackages)
listDependentPackages = self.base.findBuildTimeRequiredPackages()
if constants.rpmCheck and self.base.package in constants.testForceRPMS:
listDependentPackages.extend(self.base.findBuildTimeCheckRequiredPackages())
testPackages=set(constants.listMakeCheckRPMPkgtoInstall)-set(listInstalledPackages)-set([self.base.package])
listDependentPackages.extend(testPackages)
listDependentPackages=list(set(listDependentPackages))
pkgUtils = PackageUtils(self.base.logName, self.base.logPath)
if len(listDependentPackages) != 0:
self.base.logger.info("BuildContainer-buildPackage: Installing dependent packages..")
self.base.logger.info(listDependentPackages)
for pkg in listDependentPackages:
self.base.installPackage(pkgUtils, pkg, containerID, destLogPath, listInstalledPackages, listInstalledRPMs)
pkgUtils.installRPMSInAOneShotInContainer(containerID, destLogPath)
self.base.logger.info("Finished installing the build time dependent packages......")
self.base.logger.info("BuildContainer-buildPackage: Start building the package: " + self.base.package)
pkgUtils.adjustGCCSpecsInContainer(self.base.package, containerID, destLogPath)
pkgUtils.buildRPMSForGivenPackageInContainer(
self.base.package,
containerID,
self.base.listBuildOptionPackages,
self.base.pkgBuildOptionFile,
destLogPath)
self.base.logger.info("BuildContainer-buildPackage: Successfully built the package: " + self.base.package)
except Exception as e:
self.base.logger.error("Failed while building package:" + self.base.package)
if containerID is not None:
self.base.logger.debug("Container " + containerID.short_id + " retained for debugging.")
logFileName = os.path.join(destLogPath, self.base.package + ".log")
fileLog = os.popen('tail -n 20 ' + logFileName).read()
self.base.logger.debug(fileLog)
raise e
# Remove the container
if containerID is not None:
containerID.remove(force=True)
# Remove the dummy chroot
if chrootID is not None:
chrUtils = ChrootUtils(self.base.logName, self.base.logPath)
chrUtils.destroyChroot(chrootID)
class PackageBuilderChroot(object):
def __init__(self, mapPackageToCycles, listAvailableCyclicPackages, listBuildOptionPackages, pkgBuildOptionFile, pkgBuildType):
self.base = PackageBuilderBase(mapPackageToCycles, listAvailableCyclicPackages,
listBuildOptionPackages, pkgBuildOptionFile, pkgBuildType)
def buildPackageThreadAPI(self, package, outputMap, threadName):
self.base.buildPackageThreadAPIPrepare(package,outputMap, threadName)
try:
self.buildPackage()
outputMap[threadName]=True
except Exception as e:
# TODO: self.logger might be None |
| b8ab7fb4 |
self.base.logger.exception(e) |
| 56c77555 |
outputMap[threadName]=False
raise e
def prepareBuildRoot(self):
chrootID=None
chrootName="build-"+self.base.package
try:
chrUtils = ChrootUtils(self.base.logName,self.base.logPath)
returnVal,chrootID = chrUtils.createChroot(chrootName)
self.base.logger.debug("Created new chroot: " + chrootID)
if not returnVal:
raise Exception("Unable to prepare build root")
tUtils=ToolChainUtils(self.base.logName,self.base.logPath)
tUtils.installToolChainRPMS(chrootID, self.base.package, self.base.logPath)
except Exception as e:
if chrootID is not None:
self.base.logger.debug("Deleting chroot: " + chrootID)
chrUtils.destroyChroot(chrootID)
raise e
return chrootID
def buildPackage(self):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self.base.checkIfPackageIsAlreadyBuilt():
if not constants.rpmCheck:
self.base.logger.info("Skipping building the package:" + self.base.package)
return
elif constants.rpmCheck and self.base.package not in constants.testForceRPMS:
self.base.logger.info("Skipping testing the package:" + self.base.package)
return
chrUtils = ChrootUtils(self.base.logName,self.base.logPath)
chrootID=None
try:
chrootID = self.prepareBuildRoot()
listInstalledPackages, listInstalledRPMs = self.base.findInstalledPackages(chrootID)
listDependentPackages=self.base.findBuildTimeRequiredPackages()
if constants.rpmCheck and self.base.package in constants.testForceRPMS:
listDependentPackages.extend(self.base.findBuildTimeCheckRequiredPackages())
testPackages=set(constants.listMakeCheckRPMPkgtoInstall)-set(listInstalledPackages)-set([self.base.package])
listDependentPackages.extend(testPackages)
listDependentPackages=list(set(listDependentPackages))
pkgUtils = PackageUtils(self.base.logName,self.base.logPath)
if len(listDependentPackages) != 0:
self.base.logger.info("Installing the build time dependent packages......")
for pkg in listDependentPackages:
self.base.installPackage(pkgUtils, pkg, chrootID, self.base.logPath, listInstalledPackages, listInstalledRPMs)
pkgUtils.installRPMSInAOneShot(chrootID, self.base.logPath)
self.base.logger.info("Finished installing the build time dependent packages......")
pkgUtils.adjustGCCSpecs(self.base.package, chrootID, self.base.logPath)
pkgUtils.buildRPMSForGivenPackage(self.base.package, chrootID,self.base.listBuildOptionPackages,
self.base.pkgBuildOptionFile, self.base.logPath)
self.base.logger.info("Successfully built the package:" + self.base.package)
except Exception as e:
self.base.logger.error("Failed while building package:" + self.base.package)
self.base.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.")
logFileName = os.path.join(self.base.logPath, self.base.package + ".log")
fileLog = os.popen('tail -n 100 ' + logFileName).read()
self.base.logger.debug(fileLog)
raise e
if chrootID is not None:
chrUtils.destroyChroot(chrootID) |