from PackageUtils import PackageUtils
from Logger import Logger
from ToolChainUtils import ToolChainUtils
from CommandUtils import CommandUtils
import os.path
import sys
from constants import constants
import shutil
import docker
class BuildContainer(object):
def __init__(self, mapPackageToCycles, listAvailableCyclicPackages, listBuildOptionPackages, pkgBuildOptionFile, logName=None, logPath=None):
if logName is None:
logName = "BuildContainer"
if logPath is None:
logPath = constants.logPath
self.logName = logName
self.logPath = logPath
self.logger = Logger.getLogger(logName, logPath, True)
self.buildContainerImage = "photon_build_container:latest"
self.dockerClient = docker.from_env(version="auto")
self.mapPackageToCycles = mapPackageToCycles
self.listAvailableCyclicPackages = listAvailableCyclicPackages
self.listNodepsPackages = ["glibc","gmp","zlib","file","binutils","mpfr","mpc","gcc","ncurses","util-linux","groff","perl","texinfo","rpm","openssl","openssl-devel","go"]
self.listBuildOptionPackages = listBuildOptionPackages
self.pkgBuildOptionFile = pkgBuildOptionFile
def prepareBuildContainer(self, containerTaskName, packageName, isToolChainPackage=False):
containerID = None
mountVols = {
constants.prevPublishRPMRepo: {'bind': '/publishrpms', 'mode': 'ro'},
constants.prevPublishXRPMRepo: {'bind': '/publishxrpms', 'mode': 'ro'},
constants.rpmPath: {'bind': constants.topDirPath + "/RPMS", 'mode': 'rw'},
constants.sourceRpmPath: {'bind': constants.topDirPath + "/SRPMS", 'mode': 'rw'},
constants.logPath + "/" + self.logName: {'bind': constants.topDirPath + "/LOGS", 'mode': 'rw'},
}
containerName = containerTaskName
containerName = containerName.replace("+", "p")
try:
oldContainerID = self.dockerClient.containers.get(containerName)
if oldContainerID is not None:
oldContainerID.remove(force=True)
except docker.errors.NotFound:
sys.exc_clear()
try:
self.logger.info("BuildContainer-prepareBuildContainer: Starting build container: " + containerName)
#TODO: Is init=True equivalent of --sig-proxy?
containerID = self.dockerClient.containers.run(self.buildContainerImage,
detach=True,
name=containerName,
network_mode="host",
volumes=mountVols,
command="/bin/bash -l -c /wait.sh")
self.logger.debug("Started Photon build container for task " + containerTaskName
+ " ID: " + containerID.short_id)
if not containerID:
raise Exception("Unable to start Photon build container for task " + containerTaskName)
except Exception as e:
self.logger.debug("Unable to start Photon build container for task " + containerTaskName)
raise e
return containerID
def findPackageNameFromRPMFile(self, rpmfile):
rpmfile = os.path.basename(rpmfile)
releaseindex = rpmfile.rfind("-")
if releaseindex == -1:
self.logger.error("Invalid rpm file:" + rpmfile)
return None
versionindex=rpmfile[0:releaseindex].rfind("-")
if versionindex == -1:
self.logger.error("Invalid rpm file:" + rpmfile)
return None
packageName = rpmfile[0:versionindex]
return packageName
def findInstalledPackages(self, containerID):
pkgUtils = PackageUtils(self.logName, self.logPath)
listInstalledRPMs = pkgUtils.findInstalledRPMPackagesInContainer(containerID)
listInstalledPackages = []
for installedRPM in listInstalledRPMs:
packageName = self.findPackageNameFromRPMFile(installedRPM)
if packageName is not None:
listInstalledPackages.append(packageName)
return listInstalledPackages, listInstalledRPMs
def buildPackageThreadAPI(self, package, outputMap, threadName,):
try:
self.buildPackage(package)
outputMap[threadName] = True
except Exception as e:
self.logger.error(e)
outputMap[threadName] = False
def checkIfPackageIsAlreadyBuilt(self, package):
basePkg = constants.specData.getSpecName(package)
listRPMPackages = constants.specData.getRPMPackages(basePkg)
packageIsAlreadyBuilt = True
pkgUtils = PackageUtils(self.logName,self.logPath)
for pkg in listRPMPackages:
if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
packageIsAlreadyBuilt = False
break
return packageIsAlreadyBuilt
def buildPackage(self, package):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self.checkIfPackageIsAlreadyBuilt(package):
if not constants.rpmCheck:
self.logger.info("Skipping building the package:"+package)
return
elif constants.rpmCheck and package not in constants.testForceRPMS:
self.logger.info("Skipping testing the package:"+package)
return
#should initialize a logger based on package name
containerTaskName = "build-" + package
containerID = None
isToolChainPackage = False
if package in constants.listToolChainPackages:
isToolChainPackage = True
destLogPath = constants.logPath + "/build-"+package
try:
containerID = self.prepareBuildContainer(containerTaskName, package, isToolChainPackage)
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p "+destLogPath)
tcUtils = ToolChainUtils(self.logName, self.logPath)
if package in constants.perPackageToolChain:
self.logger.debug(constants.perPackageToolChain[package])
tcUtils.installCustomToolChainRPMSinContainer(containerID, constants.perPackageToolChain[package], package);
listInstalledPackages, listInstalledRPMs = self.findInstalledPackages(containerID)
self.logger.info(listInstalledPackages)
listDependentPackages = self.findBuildTimeRequiredPackages(package)
if constants.rpmCheck and package in constants.testForceRPMS:
listDependentPackages.extend(self.findBuildTimeCheckRequiredPackages(package))
testPackages=set(constants.listMakeCheckRPMPkgtoInstall)-set(listInstalledPackages)-set([package])
listDependentPackages.extend(testPackages)
listDependentPackages=list(set(listDependentPackages))
pkgUtils = PackageUtils(self.logName,self.logPath)
if len(listDependentPackages) != 0:
self.logger.info("BuildContainer-buildPackage: Installing dependent packages..")
self.logger.info(listDependentPackages)
for pkg in listDependentPackages:
self.installPackage(pkgUtils, pkg, containerID, destLogPath, listInstalledPackages, listInstalledRPMs)
# Special case sqlite due to package renamed from sqlite-autoconf to sqlite
if "sqlite" in listInstalledPackages or "sqlite-devel" in listInstalledPackages or "sqlite-libs" in listInstalledPackages:
if "sqlite" not in listInstalledPackages:
self.installPackage(pkgUtils, "sqlite", containerID, destLogPath, listInstalledPackages, listInstalledRPMs)
if "sqlite-devel" not in listInstalledPackages:
self.installPackage(pkgUtils, "sqlite-devel", containerID, destLogPath, listInstalledPackages, listInstalledRPMs)
if "sqlite-libs" not in listInstalledPackages:
self.installPackage(pkgUtils, "sqlite-libs", containerID, destLogPath, listInstalledPackages, listInstalledRPMs)
pkgUtils.installRPMSInAOneShotInContainer(containerID, destLogPath)
pkgUtils.adjustGCCSpecsInContainer(package, containerID, destLogPath)
pkgUtils.buildRPMSForGivenPackageInContainer(
package,
containerID,
self.listBuildOptionPackages,
self.pkgBuildOptionFile,
destLogPath)
self.logger.info("BuildContainer-buildPackage: Successfully built the package: " + package)
except Exception as e:
self.logger.error("Failed while building package:" + package)
if containerID is not None:
self.logger.debug("Container " + containerID.short_id + " retained for debugging.")
logFileName = os.path.join(destLogPath, package + ".log")
fileLog = os.popen('tail -n 20 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
# Remove the container
if containerID is not None:
containerID.remove(force=True)
def findRunTimeRequiredRPMPackages(self, rpmPackage):
listRequiredPackages = constants.specData.getRequiresForPackage(rpmPackage)
return listRequiredPackages
def findBuildTimeRequiredPackages(self, package):
listRequiredPackages = constants.specData.getBuildRequiresForPackage(package)
return listRequiredPackages
def findBuildTimeCheckRequiredPackages(self,package):
listRequiredPackages=constants.specData.getCheckBuildRequiresForPackage(package)
return listRequiredPackages
def installPackage(self, pkgUtils, package, containerID, destLogPath, listInstalledPackages, listInstalledRPMs):
latestRPM = os.path.basename(pkgUtils.findRPMFileForGivenPackage(package)).replace(".rpm", "")
if package in listInstalledPackages and latestRPM in listInstalledRPMs:
return
self.installDependentRunTimePackages(pkgUtils, package, containerID, destLogPath, listInstalledPackages, listInstalledRPMs)
noDeps = False
if self.mapPackageToCycles.has_key(package):
noDeps = True
if package in self.listNodepsPackages:
noDeps = True
if package in constants.noDepsPackageList:
noDeps = True
pkgUtils.prepRPMforInstallInContainer(package, containerID, noDeps, destLogPath)
listInstalledPackages.append(package)
listInstalledRPMs.append(latestRPM)
def installDependentRunTimePackages(self, pkgUtils, package, containerID, destLogPath, listInstalledPackages, listInstalledRPMs):
listRunTimeDependentPackages = self.findRunTimeRequiredRPMPackages(package)
if len(listRunTimeDependentPackages) != 0:
for pkg in listRunTimeDependentPackages:
if self.mapPackageToCycles.has_key(pkg) and pkg not in self.listAvailableCyclicPackages:
continue
latestPkgRPM = os.path.basename(pkgUtils.findRPMFileForGivenPackage(pkg)).replace(".rpm", "")
if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs:
continue
self.installPackage(pkgUtils, pkg, containerID, destLogPath, listInstalledPackages, listInstalledRPMs)