from PackageUtils import PackageUtils
from Logger import Logger
from ToolChainUtils import ToolChainUtils
from CommandUtils import CommandUtils
import os.path
import sys
from constants import constants
import shutil
import docker
from SpecData import SPECS
class BuildContainer(object):
def __init__(self, mapPackageToCycles, listAvailableCyclicPackages, listBuildOptionPackages, pkgBuildOptionFile, logName=None, logPath=None):
if logName is None:
logName = "BuildContainer"
if logPath is None:
logPath = constants.logPath
self.logName = logName
self.logPath = logPath
self.logger = Logger.getLogger(logName, logPath, True)
self.buildContainerImage = "photon_build_container:latest"
self.dockerClient = docker.from_env(version="auto")
self.mapPackageToCycles = mapPackageToCycles
self.listAvailableCyclicPackages = listAvailableCyclicPackages
self.listNodepsPackages = ["glibc","gmp","zlib","file","binutils","mpfr","mpc","gcc","ncurses","util-linux","groff","perl","texinfo","rpm","openssl","openssl-devel","go"]
self.listBuildOptionPackages = listBuildOptionPackages
self.pkgBuildOptionFile = pkgBuildOptionFile
def prepareBuildContainer(self, containerTaskName, packageName, isToolChainPackage=False):
containerID = None
mountVols = {
constants.prevPublishRPMRepo: {'bind': '/publishrpms', 'mode': 'ro'},
constants.prevPublishXRPMRepo: {'bind': '/publishxrpms', 'mode': 'ro'},
constants.tmpDirPath: {'bind': '/tmp', 'mode': 'rw'},
constants.rpmPath: {'bind': constants.topDirPath + "/RPMS", 'mode': 'rw'},
constants.sourceRpmPath: {'bind': constants.topDirPath + "/SRPMS", 'mode': 'rw'},
constants.logPath + "/" + self.logName: {'bind': constants.topDirPath + "/LOGS", 'mode': 'rw'},
}
containerName = containerTaskName
containerName = containerName.replace("+", "p")
try:
oldContainerID = self.dockerClient.containers.get(containerName)
if oldContainerID is not None:
oldContainerID.remove(force=True)
except docker.errors.NotFound:
sys.exc_clear()
try:
self.logger.info("BuildContainer-prepareBuildContainer: Starting build container: " + containerName)
#TODO: Is init=True equivalent of --sig-proxy?
if packageName in constants.listReqPrivilegedDockerForTest:
containerID = self.dockerClient.containers.run(self.buildContainerImage,
detach=True,
privileged=True,
name=containerName,
network_mode="host",
volumes=mountVols,
command="/bin/bash -l -c /wait.sh")
else:
containerID = self.dockerClient.containers.run(self.buildContainerImage,
detach=True,
name=containerName,
network_mode="host",
volumes=mountVols,
command="/bin/bash -l -c /wait.sh")
self.logger.debug("Started Photon build container for task " + containerTaskName
+ " ID: " + containerID.short_id)
if not containerID:
raise Exception("Unable to start Photon build container for task " + containerTaskName)
except Exception as e:
self.logger.debug("Unable to start Photon build container for task " + containerTaskName)
raise e
return containerID
def findPackageNameFromRPMFile(self, rpmfile):
rpmfile = os.path.basename(rpmfile)
releaseindex = rpmfile.rfind("-")
if releaseindex == -1:
self.logger.error("Invalid rpm file:" + rpmfile)
return None
versionindex=rpmfile[0:releaseindex].rfind("-")
if versionindex == -1:
self.logger.error("Invalid rpm file:" + rpmfile)
return None
packageName = rpmfile[0:versionindex]
return packageName
def findInstalledPackages(self, containerID):
pkgUtils = PackageUtils(self.logName, self.logPath)
listInstalledRPMs = pkgUtils.findInstalledRPMPackagesInContainer(containerID)
listInstalledPackages = []
for installedRPM in listInstalledRPMs:
packageName = self.findPackageNameFromRPMFile(installedRPM)
if packageName is not None:
listInstalledPackages.append(packageName)
return listInstalledPackages, listInstalledRPMs
def buildPackageThreadAPI(self, package, outputMap, threadName,):
try:
self.buildPackage(package)
outputMap[threadName] = True
except Exception as e:
self.logger.error(e)
outputMap[threadName] = False
def checkIfPackageIsAlreadyBuilt(self, package):
basePkg = SPECS.getData().getSpecName(package)
listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
packageIsAlreadyBuilt = True
pkgUtils = PackageUtils(self.logName,self.logPath)
for pkg in listRPMPackages:
if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
packageIsAlreadyBuilt = False
break
return packageIsAlreadyBuilt
def buildPackage(self, package):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self.checkIfPackageIsAlreadyBuilt(package):
if not constants.rpmCheck:
self.logger.info("Skipping building the package:"+package)
return
elif constants.rpmCheck and package not in constants.testForceRPMS:
self.logger.info("Skipping testing the package:"+package)
return
#should initialize a logger based on package name
containerTaskName = "build-" + package
containerID = None
isToolChainPackage = False
if package in constants.listToolChainPackages:
isToolChainPackage = True
destLogPath = constants.logPath + "/build-"+package
try:
containerID = self.prepareBuildContainer(containerTaskName, package, isToolChainPackage)
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p "+destLogPath)
tcUtils = ToolChainUtils(self.logName, self.logPath)
if package in constants.perPackageToolChain:
self.logger.debug(constants.perPackageToolChain[package])
tcUtils.installCustomToolChainRPMSinContainer(containerID, constants.perPackageToolChain[package], package);
listInstalledPackages, listInstalledRPMs = self.findInstalledPackages(containerID)
self.logger.info(listInstalledPackages)
listDependentPackages = self.findBuildTimeRequiredPackages(package)
if constants.rpmCheck and package in constants.testForceRPMS:
listDependentPackages.extend(self.findBuildTimeCheckRequiredPackages(package))
testPackages=set(constants.listMakeCheckRPMPkgtoInstall)-set(listInstalledPackages)-set([package])
listDependentPackages.extend(testPackages)
listDependentPackages=list(set(listDependentPackages))
pkgUtils = PackageUtils(self.logName,self.logPath)
if len(listDependentPackages) != 0:
self.logger.info("BuildContainer-buildPackage: Installing dependent packages..")
self.logger.info(listDependentPackages)
for pkg in listDependentPackages:
self.installPackage(pkgUtils, pkg, containerID, destLogPath, listInstalledPackages, listInstalledRPMs)
# Special case sqlite due to package renamed from sqlite-autoconf to sqlite
if "sqlite" in listInstalledPackages or "sqlite-devel" in listInstalledPackages or "sqlite-libs" in listInstalledPackages:
if "sqlite" not in listInstalledPackages:
self.installPackage(pkgUtils, "sqlite", containerID, destLogPath, listInstalledPackages, listInstalledRPMs)
if "sqlite-devel" not in listInstalledPackages:
self.installPackage(pkgUtils, "sqlite-devel", containerID, destLogPath, listInstalledPackages, listInstalledRPMs)
if "sqlite-libs" not in listInstalledPackages:
self.installPackage(pkgUtils, "sqlite-libs", containerID, destLogPath, listInstalledPackages, listInstalledRPMs)
pkgUtils.installRPMSInAOneShotInContainer(containerID, destLogPath)
pkgUtils.adjustGCCSpecsInContainer(package, containerID, destLogPath)
pkgUtils.buildRPMSForGivenPackageInContainer(
package,
containerID,
self.listBuildOptionPackages,
self.pkgBuildOptionFile,
destLogPath)
self.logger.info("BuildContainer-buildPackage: Successfully built the package: " + package)
except Exception as e:
self.logger.error("Failed while building package:" + package)
if containerID is not None:
self.logger.debug("Container " + containerID.short_id + " retained for debugging.")
logFileName = os.path.join(destLogPath, package + ".log")
fileLog = os.popen('tail -n 20 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
# Remove the container
if containerID is not None:
containerID.remove(force=True)
def findRunTimeRequiredRPMPackages(self, rpmPackage):
listRequiredPackages = SPECS.getData().getRequiresForPackage(rpmPackage)
return listRequiredPackages
def findBuildTimeRequiredPackages(self, package):
listRequiredPackages = SPECS.getData().getBuildRequiresForPackage(package)
return listRequiredPackages
def findBuildTimeCheckRequiredPackages(self,package):
listRequiredPackages=SPECS.getData().getCheckBuildRequiresForPackage(package)
return listRequiredPackages
def installPackage(self, pkgUtils, package, containerID, destLogPath, listInstalledPackages, listInstalledRPMs):
latestRPM = os.path.basename(pkgUtils.findRPMFileForGivenPackage(package)).replace(".rpm", "")
if package in listInstalledPackages and latestRPM in listInstalledRPMs:
return
self.installDependentRunTimePackages(pkgUtils, package, containerID, destLogPath, listInstalledPackages, listInstalledRPMs)
noDeps = False
if self.mapPackageToCycles.has_key(package):
noDeps = True
if package in self.listNodepsPackages:
noDeps = True
if package in constants.noDepsPackageList:
noDeps = True
pkgUtils.prepRPMforInstallInContainer(package, containerID, noDeps, destLogPath)
listInstalledPackages.append(package)
listInstalledRPMs.append(latestRPM)
def installDependentRunTimePackages(self, pkgUtils, package, containerID, destLogPath, listInstalledPackages, listInstalledRPMs):
listRunTimeDependentPackages = self.findRunTimeRequiredRPMPackages(package)
if len(listRunTimeDependentPackages) != 0:
for pkg in listRunTimeDependentPackages:
if self.mapPackageToCycles.has_key(pkg) and pkg not in self.listAvailableCyclicPackages:
continue
latestPkgRPM = os.path.basename(pkgUtils.findRPMFileForGivenPackage(pkg)).replace(".rpm", "")
if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs:
continue
self.installPackage(pkgUtils, pkg, containerID, destLogPath, listInstalledPackages, listInstalledRPMs)