| 87815216 |
import os.path
import platform
import traceback |
| 342710b6 |
import re |
| 2820c61a |
from CommandUtils import CommandUtils
from ChrootUtils import ChrootUtils
from Logger import Logger
from PackageUtils import PackageUtils
from constants import constants |
| e45f5730 |
from SpecData import SPECS |
| 2820c61a |
class ToolChainUtils(object): |
| 5fc28b16 |
|
| 87815216 |
def __init__(self, logName=None, logPath=None): |
| 2820c61a |
if logName is None:
logName = "Toolchain Utils"
if logPath is None:
logPath = constants.logPath |
| 87815216 |
self.logName = logName
self.logPath = logPath
self.logger = Logger.getLogger(logName, logPath) |
| 2820c61a |
self.adjustToolChainScript = "adjust-tool-chain.sh"
self.localegenScript = "./locale-gen.sh"
self.localegenConfig = "./locale-gen.conf" |
| 87815216 |
self.prepareBuildRootCmd = "./prepare-build-root.sh" |
| 58120db4 |
self.rpmbuildCommand = "rpmbuild" |
| 87815216 |
if os.geteuid() == 0:
self.rpmCommand = "rpm" |
| 45e37dfb |
else: |
| 87815216 |
self.rpmCommand = "fakeroot-ng rpm" |
| 45e37dfb |
|
| 87815216 |
def prepareBuildRoot(self, chrootID): |
| 387ea52d |
self.logger.info("Preparing build environment")
cmdUtils = CommandUtils() |
| 87815216 |
cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/dev")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/etc")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/proc")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/run")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/sys")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/tmp")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath)
cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath +
"/RPMS/" + platform.machine())
cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/RPMS/noarch")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/SOURCES")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/SPECS")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/LOGS")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/BUILD")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/BUILDROOT") |
| 5fc28b16 |
|
| 87815216 |
prepareChrootCmd = self.prepareBuildRootCmd + " " + chrootID
logFile = self.logPath + "/prepareBuildRoot.log"
returnVal = cmdUtils.runCommandInShell(prepareChrootCmd, logFile) |
| 387ea52d |
if not returnVal:
self.logger.error("Prepare build root script failed.Unable to prepare chroot.")
raise Exception("Prepare build root script failed") |
| 2fcd8ac0 |
|
| 87815216 |
self.logger.info("Successfully prepared chroot:" + chrootID) |
| 2fcd8ac0 |
|
| 87815216 |
def findRPMFileInGivenLocation(self, package, rpmdirPath): |
| a4a06c39 |
cmdUtils = CommandUtils() |
| 87815216 |
listFoundRPMFiles = cmdUtils.findFile(package + "-*.rpm", rpmdirPath)
listFilterRPMFiles = [] |
| a4a06c39 |
for f in listFoundRPMFiles: |
| 87815216 |
rpmFileName = os.path.basename(f)
checkRPMName = rpmFileName.replace(package, "") |
| 7acb019e |
rpmNameSplit = checkRPMName.split("-")
if len(rpmNameSplit) == 3: |
| a4a06c39 |
listFilterRPMFiles.append(f) |
| 87815216 |
if len(listFilterRPMFiles) == 1: |
| a4a06c39 |
return listFilterRPMFiles[0] |
| 87815216 |
if len(listFilterRPMFiles) == 0: |
| a4a06c39 |
return None |
| 87815216 |
if len(listFilterRPMFiles) > 1:
self.logger.error("Found multiple rpm files for given package in rpm directory." +
"Unable to determine the rpm file for package:" + package) |
| a4a06c39 |
return None |
| 5fc28b16 |
|
| 085306ae |
def buildCoreToolChainPackages(self): |
| 7f9d2e12 |
self.logger.info("Building core toolchain packages.....") |
| 87815216 |
chrootID = None |
| 7418d2bf |
pkgCount = 0 |
| 518d6a6f |
try: |
| 87815216 |
pkgUtils = PackageUtils(self.logName, self.logPath) |
| fb300e7c |
for package in constants.listCoreToolChainPackages: |
| 87815216 |
rpmPkg = pkgUtils.findRPMFileForGivenPackage(package) |
| 0fae9cdd |
if rpmPkg is not None:
continue |
| 7f9d2e12 |
self.logger.info("Building core toolchain package: " + package) |
| 87815216 |
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootName = "build-"+package
destLogPath = constants.logPath + "/build-" + package |
| 0fae9cdd |
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils() |
| 87815216 |
cmdUtils.runCommandInShell("mkdir -p " + destLogPath)
returnVal, chrootID = chrUtils.createChroot(chrootName) |
| a4a06c39 |
if not returnVal:
self.logger.error("Creating chroot failed")
raise Exception("creating chroot failed") |
| 7f9d2e12 |
self.installToolChainRPMS(chrootID, package, destLogPath) |
| 75a2daa5 |
pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath) |
| 085306ae |
pkgUtils.buildRPMSForGivenPackage(package, chrootID, destLogPath) |
| 7418d2bf |
pkgCount += 1 |
| a4a06c39 |
chrUtils.destroyChroot(chrootID) |
| 87815216 |
chrootID = None |
| 518d6a6f |
self.logger.info("Successfully built toolchain") |
| d355ea7e |
if chrootID is not None:
chrUtils.destroyChroot(chrootID) |
| 518d6a6f |
except Exception as e:
self.logger.error("Unable to build tool chain.") |
| d355ea7e |
# print stacktrace
traceback.print_exc() |
| 518d6a6f |
raise e |
| 7418d2bf |
return pkgCount |
| 87815216 |
|
| e45f5730 |
def getListDependentPackageLineContent(self, index):
listBuildRequiresPkgLineContent=SPECS.getData().getBuildRequiresForPackage(self.package, index)
listBuildRequiresPkgLineContent.extend(SPECS.getData().getCheckBuildRequiresForPackage(self.package, index))
listBuildRequiresPkgLineContent=list(set(listBuildRequiresPkgLineContent))
return listBuildRequiresPkgLineContent
def installToolChainRPMS(self, chrootID, packageName, logPath=None,index=0): |
| 7f9d2e12 |
if logPath is None: |
| 87815216 |
logPath = self.logPath |
| b85e6bd0 |
cmdUtils = CommandUtils() |
| 1ffbd987 |
self.prepareBuildRoot(chrootID)
self.logger.info("Installing Tool Chain RPMS.......") |
| 1eb9191f |
rpmFiles = ""
packages = "" |
| e45f5730 |
self.package=packageName
listBuildRequiresPackageLineContent = self.getListDependentPackageLineContent(index) |
| fb300e7c |
for package in constants.listToolChainRPMsToInstall: |
| 87815216 |
pkgUtils = PackageUtils(self.logName, self.logPath) |
| 5fc28b16 |
rpmFile = None |
| e45f5730 |
version = "*"
for depPkg in listBuildRequiresPackageLineContent:
if depPkg.package == package:
version=pkgUtils._getProperVersion(package,depPkg) |
| 6d17349b |
if constants.rpmCheck: |
| e45f5730 |
rpmFile = pkgUtils.findRPMFileForGivenPackage(package, version) |
| 6d17349b |
else:
if (packageName not in constants.listToolChainRPMsToInstall or |
| 87815216 |
constants.listToolChainRPMsToInstall.index(packageName) >
constants.listToolChainRPMsToInstall.index(package)): |
| e45f5730 |
rpmFile = pkgUtils.findRPMFileForGivenPackage(package, version) |
| 1ffbd987 |
if rpmFile is None: |
| 9874e684 |
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf |
| 9b9c0eac |
if (package == "sqlite") and (platform.machine() == "x86_64"): |
| 9874e684 |
package = "sqlite-autoconf" |
| 87815216 |
rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) |
| 1ffbd987 |
if rpmFile is None: |
| fb300e7c |
if package in constants.listOfRPMsProvidedAfterBuild: |
| 87815216 |
self.logger.info("No old version of " + package +
" exists, skip until the new version is built") |
| d77dd20b |
continue |
| 87815216 |
self.logger.error("Unable to find rpm " + package +
" in current and previous versions") |
| 36c2f510 |
raise Exception("Input Error") |
| 1ffbd987 |
rpmFiles += " " + rpmFile
packages += " " + package |
| 1eb9191f |
|
| 87815216 |
self.logger.debug("Installing toolchain rpms:" + packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chrootID +" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles)
retVal = cmdUtils.runCommandInShell(cmd, logPath + "/install_toolchain_rpms.log") |
| 7f9d2e12 |
if not retVal:
self.logger.debug("Command Executed:" + cmd) |
| 1ffbd987 |
self.logger.error("Installing tool chain failed") |
| 36c2f510 |
raise Exception("RPM installation failed") |
| 87815216 |
self.logger.info("Successfully installed default Tool Chain RPMS in Chroot:" + chrootID)
print("Building Package: ".format(packageName))
print(constants.perPackageToolChain) |
| 27a75769 |
if packageName in constants.perPackageToolChain: |
| 87815216 |
print(constants.perPackageToolChain[packageName]) |
| 8a2c2c4e |
self.installCustomToolChainRPMS(chrootID,
constants.perPackageToolChain[packageName].get(platform.machine(), []), |
| 87815216 |
packageName)
|
| 27a75769 |
def installCustomToolChainRPMS(self, chrootID, listOfToolChainPkgs, packageName): |
| 87815216 |
self.logger.info("Installing package specific tool chain RPMs for " + packageName +
".......") |
| 062d56d1 |
rpmFiles = ""
packages = "" |
| 7f9d2e12 |
cmdUtils = CommandUtils() |
| 27a75769 |
for package in listOfToolChainPkgs: |
| 87815216 |
pkgUtils = PackageUtils(self.logName, self.logPath)
print("DEBUG:" + package) |
| 342710b6 |
if re.match("openjre*", packageName) is not None or re.match("openjdk*", packageName): |
| a2cf8f06 |
# x86_64 has openjdk/jre as a published rpms but aarch64 has openjdk8/jre8
# Remove this condition after publishxrpms for x86_^4 got updated |
| 87815216 |
if ((package == "openjdk" or package == "openjre") and
platform.machine() == "aarch64"): |
| a2cf8f06 |
package = package + "8" |
| 87815216 |
rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishXRPMRepo) |
| 27a75769 |
else: |
| 87815216 |
rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) |
| 062d56d1 |
if rpmFile is None: |
| 87815216 |
self.logger.error("Unable to find rpm "+ package +
" in current and previous versions") |
| 062d56d1 |
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package
|
| 87815216 |
self.logger.debug("Installing custom rpms:" + packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles)
retVal = cmdUtils.runCommandInShell(cmd, self.logPath +
"/install_custom_toolchain_rpms.log") |
| 7f9d2e12 |
if not retVal:
self.logger.debug("Command Executed:" + cmd) |
| 062d56d1 |
self.logger.error("Installing tool chain failed")
raise Exception("RPM installation failed")
self.logger.info("Successfully installed all Tool Chain X RPMS") |
| 7418d2bf |
def installToolChainRPMSinContainer(self, containerID):
self.logger.info("Installing tool-chain RPMS in container: " + containerID.short_id)
rpmFiles = ""
packages = ""
pkgUtils = PackageUtils(self.logName, self.logPath)
for package in constants.listToolChainRPMPkgsToInstall:
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf |
| 9b9c0eac |
# if (package == "sqlite") and (platform.machine() == "x86_64"): |
| 0f1fdc4b |
# package = "sqlite-autoconf" |
| 7418d2bf |
rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild: |
| 87815216 |
self.logger.info("No old version of " + package +
" exists, skip until the new version is built") |
| 7418d2bf |
continue |
| 87815216 |
self.logger.error("Unable to find rpm " + package +
" in current and previous versions") |
| 7418d2bf |
raise Exception("Input Error")
if rpmFile.find("stage/PUBLISHRPMS"):
rpmFile = rpmFile.replace(constants.prevPublishRPMRepo, "/publishrpms")
if rpmFile.find("stage/PUBLISHXRPMS"):
rpmFile = rpmFile.replace(constants.prevPublishXRPMRepo, "/publishxrpms")
if rpmFile.find("stage/RPMS"):
rpmFile = rpmFile.replace(constants.rpmPath, constants.topDirPath + "/RPMS")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing tool-chain rpms: " + packages)
cmd = "/usr/bin/bash -l -c '/usr/bin/rpm -Uvh --force --nodeps " + rpmFiles + "'"
self.logger.info("VDBG-TCU-installToolChainRPMSinContainer: Installing rpms cmd: " + cmd)
tcInstallLog = containerID.exec_run(cmd)
# TODO: Find a way to collect exit status of the command that was run.
if not tcInstallLog:
self.logger.error("Installing tool chain in container failed")
raise Exception("RPM installation in container failed")
self.logger.info(tcInstallLog) |
| 87815216 |
self.logger.info("Successfully installed default tool-chain RPMS in container: " +
containerID.short_id) |
| 7418d2bf |
def installCustomToolChainRPMSinContainer(self, containerID, listOfToolChainPkgs, packageName):
self.logger.info("Installing package specific tool chain RPMs for " + packageName)
rpmFiles = ""
packages = ""
for package in listOfToolChainPkgs: |
| 342710b6 |
if re.match("openjre*", packageName) is not None or re.match("openjdk*", packageName): |
| 87815216 |
rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishXRPMRepo) |
| 7418d2bf |
else: |
| 87815216 |
rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) |
| 7418d2bf |
if rpmFile is None: |
| 87815216 |
self.logger.error("Unable to find rpm " + package +
" in current and previous versions") |
| 7418d2bf |
raise Exception("Input Error")
if rpmFile.find("stage/PUBLISHRPMS"):
rpmFile = rpmFile.replace(constants.prevPublishRPMRepo, "/publishrpms")
if rpmFile.find("stage/PUBLISHXRPMS"):
rpmFile = rpmFile.replace(constants.prevPublishXRPMRepo, "/publishxrpms")
if rpmFile.find("stage/RPMS"):
rpmFile = rpmFile.replace(constants.rpmPath, constants.topDirPath + "/RPMS")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing rpms: " + packages)
cmd = "rpm -Uvh --nodeps --force " + rpmFiles |
| 87815216 |
self.logger.debug("VDBG-TCU-installCustomToolChainRPMSinContainer: Installing rpms cmd: " +
cmd) |
| 7418d2bf |
tcInstallLog = containerID.exec_run(cmd)
# TODO: Find a way to collect exit status of the command that was run.
if not tcInstallLog:
self.logger.error("Installing tool chain in container failed")
raise Exception("RPM installation in container failed")
self.logger.info(tcInstallLog) |
| 87815216 |
self.logger.info("Successfully installed all tool-chain XRPMS in container: " +
containerID.short_id) |