@@ -225,10 +225,11 @@ who-needs:

 	@cd $(PHOTON_SPECDEPS_DIR) && \

 		$(PHOTON_SPECDEPS) -s $(PHOTON_SPECS_DIR) -i who-needs -p $(pkg)

-packages: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_XRPMS) $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN) generate-dep-lists

+packages: check-docker-py check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_XRPMS) $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN) generate-dep-lists

 	@echo "Building all RPMS..."

 	@cd $(PHOTON_PKG_BUILDER_DIR) && \

         $(PHOTON_PACKAGE_BUILDER) \

+                -bt $(PHOTON_BUILD_TYPE) \

                 -s $(PHOTON_SPECS_DIR) \

                 -r $(PHOTON_RPMS_DIR) \

                 -a $(PHOTON_SRPMS_DIR) \

@@ -483,6 +484,9 @@ check-tools: check-bison check-g++ check-gawk check-createrepo check-texinfo che

 check-docker:

 	@command -v docker >/dev/null 2>&1 || { echo "Package docker not installed. Aborting." >&2; exit 1; }

+check-docker-py:

+	tdnf install -y docker docker-py --refresh && systemctl daemon-reload && systemctl start docker

+

 check-bison:

 	@command -v bison >/dev/null 2>&1 || { echo "Package bison not installed. Aborting." >&2; exit 1; }

@@ -526,6 +530,7 @@ check: packages

 	@echo "Testing all RPMS ..."

 	@cd $(PHOTON_PKG_BUILDER_DIR) && \

         $(PHOTON_PACKAGE_BUILDER) \

+                -bt $(PHOTON_BUILD_TYPE) \

                 -s $(PHOTON_SPECS_DIR) \

                 -r $(PHOTON_RPMS_DIR) \

                 -a $(PHOTON_SRPMS_DIR) \

@@ -533,7 +538,7 @@ check: packages

                 -b $(PHOTON_CHROOT_PATH) \

                 -l $(PHOTON_LOGS_DIR) \

                 -p $(PHOTON_PUBLISH_RPMS_DIR) \

-		-e $(PHOTON_PUBLISH_XRPMS_DIR) \

+                -e $(PHOTON_PUBLISH_XRPMS_DIR) \

                 -c $(PHOTON_PULLSOURCES_CONFIG) \

                 -d $(PHOTON_DIST_TAG) \

                 -n $(PHOTON_BUILD_NUMBER) \

@@ -549,13 +554,14 @@ check: packages

 	@echo "Building package $(PKG_NAME) ..."

 	@cd $(PHOTON_PKG_BUILDER_DIR) && \

         $(PHOTON_PACKAGE_BUILDER) -i $(PKG_NAME)\

+                              -bt $(PHOTON_BUILD_TYPE) \

                               -b $(PHOTON_CHROOT_PATH) \

                               -s $(PHOTON_SPECS_DIR) \

                               -r $(PHOTON_RPMS_DIR) \

                               -a $(PHOTON_SRPMS_DIR) \

                               -x $(PHOTON_SRCS_DIR) \

                               -p $(PHOTON_PUBLISH_RPMS_DIR) \

-			      -e $(PHOTON_PUBLISH_XRPMS_DIR) \

+                              -e $(PHOTON_PUBLISH_XRPMS_DIR) \

                               -c $(PHOTON_PULLSOURCES_CONFIG) \

                               -d $(PHOTON_DIST_TAG) \

                               -n $(PHOTON_BUILD_NUMBER) \

@@ -1,7 +1,7 @@

 Summary:        Security client

 Name:           nss

 Version:        3.31

-Release:        2%{?dist}

+Release:        3%{?dist}

 License:        MPLv2.0

 URL:            http://ftp.mozilla.org/pub/security/nss/releases/NSS_3_31_RTM/src/%{name}-%{version}.tar.gz

 Group:          Applications/System

@@ -27,6 +27,7 @@ Requires:       nss-libs = %{version}-%{release}

 %package devel

 Summary: Development Libraries for Network Security Services

 Group: Development/Libraries

+Requires: nspr-devel

 Requires: nss = %{version}-%{release}

 %description devel

 Header files for doing development with Network Security Services.

@@ -96,6 +97,8 @@ sudo -u test ./all.sh && userdel test -r -f

 %{_libdir}/libsoftokn3.so

 %changelog

+*   Fri Jul 07 2017 Vinay Kulkarni <kulkarniv@vmware.com> 3.31-3

+-   Fix buildrequires.

 *   Thu Jun 29 2017 Xiaolin Li <xiaolinl@vmware.com> 3.31-2

 -   Fix check.

 *   Tue Jun 20 2017 Xiaolin Li <xiaolinl@vmware.com> 3.31-1

@@ -18,6 +18,7 @@ VAGRANT_BUILD=vagrant

 SRCROOT := $(realpath $(SRCROOT))

 MAKEROOT := $(realpath $(MAKEROOT))

+PHOTON_BUILD_TYPE?=chroot

 PHOTON_STAGE?=$(SRCROOT)/stage

 PHOTON_LOGS_DIR=$(PHOTON_STAGE)/LOGS

 PHOTON_RPMS_DIR=$(PHOTON_STAGE)/RPMS

new file mode 100644

@@ -0,0 +1,222 @@

+from PackageUtils import PackageUtils

+from Logger import Logger

+from ToolChainUtils import ToolChainUtils

+from CommandUtils import CommandUtils

+import os.path

+import sys

+from constants import constants

+import shutil

+import docker

+

+class BuildContainer(object):

+

+    def __init__(self, mapPackageToCycles, listAvailableCyclicPackages, listBuildOptionPackages, pkgBuildOptionFile, logName=None, logPath=None):

+        if logName is None:

+            logName = "BuildContainer"

+        if logPath is None:

+            logPath = constants.logPath

+        self.logName = logName

+        self.logPath = logPath

+        self.logger = Logger.getLogger(logName, logPath, True)

+        self.buildContainerImage = "photon_build_container:latest"

+        self.dockerClient = docker.from_env(version="auto")

+        self.mapPackageToCycles = mapPackageToCycles

+        self.listAvailableCyclicPackages = listAvailableCyclicPackages

+        self.listNodepsPackages = ["glibc","gmp","zlib","file","binutils","mpfr","mpc","gcc","ncurses","util-linux","groff","perl","texinfo","rpm","openssl","openssl-devel","go"]

+        self.listBuildOptionPackages = listBuildOptionPackages

+        self.pkgBuildOptionFile = pkgBuildOptionFile

+

+    def prepareBuildContainer(self, containerTaskName, packageName, isToolChainPackage=False):

+        containerID = None

+        mountVols = {

+                        constants.prevPublishRPMRepo: {'bind': '/publishrpms', 'mode': 'ro'},

+                        constants.prevPublishXRPMRepo: {'bind': '/publishxrpms', 'mode': 'ro'},

+                        constants.rpmPath: {'bind': constants.topDirPath + "/RPMS", 'mode': 'rw'},

+                        constants.sourceRpmPath: {'bind': constants.topDirPath + "/SRPMS", 'mode': 'rw'},

+                        constants.logPath + "/" + self.logName: {'bind': constants.topDirPath + "/LOGS", 'mode': 'rw'},

+                    }

+

+        containerName = containerTaskName

+        containerName = containerName.replace("+", "p")

+        try:

+            oldContainerID = self.dockerClient.containers.get(containerName)

+            if oldContainerID is not None:

+                oldContainerID.remove(force=True)

+        except docker.errors.NotFound:

+            sys.exc_clear()

+

+        try:

+            self.logger.info("BuildContainer-prepareBuildContainer: Starting build container: " + containerName)

+            #TODO: Is init=True equivalent of --sig-proxy?

+            containerID = self.dockerClient.containers.run(self.buildContainerImage,

+                                                           detach=True,

+                                                           name=containerName,

+                                                           network_mode="host",

+                                                           volumes=mountVols,

+                                                           command="/bin/bash -l -c /wait.sh")

+            self.logger.debug("Started Photon build container for task " + containerTaskName

+                               + " ID: " + containerID.short_id)

+            if not containerID:

+                raise Exception("Unable to start Photon build container for task " + containerTaskName)

+        except Exception as e:

+            self.logger.debug("Unable to start Photon build container for task " + containerTaskName)

+            raise e

+        return containerID

+

+    def findPackageNameFromRPMFile(self, rpmfile):

+        rpmfile = os.path.basename(rpmfile)

+        releaseindex = rpmfile.rfind("-")

+        if releaseindex == -1:

+            self.logger.error("Invalid rpm file:" + rpmfile)

+            return None

+        versionindex=rpmfile[0:releaseindex].rfind("-")

+        if versionindex == -1:

+            self.logger.error("Invalid rpm file:" + rpmfile)

+            return None

+        packageName = rpmfile[0:versionindex]

+        return packageName

+

+    def findInstalledPackages(self, containerID):

+        pkgUtils = PackageUtils(self.logName, self.logPath)

+        listInstalledRPMs = pkgUtils.findInstalledRPMPackagesInContainer(containerID)

+        listInstalledPackages = []

+        for installedRPM in listInstalledRPMs:

+            packageName = self.findPackageNameFromRPMFile(installedRPM)

+            if packageName is not None:

+                listInstalledPackages.append(packageName)

+        return listInstalledPackages, listInstalledRPMs

+

+    def buildPackageThreadAPI(self, package, outputMap, threadName,):

+        try:

+            self.buildPackage(package)

+            outputMap[threadName] = True

+        except Exception as e:

+            self.logger.error(e)

+            outputMap[threadName] = False

+

+    def checkIfPackageIsAlreadyBuilt(self, package):

+        basePkg = constants.specData.getSpecName(package)

+        listRPMPackages = constants.specData.getRPMPackages(basePkg)

+        packageIsAlreadyBuilt = True

+        pkgUtils = PackageUtils(self.logName,self.logPath)

+        for pkg in listRPMPackages:

+            if pkgUtils.findRPMFileForGivenPackage(pkg) is None:

+                packageIsAlreadyBuilt = False

+                break

+        return packageIsAlreadyBuilt

+

+    def buildPackage(self, package):

+        #do not build if RPM is already built

+        #test only if the package is in the testForceRPMS with rpmCheck

+        #build only if the package is not in the testForceRPMS with rpmCheck

+        if self.checkIfPackageIsAlreadyBuilt(package):

+            if not constants.rpmCheck:

+                self.logger.info("Skipping building the package:"+package)

+                return

+            elif constants.rpmCheck and package not in constants.testForceRPMS:

+                self.logger.info("Skipping testing the package:"+package)

+                return

+

+        #should initialize a logger based on package name

+        containerTaskName = "build-" + package

+        containerID = None

+        isToolChainPackage = False

+        if package in constants.listToolChainPackages:

+            isToolChainPackage = True

+        destLogPath = constants.logPath + "/build-"+package

+        try:

+            containerID = self.prepareBuildContainer(containerTaskName, package, isToolChainPackage)

+            if not os.path.isdir(destLogPath):

+                cmdUtils = CommandUtils()

+                cmdUtils.runCommandInShell("mkdir -p "+destLogPath)

+

+            tcUtils = ToolChainUtils(self.logName, self.logPath)

+            if package in constants.perPackageToolChain:

+                self.logger.debug(constants.perPackageToolChain[package])

+                tcUtils.installCustomToolChainRPMSinContainer(containerID, constants.perPackageToolChain[package], package);

+

+            listInstalledPackages, listInstalledRPMs = self.findInstalledPackages(containerID)

+            self.logger.info(listInstalledPackages)

+            listDependentPackages = self.findBuildTimeRequiredPackages(package)

+            if constants.rpmCheck and package in constants.testForceRPMS:

+                listDependentPackages.extend(self.findBuildTimeCheckRequiredPackages(package))

+                testPackages=set(constants.listMakeCheckRPMPkgtoInstall)-set(listInstalledPackages)-set([package])

+                listDependentPackages.extend(testPackages)

+                listDependentPackages=list(set(listDependentPackages))

+

+            pkgUtils = PackageUtils(self.logName,self.logPath)

+            if len(listDependentPackages) != 0:

+                self.logger.info("BuildContainer-buildPackage: Installing dependent packages..")

+                self.logger.info(listDependentPackages)

+                for pkg in listDependentPackages:

+                    self.installPackage(pkgUtils, pkg, containerID, destLogPath, listInstalledPackages, listInstalledRPMs)

+                # Special case sqlite due to package renamed from sqlite-autoconf to sqlite

+                if "sqlite" in listInstalledPackages or "sqlite-devel" in listInstalledPackages or "sqlite-libs" in listInstalledPackages:

+                    if "sqlite" not in listInstalledPackages:

+                        self.installPackage(pkgUtils, "sqlite", containerID, destLogPath, listInstalledPackages, listInstalledRPMs)

+                    if "sqlite-devel" not in listInstalledPackages:

+                        self.installPackage(pkgUtils, "sqlite-devel", containerID, destLogPath, listInstalledPackages, listInstalledRPMs)

+                    if "sqlite-libs" not in listInstalledPackages:

+                        self.installPackage(pkgUtils, "sqlite-libs", containerID, destLogPath, listInstalledPackages, listInstalledRPMs)

+                pkgUtils.installRPMSInAOneShotInContainer(containerID, destLogPath)

+

+            pkgUtils.adjustGCCSpecsInContainer(package, containerID, destLogPath)

+

+            pkgUtils.buildRPMSForGivenPackageInContainer(

+                                               package,

+                                               containerID,

+                                               self.listBuildOptionPackages,

+                                               self.pkgBuildOptionFile,

+                                               destLogPath)

+            self.logger.info("BuildContainer-buildPackage: Successfully built the package: " + package)

+        except Exception as e:

+            self.logger.error("Failed while building package:" + package)

+            if containerID is not None:

+                self.logger.debug("Container " + containerID.short_id + " retained for debugging.")

+            logFileName = os.path.join(destLogPath, package + ".log")

+            fileLog = os.popen('tail -n 20 ' + logFileName).read()

+            self.logger.debug(fileLog)

+            raise e

+

+        # Remove the container

+        if containerID is not None:

+            containerID.remove(force=True)

+

+    def findRunTimeRequiredRPMPackages(self, rpmPackage):

+        listRequiredPackages = constants.specData.getRequiresForPackage(rpmPackage)

+        return listRequiredPackages

+

+    def findBuildTimeRequiredPackages(self, package):

+        listRequiredPackages = constants.specData.getBuildRequiresForPackage(package)

+        return listRequiredPackages

+

+    def findBuildTimeCheckRequiredPackages(self,package):

+        listRequiredPackages=constants.specData.getCheckBuildRequiresForPackage(package)

+        return listRequiredPackages

+

+    def installPackage(self, pkgUtils, package, containerID, destLogPath, listInstalledPackages, listInstalledRPMs):

+        latestRPM = os.path.basename(pkgUtils.findRPMFileForGivenPackage(package)).replace(".rpm", "")

+        if package in listInstalledPackages and latestRPM in listInstalledRPMs:

+            return

+        self.installDependentRunTimePackages(pkgUtils, package, containerID, destLogPath, listInstalledPackages, listInstalledRPMs)

+        noDeps = False

+        if self.mapPackageToCycles.has_key(package):

+            noDeps = True

+        if package in self.listNodepsPackages:

+            noDeps = True

+        if package in constants.noDepsPackageList:

+            noDeps = True

+        pkgUtils.prepRPMforInstallInContainer(package, containerID, noDeps, destLogPath)

+        listInstalledPackages.append(package)

+        listInstalledRPMs.append(latestRPM)

+

+    def installDependentRunTimePackages(self, pkgUtils, package, containerID, destLogPath, listInstalledPackages, listInstalledRPMs):

+        listRunTimeDependentPackages = self.findRunTimeRequiredRPMPackages(package)

+        if len(listRunTimeDependentPackages) != 0:

+            for pkg in listRunTimeDependentPackages:

+                if self.mapPackageToCycles.has_key(pkg) and pkg not in self.listAvailableCyclicPackages:

+                    continue

+                latestPkgRPM = os.path.basename(pkgUtils.findRPMFileForGivenPackage(pkg)).replace(".rpm", "")

+                if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs:

+                    continue

+                self.installPackage(pkgUtils, pkg, containerID, destLogPath, listInstalledPackages, listInstalledRPMs)

new file mode 100644

@@ -0,0 +1,10 @@

+# Photon Base Build Container

+

+FROM scratch

+MAINTAINER kulkarniv@vmware.com

+

+ADD tcroot.tar.gz /

+ADD wait.sh /

+ADD adjust-gcc-specs.sh /tmp/

+

+CMD ["/usr/bin/bash", "--login"]

@@ -3,12 +3,14 @@ import os

 class Logger(object):

     @staticmethod

-    def getLogger (mymodule, logpath=None):

+    def getLogger (mymodule, logpath=None, resetFile=False):

         logfile=mymodule+".log"

         if logpath is not None:

             if not os.path.isdir(logpath):

                 os.makedirs(logpath)

             logfile=logpath+"/"+logfile

+        if resetFile:

+            open(logfile, 'w').close()

         logger=logging.getLogger(mymodule)

         if len(logger.handlers) == 0:

             #creating file handler

@@ -2,7 +2,10 @@ from PackageBuildDataGenerator import PackageBuildDataGenerator

 from Logger import Logger

 import threading

 from constants import constants

+import docker

 import os

+from ChrootUtils import ChrootUtils

+from CommandUtils import CommandUtils

 from PackageUtils import PackageUtils

 from ToolChainUtils import ToolChainUtils

 from Scheduler import Scheduler

@@ -18,6 +21,7 @@ class PackageManager(object):

         self.logName=logName

         self.logPath=logPath

         self.logger=Logger.getLogger(logName,logPath)

+        self.dockerClient = docker.from_env(version="auto")

         self.mapCyclesToPackageList={}

         self.mapPackageToCycle={}

         self.sortedPackageList=[]

@@ -28,6 +32,7 @@ class PackageManager(object):

         self.listAvailableCyclicPackages=[]

         self.listBuildOptionPackages=[]

         self.pkgBuildOptionFile=""

+        self.pkgBuildType="chroot"

     def readPackageBuildData(self, listPackages):

         try:

@@ -116,25 +121,37 @@ class PackageManager(object):

         return True

     def buildToolChain(self):

+        pkgCount = 0

         try:

             tUtils=ToolChainUtils()

-            tUtils.buildCoreToolChainPackages(self.listBuildOptionPackages, self.pkgBuildOptionFile)

+            pkgCount = tUtils.buildCoreToolChainPackages(self.listBuildOptionPackages, self.pkgBuildOptionFile)

         except Exception as e:

             self.logger.error("Unable to build tool chain")

             self.logger.error(e)

             raise e

+        return pkgCount

     def buildToolChainPackages(self, listBuildOptionPackages, pkgBuildOptionFile, buildThreads):

-        self.buildToolChain()

+        pkgCount = self.buildToolChain()

+        if self.pkgBuildType == "container":

+            # Stage 1 build container

+            #TODO image name constants.buildContainerImageName

+            if pkgCount > 0 or not self.dockerClient.images.list("photon_build_container:latest"):

+                self.createBuildContainer()

         self.buildGivenPackages(constants.listToolChainPackages, buildThreads)

+        if self.pkgBuildType == "container":

+            # Stage 2 build container

+            #TODO: rebuild container only if anything in listToolChainPackages was built

+            self.createBuildContainer()

     def buildTestPackages(self, listBuildOptionPackages, pkgBuildOptionFile, buildThreads):

         self.buildToolChain()

         self.buildGivenPackages(constants.listMakeCheckRPMPkgtoInstall, buildThreads)

-    def buildPackages(self,listPackages, listBuildOptionPackages, pkgBuildOptionFile, buildThreads):

+    def buildPackages(self,listPackages, listBuildOptionPackages, pkgBuildOptionFile, buildThreads, pkgBuildType):

         self.listBuildOptionPackages = listBuildOptionPackages

         self.pkgBuildOptionFile = pkgBuildOptionFile

+        self.pkgBuildType = pkgBuildType

         if constants.rpmCheck:

             constants.rpmCheck=False

             self.buildToolChainPackages(listBuildOptionPackages, pkgBuildOptionFile, buildThreads)

@@ -153,6 +170,7 @@ class PackageManager(object):

         ThreadPool.pkgBuildOptionFile=self.pkgBuildOptionFile

         ThreadPool.logger=self.logger

         ThreadPool.statusEvent=statusEvent

+        ThreadPool.pkgBuildType=self.pkgBuildType

     def initializeScheduler(self,statusEvent):

         Scheduler.setLog(self.logName, self.logPath)

@@ -210,3 +228,50 @@ class PackageManager(object):

         self.logger.info("Terminated")

+    def createBuildContainer(self):

+        self.logger.info("Generating photon build container..")

+        try:

+            #TODO image name constants.buildContainerImageName

+            self.dockerClient.images.remove("photon_build_container:latest", force=True)

+        except Exception as e:

+            #TODO - better handling

+            self.logger.debug("Photon build container image not found.")

+

+        # Create toolchain chroot and install toolchain RPMs

+        chrootID=None

+        try:

+            #TODO: constants.tcrootname

+            chrUtils = ChrootUtils("toolchain-chroot", self.logPath)

+            returnVal, chrootID = chrUtils.createChroot("toolchain-chroot")

+            self.logger.debug("Created tool-chain chroot: " + chrootID)

+            if not returnVal:

+                raise Exception("Unable to prepare tool-chain chroot")

+            tcUtils = ToolChainUtils("toolchain-chroot", self.logPath)

+            tcUtils.installToolChainRPMS(chrootID, "dummy")

+        except Exception as e:

+            if chrootID is not None:

+                self.logger.debug("Deleting chroot: " + chrootID)

+                chrUtils.destroyChroot(chrootID)

+            raise e

+        self.logger.info("VDBG-PU-createBuildContainer: chrootID: " + chrootID)

+

+        # Create photon build container using toolchain chroot

+        #TODO: Coalesce logging

+        cmdUtils = CommandUtils()

+        cmd = "./umount-build-root.sh " + chrootID

+        cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot1.log")

+        cmd = "cd " + chrootID + " && tar -czvf ../tcroot.tar.gz ."

+        cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot2.log")

+        cmd = "mv " + chrootID + "/../tcroot.tar.gz ."

+        cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot3.log")

+        #TODO: Container name, docker file name from constants.

+        self.dockerClient.images.build(tag="photon_build_container:latest",

+                                       path=".",

+                                       rm=True,

+                                       dockerfile="Dockerfile.photon_build_container")

+

+        # Cleanup

+        cmd = "rm -f ./tcroot.tar.gz"

+        cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot4.log")

+        chrUtils.destroyChroot(chrootID)

+        self.logger.info("Photon build container successfully created.")

@@ -28,14 +28,16 @@ class PackageUtils(object):

         self.rpmbuildBuildallOption = "-ba --clean"

         self.rpmbuildNocheckOption = "--nocheck"

         self.rpmbuildCheckOption ="-bi --clean"

-        self.rpmbuildDistOption = '--define \\\"dist %s\\\"' % constants.dist

         self.queryRpmPackageOptions = "-qa"

         self.forceRpmPackageOptions = "--force"

+        self.replaceRpmPackageOptions = "--replacepkgs"

         self.adjustGCCSpecScript="adjust-gcc-specs.sh"

         self.rpmFilesToInstallInAOneShot=""

         self.packagesToInstallInAOneShot=""

         self.noDepsRPMFilesToInstallInAOneShot=""

         self.noDepsPackagesToInstallInAOneShot=""

+        self.rpmFilesToReInstallInAOneShot=""

+        self.noDepsRPMFilesToReInstallInAOneShot=""

     def getRPMArch(self,rpmName):

         arch=""

@@ -105,32 +107,35 @@ class PackageUtils(object):

                 self.logger.error("Unable to install rpms")

                 raise Exception("RPM installation failed")

-

-    def copySourcesTobuildroot(self,listSourceFiles,package,destDir):

+    def verifyShaAndGetSourcePath(self, source, package):

         cmdUtils = CommandUtils()

-        for source in listSourceFiles:

-            # Fetch/verify sources if sha1 not None.

-            sha1 = constants.specData.getSHA1(package, source)

-            if sha1 is not None:

-                PullSources.get(source, sha1, constants.sourcePath, constants.pullsourcesConfig, self.logger)

-

-            sourcePath = cmdUtils.findFile(source,constants.sourcePath)

+        # Fetch/verify sources if sha1 not None.

+        sha1 = constants.specData.getSHA1(package, source)

+        if sha1 is not None:

+            PullSources.get(source, sha1, constants.sourcePath, constants.pullsourcesConfig, self.logger)

+

+        sourcePath = cmdUtils.findFile(source,constants.sourcePath)

+        if sourcePath is None or len(sourcePath) == 0:

+            sourcePath = cmdUtils.findFile(source,constants.specPath)

             if sourcePath is None or len(sourcePath) == 0:

-                sourcePath = cmdUtils.findFile(source,constants.specPath)

-                if sourcePath is None or len(sourcePath) == 0:

-                    if sha1 is None:

-                        self.logger.error("No sha1 found or missing source for "+source)

-                        raise Exception("No sha1 found or missing source for "+source)

-                    else:

-                        self.logger.error("Missing source: "+source+". Cannot find sources for package: "+package)

-                        raise Exception("Missing source")

-            else:

                 if sha1 is None:

-                    self.logger.error("No sha1 found for "+source)

-                    raise Exception("No sha1 found")

-            if len(sourcePath) > 1:

-                self.logger.error("Multiple sources found for source:"+source+"\n"+ ",".join(sourcePath) +"\nUnable to determine one.")

-                raise Exception("Multiple sources found")

+                    self.logger.error("No sha1 found or missing source for "+source)

+                    raise Exception("No sha1 found or missing source for "+source)

+                else:

+                    self.logger.error("Missing source: "+source+". Cannot find sources for package: "+package)

+                    raise Exception("Missing source")

+        else:

+            if sha1 is None:

+                self.logger.error("No sha1 found for "+source)

+                raise Exception("No sha1 found")

+        if len(sourcePath) > 1:

+            self.logger.error("Multiple sources found for source:"+source+"\n"+ ",".join(sourcePath) +"\nUnable to determine one.")

+            raise Exception("Multiple sources found")

+        return sourcePath

+

+    def copySourcesTobuildroot(self,listSourceFiles,package,destDir):

+        for source in listSourceFiles:

+            sourcePath = self.verifyShaAndGetSourcePath(source, package)

             self.logger.info("Copying... Source path :" + source + " Source filename: " + sourcePath[0])

             shutil.copy2(sourcePath[0], destDir)

@@ -145,6 +150,24 @@ class PackageUtils(object):

                 else:

                     shutil.copytree(source, destDir)

+    def getAdditionalBuildFiles(self, package, pkgBuildOptionFile):

+        listAdditionalFiles = []

+        macros = []

+        jsonData = open(pkgBuildOptionFile)

+        pkg_build_option_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict)

+        jsonData.close()

+        pkgs_sorted = pkg_build_option_json.items()

+        for pkg in pkgs_sorted:

+            p = str(pkg[0].encode('utf-8'))

+            if p == package:

+                filelist = pkg[1]["files"]

+                for f in filelist:

+                    listAdditionalFiles.append(f)

+                macrolist = pkg[1]["macros"]

+                for macro in macrolist:

+                    macros.append(str(macro.encode('utf-8')))

+        return listAdditionalFiles, macros

+

     def buildRPMSForGivenPackage(self,package,chrootID,listBuildOptionPackages,pkgBuildOptionFile,destLogPath=None):

         self.logger.info("Building rpm's for package:"+package)

@@ -164,23 +187,9 @@ class PackageUtils(object):

         self.copySourcesTobuildroot(listSourcesFiles,package,chrootSourcePath)

         self.copySourcesTobuildroot(listPatchFiles,package,chrootSourcePath)

-        listAdditionalFiles = []

         macros = []

         if package in listBuildOptionPackages:

-            jsonData = open(pkgBuildOptionFile)

-            pkg_build_option_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict)

-            jsonData.close()

-            pkgs_sorted = pkg_build_option_json.items()

-            for pkg in pkgs_sorted:

-                p = str(pkg[0].encode('utf-8'))

-                if p == package:

-                    filelist = pkg[1]["files"]

-                    for f in filelist:

-                        listAdditionalFiles.append(f)

-                    macrolist = pkg[1]["macros"]

-                    for macro in macrolist:

-                        macros.append(str(macro.encode('utf-8')))

-

+            listAdditionalFiles, macros = self.getAdditionalBuildFiles(package, pkgBuildOptionFile)

             self.copyAdditionalBuildFiles(listAdditionalFiles,chrootID)

         #Adding rpm macros

@@ -216,7 +225,8 @@ class PackageUtils(object):

     def buildRPM(self,specFile,logFile,chrootCmd,package,macros):

-        rpmBuildcmd=self.rpmbuildBinary+" "+self.rpmbuildBuildallOption+" "+self.rpmbuildDistOption

+        rpmbuildDistOption = '--define \\\"dist %s\\\"' % constants.dist

+        rpmBuildcmd=self.rpmbuildBinary+" "+self.rpmbuildBuildallOption+" "+rpmbuildDistOption

         if constants.rpmCheck and package in constants.testForceRPMS:

             self.logger.info("#"*(68+2*len(package)))

@@ -347,3 +357,297 @@ class PackageUtils(object):

         self.logger.error("Failed while adjusting gcc specs")

         raise Exception("Failed while adjusting gcc specs")

+

+    def copySourcesToContainer(self, listSourceFiles, package, containerID, destDir):

+        cmdUtils = CommandUtils()

+        for source in listSourceFiles:

+            sourcePath = self.verifyShaAndGetSourcePath(source, package)

+            self.logger.info("Copying source file: " + sourcePath[0])

+            copyCmd = "docker cp " + sourcePath[0] + " " + containerID.short_id + ":" + destDir

+            cmdUtils.runCommandInShell(copyCmd)

+

+    def copyAdditionalBuildFilesToContainer(self, listAdditionalFiles, containerID):

+        cmdUtils = CommandUtils()

+        #self.logger.debug("VDBG-PU-copyAdditionalBuildFilesToContainer id: " +containerID.short_id)

+        #self.logger.debug(listAdditionalFiles)

+        for additionalFile in listAdditionalFiles:

+            source = additionalFile["src"].encode('utf-8')

+            destDir = additionalFile["dst"].encode('utf-8')

+            destPath = containerID.short_id + ":" + destDir

+            #TODO: exit status of exec_run

+            containerID.exec_run("mkdir -p " + destDir)

+            if os.path.exists(source):

+                copyCmd = "docker cp " + source

+                if os.path.isfile(source):

+                    self.logger.info("Copying addl source file: " + source)

+                    copyCmd += " " + destPath

+                else:

+                    self.logger.info("Copying addl source file tree: " + source)

+                    copyCmd +=  "/. " + destPath

+                #TODO: cmd error code

+                cmdUtils.runCommandInShell(copyCmd)

+

+    def getRPMPathInContainer(self, rpmFile, containerID):

+        rpmName = os.path.basename(rpmFile)

+        #TODO: Container path from constants

+        if "PUBLISHRPMS" in rpmFile:

+            rpmPath = "/publishrpms/"

+        elif "PUBLISHXRPMS" in rpmFile:

+            rpmPath = "/publishxrpms/"

+        else:

+            rpmPath = constants.topDirPath + "/RPMS/"

+        if "noarch" in rpmFile:

+            rpmPath += "noarch/"

+        else:

+            rpmPath += "x86_64/"

+        rpmPath += rpmName

+        return rpmPath

+

+    def prepRPMforInstallInContainer(self, package, containerID, noDeps=False, destLogPath=None):

+        rpmfile = self.findRPMFileForGivenPackage(package)

+        if rpmfile is None:

+            self.logger.error("No rpm file found for package: " + package)

+            raise Exception("Missing rpm file")

+

+        rpmDestFile = self.getRPMPathInContainer(rpmfile, containerID)

+        if noDeps:

+            self.noDepsRPMFilesToInstallInAOneShot += " " + rpmDestFile

+            self.noDepsPackagesToInstallInAOneShot += " " + package

+            if package in constants.listReInstallPackages:

+                self.noDepsRPMFilesToReInstallInAOneShot += " " + rpmDestFile

+        else:

+            self.rpmFilesToInstallInAOneShot += " " + rpmDestFile

+            self.packagesToInstallInAOneShot += " " + package

+            if package in constants.listReInstallPackages:

+                self.rmpFilesToReInstallInAOneShot += " " + rpmDestFile

+

+    def installRPMSInAOneShotInContainer(self, containerID, destLogPath):

+        rpmInstallcmd = self.rpmBinary + " " + self.installRPMPackageOptions + " " + self.forceRpmPackageOptions

+

+        if self.noDepsRPMFilesToInstallInAOneShot != "":

+            self.logger.info("PackageUtils-installRPMSInAOneShotInContainer: Installing nodeps rpms: " + \

+                             self.noDepsPackagesToInstallInAOneShot)

+            logFile = destLogPath + "/install_rpms_nodeps.log"

+            cmd = rpmInstallcmd + " " + self.nodepsRPMPackageOptions + " " + self.noDepsRPMFilesToInstallInAOneShot

+            cmd = "/bin/bash -l -c '" + cmd + "'"

+            #self.logger.debug("VDBG-PU-installRPMSInAOneShotInContainer: Install nodeps cmd: " + cmd)

+            #TODO: Error code from exec_run

+            installLog = containerID.exec_run(cmd)

+            if not installLog:

+                self.logger.error("Unable to install nodeps rpms")

+                raise Exception("nodeps RPM installation failed")

+            logfile = open(logFile, 'w')

+            logfile.write(installLog)

+            logfile.close()

+

+            if self.noDepsRPMFilesToReInstallInAOneShot != "":

+                cmd = rpmInstallcmd + " " + self.nodepsRPMPackageOptions + " " + self.forceRpmPackageOptions + " " + self.noDepsRPMFilesToReInstallInAOneShot

+                cmd = "/bin/bash -l -c '" + cmd + "'"

+                #self.logger.debug("VDBG-PU-installRPMSInAOneShotInContainer: ReInstall nodeps cmd: " + cmd)

+                #TODO: Error code from exec_run

+                installLog = containerID.exec_run(cmd)

+                if not installLog:

+                    self.logger.error("Unable to re-install nodeps rpms")

+                    raise Exception("nodeps RPM re-installation failed")

+                logfile = open(logFile, 'a')

+                logfile.write(installLog)

+                logfile.close()

+

+        if self.rpmFilesToInstallInAOneShot != "":

+            self.logger.info("PackageUtils-installRPMSInAOneShotInContainer: Installing rpms: " + \

+                             self.packagesToInstallInAOneShot)

+            logFile = destLogPath + "/install_rpms.log"

+            cmd = rpmInstallcmd + " " + self.rpmFilesToInstallInAOneShot

+            cmd = "/bin/bash -l -c '" + cmd + "'"

+            #self.logger.debug("VDBG-PU-installRPMSInAOneShotInContainer: Install cmd: " + cmd)

+            #TODO: Error code from exec_run

+            installLog = containerID.exec_run(cmd)

+            if not installLog:

+                self.logger.error("Unable to install rpms")

+                raise Exception("RPM installation failed")

+            logfile = open(logFile, 'w')

+            logfile.write(installLog)

+            logfile.close()

+

+            if self.rpmFilesToReInstallInAOneShot != "":

+                cmd = rpmInstallcmd + " " + self.forceRpmPackageOptions + " " + self.rpmFilesToReInstallInAOneShot

+                cmd = "/bin/bash -l -c '" + cmd + "'"

+                #self.logger.debug("VDBG-PU-installRPMSInAOneShotInContainer: ReInstall cmd: " + cmd)

+                #TODO: Error code from exec_run

+                installLog = containerID.exec_run(cmd)

+                if not installLog:

+                    self.logger.error("Unable to re-install rpms")

+                    raise Exception("RPM re-installation failed")

+                logfile = open(logFile, 'a')

+                logfile.write(installLog)

+                logfile.close()

+

+    def findInstalledRPMPackagesInContainer(self, containerID):

+        cmd = self.rpmBinary + " " + self.queryRpmPackageOptions

+        cmd = "/bin/bash -l -c '" + cmd + "'"

+        #TODO: Error code from exec_run

+        result = containerID.exec_run(cmd)

+        if result is not None:

+            return result.split()

+        return result

+

+    def adjustGCCSpecsInContainer(self, package, containerID, logPath):

+        opt = " " + constants.specData.getSecurityHardeningOption(package)

+        adjustCmd = "/tmp/" + self.adjustGCCSpecScript + opt

+        adjustCmd = "/bin/bash -l -c '" + adjustCmd + "'"

+        logFile = logPath + "/adjustGCCSpecScript.log"

+

+        #TODO: Error code from exec_run

+        scriptLog = containerID.exec_run(adjustCmd)

+        if scriptLog:

+            logfile = open(logFile, 'w')

+            logfile.write(scriptLog)

+            logfile.close()

+            return

+

+        self.logger.debug(containerID.exec_run("ls -la /tmp/" + self.adjustGCCSpecScript))

+        self.logger.debug(containerID.exec_run("lsof /tmp/" + self.adjustGCCSpecScript))

+        self.logger.debug(containerID.exec_run("ps ax"))

+        self.logger.error("Failed while adjusting gcc specs")

+        raise Exception("Failed while adjusting gcc specs")

+

+    def buildRPMSForGivenPackageInContainer(self, package, containerID, listBuildOptionPackages,

+                                            pkgBuildOptionFile, destLogPath=None):

+        self.logger.info("Building rpm's for package " + package + " in container " + containerID.short_id)

+

+        listSourcesFiles = constants.specData.getSources(package)

+        listPatchFiles = constants.specData.getPatches(package)

+        specFile = constants.specData.getSpecFile(package)

+        specName = constants.specData.getSpecName(package) + ".spec"

+        sourcePath = constants.topDirPath + "/SOURCES/"

+        specPath = constants.topDirPath + "/SPECS/"

+        rpmLogFile = constants.topDirPath + "/LOGS/" + package + ".log"

+        destLogFile = destLogPath + "/" + package + ".log"

+        cmdUtils = CommandUtils()

+

+        #TODO: mount it in, don't copy

+        cpSpecCmd = "docker cp " + specFile + " " + containerID.short_id \

+                        + ":" + specPath + specName

+        returnVal = cmdUtils.runCommandInShell(cpSpecCmd)

+        if not returnVal:

+            self.logger.error("Error copying source SPEC file to container")

+            raise Exception("Failed copying source SPEC to container")

+

+# FIXME: some sources are located in SPECS/.. how to mount?

+#        if os.geteuid()==0:

+        #TODO: mount it in, don't copy

+        macros = []

+        self.copySourcesToContainer(listSourcesFiles, package, containerID, sourcePath)

+        #TODO: mount it in, don't copy

+        self.copySourcesToContainer(listPatchFiles, package, containerID, sourcePath)

+        if package in listBuildOptionPackages:

+            listAdditionalFiles, macros = self.getAdditionalBuildFiles(package, pkgBuildOptionFile)

+            self.copyAdditionalBuildFilesToContainer(listAdditionalFiles, containerID)

+

+        # Add rpm macros

+        listRPMMacros = constants.specData.getRPMMacros()

+        for macroName in listRPMMacros.keys():

+            macros.append(macroName + " " + listRPMMacros[macroName])

+

+        # Build RPMs

+        listRPMFiles=[]

+        listSRPMFiles=[]

+        try:

+            listRPMFiles, listSRPMFiles = self.buildRPMinContainer(

+                                                    specPath + specName,

+                                                    rpmLogFile,

+                                                    destLogFile,

+                                                    containerID,

+                                                    package,

+                                                    macros)

+        except Exception as e:

+            self.logger.error("Failed while building rpm: " + package)

+            raise e

+        self.logger.info("RPM build is successful")

+

+        # Verify RPM and SRPM files exist as success criteria

+        for rpmFile in listRPMFiles:

+            rpmName = os.path.basename(rpmFile)

+            rpmDestDir = self.getRPMDestDir(rpmName, constants.rpmPath)

+            rpmDestFile = rpmDestDir + "/" + rpmName

+            if not os.path.isfile(rpmDestFile):

+                self.logger.error("Could not find RPM file: " + rpmDestFile)

+                raise Exception("Built RPM file not found.")

+

+        for srpmFile in listSRPMFiles:

+            srpmName = os.path.basename(srpmFile)

+            srpmDestDir = self.getRPMDestDir(os.path.basename(srpmFile), constants.sourceRpmPath)

+            srpmDestFile = srpmDestDir + "/" + srpmName

+            if not os.path.isfile(srpmDestFile):

+                self.logger.error("Could not find RPM file: " + srpmDestFile)

+                raise Exception("Built SRPM file not found.")

+

+    def buildRPMinContainer(self, specFile, rpmLogFile, destLogFile, containerID, package, macros):

+

+        rpmbuildDistOption = '--define \"dist %s\"' % constants.dist

+        rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption \

+                          + " " + rpmbuildDistOption

+

+        if constants.rpmCheck and package in constants.testForceRPMS:

+            self.logger.info("#"*(68+2*len(package)))

+            if not constants.specData.isCheckAvailable(package):

+                self.logger.info("####### "+package+" MakeCheck is not available. Skipping MakeCheck TEST for "+package+ " #######")

+                rpmBuildCmd=self.rpmbuildBinary+" --clean"

+            else:

+                self.logger.info("####### "+package+" MakeCheck is available. Running MakeCheck TEST for "+package+ " #######")

+                rpmBuildCmd=self.rpmbuildBinary+" "+self.rpmbuildCheckOption

+            self.logger.info("#"*(68+2*len(package)))

+        else:

+           rpmBuildCmd+=" "+self.rpmbuildNocheckOption

+

+        for macro in macros:

+            rpmBuildCmd += ' --define \"%s\"' % macro

+        rpmBuildCmd += " " + specFile

+        rpmBuildCmd = "/bin/bash -l -c '" + rpmBuildCmd + " > " + rpmLogFile + " 2>&1'"

+        rpmBuildCmd = "docker exec " + str(containerID.short_id) + " " + rpmBuildCmd

+

+        cmdUtils = CommandUtils()

+        self.logger.info("Building rpm for package: " + package)

+        #TODO: Show running log of rpmbuildcmd

+        #TODO: Get exit status of rpmBuildCmd

+        #containerID.exec_run(rpmBuildCmd)

+        returnVal = cmdUtils.runCommandInShell(rpmBuildCmd)

+

+        if not os.path.isfile(destLogFile):

+            self.logger.error("RPM build not file not found. Building rpm failed for: " + specFile)

+            raise Exception("RPM Build failed")

+

+        if constants.rpmCheck and package in constants.testForceRPMS:

+            if not constants.specData.isCheckAvailable(package):

+                constants.testLogger.info(package+" : N/A")

+            elif returnVal:

+                constants.testLogger.info(package+" : PASS")

+            else: