@@ -17,24 +17,24 @@ class PackageBuilderBase(object):

         self.logger = None

         self.package = None

         self.version = None

+        self.doneList = None

         self.mapPackageToCycles = mapPackageToCycles

         self.listNodepsPackages = ["glibc", "gmp", "zlib", "file", "binutils", "mpfr",

                                    "mpc", "gcc", "ncurses", "util-linux", "groff", "perl",

                                    "texinfo", "rpm", "openssl", "go"]

         self.pkgBuildType = pkgBuildType

-    def buildPackageFunction(self, pkg):

+    def buildPackageFunction(self, pkg, doneList):

         packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)

         #do not build if RPM is already built

         #test only if the package is in the testForceRPMS with rpmCheck

         #build only if the package is not in the testForceRPMS with rpmCheck

-        if self._checkIfPackageIsAlreadyBuilt(packageName, packageVersion):

-            if not constants.rpmCheck:

-                return

-            elif constants.rpmCheck and self.package not in constants.testForceRPMS:

+

+        if not constants.rpmCheck or packageName in constants.testForceRPMS:

+            if self._checkIfPackageIsAlreadyBuilt(packageName, packageVersion, doneList):

                 return

-        self._buildPackagePrepareFunction(packageName, packageVersion)

+        self._buildPackagePrepareFunction(packageName, packageVersion, doneList)

         try:

             self._buildPackage()

         except Exception as e:

@@ -42,7 +42,7 @@ class PackageBuilderBase(object):

             self.logger.exception(e)

             raise e

-    def _buildPackagePrepareFunction(self, package, version):

+    def _buildPackagePrepareFunction(self, package, version, doneList):

         self.package = package

         self.version = version

         self.logName = "build-" + package + "-" + version

@@ -51,6 +51,7 @@ class PackageBuilderBase(object):

             cmdUtils = CommandUtils()

             cmdUtils.runCommandInShell("mkdir -p " + self.logPath)

         self.logger = Logger.getLogger(self.logName, self.logPath, constants.logLevel)

+        self.doneList = doneList

     def _findPackageNameAndVersionFromRPMFile(self, rpmfile):

         rpmfile = os.path.basename(rpmfile)

@@ -71,16 +72,10 @@ class PackageBuilderBase(object):

                 listInstalledPackages.append(pkg)

         return listInstalledPackages, listInstalledRPMs

-    def _checkIfPackageIsAlreadyBuilt(self, package, version):

-        basePkg = SPECS.getData().getSpecName(package)

-        listRPMPackages = SPECS.getData().getRPMPackages(basePkg, version)

-        packageIsAlreadyBuilt = True

-        pkgUtils = PackageUtils()

-        for pkg in listRPMPackages:

-            if pkgUtils.findRPMFileForGivenPackage(pkg, version) is None:

-                packageIsAlreadyBuilt = False

-                break

-        return packageIsAlreadyBuilt

+    def _checkIfPackageIsAlreadyBuilt(self, package, version, doneList):

+        basePkg = SPECS.getData().getSpecName(package) + "-" + version

+        return basePkg in doneList

+

     def _findRunTimeRequiredRPMPackages(self, rpmPackage, version):

         return SPECS.getData().getRequiresForPackage(rpmPackage, version)

@@ -93,7 +88,7 @@ class PackageBuilderBase(object):

     def _installPackage(self, pkgUtils, package, packageVersion, sandbox, destLogPath,

                         listInstalledPackages, listInstalledRPMs):

-        rpmfile = pkgUtils.findRPMFileForGivenPackage(package,packageVersion);

+        rpmfile = pkgUtils.findRPMFile(package,packageVersion);

         if rpmfile is None:

             self.logger.error("No rpm file found for package: " + package + "-" + packageVersion)

             raise Exception("Missing rpm file")

@@ -122,7 +117,7 @@ class PackageBuilderBase(object):

                     continue

                 packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)

                 latestPkgRPM = os.path.basename(

-                    pkgUtils.findRPMFileForGivenPackage(packageName, packageVersion)).replace(".rpm", "")

+                    pkgUtils.findRPMFile(packageName, packageVersion)).replace(".rpm", "")

                 if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs:

                     continue

                 self._installPackage(pkgUtils, packageName,packageVersion, sandbox, destLogPath,listInstalledPackages, listInstalledRPMs)

@@ -217,7 +212,10 @@ class PackageBuilderChroot(PackageBuilderBase):

             chroot.create(self.package + "-" + self.version)

             tUtils = ToolChainUtils(self.logName, self.logPath)

-            tUtils.installToolChainRPMS(chroot, self.package, self.version, self.logPath)

+            tUtils.installToolChainRPMS(chroot, self.package,

+                                        self.version, self.logPath,

+                                        usePublishedRPMS=True,

+                                        availablePackages=self.doneList)

             listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = (

                 self._findDependentPackagesAndInstalledRPM(chroot))

@@ -38,7 +38,7 @@ class PackageInfo(object):

                     debugrpmFile = debugrpmFiles[0]

                 pkgUtils = PackageUtils(self.logName, self.logPath)

                 for rpmPkg in listRPMPackages:

-                    rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg, version)

+                    rpmFile = pkgUtils.findRPMFile(rpmPkg, version)

                     if rpmFile is not None:

                         listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile,

                                              "debugrpm":debugrpmFile}

@@ -50,7 +50,7 @@ class PackageManager(object):

             # Stage 1 build container

             #TODO image name constants.buildContainerImageName

             if pkgCount > 0 or not self.dockerClient.images.list(constants.buildContainerImage):

-                self._createBuildContainer()

+                self._createBuildContainer(True)

         self.logger.info("Step 2 : Building stage 2 of the toolchain...")

         self.logger.info(constants.listToolChainPackages)

         self.logger.info("")

@@ -61,7 +61,7 @@ class PackageManager(object):

         if self.pkgBuildType == "container":

             # Stage 2 build container

             #TODO: rebuild container only if anything in listToolChainPackages was built

-            self._createBuildContainer()

+            self._createBuildContainer(False)

     def buildPackages(self, listPackages, buildThreads):

         if constants.rpmCheck:

@@ -91,7 +91,7 @@ class PackageManager(object):

             return False

         return True

-    # Returns list of package names which spec file has all subpackages built

+    # Returns list of base package names which spec file has all subpackages built

     # Returns set of package name and version like

     # ["name1-vers1", "name2-vers2",..]

     def _readAlreadyAvailablePackages(self):

@@ -104,14 +104,12 @@ class PackageManager(object):

                 packageIsAlreadyBuilt=True

                 listRPMPackages = SPECS.getData().getRPMPackages(package, version)

                 for rpmPkg in listRPMPackages:

-                    if pkgUtils.findRPMFileForGivenPackage(rpmPkg, version) is None:

+                    if pkgUtils.findRPMFile(rpmPkg, version) is None:

                         packageIsAlreadyBuilt=False

                         break;

                 if packageIsAlreadyBuilt:

                     listAvailablePackages.add(package+"-"+version)

-        self.logger.debug("List of Already built packages")

-        self.logger.debug(listAvailablePackages)

         return listAvailablePackages

     def _calculateParams(self, listPackages):

@@ -119,28 +117,21 @@ class PackageManager(object):

         self.mapPackageToCycle.clear()

         self.sortedPackageList = []

-        self.listOfPackagesAlreadyBuilt = list(self._readAlreadyAvailablePackages())

-

-        updateBuiltRPMSList = False

-        while not updateBuiltRPMSList:

-            updateBuiltRPMSList = True

-            listOfPackagesAlreadyBuilt = self.listOfPackagesAlreadyBuilt

-            for pkg in listOfPackagesAlreadyBuilt:

-                packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)

-                listDependentRpmPackages = SPECS.getData().getRequiresAllForPackage(packageName, packageVersion)

-                needToRebuild = False

-                for dependentPkg in listDependentRpmPackages:

-                    if dependentPkg not in self.listOfPackagesAlreadyBuilt:

-                        needToRebuild = True

-                        updateBuiltRPMSList = False

-                if needToRebuild:

-                    self.listOfPackagesAlreadyBuilt.remove(pkg)

+        self.listOfPackagesAlreadyBuilt = self._readAlreadyAvailablePackages()

+        if self.listOfPackagesAlreadyBuilt:

+            self.logger.debug("List of already available packages:")

+            self.logger.debug(self.listOfPackagesAlreadyBuilt)

         listPackagesToBuild = copy.copy(listPackages)

         for pkg in listPackages:

             if (pkg in self.listOfPackagesAlreadyBuilt and

                     not constants.rpmCheck):

                 listPackagesToBuild.remove(pkg)

+

+        if listPackagesToBuild:

+            self.logger.info("List of packages yet to be built...")

+            self.logger.info(listPackagesToBuild)

+            self.logger.info("")

         if not self._readPackageBuildData(listPackagesToBuild):

             return False

         return True

@@ -158,39 +149,23 @@ class PackageManager(object):

     def _initializeScheduler(self, statusEvent):

         Scheduler.setLog(self.logName, self.logPath, self.logLevel)

-        Scheduler.setParams(self.sortedPackageList, set(self.listOfPackagesAlreadyBuilt))

+        Scheduler.setParams(self.sortedPackageList, self.listOfPackagesAlreadyBuilt)

         Scheduler.setEvent(statusEvent)

         Scheduler.stopScheduling = False

     def _buildGivenPackages(self, listPackages, buildThreads):

         # Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..]

-        listPackageNamesAndVersions=[]

+        listPackageNamesAndVersions=set()

         for pkg in listPackages:

-            for version in SPECS.getData().getVersions(pkg):

-                listPackageNamesAndVersions.append(pkg+"-"+version)

-        alreadyBuiltRPMS = self._readAlreadyAvailablePackages()

-        if alreadyBuiltRPMS:

-            self.logger.debug("List of already available packages:")

-            self.logger.debug(alreadyBuiltRPMS)

-

-        if constants.rpmCheck:

-            listMakeCheckPackages=set()

-            for pkg in listPackages:

-                version = SPECS.getData().getHighestVersion(pkg)

-                listMakeCheckPackages.add(pkg+"-"+version)

-            listPackageNamesAndVersions = (list(set(listPackageNamesAndVersions)|(listMakeCheckPackages-alreadyBuiltRPMS)))

+            base = SPECS.getData().getSpecName(pkg)

+            for version in SPECS.getData().getVersions(base):

+                listPackageNamesAndVersions.add(base+"-"+version)

         returnVal = self._calculateParams(listPackageNamesAndVersions)

         if not returnVal:

             self.logger.error("Unable to set paramaters. Terminating the package manager.")

             raise Exception("Unable to set paramaters")

-        listBasePackageNamesAndVersions = list(map(lambda x:SPECS.getData().getBasePkg(x), listPackageNamesAndVersions))

-        listPackagesToBuild = list((set(listBasePackageNamesAndVersions) - set(alreadyBuiltRPMS)))

-        if listPackagesToBuild:

-            self.logger.info("List of packages yet to be built...")

-            self.logger.info(listPackagesToBuild)

-            self.logger.info("")

         statusEvent = threading.Event()

         self._initializeScheduler(statusEvent)

         self._initializeThreadPool(statusEvent)

@@ -225,7 +200,7 @@ class PackageManager(object):

                 self.logger.error("Build stopped unexpectedly.Unknown error.")

                 raise Exception("Unknown error")

-    def _createBuildContainer(self):

+    def _createBuildContainer(self, usePublishedRPMs):

         self.logger.debug("Generating photon build container..")

         try:

             #TODO image name constants.buildContainerImageName

@@ -241,7 +216,7 @@ class PackageManager(object):

             chroot = Chroot(self.logger)

             chroot.create("toolchain-chroot")

             tcUtils = ToolChainUtils("toolchain-chroot", self.logPath)

-            tcUtils.installToolChainRPMS(chroot)

+            tcUtils.installToolChainRPMS(chroot, usePublishedRPMS=usePublishedRPMs)

         except Exception as e:

             if chroot:

                 chroot.destroy()

@@ -40,7 +40,7 @@ class PackageUtils(object):

         self.logfnvalue = None

     def prepRPMforInstall(self, package, version, noDeps=False, destLogPath=None):

-        rpmfile = self.findRPMFileForGivenPackage(package, version)

+        rpmfile = self.findRPMFile(package, version)

         if rpmfile is None:

             self.logger.error("No rpm file found for package: " + package)

             raise Exception("Missing rpm file")

@@ -51,8 +51,11 @@ class PackageUtils(object):

             rpmDestFile = "/publishrpms/"

         elif "PUBLISHXRPMS" in rpmfile:

             rpmDestFile = "/publishxrpms/"

+        elif constants.inputRPMSPath and constants.inputRPMSPath in rpmfile:

+            rpmDestFile = "/inputrpms/"

         else:

             rpmDestFile = constants.topDirPath + "/RPMS/"

+

         if "noarch" in rpmfile:

             rpmDestFile += "noarch/"

         else:

@@ -139,29 +142,24 @@ class PackageUtils(object):

                 CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug)

         self.logger.debug("RPM build is successful")

-    def findRPMFileForGivenPackage(self, package,version="*"):

+    def findRPMFile(self, package,version="*"):

         cmdUtils = CommandUtils()

         if version == "*":

                 version = SPECS.getData().getHighestVersion(package)

         release = SPECS.getData().getRelease(package, version)

         buildarch=SPECS.getData().getBuildArch(package, version)

-        listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." +

-                                                   buildarch+".rpm",

-                                                   constants.rpmPath)], [])

+        filename= package + "-" + version + "-" + release + "." + buildarch+".rpm"

+

+        fullpath = constants.rpmPath + "/" + buildarch + "/" + filename

+        if os.path.isfile(fullpath):

+            return fullpath

+

         if constants.inputRPMSPath is not None:

-            listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release +

-                                                       "." + buildarch+".rpm",

-                                                       constants.inputRPMSPath)],

-                                    listFoundRPMFiles)

-        if len(listFoundRPMFiles) == 1:

-            return listFoundRPMFiles[0]

-        if len(listFoundRPMFiles) == 0:

-            return None

-        if len(listFoundRPMFiles) > 1:

-            self.logger.error("Found multiple rpm files for given package in rpm directory." +

-                              "Unable to determine the rpm file for package:" + package +

-                              " : " + str(listFoundRPMFiles))

-            raise Exception("Multiple rpm files found")

+            fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename

+        if os.path.isfile(fullpath):

+            return fullpath

+

+        return None

     def findInstalledRPMPackages(self, sandbox):

         rpms = None

@@ -59,6 +59,7 @@ class Chroot(Sandbox):

         cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/tmp")

         cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/publishrpms")

         cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/publishxrpms")

+        cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/inputrpms")

         cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath)

         cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/RPMS")

         cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/SRPMS")

@@ -83,6 +84,9 @@ class Chroot(Sandbox):

                                         chrootID + "/publishrpms")

             cmdUtils.runCommandInShell("mount -o ro --bind " + constants.prevPublishXRPMRepo + " " +

                                         chrootID + "/publishxrpms")

+            if constants.inputRPMSPath:

+                cmdUtils.runCommandInShell("mount -o ro --bind " + constants.inputRPMSPath + " " +

+                                            chrootID + "/inputrpms")

         self.logger.debug("Successfully created chroot:" + chrootID)

@@ -180,6 +184,9 @@ class Container(Sandbox):

             constants.dockerUnixSocket: {'bind': constants.dockerUnixSocket, 'mode': 'rw'}

             }

+        if constants.inputRPMSPath:

+            mountVols[constants.inputRPMSPath] = {'bind': '/inputrpms', 'mode': 'ro'}

+

         containerName = containerName.replace("+", "p")

         try:

             oldContainerID = self.dockerClient.containers.get(containerName)

@@ -96,6 +96,10 @@ class Scheduler(object):

             return package

     @staticmethod

+    def getDoneList():

+        return list(Scheduler.listOfAlreadyBuiltPackages)

+

+    @staticmethod

     def _getBuildRequiredPackages(pkg):

         listRequiredRPMPackages = []

         listRequiredRPMPackages.extend(SPECS.getData().getBuildRequiresForPkg(pkg))

@@ -26,7 +26,7 @@ class ToolChainUtils(object):

         else:

             self.rpmCommand = "fakeroot-ng rpm"

-    def findRPMFileInGivenLocation(self, package, rpmdirPath):

+    def _findPublishedRPM(self, package, rpmdirPath):

         listFoundRPMFiles = CommandUtils.findFile(package + "-*.rpm", rpmdirPath)

         listFilterRPMFiles = []

         for f in listFoundRPMFiles:

@@ -55,7 +55,7 @@ class ToolChainUtils(object):

             coreToolChainYetToBuild = []

             for package in constants.listCoreToolChainPackages:

                 version = SPECS.getData().getHighestVersion(package)

-                rpmPkg = pkgUtils.findRPMFileForGivenPackage(package, version)

+                rpmPkg = pkgUtils.findRPMFile(package, version)

                 if rpmPkg is not None:

                     continue

                 else:

@@ -93,7 +93,7 @@ class ToolChainUtils(object):

         listBuildRequiresPkg.extend(SPECS.getData().getCheckBuildRequiresForPackage(package, version))

         return listBuildRequiresPkg

-    def installToolChainRPMS(self, chroot, packageName=None, packageVersion=None, logPath=None):

+    def installToolChainRPMS(self, chroot, packageName=None, packageVersion=None, logPath=None, usePublishedRPMS=True, availablePackages=None):

         if logPath is None:

             logPath = self.logPath

         self.logger.debug("Installing Tool Chain RPMS.......")

@@ -105,30 +105,51 @@ class ToolChainUtils(object):

         for package in constants.listToolChainRPMsToInstall:

             pkgUtils = PackageUtils(self.logName, self.logPath)

             rpmFile = None

-            version = "*"

+            version = None

+

+            # Get proper package version

             for depPkg in listBuildRequiresPackages:

                 depPkgName, depPkgVersion = StringUtils.splitPackageNameAndVersion(depPkg)

                 if depPkgName == package:

                         version=depPkgVersion

+                        break

+            if not version:

+                version = SPECS.getData().getHighestVersion(package)

+

+            basePkg = SPECS.getData().getSpecName(package)+"-"+version

+            isAvailable = (availablePackages and basePkg in availablePackages)

+

             if constants.rpmCheck:

-                rpmFile = pkgUtils.findRPMFileForGivenPackage(package, version)

-            elif (packageName is None or 

-                    packageName not in constants.listToolChainRPMsToInstall or

-                    constants.listToolChainRPMsToInstall.index(packageName) >

+                rpmFile = pkgUtils.findRPMFile(package, version)

+

+            if rpmFile is None:

+                # Honor the toolchain list order.

+                # if index of depended package ('package') is more

+                # then index of the current package that we are

+                # building ('packageName'), then we _must_ use published

+                # `package` rpm.

+                if (packageName and

+                    packageName in constants.listToolChainRPMsToInstall and

+                    constants.listToolChainRPMsToInstall.index(packageName) <

                         constants.listToolChainRPMsToInstall.index(package)):

-                rpmFile = pkgUtils.findRPMFileForGivenPackage(package, version)

+                    isAvailable = False

+                else:

+                    rpmFile = pkgUtils.findRPMFile(package, version)

+

             if rpmFile is None:

+                if not usePublishedRPMS or isAvailable:

+                    raise Exception("%s-%s not found in available packages" % (package, version))

+

                 # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf

                 if (package == "sqlite") and (platform.machine() == "x86_64"):

                     package = "sqlite-autoconf"

-                rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)

+                rpmFile = self._findPublishedRPM(package, constants.prevPublishRPMRepo)

                 if rpmFile is None:

                     if package in constants.listOfRPMsProvidedAfterBuild:

                         self.logger.debug("No old version of " + package +

                                          " exists, skip until the new version is built")

                         continue

-                    self.logger.error("Unable to find rpm " + package +

-                                      " in current and previous versions")

+                    self.logger.error("Unable to find published rpm " + package)

                     raise Exception("Input Error")

             rpmFiles += " " + rpmFile

             packages += " " + package+"-"+version

@@ -161,7 +182,7 @@ class ToolChainUtils(object):

             else:

                 path = constants.prevPublishRPMRepo

                 sandboxPath = "/publishrpms"

-            rpmFile = self.findRPMFileInGivenLocation(package, path)

+            rpmFile = self._findPublishedRPM(package, path)

             if rpmFile is None:

                 self.logger.error("Unable to find rpm "+ package +

                                   " in current and previous versions")

@@ -21,6 +21,7 @@ class WorkerThread(threading.Thread):

         self.logger.debug("Thread " + self.name + " is starting now")

         while True:

             pkg = Scheduler.Scheduler.getNextPackageToBuild()

+            doneList = Scheduler.Scheduler.getDoneList()

             if pkg is None:

                 break

             if self.pkgBuildType == "chroot":

@@ -30,7 +31,7 @@ class WorkerThread(threading.Thread):

                 pkgBuilder = PackageBuilderContainer(self.mapPackageToCycle,

                                                      self.pkgBuildType)

             try:

-                pkgBuilder.buildPackageFunction(pkg)

+                pkgBuilder.buildPackageFunction(pkg, doneList)

             except Exception as e:

                 self.logger.exception(e)

                 buildThreadFailed = True