@@ -1,21 +1,9 @@

-import subprocess

 from Logger import Logger

 from CommandUtils import CommandUtils

-import os

-import threading

-import time

 from constants import constants

-

-chrootRootPath="/mnt"

+import os.path

 class ChrootUtils(object):

-    counter=1

-    lockForCounter=threading.Lock()

-    lockForTrackingChroots=threading.Lock()

-    lockForCreateChroot=threading.Lock()

-    activeChroots=[]

-    failureFlag=False

-    numChroots=1

     def __init__(self,logName=None,logPath=None):

         if logName is None:

@@ -26,124 +14,34 @@ class ChrootUtils(object):

         self.logPath=logPath

         self.logger=Logger.getLogger(logName,logPath)

-    def _getChrootUniqueID(self):

-        ChrootUtils.lockForCounter.acquire()

-        chrootID=chrootRootPath+"/photonroot"+str(ChrootUtils.counter)

-        while os.path.isdir(chrootID):

-            ChrootUtils.counter = ChrootUtils.counter +1

-            chrootID=chrootRootPath+"/photonroot"+str(ChrootUtils.counter)

-        ChrootUtils.counter = ChrootUtils.counter +1

-        ChrootUtils.lockForCounter.release()

-        return chrootID

-    

-    def updateParams(self):

-        process = subprocess.Popen(["df" ,chrootRootPath],shell=True,stdout=subprocess.PIPE)

-        retval = process.wait()

-        if retval != 0:

-            self.logger.error("Unable to check free space. Unknown error.")

-            return False

-        output = process.communicate()[0]

-        device, size, used, available, percent, mountpoint = output.split("\n")[1].split()

-        c =  int(available)/600000

-        ChrootUtils.numChroots=int(c)

-        self.logger.info("Updated number of chroots:"+str(ChrootUtils.numChroots))

-    

-    def checkFreeSpace(self):

-        process = subprocess.Popen(["df" ,chrootRootPath],shell=True,stdout=subprocess.PIPE)

-        retval = process.wait()

-        if retval != 0:

-            self.logger.error("Unable to check free space. Unknown error.")

-            return False

-        output = process.communicate()[0]

-        device, size, used, available, percent, mountpoint = output.split("\n")[1].split()

-        if available > 600000:

-            self.logger.info("Free Space available "+ available)

-            return True

-        return False

-    

-    def createChroot(self):

-        if ChrootUtils.failureFlag:

-            return False

-        

-        ChrootUtils.lockForCreateChroot.acquire()

-        if ChrootUtils.failureFlag:

-            ChrootUtils.lockForCreateChroot.release()

-            return False

-        

-        startTime=currentTime=time.time()

-        timeOut=1800

-        freeSpaceAvailable=False

-        while currentTime < startTime + timeOut:

-            #if self.checkFreeSpace():

-            if len(ChrootUtils.activeChroots) < ChrootUtils.numChroots:

-                freeSpaceAvailable=True

-                break

-            time.sleep(100)

-            

-        if not freeSpaceAvailable:

-            self.logger.error("Unable to create chroot. No sufficient free space is available.")

-            ChrootUtils.lockForCreateChroot.release()

-            ChrootUtils.failureFlag=True

-            return False,None

-        

-        chrootID=self._getChrootUniqueID()

+    def createChroot(self,chrootName):

+        chrootID=constants.buildRootPath+"/"+chrootName

+        if os.path.isdir(chrootID):

+            if not self.destroyChroot(chrootID):

+                self.logger.error("Given chroot "+chrootID+" is already exists. unable to destroy it ")

+                return False,None

         # need to add timeout for this step

         # http://stackoverflow.com/questions/1191374/subprocess-with-timeout

-        process = subprocess.Popen("mkdir -p "+chrootID,shell=True,stdout=subprocess.PIPE)

-        retval = process.wait()

-        

-        ChrootUtils.lockForCreateChroot.release()

-        

-        if retval != 0:

-            self.logger.error("Unable to create chroot:"+ chrootID +".Unknown error.")

-            return False,None

-        

-        ChrootUtils.lockForTrackingChroots.acquire()

-        ChrootUtils.activeChroots.append(chrootID) 

-        ChrootUtils.lockForTrackingChroots.release()           

-        return True,chrootID

-    

-    def createChroot1(self,chrootID):

-        # need to add timeout for this step

-        # http://stackoverflow.com/questions/1191374/subprocess-with-timeout

-        process = subprocess.Popen("mkdir -p "+chrootID,shell=True,stdout=subprocess.PIPE)

-        retval = process.wait()

-        

-        if retval != 0:

+        cmdUtils=CommandUtils()

+        returnVal=cmdUtils.runCommandInShell("mkdir -p "+chrootID)

+        if not returnVal:

             self.logger.error("Unable to create chroot:"+ chrootID +".Unknown error.")

             return False,None

-        

         return True,chrootID

     def destroyChroot(self,chrootID):

-        '''

-        validChroot = True

-        ChrootUtils.lockForTrackingChroots.acquire()

-        if chrootID not in ChrootUtils.activeChroots:

-            validChroot = False

-        else:

-            ChrootUtils.activeChroots.remove(chrootID)

-        ChrootUtils.lockForTrackingChroots.release()

-        if not validChroot:

-            self.logger.error("Given chroot:"+chrootID+" is not a valid chroot. It is not created by ChrootUtils.")

-            return False

-        '''

         # need to add timeout for this step

         # http://stackoverflow.com/questions/1191374/subprocess-with-timeout

-        process = subprocess.Popen("./cleanup-build-root.sh "+chrootID,shell=True,stdout=subprocess.PIPE)

-        retval = process.wait()

-        if retval != 0:

+        cmdUtils=CommandUtils()

+        returnVal=cmdUtils.runCommandInShell("./cleanup-build-root.sh "+chrootID)

+        if not returnVal:

+            self.logger.error("Unable to destroy chroot:"+ chrootID +".Unknown error.")

+            return False

+        

+        returnVal=cmdUtils.runCommandInShell("rm -rf "+chrootID)

+        if not returnVal:

             self.logger.error("Unable to destroy chroot:"+ chrootID +".Unknown error.")

             return False

-        cmdUtils=CommandUtils()

-        cmdUtils.runCommandInShell("rm -rf "+chrootID)

         self.logger.info("Successfully destroyed chroot:"+chrootID)

         return True

-    

-    def cleanUpActiveChroots(self):

-        ChrootUtils.lockForTrackingChroots.acquire()

-        listChroots=self.activeChroots[:]

-        ChrootUtils.lockForTrackingChroots.release()

-        for chrootID in listChroots:

-            self.destroyChroot(chrootID)

-        self.logger.info("Successfully destroyed all active chroots")

+    

\ No newline at end of file

@@ -20,27 +20,11 @@ class PackageBuilder(object):

         self.listAvailableCyclicPackages = listAvailableCyclicPackages

         self.listNodepsPackages = ["glibc","gmp","zlib","file","binutils","mpfr","mpc","gcc","ncurses","util-linux","groff","perl","texinfo","rpm","openssl","go"]

-    #assumes tool chain is already built

-    def prepareBuildRoot(self):

+    def prepareBuildRoot(self,chrootName):

         chrootID=None

         try:

             chrUtils = ChrootUtils(self.logName,self.logPath)

-            returnVal,chrootID = chrUtils.createChroot()

-            if not returnVal:

-                raise Exception("Unable to prepare build root")

-            tUtils=ToolChainUtils(self.logName,self.logPath)

-            tUtils.installToolChain(chrootID)

-        except Exception as e:

-            if chrootID is not None:

-                chrUtils.destroyChroot(chrootID)

-            raise e

-        return chrootID

-    

-    def prepareBuildRoot1(self,chrootID):

-        #chrootID=None

-        try:

-            chrUtils = ChrootUtils(self.logName,self.logPath)

-            returnVal,chrootID = chrUtils.createChroot1(chrootID)

+            returnVal,chrootID = chrUtils.createChroot(chrootName)

             if not returnVal:

                 raise Exception("Unable to prepare build root")

             tUtils=ToolChainUtils(self.logName,self.logPath)

@@ -85,9 +69,10 @@ class PackageBuilder(object):

     def buildPackage(self,package):

         #should initialize a logger based on package name

         chrUtils = ChrootUtils(self.logName,self.logPath)

-        chrootID="build-"+package

+        chrootName="build-"+package

+        chrootID=None

         try:

-            chrootID = self.prepareBuildRoot1(chrootID)

+            chrootID = self.prepareBuildRoot(chrootName)

             destLogPath=constants.logPath+"/build-"+package

             if not os.path.isdir(destLogPath):

                 cmdUtils = CommandUtils()

@@ -2,14 +2,11 @@ from PackageBuildDataGenerator import PackageBuildDataGenerator

 from Logger import Logger

 import threading

 from constants import constants

-import time

-from PackageBuilder import PackageBuilder

 import os

 from PackageUtils import PackageUtils

 from ToolChainUtils import ToolChainUtils

 from Scheduler import Scheduler

 from ThreadPool import ThreadPool

-from WorkerThread import WorkerThread

 import subprocess

 class PackageManager(object):

@@ -41,20 +38,6 @@ class PackageManager(object):

             return False

         return True

-    def getRequiredPackages(self,package):

-        listRequiredRPMPackages=[]

-        listRequiredRPMPackages.extend(constants.specData.getBuildRequiresForPackage(package))

-        listRequiredRPMPackages.extend(constants.specData.getRequiresAllForPackage(package))

-        

-        listRequiredPackages=[]

-        for pkg in listRequiredRPMPackages:

-            basePkg=constants.specData.getSpecName(pkg)

-            if basePkg not in listRequiredPackages:

-                listRequiredPackages.append(basePkg)

-        

-        return listRequiredPackages

-                

-    

     def readAlreadyAvailablePackages(self):

         listAvailablePackages=[]

         listRPMFiles=[]

@@ -100,68 +83,6 @@ class PackageManager(object):

         return True

-    def checkIfAnyThreadsAreCompleted(self):

-        readyToLaunchMoreThreads = False

-        listThreadsObjToRemove=[]

-        for t in self.listThreads:

-            self.logger.debug("Checking thread "+t +", whether it is completed or not")

-            #check if any thread is completed. If completed, we can start more threads.

-            if self.mapOutputThread.has_key(t):

-                output = self.mapOutputThread[t]

-                self.logger.info("Output of thread "+t+" "+str(output))

-                if not output:

-                    self.logger.error("Thread "+ t+" is failed ")

-                    self.logger.error("Unable to build package "+t)

-                    raise Exception("Package builder failed")

-                else:

-                    self.logger.debug("Still running, not completed yet")

-                    readyToLaunchMoreThreads=True

-                    self.listPackagesToBuild.remove(t)

-                    self.listOfPackagesAlreadyBuilt.append(t)

-                    listThreadsObjToRemove.append(t)

-                    if self.mapPackageToCycle.has_key(t):

-                        self.listAvailableCyclicPackages.append(t)

-        

-        if not readyToLaunchMoreThreads:

-            return False

-            

-        for k in listThreadsObjToRemove:

-            tObj=self.listThreads.pop(k)

-            if tObj.isAlive():

-                self.logger.info("Thread is alive")

-            else:

-                self.logger.info("Thread is dead")

-            self.logger.info(tObj.isAlive())

-

-        return True

-    

-    #just to be safe, we have this method

-    def checkIfAnyThreadsAreHanged(self):

-        currentTime = time.time()

-        listThreadsHanged=[]

-        for t in self.listThreads:

-            self.logger.debug("Checking thread "+t +", whether it is hanged or not")

-            if not self.mapOutputThread.has_key(t):

-                self.logger.debug("Calculating running time for thread "+t)

-                launchTime = self.mapThreadsLaunchTime[t]

-                if (currentTime - launchTime) > 7200.0:

-                    listThreadsHanged.append(t)

-            

-        if len(listThreadsHanged) > 0:

-            self.logger.info("Following threads are hanged")

-            self.logger.info(listThreadsHanged)

-            raise Exception("Threads are hanged")

-    

-    def waitTillNewThreadsCanBeSpawned(self):

-        if len(self.listThreads) == 0:

-            return

-        while True:

-            if self.checkIfAnyThreadsAreCompleted():

-                break

-            self.checkIfAnyThreadsAreHanged()

-            self.logger.info("Sleeping for 10 seconds")

-            time.sleep(10)

-    

     def buildToolChain(self):

         try:

             tUtils=ToolChainUtils()

@@ -174,7 +95,7 @@ class PackageManager(object):

         return True

     def calculatePossibleNumWorkerThreads(self):

-        process = subprocess.Popen(["df" ,"/mnt"],shell=True,stdout=subprocess.PIPE)

+        process = subprocess.Popen(["df" ,constants.buildRootPath],shell=True,stdout=subprocess.PIPE)

         retval = process.wait()

         if retval != 0:

             self.logger.error("Unable to check free space. Unknown error.")

@@ -183,7 +104,7 @@ class PackageManager(object):

         device, size, used, available, percent, mountpoint = output.split("\n")[1].split()

         c =  int(available)/600000

         numChroots=int(c)

-        self.logger.info("Updated number of chroots:"+str(numChroots))

+        self.logger.info("Possible number of worker threads:"+str(numChroots))

         return numChroots

     def buildPackages (self, listPackages):

@@ -211,6 +132,8 @@ class PackageManager(object):

         ThreadPool.listAvailableCyclicPackages=self.listAvailableCyclicPackages

         ThreadPool.logger=self.logger

         ThreadPool.statusEvent=statusEvent

+        

+        

         i=0

         while i < numWorkerThreads:

             workerName="WorkerThread"+str(i)

@@ -221,11 +144,12 @@ class PackageManager(object):

         statusEvent.wait()

         Scheduler.stopScheduling=True

-        

         setFailFlag=False

         allPackagesBuilt=False

+        

         if Scheduler.isAnyPackagesFailedToBuild():

             setFailFlag=True

+        

         if Scheduler.isAllPackagesBuilt():

             allPackagesBuilt=True

@@ -245,87 +169,3 @@ class PackageManager(object):

             w.join()

         self.logger.info("Terminated")

-        

-        

-        

-            

-        

-

-    

-    def buildPackages1 (self, listPackages):

-        

-        if not self.buildToolChain():

-            return False

-

-        returnVal=self.calculateParams(listPackages)

-        if not returnVal:

-            self.logger.error("Unable to set paramaters. Terminating the package manager.")

-            return False

-        

-        returnVal = True

-        try:

-            while len(self.listPackagesToBuild) > 0:

-                #Free some threads to launch next threads

-                self.waitTillNewThreadsCanBeSpawned()

-                

-                listOfPackagesCanBeBuild=self.findNextPackageToBuild()

-                if len(listOfPackagesCanBeBuild) == 0 and len(self.listPackagesToBuild) != 0:

-                    self.logger.info("Waiting for current threads to complete to launch building new packages")

-                

-                for pkg in listOfPackagesCanBeBuild:

-                    currentTime = time.time()

-                    pkgBuilder = PackageBuilder(self.mapPackageToCycle,self.listAvailableCyclicPackages,"build-"+pkg,constants.logPath)

-                    t = threading.Thread(target=pkgBuilder.buildPackageThreadAPI,args=(pkg,self.mapOutputThread,pkg))

-                    self.listThreads[pkg]=t

-                    self.mapThreadsLaunchTime[pkg]=currentTime

-                    self.logger.info("Launching thread for package:"+pkg)

-                    t.start()

-                    self.logger.info("Started the thread for "+pkg)

-                

-                if len(self.listThreads) == 0 and len(self.listPackagesToBuild) != 0:

-                    self.logger.error("Following packages are waiting for unknown package")

-                    self.logger.error(self.listPackagesToBuild)

-                    raise Exception("Invalid Schedule order")

-                    

-            self.logger.info( "Successfully built all the packages")

-

-        except Exception as e:

-            self.logger.error(str(e))

-            self.logger.error("Caught exception.")

-            self.logger.error("Failed and exited gracefully")

-            returnVal = False

-        finally:

-            self.waitForRemainingThreads()

-        return returnVal

-    

-    def waitForRemainingThreads(self):

-        self.logger.info("Waiting for all remaining threads to complete")

-        for t in self.listThreads:

-            self.listThreads[t].join()

-        return

-

-    def findNextPackageToBuild(self):

-        listOfPackagesNextToBuild=[]

-        self.logger.info("Checking for next possible packages to build")

-        for pkg in self.listPackagesToBuild:

-            if self.listThreads.has_key(pkg):

-                continue

-            listRequiredPackages=self.getRequiredPackages(pkg)

-            canBuild=True

-            self.logger.info("Required packages for "+ pkg + " are:")

-            self.logger.info(listRequiredPackages)

-            for reqPkg in listRequiredPackages:

-                if reqPkg not in self.listOfPackagesAlreadyBuilt:

-                    canBuild=False

-                    self.logger.info(reqPkg+" is not available. So we cannot build "+ pkg +" at this moment.")

-                    break

-            if canBuild:

-                listOfPackagesNextToBuild.append(pkg)

-        return listOfPackagesNextToBuild

-                

-                 

-            

-            

-        

-    

-    

@@ -4,7 +4,6 @@ from Logger import Logger

 from PackageUtils import PackageUtils

 import shutil

 from constants import constants

-import subprocess

 class ToolChainUtils(object):

     __built_successfull=False

@@ -44,14 +43,16 @@ class ToolChainUtils(object):

             "libpipeline", "gdbm","perl","texinfo","rpm",

             "autoconf","automake", "groff", "man-db", "man-pages","elfutils","cpio"]

-    def prepareChroot(self,chrootID,toolsArchive=""):

-        process = subprocess.Popen([self.prepareBuildRootCmd, chrootID,  constants.specPath,  constants.rpmPath, constants.toolsPath,toolsArchive])

-        returnVal = process.wait()

-        if returnVal != 0:

+    def prepareChroot(self,chrootID,toolsArchive=None):

+        cmdUtils=CommandUtils()

+        prepareChrootCmd=self.prepareBuildRootCmd+" "+chrootID+" "+constants.specPath+" "+constants.rpmPath+" "+constants.toolsPath

+        if toolsArchive is not None:

+            prepareChrootCmd=prepareChrootCmd+" "+toolsArchive

+        returnVal=cmdUtils.runCommandInShell(prepareChrootCmd)

+        if not returnVal:

             self.logger.error("Prepare build root script failed.Unable to prepare chroot.")

             raise Exception("Prepare build root script failed")

-        cmdUtils=CommandUtils()

         cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS")

         cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/x86_64")

         cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/noarch")

@@ -64,7 +65,8 @@ class ToolChainUtils(object):

         chrootID=None

         chrUtils = ChrootUtils(self.logName,self.logPath)

         try:

-            returnVal,chrootID = chrUtils.createChroot()

+            chrootName="build-toolchain"

+            returnVal,chrootID = chrUtils.createChroot(chrootName)

             if not returnVal:

                 raise Exception("creating chroot failed")

             self.prepareChroot(chrootID)

@@ -68,8 +68,6 @@ def main():

     listPackages1=["nano","swig","wget"]

-    chrUtils = ChrootUtils()

-    chrUtils.updateParams()

     pkgManager = PackageManager()

     pkgManager.buildPackages(listPackages)

@@ -10,6 +10,7 @@ class constants(object):

     topDirPath=""

     specData=None

     testPath="test"

+    buildRootPath="/mnt"

     @staticmethod

@@ -23,7 +23,10 @@ BUILDROOT=$1

 SPEC_PATH=$2

 RPM_PATH=$3

 TOOLS_PATH=$4

-TOOLS_ARCHIVE=$5

+TOOLS_ARCHIVE=""

+if [ $# -ge 5 ]; then

+    TOOLS_ARCHIVE=$5

+fi

 PARENT=/usr/src/photon

 rm -rf ${BUILDROOT}/*