| ... | ... |
@@ -113,8 +113,13 @@ class ChrootUtils(object): |
| 113 | 113 |
return False |
| 114 | 114 |
cmdUtils=CommandUtils() |
| 115 | 115 |
cmdUtils.runCommandInShell("rm -rf "+chrootID)
|
| 116 |
- ChrootUtils.lockForTrackingChroots.acquire() |
|
| 117 |
- ChrootUtils.lockForTrackingChroots.release() |
|
| 118 | 116 |
self.logger.info("Successfully destroyed chroot:"+chrootID)
|
| 119 | 117 |
return True |
| 120 |
- |
|
| 118 |
+ |
|
| 119 |
+ def cleanUpActiveChroots(self): |
|
| 120 |
+ ChrootUtils.lockForTrackingChroots.acquire() |
|
| 121 |
+ listChroots=self.activeChroots[:] |
|
| 122 |
+ ChrootUtils.lockForTrackingChroots.release() |
|
| 123 |
+ for chrootID in listChroots: |
|
| 124 |
+ self.destroyChroot(chrootID) |
|
| 125 |
+ self.logger.info("Successfully destroyed all active chroots")
|
| ... | ... |
@@ -35,7 +35,6 @@ class PackageManager(object): |
| 35 | 35 |
except: |
| 36 | 36 |
self.logger.error("unable to get sorted list")
|
| 37 | 37 |
return False |
| 38 |
- |
|
| 39 | 38 |
return True |
| 40 | 39 |
|
| 41 | 40 |
def getRequiredPackages(self,package): |
| ... | ... |
@@ -101,16 +100,17 @@ class PackageManager(object): |
| 101 | 101 |
readyToLaunchMoreThreads = False |
| 102 | 102 |
listThreadsObjToRemove=[] |
| 103 | 103 |
for t in self.listThreads: |
| 104 |
- self.logger.info("Checking thread "+t)
|
|
| 104 |
+ self.logger.debug("Checking thread "+t +", whether it is completed or not")
|
|
| 105 | 105 |
#check if any thread is completed. If completed, we can start more threads. |
| 106 | 106 |
if self.mapOutputThread.has_key(t): |
| 107 | 107 |
output = self.mapOutputThread[t] |
| 108 | 108 |
self.logger.info("Output of thread "+t+" "+str(output))
|
| 109 | 109 |
if not output: |
| 110 | 110 |
self.logger.error("Thread "+ t+" is failed ")
|
| 111 |
- #kill remaining Threads |
|
| 112 |
- return False,False |
|
| 111 |
+ self.logger.error("Unable to build package "+t)
|
|
| 112 |
+ raise Exception("Package builder failed")
|
|
| 113 | 113 |
else: |
| 114 |
+ self.logger.debug("Still running, not completed yet")
|
|
| 114 | 115 |
readyToLaunchMoreThreads=True |
| 115 | 116 |
self.listPackagesToBuild.remove(t) |
| 116 | 117 |
self.listOfPackagesAlreadyBuilt.append(t) |
| ... | ... |
@@ -119,93 +119,106 @@ class PackageManager(object): |
| 119 | 119 |
self.listAvailableCyclicPackages.append(t) |
| 120 | 120 |
|
| 121 | 121 |
if not readyToLaunchMoreThreads: |
| 122 |
- return True,False |
|
| 122 |
+ return False |
|
| 123 | 123 |
|
| 124 | 124 |
for k in listThreadsObjToRemove: |
| 125 |
- self.listThreads.pop(k) |
|
| 126 |
- |
|
| 127 |
- return True,True |
|
| 125 |
+ tObj=self.listThreads.pop(k) |
|
| 126 |
+ if tObj.isAlive(): |
|
| 127 |
+ self.logger.info("Thread is alive")
|
|
| 128 |
+ else: |
|
| 129 |
+ self.logger.info("Thread is dead")
|
|
| 130 |
+ self.logger.info(tObj.isAlive()) |
|
| 131 |
+ |
|
| 132 |
+ return True |
|
| 128 | 133 |
|
| 129 | 134 |
def checkIfAnyThreadsAreHanged(self): |
| 130 | 135 |
currentTime = time.time() |
| 131 | 136 |
listThreadsHanged=[] |
| 132 | 137 |
for t in self.listThreads: |
| 133 |
- self.logger.info("Checking thread "+t)
|
|
| 138 |
+ self.logger.debug("Checking thread "+t +", whether it is hanged or not")
|
|
| 134 | 139 |
if not self.mapOutputThread.has_key(t): |
| 135 |
- self.logger.info("Calculating running time for thread "+t)
|
|
| 140 |
+ self.logger.debug("Calculating running time for thread "+t)
|
|
| 136 | 141 |
launchTime = self.mapThreadsLaunchTime[t] |
| 137 | 142 |
if (currentTime - launchTime) > 3600.0: |
| 138 | 143 |
listThreadsHanged.append(t) |
| 139 | 144 |
|
| 140 | 145 |
if len(listThreadsHanged) > 0: |
| 141 |
- self.logger.info("Looks like following threads are hanged")
|
|
| 146 |
+ self.logger.info("Following threads are hanged")
|
|
| 142 | 147 |
self.logger.info(listThreadsHanged) |
| 143 |
- #kill all threads |
|
| 144 |
- return False |
|
| 145 |
- |
|
| 146 |
- return True |
|
| 148 |
+ raise Exception("Threads are hanged")
|
|
| 147 | 149 |
|
| 148 | 150 |
def waitTillNewThreadsCanBeSpawned(self): |
| 149 | 151 |
if len(self.listThreads) == 0: |
| 150 |
- return True |
|
| 151 |
- returnVal = False |
|
| 152 |
+ return |
|
| 152 | 153 |
while True: |
| 153 |
- sucess,Tfail = self.checkIfAnyThreadsAreCompleted() |
|
| 154 |
- if not sucess: |
|
| 155 |
- break |
|
| 156 |
- if sucess and Tfail: |
|
| 157 |
- returnVal = True |
|
| 158 |
- break |
|
| 159 |
- if not self.checkIfAnyThreadsAreHanged(): |
|
| 154 |
+ if self.checkIfAnyThreadsAreCompleted(): |
|
| 160 | 155 |
break |
| 156 |
+ self.checkIfAnyThreadsAreHanged() |
|
| 161 | 157 |
self.logger.info("Sleeping for 30 seconds")
|
| 162 | 158 |
time.sleep(30) |
| 163 |
- return returnVal |
|
| 159 |
+ |
|
| 160 |
+ def buildToolChain(self): |
|
| 161 |
+ try: |
|
| 162 |
+ tUtils=ToolChainUtils() |
|
| 163 |
+ tUtils.buildToolChain() |
|
| 164 |
+ except Exception as e: |
|
| 165 |
+ self.logger.error("Unable to build tool chain")
|
|
| 166 |
+ self.logger.error(e) |
|
| 167 |
+ return False |
|
| 164 | 168 |
|
| 165 |
- def buildPackages (self, listPackages): |
|
| 169 |
+ return True |
|
| 166 | 170 |
|
| 167 |
- tUtils=ToolChainUtils() |
|
| 168 |
- tUtils.buildToolChain() |
|
| 171 |
+ def buildPackages (self, listPackages): |
|
| 169 | 172 |
|
| 173 |
+ if not self.buildToolChain(): |
|
| 174 |
+ return False |
|
| 175 |
+ |
|
| 170 | 176 |
returnVal=self.calculateParams(listPackages) |
| 171 | 177 |
if not returnVal: |
| 172 | 178 |
self.logger.error("Unable to set paramaters. Terminating the package manager.")
|
| 173 | 179 |
return False |
| 174 |
- returnVal = True |
|
| 175 |
- while len(self.listPackagesToBuild) > 0: |
|
| 176 |
- #Free some threads to launch next threads |
|
| 177 |
- if not self.waitTillNewThreadsCanBeSpawned(): |
|
| 178 |
- returnVal = False |
|
| 179 |
- break |
|
| 180 |
- |
|
| 181 |
- listOfPackagesCanBeBuild=self.findNextPackageToBuild() |
|
| 182 |
- if len(listOfPackagesCanBeBuild) == 0 and len(self.listPackagesToBuild) != 0: |
|
| 183 |
- self.logger.info("Waiting for current threads to complete to launch building new packages")
|
|
| 184 |
- |
|
| 185 |
- for pkg in listOfPackagesCanBeBuild: |
|
| 186 |
- currentTime = time.time() |
|
| 187 |
- pkgBuilder = PackageBuilder(self.mapPackageToCycle,self.listAvailableCyclicPackages,"build-"+pkg,constants.logPath) |
|
| 188 |
- t = threading.Thread(target=pkgBuilder.buildPackageThreadAPI,args=(pkg,self.mapOutputThread,pkg)) |
|
| 189 |
- self.listThreads[pkg]=t |
|
| 190 |
- self.mapThreadsLaunchTime[pkg]=currentTime |
|
| 191 |
- self.logger.info("Launching thread for package:"+pkg)
|
|
| 192 |
- t.start() |
|
| 193 |
- self.logger.info("Started the thread for "+pkg)
|
|
| 194 |
- |
|
| 195 |
- if len(self.listThreads) == 0 and len(self.listPackagesToBuild) != 0: |
|
| 196 |
- self.logger.error("Following packages are waiting for unknown package")
|
|
| 197 |
- self.logger.error(self.listPackagesToBuild) |
|
| 198 |
- self.logger.error("Unexpected error")
|
|
| 199 |
- returnVal = False |
|
| 200 |
- break |
|
| 201 |
- |
|
| 202 |
- if not returnVal: |
|
| 203 |
- self.logger.error("Failed and exited gracefully")
|
|
| 204 |
- return False |
|
| 205 | 180 |
|
| 206 |
- self.logger.info( "Successfully built all the packages") |
|
| 207 |
- return True |
|
| 181 |
+ returnVal = True |
|
| 182 |
+ try: |
|
| 183 |
+ while len(self.listPackagesToBuild) > 0: |
|
| 184 |
+ #Free some threads to launch next threads |
|
| 185 |
+ self.waitTillNewThreadsCanBeSpawned() |
|
| 186 |
+ |
|
| 187 |
+ listOfPackagesCanBeBuild=self.findNextPackageToBuild() |
|
| 188 |
+ if len(listOfPackagesCanBeBuild) == 0 and len(self.listPackagesToBuild) != 0: |
|
| 189 |
+ self.logger.info("Waiting for current threads to complete to launch building new packages")
|
|
| 190 |
+ |
|
| 191 |
+ for pkg in listOfPackagesCanBeBuild: |
|
| 192 |
+ currentTime = time.time() |
|
| 193 |
+ pkgBuilder = PackageBuilder(self.mapPackageToCycle,self.listAvailableCyclicPackages,"build-"+pkg,constants.logPath) |
|
| 194 |
+ t = threading.Thread(target=pkgBuilder.buildPackageThreadAPI,args=(pkg,self.mapOutputThread,pkg)) |
|
| 195 |
+ self.listThreads[pkg]=t |
|
| 196 |
+ self.mapThreadsLaunchTime[pkg]=currentTime |
|
| 197 |
+ self.logger.info("Launching thread for package:"+pkg)
|
|
| 198 |
+ t.start() |
|
| 199 |
+ self.logger.info("Started the thread for "+pkg)
|
|
| 208 | 200 |
|
| 201 |
+ if len(self.listThreads) == 0 and len(self.listPackagesToBuild) != 0: |
|
| 202 |
+ self.logger.error("Following packages are waiting for unknown package")
|
|
| 203 |
+ self.logger.error(self.listPackagesToBuild) |
|
| 204 |
+ raise Exception("Invalid Schedule order")
|
|
| 205 |
+ |
|
| 206 |
+ self.logger.info( "Successfully built all the packages") |
|
| 207 |
+ |
|
| 208 |
+ except Exception as e: |
|
| 209 |
+ self.logger.error(str(e)) |
|
| 210 |
+ self.logger.error("Caught exception.")
|
|
| 211 |
+ self.logger.error("Failed and exited gracefully")
|
|
| 212 |
+ returnVal = False |
|
| 213 |
+ finally: |
|
| 214 |
+ if len(self.listThreads) > 0: |
|
| 215 |
+ self.killAllThreads() |
|
| 216 |
+ return returnVal |
|
| 217 |
+ |
|
| 218 |
+ def killAllThreads(self): |
|
| 219 |
+ self.logger.info("Killing all remaining threads")
|
|
| 220 |
+ return |
|
| 221 |
+ |
|
| 209 | 222 |
def findNextPackageToBuild(self): |
| 210 | 223 |
listOfPackagesNextToBuild=[] |
| 211 | 224 |
self.logger.info("Checking for next possible packages to build")
|
| ... | ... |
@@ -84,15 +84,13 @@ class ToolChainUtils(object): |
| 84 | 84 |
if chrootID is not None: |
| 85 | 85 |
chrUtils.destroyChroot(chrootID) |
| 86 | 86 |
|
| 87 |
- #tool chain should be updated before calling this method |
|
| 87 |
+ #Tool chain should be built before calling this method |
|
| 88 | 88 |
def installToolChain(self,chrootID): |
| 89 | 89 |
self.logger.info("Installing toolchain.....")
|
| 90 | 90 |
self.prepareChroot(chrootID,"minimal") |
| 91 | 91 |
pkgUtils= PackageUtils(self.logName,self.logPath) |
| 92 | 92 |
for package in self.listPkgsToInstallToolChain: |
| 93 | 93 |
pkgUtils.installRPM(package, chrootID, True) |
| 94 |
- if package == "glibc": |
|
| 95 |
- self.adjustToolChain(chrootID) |
|
| 96 | 94 |
cmdUtils=CommandUtils() |
| 97 | 95 |
cmdUtils.runCommandInShell("rm -rf "+ chrootID+"/tools")
|
| 98 | 96 |
cmdUtils.runCommandInShell("rm "+ chrootID+"/"+constants.topDirPath+"/RPMS/x86_64/*")
|
| ... | ... |
@@ -4,15 +4,9 @@ from optparse import OptionParser |
| 4 | 4 |
import os.path |
| 5 | 5 |
from CommandUtils import CommandUtils |
| 6 | 6 |
from Logger import Logger |
| 7 |
-from SpecData import SerializableSpecObjectsUtils |
|
| 8 |
-from ChrootUtils import ChrootUtils |
|
| 9 |
-from PackageUtils import PackageUtils |
|
| 10 |
-from ToolChainUtils import ToolChainUtils |
|
| 11 | 7 |
from constants import constants |
| 12 |
-from PackageBuildDataGenerator import PackageBuildDataGenerator |
|
| 13 | 8 |
from PackageManager import PackageManager |
| 14 | 9 |
|
| 15 |
- |
|
| 16 | 10 |
def main(): |
| 17 | 11 |
usage = "Usage: %prog [options] <package name>" |
| 18 | 12 |
parser = OptionParser(usage) |
| ... | ... |
@@ -72,61 +66,9 @@ def main(): |
| 72 | 72 |
constants.initialize(options) |
| 73 | 73 |
|
| 74 | 74 |
listPackages1=["nano","swig","wget"] |
| 75 |
- |
|
| 76 |
- #tUtils = ToolChainUtils() |
|
| 77 |
- #tUtils.buildToolChain() |
|
| 75 |
+ |
|
| 78 | 76 |
pkgManager = PackageManager() |
| 79 | 77 |
pkgManager.buildPackages(listPackages) |
| 80 |
- |
|
| 81 |
- ''' |
|
| 82 |
- chrUtils=ChrootUtils(options) |
|
| 83 |
- returnVal,chrootID1=chrUtils.createChroot() |
|
| 84 |
- logger.info("Obtained chroot"+ chrootID1)
|
|
| 85 |
- if not returnVal: |
|
| 86 |
- return False |
|
| 87 |
- chrUtils.prepareChroot(chrootID1) |
|
| 88 |
- chrUtils.destroyChroot(chrootID1) |
|
| 89 |
- |
|
| 90 |
- |
|
| 91 |
- chrUtils1=ChrootUtils(options) |
|
| 92 |
- returnVal,chrootID=chrUtils1.createChroot() |
|
| 93 |
- logger.info("Obtained chroot"+ chrootID)
|
|
| 94 |
- if not returnVal: |
|
| 95 |
- return False |
|
| 96 |
- chrUtils1.prepareChroot(chrootID) |
|
| 97 |
- |
|
| 98 |
- tUtils.installToolChain(chrootID) |
|
| 99 |
- #chrUtils1.destroyChroot(chrootID) |
|
| 100 |
- ''' |
|
| 101 |
- |
|
| 102 |
- #tUtils=ToolChainUtils() |
|
| 103 |
- #tUtils.buildToolChain() |
|
| 104 |
- |
|
| 105 |
- #pkgUtils=PackageUtils() |
|
| 106 |
- #pkgUtils.buildPackage("nano")
|
|
| 107 |
- |
|
| 108 |
- |
|
| 109 |
- |
|
| 110 |
- |
|
| 111 |
- |
|
| 112 |
- |
|
| 113 |
-''' |
|
| 114 |
- package_builder = BuildSystem(options.source_path, options.spec_path, options.rpm_path, options.build_root, options.tools_path, options.log_path) |
|
| 115 |
- |
|
| 116 |
- returnVal = True |
|
| 117 |
- if options.clean_build: |
|
| 118 |
- returnVal=package_builder.doCleanBuild() |
|
| 119 |
- elif options.build_all: |
|
| 120 |
- returnVal=package_builder.buildAllPackages() |
|
| 121 |
- elif options.install_tool_chain: |
|
| 122 |
- returnVal=package_builder.installToolchain() |
|
| 123 |
- elif options.install_package: |
|
| 124 |
- if (len(args)) != 1: |
|
| 125 |
- parser.error("Incorrect number of arguments")
|
|
| 126 |
- returnVal=False |
|
| 127 |
- else: |
|
| 128 |
- returnVal=package_builder.installPackage(args[0],options.force_build) |
|
| 129 |
- return returnVal |
|
| 130 |
-''' |
|
| 78 |
+ |
|
| 131 | 79 |
if __name__=="__main__": |
| 132 | 80 |
main() |
| 133 | 81 |
\ No newline at end of file |