... | ... |
@@ -113,8 +113,13 @@ class ChrootUtils(object): |
113 | 113 |
return False |
114 | 114 |
cmdUtils=CommandUtils() |
115 | 115 |
cmdUtils.runCommandInShell("rm -rf "+chrootID) |
116 |
- ChrootUtils.lockForTrackingChroots.acquire() |
|
117 |
- ChrootUtils.lockForTrackingChroots.release() |
|
118 | 116 |
self.logger.info("Successfully destroyed chroot:"+chrootID) |
119 | 117 |
return True |
120 |
- |
|
118 |
+ |
|
119 |
+ def cleanUpActiveChroots(self): |
|
120 |
+ ChrootUtils.lockForTrackingChroots.acquire() |
|
121 |
+ listChroots=self.activeChroots[:] |
|
122 |
+ ChrootUtils.lockForTrackingChroots.release() |
|
123 |
+ for chrootID in listChroots: |
|
124 |
+ self.destroyChroot(chrootID) |
|
125 |
+ self.logger.info("Successfully destroyed all active chroots") |
... | ... |
@@ -35,7 +35,6 @@ class PackageManager(object): |
35 | 35 |
except: |
36 | 36 |
self.logger.error("unable to get sorted list") |
37 | 37 |
return False |
38 |
- |
|
39 | 38 |
return True |
40 | 39 |
|
41 | 40 |
def getRequiredPackages(self,package): |
... | ... |
@@ -101,16 +100,17 @@ class PackageManager(object): |
101 | 101 |
readyToLaunchMoreThreads = False |
102 | 102 |
listThreadsObjToRemove=[] |
103 | 103 |
for t in self.listThreads: |
104 |
- self.logger.info("Checking thread "+t) |
|
104 |
+ self.logger.debug("Checking thread "+t +", whether it is completed or not") |
|
105 | 105 |
#check if any thread is completed. If completed, we can start more threads. |
106 | 106 |
if self.mapOutputThread.has_key(t): |
107 | 107 |
output = self.mapOutputThread[t] |
108 | 108 |
self.logger.info("Output of thread "+t+" "+str(output)) |
109 | 109 |
if not output: |
110 | 110 |
self.logger.error("Thread "+ t+" is failed ") |
111 |
- #kill remaining Threads |
|
112 |
- return False,False |
|
111 |
+ self.logger.error("Unable to build package "+t) |
|
112 |
+ raise Exception("Package builder failed") |
|
113 | 113 |
else: |
114 |
+ self.logger.debug("Still running, not completed yet") |
|
114 | 115 |
readyToLaunchMoreThreads=True |
115 | 116 |
self.listPackagesToBuild.remove(t) |
116 | 117 |
self.listOfPackagesAlreadyBuilt.append(t) |
... | ... |
@@ -119,93 +119,106 @@ class PackageManager(object): |
119 | 119 |
self.listAvailableCyclicPackages.append(t) |
120 | 120 |
|
121 | 121 |
if not readyToLaunchMoreThreads: |
122 |
- return True,False |
|
122 |
+ return False |
|
123 | 123 |
|
124 | 124 |
for k in listThreadsObjToRemove: |
125 |
- self.listThreads.pop(k) |
|
126 |
- |
|
127 |
- return True,True |
|
125 |
+ tObj=self.listThreads.pop(k) |
|
126 |
+ if tObj.isAlive(): |
|
127 |
+ self.logger.info("Thread is alive") |
|
128 |
+ else: |
|
129 |
+ self.logger.info("Thread is dead") |
|
130 |
+ self.logger.info(tObj.isAlive()) |
|
131 |
+ |
|
132 |
+ return True |
|
128 | 133 |
|
129 | 134 |
def checkIfAnyThreadsAreHanged(self): |
130 | 135 |
currentTime = time.time() |
131 | 136 |
listThreadsHanged=[] |
132 | 137 |
for t in self.listThreads: |
133 |
- self.logger.info("Checking thread "+t) |
|
138 |
+ self.logger.debug("Checking thread "+t +", whether it is hanged or not") |
|
134 | 139 |
if not self.mapOutputThread.has_key(t): |
135 |
- self.logger.info("Calculating running time for thread "+t) |
|
140 |
+ self.logger.debug("Calculating running time for thread "+t) |
|
136 | 141 |
launchTime = self.mapThreadsLaunchTime[t] |
137 | 142 |
if (currentTime - launchTime) > 3600.0: |
138 | 143 |
listThreadsHanged.append(t) |
139 | 144 |
|
140 | 145 |
if len(listThreadsHanged) > 0: |
141 |
- self.logger.info("Looks like following threads are hanged") |
|
146 |
+ self.logger.info("Following threads are hanged") |
|
142 | 147 |
self.logger.info(listThreadsHanged) |
143 |
- #kill all threads |
|
144 |
- return False |
|
145 |
- |
|
146 |
- return True |
|
148 |
+ raise Exception("Threads are hanged") |
|
147 | 149 |
|
148 | 150 |
def waitTillNewThreadsCanBeSpawned(self): |
149 | 151 |
if len(self.listThreads) == 0: |
150 |
- return True |
|
151 |
- returnVal = False |
|
152 |
+ return |
|
152 | 153 |
while True: |
153 |
- sucess,Tfail = self.checkIfAnyThreadsAreCompleted() |
|
154 |
- if not sucess: |
|
155 |
- break |
|
156 |
- if sucess and Tfail: |
|
157 |
- returnVal = True |
|
158 |
- break |
|
159 |
- if not self.checkIfAnyThreadsAreHanged(): |
|
154 |
+ if self.checkIfAnyThreadsAreCompleted(): |
|
160 | 155 |
break |
156 |
+ self.checkIfAnyThreadsAreHanged() |
|
161 | 157 |
self.logger.info("Sleeping for 30 seconds") |
162 | 158 |
time.sleep(30) |
163 |
- return returnVal |
|
159 |
+ |
|
160 |
+ def buildToolChain(self): |
|
161 |
+ try: |
|
162 |
+ tUtils=ToolChainUtils() |
|
163 |
+ tUtils.buildToolChain() |
|
164 |
+ except Exception as e: |
|
165 |
+ self.logger.error("Unable to build tool chain") |
|
166 |
+ self.logger.error(e) |
|
167 |
+ return False |
|
164 | 168 |
|
165 |
- def buildPackages (self, listPackages): |
|
169 |
+ return True |
|
166 | 170 |
|
167 |
- tUtils=ToolChainUtils() |
|
168 |
- tUtils.buildToolChain() |
|
171 |
+ def buildPackages (self, listPackages): |
|
169 | 172 |
|
173 |
+ if not self.buildToolChain(): |
|
174 |
+ return False |
|
175 |
+ |
|
170 | 176 |
returnVal=self.calculateParams(listPackages) |
171 | 177 |
if not returnVal: |
172 | 178 |
self.logger.error("Unable to set paramaters. Terminating the package manager.") |
173 | 179 |
return False |
174 |
- returnVal = True |
|
175 |
- while len(self.listPackagesToBuild) > 0: |
|
176 |
- #Free some threads to launch next threads |
|
177 |
- if not self.waitTillNewThreadsCanBeSpawned(): |
|
178 |
- returnVal = False |
|
179 |
- break |
|
180 |
- |
|
181 |
- listOfPackagesCanBeBuild=self.findNextPackageToBuild() |
|
182 |
- if len(listOfPackagesCanBeBuild) == 0 and len(self.listPackagesToBuild) != 0: |
|
183 |
- self.logger.info("Waiting for current threads to complete to launch building new packages") |
|
184 |
- |
|
185 |
- for pkg in listOfPackagesCanBeBuild: |
|
186 |
- currentTime = time.time() |
|
187 |
- pkgBuilder = PackageBuilder(self.mapPackageToCycle,self.listAvailableCyclicPackages,"build-"+pkg,constants.logPath) |
|
188 |
- t = threading.Thread(target=pkgBuilder.buildPackageThreadAPI,args=(pkg,self.mapOutputThread,pkg)) |
|
189 |
- self.listThreads[pkg]=t |
|
190 |
- self.mapThreadsLaunchTime[pkg]=currentTime |
|
191 |
- self.logger.info("Launching thread for package:"+pkg) |
|
192 |
- t.start() |
|
193 |
- self.logger.info("Started the thread for "+pkg) |
|
194 |
- |
|
195 |
- if len(self.listThreads) == 0 and len(self.listPackagesToBuild) != 0: |
|
196 |
- self.logger.error("Following packages are waiting for unknown package") |
|
197 |
- self.logger.error(self.listPackagesToBuild) |
|
198 |
- self.logger.error("Unexpected error") |
|
199 |
- returnVal = False |
|
200 |
- break |
|
201 |
- |
|
202 |
- if not returnVal: |
|
203 |
- self.logger.error("Failed and exited gracefully") |
|
204 |
- return False |
|
205 | 180 |
|
206 |
- self.logger.info( "Successfully built all the packages") |
|
207 |
- return True |
|
181 |
+ returnVal = True |
|
182 |
+ try: |
|
183 |
+ while len(self.listPackagesToBuild) > 0: |
|
184 |
+ #Free some threads to launch next threads |
|
185 |
+ self.waitTillNewThreadsCanBeSpawned() |
|
186 |
+ |
|
187 |
+ listOfPackagesCanBeBuild=self.findNextPackageToBuild() |
|
188 |
+ if len(listOfPackagesCanBeBuild) == 0 and len(self.listPackagesToBuild) != 0: |
|
189 |
+ self.logger.info("Waiting for current threads to complete to launch building new packages") |
|
190 |
+ |
|
191 |
+ for pkg in listOfPackagesCanBeBuild: |
|
192 |
+ currentTime = time.time() |
|
193 |
+ pkgBuilder = PackageBuilder(self.mapPackageToCycle,self.listAvailableCyclicPackages,"build-"+pkg,constants.logPath) |
|
194 |
+ t = threading.Thread(target=pkgBuilder.buildPackageThreadAPI,args=(pkg,self.mapOutputThread,pkg)) |
|
195 |
+ self.listThreads[pkg]=t |
|
196 |
+ self.mapThreadsLaunchTime[pkg]=currentTime |
|
197 |
+ self.logger.info("Launching thread for package:"+pkg) |
|
198 |
+ t.start() |
|
199 |
+ self.logger.info("Started the thread for "+pkg) |
|
208 | 200 |
|
201 |
+ if len(self.listThreads) == 0 and len(self.listPackagesToBuild) != 0: |
|
202 |
+ self.logger.error("Following packages are waiting for unknown package") |
|
203 |
+ self.logger.error(self.listPackagesToBuild) |
|
204 |
+ raise Exception("Invalid Schedule order") |
|
205 |
+ |
|
206 |
+ self.logger.info( "Successfully built all the packages") |
|
207 |
+ |
|
208 |
+ except Exception as e: |
|
209 |
+ self.logger.error(str(e)) |
|
210 |
+ self.logger.error("Caught exception.") |
|
211 |
+ self.logger.error("Failed and exited gracefully") |
|
212 |
+ returnVal = False |
|
213 |
+ finally: |
|
214 |
+ if len(self.listThreads) > 0: |
|
215 |
+ self.killAllThreads() |
|
216 |
+ return returnVal |
|
217 |
+ |
|
218 |
+ def killAllThreads(self): |
|
219 |
+ self.logger.info("Killing all remaining threads") |
|
220 |
+ return |
|
221 |
+ |
|
209 | 222 |
def findNextPackageToBuild(self): |
210 | 223 |
listOfPackagesNextToBuild=[] |
211 | 224 |
self.logger.info("Checking for next possible packages to build") |
... | ... |
@@ -84,15 +84,13 @@ class ToolChainUtils(object): |
84 | 84 |
if chrootID is not None: |
85 | 85 |
chrUtils.destroyChroot(chrootID) |
86 | 86 |
|
87 |
- #tool chain should be updated before calling this method |
|
87 |
+ #Tool chain should be built before calling this method |
|
88 | 88 |
def installToolChain(self,chrootID): |
89 | 89 |
self.logger.info("Installing toolchain.....") |
90 | 90 |
self.prepareChroot(chrootID,"minimal") |
91 | 91 |
pkgUtils= PackageUtils(self.logName,self.logPath) |
92 | 92 |
for package in self.listPkgsToInstallToolChain: |
93 | 93 |
pkgUtils.installRPM(package, chrootID, True) |
94 |
- if package == "glibc": |
|
95 |
- self.adjustToolChain(chrootID) |
|
96 | 94 |
cmdUtils=CommandUtils() |
97 | 95 |
cmdUtils.runCommandInShell("rm -rf "+ chrootID+"/tools") |
98 | 96 |
cmdUtils.runCommandInShell("rm "+ chrootID+"/"+constants.topDirPath+"/RPMS/x86_64/*") |
... | ... |
@@ -4,15 +4,9 @@ from optparse import OptionParser |
4 | 4 |
import os.path |
5 | 5 |
from CommandUtils import CommandUtils |
6 | 6 |
from Logger import Logger |
7 |
-from SpecData import SerializableSpecObjectsUtils |
|
8 |
-from ChrootUtils import ChrootUtils |
|
9 |
-from PackageUtils import PackageUtils |
|
10 |
-from ToolChainUtils import ToolChainUtils |
|
11 | 7 |
from constants import constants |
12 |
-from PackageBuildDataGenerator import PackageBuildDataGenerator |
|
13 | 8 |
from PackageManager import PackageManager |
14 | 9 |
|
15 |
- |
|
16 | 10 |
def main(): |
17 | 11 |
usage = "Usage: %prog [options] <package name>" |
18 | 12 |
parser = OptionParser(usage) |
... | ... |
@@ -72,61 +66,9 @@ def main(): |
72 | 72 |
constants.initialize(options) |
73 | 73 |
|
74 | 74 |
listPackages1=["nano","swig","wget"] |
75 |
- |
|
76 |
- #tUtils = ToolChainUtils() |
|
77 |
- #tUtils.buildToolChain() |
|
75 |
+ |
|
78 | 76 |
pkgManager = PackageManager() |
79 | 77 |
pkgManager.buildPackages(listPackages) |
80 |
- |
|
81 |
- ''' |
|
82 |
- chrUtils=ChrootUtils(options) |
|
83 |
- returnVal,chrootID1=chrUtils.createChroot() |
|
84 |
- logger.info("Obtained chroot"+ chrootID1) |
|
85 |
- if not returnVal: |
|
86 |
- return False |
|
87 |
- chrUtils.prepareChroot(chrootID1) |
|
88 |
- chrUtils.destroyChroot(chrootID1) |
|
89 |
- |
|
90 |
- |
|
91 |
- chrUtils1=ChrootUtils(options) |
|
92 |
- returnVal,chrootID=chrUtils1.createChroot() |
|
93 |
- logger.info("Obtained chroot"+ chrootID) |
|
94 |
- if not returnVal: |
|
95 |
- return False |
|
96 |
- chrUtils1.prepareChroot(chrootID) |
|
97 |
- |
|
98 |
- tUtils.installToolChain(chrootID) |
|
99 |
- #chrUtils1.destroyChroot(chrootID) |
|
100 |
- ''' |
|
101 |
- |
|
102 |
- #tUtils=ToolChainUtils() |
|
103 |
- #tUtils.buildToolChain() |
|
104 |
- |
|
105 |
- #pkgUtils=PackageUtils() |
|
106 |
- #pkgUtils.buildPackage("nano") |
|
107 |
- |
|
108 |
- |
|
109 |
- |
|
110 |
- |
|
111 |
- |
|
112 |
- |
|
113 |
-''' |
|
114 |
- package_builder = BuildSystem(options.source_path, options.spec_path, options.rpm_path, options.build_root, options.tools_path, options.log_path) |
|
115 |
- |
|
116 |
- returnVal = True |
|
117 |
- if options.clean_build: |
|
118 |
- returnVal=package_builder.doCleanBuild() |
|
119 |
- elif options.build_all: |
|
120 |
- returnVal=package_builder.buildAllPackages() |
|
121 |
- elif options.install_tool_chain: |
|
122 |
- returnVal=package_builder.installToolchain() |
|
123 |
- elif options.install_package: |
|
124 |
- if (len(args)) != 1: |
|
125 |
- parser.error("Incorrect number of arguments") |
|
126 |
- returnVal=False |
|
127 |
- else: |
|
128 |
- returnVal=package_builder.installPackage(args[0],options.force_build) |
|
129 |
- return returnVal |
|
130 |
-''' |
|
78 |
+ |
|
131 | 79 |
if __name__=="__main__": |
132 | 80 |
main() |
133 | 81 |
\ No newline at end of file |