Change-Id: I60d337366dd37a939d531dc89ac4319239beac01
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/4225
Tested-by: gerrit-photon <photon-checkins@vmware.com>
Reviewed-by: Alexey Makhalov <amakhalov@vmware.com>
... | ... |
@@ -7,13 +7,14 @@ from Queue import PriorityQueue |
7 | 7 |
from SpecData import SPECS |
8 | 8 |
|
9 | 9 |
class Scheduler(object): |
10 |
- |
|
10 |
+ |
|
11 | 11 |
lock=threading.Lock() |
12 | 12 |
listOfAlreadyBuiltPackages=[] |
13 | 13 |
listOfPackagesToBuild=[] |
14 | 14 |
listOfPackagesCurrentlyBuilding=[] |
15 | 15 |
sortedList=[] |
16 |
- listOfPackagesNextToBuild=PriorityQueue() |
|
16 |
+ listOfPackagesNextToBuild=[] |
|
17 |
+ queueOfPackagesNextToBuild=PriorityQueue() |
|
17 | 18 |
listOfFailedPackages=[] |
18 | 19 |
alldependencyGraph = {} |
19 | 20 |
dependencyGraph = {} |
... | ... |
@@ -23,14 +24,14 @@ class Scheduler(object): |
23 | 23 |
logger=None |
24 | 24 |
event=None |
25 | 25 |
stopScheduling=False |
26 |
- |
|
26 |
+ |
|
27 | 27 |
@staticmethod |
28 | 28 |
def setEvent(event): |
29 | 29 |
Scheduler.event=event |
30 |
- |
|
30 |
+ |
|
31 | 31 |
@staticmethod |
32 | 32 |
def setLog(logName,logPath): |
33 |
- Scheduler.logger = Logger.getLogger(logName, logPath) |
|
33 |
+ Scheduler.logger = Logger.getLogger(logName, logPath) |
|
34 | 34 |
|
35 | 35 |
@staticmethod |
36 | 36 |
def getBuildRequiredPackages(package): |
... | ... |
@@ -107,12 +108,7 @@ class Scheduler(object): |
107 | 107 |
|
108 | 108 |
@staticmethod |
109 | 109 |
def setPriorities(): |
110 |
- if constants.packageWeightsPath == None: |
|
111 |
- Scheduler.logger.info("Priority Scheduler disabled") |
|
112 |
- Scheduler.isPriorityScheduler = 0 |
|
113 |
- else: |
|
114 |
- Scheduler.parseWeights() |
|
115 |
- |
|
110 |
+ Scheduler.parseWeights() |
|
116 | 111 |
for package in Scheduler.sortedList: |
117 | 112 |
Scheduler.dependencyGraph[package] = {} |
118 | 113 |
Scheduler.alldependencyGraph[package] = {} |
... | ... |
@@ -133,37 +129,41 @@ class Scheduler(object): |
133 | 133 |
Scheduler.logger.info("set Priorities: Priority of all packages") |
134 | 134 |
Scheduler.logger.info(Scheduler.priorityMap) |
135 | 135 |
|
136 |
- |
|
137 | 136 |
@staticmethod |
138 | 137 |
def setParams(sortedList,listOfAlreadyBuiltPackages): |
138 |
+ if constants.packageWeightsPath == None: |
|
139 |
+ Scheduler.logger.info("Priority Scheduler disabled") |
|
140 |
+ Scheduler.isPriorityScheduler = 0 |
|
139 | 141 |
Scheduler.sortedList=sortedList |
140 | 142 |
Scheduler.listOfAlreadyBuiltPackages=listOfAlreadyBuiltPackages |
141 | 143 |
for x in Scheduler.sortedList: |
142 | 144 |
if x not in Scheduler.listOfAlreadyBuiltPackages or x in constants.testForceRPMS: |
143 | 145 |
Scheduler.listOfPackagesToBuild.append(x) |
144 | 146 |
Scheduler.listOfPackagesCurrentlyBuilding=[] |
145 |
- Scheduler.listOfPackagesNextToBuild=[] |
|
147 |
+ Scheduler.queueOfPackagesNextToBuild=[] |
|
146 | 148 |
Scheduler.listOfFailedPackages=[] |
147 |
- Scheduler.setPriorities() |
|
148 |
- |
|
149 |
+ if Scheduler.isPriorityScheduler == 1: |
|
150 |
+ Scheduler.setPriorities() |
|
151 |
+ |
|
149 | 152 |
@staticmethod |
150 | 153 |
def getRequiredPackages(package): |
151 | 154 |
listRequiredRPMPackages=[] |
152 | 155 |
listRequiredRPMPackages.extend(SPECS.getData().getBuildRequiresForPackage(package)) |
153 | 156 |
listRequiredRPMPackages.extend(SPECS.getData().getRequiresAllForPackage(package)) |
154 |
- |
|
157 |
+ |
|
155 | 158 |
listRequiredPackages=[] |
156 | 159 |
|
157 | 160 |
for pkg in listRequiredRPMPackages: |
158 | 161 |
basePkg=SPECS.getData().getSpecName(pkg) |
159 | 162 |
if basePkg not in listRequiredPackages: |
160 | 163 |
listRequiredPackages.append(basePkg) |
161 |
- |
|
164 |
+ |
|
162 | 165 |
return listRequiredPackages |
163 |
- |
|
166 |
+ |
|
164 | 167 |
@staticmethod |
165 | 168 |
def __getListNextPackagesReadyToBuild(): |
166 |
- listOfPackagesNextToBuild=PriorityQueue() |
|
169 |
+ queueOfPackagesNextToBuild=PriorityQueue() |
|
170 |
+ listOfPackagesNextToBuild=[] |
|
167 | 171 |
Scheduler.logger.info("Checking for next possible packages to build") |
168 | 172 |
for pkg in Scheduler.listOfPackagesToBuild: |
169 | 173 |
if pkg in Scheduler.listOfPackagesCurrentlyBuilding: |
... | ... |
@@ -178,9 +178,13 @@ class Scheduler(object): |
178 | 178 |
Scheduler.logger.info(reqPkg+" is not available. So we cannot build "+ pkg +" at this moment.") |
179 | 179 |
break |
180 | 180 |
if canBuild: |
181 |
- listOfPackagesNextToBuild.put((-Scheduler.priorityMap[pkg], pkg)) |
|
182 | 181 |
Scheduler.logger.info("Adding "+ pkg +" to the schedule list") |
183 |
- return listOfPackagesNextToBuild |
|
182 |
+ if Scheduler.isPriorityScheduler: |
|
183 |
+ queueOfPackagesNextToBuild.put((-Scheduler.priorityMap[pkg], pkg)) |
|
184 |
+ return queueOfPackagesNextToBuild |
|
185 |
+ else: |
|
186 |
+ listOfPackagesNextToBuild.append(pkg) |
|
187 |
+ return listOfPackagesNextToBuild |
|
184 | 188 |
|
185 | 189 |
@staticmethod |
186 | 190 |
def getNextPackageToBuild(): |
... | ... |
@@ -196,62 +200,74 @@ class Scheduler(object): |
196 | 196 |
if Scheduler.event is not None: |
197 | 197 |
Scheduler.event.set() |
198 | 198 |
|
199 |
- try: |
|
200 |
- if Scheduler.listOfPackagesNextToBuild.qsize() == 0: |
|
201 |
- listOfPackagesNextToBuild = Scheduler.__getListNextPackagesReadyToBuild() |
|
202 |
- Scheduler.listOfPackagesNextToBuild = listOfPackagesNextToBuild |
|
203 |
- except: |
|
204 |
- if len(Scheduler.listOfPackagesNextToBuild) == 0: |
|
205 |
- listOfPackagesNextToBuild = Scheduler.__getListNextPackagesReadyToBuild() |
|
206 |
- Scheduler.listOfPackagesNextToBuild = listOfPackagesNextToBuild |
|
207 |
- |
|
208 |
- if Scheduler.listOfPackagesNextToBuild.qsize() == 0: |
|
209 |
- Scheduler.logger.info("Released scheduler lock") |
|
210 |
- Scheduler.lock.release() |
|
211 |
- return None |
|
199 |
+ if 0 == Scheduler.isPriorityScheduler: |
|
200 |
+ if Scheduler.listOfPackagesNextToBuild is None or len(Scheduler.listOfPackagesNextToBuild) == 0: |
|
201 |
+ listOfPackagesNextToBuild=Scheduler.__getListNextPackagesReadyToBuild() |
|
202 |
+ Scheduler.listOfPackagesNextToBuild=listOfPackagesNextToBuild |
|
203 |
+ |
|
204 |
+ if Scheduler.listOfPackagesNextToBuild is None or len(Scheduler.listOfPackagesNextToBuild) == 0: |
|
205 |
+ Scheduler.logger.info("Released scheduler lock") |
|
206 |
+ Scheduler.lock.release() |
|
207 |
+ return None |
|
208 |
+ |
|
209 |
+ package=Scheduler.listOfPackagesNextToBuild.pop(0) |
|
210 |
+ |
|
211 |
+ if len(Scheduler.listOfPackagesNextToBuild) > 0: |
|
212 |
+ ThreadPool.ThreadPool.activateWorkerThreads(len(Scheduler.listOfPackagesNextToBuild)) |
|
213 |
+ else: |
|
214 |
+ try: |
|
215 |
+ if Scheduler.queueOfPackagesNextToBuild.qsize() == 0: |
|
216 |
+ queueOfPackagesNextToBuild = Scheduler.__getListNextPackagesReadyToBuild() |
|
217 |
+ Scheduler.queueOfPackagesNextToBuild = queueOfPackagesNextToBuild |
|
218 |
+ except: |
|
219 |
+ if len(Scheduler.queueOfPackagesNextToBuild) == 0: |
|
220 |
+ queueOfPackagesNextToBuild = Scheduler.__getListNextPackagesReadyToBuild() |
|
221 |
+ Scheduler.queueOfPackagesNextToBuild = queueOfPackagesNextToBuild |
|
222 |
+ |
|
223 |
+ if Scheduler.queueOfPackagesNextToBuild.qsize() == 0: |
|
224 |
+ Scheduler.logger.info("Released scheduler lock") |
|
225 |
+ Scheduler.lock.release() |
|
226 |
+ return None |
|
227 |
+ |
|
228 |
+ packageTup=Scheduler.queueOfPackagesNextToBuild.get() |
|
229 |
+ |
|
230 |
+ if packageTup[0] == 0 and Scheduler.isPriorityScheduler == 1: |
|
231 |
+ queueOfPackagesNextToBuild = Scheduler.__getListNextPackagesReadyToBuild() |
|
232 |
+ Scheduler.queueOfPackagesNextToBuild = queueOfPackagesNextToBuild |
|
233 |
+ packageTup = Scheduler.listOfPackagesNextToBuild.get() |
|
234 |
+ |
|
235 |
+ package = packageTup[1] |
|
236 |
+ Scheduler.logger.info("PackagesNextToBuild " + str(packageTup)) |
|
237 |
+ if Scheduler.queueOfPackagesNextToBuild.qsize() > 0: |
|
238 |
+ ThreadPool.ThreadPool.activateWorkerThreads(Scheduler.queueOfPackagesNextToBuild.qsize()) |
|
212 | 239 |
|
213 |
- packageTup=Scheduler.listOfPackagesNextToBuild.get() |
|
214 |
- |
|
215 |
- if packageTup[0] == 0 and Scheduler.isPriorityScheduler == 1: |
|
216 |
- listOfPackagesNextToBuild = Scheduler.__getListNextPackagesReadyToBuild() |
|
217 |
- Scheduler.listOfPackagesNextToBuild = listOfPackagesNextToBuild |
|
218 |
- if Scheduler.listOfPackagesNextToBuild.qsize() == 0: |
|
219 |
- Scheduler.logger.info("Released scheduler lock") |
|
220 |
- Scheduler.lock.release() |
|
221 |
- return None |
|
222 |
- packageTup = Scheduler.listOfPackagesNextToBuild.get() |
|
223 |
- |
|
224 |
- package = packageTup[1] |
|
225 |
- Scheduler.logger.info("PackagesNextToBuild " + str(packageTup)) |
|
226 |
- if Scheduler.listOfPackagesNextToBuild.qsize() > 0: |
|
227 |
- ThreadPool.ThreadPool.activateWorkerThreads(Scheduler.listOfPackagesNextToBuild.qsize()) |
|
228 | 240 |
Scheduler.logger.info("Released scheduler lock") |
229 | 241 |
Scheduler.lock.release() |
230 | 242 |
Scheduler.listOfPackagesCurrentlyBuilding.append(package) |
231 | 243 |
Scheduler.listOfPackagesToBuild.remove(package) |
232 | 244 |
return package |
233 |
- |
|
245 |
+ |
|
234 | 246 |
#can be synchronized TODO |
235 | 247 |
@staticmethod |
236 | 248 |
def notifyPackageBuildCompleted(package): |
237 | 249 |
if package in Scheduler.listOfPackagesCurrentlyBuilding: |
238 | 250 |
Scheduler.listOfPackagesCurrentlyBuilding.remove(package) |
239 | 251 |
Scheduler.listOfAlreadyBuiltPackages.append(package) |
240 |
- |
|
241 |
- |
|
252 |
+ |
|
253 |
+ |
|
242 | 254 |
#can be synchronized TODO |
243 | 255 |
@staticmethod |
244 | 256 |
def notifyPackageBuildFailed(package): |
245 | 257 |
if package in Scheduler.listOfPackagesCurrentlyBuilding: |
246 | 258 |
Scheduler.listOfPackagesCurrentlyBuilding.remove(package) |
247 | 259 |
Scheduler.listOfFailedPackages.append(package) |
248 |
- |
|
260 |
+ |
|
249 | 261 |
@staticmethod |
250 | 262 |
def isAllPackagesBuilt(): |
251 | 263 |
if len(Scheduler.listOfPackagesToBuild) == 0 : |
252 | 264 |
return True |
253 | 265 |
return False |
254 |
- |
|
266 |
+ |
|
255 | 267 |
@staticmethod |
256 | 268 |
def isAnyPackagesFailedToBuild(): |
257 | 269 |
if len(Scheduler.listOfFailedPackages) != 0: |