Use upward dependencies instead of downward.
Support BASE_COMMIT target for `make clean-stage-for-incremental-build`
Change-Id: I28a5e791c89cd7551074b47de967a7112515d28b
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/5803
Reviewed-by: Anish Swaminathan <anishs@vmware.com>
Tested-by: Anish Swaminathan <anishs@vmware.com>
... | ... |
@@ -235,13 +235,19 @@ who-needs: |
235 | 235 |
@cd $(PHOTON_SPECDEPS_DIR) && \ |
236 | 236 |
$(PHOTON_SPECDEPS) -s $(PHOTON_SPECS_DIR) -i who-needs -p $(pkg) |
237 | 237 |
|
238 |
-# This target analyzes top git commit and removes staged RPMS that can be affected |
|
239 |
-# by this change and should be rebuilt as part of incremental build support |
|
238 |
+# Input args: BASE_COMMIT= (optional) |
|
239 |
+# |
|
240 |
+# This target removes staged RPMS that can be affected by change(s) and should |
|
241 |
+# be rebuilt as part of incremental build support |
|
240 | 242 |
# For every spec file touched - remove all upward dependent packages (rpms) |
241 | 243 |
# If support folder was touched - do full build |
244 |
+# |
|
245 |
+# The analyzed changes are: |
|
246 |
+# - commits from BASE_COMMIT to HEAD (if BASE_COMMIT= parameter is specified) |
|
247 |
+# - local changes (if no commits specified) |
|
242 | 248 |
clean-stage-for-incremental-build: |
243 |
- @test -n "$$(git show @ SPECS)" && $(PHOTON_SPECDEPS) -s $(PHOTON_SPECS_DIR) -i remove-upward-deps -p $$(echo `git show --pretty="format:" --name-only @ | grep .spec | xargs -n1 basename 2>/dev/null` | tr ' ' :) |
|
244 |
- @test -n "$$(git show @ support)" && $(RM) -rf $(PHOTON_RPMS_DIR) |
|
249 |
+ @test -n "$$(git diff --name-only $(BASE_COMMIT) @ | grep SPECS)" && $(PHOTON_SPECDEPS) -s $(PHOTON_SPECS_DIR) -i remove-upward-deps -p $$(echo `git diff --name-only $(BASE_COMMIT) @ | grep .spec | xargs -n1 basename 2>/dev/null` | tr ' ' :) ||: |
|
250 |
+ @test -n "$$(git diff --name-only $(BASE_COMMIT) @ | grep support)" && $(RM) -rf $(PHOTON_RPMS_DIR) ||: |
|
245 | 251 |
|
246 | 252 |
packages: check-docker-py check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_XRPMS) $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN) generate-dep-lists |
247 | 253 |
@echo "Building all RPMS..." |
... | ... |
@@ -140,12 +140,10 @@ class PackageBuildDataGenerator(object): |
140 | 140 |
nextPackagesToConstructGraph.update(dependentPackages) |
141 | 141 |
|
142 | 142 |
if addRunTimeGraph: |
143 |
- packageName, packageVersion = StringUtils.splitPackageNameAndVersion(basePackage) |
|
144 |
- rpmPackages = SPECS.getData().getPackages(packageName, packageVersion) |
|
145 | 143 |
dependentPackages = set() |
146 |
- for rpmPkg in rpmPackages: |
|
147 |
- dependentRpmPackages = SPECS.getData().getRequiresAllForPackage(rpmPkg, packageVersion) |
|
148 |
- self.__runTimeDependencyGraph[rpmPkg+"-"+packageVersion] = copy.copy(set(dependentRpmPackages)) |
|
144 |
+ for rpmPkg in SPECS.getData().getPackagesForPkg(basePackage): |
|
145 |
+ dependentRpmPackages = SPECS.getData().getRequiresAllForPkg(rpmPkg) |
|
146 |
+ self.__runTimeDependencyGraph[rpmPkg] = copy.copy(set(dependentRpmPackages)) |
|
149 | 147 |
for pkg in dependentRpmPackages: |
150 | 148 |
dependentPackages.add(SPECS.getData().getBasePkg(pkg)) |
151 | 149 |
nextPackagesToConstructGraph.update(dependentPackages) |
... | ... |
@@ -412,26 +412,6 @@ class PackageUtils(object): |
412 | 412 |
rpmDestDir = rpmDir + "/" + arch |
413 | 413 |
return rpmDestDir |
414 | 414 |
|
415 |
- def _getProperVersion(self,package,parseSpecObj): |
|
416 |
- listOfVersionObjs=SPECS.getData().getSpecObj(package) |
|
417 |
- for num in listOfVersionObjs: |
|
418 |
- if parseSpecObj.compare == ">=": |
|
419 |
- if LooseVersion(num.version) >= LooseVersion(parseSpecObj.version): |
|
420 |
- return num.version |
|
421 |
- elif parseSpecObj.compare == "<=": |
|
422 |
- if LooseVersion(num.version) <= LooseVersion(parseSpecObj.version): |
|
423 |
- return num.version |
|
424 |
- elif parseSpecObj.compare == "=": |
|
425 |
- if LooseVersion(num.version) == LooseVersion(parseSpecObj.version): |
|
426 |
- return num.version |
|
427 |
- elif parseSpecObj.compare == "<": |
|
428 |
- if LooseVersion(num.version) < LooseVersion(parseSpecObj.version): |
|
429 |
- return num.version |
|
430 |
- elif parseSpecObj.compare == ">": |
|
431 |
- if LooseVersion(num.version) > LooseVersion(parseSpecObj.version): |
|
432 |
- return num.version |
|
433 |
- return "*" |
|
434 |
- |
|
435 | 415 |
def _copyRPM(self, rpmFile, destDir): |
436 | 416 |
cmdUtils = CommandUtils() |
437 | 417 |
rpmName = os.path.basename(rpmFile) |
... | ... |
@@ -234,8 +234,11 @@ class SpecObjectsUtils(object): |
234 | 234 |
return self._getSpecObjField(package, version, field=lambda x : x.listPackages) |
235 | 235 |
|
236 | 236 |
def getPackagesForPkg(self, pkg): |
237 |
+ pkgs=[] |
|
237 | 238 |
package, version = StringUtils.splitPackageNameAndVersion(pkg) |
238 |
- return self.getPackages(package, version) |
|
239 |
+ for p in self.getPackages(package, version): |
|
240 |
+ pkgs.append(p+"-"+version) |
|
241 |
+ return pkgs |
|
239 | 242 |
|
240 | 243 |
def getRPMPackages(self, package, version): |
241 | 244 |
return self._getSpecObjField(package, version, field=lambda x : x.listRPMPackages) |
... | ... |
@@ -394,26 +397,29 @@ class SpecDependencyGenerator(object): |
394 | 394 |
parent[depPkg] = specPkg |
395 | 395 |
depQue.put(depPkg) |
396 | 396 |
|
397 |
- def findTotalWhoNeedsToBuild(self, depQue, whoNeedsBuild): |
|
398 |
- while not depQue.empty(): |
|
399 |
- specPkg = depQue.get() |
|
400 |
- listPackagesRequiredToBuild = SPECS.getData().getBuildRequiresForPkg(specPkg) |
|
401 |
- for depPkg in listPackagesRequiredToBuild: |
|
402 |
- depSpecPkg = SPECS.getData().getBasePkg(depPkg) |
|
403 |
- if depSpecPkg not in whoNeedsBuild: |
|
404 |
- whoNeedsBuild.append(depSpecPkg) |
|
405 |
- depQue.put(depSpecPkg) |
|
406 |
- |
|
407 |
- def findTotalWhoNeeds(self, depQue, whoNeeds): |
|
408 |
- while not depQue.empty(): |
|
409 |
- specPkg = depQue.get() |
|
410 |
- listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(specPkg) |
|
411 |
- listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(specPkg)) |
|
412 |
- for depPkg in listPackagesRequired: |
|
413 |
- depSpecPkg = SPECS.getData().getBasePkg(depPkg) |
|
414 |
- if depSpecPkg not in whoNeeds: |
|
415 |
- whoNeeds.append(depSpecPkg) |
|
416 |
- depQue.put(depSpecPkg) |
|
397 |
+ def getBasePackagesRequired(self, pkg): |
|
398 |
+ listBasePackagesRequired=[] |
|
399 |
+ listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg) |
|
400 |
+ listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg)) |
|
401 |
+ for p in listPackagesRequired: |
|
402 |
+ basePkg = SPECS.getData().getBasePkg(p) |
|
403 |
+ if basePkg not in listBasePackagesRequired: |
|
404 |
+ listBasePackagesRequired.append(basePkg) |
|
405 |
+ return listBasePackagesRequired |
|
406 |
+ |
|
407 |
+ |
|
408 |
+ def findTotalWhoNeeds(self, depList, whoNeeds): |
|
409 |
+ while depList: |
|
410 |
+ pkg = depList.pop(0) |
|
411 |
+ for depPackage in SPECS.getData().getListPackages(): |
|
412 |
+ for version in SPECS.getData().getVersions(depPackage): |
|
413 |
+ depBasePkg = depPackage+"-"+version |
|
414 |
+ if depBasePkg in whoNeeds: |
|
415 |
+ continue |
|
416 |
+ if pkg in self.getBasePackagesRequired(depBasePkg): |
|
417 |
+ whoNeeds.append(depBasePkg) |
|
418 |
+ if depBasePkg not in depList: |
|
419 |
+ depList.append(depBasePkg) |
|
417 | 420 |
|
418 | 421 |
def printTree(self, children, curParent, depth): |
419 | 422 |
if curParent in children: |
... | ... |
@@ -430,8 +436,7 @@ class SpecDependencyGenerator(object): |
430 | 430 |
def updateLevels(self, mapDependencies, inPkg, parent, level): |
431 | 431 |
listPackages = SPECS.getData().getPackagesForPkg(inPkg) |
432 | 432 |
for depPkg in SPECS.getData().getRequiresForPkg(inPkg): |
433 |
- package, version = StringUtils.splitPackageNameAndVersion(depPkg) |
|
434 |
- if package in listPackages: |
|
433 |
+ if depPkg in listPackages: |
|
435 | 434 |
continue |
436 | 435 |
if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1: |
437 | 436 |
mapDependencies[depPkg] = level + 1 |
... | ... |
@@ -490,7 +495,7 @@ class SpecDependencyGenerator(object): |
490 | 490 |
def process(self, inputType, inputValue, displayOption, outputFile=None): |
491 | 491 |
whoNeedsList = [] |
492 | 492 |
inputPackages = [] |
493 |
- whoNeedsBuild = [] |
|
493 |
+ whatNeedsBuild = [] |
|
494 | 494 |
mapDependencies = {} |
495 | 495 |
parent = {} |
496 | 496 |
if inputType == "pkg" or inputType == "json": |
... | ... |
@@ -505,14 +510,13 @@ class SpecDependencyGenerator(object): |
505 | 505 |
else: |
506 | 506 |
return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent) |
507 | 507 |
elif inputType == "remove-upward-deps": |
508 |
- depQue = queue.Queue() |
|
508 |
+ depList = [] |
|
509 | 509 |
for specFile in inputValue.split(":"): |
510 | 510 |
if specFile in SPECS.getData().mapSpecFileNameToSpecObj: |
511 | 511 |
specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile] |
512 | 512 |
whoNeedsList.append(specObj.name+"-"+specObj.version) |
513 |
- for package in specObj.listPackages: |
|
514 |
- depQue.put(package+"-"+specObj.version) |
|
515 |
- self.findTotalWhoNeeds(depQue, whoNeedsList) |
|
513 |
+ depList.append(specObj.name+"-"+specObj.version) |
|
514 |
+ self.findTotalWhoNeeds(depList, whoNeedsList) |
|
516 | 515 |
return whoNeedsList |
517 | 516 |
|
518 | 517 |
elif inputType == "who-needs": |
... | ... |
@@ -525,10 +529,3 @@ class SpecDependencyGenerator(object): |
525 | 525 |
whoNeedsList.append(depPkg) |
526 | 526 |
print (whoNeedsList) |
527 | 527 |
return whoNeedsList |
528 |
- elif inputType == "who-needs-build": |
|
529 |
- depQue = queue.Queue() |
|
530 |
- depQue.put(inputValue) |
|
531 |
- self.findTotalWhoNeedsToBuild(depQue, whoNeedsBuild) |
|
532 |
- print ("Following specs need to be build again") |
|
533 |
- print (whoNeedsBuild) |
|
534 |
- return whoNeedsBuild |
... | ... |
@@ -59,8 +59,7 @@ def main(): |
59 | 59 |
cmdUtils.runCommandInShell2("rm -f "+rpmFile) |
60 | 60 |
# To display/print package dependencies on console |
61 | 61 |
elif (options.input_type == "pkg" or |
62 |
- options.input_type == "who-needs" or |
|
63 |
- options.input_type == "who-needs-build"): |
|
62 |
+ options.input_type == "who-needs"): |
|
64 | 63 |
specDeps.process(options.input_type, options.pkg, options.display_option) |
65 | 64 |
|
66 | 65 |
elif options.input_type == "json": |