Browse code

Package builder: fix incremental build

Use upward dependencies instead of downward.
Support BASE_COMMIT target for `make clean-stage-for-incremental-build`

Change-Id: I28a5e791c89cd7551074b47de967a7112515d28b
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/5803
Reviewed-by: Anish Swaminathan <anishs@vmware.com>
Tested-by: Anish Swaminathan <anishs@vmware.com>

Alexey Makhalov authored on 2018/09/27 07:21:23
Showing 5 changed files
... ...
@@ -235,13 +235,19 @@ who-needs:
235 235
 	@cd $(PHOTON_SPECDEPS_DIR) && \
236 236
 		$(PHOTON_SPECDEPS) -s $(PHOTON_SPECS_DIR) -i who-needs -p $(pkg)
237 237
 
238
-# This target analyzes top git commit and removes staged RPMS that can be affected
239
-# by this change and should be rebuilt as part of incremental build support
238
+# Input args: BASE_COMMIT= (optional)
239
+#
240
+# This target removes staged RPMS that can be affected by change(s) and should
241
+# be rebuilt as part of incremental build support
240 242
 # For every spec file touched - remove all upward dependent packages (rpms)
241 243
 # If support folder was touched - do full build
244
+#
245
+# The analyzed changes are:
246
+# - commits from BASE_COMMIT to HEAD (if BASE_COMMIT= parameter is specified)
247
+# - local changes (if no commits specified)
242 248
 clean-stage-for-incremental-build:
243
-	@test -n "$$(git show @ SPECS)" && $(PHOTON_SPECDEPS) -s $(PHOTON_SPECS_DIR) -i remove-upward-deps -p $$(echo `git show --pretty="format:" --name-only @ | grep .spec | xargs -n1 basename 2>/dev/null` | tr ' ' :)
244
-	@test -n "$$(git show @ support)" && $(RM) -rf $(PHOTON_RPMS_DIR)
249
+	@test -n "$$(git diff --name-only $(BASE_COMMIT) @ | grep SPECS)" && $(PHOTON_SPECDEPS) -s $(PHOTON_SPECS_DIR) -i remove-upward-deps -p $$(echo `git diff --name-only $(BASE_COMMIT) @ | grep .spec | xargs -n1 basename 2>/dev/null` | tr ' ' :) ||:
250
+	@test -n "$$(git diff --name-only $(BASE_COMMIT) @ | grep support)" && $(RM) -rf $(PHOTON_RPMS_DIR) ||:
245 251
 
246 252
 packages: check-docker-py check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_XRPMS) $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN) generate-dep-lists
247 253
 	@echo "Building all RPMS..."
... ...
@@ -140,12 +140,10 @@ class PackageBuildDataGenerator(object):
140 140
             nextPackagesToConstructGraph.update(dependentPackages)
141 141
 
142 142
         if addRunTimeGraph:
143
-            packageName, packageVersion = StringUtils.splitPackageNameAndVersion(basePackage)
144
-            rpmPackages = SPECS.getData().getPackages(packageName, packageVersion)
145 143
             dependentPackages = set()
146
-            for rpmPkg in rpmPackages:
147
-                dependentRpmPackages = SPECS.getData().getRequiresAllForPackage(rpmPkg, packageVersion)
148
-                self.__runTimeDependencyGraph[rpmPkg+"-"+packageVersion] = copy.copy(set(dependentRpmPackages))
144
+            for rpmPkg in SPECS.getData().getPackagesForPkg(basePackage):
145
+                dependentRpmPackages = SPECS.getData().getRequiresAllForPkg(rpmPkg)
146
+                self.__runTimeDependencyGraph[rpmPkg] = copy.copy(set(dependentRpmPackages))
149 147
                 for pkg in dependentRpmPackages:
150 148
                     dependentPackages.add(SPECS.getData().getBasePkg(pkg))
151 149
             nextPackagesToConstructGraph.update(dependentPackages)
... ...
@@ -412,26 +412,6 @@ class PackageUtils(object):
412 412
         rpmDestDir = rpmDir + "/" + arch
413 413
         return rpmDestDir
414 414
 
415
-    def _getProperVersion(self,package,parseSpecObj):
416
-        listOfVersionObjs=SPECS.getData().getSpecObj(package)
417
-        for num in listOfVersionObjs:
418
-                if parseSpecObj.compare == ">=":
419
-                       if LooseVersion(num.version) >= LooseVersion(parseSpecObj.version):
420
-                                return num.version
421
-                elif parseSpecObj.compare == "<=":
422
-                        if LooseVersion(num.version) <= LooseVersion(parseSpecObj.version):
423
-                                return num.version
424
-                elif parseSpecObj.compare == "=":
425
-                        if LooseVersion(num.version) == LooseVersion(parseSpecObj.version):
426
-                                return num.version
427
-                elif parseSpecObj.compare == "<":
428
-                        if LooseVersion(num.version) < LooseVersion(parseSpecObj.version):
429
-                                return num.version
430
-                elif parseSpecObj.compare == ">":
431
-                        if LooseVersion(num.version) > LooseVersion(parseSpecObj.version):
432
-                                return num.version
433
-        return "*"
434
-
435 415
     def _copyRPM(self, rpmFile, destDir):
436 416
         cmdUtils = CommandUtils()
437 417
         rpmName = os.path.basename(rpmFile)
... ...
@@ -234,8 +234,11 @@ class SpecObjectsUtils(object):
234 234
         return self._getSpecObjField(package, version, field=lambda x : x.listPackages)
235 235
 
236 236
     def getPackagesForPkg(self, pkg):
237
+        pkgs=[]
237 238
         package, version = StringUtils.splitPackageNameAndVersion(pkg)
238
-        return self.getPackages(package, version)
239
+        for p in self.getPackages(package, version):
240
+            pkgs.append(p+"-"+version)
241
+        return pkgs
239 242
 
240 243
     def getRPMPackages(self, package, version):
241 244
         return self._getSpecObjField(package, version, field=lambda x : x.listRPMPackages)
... ...
@@ -394,26 +397,29 @@ class SpecDependencyGenerator(object):
394 394
                     parent[depPkg] = specPkg
395 395
                     depQue.put(depPkg)
396 396
 
397
-    def findTotalWhoNeedsToBuild(self, depQue, whoNeedsBuild):
398
-        while not depQue.empty():
399
-            specPkg = depQue.get()
400
-            listPackagesRequiredToBuild = SPECS.getData().getBuildRequiresForPkg(specPkg)
401
-            for depPkg in listPackagesRequiredToBuild:
402
-                depSpecPkg = SPECS.getData().getBasePkg(depPkg)
403
-                if depSpecPkg not in whoNeedsBuild:
404
-                    whoNeedsBuild.append(depSpecPkg)
405
-                    depQue.put(depSpecPkg)
406
-
407
-    def findTotalWhoNeeds(self, depQue, whoNeeds):
408
-        while not depQue.empty():
409
-            specPkg = depQue.get()
410
-            listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(specPkg)
411
-            listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(specPkg))
412
-            for depPkg in listPackagesRequired:
413
-                depSpecPkg = SPECS.getData().getBasePkg(depPkg)
414
-                if depSpecPkg not in whoNeeds:
415
-                    whoNeeds.append(depSpecPkg)
416
-                    depQue.put(depSpecPkg)
397
+    def getBasePackagesRequired(self, pkg):
398
+        listBasePackagesRequired=[]
399
+        listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg)
400
+        listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg))
401
+        for p in listPackagesRequired:
402
+            basePkg = SPECS.getData().getBasePkg(p)
403
+            if basePkg not in listBasePackagesRequired:
404
+                listBasePackagesRequired.append(basePkg)
405
+        return listBasePackagesRequired
406
+
407
+
408
+    def findTotalWhoNeeds(self, depList, whoNeeds):
409
+        while depList:
410
+            pkg = depList.pop(0)
411
+            for depPackage in SPECS.getData().getListPackages():
412
+                for version in SPECS.getData().getVersions(depPackage):
413
+                    depBasePkg = depPackage+"-"+version
414
+                    if depBasePkg in whoNeeds:
415
+                        continue
416
+                    if pkg in self.getBasePackagesRequired(depBasePkg):
417
+                        whoNeeds.append(depBasePkg)
418
+                        if depBasePkg not in depList:
419
+                            depList.append(depBasePkg)
417 420
 
418 421
     def printTree(self, children, curParent, depth):
419 422
         if curParent in children:
... ...
@@ -430,8 +436,7 @@ class SpecDependencyGenerator(object):
430 430
     def updateLevels(self, mapDependencies, inPkg, parent, level):
431 431
         listPackages = SPECS.getData().getPackagesForPkg(inPkg)
432 432
         for depPkg in SPECS.getData().getRequiresForPkg(inPkg):
433
-            package, version = StringUtils.splitPackageNameAndVersion(depPkg)
434
-            if package in listPackages:
433
+            if depPkg in listPackages:
435 434
                 continue
436 435
             if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:
437 436
                 mapDependencies[depPkg] = level + 1
... ...
@@ -490,7 +495,7 @@ class SpecDependencyGenerator(object):
490 490
     def process(self, inputType, inputValue, displayOption, outputFile=None):
491 491
         whoNeedsList = []
492 492
         inputPackages = []
493
-        whoNeedsBuild = []
493
+        whatNeedsBuild = []
494 494
         mapDependencies = {}
495 495
         parent = {}
496 496
         if inputType == "pkg" or inputType == "json":
... ...
@@ -505,14 +510,13 @@ class SpecDependencyGenerator(object):
505 505
             else:
506 506
                 return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent)
507 507
         elif inputType == "remove-upward-deps":
508
-            depQue = queue.Queue()
508
+            depList = []
509 509
             for specFile in inputValue.split(":"):
510 510
                 if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
511 511
                     specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
512 512
                     whoNeedsList.append(specObj.name+"-"+specObj.version)
513
-                    for package in specObj.listPackages:
514
-                        depQue.put(package+"-"+specObj.version)
515
-            self.findTotalWhoNeeds(depQue, whoNeedsList)
513
+                    depList.append(specObj.name+"-"+specObj.version)
514
+            self.findTotalWhoNeeds(depList, whoNeedsList)
516 515
             return whoNeedsList
517 516
 
518 517
         elif inputType == "who-needs":
... ...
@@ -525,10 +529,3 @@ class SpecDependencyGenerator(object):
525 525
                         whoNeedsList.append(depPkg)
526 526
             print (whoNeedsList)
527 527
             return whoNeedsList
528
-        elif inputType == "who-needs-build":
529
-            depQue = queue.Queue()
530
-            depQue.put(inputValue)
531
-            self.findTotalWhoNeedsToBuild(depQue, whoNeedsBuild)
532
-            print ("Following specs need to be build again")
533
-            print (whoNeedsBuild)
534
-            return whoNeedsBuild
... ...
@@ -59,8 +59,7 @@ def main():
59 59
                 cmdUtils.runCommandInShell2("rm -f "+rpmFile)
60 60
         # To display/print package dependencies on console
61 61
         elif (options.input_type == "pkg" or
62
-                options.input_type == "who-needs" or
63
-                options.input_type == "who-needs-build"):
62
+                options.input_type == "who-needs"):
64 63
             specDeps.process(options.input_type, options.pkg, options.display_option)
65 64
 
66 65
         elif options.input_type == "json":