Browse code

Mutiversion support for BuildRequires and Requires dependencies in Package Builder system.

Added Version comparision and provide the appropriate version of the dependent packages in Package Builder.

Change-Id: Ib87fee8660151e4a8b0613031062c39cc9f21200
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/5347
Tested-by: gerrit-photon <photon-checkins@vmware.com>
Reviewed-by: Anish Swaminathan <anishs@vmware.com>

Ankit Jain authored on 2018/07/12 23:15:42
Showing 4 changed files
... ...
@@ -8,6 +8,7 @@ from CommandUtils import CommandUtils
8 8
 from constants import constants
9 9
 from SpecData import SPECS
10 10
 import docker
11
+from distutils.version import LooseVersion
11 12
 
12 13
 class PackageBuilderBase(object):
13 14
     def __init__(self, mapPackageToCycles, pkgBuildType):
... ...
@@ -76,7 +77,7 @@ class PackageBuilderBase(object):
76 76
         packageIsAlreadyBuilt = True
77 77
         pkgUtils = PackageUtils(self.logName, self.logPath)
78 78
         for pkg in listRPMPackages:
79
-            if pkgUtils.findRPMFileForGivenPackage(pkg, index) is None:
79
+            if pkgUtils.findRPMFileForGivenPackage(pkg,"*", index) is None:
80 80
                 packageIsAlreadyBuilt = False
81 81
                 break
82 82
         return packageIsAlreadyBuilt
... ...
@@ -90,7 +91,40 @@ class PackageBuilderBase(object):
90 90
     def _findBuildTimeCheckRequiredPackages(self, index=0):
91 91
         return SPECS.getData().getCheckBuildRequiresForPackage(self.package, index)
92 92
 
93
-    def _installPackage(self, pkgUtils, package, instanceID, destLogPath,
93
+    def _findRunTimeRequiredRPMPackagesParseObj(self,rpmPackage):
94
+        listRequiredPackages=SPECS.getData().getRequiresParseObjForPackage(rpmPackage)
95
+        return listRequiredPackages
96
+
97
+    def _findBuildTimeRequiredPackagesParseObj(self, index):
98
+        listRequiredPackages=SPECS.getData().getBuildRequiresParseObjForPackage(self.package, index)
99
+        return listRequiredPackages
100
+
101
+    def _findBuildTimeCheckRequiredPackagesParseObj(self, index):
102
+        listRequiredPackages=SPECS.getData().getCheckBuildRequiresParseObjForPackage(self.package, index)
103
+        return listRequiredPackages
104
+
105
+    def _getProperVersion(self,package,parseSpecObj):
106
+        listOfVersionObjs=self.getSpecObj(package)
107
+        for num in listOfVersionObjs:
108
+                if parseSpecObj.compare == 'gte':
109
+                       if LooseVersion(num.version) >= LooseVersion(parseSpecObj.version):
110
+                                return num.version
111
+                elif parseSpecObj.compare == 'lte':
112
+                        if LooseVersion(num.version) <= LooseVersion(parseSpecObj.version):
113
+                                return num.version
114
+                elif parseSpecObj.compare == 'eq':
115
+                        if LooseVersion(num.version) == LooseVersion(parseSpecObj.version):
116
+                                return num.version
117
+                elif parseSpecObj.compare == 'lt':
118
+                        if LooseVersion(num.version) < LooseVersion(parseSpecObj.version):
119
+                                return num.version
120
+                elif parseSpecObj.compare == 'gt':
121
+                        if LooseVersion(num.version) > LooseVersion(parseSpecObj.version):
122
+                                return num.version
123
+        return "*"
124
+
125
+
126
+    def _installPackage(self, pkgUtils, package,packageVersion, instanceID, destLogPath,
94 127
                         listInstalledPackages, listInstalledRPMs):
95 128
         latestRPM = os.path.basename(
96 129
             pkgUtils.findRPMFileForGivenPackage(package)).replace(".rpm", "")
... ...
@@ -107,13 +141,14 @@ class PackageBuilderBase(object):
107 107
                 package in constants.noDepsPackageList):
108 108
             noDeps = True
109 109
         if self.pkgBuildType == "chroot":
110
-            pkgUtils.installRPM(package, instanceID, noDeps, destLogPath)
110
+            pkgUtils.installRPM(package, packageVersion,instanceID, noDeps, destLogPath)
111 111
         elif self.pkgBuildType == "container":
112
-            pkgUtils.prepRPMforInstallInContainer(package, instanceID, noDeps, destLogPath)
112
+            pkgUtils.prepRPMforInstallInContainer(package,packageVersion, instanceID, noDeps, destLogPath)
113 113
 
114 114
     def _installDependentRunTimePackages(self, pkgUtils, package, instanceID, destLogPath,
115 115
                                          listInstalledPackages, listInstalledRPMs):
116 116
         listRunTimeDependentPackages = self._findRunTimeRequiredRPMPackages(package)
117
+        listRunTimeDependentPackagesParseObj=self._findRunTimeRequiredRPMPackagesParseObj(package)
117 118
         if listRunTimeDependentPackages:
118 119
             for pkg in listRunTimeDependentPackages:
119 120
                 if pkg in self.mapPackageToCycles:
... ...
@@ -122,26 +157,40 @@ class PackageBuilderBase(object):
122 122
                     pkgUtils.findRPMFileForGivenPackage(pkg)).replace(".rpm", "")
123 123
                 if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs:
124 124
                     continue
125
-                self._installPackage(pkgUtils, pkg, instanceID, destLogPath,
126
-                                     listInstalledPackages, listInstalledRPMs)
125
+                flag = False
126
+                for objName in listRunTimeDependentPackagesParseObj:
127
+                    if objName.package == pkg:
128
+                        properVersion=self._getProperVersion(pkg,objName)
129
+                        self._installPackage(pkgUtils, pkg,properVersion, instanceID, destLogPath,listInstalledPackages, listInstalledRPMs)
130
+                        flag = True
131
+                        break;
132
+                if flag == False:
133
+                        self._installPackage(pkgUtils, pkg,"*", instanceID, destLogPath,listInstalledPackages, listInstalledRPMs)
127 134
 
128 135
     def _findDependentPackagesAndInstalledRPM(self, instanceID, index=0):
129 136
         listInstalledPackages, listInstalledRPMs = self._findInstalledPackages(instanceID)
130 137
         self.logger.info(listInstalledPackages)
131 138
         listDependentPackages = self._findBuildTimeRequiredPackages(index)
139
+        listDependentPackagesParseObj=self._findBuildTimeRequiredPackagesParseObj(index)
132 140
         if constants.rpmCheck and self.package in constants.testForceRPMS:
133 141
             listDependentPackages.extend(self._findBuildTimeCheckRequiredPackages(index))
142
+            listDependentPackagesParseObj.extend(self._findBuildTimeCheckRequiredPackagesParseObj(index))
134 143
             testPackages = (set(constants.listMakeCheckRPMPkgtoInstall) -
135 144
                             set(listInstalledPackages) -
136 145
                             set([self.package]))
137 146
             listDependentPackages.extend(testPackages)
138 147
             listDependentPackages = list(set(listDependentPackages))
139
-        return listDependentPackages, listInstalledPackages, listInstalledRPMs
148
+            listDependentPackagesParseObj=list(set(listDependentPackagesParseObj))
149
+        return listDependentPackages, listDependentPackagesParseObj,listInstalledPackages, listInstalledRPMs
140 150
 
141 151
     @staticmethod
142 152
     def getNumOfVersions(package):
143 153
         return SPECS.getData().getNumberOfVersions(package)
144 154
 
155
+    @staticmethod
156
+    def getSpecObj(package):
157
+        return SPECS.getData().getSpecObj(package)
158
+
145 159
 class PackageBuilderContainer(PackageBuilderBase):
146 160
     def __init__(self, mapPackageToCycles, pkgBuildType):
147 161
         self.buildContainerImage = "photon_build_container:latest"
... ...
@@ -249,7 +298,7 @@ class PackageBuilderContainer(PackageBuilderBase):
249 249
                     constants.perPackageToolChain[self.package],
250 250
                     self.package)
251 251
 
252
-            listDependentPackages, listInstalledPackages, listInstalledRPMs = (
252
+            listDependentPackages,listDependentPackagesParseObj, listInstalledPackages, listInstalledRPMs = (
253 253
                 self._findDependentPackagesAndInstalledRPM(containerID, index))
254 254
 
255 255
             pkgUtils = PackageUtils(self.logName, self.logPath)
... ...
@@ -258,8 +307,15 @@ class PackageBuilderContainer(PackageBuilderBase):
258 258
                                  "Installing dependent packages..")
259 259
                 self.logger.info(listDependentPackages)
260 260
                 for pkg in listDependentPackages:
261
-                    self._installPackage(pkgUtils, pkg, containerID, destLogPath,
262
-                                         listInstalledPackages, listInstalledRPMs)
261
+                    flag = False
262
+                    for objName in listDependentPackagesParseObj:
263
+                        if objName.package == pkg:
264
+                                properVersion=self._getProperVersion(pkg,objName)
265
+                                self._installPackage(pkgUtils, pkg,properVersion, containerID, destLogPath,listInstalledPackages, listInstalledRPMs)
266
+                                flag = True
267
+                                break;
268
+                    if flag == False:
269
+                        self._installPackage(pkgUtils, pkg,"*", containerID, destLogPath,listInstalledPackages, listInstalledRPMs)
263 270
                 pkgUtils.installRPMSInAOneShotInContainer(containerID, destLogPath)
264 271
                 self.logger.info("Finished installing the build time dependent packages....")
265 272
 
... ...
@@ -329,7 +385,7 @@ class PackageBuilderChroot(PackageBuilderBase):
329 329
         chrootID = None
330 330
         try:
331 331
             chrootID = self._prepareBuildRoot()
332
-            listDependentPackages, listInstalledPackages, listInstalledRPMs = (
332
+            listDependentPackages,listDependentPackagesParseObj, listInstalledPackages, listInstalledRPMs = (
333 333
                 self._findDependentPackagesAndInstalledRPM(chrootID, index))
334 334
 
335 335
             pkgUtils = PackageUtils(self.logName, self.logPath)
... ...
@@ -337,8 +393,15 @@ class PackageBuilderChroot(PackageBuilderBase):
337 337
             if listDependentPackages:
338 338
                 self.logger.info("Installing the build time dependent packages......")
339 339
                 for pkg in listDependentPackages:
340
-                    self._installPackage(pkgUtils, pkg, chrootID, self.logPath,
341
-                                         listInstalledPackages, listInstalledRPMs)
340
+                    flag = False
341
+                    for objName in listDependentPackagesParseObj:
342
+                        if objName.package == pkg:
343
+                                properVersion=self._getProperVersion(pkg,objName)
344
+                                self._installPackage(pkgUtils, pkg,properVersion, chrootID, self.logPath,listInstalledPackages, listInstalledRPMs)
345
+                                flag = True
346
+                                break;
347
+                    if flag == False:
348
+                        self._installPackage(pkgUtils, pkg,"*", chrootID, self.logPath,listInstalledPackages, listInstalledRPMs)
342 349
                 pkgUtils.installRPMSInAOneShot(chrootID, self.logPath)
343 350
                 self.logger.info("Finished installing the build time dependent packages....")
344 351
 
... ...
@@ -41,11 +41,11 @@ class PackageUtils(object):
41 41
         self.rpmFilesToReInstallInAOneShot = ""
42 42
         self.noDepsRPMFilesToReInstallInAOneShot = ""
43 43
 
44
-    def installRPM(self, package, chrootID, noDeps=False, destLogPath=None):
44
+    def installRPM(self, package,version, chrootID, noDeps=False, destLogPath=None):
45 45
 #        self.logger.info("Installing rpm for package:"+package)
46 46
 #        self.logger.debug("No deps:"+str(noDeps))
47 47
 
48
-        rpmfile = self.findRPMFileForGivenPackage(package)
48
+        rpmfile = self.findRPMFileForGivenPackage(package,version)
49 49
         if rpmfile is None:
50 50
             self.logger.error("No rpm file found for package:" + package)
51 51
             raise Exception("Missing rpm file: " + package)
... ...
@@ -143,10 +143,12 @@ class PackageUtils(object):
143 143
         for srpmFile in listSRPMFiles:
144 144
             srpmDestFile = self._copyRPM(chrootID + "/" + srpmFile, constants.sourceRpmPath)
145 145
 
146
-    def findRPMFileForGivenPackage(self, package, index=0):
146
+    def findRPMFileForGivenPackage(self, package,version="*", index=0):
147 147
         cmdUtils = CommandUtils()
148
-        version = SPECS.getData().getVersion(package, index)
149
-        release = SPECS.getData().getRelease(package, index)
148
+        release = "*"
149
+        if version == "*":
150
+                version = SPECS.getData().getVersion(package, index)
151
+                release = SPECS.getData().getRelease(package, index)
150 152
         listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." +
151 153
                                                    platform.machine()+".rpm",
152 154
                                                    constants.rpmPath),
... ...
@@ -220,8 +222,8 @@ class PackageUtils(object):
220 220
         self.logger.error("Failed while adjusting gcc specs")
221 221
         raise Exception("Failed while adjusting gcc specs")
222 222
 
223
-    def prepRPMforInstallInContainer(self, package, containerID, noDeps=False, destLogPath=None):
224
-        rpmfile = self.findRPMFileForGivenPackage(package)
223
+    def prepRPMforInstallInContainer(self, package,version, containerID, noDeps=False, destLogPath=None):
224
+        rpmfile = self.findRPMFileForGivenPackage(package,version)
225 225
         if rpmfile is None:
226 226
             self.logger.error("No rpm file found for package: " + package)
227 227
             raise Exception("Missing rpm file")
... ...
@@ -21,6 +21,9 @@ class SpecObject(object):
21 21
         self.checkBuildRequirePackages = []
22 22
         self.installRequiresAllPackages = []
23 23
         self.installRequiresPackages = {}
24
+        self.specParseObjBuildRequirePackages = []
25
+        self.specParseObjInstallRequiresPackages=[]
26
+        self.specParseObjCheckBuildRequirePackages=[]
24 27
         self.specFile = ""
25 28
         self.listSources = []
26 29
         self.checksums = {}
... ...
@@ -47,9 +50,9 @@ class SpecObjectsUtils(object):
47 47
             specName = spec.getBasePackageName()
48 48
             specObj = SpecObject()
49 49
             specObj.name = specName
50
-            specObj.buildRequirePackages = spec.getBuildRequiresAllPackages()
51
-            specObj.installRequiresAllPackages = spec.getRequiresAllPackages()
52
-            specObj.checkBuildRequirePackages = spec.getCheckBuildRequiresAllPackages()
50
+            specObj.buildRequirePackages,specObj.specParseObjBuildRequirePackages = spec.getBuildRequiresAllPackages()
51
+            specObj.installRequiresAllPackages,specObj.specParseObjInstallRequiresPackages = spec.getRequiresAllPackages()
52
+            specObj.checkBuildRequirePackages,specObj.specParseObjCheckBuildRequirePackages = spec.getCheckBuildRequiresAllPackages()
53 53
             specObj.listPackages = spec.getPackageNames()
54 54
             specObj.specFile = specFile
55 55
             specObj.version = spec.getVersion()
... ...
@@ -107,6 +110,22 @@ class SpecObjectsUtils(object):
107 107
         specName = self.getSpecName(package)
108 108
         return self.mapSpecObjects[specName][index].checkBuildRequirePackages
109 109
 
110
+    def getBuildRequiresParseObjForPackage(self, package, index=0):
111
+        specName=self.getSpecName(package)
112
+        return self.mapSpecObjects[specName][index].specParseObjBuildRequirePackages
113
+
114
+    def getRequiresParseObjForPackage(self, package, index=0):
115
+        specName=self.getSpecName(package)
116
+        return self.mapSpecObjects[specName][index].specParseObjInstallRequiresPackages
117
+
118
+    def getCheckBuildRequiresParseObjForPackage(self, package, index=0):
119
+        specName=self.getSpecName(package)
120
+        return self.mapSpecObjects[specName][index].specParseObjCheckBuildRequirePackages
121
+
122
+    def getSpecObj(self, package):
123
+        specName=self.getSpecName(package)
124
+        return self.mapSpecObjects[specName]
125
+
110 126
     def getRelease(self, package, index=0):
111 127
         specName = self.getSpecName(package)
112 128
         return self.mapSpecObjects[specName][index].release
... ...
@@ -92,35 +92,52 @@ class Specutils(object):
92 92
 
93 93
     def getRequiresAllPackages(self):
94 94
         dependentPackages = []
95
+        specParseObjDependentPackages=[]
95 96
         for pkg in self.spec.packages.values():
96 97
             for dpkg in pkg.requires:
97 98
                 dependentPackages.append(dpkg.package)
99
+                specParseObjDependentPackages.append(dpkg)
98 100
         dependentPackages = list(set(dependentPackages))
101
+        specParseObjDependentPackages = list(set(specParseObjDependentPackages))
102
+        specParseObjDependentPackagesTemp = specParseObjDependentPackages[:]
99 103
         packageNames = self.getPackageNames()
100 104
         for pkgName in packageNames:
101 105
             if pkgName in dependentPackages:
102 106
                 dependentPackages.remove(pkgName)
103
-        return dependentPackages
107
+            for objName in specParseObjDependentPackagesTemp:
108
+                if objName.package == pkgName:
109
+                        specParseObjDependentPackages.remove(objName)
110
+        return dependentPackages,specParseObjDependentPackages
104 111
 
105 112
     def getBuildRequiresAllPackages(self):
106 113
         dependentPackages = []
114
+        specParseObjDependentPackages=[]
107 115
         for pkg in self.spec.packages.values():
108 116
             for dpkg in pkg.buildrequires:
109 117
                 dependentPackages.append(dpkg.package)
118
+                specParseObjDependentPackages.append(dpkg)
110 119
         dependentPackages = list(set(dependentPackages))
120
+        specParseObjDependentPackages = list(set(specParseObjDependentPackages))
121
+        specParseObjDependentPackagesTemp = specParseObjDependentPackages[:]
111 122
         packageNames = self.getPackageNames()
112 123
         for pkgName in packageNames:
113 124
             if pkgName in dependentPackages:
114 125
                 dependentPackages.remove(pkgName)
115
-        return dependentPackages
126
+            for objName in specParseObjDependentPackagesTemp:
127
+                if objName.package == pkgName:
128
+                        specParseObjDependentPackages.remove(objName)
129
+        return dependentPackages,specParseObjDependentPackages
116 130
 
117 131
     def getCheckBuildRequiresAllPackages(self):
118 132
         dependentPackages = []
133
+        specParseObjDependentPackages=[]
119 134
         for pkg in self.spec.packages.values():
120 135
             for dpkg in pkg.checkbuildrequires:
121 136
                 dependentPackages.append(dpkg.package)
137
+                specParseObjDependentPackages.append(dpkg)
122 138
         dependentPackages = list(set(dependentPackages))
123
-        return dependentPackages
139
+        specParseObjDependentPackages = list(set(specParseObjDependentPackages))
140
+        return dependentPackages,specParseObjDependentPackages
124 141
 
125 142
     def getRequires(self, pkgName):
126 143
         dependentPackages = []