Browse code

python3 builder.

Change-Id: I28f49a454df3be7c3275c80465c7a91e9172f241
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/4413
Tested-by: gerrit-photon <photon-checkins@vmware.com>
Reviewed-by: Xiaolin Li <xiaolinl@vmware.com>

xiaolin-vmware authored on 2018/01/04 07:25:14
Showing 26 changed files
... ...
@@ -528,7 +528,7 @@ check-docker-service:
528 528
 	@docker ps >/dev/null 2>&1 || { echo "Docker service is not running. Aborting." >&2; exit 1; }
529 529
 
530 530
 check-docker-py:
531
-	@python -c "import docker; assert docker.__version__ == '$(PHOTON_DOCKER_PY_VER)'" >/dev/null 2>&1 || { echo "Error: Python package docker-py 2.3.0 not installed.\nPlease use: pip install docker==2.3.0" >&2; exit 1; }
531
+	@python3 -c "import docker; assert docker.__version__ == '$(PHOTON_DOCKER_PY_VER)'" >/dev/null 2>&1 || { echo "Error: Python3 package docker-py3 2.3.0 not installed.\nPlease use: pip3 install docker==2.3.0" >&2; exit 1; }
532 532
 
533 533
 check-bison:
534 534
 	@command -v bison >/dev/null 2>&1 || { echo "Package bison not installed. Aborting." >&2; exit 1; }
... ...
@@ -1,47 +1,47 @@
1
+import os.path
1 2
 from Logger import Logger
2 3
 from CommandUtils import CommandUtils
3 4
 from constants import constants
4
-import os.path
5 5
 
6 6
 class ChrootUtils(object):
7
-    
8
-    def __init__(self,logName=None,logPath=None):
7
+
8
+    def __init__(self, logName=None, logPath=None):
9 9
         if logName is None:
10 10
             logName = "ChrootUtils"
11 11
         if logPath is None:
12 12
             logPath = constants.logPath
13
-        self.logName=logName
14
-        self.logPath=logPath
15
-        self.logger=Logger.getLogger(logName,logPath)
16
-        
17
-    def createChroot(self,chrootName):
18
-        chrootID=constants.buildRootPath+"/"+chrootName
13
+        self.logName = logName
14
+        self.logPath = logPath
15
+        self.logger = Logger.getLogger(logName, logPath)
16
+
17
+    def createChroot(self, chrootName):
18
+        chrootID = constants.buildRootPath + "/" + chrootName
19 19
         if os.path.isdir(chrootID):
20 20
             if not self.destroyChroot(chrootID):
21
-                self.logger.error("Given chroot "+chrootID+" is already exists. unable to destroy it ")
22
-                return False,None
21
+                self.logger.error("Given chroot " + chrootID +
22
+                                  " is already exists. unable to destroy it ")
23
+                return False, None
23 24
         # need to add timeout for this step
24 25
         # http://stackoverflow.com/questions/1191374/subprocess-with-timeout
25
-        cmdUtils=CommandUtils()
26
-        returnVal=cmdUtils.runCommandInShell("mkdir -p "+chrootID)
26
+        cmdUtils = CommandUtils()
27
+        returnVal = cmdUtils.runCommandInShell("mkdir -p " + chrootID)
27 28
         if not returnVal:
28
-            self.logger.error("Unable to create chroot:"+ chrootID +".Unknown error.")
29
-            return False,None
30
-        return True,chrootID
31
-    
32
-    def destroyChroot(self,chrootID):
29
+            self.logger.error("Unable to create chroot:" + chrootID + ".Unknown error.")
30
+            return False, None
31
+        return True, chrootID
32
+
33
+    def destroyChroot(self, chrootID):
33 34
         # need to add timeout for this step
34 35
         # http://stackoverflow.com/questions/1191374/subprocess-with-timeout
35
-        cmdUtils=CommandUtils()
36
-        returnVal=cmdUtils.runCommandInShell("./clean-up-chroot.py "+chrootID)
36
+        cmdUtils = CommandUtils()
37
+        returnVal = cmdUtils.runCommandInShell("./clean-up-chroot.py " + chrootID)
37 38
         if not returnVal:
38
-            self.logger.error("Unable to destroy chroot:"+ chrootID +".Unknown error.")
39
+            self.logger.error("Unable to destroy chroot:" + chrootID + ".Unknown error.")
39 40
             return False
40
-        
41
-        returnVal=cmdUtils.runCommandInShell("rm -rf "+chrootID)
41
+
42
+        returnVal = cmdUtils.runCommandInShell("rm -rf " + chrootID)
42 43
         if not returnVal:
43
-            self.logger.error("Unable to destroy chroot:"+ chrootID +".Unknown error.")
44
+            self.logger.error("Unable to destroy chroot:" + chrootID + ".Unknown error.")
44 45
             return False
45
-        self.logger.info("Successfully destroyed chroot:"+chrootID)
46
+        self.logger.info("Successfully destroyed chroot:" + chrootID)
46 47
         return True
47
-    
... ...
@@ -5,9 +5,11 @@ class CommandUtils(object):
5 5
     def __init__(self):
6 6
         self.findBinary = "find"
7 7
 
8
-    def findFile (self, filename, sourcePath):
9
-        process = subprocess.Popen([self.findBinary, "-L", sourcePath, "-name", filename, "-not", "-type", "d"],  stdout=subprocess.PIPE)
10
-        # We don't check the return val here because find could return 1 but still be able to find
8
+    def findFile(self, filename, sourcePath):
9
+        process = subprocess.Popen([self.findBinary, "-L", sourcePath, "-name", filename,
10
+                                    "-not", "-type", "d"], stdout=subprocess.PIPE)
11
+        # We don't check the return val here because find could return 1 but still be
12
+        # able to find
11 13
         # the result. We shouldn't blindly return None without even checking the result.
12 14
         # The reason we want to suppress this is because for built RPMs, we first copy it to
13 15
         # the location with a random name and move it to the real name. find will complain our
... ...
@@ -17,30 +19,30 @@ class CommandUtils(object):
17 17
 
18 18
         #if returnVal != 0:
19 19
         #    return None
20
-        result=process.communicate()[0]
20
+        result = process.communicate()[0]
21 21
         if result is None:
22 22
             return None
23
-        return result.split()
23
+        return result.decode().split()
24 24
 
25
-    def runCommandInShell(self,cmd,logfilePath=None,chrootCmd=None):
25
+    @staticmethod
26
+    def runCommandInShell(cmd, logfilePath=None, chrootCmd=None):
26 27
         if chrootCmd is not None:
27
-            cmd = chrootCmd+" "+cmd
28
+            cmd = chrootCmd + " " + cmd
28 29
         if logfilePath is None:
29
-            logfilePath=os.devnull
30
-        logfile=open(logfilePath,"w")
31
-        process = subprocess.Popen("%s" %cmd,shell=True,stdout=logfile,stderr=logfile)
30
+            logfilePath = os.devnull
31
+        logfile = open(logfilePath, "w")
32
+        process = subprocess.Popen("%s" %cmd, shell=True, stdout=logfile, stderr=logfile)
32 33
         retval = process.wait()
33 34
         logfile.close()
34
-        if retval==0:
35
+        if retval == 0:
35 36
             return True
36 37
         return False
37
-
38
-    def runCommandInShell2(self,cmd,chrootCmd=None):
38
+    @staticmethod
39
+    def runCommandInShell2(cmd, chrootCmd=None):
39 40
         if chrootCmd is not None:
40
-            cmd = chrootCmd+" "+cmd
41
-        process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE)
41
+            cmd = chrootCmd + " " + cmd
42
+        process = subprocess.Popen("%s" %cmd, shell=True, stdout=subprocess.PIPE)
42 43
         retval = process.wait()
43 44
         if retval != 0:
44 45
             return None
45 46
         return process.communicate()[0]
46
-
... ...
@@ -1,22 +1,22 @@
1
-import logging
2 1
 import os
2
+import logging
3 3
 
4 4
 class Logger(object):
5 5
     @staticmethod
6
-    def getLogger (mymodule, logpath=None, resetFile=False):
7
-        logfile=mymodule+".log"
6
+    def getLogger(mymodule, logpath=None, resetFile=False):
7
+        logfile = mymodule + ".log"
8 8
         if logpath is not None:
9 9
             if not os.path.isdir(logpath):
10 10
                 os.makedirs(logpath)
11
-            logfile=logpath+"/"+logfile
11
+            logfile = logpath + "/" + logfile
12 12
         if resetFile:
13 13
             open(logfile, 'w').close()
14
-        logger=logging.getLogger(mymodule)
14
+        logger = logging.getLogger(mymodule)
15 15
         if len(logger.handlers) == 0:
16 16
             #creating file handler
17
-            fhandler=logging.FileHandler(logfile)
17
+            fhandler = logging.FileHandler(logfile)
18 18
             # create console handler
19
-            ch=logging.StreamHandler()
19
+            ch = logging.StreamHandler()
20 20
             formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
21 21
             # add formatter to handler
22 22
             fhandler.setFormatter(formatter)
... ...
@@ -25,15 +25,15 @@ class Logger(object):
25 25
             logger.addHandler(ch)
26 26
             logger.addHandler(fhandler)
27 27
             logger.setLevel(logging.DEBUG)
28
-            logger.info("--------------------------------------------------------------------------")
28
+            logger.info("-" * 75)
29 29
             logger.info("Starting Log")
30
-            logger.info("--------------------------------------------------------------------------")
30
+            logger.info("-" * 75)
31 31
         return logger
32 32
 
33 33
 if __name__ == "__main__":
34 34
     #Logger.getLogger("my module")
35
-    t1 =  Logger.getLogger("my module")
35
+    t1 = Logger.getLogger("my module")
36 36
     t1.info("test1")
37
-    t2  = Logger.getLogger("my module")
37
+    t2 = Logger.getLogger("my module")
38 38
     t2.info("test2")
39 39
     t1.info("test3")
... ...
@@ -1,36 +1,36 @@
1 1
 import os.path
2 2
 
3 3
 class MiscUtils(object):
4
-    
4
+
5 5
     @staticmethod
6
-    def isOutdated(listInputFiles,listOutputFiles):
7
-        thresholdTimeStamp=None
6
+    def isOutdated(listInputFiles, listOutputFiles):
7
+        thresholdTimeStamp = None
8 8
         if len(listInputFiles) == 0:
9 9
             return False
10 10
         if len(listOutputFiles) == 0:
11 11
             return True
12 12
         for f in listOutputFiles:
13
-            t=os.path.getmtime(f)
13
+            t = os.path.getmtime(f)
14 14
             if thresholdTimeStamp is None:
15 15
                 thresholdTimeStamp = t
16 16
             if t < thresholdTimeStamp:
17
-                thresholdTimeStamp = t 
17
+                thresholdTimeStamp = t
18 18
         for f in listInputFiles:
19
-            t=os.path.getmtime(f)
19
+            t = os.path.getmtime(f)
20 20
             if t > thresholdTimeStamp:
21 21
                 return True
22 22
         return False
23
-    
23
+
24 24
     @staticmethod
25
-    def getListSpecFiles(listSpecFiles,path):
25
+    def getListSpecFiles(listSpecFiles, path):
26 26
         for dirEntry in os.listdir(path):
27 27
             dirEntryPath = os.path.join(path, dirEntry)
28 28
             if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(".spec"):
29 29
                 listSpecFiles.append(dirEntryPath)
30 30
             elif os.path.isdir(dirEntryPath):
31
-                MiscUtils.getListSpecFiles(listSpecFiles,dirEntryPath)
32
-    
33
-if __name__=="__main__":
34
-    listInputFiles=["SpecParser.py","Logger.py"]
35
-    listOutputFiles=["builder.py"]
36
-    print MiscUtils.isOutdated(listInputFiles, listOutputFiles)
37 31
\ No newline at end of file
32
+                MiscUtils.getListSpecFiles(listSpecFiles, dirEntryPath)
33
+
34
+if __name__ == "__main__":
35
+    listInputFiles = ["SpecParser.py", "Logger.py"]
36
+    listOutputFiles = ["builder.py"]
37
+    print(MiscUtils.isOutdated(listInputFiles, listOutputFiles))
... ...
@@ -1,204 +1,205 @@
1
+import copy
1 2
 from Logger import Logger
2 3
 from constants import constants
3
-from sets import Set
4
-import copy
5 4
 from SpecData import SPECS
6 5
 
7
-
8
-
9 6
 def removeDuplicateEntriesInList(myList):
10
-    myListCopy=[]
7
+    myListCopy = []
11 8
     for p in myList:
12 9
         if p not in myListCopy:
13 10
             myListCopy.append(p)
14 11
     return myListCopy
15
-      
12
+
16 13
 class PackageBuildDataGenerator(object):
17
-    
18
-    cycleCount=0
19
-    
20
-    def __init__(self,logName=None,logPath=None):
14
+
15
+    cycleCount = 0
16
+
17
+    def __init__(self, logName=None, logPath=None):
21 18
         if logName is None:
22 19
             logName = "PackageBuildDataGenerator"
23 20
         if logPath is None:
24 21
             logPath = constants.logPath
25
-        self.logName=logName
26
-        self.logPath=logPath
27
-        self.logger=Logger.getLogger(logName,logPath)
28
-        self.__mapCyclesToPackageList={}
29
-        self.__mapPackageToCycle={}
30
-        self.__buildDependencyGraph={}
31
-        self.__runTimeDependencyGraph={}
32
-        self.__sortedPackageList=[]
33
-        self.__sortedBuildDependencyGraph={}
34
-    
35
-    def getPackageBuildData(self,listPackages):
22
+        self.logName = logName
23
+        self.logPath = logPath
24
+        self.logger = Logger.getLogger(logName, logPath)
25
+        self.__mapCyclesToPackageList = {}
26
+        self.__mapPackageToCycle = {}
27
+        self.__buildDependencyGraph = {}
28
+        self.__runTimeDependencyGraph = {}
29
+        self.__sortedPackageList = []
30
+        self.__sortedBuildDependencyGraph = {}
31
+
32
+    def getPackageBuildData(self, listPackages):
36 33
         self.__readDependencyGraphAndCyclesForGivenPackages(listPackages)
37 34
         self.__getSortedBuildOrderListForGivenPackages(listPackages)
38 35
         return self.__mapCyclesToPackageList, self.__mapPackageToCycle, self.__sortedPackageList
39 36
 
40
-    #todo    
41
-    def findCompleteListOfPackagesRequiredToBuildGivenPackages(self,listPackages):
42
-        return self.__buildDependencyGraph.keys()    
43
- 
44
-    def createSortListForPkg(self,pkg):
45
-        runTimeDepPkgList=self.__runTimeDependencyGraph[pkg]
37
+    #todo
38
+    def findCompleteListOfPackagesRequiredToBuildGivenPackages(self, listPackages):
39
+        return list(self.__buildDependencyGraph.keys())
40
+
41
+    def createSortListForPkg(self, pkg):
42
+        runTimeDepPkgList = self.__runTimeDependencyGraph[pkg]
46 43
         runTimeDepPkgList.append(pkg)
47
-        sortListForPkg=[]
48
-        
44
+        sortListForPkg = []
45
+
49 46
         for p in runTimeDepPkgList:
50
-            basePkg=SPECS.getData().getSpecName(p)
47
+            basePkg = SPECS.getData().getSpecName(p)
51 48
             for bPkg in self.__sortedBuildDependencyGraph[basePkg]:
52 49
                 if bPkg not in sortListForPkg:
53 50
                     sortListForPkg.append(bPkg)
54
-        
51
+
55 52
         return sortListForPkg
56
-    
57
-    def getCircularDependentPackages(self,pkg):
58
-        circularDependentPackages=[]
59
-        if self.__mapPackageToCycle.has_key(pkg):
60
-            circularDependentPackages.extend(self.__mapCyclesToPackageList[self.__mapPackageToCycle[pkg]])
53
+
54
+    def getCircularDependentPackages(self, pkg):
55
+        circularDependentPackages = []
56
+        if pkg in self.__mapPackageToCycle:
57
+            circularDependentPackages.extend(
58
+                self.__mapCyclesToPackageList[self.__mapPackageToCycle[pkg]])
61 59
             circularDependentPackages.remove(pkg)
62 60
         return circularDependentPackages
63
-    
64
-    def __getSortedBuildOrderListForGivenPackages(self,listPackages):
65
-        
66
-        alreadyProcessedPackages=[]
67
-        sortedList=[]
68
-        completeListPackagesToBuild=self.findCompleteListOfPackagesRequiredToBuildGivenPackages(listPackages)
61
+
62
+    def __getSortedBuildOrderListForGivenPackages(self, listPackages):
63
+
64
+        alreadyProcessedPackages = []
65
+        sortedList = []
66
+        completeListPackagesToBuild = self.findCompleteListOfPackagesRequiredToBuildGivenPackages(
67
+            listPackages)
69 68
         packageIndexInSortedList = 0
70
-        prevSortListLen=0
71
-        
69
+        prevSortListLen = 0
70
+
72 71
         while completeListPackagesToBuild:
73
-            
72
+
74 73
             # find next package to process
75
-            pkg=None
76
-            index=-1
77
-            lenList=len(sortedList)
74
+            pkg = None
75
+            index = -1
76
+            lenList = len(sortedList)
78 77
             for  i in range(lenList):
79 78
                 if sortedList[i] in alreadyProcessedPackages:
80 79
                     continue
81 80
                 pkg = sortedList[i]
82 81
                 packageIndexInSortedList = i
83 82
                 break
84
-                
83
+
85 84
             if pkg is None:
86 85
                 pkg = completeListPackagesToBuild.pop()
87 86
                 packageIndexInSortedList = len(sortedList)
88 87
 
89 88
             #creating sort list for package
90
-            sortListForPkg=self.createSortListForPkg(pkg)
91
-            
89
+            sortListForPkg = self.createSortListForPkg(pkg)
90
+
92 91
             #remove any cyclic packages in sortListForPkg if they already exists in sortedList
93
-            circularDependentPackages=self.getCircularDependentPackages(pkg)
92
+            circularDependentPackages = self.getCircularDependentPackages(pkg)
94 93
             for p in circularDependentPackages:
95 94
                 if p in sortedList and p in sortListForPkg:
96 95
                     sortListForPkg.remove(p)
97
-            
96
+
98 97
             # insert sort list of package in global sorted list
99
-            index=packageIndexInSortedList
100
-            subList=[]
98
+            index = packageIndexInSortedList
99
+            subList = []
101 100
             if packageIndexInSortedList > 0:
102
-                subList=sortedList[:packageIndexInSortedList]
101
+                subList = sortedList[:packageIndexInSortedList]
103 102
             for p in sortListForPkg:
104 103
                 if  p not in subList:
105 104
                     sortedList.insert(index, p)
106 105
                     index = index + 1
107
-            
106
+
108 107
             alreadyProcessedPackages.append(p)
109
-        
108
+
110 109
             # Remove duplicate entries in sorted list in intervals
111
-            if (len(sortedList)-prevSortListLen) > 100 :
110
+            if (len(sortedList) - prevSortListLen) > 100:
112 111
                 self.logger.info("Removing duplicates in sortedList")
113 112
                 sortedList = removeDuplicateEntriesInList(sortedList)
114 113
             else:
115
-                prevSortListLen=len(sortedList)
116
-                
114
+                prevSortListLen = len(sortedList)
115
+
117 116
         self.logger.info("Removing duplicates in sorted list")
118 117
         sortedList = removeDuplicateEntriesInList(sortedList)
119
-        
118
+
120 119
         self.logger.info("Sorted list:")
121 120
         self.logger.info(sortedList)
122
-        self.__sortedPackageList=sortedList
123
-    
124
-    def __constructBuildAndRunTimeDependencyGraph(self,package):
125
-        basePackage=SPECS.getData().getSpecName(package)
126
-
127
-        addBuildTimeGraph=True
128
-        addRunTimeGraph=True
129
-        if self.__buildDependencyGraph.has_key(basePackage):
121
+        self.__sortedPackageList = sortedList
122
+
123
+    def __constructBuildAndRunTimeDependencyGraph(self, package):
124
+        basePackage = SPECS.getData().getSpecName(package)
125
+
126
+        addBuildTimeGraph = True
127
+        addRunTimeGraph = True
128
+        if basePackage in self.__buildDependencyGraph:
130 129
             addBuildTimeGraph = False
131
-        if self.__runTimeDependencyGraph.has_key(basePackage):
132
-            addRunTimeGraph=False
133
-        
134
-        nextPackagesToConstructGraph=[]
130
+        if basePackage in self.__runTimeDependencyGraph:
131
+            addRunTimeGraph = False
132
+
133
+        nextPackagesToConstructGraph = []
135 134
         if addBuildTimeGraph:
136
-            listDependentRpmPackages=SPECS.getData().getBuildRequiresForPackage(basePackage)
137
-            listDependentPackages=[]
135
+            listDependentRpmPackages = SPECS.getData().getBuildRequiresForPackage(basePackage)
136
+            listDependentPackages = []
138 137
             for rpmPkg in listDependentRpmPackages:
139
-                basePkg=SPECS.getData().getSpecName(rpmPkg)
138
+                basePkg = SPECS.getData().getSpecName(rpmPkg)
140 139
                 if basePkg not in listDependentPackages:
141 140
                     listDependentPackages.append(basePkg)
142
-            self.__buildDependencyGraph[basePackage]=listDependentPackages
141
+            self.__buildDependencyGraph[basePackage] = listDependentPackages
143 142
             nextPackagesToConstructGraph.extend(listDependentPackages)
144
-        
143
+
145 144
         if addRunTimeGraph:
146
-            listRpmPackages=SPECS.getData().getPackages(basePackage)
145
+            listRpmPackages = SPECS.getData().getPackages(basePackage)
147 146
             for rpmPkg in listRpmPackages:
148
-                listDependentRpmPackages=SPECS.getData().getRequiresAllForPackage(rpmPkg)
149
-                self.__runTimeDependencyGraph[rpmPkg]=listDependentRpmPackages[:]
147
+                listDependentRpmPackages = SPECS.getData().getRequiresAllForPackage(rpmPkg)
148
+                self.__runTimeDependencyGraph[rpmPkg] = listDependentRpmPackages[:]
150 149
                 nextPackagesToConstructGraph.extend(listDependentRpmPackages)
151 150
 
152 151
         for pkg in nextPackagesToConstructGraph:
153 152
             self.__constructBuildAndRunTimeDependencyGraph(pkg)
154
-    
155
-    def __readDependencyGraphAndCyclesForGivenPackages(self,listPackages):
153
+
154
+    def __readDependencyGraphAndCyclesForGivenPackages(self, listPackages):
156 155
         self.logger.info("Reading dependency graph to check for cycles")
157 156
         for pkg in listPackages:
158 157
             self.__constructBuildAndRunTimeDependencyGraph(pkg)
159
-        packagesToBUild=self.__buildDependencyGraph.keys()
160
-        for pkg in packagesToBUild:
161
-            sortedPackageList,circularDependentPackages = self.topologicalSortPackages(self.__buildDependencyGraph,pkg)
162
-            if len(circularDependentPackages) > 0 :
158
+
159
+        for pkg in self.__buildDependencyGraph.keys():
160
+            sortedPackageList, circularDependentPackages = self.topologicalSortPackages(
161
+                self.__buildDependencyGraph, pkg)
162
+            if len(circularDependentPackages) > 0:
163 163
                 self.logger.error("Found circular dependency")
164 164
                 self.logger.error(circularDependentPackages)
165 165
                 raise Exception("Build Time Circular Dependency")
166
-            self.__sortedBuildDependencyGraph[pkg]=sortedPackageList
167
-        sortedPackageList,circularDependentPackages = self.topologicalSortPackages(self.__runTimeDependencyGraph)
168
-        if len(circularDependentPackages) > 0 :
166
+            self.__sortedBuildDependencyGraph[pkg] = sortedPackageList
167
+        sortedPackageList, circularDependentPackages = self.topologicalSortPackages(
168
+            self.__runTimeDependencyGraph)
169
+        if len(circularDependentPackages) > 0:
169 170
             self.__findCircularDependencies(circularDependentPackages)
170
-    
171
+
171 172
     def topologicalSortPackages(self, dependencyGraph, package=None):
172
-        noDepPackages = Set()
173
+        noDepPackages = set()
173 174
         sortedPackageList = []
174 175
         dependentOfPackage = dict()
175
-        
176
-        dependentPackages={}
176
+
177
+        dependentPackages = {}
177 178
         if package is None:
178
-            dependentPackages=copy.deepcopy(dependencyGraph)
179
+            dependentPackages = copy.deepcopy(dependencyGraph)
179 180
         else:
180
-            listDepPkgs= Set()
181
+            listDepPkgs = set()
181 182
             listDepPkgs.add(package)
182 183
             while listDepPkgs:
183 184
                 pkg = listDepPkgs.pop()
184
-                if dependentPackages.has_key(pkg):
185
+                if pkg in dependentPackages:
185 186
                     continue
186
-                dependentPackages[pkg]=dependencyGraph[pkg][:]
187
+                dependentPackages[pkg] = dependencyGraph[pkg][:]
187 188
                 for depPkg in dependencyGraph[pkg]:
188 189
                     listDepPkgs.add(depPkg)
189
-                    
190
+
190 191
         #Find packages with no dependencies and generate dependentof_package edge list
191 192
         for pkg in dependentPackages:
192 193
             if len(dependentPackages[pkg]) == 0:
193 194
                 noDepPackages.add(pkg)
194 195
             else:
195 196
                 for depPkg in dependentPackages[pkg]:
196
-                    if not dependentOfPackage.has_key(depPkg):
197
-                        dependentOfPackage[depPkg]=[pkg]
197
+                    if depPkg not in dependentOfPackage:
198
+                        dependentOfPackage[depPkg] = [pkg]
198 199
                     else:
199 200
                         if pkg not in dependentOfPackage[depPkg]:
200 201
                             dependentOfPackage[depPkg].append(pkg)
201
-        
202
+
202 203
         while noDepPackages:
203 204
             pkg = noDepPackages.pop()
204 205
             sortedPackageList.append(pkg)
... ...
@@ -206,29 +207,27 @@ class PackageBuildDataGenerator(object):
206 206
                 for childPkg in list(dependentOfPackage.get(pkg)):
207 207
                     dependentOfPackage.get(pkg).remove(childPkg)
208 208
                     dependentPackages[childPkg].remove(pkg)
209
-                    if len(dependentPackages[childPkg])==0:
209
+                    if len(dependentPackages[childPkg]) == 0:
210 210
                         noDepPackages.add(childPkg)
211
-        
211
+
212 212
         # creating circular dependency graph for given dependency graph
213
-        circularDependencyGraph={}
214
-        listCircularPkg = dependentPackages.keys()
215
-        for pkg in listCircularPkg:
213
+        circularDependencyGraph = {}
214
+        for pkg in dependentPackages.keys():
216 215
             if len(dependentPackages[pkg]) != 0:
217
-                circularDependencyGraph[pkg]=dependentPackages[pkg]
218
-            
219
-        #return (non-circular dependent package in sorted order and circular dependent package list in a dependencyGraph)
220
-        return sortedPackageList,circularDependencyGraph
221
-    
222
-    
223
-    def __constructDependencyMap(self,cyclicDependencyGraph):
216
+                circularDependencyGraph[pkg] = dependentPackages[pkg]
217
+
218
+        #return (non-circular dependent package in sorted order and circular dependent
219
+        #package list in a dependencyGraph)
220
+        return sortedPackageList, circularDependencyGraph
221
+
222
+    def __constructDependencyMap(self, yclicDependencyGraph):
224 223
         self.logger.info("Constructing dependency map from circular dependency graph.....")
225
-        constructDependencyMap={}
226
-        listNodes=cyclicDependencyGraph.keys()
227
-        for node in listNodes:
228
-            tmpDepNodeList=[]
224
+        constructDependencyMap = {}
225
+        for node in cyclicDependencyGraph.keys():
226
+            tmpDepNodeList = []
229 227
             tmpDepNodeList.append(node)
230
-            depNodeList=[]
231
-            while len(tmpDepNodeList)!=0:
228
+            depNodeList = []
229
+            while len(tmpDepNodeList) != 0:
232 230
                 currentNode = tmpDepNodeList.pop()
233 231
                 addDepNodeList = cyclicDependencyGraph[currentNode]
234 232
                 depNodeList.append(currentNode)
... ...
@@ -239,50 +238,43 @@ class PackageBuildDataGenerator(object):
239 239
                         if depNode not in tmpDepNodeList:
240 240
                             tmpDepNodeList.append(depNode)
241 241
             depNodeList.remove(node)
242
-            constructDependencyMap[node]=depNodeList
242
+            constructDependencyMap[node] = depNodeList
243 243
         self.logger.info("Dependency Map:")
244 244
         self.logger.info(constructDependencyMap)
245 245
         return constructDependencyMap
246
-        
247
-        
248
-    def __findCircularDependencies(self,cyclicDependencyGraph):
246
+
247
+    def __findCircularDependencies(self, cyclicDependencyGraph):
249 248
         self.logger.info("Looking for circular dependencies")
250 249
         if len(cyclicDependencyGraph) == 0:
251 250
             return
252 251
         #step1: construct dependency map from dependency graph
253
-        constructDependencyMap=self.__constructDependencyMap(cyclicDependencyGraph)
254
-        
252
+        constructDependencyMap = self.__constructDependencyMap(cyclicDependencyGraph)
253
+
255 254
         #step2: find cycles in dependency map
256 255
         self.logger.info("Finding and adding cycles using constructed dependency map......")
257
-        cycleCount=0
258
-        listNodes=cyclicDependencyGraph.keys()
259
-        for node in listNodes:
260
-            listDepPkg=constructDependencyMap[node]
261
-            cycPkgs=[]
262
-            if not self.__mapPackageToCycle.has_key(node):
256
+        cycleCount = 0
257
+        for node in cyclicDependencyGraph.keys():
258
+            listDepPkg = constructDependencyMap[node]
259
+            cycPkgs = []
260
+            if node not in self.__mapPackageToCycle:
263 261
                 for depPkg in listDepPkg:
264 262
                     x = constructDependencyMap[depPkg]
265 263
                     if node in x:
266 264
                         cycPkgs.append(depPkg)
267
-                
265
+
268 266
                 if len(cycPkgs) != 0:
269 267
                     cycPkgs.append(node)
270
-                    cycleName="cycle"+str(PackageBuildDataGenerator.cycleCount)
271
-                    PackageBuildDataGenerator.cycleCount=PackageBuildDataGenerator.cycleCount+1
268
+                    cycleName = "cycle" + str(PackageBuildDataGenerator.cycleCount)
269
+                    PackageBuildDataGenerator.cycleCount += 1
272 270
                     for x in cycPkgs:
273
-                        self.__mapPackageToCycle[x]=cycleName
274
-                    self.__mapCyclesToPackageList[cycleName]=cycPkgs
271
+                        self.__mapPackageToCycle[x] = cycleName
272
+                    self.__mapCyclesToPackageList[cycleName] = cycPkgs
275 273
                     self.logger.info("New circular dependency found:")
276
-                    self.logger.info(cycleName+" "+ ",".join(cycPkgs))
277
-                    cycleCount = cycleCount + 1
278
-        
279
-        if cycleCount > 0 :
280
-            self.logger.info("Found "+str(cycleCount) + " cycles.")
274
+                    self.logger.info(cycleName + " " + ",".join(cycPkgs))
275
+                    cycleCount += 1
276
+
277
+        if cycleCount > 0:
278
+            self.logger.info("Found " + str(cycleCount) + " cycles.")
281 279
             self.logger.info("Successfully added all detected circular dependencies to list.")
282 280
         else:
283 281
             self.logger.info("No circular dependencies found.")
284
-        
285
-    
286
-
287
-    
288
-    
... ...
@@ -1,49 +1,51 @@
1
+import sys
2
+import os.path
1 3
 from PackageUtils import PackageUtils
2 4
 from Logger import Logger
3 5
 from ChrootUtils import ChrootUtils
4 6
 from ToolChainUtils import ToolChainUtils
5 7
 from CommandUtils import CommandUtils
6
-import os.path
7 8
 from constants import constants
8
-import shutil
9 9
 from SpecData import SPECS
10 10
 import docker
11
-import sys
12 11
 
13 12
 class PackageBuilderBase(object):
14
-    def __init__(self,mapPackageToCycles,listAvailableCyclicPackages,listBuildOptionPackages,pkgBuildOptionFile, pkgBuildType):
13
+    def __init__(self, mapPackageToCycles, listAvailableCyclicPackages,
14
+                 listBuildOptionPackages, pkgBuildOptionFile, pkgBuildType):
15 15
         # will be initialized in buildPackageThreadAPI()
16
-        self.logName=None
17
-        self.logPath=None
18
-        self.logger=None
19
-        self.package=None
16
+        self.logName = None
17
+        self.logPath = None
18
+        self.logger = None
19
+        self.package = None
20 20
         self.mapPackageToCycles = mapPackageToCycles
21 21
         self.listAvailableCyclicPackages = listAvailableCyclicPackages
22
-        self.listNodepsPackages = ["glibc","gmp","zlib","file","binutils","mpfr","mpc","gcc","ncurses","util-linux","groff","perl","texinfo","rpm","openssl","go"]
23
-        self.listBuildOptionPackages=listBuildOptionPackages
24
-        self.pkgBuildOptionFile=pkgBuildOptionFile
22
+        self.listNodepsPackages = ["glibc", "gmp", "zlib", "file", "binutils", "mpfr",
23
+                                   "mpc", "gcc", "ncurses", "util-linux", "groff", "perl",
24
+                                   "texinfo", "rpm", "openssl", "go"]
25
+        self.listBuildOptionPackages = listBuildOptionPackages
26
+        self.pkgBuildOptionFile = pkgBuildOptionFile
25 27
         self.pkgBuildType = pkgBuildType
26 28
 
27
-    def buildPackageThreadAPIPrepare(self,package,outputMap, threadName):
28
-        self.package=package
29
-        self.logName="build-"+package
30
-        self.logPath=constants.logPath+"/build-"+package
29
+    def buildPackageThreadAPIPrepare(self, package, outputMap, threadName):
30
+        self.package = package
31
+        self.logName = "build-" + package
32
+        self.logPath = constants.logPath + "/build-" + package
31 33
         if not os.path.isdir(self.logPath):
32 34
             cmdUtils = CommandUtils()
33
-            cmdUtils.runCommandInShell("mkdir -p "+self.logPath)
34
-        self.logger=Logger.getLogger(self.logName,self.logPath)
35
+            cmdUtils.runCommandInShell("mkdir -p " + self.logPath)
36
+        self.logger = Logger.getLogger(self.logName, self.logPath)
35 37
 
36
-    def findPackageNameFromRPMFile(self,rpmfile):
37
-        rpmfile=os.path.basename(rpmfile)
38
-        releaseindex=rpmfile.rfind("-")
38
+    def findPackageNameFromRPMFile(self, rpmfile):
39
+        rpmfile = os.path.basename(rpmfile)
40
+        releaseindex = rpmfile.rfind("-")
39 41
         if releaseindex == -1:
40
-            self.logger.error("Invalid rpm file:"+rpmfile)
42
+            self.logger.error("Invalid rpm file:" + rpmfile)
41 43
             return None
42
-        versionindex=rpmfile[0:releaseindex].rfind("-")
44
+        versionindex = rpmfile[0:releaseindex].rfind("-")
43 45
         if versionindex == -1:
44
-            self.logger.error("Invalid rpm file:"+rpmfile)
46
+            self.logger.error("Invalid rpm file:" + rpmfile)
45 47
             return None
46
-        packageName=rpmfile[0:versionindex]
48
+        packageName = rpmfile[0:versionindex]
47 49
         return packageName
48 50
 
49 51
     def findInstalledPackages(self, instanceID):
... ...
@@ -52,69 +54,76 @@ class PackageBuilderBase(object):
52 52
             listInstalledRPMs = pkgUtils.findInstalledRPMPackages(instanceID)
53 53
         elif self.pkgBuildType == "container":
54 54
             listInstalledRPMs = pkgUtils.findInstalledRPMPackagesInContainer(instanceID)
55
-        listInstalledPackages=[]
55
+        listInstalledPackages = []
56 56
         for installedRPM in listInstalledRPMs:
57
-            packageName=self.findPackageNameFromRPMFile(installedRPM)
57
+            packageName = self.findPackageNameFromRPMFile(installedRPM)
58 58
             if packageName is not None:
59 59
                 listInstalledPackages.append(packageName)
60 60
         return listInstalledPackages, listInstalledRPMs
61 61
 
62 62
     def checkIfPackageIsAlreadyBuilt(self):
63
-        basePkg=SPECS.getData().getSpecName(self.package)
64
-        listRPMPackages=SPECS.getData().getRPMPackages(basePkg)
65
-        packageIsAlreadyBuilt=True
66
-        pkgUtils = PackageUtils(self.logName,self.logPath)
63
+        basePkg = SPECS.getData().getSpecName(self.package)
64
+        listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
65
+        packageIsAlreadyBuilt = True
66
+        pkgUtils = PackageUtils(self.logName, self.logPath)
67 67
         for pkg in listRPMPackages:
68 68
             if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
69
-                packageIsAlreadyBuilt=False
69
+                packageIsAlreadyBuilt = False
70 70
                 break
71 71
         return packageIsAlreadyBuilt
72 72
 
73
-    def findRunTimeRequiredRPMPackages(self,rpmPackage):
74
-        listRequiredPackages=SPECS.getData().getRequiresForPackage(rpmPackage)
73
+    def findRunTimeRequiredRPMPackages(self, rpmPackage):
74
+        listRequiredPackages = SPECS.getData().getRequiresForPackage(rpmPackage)
75 75
         return listRequiredPackages
76 76
 
77 77
     def findBuildTimeRequiredPackages(self):
78
-        listRequiredPackages=SPECS.getData().getBuildRequiresForPackage(self.package)
78
+        listRequiredPackages = SPECS.getData().getBuildRequiresForPackage(self.package)
79 79
         return listRequiredPackages
80 80
 
81 81
     def findBuildTimeCheckRequiredPackages(self):
82
-        listRequiredPackages=SPECS.getData().getCheckBuildRequiresForPackage(self.package)
82
+        listRequiredPackages = SPECS.getData().getCheckBuildRequiresForPackage(self.package)
83 83
         return listRequiredPackages
84 84
 
85
-    def installPackage(self, pkgUtils, package, instanceID, destLogPath, listInstalledPackages, listInstalledRPMs):
86
-        latestRPM = os.path.basename(pkgUtils.findRPMFileForGivenPackage(package)).replace(".rpm", "")
85
+    def installPackage(self, pkgUtils, package, instanceID, destLogPath,
86
+                       listInstalledPackages, listInstalledRPMs):
87
+        latestRPM = os.path.basename(
88
+            pkgUtils.findRPMFileForGivenPackage(package)).replace(".rpm", "")
87 89
         if package in listInstalledPackages and latestRPM in listInstalledRPMs:
88 90
             return
89 91
         # mark it as installed -  to avoid cyclic recursion
90 92
         listInstalledPackages.append(package)
91 93
         listInstalledRPMs.append(latestRPM)
92
-        self.installDependentRunTimePackages(pkgUtils,package,instanceID,destLogPath,listInstalledPackages, listInstalledRPMs)
93
-        noDeps=False
94
-        if self.mapPackageToCycles.has_key(package):
94
+        self.installDependentRunTimePackages(pkgUtils, package, instanceID, destLogPath,
95
+                                             listInstalledPackages, listInstalledRPMs)
96
+        noDeps = False
97
+        if package in self.mapPackageToCycles:
95 98
             noDeps = True
96 99
         if package in self.listNodepsPackages:
97
-            noDeps=True
100
+            noDeps = True
98 101
         if package in constants.noDepsPackageList:
99
-            noDeps=True
102
+            noDeps = True
100 103
         if self.pkgBuildType == "chroot":
101
-            pkgUtils.installRPM(package,instanceID,noDeps,destLogPath)
104
+            pkgUtils.installRPM(package, instanceID, noDeps, destLogPath)
102 105
         elif self.pkgBuildType == "container":
103 106
             pkgUtils.prepRPMforInstallInContainer(package, instanceID, noDeps, destLogPath)
104 107
 
105
-    def installDependentRunTimePackages(self,pkgUtils,package,instanceID,destLogPath,listInstalledPackages, listInstalledRPMs):
106
-        listRunTimeDependentPackages=self.findRunTimeRequiredRPMPackages(package)
108
+    def installDependentRunTimePackages(self, pkgUtils, package, instanceID, destLogPath,
109
+                                        listInstalledPackages, listInstalledRPMs):
110
+        listRunTimeDependentPackages = self.findRunTimeRequiredRPMPackages(package)
107 111
         if len(listRunTimeDependentPackages) != 0:
108 112
             for pkg in listRunTimeDependentPackages:
109
-                if self.mapPackageToCycles.has_key(pkg) and pkg not in self.listAvailableCyclicPackages:
113
+                if pkg in self.mapPackageToCycles and pkg not in self.listAvailableCyclicPackages:
110 114
                     continue
111
-                latestPkgRPM = os.path.basename(pkgUtils.findRPMFileForGivenPackage(pkg)).replace(".rpm", "") 
115
+                latestPkgRPM = os.path.basename(
116
+                    pkgUtils.findRPMFileForGivenPackage(pkg)).replace(".rpm", "")
112 117
                 if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs:
113 118
                     continue
114
-                self.installPackage(pkgUtils, pkg, instanceID, destLogPath, listInstalledPackages, listInstalledRPMs)
119
+                self.installPackage(pkgUtils, pkg, instanceID, destLogPath,
120
+                                    listInstalledPackages, listInstalledRPMs)
115 121
 
116 122
 class PackageBuilderContainer(object):
117
-    def __init__(self, mapPackageToCycles, listAvailableCyclicPackages, listBuildOptionPackages, pkgBuildOptionFile, pkgBuildType):
123
+    def __init__(self, mapPackageToCycles, listAvailableCyclicPackages, listBuildOptionPackages,
124
+                 pkgBuildOptionFile, pkgBuildType):
118 125
         self.buildContainerImage = "photon_build_container:latest"
119 126
         self.dockerClient = docker.from_env(version="auto")
120 127
 
... ...
@@ -125,19 +134,20 @@ class PackageBuilderContainer(object):
125 125
         self.base.buildPackageThreadAPIPrepare(package, outputMap, threadName)
126 126
         try:
127 127
             self.buildPackage()
128
-            outputMap[threadName]=True
128
+            outputMap[threadName] = True
129 129
         except Exception as e:
130 130
             # TODO: self.logger might be None
131 131
             self.base.logger.exception(e)
132
-            outputMap[threadName]=False
132
+            outputMap[threadName] = False
133 133
             raise e
134 134
 
135
-    def prepareBuildContainer(self, containerTaskName, packageName, isToolChainPackage=False):
135
+    def prepareBuildContainer(self, containerTaskName, packageName,
136
+                              isToolChainPackage=False):
136 137
         # Prepare an empty chroot environment to let docker use the BUILD folder.
137 138
         # This avoids docker using overlayFS which will cause make check failure.
138
-        chrootName="build-"+packageName
139
+        chrootName = "build-" + packageName
139 140
         chrUtils = ChrootUtils(self.base.logName, self.base.logPath)
140
-        returnVal,chrootID = chrUtils.createChroot(chrootName)
141
+        returnVal, chrootID = chrUtils.createChroot(chrootName)
141 142
         if not returnVal:
142 143
             raise Exception("Unable to prepare build root")
143 144
         cmdUtils = CommandUtils()
... ...
@@ -146,15 +156,17 @@ class PackageBuilderContainer(object):
146 146
 
147 147
         containerID = None
148 148
         mountVols = {
149
-                        constants.prevPublishRPMRepo: {'bind': '/publishrpms', 'mode': 'ro'},
150
-                        constants.prevPublishXRPMRepo: {'bind': '/publishxrpms', 'mode': 'ro'},
151
-                        constants.tmpDirPath: {'bind': '/tmp', 'mode': 'rw'},
152
-                        constants.rpmPath: {'bind': constants.topDirPath + "/RPMS", 'mode': 'rw'},
153
-                        constants.sourceRpmPath: {'bind': constants.topDirPath + "/SRPMS", 'mode': 'rw'},
154
-                        constants.logPath + "/" + self.base.logName: {'bind': constants.topDirPath + "/LOGS", 'mode': 'rw'},
155
-                        chrootID + constants.topDirPath + "/BUILD": {'bind': constants.topDirPath + "/BUILD", 'mode': 'rw'},
156
-                        constants.dockerUnixSocket: {'bind': constants.dockerUnixSocket, 'mode': 'rw'}
157
-                    }
149
+            constants.prevPublishRPMRepo: {'bind': '/publishrpms', 'mode': 'ro'},
150
+            constants.prevPublishXRPMRepo: {'bind': '/publishxrpms', 'mode': 'ro'},
151
+            constants.tmpDirPath: {'bind': '/tmp', 'mode': 'rw'},
152
+            constants.rpmPath: {'bind': constants.topDirPath + "/RPMS", 'mode': 'rw'},
153
+            constants.sourceRpmPath: {'bind': constants.topDirPath + "/SRPMS", 'mode': 'rw'},
154
+            constants.logPath + "/" + self.base.logName: {'bind': constants.topDirPath + "/LOGS",
155
+                                                          'mode': 'rw'},
156
+            chrootID + constants.topDirPath + "/BUILD": {'bind': constants.topDirPath + "/BUILD",
157
+                                                         'mode': 'rw'},
158
+            constants.dockerUnixSocket: {'bind': constants.dockerUnixSocket, 'mode': 'rw'}
159
+            }
158 160
 
159 161
         containerName = containerTaskName
160 162
         containerName = containerName.replace("+", "p")
... ...
@@ -163,10 +175,14 @@ class PackageBuilderContainer(object):
163 163
             if oldContainerID is not None:
164 164
                 oldContainerID.remove(force=True)
165 165
         except docker.errors.NotFound:
166
-            sys.exc_clear()
166
+            try:
167
+                sys.exc_clear()
168
+            except:
169
+                pass
167 170
 
168 171
         try:
169
-            self.base.logger.info("BuildContainer-prepareBuildContainer: Starting build container: " + containerName)
172
+            self.base.logger.info("BuildContainer-prepareBuildContainer: " +
173
+                                  "Starting build container: " + containerName)
170 174
             #TODO: Is init=True equivalent of --sig-proxy?
171 175
             privilegedDocker = False
172 176
             cap_list = ['SYS_PTRACE']
... ...
@@ -174,20 +190,22 @@ class PackageBuilderContainer(object):
174 174
                 privilegedDocker = True
175 175
 
176 176
             containerID = self.dockerClient.containers.run(self.buildContainerImage,
177
-                                                               detach=True,
178
-                                                               cap_add=cap_list,
179
-                                                               privileged=privilegedDocker,
180
-                                                               name=containerName,
181
-                                                               network_mode="host",
182
-                                                               volumes=mountVols,
183
-                                                               command="/bin/bash -l -c /wait.sh")
184
-
185
-            self.base.logger.debug("Started Photon build container for task " + containerTaskName
186
-                               + " ID: " + containerID.short_id)
177
+                                                           detach=True,
178
+                                                           cap_add=cap_list,
179
+                                                           privileged=privilegedDocker,
180
+                                                           name=containerName,
181
+                                                           network_mode="host",
182
+                                                           volumes=mountVols,
183
+                                                           command="/bin/bash -l -c /wait.sh")
184
+
185
+            self.base.logger.debug("Started Photon build container for task " + containerTaskName +
186
+                                   " ID: " + containerID.short_id)
187 187
             if not containerID:
188
-                raise Exception("Unable to start Photon build container for task " + containerTaskName)
188
+                raise Exception("Unable to start Photon build container for task " +
189
+                                containerTaskName)
189 190
         except Exception as e:
190
-            self.base.logger.debug("Unable to start Photon build container for task " + containerTaskName)
191
+            self.base.logger.debug("Unable to start Photon build container for task " +
192
+                                   containerTaskName)
191 193
             raise e
192 194
         return containerID, chrootID
193 195
 
... ...
@@ -212,46 +230,55 @@ class PackageBuilderContainer(object):
212 212
             isToolChainPackage = True
213 213
         destLogPath = constants.logPath + "/build-" + self.base.package
214 214
         try:
215
-            containerID, chrootID = self.prepareBuildContainer(containerTaskName, self.base.package, isToolChainPackage)
215
+            containerID, chrootID = self.prepareBuildContainer(
216
+                containerTaskName, self.base.package, isToolChainPackage)
216 217
 
217 218
             tcUtils = ToolChainUtils(self.base.logName, self.base.logPath)
218 219
             if self.base.package in constants.perPackageToolChain:
219 220
                 self.base.logger.debug(constants.perPackageToolChain[self.base.package])
220
-                tcUtils.installCustomToolChainRPMSinContainer(containerID,
221
-                                                              constants.perPackageToolChain[self.base.package],
222
-                                                              self.base.package);
221
+                tcUtils.installCustomToolChainRPMSinContainer(
222
+                    containerID,
223
+                    constants.perPackageToolChain[self.base.package],
224
+                    self.base.package)
223 225
 
224 226
             listInstalledPackages, listInstalledRPMs = self.base.findInstalledPackages(containerID)
225 227
             self.base.logger.info(listInstalledPackages)
226 228
             listDependentPackages = self.base.findBuildTimeRequiredPackages()
227 229
             if constants.rpmCheck and self.base.package in constants.testForceRPMS:
228 230
                 listDependentPackages.extend(self.base.findBuildTimeCheckRequiredPackages())
229
-                testPackages=set(constants.listMakeCheckRPMPkgtoInstall)-set(listInstalledPackages)-set([self.base.package])
231
+                testPackages = (set(constants.listMakeCheckRPMPkgtoInstall) -
232
+                                set(listInstalledPackages) -
233
+                                set([self.base.package]))
230 234
                 listDependentPackages.extend(testPackages)
231
-                listDependentPackages=list(set(listDependentPackages))
235
+                listDependentPackages = list(set(listDependentPackages))
232 236
 
233 237
             pkgUtils = PackageUtils(self.base.logName, self.base.logPath)
234 238
             if len(listDependentPackages) != 0:
235
-                self.base.logger.info("BuildContainer-buildPackage: Installing dependent packages..")
239
+                self.base.logger.info("BuildContainer-buildPackage: " +
240
+                                      "Installing dependent packages..")
236 241
                 self.base.logger.info(listDependentPackages)
237 242
                 for pkg in listDependentPackages:
238
-                    self.base.installPackage(pkgUtils, pkg, containerID, destLogPath, listInstalledPackages, listInstalledRPMs)
243
+                    self.base.installPackage(pkgUtils, pkg, containerID, destLogPath,
244
+                                             listInstalledPackages, listInstalledRPMs)
239 245
                 pkgUtils.installRPMSInAOneShotInContainer(containerID, destLogPath)
240
-                self.base.logger.info("Finished installing the build time dependent packages......")
246
+                self.base.logger.info("Finished installing the build time dependent packages....")
241 247
 
242
-            self.base.logger.info("BuildContainer-buildPackage: Start building the package: " + self.base.package)
248
+            self.base.logger.info("BuildContainer-buildPackage: Start building the package: " +
249
+                                  self.base.package)
243 250
             pkgUtils.adjustGCCSpecsInContainer(self.base.package, containerID, destLogPath)
244 251
             pkgUtils.buildRPMSForGivenPackageInContainer(
245
-                                               self.base.package,
246
-                                               containerID,
247
-                                               self.base.listBuildOptionPackages,
248
-                                               self.base.pkgBuildOptionFile,
249
-                                               destLogPath)
250
-            self.base.logger.info("BuildContainer-buildPackage: Successfully built the package: " + self.base.package)
252
+                self.base.package,
253
+                containerID,
254
+                self.base.listBuildOptionPackages,
255
+                self.base.pkgBuildOptionFile,
256
+                destLogPath)
257
+            self.base.logger.info("BuildContainer-buildPackage: Successfully built the package: " +
258
+                                  self.base.package)
251 259
         except Exception as e:
252 260
             self.base.logger.error("Failed while building package:" + self.base.package)
253 261
             if containerID is not None:
254
-                self.base.logger.debug("Container " + containerID.short_id + " retained for debugging.")
262
+                self.base.logger.debug("Container " + containerID.short_id +
263
+                                       " retained for debugging.")
255 264
             logFileName = os.path.join(destLogPath, self.base.package + ".log")
256 265
             fileLog = os.popen('tail -n 20 ' + logFileName).read()
257 266
             self.base.logger.debug(fileLog)
... ...
@@ -266,31 +293,32 @@ class PackageBuilderContainer(object):
266 266
             chrUtils.destroyChroot(chrootID)
267 267
 
268 268
 class PackageBuilderChroot(object):
269
-    def __init__(self, mapPackageToCycles, listAvailableCyclicPackages, listBuildOptionPackages, pkgBuildOptionFile, pkgBuildType):
269
+    def __init__(self, mapPackageToCycles, listAvailableCyclicPackages, listBuildOptionPackages,
270
+                 pkgBuildOptionFile, pkgBuildType):
270 271
         self.base = PackageBuilderBase(mapPackageToCycles, listAvailableCyclicPackages,
271 272
                                        listBuildOptionPackages, pkgBuildOptionFile, pkgBuildType)
272 273
 
273 274
     def buildPackageThreadAPI(self, package, outputMap, threadName):
274
-        self.base.buildPackageThreadAPIPrepare(package,outputMap, threadName)
275
+        self.base.buildPackageThreadAPIPrepare(package, outputMap, threadName)
275 276
         try:
276 277
             self.buildPackage()
277
-            outputMap[threadName]=True
278
+            outputMap[threadName] = True
278 279
         except Exception as e:
279 280
             # TODO: self.logger might be None
280 281
             self.base.logger.exception(e)
281
-            outputMap[threadName]=False
282
+            outputMap[threadName] = False
282 283
             raise e
283 284
 
284 285
     def prepareBuildRoot(self):
285
-        chrootID=None
286
-        chrootName="build-"+self.base.package
286
+        chrootID = None
287
+        chrootName = "build-" + self.base.package
287 288
         try:
288
-            chrUtils = ChrootUtils(self.base.logName,self.base.logPath)
289
-            returnVal,chrootID = chrUtils.createChroot(chrootName)
289
+            chrUtils = ChrootUtils(self.base.logName, self.base.logPath)
290
+            returnVal, chrootID = chrUtils.createChroot(chrootName)
290 291
             self.base.logger.debug("Created new chroot: " + chrootID)
291 292
             if not returnVal:
292 293
                 raise Exception("Unable to prepare build root")
293
-            tUtils=ToolChainUtils(self.base.logName,self.base.logPath)
294
+            tUtils = ToolChainUtils(self.base.logName, self.base.logPath)
294 295
             tUtils.installToolChainRPMS(chrootID, self.base.package, self.base.logPath)
295 296
         except Exception as e:
296 297
             if chrootID is not None:
... ...
@@ -311,33 +339,39 @@ class PackageBuilderChroot(object):
311 311
                 self.base.logger.info("Skipping testing the package:" + self.base.package)
312 312
                 return
313 313
 
314
-        chrUtils = ChrootUtils(self.base.logName,self.base.logPath)
315
-        chrootID=None
314
+        chrUtils = ChrootUtils(self.base.logName, self.base.logPath)
315
+        chrootID = None
316 316
         try:
317 317
             chrootID = self.prepareBuildRoot()
318 318
             listInstalledPackages, listInstalledRPMs = self.base.findInstalledPackages(chrootID)
319
-            listDependentPackages=self.base.findBuildTimeRequiredPackages()
319
+            listDependentPackages = self.base.findBuildTimeRequiredPackages()
320 320
             if constants.rpmCheck and self.base.package in constants.testForceRPMS:
321 321
                 listDependentPackages.extend(self.base.findBuildTimeCheckRequiredPackages())
322
-                testPackages=set(constants.listMakeCheckRPMPkgtoInstall)-set(listInstalledPackages)-set([self.base.package])
322
+                testPackages = (set(constants.listMakeCheckRPMPkgtoInstall) -
323
+                                set(listInstalledPackages) -
324
+                                set([self.base.package]))
323 325
                 listDependentPackages.extend(testPackages)
324
-                listDependentPackages=list(set(listDependentPackages))
326
+                listDependentPackages = list(set(listDependentPackages))
325 327
 
326
-            pkgUtils = PackageUtils(self.base.logName,self.base.logPath)
328
+            pkgUtils = PackageUtils(self.base.logName, self.base.logPath)
327 329
             if len(listDependentPackages) != 0:
328 330
                 self.base.logger.info("Installing the build time dependent packages......")
329 331
                 for pkg in listDependentPackages:
330
-                    self.base.installPackage(pkgUtils, pkg, chrootID, self.base.logPath, listInstalledPackages, listInstalledRPMs)
332
+                    self.base.installPackage(pkgUtils, pkg, chrootID, self.base.logPath,
333
+                                             listInstalledPackages, listInstalledRPMs)
331 334
                 pkgUtils.installRPMSInAOneShot(chrootID, self.base.logPath)
332
-                self.base.logger.info("Finished installing the build time dependent packages......")
335
+                self.base.logger.info("Finished installing the build time dependent packages....")
333 336
 
334 337
             pkgUtils.adjustGCCSpecs(self.base.package, chrootID, self.base.logPath)
335
-            pkgUtils.buildRPMSForGivenPackage(self.base.package, chrootID,self.base.listBuildOptionPackages,
336
-                                              self.base.pkgBuildOptionFile, self.base.logPath)
338
+            pkgUtils.buildRPMSForGivenPackage(self.base.package, chrootID,
339
+                                              self.base.listBuildOptionPackages,
340
+                                              self.base.pkgBuildOptionFile,
341
+                                              self.base.logPath)
337 342
             self.base.logger.info("Successfully built the package:" + self.base.package)
338 343
         except Exception as e:
339 344
             self.base.logger.error("Failed while building package:" + self.base.package)
340
-            self.base.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.")
345
+            self.base.logger.debug("Chroot with ID: " + chrootID +
346
+                                   " not deleted for debugging.")
341 347
             logFileName = os.path.join(self.base.logPath, self.base.package + ".log")
342 348
             fileLog = os.popen('tail -n 100 ' + logFileName).read()
343 349
             self.base.logger.debug(fileLog)
... ...
@@ -1,7 +1,7 @@
1 1
 import json
2
+import os.path
2 3
 from Logger import Logger
3 4
 from constants import constants
4
-import os.path
5 5
 from CommandUtils import CommandUtils
6 6
 from PackageUtils import PackageUtils
7 7
 from SpecData import SPECS
... ...
@@ -13,13 +13,13 @@ class PackageInfo(object):
13 13
             logName = "PackageInfo"
14 14
         if logPath is None:
15 15
             logPath = constants.logPath
16
-        self.logName=logName
17
-        self.logPath=logPath
18
-        self.logger=Logger.getLogger(logName,logPath)
16
+        self.logName = logName
17
+        self.logPath = logPath
18
+        self.logger = Logger.getLogger(logName, logPath)
19 19
         self.pkgList = {}
20 20
 
21 21
     def loadPackagesData(self):
22
-        listPackages =  SPECS.getData().getListPackages()
22
+        listPackages = SPECS.getData().getListPackages()
23 23
         listPackages.sort()
24 24
         listRPMFiles = []
25 25
         cmdUtils = CommandUtils()
... ...
@@ -27,33 +27,34 @@ class PackageInfo(object):
27 27
             release = SPECS.getData().getRelease(package)
28 28
             version = SPECS.getData().getVersion(package)
29 29
             listRPMPackages = SPECS.getData().getRPMPackages(package)
30
-            srpmFileName = package+"-"+version+"-"+release+".src.rpm"
30
+            srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
31 31
             srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
32 32
             srpmFile = None
33 33
             if len(srpmFiles) == 1:
34 34
                 srpmFile = srpmFiles[0]
35
-            debugrpmFileName = package+"-debuginfo-"+version+"-"+release+"*"
35
+            debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
36 36
             debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
37 37
             debugrpmFile = None
38 38
             if len(debugrpmFiles) == 1:
39 39
                 debugrpmFile = debugrpmFiles[0]
40
-            pkgUtils = PackageUtils(self.logName,self.logPath)
40
+            pkgUtils = PackageUtils(self.logName, self.logPath)
41 41
             for rpmPkg in listRPMPackages:
42 42
                 rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg)
43 43
                 if rpmFile is not None:
44 44
                     listRPMFiles.append(rpmFile)
45
-                    listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile, "debugrpm":debugrpmFile}
45
+                    listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile,
46
+                                         "debugrpm":debugrpmFile}
46 47
                     self.pkgList[rpmPkg] = listPkgAttributes
47
-                    self.logger.debug("Added "+rpmPkg +" rpm package to the list")
48
+                    self.logger.debug("Added " + rpmPkg + " rpm package to the list")
48 49
                 else:
49
-                    self.logger.error("Missing rpm file for package:"+rpmPkg)
50
+                    self.logger.error("Missing rpm file for package:" + rpmPkg)
50 51
 
51 52
     def writePkgListToFile(self, fileName):
52
-         self.logger.info("Writing package list to the json file")
53
-         cmdUtils=CommandUtils()
54
-         dirPath=os.path.basename(fileName)
55
-         if not os.path.isdir(dirPath):
56
-             cmdUtils.runCommandInShell("mkdir -p "+dirPath)
57
-         pkgInfoFile = open(fileName,'w+')
58
-         json.dump(self.pkgList, pkgInfoFile, indent=4)
59
-         pkgInfoFile.close()
53
+        self.logger.info("Writing package list to the json file")
54
+        cmdUtils = CommandUtils()
55
+        dirPath = os.path.basename(fileName)
56
+        if not os.path.isdir(dirPath):
57
+            cmdUtils.runCommandInShell("mkdir -p " + dirPath)
58
+        pkgInfoFile = open(fileName, 'w+')
59
+        json.dump(self.pkgList, pkgInfoFile, indent=4)
60
+        pkgInfoFile.close()
... ...
@@ -1,9 +1,9 @@
1
+import os
2
+import threading
1 3
 from PackageBuildDataGenerator import PackageBuildDataGenerator
2 4
 from Logger import Logger
3
-import threading
4 5
 from constants import constants
5 6
 import docker
6
-import os
7 7
 from ChrootUtils import ChrootUtils
8 8
 from CommandUtils import CommandUtils
9 9
 from PackageUtils import PackageUtils
... ...
@@ -14,67 +14,70 @@ from SpecData import SPECS
14 14
 
15 15
 class PackageManager(object):
16 16
 
17
-    def __init__(self,logName=None,logPath=None,pkgBuildType="chroot"):
17
+    def __init__(self, logName=None, logPath=None, pkgBuildType="chroot"):
18 18
         if logName is None:
19 19
             logName = "PackageManager"
20 20
         if logPath is None:
21 21
             logPath = constants.logPath
22
-        self.logName=logName
23
-        self.logPath=logPath
24
-        self.logger=Logger.getLogger(logName,logPath)
25
-        self.mapCyclesToPackageList={}
26
-        self.mapPackageToCycle={}
27
-        self.sortedPackageList=[]
22
+        self.logName = logName
23
+        self.logPath = logPath
24
+        self.logger = Logger.getLogger(logName, logPath)
25
+        self.mapCyclesToPackageList = {}
26
+        self.mapPackageToCycle = {}
27
+        self.sortedPackageList = []
28 28
         self.listOfPackagesAlreadyBuilt = []
29
-        self.listThreads={}
30
-        self.mapOutputThread={}
31
-        self.mapThreadsLaunchTime={}
32
-        self.listAvailableCyclicPackages=[]
33
-        self.listBuildOptionPackages=[]
34
-        self.pkgBuildOptionFile=""
35
-        self.pkgBuildType=pkgBuildType
29
+        self.listThreads = {}
30
+        self.mapOutputThread = {}
31
+        self.mapThreadsLaunchTime = {}
32
+        self.listAvailableCyclicPackages = []
33
+        self.listBuildOptionPackages = []
34
+        self.pkgBuildOptionFile = ""
35
+        self.pkgBuildType = pkgBuildType
36 36
         if self.pkgBuildType == "container":
37 37
             self.dockerClient = docker.from_env(version="auto")
38 38
 
39 39
     def readPackageBuildData(self, listPackages):
40 40
         try:
41
-            pkgBuildDataGen = PackageBuildDataGenerator(self.logName,self.logPath)
42
-            self.mapCyclesToPackageList,self.mapPackageToCycle,self.sortedPackageList = pkgBuildDataGen.getPackageBuildData(listPackages)
43
-        except:
41
+            pkgBuildDataGen = PackageBuildDataGenerator(self.logName, self.logPath)
42
+            self.mapCyclesToPackageList, self.mapPackageToCycle, self.sortedPackageList = (
43
+                pkgBuildDataGen.getPackageBuildData(listPackages))
44
+
45
+        except Exception as e:
46
+            self.logger.exception(e)
44 47
             self.logger.error("unable to get sorted list")
45 48
             return False
46 49
         return True
47 50
 
48 51
     def readAlreadyAvailablePackages(self):
49
-        listAvailablePackages=[]
50
-        listFoundRPMPackages=[]
51
-        listRPMFiles=[]
52
-        listDirectorys=[]
52
+        listAvailablePackages = []
53
+        listFoundRPMPackages = []
54
+        listRPMFiles = []
55
+        listDirectorys = []
53 56
         listDirectorys.append(constants.rpmPath)
54 57
         if constants.inputRPMSPath is not None:
55 58
             listDirectorys.append(constants.inputRPMSPath)
56 59
 
57 60
         while len(listDirectorys) > 0:
58
-            dirPath=listDirectorys.pop()
61
+            dirPath = listDirectorys.pop()
59 62
             for dirEntry in os.listdir(dirPath):
60 63
                 dirEntryPath = os.path.join(dirPath, dirEntry)
61 64
                 if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(".rpm"):
62 65
                     listRPMFiles.append(dirEntryPath)
63 66
                 elif os.path.isdir(dirEntryPath):
64 67
                     listDirectorys.append(dirEntryPath)
65
-        pkgUtils = PackageUtils(self.logName,self.logPath)
68
+        pkgUtils = PackageUtils(self.logName, self.logPath)
66 69
         for rpmfile in listRPMFiles:
67
-            package,version,release = pkgUtils.findPackageInfoFromRPMFile(rpmfile)
70
+            package, version, release = pkgUtils.findPackageInfoFromRPMFile(rpmfile)
68 71
             if SPECS.getData().isRPMPackage(package):
69
-                specVersion=SPECS.getData().getVersion(package)
70
-                specRelease=SPECS.getData().getRelease(package)
72
+                specVersion = SPECS.getData().getVersion(package)
73
+                specRelease = SPECS.getData().getRelease(package)
71 74
                 if version == specVersion and release == specRelease:
72 75
                     listFoundRPMPackages.append(package)
73 76
         #Mark package available only if all sub packages are available
74 77
         for package in listFoundRPMPackages:
75 78
             basePkg = SPECS.getData().getSpecName(package)
76 79
             if basePkg in listAvailablePackages:
77
-                continue;
80
+                continue
78 81
             listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
79 82
             packageIsAlreadyBuilt = True
80 83
             for rpmpkg in listRPMPackages:
... ...
@@ -86,14 +89,14 @@ class PackageManager(object):
86 86
         self.logger.info(listAvailablePackages)
87 87
         return listAvailablePackages
88 88
 
89
-    def calculateParams(self,listPackages):
89
+    def calculateParams(self, listPackages):
90 90
         self.listThreads.clear()
91 91
         self.mapOutputThread.clear()
92 92
         self.mapThreadsLaunchTime.clear()
93
-        self.listAvailableCyclicPackages=[]
93
+        self.listAvailableCyclicPackages = []
94 94
         self.mapCyclesToPackageList.clear()
95 95
         self.mapPackageToCycle.clear()
96
-        self.sortedPackageList=[]
96
+        self.sortedPackageList = []
97 97
 
98 98
         listOfPackagesAlreadyBuilt = []
99 99
         listOfPackagesAlreadyBuilt = self.readAlreadyAvailablePackages()
... ...
@@ -113,9 +116,10 @@ class PackageManager(object):
113 113
                 if needToRebuild:
114 114
                     self.listOfPackagesAlreadyBuilt.remove(pkg)
115 115
 
116
-        listPackagesToBuild=listPackages[:]
116
+        listPackagesToBuild = listPackages[:]
117 117
         for pkg in listPackages:
118
-            if pkg in self.listOfPackagesAlreadyBuilt and not constants.rpmCheck:
118
+            if (pkg in self.listOfPackagesAlreadyBuilt and
119
+                    not constants.rpmCheck):
119 120
                 listPackagesToBuild.remove(pkg)
120 121
 
121 122
         if not self.readPackageBuildData(listPackagesToBuild):
... ...
@@ -125,8 +129,9 @@ class PackageManager(object):
125 125
     def buildToolChain(self):
126 126
         pkgCount = 0
127 127
         try:
128
-            tUtils=ToolChainUtils()
129
-            pkgCount = tUtils.buildCoreToolChainPackages(self.listBuildOptionPackages, self.pkgBuildOptionFile)
128
+            tUtils = ToolChainUtils()
129
+            pkgCount = tUtils.buildCoreToolChainPackages(self.listBuildOptionPackages,
130
+                                                         self.pkgBuildOptionFile)
130 131
         except Exception as e:
131 132
             self.logger.error("Unable to build tool chain")
132 133
             self.logger.error(e)
... ...
@@ -150,71 +155,73 @@ class PackageManager(object):
150 150
         self.buildToolChain()
151 151
         self.buildGivenPackages(constants.listMakeCheckRPMPkgtoInstall, buildThreads)
152 152
 
153
-    def buildPackages(self,listPackages, listBuildOptionPackages, pkgBuildOptionFile, buildThreads, pkgBuildType):
153
+    def buildPackages(self, listPackages, listBuildOptionPackages, pkgBuildOptionFile,
154
+                      buildThreads, pkgBuildType):
154 155
         self.listBuildOptionPackages = listBuildOptionPackages
155 156
         self.pkgBuildOptionFile = pkgBuildOptionFile
156 157
         self.pkgBuildType = pkgBuildType
157 158
         if constants.rpmCheck:
158
-            constants.rpmCheck=False
159
+            constants.rpmCheck = False
159 160
             self.buildToolChainPackages(listBuildOptionPackages, pkgBuildOptionFile, buildThreads)
160 161
             self.buildTestPackages(listBuildOptionPackages, pkgBuildOptionFile, buildThreads)
161
-            constants.rpmCheck=True
162
+            constants.rpmCheck = True
162 163
             self.buildGivenPackages(listPackages, buildThreads)
163 164
         else:
164 165
             self.buildToolChainPackages(listBuildOptionPackages, pkgBuildOptionFile, buildThreads)
165 166
             self.buildGivenPackages(listPackages, buildThreads)
166 167
 
167
-    def initializeThreadPool(self,statusEvent):
168
+    def initializeThreadPool(self, statusEvent):
168 169
         ThreadPool.clear()
169
-        ThreadPool.mapPackageToCycle=self.mapPackageToCycle
170
-        ThreadPool.listAvailableCyclicPackages=self.listAvailableCyclicPackages
171
-        ThreadPool.listBuildOptionPackages=self.listBuildOptionPackages
172
-        ThreadPool.pkgBuildOptionFile=self.pkgBuildOptionFile
173
-        ThreadPool.logger=self.logger
174
-        ThreadPool.statusEvent=statusEvent
175
-        ThreadPool.pkgBuildType=self.pkgBuildType
170
+        ThreadPool.mapPackageToCycle = self.mapPackageToCycle
171
+        ThreadPool.listAvailableCyclicPackages = self.listAvailableCyclicPackages
172
+        ThreadPool.listBuildOptionPackages = self.listBuildOptionPackages