Browse code

Replace PARALLEL make option with THREADS.

YustasSwamp authored on 2015/07/07 03:09:05
Showing 3 changed files
... ...
@@ -42,7 +42,7 @@ CONTAIN := $(TOOLS_BIN)/contain
42 42
 check check-bison check-g++ check-gawk check-createrepo check-vagrant check-packer check-packer-ovf-plugin check-sanity \
43 43
 clean-install clean-chroot
44 44
 
45
-PARALLEL=False
45
+THREADS?=1
46 46
 
47 47
 all: iso micro-iso minimal-iso docker-image
48 48
 
... ...
@@ -70,7 +70,7 @@ packages-micro: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES)
70 70
                 -l $(PHOTON_LOGS_DIR) \
71 71
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
72 72
                 -j $(PHOTON_DATA_DIR)/build_install_options_micro.json \
73
-                -e ${PARALLEL}
73
+                -t ${THREADS}
74 74
 
75 75
 minimal: minimal-iso
76 76
 
... ...
@@ -96,7 +96,7 @@ packages-minimal: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES)
96 96
                 -l $(PHOTON_LOGS_DIR) \
97 97
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
98 98
                 -j $(PHOTON_DATA_DIR)/build_install_options_minimal.json \
99
-                -e ${PARALLEL}
99
+                -t ${THREADS}
100 100
 
101 101
 iso: check $(PHOTON_STAGE) $(PHOTON_PACKAGES)
102 102
 	@echo "Building Photon FUll ISO..."
... ...
@@ -120,7 +120,7 @@ packages: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN)
120 120
                 -l $(PHOTON_LOGS_DIR) \
121 121
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
122 122
                 -j $(PHOTON_DATA_DIR)/build_install_options_all.json \
123
-                -e ${PARALLEL}
123
+                -t ${THREADS}
124 124
 
125 125
 tool-chain-stage1: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN)
126 126
 	@echo "Building all RPMS..."
... ...
@@ -133,7 +133,7 @@ tool-chain-stage1: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN)
133 133
                 -l $(PHOTON_LOGS_DIR) \
134 134
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
135 135
                 -j $(PHOTON_DATA_DIR)/build_install_options_all.json \
136
-                -e ${PARALLEL} \
136
+                -t ${THREADS} \
137 137
                 -m stage1
138 138
 
139 139
 tool-chain-stage2: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN)
... ...
@@ -147,7 +147,7 @@ tool-chain-stage2: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN)
147 147
                 -l $(PHOTON_LOGS_DIR) \
148 148
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
149 149
                 -j $(PHOTON_DATA_DIR)/build_install_options_all.json \
150
-                -e ${PARALLEL} \
150
+                -t ${THREADS} \
151 151
                 -m stage2
152 152
 
153 153
 
... ...
@@ -28,7 +28,6 @@ class PackageManager(object):
28 28
         self.mapThreadsLaunchTime={}
29 29
         self.listAvailableCyclicPackages=[]
30 30
         self.listPackagesToBuild=[]
31
-        self.maxNumThreads=20
32 31
         
33 32
     def readPackageBuildData(self, listPackages):
34 33
         try:
... ...
@@ -93,26 +92,13 @@ class PackageManager(object):
93 93
             self.logger.error(e)
94 94
             raise e
95 95
     
96
-    def calculatePossibleNumWorkerThreads(self):
97
-        process = subprocess.Popen(["df" ,constants.buildRootPath],shell=True,stdout=subprocess.PIPE)
98
-        retval = process.wait()
99
-        if retval != 0:
100
-            self.logger.error("Unable to check free space. Unknown error.")
101
-            return False
102
-        output = process.communicate()[0]
103
-        device, size, used, available, percent, mountpoint = output.split("\n")[1].split()
104
-        c =  int(available)/600000
105
-        numChroots=int(c)
106
-        self.logger.info("Possible number of worker threads:"+str(numChroots))
107
-        return numChroots
108
-    
109
-    def buildToolChainPackages(self, parallelBuild):
96
+    def buildToolChainPackages(self, buildThreads):
110 97
         self.buildToolChain()
111
-        self.buildGivenPackages(constants.listToolChainPackages, parallelBuild)
98
+        self.buildGivenPackages(constants.listToolChainPackages, buildThreads)
112 99
         
113
-    def buildPackages(self,listPackages, parallelBuild):
114
-        self.buildToolChainPackages(parallelBuild)
115
-        self.buildGivenPackages(listPackages, parallelBuild)
100
+    def buildPackages(self,listPackages, buildThreads):
101
+        self.buildToolChainPackages(buildThreads)
102
+        self.buildGivenPackages(listPackages, buildThreads)
116 103
     
117 104
     def initializeThreadPool(self,statusEvent):
118 105
         ThreadPool.clear()
... ...
@@ -127,29 +113,18 @@ class PackageManager(object):
127 127
         Scheduler.setEvent(statusEvent)
128 128
         Scheduler.stopScheduling=False
129 129
     
130
-    def buildGivenPackages (self, listPackages, parallelBuild):
131
-        defaultThreads = 1
132
-        if parallelBuild == True:
133
-            defaultThreads = 8
134
-
130
+    def buildGivenPackages (self, listPackages, buildThreads):
135 131
         returnVal=self.calculateParams(listPackages)
136 132
         if not returnVal:
137 133
             self.logger.error("Unable to set paramaters. Terminating the package manager.")
138 134
             raise Exception("Unable to set paramaters")
139 135
         
140 136
         statusEvent=threading.Event()
141
-        numWorkerThreads=self.calculatePossibleNumWorkerThreads()
142
-        if numWorkerThreads > self.maxNumThreads:
143
-            numWorkerThreads = defaultThreads
144
-        if numWorkerThreads == 0:
145
-            self.logger.error("Unable to create worker threads. Terminating the package manager.")
146
-            raise Exception("No Space.")
147
-
148 137
         self.initializeScheduler(statusEvent)
149 138
         self.initializeThreadPool(statusEvent)
150 139
         
151 140
         i=0
152
-        while i < numWorkerThreads:
141
+        while i < buildThreads:
153 142
             workerName="WorkerThread"+str(i)
154 143
             ThreadPool.addWorkerThread(workerName)
155 144
             ThreadPool.startWorkerThread(workerName)
... ...
@@ -22,7 +22,7 @@ def main():
22 22
     parser.add_option("-z",  "--top-dir-path", dest="topDirPath",  default="/usr/src/photon")
23 23
     parser.add_option("-j",  "--json-file", dest="inputJSONFile",  default="../../common/data/build_install_options_all.json")
24 24
     parser.add_option("-b",  "--build-root-path", dest="buildRootPath",  default="/mnt")
25
-    parser.add_option("-e",  "--parallel-build", dest="parallelBuild",  default=False)
25
+    parser.add_option("-t",  "--threads", dest="buildThreads",  default=1, type="int", help="Numbeer of working threads")
26 26
     parser.add_option("-m",  "--tool-chain-stage", dest="toolChainStage",  default="None")
27 27
 
28 28
     (options,  args) = parser.parse_args()
... ...
@@ -30,11 +30,6 @@ def main():
30 30
     if not os.path.isdir(options.logPath):
31 31
         cmdUtils.runCommandInShell("mkdir -p "+options.logPath)
32 32
     
33
-    if options.parallelBuild in ["TRUE", "True", "true"]:
34
-        parallelBuild = True
35
-    else:
36
-        parallelBuild = False
37
-
38 33
     logger=Logger.getLogger(options.logPath+"/Main")
39 34
     
40 35
     errorFlag=False
... ...
@@ -113,11 +108,11 @@ def main():
113 113
             pkgManager.buildToolChain()
114 114
         elif options.toolChainStage == "stage2":
115 115
             pkgManager = PackageManager()
116
-            pkgManager.buildToolChainPackages(parallelBuild)
116
+            pkgManager.buildToolChainPackages(options.buildThreads)
117 117
         elif options.installPackage:
118
-            buildAPackage(package, parallelBuild)
118
+            buildAPackage(package, options.buildThreads)
119 119
         else:
120
-            buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption,logger, parallelBuild)
120
+            buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption,logger, options.buildThreads)
121 121
     except Exception as e:
122 122
         logger.error("Caught an exception")
123 123
         logger.error(str(e))
... ...
@@ -125,19 +120,19 @@ def main():
125 125
     
126 126
     sys.exit(0)
127 127
 
128
-def buildToolChain(parallelBuild):
128
+def buildToolChain(buildThreads):
129 129
     listPackages=[]
130 130
     listPackages.append(package)
131 131
     pkgManager = PackageManager()
132
-    pkgManager.buildPackages(listPackages, parallelBuild)
132
+    pkgManager.buildPackages(listPackages, buildThreads)
133 133
 
134
-def buildAPackage(package, parallelBuild):
134
+def buildAPackage(package, buildThreads):
135 135
     listPackages=[]
136 136
     listPackages.append(package)
137 137
     pkgManager = PackageManager()
138
-    pkgManager.buildPackages(listPackages, parallelBuild)
138
+    pkgManager.buildPackages(listPackages, buildThreads)
139 139
 
140
-def buildPackagesFromGivenJSONFile(inputJSONFile,buildOption,logger, parallelBuild):
140
+def buildPackagesFromGivenJSONFile(inputJSONFile,buildOption,logger, buildThreads):
141 141
     listPackages = get_all_package_names(inputJSONFile)
142 142
 
143 143
     listPackagesToBuild=[]
... ...
@@ -147,7 +142,7 @@ def buildPackagesFromGivenJSONFile(inputJSONFile,buildOption,logger, parallelBui
147 147
     logger.info("List of packages to build:")
148 148
     logger.info(listPackagesToBuild)
149 149
     pkgManager = PackageManager()
150
-    pkgManager.buildPackages(listPackagesToBuild, parallelBuild)
150
+    pkgManager.buildPackages(listPackagesToBuild, buildThreads)
151 151
     
152 152
 def get_all_package_names(build_install_option):
153 153
     base_path = os.path.dirname(build_install_option)