Browse code

Add parallel build optional. Usage: sudo make iso PARALLEL=True . By default parallel bulid will be disabled.

Touseef Liaqat authored on 2015/06/30 06:14:13
Showing 3 changed files
... ...
@@ -42,6 +42,8 @@ CONTAIN := $(TOOLS_BIN)/contain
42 42
 check check-bison check-g++ check-gawk check-createrepo check-vagrant check-packer check-packer-ovf-plugin check-sanity \
43 43
 clean-install clean-chroot
44 44
 
45
+PARALLEL=False
46
+
45 47
 all: iso
46 48
 
47 49
 micro: check $(PHOTON_STAGE) $(PHOTON_PACKAGES_MICRO)
... ...
@@ -65,7 +67,8 @@ packages-micro: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES)
65 65
                 -b $(PHOTON_CHROOT_PATH) \
66 66
                 -l $(PHOTON_LOGS_DIR) \
67 67
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
68
-                -j $(PHOTON_DATA_DIR)/build_install_options_micro.json
68
+                -j $(PHOTON_DATA_DIR)/build_install_options_micro.json \
69
+                -e ${PARALLEL}
69 70
 
70 71
 minimal: check $(PHOTON_STAGE) $(PHOTON_PACKAGES_MINIMAL)
71 72
 	@echo "Building Photon Minimal ISO..."
... ...
@@ -88,7 +91,8 @@ packages-minimal: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES)
88 88
                 -b $(PHOTON_CHROOT_PATH) \
89 89
                 -l $(PHOTON_LOGS_DIR) \
90 90
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
91
-                -j $(PHOTON_DATA_DIR)/build_install_options_minimal.json
91
+                -j $(PHOTON_DATA_DIR)/build_install_options_minimal.json \
92
+                -e ${PARALLEL}
92 93
 
93 94
 iso: check $(PHOTON_STAGE) $(PHOTON_PACKAGES)
94 95
 	@echo "Building Photon FUll ISO..."
... ...
@@ -111,7 +115,8 @@ packages: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN)
111 111
                 -b $(PHOTON_CHROOT_PATH) \
112 112
                 -l $(PHOTON_LOGS_DIR) \
113 113
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
114
-                -j $(PHOTON_DATA_DIR)/build_install_options_all.json
114
+                -j $(PHOTON_DATA_DIR)/build_install_options_all.json \
115
+                -e ${PARALLEL}
115 116
 
116 117
 packages-cached:
117 118
 	@echo "Using cached RPMS..."
... ...
@@ -105,13 +105,13 @@ class PackageManager(object):
105 105
         self.logger.info("Possible number of worker threads:"+str(numChroots))
106 106
         return numChroots
107 107
     
108
-    def buildToolChainPackages(self):
108
+    def buildToolChainPackages(self, parallelBuild):
109 109
         self.buildToolChain()
110
-        self.buildGivenPackages(constants.listToolChainPackages)
110
+        self.buildGivenPackages(constants.listToolChainPackages, parallelBuild)
111 111
         
112
-    def buildPackages(self,listPackages):
113
-        self.buildToolChainPackages()
114
-        self.buildGivenPackages(listPackages)
112
+    def buildPackages(self,listPackages, parallelBuild):
113
+        self.buildToolChainPackages(parallelBuild)
114
+        self.buildGivenPackages(listPackages, parallelBuild)
115 115
     
116 116
     def initializeThreadPool(self,statusEvent):
117 117
         ThreadPool.clear()
... ...
@@ -126,7 +126,11 @@ class PackageManager(object):
126 126
         Scheduler.setEvent(statusEvent)
127 127
         Scheduler.stopScheduling=False
128 128
     
129
-    def buildGivenPackages (self, listPackages):
129
+    def buildGivenPackages (self, listPackages, parallelBuild):
130
+        defaultThreads = 1
131
+        if parallelBuild == True:
132
+            defaultThreads = 8
133
+
130 134
         returnVal=self.calculateParams(listPackages)
131 135
         if not returnVal:
132 136
             self.logger.error("Unable to set paramaters. Terminating the package manager.")
... ...
@@ -135,11 +139,11 @@ class PackageManager(object):
135 135
         statusEvent=threading.Event()
136 136
         numWorkerThreads=self.calculatePossibleNumWorkerThreads()
137 137
         if numWorkerThreads > 8:
138
-            numWorkerThreads = 1
138
+            numWorkerThreads = defaultThreads
139 139
         if numWorkerThreads == 0:
140 140
             self.logger.error("Unable to create worker threads. Terminating the package manager.")
141 141
             raise Exception("No Space.")
142
-         
142
+
143 143
         self.initializeScheduler(statusEvent)
144 144
         self.initializeThreadPool(statusEvent)
145 145
         
... ...
@@ -22,6 +22,7 @@ def main():
22 22
     parser.add_option("-z",  "--top-dir-path", dest="topDirPath",  default="/usr/src/photon")
23 23
     parser.add_option("-j",  "--json-file", dest="inputJSONFile",  default="input.json")
24 24
     parser.add_option("-b",  "--build-root-path", dest="buildRootPath",  default="/mnt")
25
+    parser.add_option("-e",  "--parallel-build", dest="parallelBuild",  default=False)
25 26
     
26 27
     
27 28
     (options,  args) = parser.parse_args()
... ...
@@ -29,6 +30,11 @@ def main():
29 29
     if not os.path.isdir(options.logPath):
30 30
         cmdUtils.runCommandInShell("mkdir -p "+options.logPath)
31 31
     
32
+    if options.parallelBuild in ["TRUE", "True", "true"]:
33
+        parallelBuild = True
34
+    else:
35
+        parallelBuild = False
36
+
32 37
     logger=Logger.getLogger(options.logPath+"/Main")
33 38
     
34 39
     errorFlag=False
... ...
@@ -82,7 +88,7 @@ def main():
82 82
         logger.info("JSON File :" + options.inputJSONFile)
83 83
     else:
84 84
         logger.info("Package to build:"+package)
85
-    
85
+
86 86
     '''    
87 87
     listPackages=["acl","attr","autoconf","automake","bash","bc","bindutils","binutils","bison","boost","btrfs-progs","bzip2","ca-certificates","cdrkit","check",
88 88
                   "cloud-init","cmake","coreutils","cpio","cracklib","createrepo","curl","cyrus-sasl","db","dbus","deltarpm","diffutils","docbook-xml","docbook-xsl",
... ...
@@ -103,9 +109,9 @@ def main():
103 103
     try:
104 104
         constants.initialize(options)
105 105
         if options.installPackage:
106
-            buildAPackage(package)
106
+            buildAPackage(package, parallelBuild)
107 107
         else:
108
-            buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption,logger)
108
+            buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption,logger, parallelBuild)
109 109
     except Exception as e:
110 110
         logger.error("Caught an exception")
111 111
         logger.error(str(e))
... ...
@@ -113,13 +119,13 @@ def main():
113 113
     
114 114
     sys.exit(0)
115 115
 
116
-def buildAPackage(package):
116
+def buildAPackage(package, parallelBuild):
117 117
     listPackages=[]
118 118
     listPackages.append(package)
119 119
     pkgManager = PackageManager()
120
-    pkgManager.buildPackages(listPackages)
120
+    pkgManager.buildPackages(listPackages, parallelBuild)
121 121
 
122
-def buildPackagesFromGivenJSONFile(inputJSONFile,buildOption,logger):
122
+def buildPackagesFromGivenJSONFile(inputJSONFile,buildOption,logger, parallelBuild):
123 123
     listPackages = get_all_package_names(inputJSONFile)
124 124
 
125 125
     listPackagesToBuild=[]
... ...
@@ -129,7 +135,7 @@ def buildPackagesFromGivenJSONFile(inputJSONFile,buildOption,logger):
129 129
     logger.info("List of packages to build:")
130 130
     logger.info(listPackagesToBuild)
131 131
     pkgManager = PackageManager()
132
-    pkgManager.buildPackages(listPackagesToBuild)
132
+    pkgManager.buildPackages(listPackagesToBuild, parallelBuild)
133 133
     
134 134
 def get_all_package_names(build_install_option):
135 135
     base_path = os.path.dirname(build_install_option)