| ... | ... |
@@ -42,6 +42,8 @@ CONTAIN := $(TOOLS_BIN)/contain |
| 42 | 42 |
check check-bison check-g++ check-gawk check-createrepo check-vagrant check-packer check-packer-ovf-plugin check-sanity \ |
| 43 | 43 |
clean-install clean-chroot |
| 44 | 44 |
|
| 45 |
+PARALLEL=False |
|
| 46 |
+ |
|
| 45 | 47 |
all: iso |
| 46 | 48 |
|
| 47 | 49 |
micro: check $(PHOTON_STAGE) $(PHOTON_PACKAGES_MICRO) |
| ... | ... |
@@ -65,7 +67,8 @@ packages-micro: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) |
| 65 | 65 |
-b $(PHOTON_CHROOT_PATH) \ |
| 66 | 66 |
-l $(PHOTON_LOGS_DIR) \ |
| 67 | 67 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
| 68 |
- -j $(PHOTON_DATA_DIR)/build_install_options_micro.json |
|
| 68 |
+ -j $(PHOTON_DATA_DIR)/build_install_options_micro.json \ |
|
| 69 |
+ -e ${PARALLEL}
|
|
| 69 | 70 |
|
| 70 | 71 |
minimal: check $(PHOTON_STAGE) $(PHOTON_PACKAGES_MINIMAL) |
| 71 | 72 |
@echo "Building Photon Minimal ISO..." |
| ... | ... |
@@ -88,7 +91,8 @@ packages-minimal: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) |
| 88 | 88 |
-b $(PHOTON_CHROOT_PATH) \ |
| 89 | 89 |
-l $(PHOTON_LOGS_DIR) \ |
| 90 | 90 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
| 91 |
- -j $(PHOTON_DATA_DIR)/build_install_options_minimal.json |
|
| 91 |
+ -j $(PHOTON_DATA_DIR)/build_install_options_minimal.json \ |
|
| 92 |
+ -e ${PARALLEL}
|
|
| 92 | 93 |
|
| 93 | 94 |
iso: check $(PHOTON_STAGE) $(PHOTON_PACKAGES) |
| 94 | 95 |
@echo "Building Photon FUll ISO..." |
| ... | ... |
@@ -111,7 +115,8 @@ packages: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN) |
| 111 | 111 |
-b $(PHOTON_CHROOT_PATH) \ |
| 112 | 112 |
-l $(PHOTON_LOGS_DIR) \ |
| 113 | 113 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
| 114 |
- -j $(PHOTON_DATA_DIR)/build_install_options_all.json |
|
| 114 |
+ -j $(PHOTON_DATA_DIR)/build_install_options_all.json \ |
|
| 115 |
+ -e ${PARALLEL}
|
|
| 115 | 116 |
|
| 116 | 117 |
packages-cached: |
| 117 | 118 |
@echo "Using cached RPMS..." |
| ... | ... |
@@ -105,13 +105,13 @@ class PackageManager(object): |
| 105 | 105 |
self.logger.info("Possible number of worker threads:"+str(numChroots))
|
| 106 | 106 |
return numChroots |
| 107 | 107 |
|
| 108 |
- def buildToolChainPackages(self): |
|
| 108 |
+ def buildToolChainPackages(self, parallelBuild): |
|
| 109 | 109 |
self.buildToolChain() |
| 110 |
- self.buildGivenPackages(constants.listToolChainPackages) |
|
| 110 |
+ self.buildGivenPackages(constants.listToolChainPackages, parallelBuild) |
|
| 111 | 111 |
|
| 112 |
- def buildPackages(self,listPackages): |
|
| 113 |
- self.buildToolChainPackages() |
|
| 114 |
- self.buildGivenPackages(listPackages) |
|
| 112 |
+ def buildPackages(self,listPackages, parallelBuild): |
|
| 113 |
+ self.buildToolChainPackages(parallelBuild) |
|
| 114 |
+ self.buildGivenPackages(listPackages, parallelBuild) |
|
| 115 | 115 |
|
| 116 | 116 |
def initializeThreadPool(self,statusEvent): |
| 117 | 117 |
ThreadPool.clear() |
| ... | ... |
@@ -126,7 +126,11 @@ class PackageManager(object): |
| 126 | 126 |
Scheduler.setEvent(statusEvent) |
| 127 | 127 |
Scheduler.stopScheduling=False |
| 128 | 128 |
|
| 129 |
- def buildGivenPackages (self, listPackages): |
|
| 129 |
+ def buildGivenPackages (self, listPackages, parallelBuild): |
|
| 130 |
+ defaultThreads = 1 |
|
| 131 |
+ if parallelBuild == True: |
|
| 132 |
+ defaultThreads = 8 |
|
| 133 |
+ |
|
| 130 | 134 |
returnVal=self.calculateParams(listPackages) |
| 131 | 135 |
if not returnVal: |
| 132 | 136 |
self.logger.error("Unable to set paramaters. Terminating the package manager.")
|
| ... | ... |
@@ -135,11 +139,11 @@ class PackageManager(object): |
| 135 | 135 |
statusEvent=threading.Event() |
| 136 | 136 |
numWorkerThreads=self.calculatePossibleNumWorkerThreads() |
| 137 | 137 |
if numWorkerThreads > 8: |
| 138 |
- numWorkerThreads = 1 |
|
| 138 |
+ numWorkerThreads = defaultThreads |
|
| 139 | 139 |
if numWorkerThreads == 0: |
| 140 | 140 |
self.logger.error("Unable to create worker threads. Terminating the package manager.")
|
| 141 | 141 |
raise Exception("No Space.")
|
| 142 |
- |
|
| 142 |
+ |
|
| 143 | 143 |
self.initializeScheduler(statusEvent) |
| 144 | 144 |
self.initializeThreadPool(statusEvent) |
| 145 | 145 |
|
| ... | ... |
@@ -22,6 +22,7 @@ def main(): |
| 22 | 22 |
parser.add_option("-z", "--top-dir-path", dest="topDirPath", default="/usr/src/photon")
|
| 23 | 23 |
parser.add_option("-j", "--json-file", dest="inputJSONFile", default="input.json")
|
| 24 | 24 |
parser.add_option("-b", "--build-root-path", dest="buildRootPath", default="/mnt")
|
| 25 |
+ parser.add_option("-e", "--parallel-build", dest="parallelBuild", default=False)
|
|
| 25 | 26 |
|
| 26 | 27 |
|
| 27 | 28 |
(options, args) = parser.parse_args() |
| ... | ... |
@@ -29,6 +30,11 @@ def main(): |
| 29 | 29 |
if not os.path.isdir(options.logPath): |
| 30 | 30 |
cmdUtils.runCommandInShell("mkdir -p "+options.logPath)
|
| 31 | 31 |
|
| 32 |
+ if options.parallelBuild in ["TRUE", "True", "true"]: |
|
| 33 |
+ parallelBuild = True |
|
| 34 |
+ else: |
|
| 35 |
+ parallelBuild = False |
|
| 36 |
+ |
|
| 32 | 37 |
logger=Logger.getLogger(options.logPath+"/Main") |
| 33 | 38 |
|
| 34 | 39 |
errorFlag=False |
| ... | ... |
@@ -82,7 +88,7 @@ def main(): |
| 82 | 82 |
logger.info("JSON File :" + options.inputJSONFile)
|
| 83 | 83 |
else: |
| 84 | 84 |
logger.info("Package to build:"+package)
|
| 85 |
- |
|
| 85 |
+ |
|
| 86 | 86 |
''' |
| 87 | 87 |
listPackages=["acl","attr","autoconf","automake","bash","bc","bindutils","binutils","bison","boost","btrfs-progs","bzip2","ca-certificates","cdrkit","check", |
| 88 | 88 |
"cloud-init","cmake","coreutils","cpio","cracklib","createrepo","curl","cyrus-sasl","db","dbus","deltarpm","diffutils","docbook-xml","docbook-xsl", |
| ... | ... |
@@ -103,9 +109,9 @@ def main(): |
| 103 | 103 |
try: |
| 104 | 104 |
constants.initialize(options) |
| 105 | 105 |
if options.installPackage: |
| 106 |
- buildAPackage(package) |
|
| 106 |
+ buildAPackage(package, parallelBuild) |
|
| 107 | 107 |
else: |
| 108 |
- buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption,logger) |
|
| 108 |
+ buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption,logger, parallelBuild) |
|
| 109 | 109 |
except Exception as e: |
| 110 | 110 |
logger.error("Caught an exception")
|
| 111 | 111 |
logger.error(str(e)) |
| ... | ... |
@@ -113,13 +119,13 @@ def main(): |
| 113 | 113 |
|
| 114 | 114 |
sys.exit(0) |
| 115 | 115 |
|
| 116 |
-def buildAPackage(package): |
|
| 116 |
+def buildAPackage(package, parallelBuild): |
|
| 117 | 117 |
listPackages=[] |
| 118 | 118 |
listPackages.append(package) |
| 119 | 119 |
pkgManager = PackageManager() |
| 120 |
- pkgManager.buildPackages(listPackages) |
|
| 120 |
+ pkgManager.buildPackages(listPackages, parallelBuild) |
|
| 121 | 121 |
|
| 122 |
-def buildPackagesFromGivenJSONFile(inputJSONFile,buildOption,logger): |
|
| 122 |
+def buildPackagesFromGivenJSONFile(inputJSONFile,buildOption,logger, parallelBuild): |
|
| 123 | 123 |
listPackages = get_all_package_names(inputJSONFile) |
| 124 | 124 |
|
| 125 | 125 |
listPackagesToBuild=[] |
| ... | ... |
@@ -129,7 +135,7 @@ def buildPackagesFromGivenJSONFile(inputJSONFile,buildOption,logger): |
| 129 | 129 |
logger.info("List of packages to build:")
|
| 130 | 130 |
logger.info(listPackagesToBuild) |
| 131 | 131 |
pkgManager = PackageManager() |
| 132 |
- pkgManager.buildPackages(listPackagesToBuild) |
|
| 132 |
+ pkgManager.buildPackages(listPackagesToBuild, parallelBuild) |
|
| 133 | 133 |
|
| 134 | 134 |
def get_all_package_names(build_install_option): |
| 135 | 135 |
base_path = os.path.dirname(build_install_option) |