... | ... |
@@ -42,7 +42,7 @@ CONTAIN := $(TOOLS_BIN)/contain |
42 | 42 |
check check-bison check-g++ check-gawk check-createrepo check-vagrant check-packer check-packer-ovf-plugin check-sanity \ |
43 | 43 |
clean-install clean-chroot |
44 | 44 |
|
45 |
-PARALLEL=False |
|
45 |
+THREADS?=1 |
|
46 | 46 |
|
47 | 47 |
all: iso micro-iso minimal-iso docker-image |
48 | 48 |
|
... | ... |
@@ -70,7 +70,7 @@ packages-micro: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) |
70 | 70 |
-l $(PHOTON_LOGS_DIR) \ |
71 | 71 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
72 | 72 |
-j $(PHOTON_DATA_DIR)/build_install_options_micro.json \ |
73 |
- -e ${PARALLEL} |
|
73 |
+ -t ${THREADS} |
|
74 | 74 |
|
75 | 75 |
minimal: minimal-iso |
76 | 76 |
|
... | ... |
@@ -96,7 +96,7 @@ packages-minimal: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) |
96 | 96 |
-l $(PHOTON_LOGS_DIR) \ |
97 | 97 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
98 | 98 |
-j $(PHOTON_DATA_DIR)/build_install_options_minimal.json \ |
99 |
- -e ${PARALLEL} |
|
99 |
+ -t ${THREADS} |
|
100 | 100 |
|
101 | 101 |
iso: check $(PHOTON_STAGE) $(PHOTON_PACKAGES) |
102 | 102 |
@echo "Building Photon FUll ISO..." |
... | ... |
@@ -120,7 +120,7 @@ packages: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN) |
120 | 120 |
-l $(PHOTON_LOGS_DIR) \ |
121 | 121 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
122 | 122 |
-j $(PHOTON_DATA_DIR)/build_install_options_all.json \ |
123 |
- -e ${PARALLEL} |
|
123 |
+ -t ${THREADS} |
|
124 | 124 |
|
125 | 125 |
tool-chain-stage1: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN) |
126 | 126 |
@echo "Building all RPMS..." |
... | ... |
@@ -133,7 +133,7 @@ tool-chain-stage1: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN) |
133 | 133 |
-l $(PHOTON_LOGS_DIR) \ |
134 | 134 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
135 | 135 |
-j $(PHOTON_DATA_DIR)/build_install_options_all.json \ |
136 |
- -e ${PARALLEL} \ |
|
136 |
+ -t ${THREADS} \ |
|
137 | 137 |
-m stage1 |
138 | 138 |
|
139 | 139 |
tool-chain-stage2: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN) |
... | ... |
@@ -147,7 +147,7 @@ tool-chain-stage2: check $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTAIN) |
147 | 147 |
-l $(PHOTON_LOGS_DIR) \ |
148 | 148 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
149 | 149 |
-j $(PHOTON_DATA_DIR)/build_install_options_all.json \ |
150 |
- -e ${PARALLEL} \ |
|
150 |
+ -t ${THREADS} \ |
|
151 | 151 |
-m stage2 |
152 | 152 |
|
153 | 153 |
|
... | ... |
@@ -28,7 +28,6 @@ class PackageManager(object): |
28 | 28 |
self.mapThreadsLaunchTime={} |
29 | 29 |
self.listAvailableCyclicPackages=[] |
30 | 30 |
self.listPackagesToBuild=[] |
31 |
- self.maxNumThreads=20 |
|
32 | 31 |
|
33 | 32 |
def readPackageBuildData(self, listPackages): |
34 | 33 |
try: |
... | ... |
@@ -93,26 +92,13 @@ class PackageManager(object): |
93 | 93 |
self.logger.error(e) |
94 | 94 |
raise e |
95 | 95 |
|
96 |
- def calculatePossibleNumWorkerThreads(self): |
|
97 |
- process = subprocess.Popen(["df" ,constants.buildRootPath],shell=True,stdout=subprocess.PIPE) |
|
98 |
- retval = process.wait() |
|
99 |
- if retval != 0: |
|
100 |
- self.logger.error("Unable to check free space. Unknown error.") |
|
101 |
- return False |
|
102 |
- output = process.communicate()[0] |
|
103 |
- device, size, used, available, percent, mountpoint = output.split("\n")[1].split() |
|
104 |
- c = int(available)/600000 |
|
105 |
- numChroots=int(c) |
|
106 |
- self.logger.info("Possible number of worker threads:"+str(numChroots)) |
|
107 |
- return numChroots |
|
108 |
- |
|
109 |
- def buildToolChainPackages(self, parallelBuild): |
|
96 |
+ def buildToolChainPackages(self, buildThreads): |
|
110 | 97 |
self.buildToolChain() |
111 |
- self.buildGivenPackages(constants.listToolChainPackages, parallelBuild) |
|
98 |
+ self.buildGivenPackages(constants.listToolChainPackages, buildThreads) |
|
112 | 99 |
|
113 |
- def buildPackages(self,listPackages, parallelBuild): |
|
114 |
- self.buildToolChainPackages(parallelBuild) |
|
115 |
- self.buildGivenPackages(listPackages, parallelBuild) |
|
100 |
+ def buildPackages(self,listPackages, buildThreads): |
|
101 |
+ self.buildToolChainPackages(buildThreads) |
|
102 |
+ self.buildGivenPackages(listPackages, buildThreads) |
|
116 | 103 |
|
117 | 104 |
def initializeThreadPool(self,statusEvent): |
118 | 105 |
ThreadPool.clear() |
... | ... |
@@ -127,29 +113,18 @@ class PackageManager(object): |
127 | 127 |
Scheduler.setEvent(statusEvent) |
128 | 128 |
Scheduler.stopScheduling=False |
129 | 129 |
|
130 |
- def buildGivenPackages (self, listPackages, parallelBuild): |
|
131 |
- defaultThreads = 1 |
|
132 |
- if parallelBuild == True: |
|
133 |
- defaultThreads = 8 |
|
134 |
- |
|
130 |
+ def buildGivenPackages (self, listPackages, buildThreads): |
|
135 | 131 |
returnVal=self.calculateParams(listPackages) |
136 | 132 |
if not returnVal: |
137 | 133 |
self.logger.error("Unable to set paramaters. Terminating the package manager.") |
138 | 134 |
raise Exception("Unable to set paramaters") |
139 | 135 |
|
140 | 136 |
statusEvent=threading.Event() |
141 |
- numWorkerThreads=self.calculatePossibleNumWorkerThreads() |
|
142 |
- if numWorkerThreads > self.maxNumThreads: |
|
143 |
- numWorkerThreads = defaultThreads |
|
144 |
- if numWorkerThreads == 0: |
|
145 |
- self.logger.error("Unable to create worker threads. Terminating the package manager.") |
|
146 |
- raise Exception("No Space.") |
|
147 |
- |
|
148 | 137 |
self.initializeScheduler(statusEvent) |
149 | 138 |
self.initializeThreadPool(statusEvent) |
150 | 139 |
|
151 | 140 |
i=0 |
152 |
- while i < numWorkerThreads: |
|
141 |
+ while i < buildThreads: |
|
153 | 142 |
workerName="WorkerThread"+str(i) |
154 | 143 |
ThreadPool.addWorkerThread(workerName) |
155 | 144 |
ThreadPool.startWorkerThread(workerName) |
... | ... |
@@ -22,7 +22,7 @@ def main(): |
22 | 22 |
parser.add_option("-z", "--top-dir-path", dest="topDirPath", default="/usr/src/photon") |
23 | 23 |
parser.add_option("-j", "--json-file", dest="inputJSONFile", default="../../common/data/build_install_options_all.json") |
24 | 24 |
parser.add_option("-b", "--build-root-path", dest="buildRootPath", default="/mnt") |
25 |
- parser.add_option("-e", "--parallel-build", dest="parallelBuild", default=False) |
|
25 |
+ parser.add_option("-t", "--threads", dest="buildThreads", default=1, type="int", help="Numbeer of working threads") |
|
26 | 26 |
parser.add_option("-m", "--tool-chain-stage", dest="toolChainStage", default="None") |
27 | 27 |
|
28 | 28 |
(options, args) = parser.parse_args() |
... | ... |
@@ -30,11 +30,6 @@ def main(): |
30 | 30 |
if not os.path.isdir(options.logPath): |
31 | 31 |
cmdUtils.runCommandInShell("mkdir -p "+options.logPath) |
32 | 32 |
|
33 |
- if options.parallelBuild in ["TRUE", "True", "true"]: |
|
34 |
- parallelBuild = True |
|
35 |
- else: |
|
36 |
- parallelBuild = False |
|
37 |
- |
|
38 | 33 |
logger=Logger.getLogger(options.logPath+"/Main") |
39 | 34 |
|
40 | 35 |
errorFlag=False |
... | ... |
@@ -113,11 +108,11 @@ def main(): |
113 | 113 |
pkgManager.buildToolChain() |
114 | 114 |
elif options.toolChainStage == "stage2": |
115 | 115 |
pkgManager = PackageManager() |
116 |
- pkgManager.buildToolChainPackages(parallelBuild) |
|
116 |
+ pkgManager.buildToolChainPackages(options.buildThreads) |
|
117 | 117 |
elif options.installPackage: |
118 |
- buildAPackage(package, parallelBuild) |
|
118 |
+ buildAPackage(package, options.buildThreads) |
|
119 | 119 |
else: |
120 |
- buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption,logger, parallelBuild) |
|
120 |
+ buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption,logger, options.buildThreads) |
|
121 | 121 |
except Exception as e: |
122 | 122 |
logger.error("Caught an exception") |
123 | 123 |
logger.error(str(e)) |
... | ... |
@@ -125,19 +120,19 @@ def main(): |
125 | 125 |
|
126 | 126 |
sys.exit(0) |
127 | 127 |
|
128 |
-def buildToolChain(parallelBuild): |
|
128 |
+def buildToolChain(buildThreads): |
|
129 | 129 |
listPackages=[] |
130 | 130 |
listPackages.append(package) |
131 | 131 |
pkgManager = PackageManager() |
132 |
- pkgManager.buildPackages(listPackages, parallelBuild) |
|
132 |
+ pkgManager.buildPackages(listPackages, buildThreads) |
|
133 | 133 |
|
134 |
-def buildAPackage(package, parallelBuild): |
|
134 |
+def buildAPackage(package, buildThreads): |
|
135 | 135 |
listPackages=[] |
136 | 136 |
listPackages.append(package) |
137 | 137 |
pkgManager = PackageManager() |
138 |
- pkgManager.buildPackages(listPackages, parallelBuild) |
|
138 |
+ pkgManager.buildPackages(listPackages, buildThreads) |
|
139 | 139 |
|
140 |
-def buildPackagesFromGivenJSONFile(inputJSONFile,buildOption,logger, parallelBuild): |
|
140 |
+def buildPackagesFromGivenJSONFile(inputJSONFile,buildOption,logger, buildThreads): |
|
141 | 141 |
listPackages = get_all_package_names(inputJSONFile) |
142 | 142 |
|
143 | 143 |
listPackagesToBuild=[] |
... | ... |
@@ -147,7 +142,7 @@ def buildPackagesFromGivenJSONFile(inputJSONFile,buildOption,logger, parallelBui |
147 | 147 |
logger.info("List of packages to build:") |
148 | 148 |
logger.info(listPackagesToBuild) |
149 | 149 |
pkgManager = PackageManager() |
150 |
- pkgManager.buildPackages(listPackagesToBuild, parallelBuild) |
|
150 |
+ pkgManager.buildPackages(listPackagesToBuild, buildThreads) |
|
151 | 151 |
|
152 | 152 |
def get_all_package_names(build_install_option): |
153 | 153 |
base_path = os.path.dirname(build_install_option) |