This patchset contains following changes:
i) Using kube config instead of token to bring uniformity,
since we have to use kube config for kubectl cp operation
and we dont make to make changes in cluster(role binding etc)
when we use kube config.
ii) Ctrl C or SIGINT to stop the build at any time.
Whenever used it will deleted existing infrastructure of pods
and deploymets and then exit the build.
iii) Building images like ami,gce,azure with distirbuted build
command.
Change-Id: I5513cd2b0babbb21e00b2d88635867110b2d9877
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/11183
Tested-by: gerrit-photon <photon-checkins@vmware.com>
Reviewed-by: Ankit Jain <ankitja@vmware.com>
... | ... |
@@ -575,19 +575,14 @@ class RpmBuildTarget: |
575 | 575 |
Builder.buildPackagesForAllSpecs(Build_Config.buildThreads, Build_Config.pkgBuildType, Build_Config.pkgInfoFile, self.logger) |
576 | 576 |
|
577 | 577 |
def distributed_build(): |
578 |
- from DistributedBuilder import DistributedBuilder |
|
579 |
- print("Building RPMS through kubernetes ...") |
|
580 |
- os.chdir(str(PurePath(curDir, "support", "package-builder"))) |
|
578 |
+ import DistributedBuilder |
|
579 |
+ if not check_prerequesite["vixdiskutil"]: |
|
580 |
+ vixdiskutil() |
|
581 |
+ print("Distributed Building using kubernetes ...") |
|
581 | 582 |
with open(Build_Config.distributedBuildFile, 'r') as configFile: |
582 | 583 |
distributedBuildConfig = json.load(configFile) |
583 | 584 |
|
584 |
- distributedBuilder = DistributedBuilder(distributedBuildConfig) |
|
585 |
- distributedBuilder.create() |
|
586 |
- distributedBuilder.getLogs() |
|
587 |
- distributedBuilder.monitorJob() |
|
588 |
- distributedBuilder.copyFromNfs() |
|
589 |
- distributedBuilder.deleteBuild() |
|
590 |
- distributedBuilder.clean() |
|
585 |
+ DistributedBuilder.main(distributedBuildConfig) |
|
591 | 586 |
|
592 | 587 |
def tool_chain_stage1(self): |
593 | 588 |
from PackageManager import PackageManager |
... | ... |
@@ -822,13 +817,15 @@ class BuildImage: |
822 | 822 |
CheckTools.check_photon_installer() |
823 | 823 |
if not check_prerequesite["photon-stage"]: |
824 | 824 |
BuildEnvironmentSetup.photon_stage() |
825 |
- if not check_prerequesite["vixdiskutil"] and constants.buildArch == "x86_64": |
|
825 |
+ |
|
826 |
+ local_build = not configdict['photon-build-param']['start-scheduler-server'] |
|
827 |
+ if not check_prerequesite["vixdiskutil"] and constants.buildArch == "x86_64" and local_build: |
|
826 | 828 |
vixdiskutil() |
827 | 829 |
rpmBuildTarget = None |
828 | 830 |
if not check_prerequesite["packages"]: |
829 | 831 |
rpmBuildTarget = RpmBuildTarget() |
830 | 832 |
rpmBuildTarget.packages() |
831 |
- if not check_prerequesite["ostree-repo"]: |
|
833 |
+ if not check_prerequesite["ostree-repo"] and local_build: |
|
832 | 834 |
RpmBuildTarget.ostree_repo() |
833 | 835 |
print("Building " + self.img_name + " image") |
834 | 836 |
imagebuilder.createImage(self) |
... | ... |
@@ -928,7 +925,7 @@ def initialize_constants(): |
928 | 928 |
constants.setKatBuild(configdict["photon-build-param"].get("kat-build", False)) |
929 | 929 |
Build_Config.setConfFile(configdict["additional-path"]["conf-file"]) |
930 | 930 |
Build_Config.setPkgToBeCopiedConfFile(configdict.get("additional-path", {}).get("pkg-to-be-copied-conf-file")) |
931 |
- Build_Config.setDistributedBuildFile(configdict.get("additional-path", {}).get("distributed-build-option-file", PurePath(curDir, "common", "data", "distributed_build_options.json"))) |
|
931 |
+ Build_Config.setDistributedBuildFile(os.path.join(Build_Config.dataDir, "distributed_build_options.json")) |
|
932 | 932 |
Builder.get_packages_with_build_options(configdict['photon-build-param']['pkg-build-options']) |
933 | 933 |
Build_Config.setCommonDir(PurePath(curDir, "common", "data")) |
934 | 934 |
constants.setStartSchedulerServer(configdict["photon-build-param"]['start-scheduler-server']) |
... | ... |
@@ -7,13 +7,12 @@ import json |
7 | 7 |
import subprocess |
8 | 8 |
import uuid |
9 | 9 |
import sys |
10 |
+import signal |
|
10 | 11 |
from argparse import ArgumentParser |
11 | 12 |
from Logger import Logger |
12 | 13 |
from constants import constants |
13 | 14 |
from kubernetes import client, config, watch |
14 | 15 |
from kubernetes import stream |
15 |
-import urllib3 |
|
16 |
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) |
|
17 | 16 |
|
18 | 17 |
class DistributedBuilder: |
19 | 18 |
|
... | ... |
@@ -28,21 +27,11 @@ class DistributedBuilder: |
28 | 28 |
self.distributedBuildConfig = distributedBuildConfig |
29 | 29 |
self.buildGuid = self.getBuildGuid() |
30 | 30 |
|
31 |
- self.aApiClient = self.getClusterAuthorization() |
|
31 |
+ self.aApiClient = config.load_kube_config() |
|
32 | 32 |
self.coreV1ApiInstance = client.CoreV1Api(self.aApiClient) |
33 | 33 |
self.batchV1ApiInstance = client.BatchV1Api(self.aApiClient) |
34 | 34 |
self.AppsV1ApiInstance = client.AppsV1Api(self.aApiClient) |
35 | 35 |
|
36 |
- |
|
37 |
- def getClusterAuthorization(self): |
|
38 |
- aToken = self.distributedBuildConfig["kubernetesAuthorizationToken"] |
|
39 |
- aConfiguration = client.Configuration() |
|
40 |
- aConfiguration.host = "https://" + self.distributedBuildConfig["kubernetes-master-ip"] + ":" + self.distributedBuildConfig["kubernetes-master-port"] |
|
41 |
- aConfiguration.verify_ssl = False |
|
42 |
- aConfiguration.ssl_ca_cert = None |
|
43 |
- aConfiguration.api_key = {"authorization": "Bearer " + aToken} |
|
44 |
- return client.ApiClient(aConfiguration) |
|
45 |
- |
|
46 | 36 |
def getBuildGuid(self): |
47 | 37 |
guid = str(uuid.uuid4()).split("-")[1] |
48 | 38 |
guid = guid.lower() |
... | ... |
@@ -171,15 +160,16 @@ class DistributedBuilder: |
171 | 171 |
self.logger.error("Exception when calling BatchV1Api->delete_namespaced_job: %s\n" % e.reason) |
172 | 172 |
|
173 | 173 |
def deleteBuild(self): |
174 |
- count = 2 |
|
175 |
- while count: |
|
176 |
- pod = "nfspod" + "-" + self.buildGuid |
|
177 |
- cmd = ['/bin/sh', '-c', 'cd /root; ls; rm -rf ' + 'build-' + self.buildGuid] |
|
178 |
- resp = stream.stream(self.coreV1ApiInstance.connect_get_namespaced_pod_exec, pod, 'default', command=cmd, stderr=True, stdin=False, stdout=True, tty=False) |
|
179 |
- self.logger.info("%s"%resp) |
|
180 |
- count -= 1 |
|
181 |
- |
|
182 |
- self.logger.info("Deleted Build Successfully") |
|
174 |
+ self.logger.info("Removing Build folder ...") |
|
175 |
+ pod = "nfspod" + "-" + self.buildGuid |
|
176 |
+ cmd = ['/bin/sh', '-c', 'rm -rf ' + '/root/build-' + self.buildGuid] |
|
177 |
+ try: |
|
178 |
+ resp = stream.stream(self.coreV1ApiInstance.connect_get_namespaced_pod_exec, pod, 'default', command=cmd, \ |
|
179 |
+ stderr=True, stdin=False, stdout=True, tty=False, _preload_content=False) |
|
180 |
+ resp.run_forever(timeout=10) |
|
181 |
+ self.logger.info("Deleted Build folder Successfully...") |
|
182 |
+ except client.rest.ApiException as e: |
|
183 |
+ self.logger.error("Exception when calling CoreV1Api->connect_namespaced_pod_exec: %s\n" % e.reason) |
|
183 | 184 |
|
184 | 185 |
def deleteNfsPod(self): |
185 | 186 |
try: |
... | ... |
@@ -206,7 +196,6 @@ class DistributedBuilder: |
206 | 206 |
self.logger.error("Exception when calling AppsV1Api->delete_namespaced_deployment: %s\n" % e.reason) |
207 | 207 |
|
208 | 208 |
def copyToNfs(self): |
209 |
- self.createNfsPod() |
|
210 | 209 |
podName = "nfspod" + "-" + self.buildGuid |
211 | 210 |
while True: |
212 | 211 |
resp = self.coreV1ApiInstance.read_namespaced_pod(name=podName, namespace='default') |
... | ... |
@@ -214,21 +203,19 @@ class DistributedBuilder: |
214 | 214 |
if status == 'Running': |
215 | 215 |
break |
216 | 216 |
|
217 |
- cmd = "kubectl cp " +str( os.path.join(os.path.dirname(__file__)).replace('support/package-builder', '')) \ |
|
217 |
+ cmd = "kubectl cp " + str( os.path.join(os.path.dirname(__file__)).replace('support/package-builder', '')) \ |
|
218 | 218 |
+ " " + podName + ":/root/" + "build-" + self.buildGuid + "/photon" |
219 | 219 |
self.logger.info("%s"%cmd) |
220 | 220 |
process = subprocess.Popen("%s" %cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
221 | 221 |
retval = process.wait() |
222 | 222 |
if retval == 0: |
223 | 223 |
self.logger.info("kubectl cp successfull.") |
224 |
- self.deleteNfsPod() |
|
225 | 224 |
else: |
226 | 225 |
self.logger.error("kubectl cp failed.") |
227 | 226 |
self.clean() |
228 | 227 |
sys.exit(1) |
229 | 228 |
|
230 | 229 |
def copyFromNfs(self): |
231 |
- self.createNfsPod() |
|
232 | 230 |
podName = "nfspod" + "-" + self.buildGuid |
233 | 231 |
while True: |
234 | 232 |
resp = self.coreV1ApiInstance.read_namespaced_pod(name=podName, namespace='default') |
... | ... |
@@ -244,7 +231,6 @@ class DistributedBuilder: |
244 | 244 |
self.logger.info("kubectl cp successfull.") |
245 | 245 |
else: |
246 | 246 |
self.logger.error("kubectl cp failed.") |
247 |
- self.deleteBuild() |
|
248 | 247 |
self.clean() |
249 | 248 |
sys.exit(1) |
250 | 249 |
|
... | ... |
@@ -279,12 +265,19 @@ class DistributedBuilder: |
279 | 279 |
self.logger.error(e) |
280 | 280 |
self.logger.info("pod terminated") |
281 | 281 |
|
282 |
+ def signal_handler(self, signal, frame): |
|
283 |
+ self.logger.info("SIGINT received") |
|
284 |
+ self.logger.info("Stopping Build ...") |
|
285 |
+ self.clean() |
|
286 |
+ sys.exit(0) |
|
287 |
+ |
|
282 | 288 |
def clean(self): |
283 | 289 |
self.logger.info("-"*45) |
284 | 290 |
self.logger.info("") |
285 | 291 |
self.logger.info("Cleaning up ...") |
286 |
- self.deleteMasterJob() |
|
292 |
+ self.deleteBuild() |
|
287 | 293 |
self.deleteNfsPod() |
294 |
+ self.deleteMasterJob() |
|
288 | 295 |
self.deleteMasterService() |
289 | 296 |
self.deleteDeployment() |
290 | 297 |
self.deletePersistentVolumeClaim() |
... | ... |
@@ -295,11 +288,21 @@ class DistributedBuilder: |
295 | 295 |
self.logger.info("") |
296 | 296 |
self.createPersistentVolume() |
297 | 297 |
self.createPersistentVolumeClaim() |
298 |
+ self.createNfsPod() |
|
298 | 299 |
self.copyToNfs() |
299 | 300 |
self.createMasterService() |
300 | 301 |
self.createMasterJob() |
301 | 302 |
self.createDeployment() |
302 | 303 |
|
304 |
+def main(distributedBuildConfig): |
|
305 |
+ distributedBuilder = DistributedBuilder(distributedBuildConfig) |
|
306 |
+ signal.signal(signal.SIGINT, distributedBuilder.signal_handler) |
|
307 |
+ distributedBuilder.create() |
|
308 |
+ distributedBuilder.getLogs() |
|
309 |
+ distributedBuilder.monitorJob() |
|
310 |
+ distributedBuilder.copyFromNfs() |
|
311 |
+ distributedBuilder.clean() |
|
312 |
+ |
|
303 | 313 |
if __name__ == "__main__": |
304 | 314 |
|
305 | 315 |
parser = ArgumentParser() |
... | ... |
@@ -310,14 +313,6 @@ if __name__ == "__main__": |
310 | 310 |
options = parser.parse_args() |
311 | 311 |
constants.setLogPath(options.logPath) |
312 | 312 |
constants.setLogLevel(options.logLevel) |
313 |
- |
|
314 |
- with open(os.path.join(os.path.dirname(__file__), options.distributedBuildOptionFile), 'r') as configFile: |
|
313 |
+ with open(os.path.join(os.path.dirname(__file__), options.distributedBuildFile), 'r') as configFile: |
|
315 | 314 |
distributedBuildConfig = json.load(configFile) |
316 |
- |
|
317 |
- distributedBuilder = DistributedBuilder(distributedBuildConfig) |
|
318 |
- distributedBuilder.create() |
|
319 |
- distributedBuilder.getLogs() |
|
320 |
- distributedBuilder.monitorJob() |
|
321 |
- distributedBuilder.copyFromNfs() |
|
322 |
- distributedBuilder.deleteBuild() |
|
323 |
- distributedBuilder.clean() |
|
315 |
+ main(distributedBuildConfig) |
... | ... |
@@ -22,11 +22,19 @@ spec: |
22 | 22 |
- name: sys |
23 | 23 |
mountPath: /sys |
24 | 24 |
readOnly: false |
25 |
+ - name: dev |
|
26 |
+ mountPath: /dev |
|
27 |
+ readOnly: false |
|
25 | 28 |
workingDir: /root/photon |
26 | 29 |
command: ["/bin/bash"] |
27 |
- args: ["-c","tdnf remove -y toybox && \ |
|
28 |
- tdnf install -y build-essential createrepo texinfo wget sudo \ |
|
29 |
- findutils python3-pip git && \ |
|
30 |
+ args: ["-c","tdnf install -y build-essential createrepo texinfo wget sudo \ |
|
31 |
+ fuse gdbm gptfdisk grub2 grub2-efi grub2-lang grub2-pc \ |
|
32 |
+ qemu-img kmod kpartx dosfstools efibootmgr ncurses e2fsprogs \ |
|
33 |
+ python3 python3-asn1crypto python3-certifi python3-cffi \ |
|
34 |
+ python3-chardet python3-cryptography python3-curses \ |
|
35 |
+ python3-idna python3-packaging python3-pyasn1 python3-pycparser \ |
|
36 |
+ python3-pyOpenSSL python3-pyparsing python3-requests python3-six \ |
|
37 |
+ util-linux rpm glibc-iconv findutils python3-pip git && \ |
|
30 | 38 |
pip3 install flask requests pyOpenSSL docker==2.3.0 && \ |
31 | 39 |
chmod 755 /sys "] |
32 | 40 |
volumes: |
... | ... |
@@ -36,4 +44,7 @@ spec: |
36 | 36 |
- name: sys |
37 | 37 |
hostPath: |
38 | 38 |
path: /sys |
39 |
+ - name: dev |
|
40 |
+ hostPath: |
|
41 |
+ path: /dev |
|
39 | 42 |
backoffLimit: 1 |