Browse code

DistributedBuild: Additional changes

This patchset contains following changes:
i) Using kube config instead of token to bring uniformity,
since we have to use kube config for kubectl cp operation
and we dont make to make changes in cluster(role binding etc)
when we use kube config.

ii) Ctrl C or SIGINT to stop the build at any time.
Whenever used it will deleted existing infrastructure of pods
and deploymets and then exit the build.

iii) Building images like ami,gce,azure with distirbuted build
command.

Change-Id: I5513cd2b0babbb21e00b2d88635867110b2d9877
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/11183
Tested-by: gerrit-photon <photon-checkins@vmware.com>
Reviewed-by: Ankit Jain <ankitja@vmware.com>

Prashant Singh Chauhan authored on 2020/09/15 01:44:11
Showing 4 changed files
... ...
@@ -575,19 +575,14 @@ class RpmBuildTarget:
575 575
         Builder.buildPackagesForAllSpecs(Build_Config.buildThreads, Build_Config.pkgBuildType, Build_Config.pkgInfoFile, self.logger)
576 576
 
577 577
     def distributed_build():
578
-        from DistributedBuilder import DistributedBuilder
579
-        print("Building RPMS through kubernetes ...")
580
-        os.chdir(str(PurePath(curDir, "support", "package-builder")))
578
+        import DistributedBuilder
579
+        if not check_prerequesite["vixdiskutil"]:
580
+            vixdiskutil()
581
+        print("Distributed Building using kubernetes ...")
581 582
         with open(Build_Config.distributedBuildFile, 'r') as configFile:
582 583
             distributedBuildConfig = json.load(configFile)
583 584
 
584
-        distributedBuilder = DistributedBuilder(distributedBuildConfig)
585
-        distributedBuilder.create()
586
-        distributedBuilder.getLogs()
587
-        distributedBuilder.monitorJob()
588
-        distributedBuilder.copyFromNfs()
589
-        distributedBuilder.deleteBuild()
590
-        distributedBuilder.clean()
585
+        DistributedBuilder.main(distributedBuildConfig)
591 586
 
592 587
     def tool_chain_stage1(self):
593 588
         from PackageManager import PackageManager
... ...
@@ -822,13 +817,15 @@ class BuildImage:
822 822
             CheckTools.check_photon_installer()
823 823
         if not check_prerequesite["photon-stage"]:
824 824
             BuildEnvironmentSetup.photon_stage()
825
-        if not check_prerequesite["vixdiskutil"] and constants.buildArch == "x86_64":
825
+
826
+        local_build = not configdict['photon-build-param']['start-scheduler-server']
827
+        if not check_prerequesite["vixdiskutil"] and constants.buildArch == "x86_64" and local_build:
826 828
             vixdiskutil()
827 829
         rpmBuildTarget = None
828 830
         if not check_prerequesite["packages"]:
829 831
             rpmBuildTarget = RpmBuildTarget()
830 832
             rpmBuildTarget.packages()
831
-        if not check_prerequesite["ostree-repo"]:
833
+        if not check_prerequesite["ostree-repo"] and local_build:
832 834
             RpmBuildTarget.ostree_repo()
833 835
         print("Building " + self.img_name + " image")
834 836
         imagebuilder.createImage(self)
... ...
@@ -928,7 +925,7 @@ def initialize_constants():
928 928
     constants.setKatBuild(configdict["photon-build-param"].get("kat-build", False))
929 929
     Build_Config.setConfFile(configdict["additional-path"]["conf-file"])
930 930
     Build_Config.setPkgToBeCopiedConfFile(configdict.get("additional-path", {}).get("pkg-to-be-copied-conf-file"))
931
-    Build_Config.setDistributedBuildFile(configdict.get("additional-path", {}).get("distributed-build-option-file", PurePath(curDir, "common", "data", "distributed_build_options.json")))
931
+    Build_Config.setDistributedBuildFile(os.path.join(Build_Config.dataDir, "distributed_build_options.json"))
932 932
     Builder.get_packages_with_build_options(configdict['photon-build-param']['pkg-build-options'])
933 933
     Build_Config.setCommonDir(PurePath(curDir, "common", "data"))
934 934
     constants.setStartSchedulerServer(configdict["photon-build-param"]['start-scheduler-server'])
... ...
@@ -1,8 +1,5 @@
1 1
 {
2 2
     "command": "make packages SCHEDULER_SERVER=enable",
3
-    "kubernetesAuthorizationToken": "",
4
-    "kubernetes-master-ip": "",
5
-    "kubernetes-master-port": "6443",
6 3
     "nfs-server-ip": "",
7 4
     "nfs-server-path": "",
8 5
     "pods": 1
... ...
@@ -7,13 +7,12 @@ import json
7 7
 import subprocess
8 8
 import uuid
9 9
 import sys
10
+import signal
10 11
 from argparse import ArgumentParser
11 12
 from Logger import Logger
12 13
 from constants import constants
13 14
 from kubernetes import client, config, watch
14 15
 from kubernetes import stream
15
-import urllib3
16
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
17 16
 
18 17
 class DistributedBuilder:
19 18
 
... ...
@@ -28,21 +27,11 @@ class DistributedBuilder:
28 28
         self.distributedBuildConfig = distributedBuildConfig
29 29
         self.buildGuid = self.getBuildGuid()
30 30
 
31
-        self.aApiClient = self.getClusterAuthorization()
31
+        self.aApiClient = config.load_kube_config()
32 32
         self.coreV1ApiInstance = client.CoreV1Api(self.aApiClient)
33 33
         self.batchV1ApiInstance = client.BatchV1Api(self.aApiClient)
34 34
         self.AppsV1ApiInstance = client.AppsV1Api(self.aApiClient)
35 35
 
36
-
37
-    def getClusterAuthorization(self):
38
-        aToken = self.distributedBuildConfig["kubernetesAuthorizationToken"]
39
-        aConfiguration = client.Configuration()
40
-        aConfiguration.host = "https://" + self.distributedBuildConfig["kubernetes-master-ip"] + ":" + self.distributedBuildConfig["kubernetes-master-port"]
41
-        aConfiguration.verify_ssl = False
42
-        aConfiguration.ssl_ca_cert = None
43
-        aConfiguration.api_key = {"authorization": "Bearer " + aToken}
44
-        return client.ApiClient(aConfiguration)
45
-
46 36
     def getBuildGuid(self):
47 37
          guid = str(uuid.uuid4()).split("-")[1]
48 38
          guid = guid.lower()
... ...
@@ -171,15 +160,16 @@ class DistributedBuilder:
171 171
            self.logger.error("Exception when calling BatchV1Api->delete_namespaced_job: %s\n" % e.reason)
172 172
 
173 173
     def deleteBuild(self):
174
-        count = 2
175
-        while count:
176
-            pod = "nfspod" + "-" + self.buildGuid
177
-            cmd = ['/bin/sh', '-c', 'cd /root; ls; rm -rf ' + 'build-' + self.buildGuid]
178
-            resp = stream.stream(self.coreV1ApiInstance.connect_get_namespaced_pod_exec, pod, 'default', command=cmd, stderr=True, stdin=False, stdout=True, tty=False)
179
-            self.logger.info("%s"%resp)
180
-            count -= 1
181
-
182
-        self.logger.info("Deleted Build Successfully")
174
+        self.logger.info("Removing Build folder ...")
175
+        pod = "nfspod" + "-" + self.buildGuid
176
+        cmd = ['/bin/sh', '-c', 'rm -rf ' + '/root/build-' + self.buildGuid]
177
+        try:
178
+            resp = stream.stream(self.coreV1ApiInstance.connect_get_namespaced_pod_exec, pod, 'default', command=cmd, \
179
+                   stderr=True, stdin=False, stdout=True, tty=False, _preload_content=False)
180
+            resp.run_forever(timeout=10)
181
+            self.logger.info("Deleted Build folder Successfully...")
182
+        except client.rest.ApiException as e:
183
+            self.logger.error("Exception when calling CoreV1Api->connect_namespaced_pod_exec: %s\n" % e.reason)
183 184
 
184 185
     def deleteNfsPod(self):
185 186
         try:
... ...
@@ -206,7 +196,6 @@ class DistributedBuilder:
206 206
             self.logger.error("Exception when calling AppsV1Api->delete_namespaced_deployment: %s\n" % e.reason)
207 207
 
208 208
     def copyToNfs(self):
209
-        self.createNfsPod()
210 209
         podName = "nfspod" + "-" + self.buildGuid
211 210
         while True:
212 211
             resp = self.coreV1ApiInstance.read_namespaced_pod(name=podName, namespace='default')
... ...
@@ -214,21 +203,19 @@ class DistributedBuilder:
214 214
             if status == 'Running':
215 215
                 break
216 216
 
217
-        cmd = "kubectl cp " +str( os.path.join(os.path.dirname(__file__)).replace('support/package-builder', '')) \
217
+        cmd = "kubectl cp " + str( os.path.join(os.path.dirname(__file__)).replace('support/package-builder', '')) \
218 218
                + " " + podName + ":/root/" + "build-" + self.buildGuid + "/photon"
219 219
         self.logger.info("%s"%cmd)
220 220
         process = subprocess.Popen("%s" %cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
221 221
         retval = process.wait()
222 222
         if retval == 0:
223 223
             self.logger.info("kubectl cp successfull.")
224
-            self.deleteNfsPod()
225 224
         else:
226 225
             self.logger.error("kubectl cp failed.")
227 226
             self.clean()
228 227
             sys.exit(1)
229 228
 
230 229
     def copyFromNfs(self):
231
-        self.createNfsPod()
232 230
         podName = "nfspod" + "-" + self.buildGuid
233 231
         while True:
234 232
             resp = self.coreV1ApiInstance.read_namespaced_pod(name=podName, namespace='default')
... ...
@@ -244,7 +231,6 @@ class DistributedBuilder:
244 244
             self.logger.info("kubectl cp successfull.")
245 245
         else:
246 246
             self.logger.error("kubectl cp failed.")
247
-            self.deleteBuild()
248 247
             self.clean()
249 248
             sys.exit(1)
250 249
 
... ...
@@ -279,12 +265,19 @@ class DistributedBuilder:
279 279
             self.logger.error(e)
280 280
         self.logger.info("pod terminated")
281 281
 
282
+    def signal_handler(self, signal, frame):
283
+        self.logger.info("SIGINT received")
284
+        self.logger.info("Stopping Build ...")
285
+        self.clean()
286
+        sys.exit(0)
287
+
282 288
     def clean(self):
283 289
         self.logger.info("-"*45)
284 290
         self.logger.info("")
285 291
         self.logger.info("Cleaning up ...")
286
-        self.deleteMasterJob()
292
+        self.deleteBuild()
287 293
         self.deleteNfsPod()
294
+        self.deleteMasterJob()
288 295
         self.deleteMasterService()
289 296
         self.deleteDeployment()
290 297
         self.deletePersistentVolumeClaim()
... ...
@@ -295,11 +288,21 @@ class DistributedBuilder:
295 295
         self.logger.info("")
296 296
         self.createPersistentVolume()
297 297
         self.createPersistentVolumeClaim()
298
+        self.createNfsPod()
298 299
         self.copyToNfs()
299 300
         self.createMasterService()
300 301
         self.createMasterJob()
301 302
         self.createDeployment()
302 303
 
304
+def main(distributedBuildConfig):
305
+    distributedBuilder = DistributedBuilder(distributedBuildConfig)
306
+    signal.signal(signal.SIGINT, distributedBuilder.signal_handler)
307
+    distributedBuilder.create()
308
+    distributedBuilder.getLogs()
309
+    distributedBuilder.monitorJob()
310
+    distributedBuilder.copyFromNfs()
311
+    distributedBuilder.clean()
312
+
303 313
 if __name__ == "__main__":
304 314
 
305 315
     parser = ArgumentParser()
... ...
@@ -310,14 +313,6 @@ if __name__ == "__main__":
310 310
     options = parser.parse_args()
311 311
     constants.setLogPath(options.logPath)
312 312
     constants.setLogLevel(options.logLevel)
313
-
314
-    with open(os.path.join(os.path.dirname(__file__), options.distributedBuildOptionFile), 'r') as configFile:
313
+    with open(os.path.join(os.path.dirname(__file__), options.distributedBuildFile), 'r') as configFile:
315 314
         distributedBuildConfig = json.load(configFile)
316
-
317
-    distributedBuilder = DistributedBuilder(distributedBuildConfig)
318
-    distributedBuilder.create()
319
-    distributedBuilder.getLogs()
320
-    distributedBuilder.monitorJob()
321
-    distributedBuilder.copyFromNfs()
322
-    distributedBuilder.deleteBuild()
323
-    distributedBuilder.clean()
315
+    main(distributedBuildConfig)
... ...
@@ -22,11 +22,19 @@ spec:
22 22
         - name: sys
23 23
           mountPath: /sys
24 24
           readOnly: false
25
+        - name: dev
26
+          mountPath: /dev
27
+          readOnly: false
25 28
         workingDir: /root/photon
26 29
         command: ["/bin/bash"]
27
-        args: ["-c","tdnf remove -y toybox && \
28
-                     tdnf install -y build-essential createrepo texinfo wget sudo \
29
-                     findutils python3-pip git && \
30
+        args: ["-c","tdnf install -y build-essential createrepo texinfo wget sudo \
31
+                     fuse gdbm gptfdisk grub2 grub2-efi grub2-lang grub2-pc \
32
+                     qemu-img kmod kpartx dosfstools efibootmgr ncurses e2fsprogs \
33
+                     python3 python3-asn1crypto python3-certifi python3-cffi \
34
+                     python3-chardet python3-cryptography python3-curses \
35
+                     python3-idna python3-packaging python3-pyasn1 python3-pycparser \
36
+                     python3-pyOpenSSL python3-pyparsing python3-requests python3-six \
37
+                     util-linux rpm glibc-iconv findutils python3-pip git && \
30 38
                      pip3 install flask requests pyOpenSSL docker==2.3.0 && \
31 39
                      chmod 755 /sys "]
32 40
       volumes:
... ...
@@ -36,4 +44,7 @@ spec:
36 36
       - name: sys
37 37
         hostPath:
38 38
           path: /sys
39
+      - name: dev
40
+        hostPath:
41
+          path: /dev
39 42
   backoffLimit: 1