Browse code

build system: configurable sources repo support

make PHOTON_PULLSOURCES_CONFIG=myrepo.conf will pull sources from
alternative repo (myrepo) instead of bintray.

--- myrepo.conf format ---
{
"baseurl":"http://myhost/path/to/sources/",
"user":"",
"apikey":""
}

Change-Id: I29e49d1613f803726c039f5b08c8eb30c33651be
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/2708
Reviewed-by: Vinay Kulkarni <kulkarniv@vmware.com>
Tested-by: gerrit-photon <photon-checkins@vmware.com>

Alexey Makhalov authored on 2017/05/19 06:22:19
Showing 7 changed files
... ...
@@ -107,7 +107,7 @@ packages-micro: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOUR
107 107
                 -b $(PHOTON_CHROOT_PATH) \
108 108
                 -l $(PHOTON_LOGS_DIR) \
109 109
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
110
-                -c $(PHOTON_BINTRAY_CONFIG) \
110
+                -c $(PHOTON_PULLSOURCES_CONFIG) \
111 111
                 -d $(PHOTON_DIST_TAG) \
112 112
                 -n $(PHOTON_BUILD_NUMBER) \
113 113
                 -v $(PHOTON_RELEASE_VERSION) \
... ...
@@ -161,7 +161,7 @@ packages-minimal: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_RPMS) $(PHOTON_SO
161 161
                 -b $(PHOTON_CHROOT_PATH) \
162 162
                 -l $(PHOTON_LOGS_DIR) \
163 163
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
164
-                -c $(PHOTON_BINTRAY_CONFIG) \
164
+                -c $(PHOTON_PULLSOURCES_CONFIG) \
165 165
                 -d $(PHOTON_DIST_TAG) \
166 166
                 -n $(PHOTON_BUILD_NUMBER) \
167 167
                 -v $(PHOTON_RELEASE_VERSION) \
... ...
@@ -242,7 +242,7 @@ packages: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_XRPMS) $(PHOTON_PUBLISH_R
242 242
                 -l $(PHOTON_LOGS_DIR) \
243 243
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
244 244
                 -e $(PHOTON_PUBLISH_XRPMS_DIR) \
245
-                -c $(PHOTON_BINTRAY_CONFIG) \
245
+                -c $(PHOTON_PULLSOURCES_CONFIG) \
246 246
                 -d $(PHOTON_DIST_TAG) \
247 247
                 -n $(PHOTON_BUILD_NUMBER) \
248 248
                 -v $(PHOTON_RELEASE_VERSION) \
... ...
@@ -262,7 +262,7 @@ updated-packages: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_XRPMS) $(PHOTON_P
262 262
                 -l $(PHOTON_LOGS_DIR) \
263 263
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
264 264
                 -e $(PHOTON_PUBLISH_XRPMS_DIR) \
265
-                -c $(PHOTON_BINTRAY_CONFIG) \
265
+                -c $(PHOTON_PULLSOURCES_CONFIG) \
266 266
                 -d $(PHOTON_DIST_TAG) \
267 267
                 -n $(PHOTON_BUILD_NUMBER) \
268 268
                 -v $(PHOTON_RELEASE_VERSION) \
... ...
@@ -282,7 +282,7 @@ tool-chain-stage1: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_RPMS) $(PHOTON_S
282 282
                 -l $(PHOTON_LOGS_DIR) \
283 283
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
284 284
                 -t ${THREADS} \
285
-                -c $(PHOTON_BINTRAY_CONFIG) \
285
+                -c $(PHOTON_PULLSOURCES_CONFIG) \
286 286
                 -d $(PHOTON_DIST_TAG) \
287 287
                 -n $(PHOTON_BUILD_NUMBER) \
288 288
                 -v $(PHOTON_RELEASE_VERSION) \
... ...
@@ -301,7 +301,7 @@ tool-chain-stage2: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_RPMS) $(PHOTON_S
301 301
                 -l $(PHOTON_LOGS_DIR) \
302 302
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
303 303
                 -t ${THREADS} \
304
-                -c $(PHOTON_BINTRAY_CONFIG) \
304
+                -c $(PHOTON_PULLSOURCES_CONFIG) \
305 305
                 -d $(PHOTON_DIST_TAG) \
306 306
                 -n $(PHOTON_BUILD_NUMBER) \
307 307
                 -v $(PHOTON_RELEASE_VERSION) \
... ...
@@ -317,7 +317,6 @@ packages-cached:
317 317
      $(CP) -f $(PHOTON_CACHE_PATH)/RPMS/x86_64/* $(PHOTON_RPMS_DIR_X86_64)/
318 318
 
319 319
 sources:
320
-	@echo "Pulling sources from bintray...(nothing to do)"
321 320
 	@$(MKDIR) -p $(PHOTON_SRCS_DIR)
322 321
 
323 322
 sources-cached:
... ...
@@ -525,7 +524,7 @@ check: packages
525 525
                 -l $(PHOTON_LOGS_DIR) \
526 526
                 -p $(PHOTON_PUBLISH_RPMS_DIR) \
527 527
 		-e $(PHOTON_PUBLISH_XRPMS_DIR) \
528
-                -c $(PHOTON_BINTRAY_CONFIG) \
528
+                -c $(PHOTON_PULLSOURCES_CONFIG) \
529 529
                 -d $(PHOTON_DIST_TAG) \
530 530
                 -n $(PHOTON_BUILD_NUMBER) \
531 531
                 -v $(PHOTON_RELEASE_VERSION) \
... ...
@@ -547,7 +546,7 @@ check: packages
547 547
                               -x $(PHOTON_SRCS_DIR) \
548 548
                               -p $(PHOTON_PUBLISH_RPMS_DIR) \
549 549
 			      -e $(PHOTON_PUBLISH_XRPMS_DIR) \
550
-                              -c $(PHOTON_BINTRAY_CONFIG) \
550
+                              -c $(PHOTON_PULLSOURCES_CONFIG) \
551 551
                               -d $(PHOTON_DIST_TAG) \
552 552
                               -n $(PHOTON_BUILD_NUMBER) \
553 553
                               -v $(PHOTON_RELEASE_VERSION) \
... ...
@@ -40,7 +40,11 @@ PHOTON_INSTALLER=$(PHOTON_INSTALLER_DIR)/photonInstaller.py
40 40
 PHOTON_SPECDEPS_DIR=$(SRCROOT)/support/SpecDeps
41 41
 PHOTON_SPECDEPS=$(PHOTON_SPECDEPS_DIR)/SpecDeps.py
42 42
 PHOTON_PACKAGE_BUILDER=$(PHOTON_PKG_BUILDER_DIR)/builder.py
43
-PHOTON_BINTRAY_CONFIG?=$(PHOTON_PKG_BUILDER_DIR)/pullsources.conf
43
+ifdef PHOTON_PULLSOURCES_CONFIG
44
+PHOTON_PULLSOURCES_CONFIG=$(abspath $(PHOTON_PULLSOURCES_CONFIG))
45
+else
46
+PHOTON_PULLSOURCES_CONFIG?=$(PHOTON_PKG_BUILDER_DIR)/bintray.conf
47
+endif
44 48
 PHOTON_PULL_PUBLISH_RPMS=$(PHOTON_PULL_PUBLISH_RPMS_DIR)/pullpublishrpms.sh
45 49
 PHOTON_PULL_PUBLISH_X_RPMS=$(PHOTON_PULL_PUBLISH_RPMS_DIR)/pullpublishXrpms.sh
46 50
 PHOTON_CLOUD_IMAGE_BUILDER=$(PHOTON_CLOUD_IMAGE_BUILDER_DIR)/cloud-image-build.sh
... ...
@@ -112,7 +112,7 @@ class PackageUtils(object):
112 112
             # Fetch/verify sources if sha1 not None.
113 113
             sha1 = constants.specData.getSHA1(package, source)
114 114
             if sha1 is not None:
115
-                PullSources.get(source, sha1, constants.sourcePath, constants.pullsourcesConfig)
115
+                PullSources.get(source, sha1, constants.sourcePath, constants.pullsourcesConfig, self.logger)
116 116
 
117 117
             sourcePath = cmdUtils.findFile(source,constants.sourcePath)
118 118
             if sourcePath is None or len(sourcePath) == 0:
... ...
@@ -1,11 +1,11 @@
1 1
 #! /usr/bin/python2
2 2
 #
3
-#    Copyright (C) 2015 VMware, Inc. All rights reserved.
3
+#    Copyright (C) 2015-2017 VMware, Inc. All rights reserved.
4 4
 #    pullsources.py 
5
-#    Allows pulling packages'sources from a bintary  
6
-#    repository.
5
+#    Allows pulling packages'sources from a source repository.
7 6
 #
8 7
 #    Author(s): Mahmoud Bassiouny (mbassiouny@vmware.com)
8
+#               Alexey Makhalov (amakhalov@vmware.com)
9 9
 #
10 10
 
11 11
 import json
... ...
@@ -25,7 +25,7 @@ def getFileHash(filepath):
25 25
         f.close()
26 26
     return sha1.hexdigest()
27 27
 
28
-def get(source, sha1, sourcesPath, configs):
28
+def get(source, sha1, sourcesPath, configs, logger):
29 29
     cmdUtils = CommandUtils()
30 30
     sourcePath = cmdUtils.findFile(source, sourcesPath)
31 31
     if sourcePath is not None and len(sourcePath) > 0:
... ...
@@ -35,22 +35,23 @@ def get(source, sha1, sourcesPath, configs):
35 35
             # Use file from sourcesPath
36 36
             return
37 37
         else:
38
-            print 'sha1 of %s does not match. %s vs %s' % (sourcePath[0], sha1, getFileHash(sourcePath[0]))
38
+            logger.info("sha1 of "+sourcePath[0]+" does not match. "+sha1+" vs "+getFileHash(sourcePath[0]))
39 39
     configFiles=configs.split(":")
40 40
     for config in configFiles:
41
-        p = pullSources(config)
41
+        p = pullSources(config, logger)
42 42
         package_path = os.path.join(sourcesPath, source)
43 43
         try: 
44 44
             p.downloadFileHelper(source, package_path, sha1)
45 45
             return
46 46
         except Exception as e:
47
-            print e
47
+            logger.error(e)
48 48
     raise Exception("Missing source: "+source)
49 49
 
50 50
 class pullSources:
51 51
 
52
-    def __init__(self, conf_file):
52
+    def __init__(self, conf_file, logger):
53 53
         self._config = {}
54
+        self.logger=logger
54 55
         self.loadConfig(conf_file)
55 56
 
56 57
         # generate the auth
... ...
@@ -65,14 +66,9 @@ class pullSources:
65 65
 
66 66
     def downloadFile(self, filename, file_path):
67 67
         #form url: https://dl.bintray.com/vmware/photon_sources/1.0/<filename>.
68
-        url = '%s/%s/%s/%s/%s' % \
69
-              (self._config['baseurl'],\
70
-               self._config['subject'],\
71
-               self._config['repo'],\
72
-               self._config['version'],\
73
-               filename)
68
+        url = '%s/%s' % (self._config['baseurl'], filename)
74 69
 
75
-        print '%s: Downloading %s...' % (str(datetime.datetime.today()), url)
70
+        self.logger.info("Downloading: "+url)
76 71
 
77 72
         with open(file_path, 'wb') as handle:
78 73
             response = requests.get(url, auth=self._auth, stream=True)
79 74
new file mode 100644
... ...
@@ -0,0 +1,5 @@
0
+{
1
+  "baseurl":"https://dl.bintray.com/vmware/photon_sources/1.0",
2
+  "user":"",
3
+  "apikey":""
4
+}
... ...
@@ -192,7 +192,7 @@ def buildSourcesList(yamlDir, logger, singleFile=True):
192 192
             sourceName=listSourceNames[0]
193 193
             sha1 = constants.specData.getSHA1(package, sourceName)
194 194
             if sha1 is not None:
195
-                PullSources.get(sourceName, sha1, yamlDir, constants.pullsourcesConfig)
195
+                PullSources.get(sourceName, sha1, yamlDir, constants.pullsourcesConfig, logger)
196 196
 
197 197
         if not singleFile:
198 198
             yamlFile = open(yamlDir+"/"+ossname+"-"+ossversion+".yaml", "w")
199 199
deleted file mode 100644
... ...
@@ -1,8 +0,0 @@
1
-{
2
-  "baseurl":"https://dl.bintray.com",
3
-  "subject":"vmware",
4
-  "repo":"photon_sources",
5
-  "version":"1.0",
6
-  "user":"", 
7
-  "apikey":""
8
-}