make PHOTON_PULLSOURCES_CONFIG=myrepo.conf will pull sources from
alternative repo (myrepo) instead of bintray.
--- myrepo.conf format ---
{
"baseurl":"http://myhost/path/to/sources/",
"user":"",
"apikey":""
}
Change-Id: I29e49d1613f803726c039f5b08c8eb30c33651be
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/2708
Reviewed-by: Vinay Kulkarni <kulkarniv@vmware.com>
Tested-by: gerrit-photon <photon-checkins@vmware.com>
... | ... |
@@ -107,7 +107,7 @@ packages-micro: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOUR |
107 | 107 |
-b $(PHOTON_CHROOT_PATH) \ |
108 | 108 |
-l $(PHOTON_LOGS_DIR) \ |
109 | 109 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
110 |
- -c $(PHOTON_BINTRAY_CONFIG) \ |
|
110 |
+ -c $(PHOTON_PULLSOURCES_CONFIG) \ |
|
111 | 111 |
-d $(PHOTON_DIST_TAG) \ |
112 | 112 |
-n $(PHOTON_BUILD_NUMBER) \ |
113 | 113 |
-v $(PHOTON_RELEASE_VERSION) \ |
... | ... |
@@ -161,7 +161,7 @@ packages-minimal: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_RPMS) $(PHOTON_SO |
161 | 161 |
-b $(PHOTON_CHROOT_PATH) \ |
162 | 162 |
-l $(PHOTON_LOGS_DIR) \ |
163 | 163 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
164 |
- -c $(PHOTON_BINTRAY_CONFIG) \ |
|
164 |
+ -c $(PHOTON_PULLSOURCES_CONFIG) \ |
|
165 | 165 |
-d $(PHOTON_DIST_TAG) \ |
166 | 166 |
-n $(PHOTON_BUILD_NUMBER) \ |
167 | 167 |
-v $(PHOTON_RELEASE_VERSION) \ |
... | ... |
@@ -242,7 +242,7 @@ packages: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_XRPMS) $(PHOTON_PUBLISH_R |
242 | 242 |
-l $(PHOTON_LOGS_DIR) \ |
243 | 243 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
244 | 244 |
-e $(PHOTON_PUBLISH_XRPMS_DIR) \ |
245 |
- -c $(PHOTON_BINTRAY_CONFIG) \ |
|
245 |
+ -c $(PHOTON_PULLSOURCES_CONFIG) \ |
|
246 | 246 |
-d $(PHOTON_DIST_TAG) \ |
247 | 247 |
-n $(PHOTON_BUILD_NUMBER) \ |
248 | 248 |
-v $(PHOTON_RELEASE_VERSION) \ |
... | ... |
@@ -262,7 +262,7 @@ updated-packages: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_XRPMS) $(PHOTON_P |
262 | 262 |
-l $(PHOTON_LOGS_DIR) \ |
263 | 263 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
264 | 264 |
-e $(PHOTON_PUBLISH_XRPMS_DIR) \ |
265 |
- -c $(PHOTON_BINTRAY_CONFIG) \ |
|
265 |
+ -c $(PHOTON_PULLSOURCES_CONFIG) \ |
|
266 | 266 |
-d $(PHOTON_DIST_TAG) \ |
267 | 267 |
-n $(PHOTON_BUILD_NUMBER) \ |
268 | 268 |
-v $(PHOTON_RELEASE_VERSION) \ |
... | ... |
@@ -282,7 +282,7 @@ tool-chain-stage1: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_RPMS) $(PHOTON_S |
282 | 282 |
-l $(PHOTON_LOGS_DIR) \ |
283 | 283 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
284 | 284 |
-t ${THREADS} \ |
285 |
- -c $(PHOTON_BINTRAY_CONFIG) \ |
|
285 |
+ -c $(PHOTON_PULLSOURCES_CONFIG) \ |
|
286 | 286 |
-d $(PHOTON_DIST_TAG) \ |
287 | 287 |
-n $(PHOTON_BUILD_NUMBER) \ |
288 | 288 |
-v $(PHOTON_RELEASE_VERSION) \ |
... | ... |
@@ -301,7 +301,7 @@ tool-chain-stage2: check-tools $(PHOTON_STAGE) $(PHOTON_PUBLISH_RPMS) $(PHOTON_S |
301 | 301 |
-l $(PHOTON_LOGS_DIR) \ |
302 | 302 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
303 | 303 |
-t ${THREADS} \ |
304 |
- -c $(PHOTON_BINTRAY_CONFIG) \ |
|
304 |
+ -c $(PHOTON_PULLSOURCES_CONFIG) \ |
|
305 | 305 |
-d $(PHOTON_DIST_TAG) \ |
306 | 306 |
-n $(PHOTON_BUILD_NUMBER) \ |
307 | 307 |
-v $(PHOTON_RELEASE_VERSION) \ |
... | ... |
@@ -317,7 +317,6 @@ packages-cached: |
317 | 317 |
$(CP) -f $(PHOTON_CACHE_PATH)/RPMS/x86_64/* $(PHOTON_RPMS_DIR_X86_64)/ |
318 | 318 |
|
319 | 319 |
sources: |
320 |
- @echo "Pulling sources from bintray...(nothing to do)" |
|
321 | 320 |
@$(MKDIR) -p $(PHOTON_SRCS_DIR) |
322 | 321 |
|
323 | 322 |
sources-cached: |
... | ... |
@@ -525,7 +524,7 @@ check: packages |
525 | 525 |
-l $(PHOTON_LOGS_DIR) \ |
526 | 526 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
527 | 527 |
-e $(PHOTON_PUBLISH_XRPMS_DIR) \ |
528 |
- -c $(PHOTON_BINTRAY_CONFIG) \ |
|
528 |
+ -c $(PHOTON_PULLSOURCES_CONFIG) \ |
|
529 | 529 |
-d $(PHOTON_DIST_TAG) \ |
530 | 530 |
-n $(PHOTON_BUILD_NUMBER) \ |
531 | 531 |
-v $(PHOTON_RELEASE_VERSION) \ |
... | ... |
@@ -547,7 +546,7 @@ check: packages |
547 | 547 |
-x $(PHOTON_SRCS_DIR) \ |
548 | 548 |
-p $(PHOTON_PUBLISH_RPMS_DIR) \ |
549 | 549 |
-e $(PHOTON_PUBLISH_XRPMS_DIR) \ |
550 |
- -c $(PHOTON_BINTRAY_CONFIG) \ |
|
550 |
+ -c $(PHOTON_PULLSOURCES_CONFIG) \ |
|
551 | 551 |
-d $(PHOTON_DIST_TAG) \ |
552 | 552 |
-n $(PHOTON_BUILD_NUMBER) \ |
553 | 553 |
-v $(PHOTON_RELEASE_VERSION) \ |
... | ... |
@@ -40,7 +40,11 @@ PHOTON_INSTALLER=$(PHOTON_INSTALLER_DIR)/photonInstaller.py |
40 | 40 |
PHOTON_SPECDEPS_DIR=$(SRCROOT)/support/SpecDeps |
41 | 41 |
PHOTON_SPECDEPS=$(PHOTON_SPECDEPS_DIR)/SpecDeps.py |
42 | 42 |
PHOTON_PACKAGE_BUILDER=$(PHOTON_PKG_BUILDER_DIR)/builder.py |
43 |
-PHOTON_BINTRAY_CONFIG?=$(PHOTON_PKG_BUILDER_DIR)/pullsources.conf |
|
43 |
+ifdef PHOTON_PULLSOURCES_CONFIG |
|
44 |
+PHOTON_PULLSOURCES_CONFIG=$(abspath $(PHOTON_PULLSOURCES_CONFIG)) |
|
45 |
+else |
|
46 |
+PHOTON_PULLSOURCES_CONFIG?=$(PHOTON_PKG_BUILDER_DIR)/bintray.conf |
|
47 |
+endif |
|
44 | 48 |
PHOTON_PULL_PUBLISH_RPMS=$(PHOTON_PULL_PUBLISH_RPMS_DIR)/pullpublishrpms.sh |
45 | 49 |
PHOTON_PULL_PUBLISH_X_RPMS=$(PHOTON_PULL_PUBLISH_RPMS_DIR)/pullpublishXrpms.sh |
46 | 50 |
PHOTON_CLOUD_IMAGE_BUILDER=$(PHOTON_CLOUD_IMAGE_BUILDER_DIR)/cloud-image-build.sh |
... | ... |
@@ -112,7 +112,7 @@ class PackageUtils(object): |
112 | 112 |
# Fetch/verify sources if sha1 not None. |
113 | 113 |
sha1 = constants.specData.getSHA1(package, source) |
114 | 114 |
if sha1 is not None: |
115 |
- PullSources.get(source, sha1, constants.sourcePath, constants.pullsourcesConfig) |
|
115 |
+ PullSources.get(source, sha1, constants.sourcePath, constants.pullsourcesConfig, self.logger) |
|
116 | 116 |
|
117 | 117 |
sourcePath = cmdUtils.findFile(source,constants.sourcePath) |
118 | 118 |
if sourcePath is None or len(sourcePath) == 0: |
... | ... |
@@ -1,11 +1,11 @@ |
1 | 1 |
#! /usr/bin/python2 |
2 | 2 |
# |
3 |
-# Copyright (C) 2015 VMware, Inc. All rights reserved. |
|
3 |
+# Copyright (C) 2015-2017 VMware, Inc. All rights reserved. |
|
4 | 4 |
# pullsources.py |
5 |
-# Allows pulling packages'sources from a bintary |
|
6 |
-# repository. |
|
5 |
+# Allows pulling packages'sources from a source repository. |
|
7 | 6 |
# |
8 | 7 |
# Author(s): Mahmoud Bassiouny (mbassiouny@vmware.com) |
8 |
+# Alexey Makhalov (amakhalov@vmware.com) |
|
9 | 9 |
# |
10 | 10 |
|
11 | 11 |
import json |
... | ... |
@@ -25,7 +25,7 @@ def getFileHash(filepath): |
25 | 25 |
f.close() |
26 | 26 |
return sha1.hexdigest() |
27 | 27 |
|
28 |
-def get(source, sha1, sourcesPath, configs): |
|
28 |
+def get(source, sha1, sourcesPath, configs, logger): |
|
29 | 29 |
cmdUtils = CommandUtils() |
30 | 30 |
sourcePath = cmdUtils.findFile(source, sourcesPath) |
31 | 31 |
if sourcePath is not None and len(sourcePath) > 0: |
... | ... |
@@ -35,22 +35,23 @@ def get(source, sha1, sourcesPath, configs): |
35 | 35 |
# Use file from sourcesPath |
36 | 36 |
return |
37 | 37 |
else: |
38 |
- print 'sha1 of %s does not match. %s vs %s' % (sourcePath[0], sha1, getFileHash(sourcePath[0])) |
|
38 |
+ logger.info("sha1 of "+sourcePath[0]+" does not match. "+sha1+" vs "+getFileHash(sourcePath[0])) |
|
39 | 39 |
configFiles=configs.split(":") |
40 | 40 |
for config in configFiles: |
41 |
- p = pullSources(config) |
|
41 |
+ p = pullSources(config, logger) |
|
42 | 42 |
package_path = os.path.join(sourcesPath, source) |
43 | 43 |
try: |
44 | 44 |
p.downloadFileHelper(source, package_path, sha1) |
45 | 45 |
return |
46 | 46 |
except Exception as e: |
47 |
- print e |
|
47 |
+ logger.error(e) |
|
48 | 48 |
raise Exception("Missing source: "+source) |
49 | 49 |
|
50 | 50 |
class pullSources: |
51 | 51 |
|
52 |
- def __init__(self, conf_file): |
|
52 |
+ def __init__(self, conf_file, logger): |
|
53 | 53 |
self._config = {} |
54 |
+ self.logger=logger |
|
54 | 55 |
self.loadConfig(conf_file) |
55 | 56 |
|
56 | 57 |
# generate the auth |
... | ... |
@@ -65,14 +66,9 @@ class pullSources: |
65 | 65 |
|
66 | 66 |
def downloadFile(self, filename, file_path): |
67 | 67 |
#form url: https://dl.bintray.com/vmware/photon_sources/1.0/<filename>. |
68 |
- url = '%s/%s/%s/%s/%s' % \ |
|
69 |
- (self._config['baseurl'],\ |
|
70 |
- self._config['subject'],\ |
|
71 |
- self._config['repo'],\ |
|
72 |
- self._config['version'],\ |
|
73 |
- filename) |
|
68 |
+ url = '%s/%s' % (self._config['baseurl'], filename) |
|
74 | 69 |
|
75 |
- print '%s: Downloading %s...' % (str(datetime.datetime.today()), url) |
|
70 |
+ self.logger.info("Downloading: "+url) |
|
76 | 71 |
|
77 | 72 |
with open(file_path, 'wb') as handle: |
78 | 73 |
response = requests.get(url, auth=self._auth, stream=True) |
... | ... |
@@ -192,7 +192,7 @@ def buildSourcesList(yamlDir, logger, singleFile=True): |
192 | 192 |
sourceName=listSourceNames[0] |
193 | 193 |
sha1 = constants.specData.getSHA1(package, sourceName) |
194 | 194 |
if sha1 is not None: |
195 |
- PullSources.get(sourceName, sha1, yamlDir, constants.pullsourcesConfig) |
|
195 |
+ PullSources.get(sourceName, sha1, yamlDir, constants.pullsourcesConfig, logger) |
|
196 | 196 |
|
197 | 197 |
if not singleFile: |
198 | 198 |
yamlFile = open(yamlDir+"/"+ossname+"-"+ossversion+".yaml", "w") |