# pullsources.py
# Allows pulling packages'sources from a source repository.
#
# Author(s): Mahmoud Bassiouny (mbassiouny@vmware.com)
# Alexey Makhalov (amakhalov@vmware.com)
#
import json
import os
import hashlib
import requests
import string
import random
from CommandUtils import CommandUtils
def getFileHash(filepath):
sha1 = hashlib.sha1()
f = open(filepath, 'rb')
try:
sha1.update(f.read())
finally:
f.close()
return sha1.hexdigest()
def get(package, source, sha1, sourcesPath, URLs, logger):
cmdUtils = CommandUtils()
sourcePath = cmdUtils.findFile(source, sourcesPath)
if sourcePath is not None and len(sourcePath) > 0:
if len(sourcePath) > 1:
raise Exception("Multiple sources found for source:" + source + "\n" +
",".join(sourcePath) +"\nUnable to determine one.")
if sha1 == getFileHash(sourcePath[0]):
# Use file from sourcesPath
return
else:
logger.info("sha1 of " + sourcePath[0] + " does not match. " + sha1 +
" vs " + getFileHash(sourcePath[0]))
for baseurl in URLs:
#form url: https://dl.bintray.com/vmware/photon_sources/1.0/<filename>.
url = '%s/%s' % (baseurl, source)
destfile = os.path.join(sourcesPath, source)
logger.debug("Downloading: " + url)
try:
downloadFile(url, destfile)
if sha1 != getFileHash(destfile):
raise Exception('Invalid sha1 for package %s file %s' % package, source)
return
except requests.exceptions.HTTPError as e:
logger.exception(e)
# on any HTTP errors - try next config
continue
except Exception as e:
logger.exception(e)
raise Exception("Missing source: " + source)
def downloadFile(url, destfile):
# We need to provide atomicity for file downloads. That is,
# the file should be visible in its canonical location only
# when the download is complete. To achieve that, first
# download to a temporary location (on the same filesystem)
# and then rename it to the final destination filename.
temp_file = destfile + "-" + \
"".join([random.choice(
string.ascii_letters + string.digits) for _ in range(6)])
response = requests.get(url, stream=True)
if not response.ok:
# Something went wrong
response.raise_for_status()
with open(temp_file, 'wb') as handle:
for block in response.iter_content(1024):
if not block:
break
handle.write(block)
handle.flush()
response.close()
if os.path.exists(destfile):
os.remove(temp_file)
else:
os.rename(temp_file, destfile)
return destfile