#! /usr/bin/python2 # # Copyright (C) 2015 VMware, Inc. All rights reserved. # photonpublish.py # Allows pushing rpms and other artifacts to # a bintray repository. # Allows queying a repository to get existing # file details. # # Author(s): Priyesh Padmavilasom # import sys import getopt import json import glob import os import hashlib import requests from requests.auth import HTTPBasicAuth from publishutils import publishUtils from publishconst import publishConst const = publishConst() class photonPublish: def __init__(self, context): self._context = context self._config = {} self.loadConfig() def loadConfig(self): confFile = self._context['config'] if(len(confFile) == 0): return with open(confFile) as jsonFile: self._config = json.load(jsonFile) #override cmdline supplied params if('user' in self._context and len(self._context['user']) > 0): self._config['user'] = self._context['user'] if('apikey' in self._context and len(self._context['apikey']) > 0): self._config['apikey'] = self._context['apikey'] #get details of existing files in remote def getPackages(self): auth = HTTPBasicAuth(self._config['user'], self._config['apikey']) #form url: https://api.com/content/vmware/photon/releases/0.9 for eg. url = '%s/packages/%s/%s/%s/versions/%s/files?include_unpublished=1' % \ (self._config['baseurl'],\ self._config['subject'],\ self._config['repo'],\ self._config['package'],\ self._config['version']) req = requests.get(url, auth=auth) if(req.status_code >= 300): raise Exception(req.text) return req.json() #return list of unpublished content #works with details from config file def getUnpublished(self): result = [] pkgs = self.getPackages() for pkg in pkgs: if not pkg[const.published]: result.append(pkg) return result #Check if the local path has any diffs with the #remote repo. Compare sha1 hash def check(self, pkgsRoot): result = { const.updates:[], const.new:[], const.obsoletes:[], const.verified:[] } localFiles = publishUtils.getFilesWithRelativePath(pkgsRoot) pkgs = self.getPackages() for pkg in pkgs: remotePath = pkg[const.path] if(remotePath in localFiles): localFiles.remove(remotePath) localPath = os.path.join(pkgsRoot, remotePath) if(os.path.isfile(localPath)): sha1 = publishUtils.sha1OfFile(localPath) if(sha1 == pkg[const.sha1]): result[const.verified].append(pkg) else: result[const.updates].append(pkg) else: result[const.obsoletes].append(pkg) for newFile in localFiles: result[const.new].append({const.path:newFile}) return result #exec results from a check against remote #this will make remote and local in sync def syncRemote(self, root, checkResults): updateCount = len(checkResults[const.updates]) newCount = len(checkResults[const.new]) if(updateCount + newCount == 0): print 'Remote is up to date.' return #push updates print 'Updating %d files' % updateCount for new in checkResults[const.updates]: filePath = new[const.path] fileName = os.path.basename(filePath) fullPath = os.path.join(root, filePath) pathName = filePath.rstrip(fileName).rstrip('/') self.updateFile(fullPath, pathName) #push new files print 'Pushing %d new files' % newCount for new in checkResults[const.new]: filePath = new[const.path] fileName = os.path.basename(filePath) fullPath = os.path.join(root, filePath) pathName = filePath.rstrip(fileName).rstrip('/') self.pushFile(fullPath, pathName) #push a folder with rpms to a specified pathname #in the remote repo def push(self, filePath, pathName): results = [] filesToPush = glob.glob(filePath) for fileToPush in filesToPush: result = self.pushFile(fileToPush, pathName) print result results.append(result) def pushFile(self, fileToPush, pathName): print 'Pushing file %s to path %s' % (fileToPush, pathName) result = {} auth = HTTPBasicAuth(self._config['user'], self._config['apikey']) fileName = os.path.basename(fileToPush) if(len(pathName) > 0): pathAndFileName = '%s/%s' % (pathName, fileName) else: pathAndFileName = fileName #form url: https://api.com/content/vmware/photon/releases/0.9 for eg. url = '%s/content/%s/%s/%s/%s/%s' % \ (self._config['baseurl'],\ self._config['subject'],\ self._config['repo'],\ self._config['package'],\ self._config['version'],\ pathAndFileName) headers={'Content-Type': 'application/octet-stream'} with open(fileToPush, 'rb') as chunkedData: req = requests.put(url, auth=auth, data=chunkedData, headers=headers) if(req.status_code >= 300): raise Exception(req.text) result['destPath'] = pathAndFileName result['sourcePath'] = fileToPush result['returnCode'] = req.status_code result['msg'] = req.text return result def updateFile(self, fileToPush, pathName): print 'Updating file %s at path %s' % (fileToPush, pathName) result = {} auth = HTTPBasicAuth(self._config['user'], self._config['apikey']) fileName = os.path.basename(fileToPush) if(len(pathName) > 0): pathAndFileName = '%s/%s' % (pathName, fileName) else: pathAndFileName = fileName #form url: https://api.com/content/vmware/photon/releases/0.9 for eg. url = '%s/content/%s/%s/%s/%s/%s?override=1' % \ (self._config['baseurl'],\ self._config['subject'],\ self._config['repo'],\ self._config['package'],\ self._config['version'],\ pathAndFileName) headers={'Content-Type': 'application/octet-stream'} with open(fileToPush, 'rb') as chunkedData: req = requests.put(url, auth=auth, data=chunkedData, headers=headers) if(req.status_code >= 300): raise Exception(req.text) result['destPath'] = pathAndFileName result['sourcePath'] = fileToPush result['returnCode'] = req.status_code result['msg'] = req.text return result #publishes pending content. Works with details from conf file. def publish(self): print 'Publishing pending files to %s/%s/%s' \ % (self._config['repo'], self._config['package'], self._config['version']) result = {} auth = HTTPBasicAuth(self._config['user'], self._config['apikey']) #form url: https://api.com/content/vmware/photon/releases/0.9 for eg. url = '%s/content/%s/%s/%s/%s/publish' % \ (self._config['baseurl'],\ self._config['subject'],\ self._config['repo'],\ self._config['package'],\ self._config['version']) req = requests.post(url, auth=auth) if(req.status_code >= 300): raise Exception(req.text) return req.json() def showUsage(): print 'photonpublish.py --files /rpms/*.rpm' print 'if you need to override config, --config /conf.conf' print 'if you need to override user/apikey, provide' print '--user username --apikey apikey' def validate(context): return len(context['config']) > 0 and len(context['files']) > 0 #test photonPublish def main(argv): try: context = { 'config':'/etc/photonpublish.conf', 'user':'', 'apikey':'', 'files':'', 'path':'', 'mode':'' } opts, args = getopt.getopt( sys.argv[1:], '', ['config=','user=','apikey=','files=','path=','mode=']) for opt, arg in opts: if opt == '--config': context['config'] = arg elif opt == '--user': context['user'] = arg elif opt == '--apikey': context['apikey'] = arg elif opt == '--files': context['files'] = arg elif opt == '--path': context['path'] = arg elif opt == '--mode': context['mode'] = arg if(not validate(context)): showUsage() sys.exit(1) publish = photonPublish(context) if(context['mode'] == 'upload'): publish.push(context['files'], context['path']) elif(context['mode'] == 'check'): print publish.check(context['files']) except Exception, e: print "Error: %s" % e sys.exit(1) if __name__ == "__main__": main(sys.argv)