#!/usr/bin/env python3 # pylint: disable=invalid-name,missing-docstring import os import json import sys import traceback from argparse import ArgumentParser from Logger import Logger from constants import constants from CommandUtils import CommandUtils from SpecData import SPECS def main(): usage = "Usage: %prog [options] " parser = ArgumentParser(usage) parser.add_argument("-s", "--spec-path", dest="specPath", default="../../SPECS") parser.add_argument("-l", "--log-path", dest="logPath", default="../../stage/LOGS") parser.add_argument("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS") parser.add_argument("-j", "--output-dir", dest="outputDirPath", default="../../stage/") parser.add_argument("-z", "--log-level", dest="logLevel", default="info") parser.add_argument("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf") parser.add_argument("-f", "--pkg-blacklist-file", dest="pkgBlacklistFile", default=None) parser.add_argument("-p", "--generate-pkg-list", dest="generatePkgList", default=False, action="store_true") parser.add_argument("-y", "--generate-yaml-files", dest="generateYamlFiles", default=False, action="store_true") parser.add_argument("-d", "--dist-tag", dest="dist", default="") options = parser.parse_args() errorFlag = False cmdUtils = CommandUtils() try: logName = "GenerateYamlFiles" logger = Logger.getLogger(logName, options.logPath, options.logLevel) if options.generateYamlFiles: if (options.pkgBlacklistFile is not None and options.pkgBlacklistFile != "" and not os.path.isfile(options.pkgBlacklistFile)): logger.error("Given package blacklist file is not valid:" + options.pkgBlacklistFile) errorFlag = True if not os.path.isdir(options.specPath): logger.error("Given Specs Path is not a directory:" + options.specPath) errorFlag = True if not os.path.isdir(options.sourceRpmPath): logger.error("Given SRPM Path is not a directory:" + options.sourceRpmPath) errorFlag = True if options.generateYamlFiles and not os.path.isfile(options.pullsourcesConfig): logger.error("Given Source config file is not a valid file:" + options.pullsourcesConfig) errorFlag = True if options.dist: dist_tag = options.dist logger.info("release tag is %s" % (dist_tag)) if errorFlag: logger.error("Found some errors. Please fix input options and re-run it.") sys.exit(1) if options.generateYamlFiles: if not os.path.isdir(options.outputDirPath): cmdUtils.runCommandInShell("mkdir -p "+options.outputDirPath) constants.setSpecPath(options.specPath) constants.setSourceRpmPath(options.sourceRpmPath) constants.setLogPath(options.logPath) constants.setLogLevel(options.logLevel) constants.setPullSourcesURL(get_baseurl(options.pullsourcesConfig)) constants.initialize() # parse SPECS folder SPECS() if options.generatePkgList: buildPackagesList(options.outputDirPath + "/packages_list.csv") elif options.generateYamlFiles: blackListPkgs = readBlackListPackages(options.pkgBlacklistFile) buildSourcesList(options.outputDirPath, blackListPkgs, logger) buildSRPMList(options.sourceRpmPath, options.outputDirPath, blackListPkgs, dist_tag, logger) except Exception as e: print("Caught Exception: " + str(e)) traceback.print_exc() sys.exit(1) sys.exit(0) def get_baseurl(self, conf_file): with open(conf_file) as jsonFile: config = json.load(jsonFile) return config['baseurl'] def buildPackagesList(csvFilename): with open(csvFilename, "w") as csvFile: csvFile.write("Package,Version,License,URL,Sources,Patches\n") listPackages = SPECS.getData().getListPackages() listPackages.sort() for package in listPackages: name = package for version in SPECS.getData().getVersions(package): packagelicense = SPECS.getData().getLicense(package, version) listPatches = SPECS.getData().getPatches(package, version) url = SPECS.getData().getURL(package, version) listSourceNames = SPECS.getData().getSources(package, version) sources = "" patches = "" if listPatches is not None: patches = " ".join(listPatches) if listSourceNames is not None: sources = " ".join(listSourceNames) csvFile.write(name + "," + version + "," + packagelicense + "," + url + "," + sources + "," + patches + "\n") def readBlackListPackages(pkgBlackListFile): blackListPkgs = [] if pkgBlackListFile is not None and pkgBlackListFile != "": with open(pkgBlackListFile) as jsonFile: config = json.load(jsonFile) if 'packages' in config: blackListPkgs = config['packages'] return blackListPkgs def buildSourcesList(yamlDir, blackListPkgs, logger, singleFile=True): cmdUtils = CommandUtils() yamlSourceDir = os.path.join(yamlDir, "yaml_sources") if not os.path.isdir(yamlSourceDir): cmdUtils.runCommandInShell("mkdir -p " + yamlSourceDir) if singleFile: yamlFile = open(yamlSourceDir + "/sources_list.yaml", "w") listPackages = SPECS.getData().getListPackages() listPackages.sort() import PullSources for package in listPackages: if package in blackListPkgs: continue ossname = package for version in SPECS.getData().getVersions(package): modified = False listPatches = SPECS.getData().getPatches(package, version) if listPatches: modified = True url = SPECS.getData().getSourceURL(package, version) if url is None: url = SPECS.getData().getURL(package, version) sourceName = None listSourceNames = SPECS.getData().getSources(package, version) if listSourceNames: sourceName = listSourceNames[0] sha1 = SPECS.getData().getSHA1(package, version, sourceName) if sha1 is not None: PullSources.get(package, sourceName, sha1, yamlSourceDir, constants.getPullSourcesURLs(package), logger) if not singleFile: yamlFile = open(yamlSourceDir + "/" + ossname + "-" + version + ".yaml", "w") yamlFile.write("vmwsource:" + ossname + ":" + version + ":\n") yamlFile.write(" repository: VMWsource\n") yamlFile.write(" name: '" + ossname + "'\n") yamlFile.write(" version: '" + version + "'\n") yamlFile.write(" url: " + str(url) + "\n") yamlFile.write(" license: UNKNOWN\n") if sourceName is not None: yamlFile.write(" vmwsource-distribution: " + str(sourceName) + "\n") if modified: yamlFile.write(" modified: true\n") yamlFile.write("\n") if not singleFile: yamlFile.close() if singleFile: yamlFile.close() logger.debug("Generated source yaml files for all packages") def buildSRPMList(srpmPath, yamlDir, blackListPkgs, dist_tag, logger, singleFile=True): cmdUtils = CommandUtils() yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms") if not os.path.isdir(yamlSrpmDir): cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir) if singleFile: yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w") listPackages = SPECS.getData().getListPackages() listPackages.sort() for package in listPackages: if package in blackListPkgs: continue ossname = package for ossversion in SPECS.getData().getVersions(package): ossrelease = SPECS.getData().getRelease(package, ossversion) srpm_file_name = "%s-%s-%s%s.src.rpm" % (ossname, ossversion, ossrelease, dist_tag) logger.info("srpm name is %s" % (srpm_file_name)) listFoundSRPMFiles = cmdUtils.findFile(srpm_file_name, srpmPath) srpmName = None if len(listFoundSRPMFiles) == 1: srpmFullPath = listFoundSRPMFiles[0] srpmName = os.path.basename(srpmFullPath) cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/" returnVal = cmdUtils.runCommandInShell(cpcmd) if not returnVal: logger.error("Copy SRPM File is failed for package:" + ossname) else: logger.error("SRPM file is not found:" + ossname) if not singleFile: yamlFile = open(yamlSrpmDir + "/" + ossname + "-" + ossversion + "-" + ossrelease + ".yaml", "w") yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" + ossrelease + ":\n") yamlFile.write(" repository: BaseOS\n") yamlFile.write(" name: '" + ossname + "'\n") yamlFile.write(" version: '" + ossversion + "-" + ossrelease + "'\n") yamlFile.write(" url: 'http://www.vmware.com'\n") yamlFile.write(" baseos-style: rpm\n") yamlFile.write(" baseos-source: '" + str(srpmName) + "'\n") yamlFile.write(" baseos-osname: 'photon'\n") yamlFile.write("\n") if not singleFile: yamlFile.close() if singleFile: yamlFile.close() logger.debug("Generated SRPM yaml files for all packages") if __name__ == "__main__": main()