support/package-builder/GenerateOSSFiles.py
accc8120
 #!/usr/bin/env python3
326d5ca8
 # pylint: disable=invalid-name,missing-docstring
accc8120
 import os
 import json
 import sys
 import traceback
 from argparse import ArgumentParser
326d5ca8
 from Logger import Logger
accc8120
 from constants import constants
 from CommandUtils import CommandUtils
 from SpecData import SPECS
 
 
 
 def main():
     usage = "Usage: %prog [options] <package name>"
     parser = ArgumentParser(usage)
326d5ca8
     parser.add_argument("-s", "--spec-path", dest="specPath",
                         default="../../SPECS")
     parser.add_argument("-l", "--log-path", dest="logPath",
                         default="../../stage/LOGS")
     parser.add_argument("-a", "--source-rpm-path", dest="sourceRpmPath",
                         default="../../stage/SRPMS")
     parser.add_argument("-j", "--output-dir", dest="outputDirPath",
                         default="../../stage/")
     parser.add_argument("-c", "--pullsources-config", dest="pullsourcesConfig",
                         default="pullsources.conf")
     parser.add_argument("-f", "--pkg-blacklist-file", dest="pkgBlacklistFile",
                         default=None)
     parser.add_argument("-p", "--generate-pkg-list", dest="generatePkgList",
                         default=False, action="store_true")
     parser.add_argument("-y", "--generate-yaml-files", dest="generateYamlFiles",
                         default=False, action="store_true")
accc8120
 
     options = parser.parse_args()
     errorFlag = False
     cmdUtils = CommandUtils()
 
     try:
         if not os.path.isdir(options.logPath):
             cmdUtils.runCommandInShell("mkdir -p " + options.logPath)
         logger = Logger.getLogger(options.logPath + "/generateYamlFiles")
 
         if options.generateYamlFiles:
             if (options.pkgBlacklistFile is not None and
                     options.pkgBlacklistFile != "" and
                     not os.path.isfile(options.pkgBlacklistFile)):
326d5ca8
                 logger.error("Given package blacklist file is not valid:"
                              + options.pkgBlacklistFile)
accc8120
                 errorFlag = True
 
         if not os.path.isdir(options.specPath):
             logger.error("Given Specs Path is not a directory:" + options.specPath)
             errorFlag = True
 
         if not os.path.isdir(options.sourceRpmPath):
             logger.error("Given SRPM Path is not a directory:" + options.sourceRpmPath)
             errorFlag = True
 
         if options.generateYamlFiles and not os.path.isfile(options.pullsourcesConfig):
326d5ca8
             logger.error("Given Source config file is not a valid file:"
                          + options.pullsourcesConfig)
accc8120
             errorFlag = True
 
         if errorFlag:
             logger.error("Found some errors. Please fix input options and re-run it.")
             sys.exit(1)
 
         if options.generateYamlFiles:
             if not os.path.isdir(options.outputDirPath):
                 cmdUtils.runCommandInShell("mkdir -p "+options.outputDirPath)
 
         constants.setSpecPath(options.specPath)
         constants.setSourceRpmPath(options.sourceRpmPath)
         constants.setLogPath(options.logPath)
         constants.setPullSourcesConfig(options.pullsourcesConfig)
         constants.initialize()
 
         # parse SPECS folder
         SPECS()
 
         if options.generatePkgList:
             buildPackagesList(options.outputDirPath + "/packages_list.csv")
         elif options.generateYamlFiles:
             blackListPkgs = readBlackListPackages(options.pkgBlacklistFile)
             buildSourcesList(options.outputDirPath, blackListPkgs, logger)
             buildSRPMList(options.sourceRpmPath, options.outputDirPath, blackListPkgs, logger)
 
     except Exception as e:
326d5ca8
         print("Caught Exception: " + str(e))
accc8120
         traceback.print_exc()
         sys.exit(1)
 
     sys.exit(0)
 
 
 def buildPackagesList(csvFilename):
326d5ca8
     with open(csvFilename, "w") as csvFile:
         csvFile.write("Package,Version,License,URL,Sources,Patches\n")
         listPackages = SPECS.getData().getListPackages()
         listPackages.sort()
         for package in listPackages:
             name = package
             version = SPECS.getData().getVersion(package)
             packagelicense = SPECS.getData().getLicense(package)
             listPatches = SPECS.getData().getPatches(package)
             url = SPECS.getData().getURL(package)
             listSourceNames = SPECS.getData().getSources(package)
             sources = ""
             patches = ""
             if listPatches is not None:
                 patches = " ".join(listPatches)
             if listSourceNames is not None:
                 sources = " ".join(listSourceNames)
             csvFile.write(name + "," + version + "," + packagelicense + "," + url + "," +
                           sources + "," + patches + "\n")
accc8120
 
 def readBlackListPackages(pkgBlackListFile):
     blackListPkgs = []
     if pkgBlackListFile is not None and pkgBlackListFile != "":
         with open(pkgBlackListFile) as jsonFile:
             config = json.load(jsonFile)
             if 'packages' in config:
                 blackListPkgs = config['packages']
     return blackListPkgs
 
 
 def buildSourcesList(yamlDir, blackListPkgs, logger, singleFile=True):
     cmdUtils = CommandUtils()
     yamlSourceDir = os.path.join(yamlDir, "yaml_sources")
     if not os.path.isdir(yamlSourceDir):
         cmdUtils.runCommandInShell("mkdir -p " + yamlSourceDir)
     if singleFile:
         yamlFile = open(yamlSourceDir + "/sources_list.yaml", "w")
     listPackages = SPECS.getData().getListPackages()
     listPackages.sort()
     import PullSources
     for package in listPackages:
         if package in blackListPkgs:
             continue
         ossname = package
         ossversion = SPECS.getData().getVersion(package)
         modified = False
         listPatches = SPECS.getData().getPatches(package)
326d5ca8
         if listPatches:
accc8120
             modified = True
         url = SPECS.getData().getSourceURL(package)
         if url is None:
             url = SPECS.getData().getURL(package)
 
         sourceName = None
         listSourceNames = SPECS.getData().getSources(package)
326d5ca8
         if listSourceNames:
accc8120
             sourceName = listSourceNames[0]
             sha1 = SPECS.getData().getSHA1(package, sourceName)
             if sha1 is not None:
a00911af
                 PullSources.get(package, sourceName, sha1, yamlSourceDir,
accc8120
                                 constants.pullsourcesConfig, logger)
 
         if not singleFile:
             yamlFile = open(yamlSourceDir + "/" + ossname + "-" + ossversion + ".yaml", "w")
         yamlFile.write("vmwsource:" + ossname + ":" + ossversion + ":\n")
         yamlFile.write("  repository: VMWsource\n")
         yamlFile.write("  name: '" + ossname + "'\n")
         yamlFile.write("  version: '" + ossversion + "'\n")
         yamlFile.write("  url: " + str(url) + "\n")
         yamlFile.write("  license: UNKNOWN\n")
         if sourceName is not None:
             yamlFile.write("  vmwsource-distribution: " + str(sourceName) + "\n")
         if modified:
             yamlFile.write("  modified: true\n")
         yamlFile.write("\n")
         if not singleFile:
             yamlFile.close()
 
     if singleFile:
         yamlFile.close()
     logger.info("Generated source yaml files for all packages")
 
 
 def buildSRPMList(srpmPath, yamlDir, blackListPkgs, logger, singleFile=True):
     cmdUtils = CommandUtils()
     yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms")
     if not os.path.isdir(yamlSrpmDir):
         cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir)
     if singleFile:
         yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w")
     listPackages = SPECS.getData().getListPackages()
     listPackages.sort()
     for package in listPackages:
         if package in blackListPkgs:
             continue
         ossname = package
         ossversion = SPECS.getData().getVersion(package)
         ossrelease = SPECS.getData().getRelease(package)
 
326d5ca8
         listFoundSRPMFiles = cmdUtils.findFile(ossname + "-" + ossversion + "-" + ossrelease
                                                + ".src.rpm",
accc8120
                                                srpmPath)
         srpmName = None
         if len(listFoundSRPMFiles) == 1:
             srpmFullPath = listFoundSRPMFiles[0]
             srpmName = os.path.basename(srpmFullPath)
             cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/"
             returnVal = cmdUtils.runCommandInShell(cpcmd)
             if not returnVal:
                 logger.error("Copy SRPM File is failed for package:" + ossname)
         else:
             logger.error("SRPM file is not found:" + ossname)
 
         if not singleFile:
326d5ca8
             yamlFile = open(yamlSrpmDir + "/" + ossname + "-" + ossversion + "-"
                             + ossrelease + ".yaml", "w")
accc8120
 
         yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" + ossrelease + ":\n")
         yamlFile.write("  repository: BaseOS\n")
         yamlFile.write("  name: '" + ossname + "'\n")
         yamlFile.write("  version: '" + ossversion + "-" + ossrelease + "'\n")
e0724b04
         yamlFile.write("  url: 'http://www.vmware.com'\n")
accc8120
         yamlFile.write("  baseos-style: rpm\n")
         yamlFile.write("  baseos-source: '" + str(srpmName) + "'\n")
         yamlFile.write("  baseos-osname: 'photon'\n")
         yamlFile.write("\n")
         if not singleFile:
             yamlFile.close()
 
     if singleFile:
         yamlFile.close()
     logger.info("Generated SRPM yaml files for all packages")
 
 
 if __name__ == "__main__":
     main()