#!/usr/bin/env python from optparse import OptionParser import os.path from CommandUtils import CommandUtils from Logger import Logger from constants import constants from PackageManager import PackageManager import json import sys from SpecUtils import Specutils from StringUtils import StringUtils import collections import traceback from PackageInfo import SourcePackageInfo def main(): usage = "Usage: %prog [options] <package name>" parser = OptionParser(usage) parser.add_option("-s", "--spec-path", dest="specPath", default="../../SPECS") parser.add_option("-x", "--source-path", dest="sourcePath", default="../../stage/SOURCES") parser.add_option("-r", "--rpm-path", dest="rpmPath", default="../../stage/RPMS") parser.add_option("-i", "--install-package", dest="installPackage", default=False, action ="store_true") parser.add_option("-p", "--publish-RPMS-path", dest="publishRPMSPath", default="../../stage/PUBLISHRPMS") parser.add_option("-l", "--log-path", dest="logPath", default="../../stage/LOGS") parser.add_option("-o", "--build-option", dest="buildOption", default="full") parser.add_option("-z", "--top-dir-path", dest="topDirPath", default="/usr/src/photon") parser.add_option("-j", "--json-file", dest="inputJSONFile", default="../../common/data/build_install_options_all.json") parser.add_option("-b", "--build-root-path", dest="buildRootPath", default="/mnt") parser.add_option("-t", "--threads", dest="buildThreads", default=1, type="int", help="Number of working threads") parser.add_option("-m", "--tool-chain-stage", dest="toolChainStage", default="None") parser.add_option("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf") parser.add_option("-d", "--dist", dest="dist", default="") parser.add_option("-k", "--input-RPMS-path", dest="inputRPMSPath", default=None) parser.add_option("-n", "--build-number", dest="buildNumber", default="0000000") parser.add_option("-v", "--release-version", dest="releaseVersion", default="NNNnNNN") parser.add_option("-u", "--enable-rpmcheck", dest="rpmCheck", default=False, action ="store_true") parser.add_option("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS") parser.add_option("-w", "--pkginfo-file", dest="pkgInfoFile", default="../../common/data/pkg_info.json") parser.add_option("-g", "--pkg-build-option-file", dest="pkgBuildOptionFile", default="../../common/data/pkg_build_options.json") (options, args) = parser.parse_args() cmdUtils=CommandUtils() if not os.path.isdir(options.logPath): cmdUtils.runCommandInShell("mkdir -p "+options.logPath) logger=Logger.getLogger(options.logPath+"/Main") errorFlag=False package = None pkgInfoJsonFile=options.pkgInfoFile if not os.path.isdir(options.sourcePath): logger.error("Given Sources Path is not a directory:"+options.sourcePath) errorFlag = True if not os.path.isdir(options.specPath): logger.error("Given Specs Path is not a directory:"+options.specPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath): logger.error("Given RPMS Path is not a directory:"+options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath+"/x86_64"): logger.error("Given RPMS Path is missing x86_64 sub-directory:"+options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath+"/noarch"): logger.error("Given RPMS Path is missing noarch sub-directory:"+options.publishRPMSPath) errorFlag = True if not os.path.isfile(options.inputJSONFile) and not options.installPackage: logger.error("Given JSON File is not a file:"+options.inputJSONFile) errorFlag = True if not os.path.isfile(options.pkgBuildOptionFile): logger.warning("Given JSON File is not a file:"+options.pkgBuildOptionFile) if options.inputRPMSPath is not None and not os.path.isdir(options.inputRPMSPath): logger.error("Given input RPMS Path is not a directory:"+options.publishRPMSPath) errorFlag = True if options.installPackage : if len(args) != 1: logger.error("Please provide package name") errorFlag = True else: package=args[0] if errorFlag: logger.error("Found some errors. Please fix input options and re-run it.") return False if not os.path.isdir(options.rpmPath): cmdUtils.runCommandInShell("mkdir -p "+options.rpmPath+"/x86_64") cmdUtils.runCommandInShell("mkdir -p "+options.rpmPath+"/noarch") if not os.path.isdir(options.sourceRpmPath): cmdUtils.runCommandInShell("mkdir -p "+options.sourceRpmPath) if not os.path.isdir(options.buildRootPath): cmdUtils.runCommandInShell("mkdir -p "+options.buildRootPath) logger.info("Source Path :"+options.sourcePath) logger.info("Spec Path :" + options.specPath) logger.info("Rpm Path :" + options.rpmPath) logger.info("Log Path :" + options.logPath) logger.info("Top Dir Path :" + options.topDirPath) logger.info("Publish RPMS Path :" + options.publishRPMSPath) if not options.installPackage: logger.info("JSON File :" + options.inputJSONFile) else: logger.info("Package to build:"+package) listBuildOptionPackages = get_packages_with_build_options(options.pkgBuildOptionFile) try: constants.initialize(options) SourcePackageInfo.setLogging() SourcePackageInfo.loadPkgInfoFromFile(pkgInfoJsonFile) if package == "packages_list": buildPackagesList(options.specPath, options.buildRootPath+"/../packages_list.csv") elif package == "sources_list": if not os.path.isdir("../../stage/yaml_sources"): cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_sources") buildSourcesList(options.specPath,"../../stage/yaml_sources", options.buildRootPath+"/../",logger) elif package == "srpms_list": if not os.path.isdir("../../stage/yaml_srpms"): cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_srpms") buildSRPMList(options.specPath,options.sourceRpmPath,"../../stage/yaml_srpms",logger) elif options.toolChainStage == "stage1": pkgManager = PackageManager() pkgManager.buildToolChain() elif options.toolChainStage == "stage2": pkgManager = PackageManager() pkgManager.buildToolChainPackages(options.buildThreads) elif options.installPackage: buildAPackage(package, listBuildOptionPackages, options.pkgBuildOptionFile, options.buildThreads) else: buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption, listBuildOptionPackages, options.pkgBuildOptionFile, logger, options.buildThreads) except Exception as e: logger.error("Caught an exception") logger.error(str(e)) # print stacktrace traceback.print_exc() sys.exit(1) logger.info("Writing Package info to the file:"+pkgInfoJsonFile) SourcePackageInfo.writePkgListToFile(pkgInfoJsonFile) sys.exit(0) def buildToolChain(buildThreads): pkgManager = PackageManager() pkgManager.buildToolChainPackages(buildThreads) def buildPackagesList(specPath, csvFilename): csvFile = open(csvFilename, "w") csvFile.write("Package,Version,License,URL,Sources,Patches\n") lst = os.listdir(specPath) lst.sort() for dirEntry in lst: specDir = os.path.join(specPath, dirEntry) if os.path.isdir(specDir): for specEntry in os.listdir(specDir): specFile = os.path.join(specDir, specEntry) if os.path.isfile(specFile) and specFile.endswith(".spec"): spec=Specutils(specFile) name=spec.getBasePackageName() version=spec.getRPMVersion(name) license=spec.getLicense(name) url=spec.getURL(name) ss=spec.getSourceURLs() sources="" for s in ss: if (s.startswith("http") or s.startswith("ftp")): if sources != "": sources += " " sources += s patches="" ps=spec.getPatchNames() for p in ps: if patches != "": patches += " " patches += p csvFile.write(name+","+version+","+license+","+url+","+sources+","+patches+"\n") csvFile.close() def buildSourcesList(specPath,sourcePath, yamlDir, logger, singleFile=False): strUtils = StringUtils() if singleFile: yamlFile = open(yamlDir+"sources_list.yaml", "w") lst = os.listdir(specPath) lst.sort() import PullSources for dirEntry in lst: specDir = os.path.join(specPath, dirEntry) if os.path.isdir(specDir): for specEntry in os.listdir(specDir): specFile = os.path.join(specDir, specEntry) if os.path.isfile(specFile) and specFile.endswith(".spec"): spec=Specutils(specFile) modified = len(spec.getPatchNames()) > 0 listSourceURLs=spec.getSourceURLs() ossname = spec.getBasePackageName() ossversion = spec.getVersion() url = None listSourceNames = spec.getSourceNames() sourceName = None if len(listSourceNames) >0: sourceName=listSourceNames[0] sha1 = spec.getChecksumForSource(sourceName) if sha1 is not None: PullSources.get(sourceName, sha1, sourcePath, constants.pullsourcesConfig) if len(listSourceURLs) > 0: sourceURL = listSourceURLs[0] if sourceURL.startswith("http") or sourceURL.startswith("ftp"): url = sourceURL; else: url=spec.getURL(ossname) if not singleFile: yamlFile = open(yamlDir+"/"+ossname+"-"+ossversion+".yaml", "w") yamlFile.write("vmwsource:"+ossname+":"+ossversion+":\n") yamlFile.write(" repository: VMWsource\n") yamlFile.write(" name: '"+ossname+"'\n") yamlFile.write(" version: '"+ossversion+"'\n") yamlFile.write(" url: "+str(url)+"\n") yamlFile.write(" license: UNKNOWN\n") if sourceName is not None: yamlFile.write(" vmwsource-distribution: "+str(sourceName)+"\n") if modified: yamlFile.write(" modified: true\n") yamlFile.write("\n") if not singleFile: yamlFile.close() if singleFile: yamlFile.close() logger.info("Generated source yaml files for all packages") def buildSRPMList(specPath,srpmPath, yamlDir, logger, singleFile=False): strUtils = StringUtils() if singleFile: yamlFile = open(yamlDir+"srpm_list.yaml", "w") lst = os.listdir(specPath) lst.sort() cmdUtils = CommandUtils() for dirEntry in lst: specDir = os.path.join(specPath, dirEntry) if os.path.isdir(specDir): for specEntry in os.listdir(specDir): specFile = os.path.join(specDir, specEntry) if os.path.isfile(specFile) and specFile.endswith(".spec"): spec=Specutils(specFile) ossname = spec.getBasePackageName() ossversion = spec.getVersion() ossrelease = spec.getRelease() listFoundSRPMFiles = cmdUtils.findFile(ossname+"-"+ossversion+"-"+ossrelease+".src.rpm",srpmPath) srpmName = None if len(listFoundSRPMFiles) == 1: srpmFullPath = listFoundSRPMFiles[0]; srpmName = os.path.basename(srpmFullPath) cpcmd = "cp "+ srpmFullPath +" "+yamlDir+"/" returnVal = cmdUtils.runCommandInShell(cpcmd) if not returnVal: logger.error("Copy SRPM File is failed for package:"+ossname) else: logger.error("SRPM file is not found:" +ossname) if not singleFile: yamlFile = open(yamlDir+"/"+ossname+"-"+ossversion+"-"+ossrelease+".yaml", "w") yamlFile.write("baseos:"+ossname+":"+ossversion+"-"+ossrelease+":\n") yamlFile.write(" repository: BaseOS\n") yamlFile.write(" name: '"+ossname+"'\n") yamlFile.write(" version: '"+ossversion+"-"+ossrelease+"'\n") yamlFile.write(" baseos-style: rpm\n") yamlFile.write(" baseos-source: '"+str(srpmName)+"'\n") yamlFile.write(" baseos-osname: 'Photon OS'\n") yamlFile.write("\n") if not singleFile: yamlFile.close() if singleFile: yamlFile.close() logger.info("Generated srpm yaml files for all packages") def buildAPackage(package, listBuildOptionPackages, pkgBuildOptionFile, buildThreads): listPackages=[] listPackages.append(package) pkgManager = PackageManager() pkgManager.buildPackages(listPackages, listBuildOptionPackages, pkgBuildOptionFile, buildThreads) def buildPackagesFromGivenJSONFile(inputJSONFile, buildOption, listBuildOptionPackages, pkgBuildOptionFile, logger, buildThreads): listPackages = get_all_package_names(inputJSONFile) listPackagesToBuild=[] for pkg in listPackages: p = pkg.encode('utf-8') listPackagesToBuild.append(str(p)) logger.info("List of packages to build:") logger.info(listPackagesToBuild) pkgManager = PackageManager() pkgManager.buildPackages(listPackagesToBuild, listBuildOptionPackages, pkgBuildOptionFile, buildThreads) def get_packages_with_build_options(pkg_build_options_file): packages = [] if os.path.exists(pkg_build_options_file): jsonData = open(pkg_build_options_file) pkg_build_option_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict) jsonData.close() pkgs_sorted = pkg_build_option_json.items() for pkg in pkgs_sorted: p = pkg[0].encode('utf-8') packages.append(str(p)) return packages def get_all_package_names(build_install_option): base_path = os.path.dirname(build_install_option) jsonData = open(build_install_option) option_list_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict) jsonData.close() options_sorted = option_list_json.items() packages = [] for install_option in options_sorted: filename = os.path.join(base_path, install_option[1]["file"]) jsonData=open(filename) package_list_json = json.load(jsonData) jsonData.close() packages = packages + package_list_json["packages"] return packages if __name__=="__main__": main()