| 3ad60002 |
#!/usr/bin/env python
|
| 2820c61a |
from optparse import OptionParser
import os.path
from CommandUtils import CommandUtils
from Logger import Logger
from constants import constants
from PackageManager import PackageManager |
| 07f00ab3 |
import json |
| 8846e378 |
import sys |
| 920a9773 |
from SpecUtils import Specutils |
| 4184f836 |
from StringUtils import StringUtils |
| dc0dfc30 |
import collections |
| d355ea7e |
import traceback |
| 2cfb758d |
from PackageInfo import SourcePackageInfo |
| b650a058 |
|
| 2820c61a |
def main():
usage = "Usage: %prog [options] <package name>"
parser = OptionParser(usage) |
| 8a84f80c |
parser.add_option("-s", "--spec-path", dest="specPath", default="../../SPECS")
parser.add_option("-x", "--source-path", dest="sourcePath", default="../../stage/SOURCES")
parser.add_option("-r", "--rpm-path", dest="rpmPath", default="../../stage/RPMS") |
| 2820c61a |
parser.add_option("-i", "--install-package", dest="installPackage", default=False, action ="store_true") |
| 8a84f80c |
parser.add_option("-p", "--publish-RPMS-path", dest="publishRPMSPath", default="../../stage/PUBLISHRPMS")
parser.add_option("-l", "--log-path", dest="logPath", default="../../stage/LOGS") |
| 07f00ab3 |
parser.add_option("-o", "--build-option", dest="buildOption", default="full") |
| 2820c61a |
parser.add_option("-z", "--top-dir-path", dest="topDirPath", default="/usr/src/photon") |
| 8a84f80c |
parser.add_option("-j", "--json-file", dest="inputJSONFile", default="../../common/data/build_install_options_all.json") |
| d23d369f |
parser.add_option("-b", "--build-root-path", dest="buildRootPath", default="/mnt") |
| 90d8acae |
parser.add_option("-t", "--threads", dest="buildThreads", default=1, type="int", help="Number of working threads") |
| db36e75e |
parser.add_option("-m", "--tool-chain-stage", dest="toolChainStage", default="None") |
| 3cc43c92 |
parser.add_option("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf") |
| e9fc4bb0 |
parser.add_option("-d", "--dist", dest="dist", default="") |
| 42ffccb5 |
parser.add_option("-k", "--input-RPMS-path", dest="inputRPMSPath", default=None) |
| 2c153d29 |
parser.add_option("-n", "--build-number", dest="buildNumber", default="0000000")
parser.add_option("-v", "--release-version", dest="releaseVersion", default="NNNnNNN") |
| bc90d582 |
parser.add_option("-u", "--enable-rpmcheck", dest="rpmCheck", default=False, action ="store_true") |
| d2526915 |
parser.add_option("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS") |
| 2cfb758d |
parser.add_option("-w", "--pkginfo-file", dest="pkgInfoFile", default="../../common/data/pkg_info.json") |
| 90d8acae |
parser.add_option("-g", "--pkg-build-option-file", dest="pkgBuildOptionFile", default="../../common/data/pkg_build_options.json") |
| b5e09fac |
parser.add_option("-q", "--rpmcheck-stop-on-error", dest="rpmCheckStopOnError", default=False, action ="store_true") |
| 8a84f80c |
|
| 2820c61a |
(options, args) = parser.parse_args() |
| 44991444 |
cmdUtils=CommandUtils()
if not os.path.isdir(options.logPath):
cmdUtils.runCommandInShell("mkdir -p "+options.logPath) |
| 2cfb758d |
|
| 07f00ab3 |
logger=Logger.getLogger(options.logPath+"/Main")
errorFlag=False |
| b650a058 |
package = None |
| 2cfb758d |
pkgInfoJsonFile=options.pkgInfoFile |
| 07f00ab3 |
if not os.path.isdir(options.sourcePath):
logger.error("Given Sources Path is not a directory:"+options.sourcePath)
errorFlag = True
if not os.path.isdir(options.specPath):
logger.error("Given Specs Path is not a directory:"+options.specPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath):
logger.error("Given RPMS Path is not a directory:"+options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath+"/x86_64"):
logger.error("Given RPMS Path is missing x86_64 sub-directory:"+options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath+"/noarch"):
logger.error("Given RPMS Path is missing noarch sub-directory:"+options.publishRPMSPath)
errorFlag = True
if not os.path.isfile(options.inputJSONFile) and not options.installPackage:
logger.error("Given JSON File is not a file:"+options.inputJSONFile) |
| 90d8acae |
errorFlag = True
if not os.path.isfile(options.pkgBuildOptionFile):
logger.warning("Given JSON File is not a file:"+options.pkgBuildOptionFile) |
| 1fd3eff3 |
|
| 42ffccb5 |
if options.inputRPMSPath is not None and not os.path.isdir(options.inputRPMSPath):
logger.error("Given input RPMS Path is not a directory:"+options.publishRPMSPath)
errorFlag = True
|
| 1fd3eff3 |
if options.installPackage :
if len(args) != 1:
logger.error("Please provide package name")
errorFlag = True
else:
package=args[0] |
| 07f00ab3 |
if errorFlag:
logger.error("Found some errors. Please fix input options and re-run it.")
return False |
| 2820c61a |
if not os.path.isdir(options.rpmPath): |
| 518d6a6f |
cmdUtils.runCommandInShell("mkdir -p "+options.rpmPath+"/x86_64")
cmdUtils.runCommandInShell("mkdir -p "+options.rpmPath+"/noarch") |
| d2526915 |
if not os.path.isdir(options.sourceRpmPath):
cmdUtils.runCommandInShell("mkdir -p "+options.sourceRpmPath) |
| 2820c61a |
|
| d23d369f |
if not os.path.isdir(options.buildRootPath):
cmdUtils.runCommandInShell("mkdir -p "+options.buildRootPath) |
| 07f00ab3 |
|
| 2820c61a |
logger.info("Source Path :"+options.sourcePath)
logger.info("Spec Path :" + options.specPath)
logger.info("Rpm Path :" + options.rpmPath)
logger.info("Log Path :" + options.logPath)
logger.info("Top Dir Path :" + options.topDirPath) |
| 07f00ab3 |
logger.info("Publish RPMS Path :" + options.publishRPMSPath) |
| b5e09fac |
|
| 07f00ab3 |
if not options.installPackage:
logger.info("JSON File :" + options.inputJSONFile)
else:
logger.info("Package to build:"+package) |
| 90d8acae |
listBuildOptionPackages = get_packages_with_build_options(options.pkgBuildOptionFile) |
| 8bcb2fdd |
|
| 07f00ab3 |
try:
constants.initialize(options) |
| 2cfb758d |
SourcePackageInfo.setLogging()
SourcePackageInfo.loadPkgInfoFromFile(pkgInfoJsonFile) |
| 4184f836 |
if package == "packages_list":
buildPackagesList(options.specPath, options.buildRootPath+"/../packages_list.csv")
elif package == "sources_list": |
| 7253ce7d |
if not os.path.isdir("../../stage/yaml_sources"):
cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_sources")
buildSourcesList(options.specPath,"../../stage/yaml_sources", options.buildRootPath+"/../",logger)
elif package == "srpms_list":
if not os.path.isdir("../../stage/yaml_srpms"):
cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_srpms")
buildSRPMList(options.specPath,options.sourceRpmPath,"../../stage/yaml_srpms",logger) |
| 920a9773 |
elif options.toolChainStage == "stage1": |
| db36e75e |
pkgManager = PackageManager()
pkgManager.buildToolChain()
elif options.toolChainStage == "stage2":
pkgManager = PackageManager() |
| 6a4bde38 |
pkgManager.buildToolChainPackages(options.buildThreads) |
| db36e75e |
elif options.installPackage: |
| 90d8acae |
buildAPackage(package, listBuildOptionPackages, options.pkgBuildOptionFile, options.buildThreads) |
| 07f00ab3 |
else: |
| 90d8acae |
buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption, listBuildOptionPackages, options.pkgBuildOptionFile, logger, options.buildThreads) |
| 07f00ab3 |
except Exception as e:
logger.error("Caught an exception")
logger.error(str(e)) |
| d355ea7e |
# print stacktrace
traceback.print_exc() |
| 8846e378 |
sys.exit(1) |
| 2cfb758d |
logger.info("Writing Package info to the file:"+pkgInfoJsonFile)
SourcePackageInfo.writePkgListToFile(pkgInfoJsonFile) |
| 8846e378 |
sys.exit(0) |
| 07f00ab3 |
|
| 6a4bde38 |
def buildToolChain(buildThreads): |
| db36e75e |
pkgManager = PackageManager() |
| af3575c9 |
pkgManager.buildToolChainPackages(buildThreads) |
| db36e75e |
|
| 4184f836 |
def buildPackagesList(specPath, csvFilename):
csvFile = open(csvFilename, "w")
csvFile.write("Package,Version,License,URL,Sources,Patches\n") |
| 920a9773 |
lst = os.listdir(specPath)
lst.sort()
for dirEntry in lst:
specDir = os.path.join(specPath, dirEntry)
if os.path.isdir(specDir):
for specEntry in os.listdir(specDir):
specFile = os.path.join(specDir, specEntry)
if os.path.isfile(specFile) and specFile.endswith(".spec"):
spec=Specutils(specFile) |
| 166760fe |
name=spec.getBasePackageName() |
| 920a9773 |
version=spec.getRPMVersion(name)
license=spec.getLicense(name)
url=spec.getURL(name) |
| 166760fe |
ss=spec.getSourceURLs()
sources=""
for s in ss:
if (s.startswith("http") or s.startswith("ftp")):
if sources != "":
sources += " "
sources += s |
| 920a9773 |
patches=""
ps=spec.getPatchNames()
for p in ps:
if patches != "":
patches += " "
patches += p |
| 4184f836 |
csvFile.write(name+","+version+","+license+","+url+","+sources+","+patches+"\n")
csvFile.close()
|
| 7253ce7d |
def buildSourcesList(specPath,sourcePath, yamlDir, logger, singleFile=False): |
| 4184f836 |
strUtils = StringUtils()
if singleFile:
yamlFile = open(yamlDir+"sources_list.yaml", "w")
lst = os.listdir(specPath)
lst.sort() |
| 7253ce7d |
import PullSources |
| 4184f836 |
for dirEntry in lst:
specDir = os.path.join(specPath, dirEntry)
if os.path.isdir(specDir):
for specEntry in os.listdir(specDir):
specFile = os.path.join(specDir, specEntry)
if os.path.isfile(specFile) and specFile.endswith(".spec"):
spec=Specutils(specFile)
modified = len(spec.getPatchNames()) > 0 |
| 259d59d0 |
listSourceURLs=spec.getSourceURLs()
ossname = spec.getBasePackageName()
ossversion = spec.getVersion()
url = None |
| 7253ce7d |
listSourceNames = spec.getSourceNames()
sourceName = None
if len(listSourceNames) >0:
sourceName=listSourceNames[0]
sha1 = spec.getChecksumForSource(sourceName)
if sha1 is not None:
PullSources.get(sourceName, sha1, sourcePath, constants.pullsourcesConfig)
|
| 259d59d0 |
if len(listSourceURLs) > 0:
sourceURL = listSourceURLs[0]
if sourceURL.startswith("http") or sourceURL.startswith("ftp"):
url = sourceURL;
else:
url=spec.getURL(ossname)
if not singleFile: |
| 7253ce7d |
yamlFile = open(yamlDir+"/"+ossname+"-"+ossversion+".yaml", "w") |
| 259d59d0 |
yamlFile.write("vmwsource:"+ossname+":"+ossversion+":\n")
yamlFile.write(" repository: VMWsource\n")
yamlFile.write(" name: '"+ossname+"'\n")
yamlFile.write(" version: '"+ossversion+"'\n")
yamlFile.write(" url: "+str(url)+"\n")
yamlFile.write(" license: UNKNOWN\n") |
| 7253ce7d |
if sourceName is not None:
yamlFile.write(" vmwsource-distribution: "+str(sourceName)+"\n") |
| 259d59d0 |
if modified:
yamlFile.write(" modified: true\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close() |
| 4184f836 |
if singleFile:
yamlFile.close() |
| 7253ce7d |
logger.info("Generated source yaml files for all packages")
def buildSRPMList(specPath,srpmPath, yamlDir, logger, singleFile=False):
strUtils = StringUtils()
if singleFile:
yamlFile = open(yamlDir+"srpm_list.yaml", "w")
lst = os.listdir(specPath)
lst.sort()
cmdUtils = CommandUtils()
for dirEntry in lst:
specDir = os.path.join(specPath, dirEntry)
if os.path.isdir(specDir):
for specEntry in os.listdir(specDir):
specFile = os.path.join(specDir, specEntry)
if os.path.isfile(specFile) and specFile.endswith(".spec"):
spec=Specutils(specFile)
ossname = spec.getBasePackageName()
ossversion = spec.getVersion()
ossrelease = spec.getRelease()
listFoundSRPMFiles = cmdUtils.findFile(ossname+"-"+ossversion+"-"+ossrelease+".src.rpm",srpmPath)
srpmName = None
if len(listFoundSRPMFiles) == 1:
srpmFullPath = listFoundSRPMFiles[0];
srpmName = os.path.basename(srpmFullPath)
cpcmd = "cp "+ srpmFullPath +" "+yamlDir+"/"
returnVal = cmdUtils.runCommandInShell(cpcmd)
if not returnVal:
logger.error("Copy SRPM File is failed for package:"+ossname)
else:
logger.error("SRPM file is not found:" +ossname)
if not singleFile:
yamlFile = open(yamlDir+"/"+ossname+"-"+ossversion+"-"+ossrelease+".yaml", "w")
yamlFile.write("baseos:"+ossname+":"+ossversion+"-"+ossrelease+":\n")
yamlFile.write(" repository: BaseOS\n")
yamlFile.write(" name: '"+ossname+"'\n")
yamlFile.write(" version: '"+ossversion+"-"+ossrelease+"'\n")
yamlFile.write(" baseos-style: rpm\n")
yamlFile.write(" baseos-source: '"+str(srpmName)+"'\n")
yamlFile.write(" baseos-osname: 'Photon OS'\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.info("Generated srpm yaml files for all packages") |
| 920a9773 |
|
| 90d8acae |
def buildAPackage(package, listBuildOptionPackages, pkgBuildOptionFile, buildThreads): |
| 07f00ab3 |
listPackages=[]
listPackages.append(package)
pkgManager = PackageManager() |
| b5e09fac |
if constants.rpmCheck:
constants.setTestForceRPMS(listPackages[:]) |
| 90d8acae |
pkgManager.buildPackages(listPackages, listBuildOptionPackages, pkgBuildOptionFile, buildThreads) |
| 07f00ab3 |
|
| 90d8acae |
def buildPackagesFromGivenJSONFile(inputJSONFile, buildOption, listBuildOptionPackages, pkgBuildOptionFile, logger, buildThreads): |
| 5d05cfcc |
listPackages = get_all_package_names(inputJSONFile)
|
| b650a058 |
listPackagesToBuild=[]
for pkg in listPackages:
p = pkg.encode('utf-8')
listPackagesToBuild.append(str(p)) |
| 07f00ab3 |
logger.info("List of packages to build:") |
| b650a058 |
logger.info(listPackagesToBuild) |
| b5e09fac |
if constants.rpmCheck:
constants.setTestForceRPMS(listPackagesToBuild[:]) |
| 2820c61a |
pkgManager = PackageManager() |
| 90d8acae |
pkgManager.buildPackages(listPackagesToBuild, listBuildOptionPackages, pkgBuildOptionFile, buildThreads)
def get_packages_with_build_options(pkg_build_options_file):
packages = []
if os.path.exists(pkg_build_options_file):
jsonData = open(pkg_build_options_file)
pkg_build_option_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict)
jsonData.close()
pkgs_sorted = pkg_build_option_json.items()
for pkg in pkgs_sorted:
p = pkg[0].encode('utf-8')
packages.append(str(p))
return packages |
| a5e4be9f |
|
| 5d05cfcc |
def get_all_package_names(build_install_option):
base_path = os.path.dirname(build_install_option)
jsonData = open(build_install_option) |
| dc0dfc30 |
option_list_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict) |
| 5d05cfcc |
jsonData.close() |
| dc0dfc30 |
options_sorted = option_list_json.items() |
| 5d05cfcc |
packages = []
for install_option in options_sorted:
filename = os.path.join(base_path, install_option[1]["file"])
jsonData=open(filename)
package_list_json = json.load(jsonData)
jsonData.close()
packages = packages + package_list_json["packages"]
return packages
|
| 2820c61a |
if __name__=="__main__": |
| 2fcd8ac0 |
main() |