Change-Id: I5f23f4d002f02d4c7fcec7afee7da90dca179594
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/2941
Tested-by: gerrit-photon <photon-checkins@vmware.com>
Reviewed-by: Anish Swaminathan <anishs@vmware.com>
... | ... |
@@ -60,7 +60,7 @@ IMGCONVERTER := $(TOOLS_BIN)/imgconverter |
60 | 60 |
|
61 | 61 |
.PHONY : all iso clean photon-build-machine photon-vagrant-build photon-vagrant-local cloud-image \ |
62 | 62 |
check check-docker check-bison check-g++ check-gawk check-createrepo check-kpartx check-vagrant check-packer check-packer-ovf-plugin check-sanity \ |
63 |
-clean-install clean-chroot build-updated-packages |
|
63 |
+clean-install clean-chroot build-updated-packages generate-yaml-files |
|
64 | 64 |
|
65 | 65 |
THREADS?=1 |
66 | 66 |
|
... | ... |
@@ -540,6 +540,27 @@ check-packer-ovf-plugin: |
540 | 540 |
$(PHOTON_RPMCHECK_OPTION) \ |
541 | 541 |
-l $(PHOTON_LOGS_DIR) |
542 | 542 |
|
543 |
+generate-yaml-files: check $(PHOTON_STAGE) $(PHOTON_PACKAGES) |
|
544 |
+ @echo "Generating yaml files for packages ..." |
|
545 |
+ @cd $(PHOTON_PKG_BUILDER_DIR) && \ |
|
546 |
+ $(PHOTON_PACKAGE_BUILDER) -y \ |
|
547 |
+ -b $(PHOTON_CHROOT_PATH) \ |
|
548 |
+ -s $(PHOTON_SPECS_DIR) \ |
|
549 |
+ -r $(PHOTON_RPMS_DIR) \ |
|
550 |
+ -a $(PHOTON_SRPMS_DIR) \ |
|
551 |
+ -x $(PHOTON_SRCS_DIR) \ |
|
552 |
+ -p $(PHOTON_PUBLISH_RPMS_DIR) \ |
|
553 |
+ -e $(PHOTON_PUBLISH_XRPMS_DIR) \ |
|
554 |
+ -c $(PHOTON_PULLSOURCES_CONFIG) \ |
|
555 |
+ -d $(PHOTON_DIST_TAG) \ |
|
556 |
+ -n $(PHOTON_BUILD_NUMBER) \ |
|
557 |
+ -v $(PHOTON_RELEASE_VERSION) \ |
|
558 |
+ -g $(PHOTON_DATA_DIR)/pkg_build_options.json \ |
|
559 |
+ $(PHOTON_RPMCHECK_OPTION) \ |
|
560 |
+ -l $(PHOTON_LOGS_DIR) \ |
|
561 |
+ -j $(PHOTON_STAGE) \ |
|
562 |
+ -f $(PHOTON_PKG_BLACKLIST_FILE) |
|
563 |
+ |
|
543 | 564 |
$(TOOLS_BIN): |
544 | 565 |
mkdir -p $(TOOLS_BIN) |
545 | 566 |
|
... | ... |
@@ -37,6 +37,9 @@ def main(): |
37 | 37 |
parser.add_option("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS") |
38 | 38 |
parser.add_option("-w", "--pkginfo-file", dest="pkgInfoFile", default="../../stage/pkg_info.json") |
39 | 39 |
parser.add_option("-g", "--pkg-build-option-file", dest="pkgBuildOptionFile", default="../../common/data/pkg_build_options.json") |
40 |
+ parser.add_option("-y", "--generate-pkg-yaml-files", dest="generatePkgYamlFiles", default=False, action ="store_true") |
|
41 |
+ parser.add_option("-j", "--pkg-yaml-dir-path", dest="pkgYamlDirPath", default="../../stage/") |
|
42 |
+ parser.add_option("-f", "--pkg-blacklist-file", dest="pkgBlacklistFile", default=None) |
|
40 | 43 |
|
41 | 44 |
(options, args) = parser.parse_args() |
42 | 45 |
cmdUtils=CommandUtils() |
... | ... |
@@ -76,6 +79,11 @@ def main(): |
76 | 76 |
logger.error("Given input RPMS Path is not a directory:"+options.inputRPMSPath) |
77 | 77 |
errorFlag = True |
78 | 78 |
|
79 |
+ if options.generatePkgYamlFiles: |
|
80 |
+ if options.pkgBlacklistFile is not None and options.pkgBlacklistFile != "" and not os.path.isfile(options.pkgBlacklistFile): |
|
81 |
+ logger.error("Given package blacklist file is not valid:"+options.pkgBlacklistFile) |
|
82 |
+ errorFlag = True |
|
83 |
+ |
|
79 | 84 |
if options.installPackage : |
80 | 85 |
if len(args) != 1: |
81 | 86 |
logger.error("Please provide package name") |
... | ... |
@@ -98,6 +106,10 @@ def main(): |
98 | 98 |
if not os.path.isdir(options.buildRootPath): |
99 | 99 |
cmdUtils.runCommandInShell("mkdir -p "+options.buildRootPath) |
100 | 100 |
|
101 |
+ if options.generatePkgYamlFiles: |
|
102 |
+ if not os.path.isdir(options.pkgYamlDirPath): |
|
103 |
+ cmdUtils.runCommandInShell("mkdir -p "+options.pkgYamlDirPath) |
|
104 |
+ |
|
101 | 105 |
logger.info("Source Path :"+options.sourcePath) |
102 | 106 |
logger.info("Spec Path :" + options.specPath) |
103 | 107 |
logger.info("Rpm Path :" + options.rpmPath) |
... | ... |
@@ -114,14 +126,10 @@ def main(): |
114 | 114 |
constants.initialize(options) |
115 | 115 |
if package == "packages_list": |
116 | 116 |
buildPackagesList(options.buildRootPath+"/../packages_list.csv") |
117 |
- elif package == "sources_list": |
|
118 |
- if not os.path.isdir("../../stage/yaml_sources"): |
|
119 |
- cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_sources") |
|
120 |
- buildSourcesList('../../stage/yaml_sources',logger) |
|
121 |
- elif package == "srpms_list": |
|
122 |
- if not os.path.isdir("../../stage/yaml_srpms"): |
|
123 |
- cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_srpms") |
|
124 |
- buildSRPMList(options.sourceRpmPath,"../../stage/yaml_srpms",logger) |
|
117 |
+ elif options.generatePkgYamlFiles: |
|
118 |
+ blackListPkgs = readBlackListPackages(options.pkgBlacklistFile) |
|
119 |
+ buildSourcesList(options.pkgYamlDirPath, blackListPkgs, logger) |
|
120 |
+ buildSRPMList(options.sourceRpmPath, options.pkgYamlDirPath, blackListPkgs, logger) |
|
125 | 121 |
elif options.toolChainStage == "stage1": |
126 | 122 |
pkgManager = PackageManager() |
127 | 123 |
pkgManager.buildToolChain() |
... | ... |
@@ -166,14 +174,28 @@ def buildPackagesList(csvFilename): |
166 | 166 |
csvFile.write(name+","+version+","+license+","+url+","+sources+","+patches+"\n") |
167 | 167 |
csvFile.close() |
168 | 168 |
|
169 |
-def buildSourcesList(yamlDir, logger, singleFile=True): |
|
170 |
- strUtils = StringUtils() |
|
169 |
+def readBlackListPackages(pkgBlackListFile): |
|
170 |
+ blackListPkgs = [] |
|
171 |
+ if pkgBlackListFile is not None and pkgBlackListFile != "": |
|
172 |
+ with open(pkgBlackListFile) as jsonFile: |
|
173 |
+ config = json.load(jsonFile) |
|
174 |
+ if 'packages' in config: |
|
175 |
+ blackListPkgs = config['packages'] |
|
176 |
+ return blackListPkgs |
|
177 |
+ |
|
178 |
+def buildSourcesList(yamlDir, blackListPkgs, logger, singleFile=True): |
|
179 |
+ cmdUtils = CommandUtils() |
|
180 |
+ yamlSourceDir = os.path.join(yamlDir, "yaml_sources") |
|
181 |
+ if not os.path.isdir(yamlSourceDir): |
|
182 |
+ cmdUtils.runCommandInShell("mkdir -p "+yamlSourceDir) |
|
171 | 183 |
if singleFile: |
172 |
- yamlFile = open(yamlDir+"/sources_list.yaml", "w") |
|
184 |
+ yamlFile = open(yamlSourceDir+"/sources_list.yaml", "w") |
|
173 | 185 |
listPackages = constants.specData.getListPackages() |
174 | 186 |
listPackages.sort() |
175 | 187 |
import PullSources |
176 | 188 |
for package in listPackages: |
189 |
+ if package in blackListPkgs: |
|
190 |
+ continue |
|
177 | 191 |
ossname = package |
178 | 192 |
ossversion = constants.specData.getVersion(package) |
179 | 193 |
modified = False |
... | ... |
@@ -193,7 +215,7 @@ def buildSourcesList(yamlDir, logger, singleFile=True): |
193 | 193 |
PullSources.get(sourceName, sha1, yamlDir, constants.pullsourcesConfig, logger) |
194 | 194 |
|
195 | 195 |
if not singleFile: |
196 |
- yamlFile = open(yamlDir+"/"+ossname+"-"+ossversion+".yaml", "w") |
|
196 |
+ yamlFile = open(yamlSourceDir+"/"+ossname+"-"+ossversion+".yaml", "w") |
|
197 | 197 |
yamlFile.write("vmwsource:"+ossname+":"+ossversion+":\n") |
198 | 198 |
yamlFile.write(" repository: VMWsource\n") |
199 | 199 |
yamlFile.write(" name: '"+ossname+"'\n") |
... | ... |
@@ -212,14 +234,18 @@ def buildSourcesList(yamlDir, logger, singleFile=True): |
212 | 212 |
yamlFile.close() |
213 | 213 |
logger.info("Generated source yaml files for all packages") |
214 | 214 |
|
215 |
-def buildSRPMList(srpmPath, yamlDir, logger, singleFile=True): |
|
216 |
- strUtils = StringUtils() |
|
215 |
+def buildSRPMList(srpmPath, yamlDir, blackListPkgs, logger, singleFile=True): |
|
216 |
+ cmdUtils = CommandUtils() |
|
217 |
+ yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms") |
|
218 |
+ if not os.path.isdir(yamlSrpmDir): |
|
219 |
+ cmdUtils.runCommandInShell("mkdir -p "+yamlSrpmDir) |
|
217 | 220 |
if singleFile: |
218 |
- yamlFile = open(yamlDir+"/srpm_list.yaml", "w") |
|
221 |
+ yamlFile = open(yamlSrpmDir+"/srpm_list.yaml", "w") |
|
219 | 222 |
listPackages = constants.specData.getListPackages() |
220 | 223 |
listPackages.sort() |
221 |
- cmdUtils = CommandUtils() |
|
222 | 224 |
for package in listPackages: |
225 |
+ if package in blackListPkgs: |
|
226 |
+ continue |
|
223 | 227 |
ossname = package |
224 | 228 |
ossversion = constants.specData.getVersion(package) |
225 | 229 |
ossrelease = constants.specData.getRelease(package) |
... | ... |
@@ -237,7 +263,7 @@ def buildSRPMList(srpmPath, yamlDir, logger, singleFile=True): |
237 | 237 |
logger.error("SRPM file is not found:" +ossname) |
238 | 238 |
|
239 | 239 |
if not singleFile: |
240 |
- yamlFile = open(yamlDir+"/"+ossname+"-"+ossversion+"-"+ossrelease+".yaml", "w") |
|
240 |
+ yamlFile = open(yamlSrpmDir+"/"+ossname+"-"+ossversion+"-"+ossrelease+".yaml", "w") |
|
241 | 241 |
|
242 | 242 |
yamlFile.write("baseos:"+ossname+":"+ossversion+"-"+ossrelease+":\n") |
243 | 243 |
yamlFile.write(" repository: BaseOS\n") |