@@ -404,16 +404,16 @@ $(PHOTON_STAGE):

 generate-dep-lists:

 	$(RMDIR) $(PHOTON_GENERATED_DATA_DIR)

 	$(MKDIR) -p $(PHOTON_GENERATED_DATA_DIR)

-	@for f in $$(ls $(PHOTON_DATA_DIR)/build_install_options*.json) ; \

-	do \

-		cp $$f $(PHOTON_GENERATED_DATA_DIR); \

-		echo "Generating the install time dependency list for " $$f; \

-		cd $(PHOTON_SPECDEPS_DIR) && \

-		$(PHOTON_SPECDEPS) \

+	cd $(PHOTON_SPECDEPS_DIR) && \

+	$(PHOTON_SPECDEPS) \

 		-s $(PHOTON_SPECS_DIR) \

 		-t $(PHOTON_STAGE) \

-		--input-type=json --file $$f -d json -a $(PHOTON_DATA_DIR); \

-	done

+		-l $(PHOTON_LOGS_DIR) \

+		-p $(PHOTON_GENERATED_DATA_DIR) \

+		--input-type=json \

+		--file "$$(ls $(PHOTON_DATA_DIR)/build_install_options*.json)" \

+		-d json \

+		-a $(PHOTON_DATA_DIR)

 photon-docker-image:

 	createrepo $(PHOTON_RPMS_DIR)

@@ -617,22 +617,11 @@ check: packages

 generate-yaml-files: check-tools $(PHOTON_STAGE) $(PHOTON_PACKAGES)

 	@echo "Generating yaml files for packages ..."

 	@cd $(PHOTON_PKG_BUILDER_DIR) && \

-        $(PHOTON_PACKAGE_BUILDER) -y \

-                              -b $(PHOTON_CHROOT_PATH) \

+        $(PHOTON_GENERATE_OSS_FILES) -y \

                               -s $(PHOTON_SPECS_DIR) \

-                              -r $(PHOTON_RPMS_DIR) \

                               -a $(PHOTON_SRPMS_DIR) \

-                              -x $(PHOTON_SRCS_DIR) \

-                              -p $(PHOTON_PUBLISH_RPMS_DIR) \

-                              -e $(PHOTON_PUBLISH_XRPMS_DIR) \

-                              -c $(PHOTON_PULLSOURCES_CONFIG) \

-                              -d $(PHOTON_DIST_TAG) \

-                              -n $(PHOTON_BUILD_NUMBER) \

-                              -v $(PHOTON_RELEASE_VERSION) \

-                              -g $(PHOTON_DATA_DIR)/pkg_build_options.json \

-                              $(PHOTON_RPMCHECK_OPTION) \

                               -l $(PHOTON_LOGS_DIR) \

-                              -j $(PHOTON_STAGE) \

+                              -c $(PHOTON_PULLSOURCES_CONFIG) \

                               -f $(PHOTON_PKG_BLACKLIST_FILE)

 $(TOOLS_BIN):

@@ -125,7 +125,7 @@

         "pycurl3",

         "python3-yum-metadata-parser",

         "strace",

-        "cracklib-python3",

+        "python3-cracklib",

         "haveged",

         "vim-extra",

         "postgresql",

@@ -42,6 +42,7 @@ PHOTON_INSTALLER=$(PHOTON_INSTALLER_DIR)/photonInstaller.py

 PHOTON_SPECDEPS_DIR=$(SRCROOT)/support/package-builder

 PHOTON_SPECDEPS=$(PHOTON_SPECDEPS_DIR)/SpecDeps.py

 PHOTON_PACKAGE_BUILDER=$(PHOTON_PKG_BUILDER_DIR)/builder.py

+PHOTON_GENERATE_OSS_FILES=$(PHOTON_PKG_BUILDER_DIR)/GenerateOSSFiles.py

 ifdef PHOTON_PULLSOURCES_CONFIG

 PHOTON_PULLSOURCES_CONFIG:=$(abspath $(PHOTON_PULLSOURCES_CONFIG))

 else

new file mode 100755

@@ -0,0 +1,221 @@

+#!/usr/bin/env python3

+

+import os

+import json

+import sys

+import traceback

+from Logger import Logger

+from argparse import ArgumentParser

+from constants import constants

+from CommandUtils import CommandUtils

+from SpecData import SPECS

+

+

+

+def main():

+    usage = "Usage: %prog [options] <package name>"

+    parser = ArgumentParser(usage)

+    parser.add_argument("-s", "--spec-path", dest="specPath", default="../../SPECS")

+    parser.add_argument("-l", "--log-path", dest="logPath", default="../../stage/LOGS")

+    parser.add_argument("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS")

+    parser.add_argument("-j", "--output-dir", dest="outputDirPath", default="../../stage/")

+    parser.add_argument("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf")

+    parser.add_argument("-f", "--pkg-blacklist-file", dest="pkgBlacklistFile", default=None)

+    parser.add_argument("-p", "--generate-pkg-list", dest="generatePkgList", default=False, action="store_true")

+    parser.add_argument("-y", "--generate-yaml-files", dest="generateYamlFiles", default=False, action="store_true")

+

+    options = parser.parse_args()

+    errorFlag = False

+    cmdUtils = CommandUtils()

+

+    try:

+        if not os.path.isdir(options.logPath):

+            cmdUtils.runCommandInShell("mkdir -p " + options.logPath)

+        logger = Logger.getLogger(options.logPath + "/generateYamlFiles")

+

+        if options.generateYamlFiles:

+            if (options.pkgBlacklistFile is not None and

+                    options.pkgBlacklistFile != "" and

+                    not os.path.isfile(options.pkgBlacklistFile)):

+                logger.error("Given package blacklist file is not valid:" + options.pkgBlacklistFile)

+                errorFlag = True

+

+        if not os.path.isdir(options.specPath):

+            logger.error("Given Specs Path is not a directory:" + options.specPath)

+            errorFlag = True

+

+        if not os.path.isdir(options.sourceRpmPath):

+            logger.error("Given SRPM Path is not a directory:" + options.sourceRpmPath)

+            errorFlag = True

+

+        if options.generateYamlFiles and not os.path.isfile(options.pullsourcesConfig):

+            logger.error("Given Source config file is not a valid file:" + options.pullsourcesConfig)

+            errorFlag = True

+

+        if errorFlag:

+            logger.error("Found some errors. Please fix input options and re-run it.")

+            sys.exit(1)

+

+        if options.generateYamlFiles:

+            if not os.path.isdir(options.outputDirPath):

+                cmdUtils.runCommandInShell("mkdir -p "+options.outputDirPath)

+

+        constants.setSpecPath(options.specPath)

+        constants.setSourceRpmPath(options.sourceRpmPath)

+        constants.setLogPath(options.logPath)

+        constants.setPullSourcesConfig(options.pullsourcesConfig)

+        constants.initialize()

+

+        # parse SPECS folder

+        SPECS()

+

+        if options.generatePkgList:

+            buildPackagesList(options.outputDirPath + "/packages_list.csv")

+        elif options.generateYamlFiles:

+            blackListPkgs = readBlackListPackages(options.pkgBlacklistFile)

+            buildSourcesList(options.outputDirPath, blackListPkgs, logger)

+            buildSRPMList(options.sourceRpmPath, options.outputDirPath, blackListPkgs, logger)

+

+    except Exception as e:

+        print("Caught Exception: "+str(e))

+        traceback.print_exc()

+        sys.exit(1)

+

+    sys.exit(0)

+

+

+def buildPackagesList(csvFilename):

+    csvFile = open(csvFilename, "w")

+    csvFile.write("Package,Version,License,URL,Sources,Patches\n")

+    listPackages = SPECS.getData().getListPackages()

+    listPackages.sort()

+    for package in listPackages:

+        name = package

+        version = SPECS.getData().getVersion(package)

+        license = SPECS.getData().getLicense(package)

+        listPatches = SPECS.getData().getPatches(package)

+        url = SPECS.getData().getURL(package)

+        listSourceNames = SPECS.getData().getSources(package)

+        sources = ""

+        patches = ""

+        if listPatches is not None:

+            patches = " ".join(listPatches)

+        if listSourceNames is not None:

+            sources = " ".join(listSourceNames)

+        csvFile.write(name + "," + version + "," + license + "," + url + "," + sources + "," + patches + "\n")

+    csvFile.close()

+

+

+def readBlackListPackages(pkgBlackListFile):

+    blackListPkgs = []

+    if pkgBlackListFile is not None and pkgBlackListFile != "":

+        with open(pkgBlackListFile) as jsonFile:

+            config = json.load(jsonFile)

+            if 'packages' in config:

+                blackListPkgs = config['packages']

+    return blackListPkgs

+

+

+def buildSourcesList(yamlDir, blackListPkgs, logger, singleFile=True):

+    cmdUtils = CommandUtils()

+    yamlSourceDir = os.path.join(yamlDir, "yaml_sources")

+    if not os.path.isdir(yamlSourceDir):

+        cmdUtils.runCommandInShell("mkdir -p " + yamlSourceDir)

+    if singleFile:

+        yamlFile = open(yamlSourceDir + "/sources_list.yaml", "w")

+    listPackages = SPECS.getData().getListPackages()

+    listPackages.sort()

+    import PullSources

+    for package in listPackages:

+        if package in blackListPkgs:

+            continue

+        ossname = package

+        ossversion = SPECS.getData().getVersion(package)

+        modified = False

+        listPatches = SPECS.getData().getPatches(package)

+        if listPatches is not None and len(listPatches) > 0:

+            modified = True

+        url = SPECS.getData().getSourceURL(package)

+        if url is None:

+            url = SPECS.getData().getURL(package)

+

+        sourceName = None

+        listSourceNames = SPECS.getData().getSources(package)

+        if len(listSourceNames) > 0:

+            sourceName = listSourceNames[0]

+            sha1 = SPECS.getData().getSHA1(package, sourceName)

+            if sha1 is not None:

+                PullSources.get(sourceName, sha1, yamlSourceDir,

+                                constants.pullsourcesConfig, logger)

+

+        if not singleFile:

+            yamlFile = open(yamlSourceDir + "/" + ossname + "-" + ossversion + ".yaml", "w")

+        yamlFile.write("vmwsource:" + ossname + ":" + ossversion + ":\n")

+        yamlFile.write("  repository: VMWsource\n")

+        yamlFile.write("  name: '" + ossname + "'\n")

+        yamlFile.write("  version: '" + ossversion + "'\n")

+        yamlFile.write("  url: " + str(url) + "\n")

+        yamlFile.write("  license: UNKNOWN\n")

+        if sourceName is not None:

+            yamlFile.write("  vmwsource-distribution: " + str(sourceName) + "\n")

+        if modified:

+            yamlFile.write("  modified: true\n")

+        yamlFile.write("\n")

+        if not singleFile:

+            yamlFile.close()

+

+    if singleFile:

+        yamlFile.close()

+    logger.info("Generated source yaml files for all packages")

+

+

+def buildSRPMList(srpmPath, yamlDir, blackListPkgs, logger, singleFile=True):

+    cmdUtils = CommandUtils()

+    yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms")

+    if not os.path.isdir(yamlSrpmDir):

+        cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir)

+    if singleFile:

+        yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w")

+    listPackages = SPECS.getData().getListPackages()

+    listPackages.sort()

+    for package in listPackages:

+        if package in blackListPkgs:

+            continue

+        ossname = package

+        ossversion = SPECS.getData().getVersion(package)

+        ossrelease = SPECS.getData().getRelease(package)

+

+        listFoundSRPMFiles = cmdUtils.findFile(ossname + "-" + ossversion + "-" + ossrelease + ".src.rpm",

+                                               srpmPath)

+        srpmName = None

+        if len(listFoundSRPMFiles) == 1:

+            srpmFullPath = listFoundSRPMFiles[0]

+            srpmName = os.path.basename(srpmFullPath)

+            cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/"

+            returnVal = cmdUtils.runCommandInShell(cpcmd)

+            if not returnVal:

+                logger.error("Copy SRPM File is failed for package:" + ossname)

+        else:

+            logger.error("SRPM file is not found:" + ossname)

+

+        if not singleFile:

+            yamlFile = open(yamlSrpmDir + "/" + ossname + "-" + ossversion + "-" + ossrelease + ".yaml", "w")

+

+        yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" + ossrelease + ":\n")

+        yamlFile.write("  repository: BaseOS\n")

+        yamlFile.write("  name: '" + ossname + "'\n")

+        yamlFile.write("  version: '" + ossversion + "-" + ossrelease + "'\n")

+        yamlFile.write("  baseos-style: rpm\n")

+        yamlFile.write("  baseos-source: '" + str(srpmName) + "'\n")

+        yamlFile.write("  baseos-osname: 'photon'\n")

+        yamlFile.write("\n")

+        if not singleFile:

+            yamlFile.close()

+

+    if singleFile:

+        yamlFile.close()

+    logger.info("Generated SRPM yaml files for all packages")

+

+

+if __name__ == "__main__":

+    main()

deleted file mode 100644

@@ -1,16 +0,0 @@

-import commands

-

-class RepoQueryDependency(object):

-    def __init__(self, repoFile):

-        self.repo_file = repoFile

-    def getRequiresList(self,pkg):

-        cmd = ("repoquery -c " + self.repo_file + " -R -q " + pkg +

-               " | xargs repoquery -c " + self.repo_file +

-               " --whatprovides -q | sed 's/-[0-9]/ /g' | cut -f 1 -d ' ' | sort | uniq ")

-        status,output = commands.getstatusoutput(cmd)

-        if status == 0:

-            outList = output.split('\n')

-            if "" in outList: outList.remove("")

-            if "Options:" in outList: outList.remove("Options:")

-            if "Usage:" in outList: outList.remove("Usage:")

-            return outList

@@ -8,6 +8,8 @@ from SpecUtils import Specutils

 from Logger import Logger

 from constants import constants

+

+

 class SerializableSpecObject(object):

     def __init__(self):

         self.listPackages = []

@@ -29,6 +31,7 @@ class SerializableSpecObject(object):

         self.license = ""

         self.specDefs = {}

+

 class SerializableSpecObjectsUtils(object):

     def __init__(self, logPath):

@@ -71,7 +74,7 @@ class SerializableSpecObjectsUtils(object):

                 self.mapPackageToSpec[specPkg] = specName

                 if spec.getIsRPMPackage(specPkg):

                     specObj.listRPMPackages.append(specPkg)

-            if skipUpdating == False:

+            if not skipUpdating:

                 self.mapSerializableSpecObjects[specName] = specObj

     def getListSpecFiles(self, listSpecFiles, path):

@@ -135,10 +138,11 @@ class SerializableSpecObjectsUtils(object):

         specName = self.getSpecName(package)

         return self.mapSerializableSpecObjects[specName].listRPMPackages

-    def getReleaseNum(self, releaseVal):

-        id = releaseVal.find("%")

-        if id != -1:

-            return releaseVal[0:id]

+    @staticmethod

+    def getReleaseNum(releaseVal):

+        releaseNum = releaseVal.find("%")

+        if releaseNum != -1:

+            return releaseVal[0:releaseNum]

         else:

             return releaseVal

@@ -223,6 +227,7 @@ class SerializableSpecObjectsUtils(object):

             self.logger.info("security_hardening: " + specObj.securityHardening)

             self.logger.info("------------------------------------------------")

+

 class SPECS(object):

     __instance = None

     specData = None

@@ -244,7 +249,7 @@ class SPECS(object):

     def initialize(self):

         # Preparse some files

-        #adding openjre8 version rpm macro

+        # adding openjre8 version rpm macro

         if platform.machine() == "x86_64":

             spec = Specutils(constants.specPath + "/openjdk8/openjdk8.spec")

         else:

@@ -252,16 +257,16 @@ class SPECS(object):

         java8version = spec.getVersion()

         constants.addMacro("JAVA8_VERSION", java8version)

-        #adding kernelversion rpm macro

+        # adding kernelversion rpm macro

         spec = Specutils(constants.specPath + "/linux/linux.spec")

         kernelversion = spec.getVersion()

         constants.addMacro("KERNEL_VERSION", kernelversion)

-        #adding kernelrelease rpm macro

+        # adding kernelrelease rpm macro

         kernelrelease = spec.getRelease()

         constants.addMacro("KERNEL_RELEASE", kernelrelease)

-        #adding kernelsubrelease rpm macro

+        # adding kernelsubrelease rpm macro

         a, b, c = kernelversion.split(".")

         kernelsubrelease = ('%02d%02d%03d%03d' % (int(a),

                                                   int(b), int(c),

@@ -274,177 +279,79 @@ class SPECS(object):

         self.specData = SerializableSpecObjectsUtils(constants.logPath)

         self.specData.readSpecsAndConvertToSerializableObjects(constants.specPath)

-# Little bit of duplication

-# Used by SpecVerify and SpecDeps only

-class SerializedSpecObjects(object):

-    def __init__(self, inputDataDir, stageDir):

-        self.mapSerializableSpecObjects = {}

-        self.mapPackageToSpec = {}

-        self.jsonFilesOutPath = stageDir + "/common/data/"

-        self.inputDataDir = inputDataDir

+class SpecDependencyGenerator(object):

-    def findTotalRequires(self, allDeps, depQue, parent, displayOption):

+    def findTotalRequires(self, mapDependencies, depQue, parent):

         while not depQue.empty():

             specPkg = depQue.get()

-            specName = self.getSpecName(specPkg)

-            if specName is None:

+            try:

+                listRequiredPackages = SPECS.getData().getRequiresForPackage(specPkg)

+            except Exception as e:

+                print("Caught Exception:"+str(e))

                 print(specPkg + " is missing")

-            specObj = self.mapSerializableSpecObjects[specName]

-            for depPkg in specObj.installRequiresPackages[specPkg]:

-                if depPkg in allDeps:

-                    if allDeps[depPkg] < allDeps[specPkg] + 1:

-                        allDeps[depPkg] = allDeps[specPkg] + 1

+

+            for depPkg in listRequiredPackages:

+                if depPkg in mapDependencies:

+                    if mapDependencies[depPkg] < mapDependencies[specPkg] + 1:

+                        mapDependencies[depPkg] = mapDependencies[specPkg] + 1

                         parent[depPkg] = specPkg

-                        self.updateLevels(allDeps, depPkg, parent, allDeps[depPkg])

+                        self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])

                 else:

-                    allDeps[depPkg] = allDeps[specPkg] + 1

+                    mapDependencies[depPkg] = mapDependencies[specPkg] + 1

                     parent[depPkg] = specPkg

                     depQue.put(depPkg)

-    def findTotalWhoNeedsToBuild(self, depQue, whoBuildDeps, whoBuildDepSet, displayOption):

+    def findTotalWhoNeedsToBuild(self, depQue, whoNeedsBuild):

         while not depQue.empty():

             specPkg = depQue.get()

-            specName = self.getSpecName(specPkg)

-            spec = Specutils(self.getSpecFile(specPkg))

-            RPMName = spec.getRPMName(specPkg)

-            debuginfoRPMName = spec.getDebuginfoRPMName(specPkg)

-            whoBuildDepSet.add(RPMName)

-            whoBuildDepSet.add(debuginfoRPMName)

-            if specName is None:

-                print(specPkg + " is missing")

-            if specPkg not in whoBuildDeps:

-                continue

-            for depPkg in whoBuildDeps[specPkg]:

-                depQue.put(depPkg)

-

-    def printTree(self, allDeps, children, curParent, depth):

+            listPackagesRequiredToBuild = SPECS.getData().getBuildRequiresForPackage(specPkg)

+            for depPkg in listPackagesRequiredToBuild:

+                depSpecPkg = SPECS.getData().getSpecName(depPkg)

+                if depSpecPkg not in whoNeedsBuild:

+                    whoNeedsBuild.append(depSpecPkg)

+                    depQue.put(depSpecPkg)

+

+    def printTree(self, children, curParent, depth):

         if curParent in children:

             for child in children[curParent]:

-                print("\t" * depth + child)

-                self.printTree(allDeps, children, child, depth + 1)

+                print ("\t" * depth + child)

+                self.printTree(children, child, depth + 1)

-    def get_all_package_names(self, jsonFilePath):

-        base_path = os.path.dirname(jsonFilePath)

+    def getAllPackageNames(self, jsonFilePath):

         jsonData = open(jsonFilePath)

         option_list_json = json.load(jsonData)

         jsonData.close()

         packages = option_list_json["packages"]

         return packages

-    def updateLevels(self, allDeps, inPkg, parent, level):

-        specName = self.getSpecName(inPkg)

-        specObj = self.mapSerializableSpecObjects[specName]

-        for depPkg in specObj.installRequiresPackages[inPkg]:

-            # ignore circular deps within single spec file

-            if (depPkg in specObj.installRequiresPackages and

-                    inPkg in specObj.installRequiresPackages[depPkg] and

-                    self.getSpecName(depPkg) == specName):

+    def updateLevels(self, mapDependencies, inPkg, parent, level):

+        listPackages = SPECS.getData().getPackages(inPkg)

+        for depPkg in SPECS.getData().getRequiresForPackage(inPkg):

+            if depPkg in listPackages:

                 continue

-            if depPkg in allDeps and allDeps[depPkg] < level + 1:

-                allDeps[depPkg] = level + 1

+            if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:

+                mapDependencies[depPkg] = level + 1

                 parent[depPkg] = inPkg

-                self.updateLevels(allDeps, depPkg, parent, allDeps[depPkg])

+                self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])

-    def readSpecsAndConvertToSerializableObjects(self, specFilesPath, inputType,

-                                                 inputValue, displayOption):

-        children = {}

-        listSpecFiles = []

-        whoNeedsList = []

-        whoBuildDepSet = set()

-        # list of all RPMS not built from photon and that must be blindly copied.

-        independentRPMS = []

-        whoBuildDeps = {}

-        allDeps = {}

-        parent = {}

+    def calculateSpecDependency(self, inputPackages, mapDependencies, parent):

         depQue = queue.Queue()

-        packageFound = False

-        self.getListSpecFiles(listSpecFiles, specFilesPath)

-        for specFile in listSpecFiles:

-            spec = Specutils(specFile)

-            specName = spec.getBasePackageName()

-            specObj = SerializableSpecObject()

-            specObj.name = specName

-            specObj.buildRequirePackages = spec.getBuildRequiresAllPackages()

-            specObj.installRequiresAllPackages = spec.getRequiresAllPackages()

-            specObj.listPackages = spec.getPackageNames()

-            specObj.specFile = specFile

-            specObj.version = spec.getVersion()

-            specObj.release = spec.getRelease()

-            specObj.listSources = spec.getSourceNames()

-            specObj.listPatches = spec.getPatchNames()

-            specObj.securityHardening = spec.getSecurityHardeningOption()

-            for specPkg in specObj.listPackages:

-                specObj.installRequiresPackages[specPkg] = spec.getRequires(specPkg)

-                if inputType == "pkg" and inputValue == specPkg:

-                # all the first level dependencies to a dictionary and queue

-                    packageFound = True

-                    for depPkg in specObj.installRequiresPackages[specPkg]:

-                        if depPkg not in allDeps:

-                            allDeps[depPkg] = 0

-                            parent[depPkg] = ""

-                            depQue.put(depPkg)

-                elif (inputType == "who-needs" and

-                      inputValue in specObj.installRequiresPackages[specPkg]):

-                    whoNeedsList.append(specPkg)

-                elif inputType == "who-needs-build":

-                    for bdrq in specObj.buildRequirePackages:

-                        if bdrq in whoBuildDeps:

-                            whoBuildDeps[bdrq].add(specPkg)

-                        else:

-                            whoBuildDeps[bdrq] = set()

-                            whoBuildDeps[bdrq].add(specPkg)

-                    if inputValue == specPkg:

-                        packageFound = True

-                        for depPkg in specObj.listPackages:

-                            depQue.put(depPkg)

-

-                self.mapPackageToSpec[specPkg] = specName

-            self.mapSerializableSpecObjects[specName] = specObj

-

-        # Generate dependencies for individual packages

-        if inputType == "pkg":

-            if packageFound == True:

-                self.findTotalRequires(allDeps, depQue, parent, displayOption)

+        for pkg in inputPackages:

+            if SPECS.getData().isRPMPackage(pkg):

+                if pkg not in mapDependencies:

+                    mapDependencies[pkg] = 0

+                    parent[pkg] = ""

+                    depQue.put(pkg)

+                    self.findTotalRequires(mapDependencies, depQue, parent)

             else:

-                print("No spec file builds a package named {}".format(inputValue))

-                return

-

-        # Generate dependencies for all packages in the given JSON input file

-        elif inputType == "json":

-            filePath = self.inputDataDir + "/" + inputValue

-            data = self.get_all_package_names(filePath)

-            for pkg in data:

-                if pkg not in allDeps:

-                    spName = self.getSpecName(pkg)

-                    if spName is not None:

-                        allDeps[pkg] = 0

-                        parent[pkg] = ""

-                        depQue.put(pkg)

-                        self.findTotalRequires(allDeps, depQue, parent, displayOption)

-                    else:

-                        independentRPMS.append(pkg)

-

-        #Generating the list of packages that requires the given input package at install time

-        elif inputType == "who-needs":

-            print(whoNeedsList)

-            return

+                print("Could not find spec for "+pkg)

-        #Generating the list of packages that the modified package will affect at build time

-        elif inputType == "who-needs-build":

-            if packageFound == True:

-                self.findTotalWhoNeedsToBuild(depQue, whoBuildDeps, whoBuildDepSet, displayOption)

-                print(whoBuildDepSet)

-            else:

-                print("No spec file builds a package named {}".format(inputValue))

-            return

-

-        # construct the sorted list of all packages (sorted by dependency)

+    def displayDependencies(self, displayOption, inputType, inputValue, allDeps, parent):

+        children = {}

         sortedList = []

         for elem in sorted(allDeps.items(), key=operator.itemgetter(1), reverse=True):

             sortedList.append(elem[0])

-        sortedList.extend(independentRPMS)

-

         # construct all children nodes

         if displayOption == "tree":

             for k, v in parent.iteritems():

@@ -456,95 +363,52 @@ class SerializedSpecObjects(object):

             if "" in children:

                 for child in children[""]:

                     print(child)

-                    self.printTree(allDeps, children, child, 1)

-                for pkg in independentRPMS:

-                    print(pkg)

+                    self.printTree(children, child, 1)

                 print("*" * 18 + " {} ".format(len(sortedList)) +

                       "packages in total " + "*" * 18)

             else:

                 if inputType == "pkg" and len(children) > 0:

-                    print("cyclic dependency detected, mappings:")

-                    print(children)

+                    print ("cyclic dependency detected, mappings: \n", children)

         # To display a flat list of all packages

         elif displayOption == "list":

-            print(sortedList)

+            print (sortedList)

         # To generate a new JSON file based on given input json file

         elif displayOption == "json" and inputType == "json":

-            d = {}

-            d['packages'] = sortedList

-            outFilePath = self.jsonFilesOutPath + inputValue

-            with open(outFilePath, 'w') as outfile:

+            d = {'packages': sortedList}

+            with open(inputValue, 'w') as outfile:

                 json.dump(d, outfile)

-        return sortedList

-    def getListSpecFiles(self, listSpecFiles, path):

-        for dirEntry in os.listdir(path):

-            dirEntryPath = os.path.join(path, dirEntry)

-            if (os.path.isfile(dirEntryPath) and

-                    dirEntryPath.endswith(".spec") and

-                    os.path.basename(dirEntryPath) not in

-                    constants.skipSpecsForArch.get(platform.machine(), [])):

-                listSpecFiles.append(dirEntryPath)

-            elif os.path.isdir(dirEntryPath):

-                self.getListSpecFiles(listSpecFiles, dirEntryPath)

-

-    def getBuildRequiresForPackage(self, package):

-        specName = self.getSpecName(package)

-        return self.mapSerializableSpecObjects[specName].buildRequirePackages

-

-    def getRequiresForPackage(self, package):

-        specName = self.getSpecName(package)

-        if package in self.mapSerializableSpecObjects[specName].installRequiresPackages:

-            return self.mapSerializableSpecObjects[specName].installRequiresPackages[package]

-        return None

-

-    def getRelease(self, package):

-        specName = self.getSpecName(package)

-        return self.mapSerializableSpecObjects[specName].release

-

-    def getVersion(self, package):

-        specName = self.getSpecName(package)

-        return self.mapSerializableSpecObjects[specName].version

-

-    def getSpecFile(self, package):

-        specName = self.getSpecName(package)

-        return self.mapSerializableSpecObjects[specName].specFile

-

-    def getPatches(self, package):

-        specName = self.getSpecName(package)

-        return self.mapSerializableSpecObjects[specName].listPatches

-

-    def getSources(self, package):

-        specName = self.getSpecName(package)

-        return self.mapSerializableSpecObjects[specName].listSources

-

-    def getPackages(self, package):

-        specName = self.getSpecName(package)

-        return self.mapSerializableSpecObjects[specName].listPackages

+        return sortedList

-    def getSpecName(self, package):

-        if package in self.mapPackageToSpec:

-            specName = self.mapPackageToSpec[package]

-            if specName in self.mapSerializableSpecObjects:

-                return specName

+    def process(self, inputType, inputValue, displayOption, outputFile=None):

+        whoNeedsList = []

+        inputPackages = []

+        whoNeedsBuild = []

+        mapDependencies = {}

+        parent = {}

+        if inputType == "pkg" or inputType == "json":

+            if inputType == "pkg":

+                inputPackages.append(inputValue)

             else:

-                print("SpecDeps: Could not able to find " + package + " package from specs")

-                raise Exception("Invalid package:" + package)

-        else:

-            return None

+                inputPackages = self.getAllPackageNames(inputValue)

-    def isRPMPackage(self, package):

-        if package in self.mapPackageToSpec:

-            specName = self.mapPackageToSpec[package]

-        if specName in self.mapSerializableSpecObjects:

-            return True

-        return False

-

-    def getSecurityHardeningOption(self, package):

-        specName = self.getSpecName(package)

-        return self.mapSerializableSpecObjects[specName].securityHardening

-

-    def getSpecDetails(self, name):

-        print(self.mapSerializableSpecObjects[name].installRequiresAllPackages)

+            self.calculateSpecDependency(inputPackages, mapDependencies, parent)

+            if outputFile is not None:

+                return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent)

+            else:

+                return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent)

+        elif inputType == "who-needs":

+            for depPkg in SPECS.getData().mapPackageToSpec:

+                if inputValue in SPECS.getData().getRequiresForPackage(depPkg):

+                    whoNeedsList.append(depPkg)

+            print (whoNeedsList)

+            return whoNeedsList

+        elif inputType == "who-needs-build":

+            depQue = queue.Queue()

+            depQue.put(inputValue)

+            self.findTotalWhoNeedsToBuild(depQue, whoNeedsBuild)

+            print ("Following specs need to be build again")

+            print (whoNeedsBuild)

+            return whoNeedsBuild

@@ -6,14 +6,19 @@

 import sys

 import os

 from argparse import ArgumentParser

-from SpecData import SerializedSpecObjects

+import shutil

+import traceback

+from SpecData import SpecDependencyGenerator

 from jsonwrapper import JsonWrapper

+from constants import constants

+from CommandUtils import CommandUtils

 DEFAULT_INPUT_TYPE = "pkg"

 DEFAULT_DISPLAY_OPTION = "tree"

 SPEC_FILE_DIR = "../../SPECS"

 LOG_FILE_DIR = "../../stage/LOGS"

+

 def main():

     usage = "Usage: %prog [options]"

     parser = ArgumentParser(usage)

@@ -22,46 +27,58 @@ def main():

     parser.add_argument("-f", "--file", dest="json_file", default="packages_minimal.json")

     parser.add_argument("-d", "--disp", dest="display_option", default=DEFAULT_DISPLAY_OPTION)

     parser.add_argument("-s", "--spec-dir", dest="spec_dir", default=SPEC_FILE_DIR)

+    parser.add_argument("-l", "--log-dir", dest="log_dir", default=LOG_FILE_DIR)

     parser.add_argument("-t", "--stage-dir", dest="stage_dir", default="../../stage")

-    parser.add_argument("-a", "--input-data-dir", dest="input_data_dir",

-                        default="../../common/data/")

+    parser.add_argument("-a", "--input-data-dir", dest="input_data_dir", default="../../common/data/")

+    parser.add_argument("-o", "--output-dir", dest="output_dir", default="../../stage/common/data")

     options = parser.parse_args()

+    constants.setSpecPath(options.spec_dir)

+    constants.setLogPath(options.log_dir)

+    constants.initialize()

+

+    cmdUtils = CommandUtils()

+

+    if not os.path.isdir(options.output_dir):

+        cmdUtils.runCommandInShell2("mkdir -p "+options.output_dir)

+

     if not options.input_data_dir.endswith('/'):

         options.input_data_dir += '/'

     try:

-        specDeps = SerializedSpecObjects(options.input_data_dir, options.stage_dir)

-        displayOption = options.display_option

-        abs_path = os.path.abspath(__file__)

-        dir_name = os.path.dirname(abs_path)

-        os.chdir(dir_name)

+        specDeps = SpecDependencyGenerator()

         # To display/print package dependencies on console

-        if(options.input_type == "pkg" or

-           options.input_type == "who-needs" or

-           options.input_type == "who-needs-build"):

-            targetName = options.pkg

-            specDeps.readSpecsAndConvertToSerializableObjects(options.spec_dir,

-                                                              options.input_type,

-                                                              targetName, displayOption)

+        if (options.input_type == "pkg" or

+                options.input_type == "who-needs" or

+                options.input_type == "who-needs-build"):

+            specDeps.process(options.input_type, options.pkg, options.display_option)

+

         elif options.input_type == "json":

-        # Generate the expanded package dependencies json file based on package_list_file

-            json_wrapper_option_list = JsonWrapper(options.json_file)

-            option_list_json = json_wrapper_option_list.read()

-            options_sorted = option_list_json.items()

-            for install_option in options_sorted:

-                if displayOption == "tree" and install_option[1]["title"] == "ISO Packages":

-                    continue

-                specDeps.readSpecsAndConvertToSerializableObjects(

-                    options.spec_dir,

-                    options.input_type, install_option[1]["file"],

-                    displayOption)

-    except Exception as _:

+            list_json_files = options.json_file.split("\n")

+            # Generate the expanded package dependencies json file based on package_list_file

+            for json_file in list_json_files:

+                shutil.copy2(json_file, options.output_dir)

+                json_wrapper_option_list = JsonWrapper(json_file)

+                option_list_json = json_wrapper_option_list.read()

+                options_sorted = option_list_json.items()

+                for install_option in options_sorted:

+                    output_file = None

+                    input_value = os.path.join(os.path.dirname(json_file), install_option[1]["file"])

+                    if options.display_option == "tree" and install_option[1]["title"] == "ISO Packages":

+                        continue

+                    if options.display_option == "json":

+                        output_file = os.path.join(options.output_dir, install_option[1]["file"])

+                    print ("Generating the install time dependency list for " + json_file)

+                    specDeps.process(options.input_type, input_value, options.display_option, output_file)

+    except Exception as e:

+        traceback.print_exc()

+        sys.stderr.write(str(e))

         sys.stderr.write("Failed to generate dependency lists from spec files\n")

         sys.exit(1)

     sys.stderr.write("Successfully generated dependency lists from spec files\n")

     sys.exit(0)

+

 if __name__ == "__main__":

     main()

deleted file mode 100755

@@ -1,73 +0,0 @@

-#!/usr/bin/python2

-#

-#Copyright (C) 2015 vmware inc.

-#

-#Author:    Harish    Udaiya    Kumar    <hudaiyakumar@vmware.com>

-from SpecUtils import Specutils

-from SpecData import SerializableSpecObject

-from SpecData import SerializedSpecObjects

-from RepoDeps import RepoQueryDependency

-import sys

-import os

-from optparse import OptionParser

-

-DEFAULT_INPUT_TYPE      =    "json"

-DEFAULT_DISPLAY_OPTION  =    "tree"

-SPEC_FILE_DIR           =    "../../SPECS"

-INPUT_DATA_DIR          =    "../../common/data"

-

-def reportMissing(pkg,specDepList, repoDepList, excludeList):

-    missingList = [];

-    if None != repoDepList:

-        for repoItem in repoDepList:

-            if repoItem not in excludeList and (None == specDepList or repoItem  not in specDepList):

-                missingList.append(repoItem)

-    if missingList:

-        if len(pkg) <= 7:

-            print pkg, "missing\t\t->", missingList

-        else:

-            print pkg, "missing\t->", missingList

-