1. photon
  2. Commit 2820c61a4ea4febc005d2c58e1f3c167bd007a21
Browse code

Resturctured the build code

Divya Thaluru authored on 2015/05/27 01:57:21
Showing 17 changed files
support/package-builder/ChrootUtils.py
History View file @ 2820c61
1 1 
new file mode 100644
... ... 
@@ -0,0 +1,120 @@
0 
+import subprocess
1 
+from Logger import Logger
2 
+#from ToolChainUtils import ToolChainUtils
3 
+from CommandUtils import CommandUtils
4 
+import os
5 
+import threading
6 
+import time
7 
+from time import sleep
8 
+from constants import constants
9 
+
10 
+chrootRootPath="/mnt"
11 
+
12 
+class ChrootUtils(object):
13 
+    counter=1
14 
+    lockForCounter=threading.Lock()
15 
+    lockForTrackingChroots=threading.Lock()
16 
+    lockForCreateChroot=threading.Lock()
17 
+    activeChroots=[]
18 
+    failureFlag=False
19 
+
20 
+    def __init__(self,logName=None,logPath=None):
21 
+        if logName is None:
22 
+            logName = "ChrootUtils"
23 
+        if logPath is None:
24 
+            logPath = constants.logPath
25 
+        self.logName=logName
26 
+        self.logPath=logPath
27 
+        self.logger=Logger.getLogger(logName,logPath)
28 
+
29 
+    def _getChrootUniqueID(self):
30 
+        ChrootUtils.lockForCounter.acquire()
31 
+        chrootID=chrootRootPath+"/photonroot"+str(ChrootUtils.counter)
32 
+        while os.path.isdir(chrootID):
33 
+            ChrootUtils.counter = ChrootUtils.counter +1
34 
+            chrootID=chrootRootPath+"/photonroot"+str(ChrootUtils.counter)
35 
+        ChrootUtils.counter = ChrootUtils.counter +1
36 
+        ChrootUtils.lockForCounter.release()
37 
+        return chrootID
38 
+
39 
+    def checkFreeSpace(self):
40 
+        process = subprocess.Popen(["df" ,chrootRootPath],shell=True,stdout=subprocess.PIPE)
41 
+        retval = process.wait()
42 
+        if retval != 0:
43 
+            self.logger.error("Unable to check free space. Unknown error.")
44 
+            return False
45 
+        output = process.communicate()[0]
46 
+        device, size, used, available, percent, mountpoint = output.split("\n")[1].split()
47 
+        if available > 600000:
48 
+            self.logger.info("Free Space available "+ available)
49 
+            return True
50 
+        return False
51 
+
52 
+    def createChroot(self):
53 
+        if ChrootUtils.failureFlag:
54 
+            return False
55 
+
56 
+        ChrootUtils.lockForCreateChroot.acquire()
57 
+        if ChrootUtils.failureFlag:
58 
+            ChrootUtils.lockForCreateChroot.release()
59 
+            return False
60 
+
61 
+        startTime=currentTime=time.time()
62 
+        timeOut=1800
63 
+        freeSpaceAvailable=False
64 
+        while currentTime < startTime + timeOut:
65 
+            #if self.checkFreeSpace():
66 
+            if len(ChrootUtils.activeChroots) < 2:
67 
+                freeSpaceAvailable=True
68 
+                break
69 
+            sleep(100)
70 
+
71 
+        if not freeSpaceAvailable:
72 
+            self.logger.error("Unable to create chroot. No sufficient free space is available.")
73 
+            ChrootUtils.lockForCreateChroot.release()
74 
+            ChrootUtils.failureFlag=True
75 
+            return False,None
76 
+
77 
+        chrootID=self._getChrootUniqueID()
78 
+        # need to add timeout for this step
79 
+        # http://stackoverflow.com/questions/1191374/subprocess-with-timeout
80 
+        process = subprocess.Popen("mkdir -p "+chrootID,shell=True,stdout=subprocess.PIPE)
81 
+        retval = process.wait()
82 
+
83 
+        ChrootUtils.lockForCreateChroot.release()
84 
+
85 
+        if retval != 0:
86 
+            self.logger.error("Unable to create chroot:"+ chrootID +".Unknown error.")
87 
+            return False,None
88 
+
89 
+        ChrootUtils.lockForTrackingChroots.acquire()
90 
+        ChrootUtils.activeChroots.append(chrootID)
91 
+        ChrootUtils.lockForTrackingChroots.release()
92 
+        return True,chrootID
93 
+
94 
+    def destroyChroot(self,chrootID):
95 
+        validChroot = True
96 
+        ChrootUtils.lockForTrackingChroots.acquire()
97 
+        if chrootID not in ChrootUtils.activeChroots:
98 
+            validChroot = False
99 
+        else:
100 
+            ChrootUtils.activeChroots.remove(chrootID)
101 
+        ChrootUtils.lockForTrackingChroots.release()
102 
+        if not validChroot:
103 
+            self.logger.error("Given chroot:"+chrootID+" is not a valid chroot. It is not created by ChrootUtils.")
104 
+            return False
105 
+
106 
+        # need to add timeout for this step
107 
+        # http://stackoverflow.com/questions/1191374/subprocess-with-timeout
108 
+        process = subprocess.Popen("./cleanup-build-root.sh "+chrootID,shell=True,stdout=subprocess.PIPE)
109 
+        retval = process.wait()
110 
+        if retval != 0:
111 
+            self.logger.error("Unable to destroy chroot:"+ chrootID +".Unknown error.")
112 
+            return False
113 
+        cmdUtils=CommandUtils()
114 
+        cmdUtils.run_command("rm -rf "+chrootID)
115 
+        ChrootUtils.lockForTrackingChroots.acquire()
116 
+        ChrootUtils.lockForTrackingChroots.release()
117 
+        self.logger.info("Successfully destroyed chroot:"+chrootID)
118 
+        return True
119 
+
support/package-builder/CommandUtils.py
History View file @ 2820c61
0 120 
new file mode 100644
... ... 
@@ -0,0 +1,93 @@
0 
+import subprocess
1 
+import os
2 
+import re
3 
+
4 
+class CommandUtils(object):
5 
+    def __init__(self):
6 
+        self.rpmbuild_binary = "rpmbuild"
7 
+        self.rpmbuild_buildall_option = "-ba"
8 
+        self.rpmbuild_nocheck_option = "--nocheck"
9 
+        self.rpm_binary = "rpm"
10 
+        self.install_rpm_package_options = "-Uvh"
11 
+        self.nodeps_rpm_package_options = "--nodeps"
12 
+        self.query_rpm_package_options = "-qa"
13 
+        self.force_rpm_package_options = "--force"
14 
+        self.find_command = "find"
15 
+
16 
+
17 
+    def find_file (self, filename, sourcePath):
18 
+        process = subprocess.Popen([self.find_command,  sourcePath,  "-name", filename],  stdout=subprocess.PIPE)
19 
+        returnVal = process.wait()
20 
+        result=process.communicate()[0]
21 
+        if result is None:
22 
+            return None
23 
+        return result.split()
24 
+
25 
+    def run_command(self,cmd,logfilePath=None,chrootCmd=None):
26 
+        if chrootCmd is not None:
27 
+            cmd = chrootCmd+" "+cmd
28 
+        if logfilePath is None:
29 
+            logfilePath=os.devnull
30 
+        logfile=open(logfilePath,"w")
31 
+        process = subprocess.Popen("%s" %cmd,shell=True,stdout=logfile)
32 
+        retval = process.wait()
33 
+        logfile.close()
34 
+        if retval==0:
35 
+            return True
36 
+        return False
37 
+
38 
+    def find_installed_rpm_packages(self, chrootCmd=None):
39 
+        cmd = self.rpm_binary+" "+self.query_rpm_package_options
40 
+        if chrootCmd is not None:
41 
+            cmd = chrootCmd+" "+cmd
42 
+        #process may hang if buffer is full
43 
+        process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE)
44 
+        retval = process.wait()
45 
+        result = process.communicate()[0]
46 
+        return result.split()
47 
+
48 
+    def installRPM(self, rpmfilepath,logfilePath, chrootCmd=None, withNodeps=False):
49 
+        print "Installing RPM...",rpmfilepath
50 
+        rpmInstallcmd=self.rpm_binary+" "+ self.install_rpm_package_options
51 
+
52 
+        if withNodeps:
53 
+            rpmInstallcmd+=" "+self.nodeps_rpm_package_options
54 
+
55 
+        rpmInstallcmd+=" "+rpmfilepath
56 
+        if chrootCmd is not None:
57 
+            rpmInstallcmd= chrootCmd +" "+rpmInstallcmd
58 
+        logfile=open(logfilePath,'w')
59 
+        process = subprocess.Popen("%s" %rpmInstallcmd,shell=True,stdout=logfile,stderr=logfile)
60 
+        retval = process.wait()
61 
+        logfile.close()
62 
+        if retval != 0:
63 
+            return False
64 
+        return True
65 
+
66 
+    def buildRPM(self, specfilepath, logfilePath, chrootCmd=None):
67 
+        print "Building RPM...",specfilepath
68 
+        rpmBuildcmd= self.rpmbuild_binary+" "+self.rpmbuild_buildall_option+" "+self.rpmbuild_nocheck_option
69 
+        rpmBuildcmd+=" "+specfilepath
70 
+        if chrootCmd is not None:
71 
+            rpmBuildcmd=chrootCmd +" "+rpmBuildcmd
72 
+
73 
+        logfile=open(logfilePath,'w')
74 
+        process = subprocess.Popen("%s" %rpmBuildcmd,shell=True,stdout=logfile,stderr=logfile)
75 
+        retval = process.wait()
76 
+        logfile.close()
77 
+        if retval != 0:
78 
+            print "Building rpm is failed"
79 
+            return False, None
80 
+
81 
+        #Extracting rpms created from log file
82 
+        logfile=open(logfilePath,'r')
83 
+        fileContents=logfile.readlines()
84 
+        logfile.close()
85 
+        listRpms=[]
86 
+        for i in range(0,len(fileContents)):
87 
+            if re.search("^Wrote:",fileContents[i]):
88 
+                listcontents=fileContents[i].split()
89 
+                if (len(listcontents) == 2) and listcontents[1].strip()[-4:] == ".rpm" and listcontents[1].find("/RPMS/") != -1:
90 
+                    listRpms.append(listcontents[1])
91 
+
92 
+        return True, listRpms
0 93 
\ No newline at end of file
support/package-builder/Logger.py
History View file @ 2820c61
1 94 
new file mode 100644
... ... 
@@ -0,0 +1,41 @@
0 
+import logging
1 
+import os
2 
+
3 
+class Logger(object):
4 
+    @staticmethod
5 
+    def getLogger (mymodule, logpath=None):
6 
+        logfile=mymodule+".log"
7 
+        if logpath is not None:
8 
+            if not os.path.isdir(logpath):
9 
+                os.makedirs(logpath)
10 
+            logfile=logpath+"/"+logfile
11 
+        logger=logging.getLogger(mymodule)
12 
+        if len(logger.handlers) == 0:
13 
+            #creating file handler
14 
+            fhandler=logging.FileHandler(logfile)
15 
+            # create console handler
16 
+            ch=logging.StreamHandler()
17 
+            formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
18 
+            # add formatter to handler
19 
+            fhandler.setFormatter(formatter)
20 
+            ch.setFormatter(formatter)
21 
+
22 
+            logger.addHandler(ch)
23 
+            logger.addHandler(fhandler)
24 
+            logger.setLevel(logging.DEBUG)
25 
+            logger.info("--------------------------------------------------------------------------")
26 
+            logger.info("Staring Log")
27 
+            logger.info("--------------------------------------------------------------------------")
28 
+        return logger
29 
+
30 
+if __name__ == "__main__":
31 
+    #Logger.getLogger("my module")
32 
+    t1 =  Logger.getLogger("my module")
33 
+    t1.info("test1")
34 
+    t2  = Logger.getLogger("my module")
35 
+    t2.info("test2")
36 
+    t1.info("test3")
37 
+
38 
+
39 
+
40 
+
support/package-builder/Main.py
History View file @ 2820c61
0 41 
new file mode 100644
... ... 
@@ -0,0 +1,130 @@
0 
+from optparse import OptionParser
1 
+import os.path
2 
+from CommandUtils import CommandUtils
3 
+from Logger import Logger
4 
+from SpecData import SerializableSpecObjectsUtils
5 
+from ChrootUtils import ChrootUtils
6 
+from PackageUtils import PackageUtils
7 
+from ToolChainUtils import ToolChainUtils
8 
+from constants import constants
9 
+from PackageBuildDataGenerator import PackageBuildDataGenerator
10 
+from PackageManager import PackageManager
11 
+
12 
+
13 
+def main():
14 
+    usage = "Usage: %prog [options] <package name>"
15 
+    parser = OptionParser(usage)
16 
+    parser.add_option("-s",  "--spec-path",  dest="specPath",  default="/workspace1/myrepos/photon/SPECS")
17 
+    parser.add_option("-o",  "--source-path",  dest="sourcePath",  default="/workspace1/mysources")
18 
+    parser.add_option("-r",  "--rpm-path",  dest="rpmPath",  default="/workspace1/mystage/RPMS")
19 
+    parser.add_option("-t",  "--install-tool-chain",  dest="installToolChain",  default=False,  action ="store_true")
20 
+    parser.add_option("-i",  "--install-package", dest="installPackage",  default=False,  action ="store_true")
21 
+    parser.add_option("-c",  "--clean-build", dest="cleanBuild",  default=False,  action ="store_true")
22 
+    parser.add_option("-p",  "--tools-path", dest="toolsPath",  default="/workspace1/mystage")
23 
+    parser.add_option("-l",  "--log-path", dest="logPath",  default="/workspace1/LOGS")
24 
+    parser.add_option("-a",  "--build-all", dest="buildAll",  default=False,  action ="store_true")
25 
+    parser.add_option("-f",  "--force", dest="forceBuild",  default=False,  action ="store_true")
26 
+    parser.add_option("-x",  "--incremental-build", dest="incrementalBuild",  default=True,  action ="store_true")
27 
+    parser.add_option("-z",  "--top-dir-path", dest="topDirPath",  default="/usr/src/photon")
28 
+
29 
+
30 
+    (options,  args) = parser.parse_args()
31 
+
32 
+    cmdUtils=CommandUtils()
33 
+    if not os.path.isdir(options.logPath):
34 
+        cmdUtils.run_command("mkdir -p "+options.logPath)
35 
+
36 
+    if not os.path.isdir(options.rpmPath):
37 
+        cmdUtils.run_command("mkdir -p "+options.rpmPath+"/x86_64")
38 
+        cmdUtils.run_command("mkdir -p "+options.rpmPath+"/noarch")
39 
+
40 
+    logger=Logger.getLogger(options.logPath+"/Main")
41 
+    logger.info("Source Path :"+options.sourcePath)
42 
+    logger.info("Spec Path :" + options.specPath)
43 
+    logger.info("Rpm Path :" + options.rpmPath)
44 
+    logger.info("Tools Path :" + options.toolsPath)
45 
+    logger.info("Log Path :" + options.logPath)
46 
+    logger.info("Top Dir Path :" + options.topDirPath)
47 
+
48 
+    if not os.path.isfile(options.toolsPath+"/tools-build.tar"):
49 
+        logger.error("Missing tools-build tar from tools path"+options.toolsPath)
50 
+        logger.error("Please provide correct tools path and continue")
51 
+        return False
52 
+
53 
+    listPackages=["acl","attr","autoconf","automake","bash","bc","bindutils","binutils","bison","boost","btrfs-progs","bzip2","ca-certificates","cdrkit","check",
54 
+                  "cloud-init","cmake","coreutils","cpio","cracklib","createrepo","curl","cyrus-sasl","db","dbus","deltarpm","diffutils","docbook-xml","docbook-xsl",
55 
+                  "docker","dparted","dracut","e2fsprogs","elfutils","etcd","expat","file","filesystem","findutils","flex","gawk","gcc","gdb","gdbm","gettext","git",
56 
+                  "glib","glibc","glibmm","gmp","go","gobject-introspection","google-daemon","google-startup-scripts","gperf","gpgme","gptfdisk","grep","groff",
57 
+                  "grub","gtk-doc","gzip","haveged","hawkey","iana-etc","inetutils","intltool","iproute2","iptables","itstool","json-glib","kbd","kmod","krb5",
58 
+                  "kubernetes","less","libaio","libassuan","libcap","libdnet","libffi","libgpg-error","libgsystem","libhif","libmspack","libpcap","libpipeline",
59 
+                  "librepo","libselinux","libsepol","libsigc++","libsolv","libtool","libxml2","libxslt","libyaml","linux","linux-api-headers","Linux-PAM","lua",
60 
+                  "lvm2","lzo","m4","make","man-db","man-pages","mercurial","mpc","mpfr","nano","ncurses","nspr","nss","ntp","openldap","openssh","openssl",
61 
+                  "open-vm-tools","ostree","parted","patch","pcre","perl","perl-common-sense","perl-Config-IniFiles","perl-DBD-SQLite","perl-DBI","perl-DBIx-Simple",
62 
+                  "perl-Exporter-Tiny","perl-JSON-XS","perl-libintl","perl-List-MoreUtils","perl-Module-Install","perl-Module-ScanDeps","perl-Types-Serialiser",
63 
+                  "perl-WWW-Curl","perl-YAML","perl-YAML-Tiny","photon-release","pkg-config","popt","procps-ng","psmisc","pycurl","pygobject","python2",
64 
+                  "python-configobj","python-iniparse","python-jsonpatch","python-jsonpointer","python-prettytable","python-requests","python-setuptools",
65 
+                  "python-six","PyYAML","readline","rocket","rpm","rpm-ostree","rpm-ostree-toolbox","ruby","sed","shadow","sqlite-autoconf","strace","sudo",
66 
+                  "swig","systemd","tar","tcpdump","tcsh","tdnf","texinfo","thin-provisioning-tools","tzdata","unzip","urlgrabber","util-linux","vim","wget",
67 
+                  "which","xerces-c","XML-Parser","xml-security-c","xz","yum","yum-metadata-parser","zlib"]
68 
+
69 
+    constants.initialize(options)
70 
+
71 
+    listPackages1=["nano","swig","wget"]
72 
+
73 
+    #tUtils = ToolChainUtils()
74 
+    #tUtils.buildToolChain()
75 
+    pkgManager = PackageManager()
76 
+    pkgManager.buildPackages(listPackages1)
77 
+
78 
+    '''
79 
+    chrUtils=ChrootUtils(options)
80 
+    returnVal,chrootID1=chrUtils.createChroot()
81 
+    logger.info("Obtained chroot"+ chrootID1)
82 
+    if not returnVal:
83 
+        return False
84 
+    chrUtils.prepareChroot(chrootID1)
85 
+    chrUtils.destroyChroot(chrootID1)
86 
+
87 
+
88 
+    chrUtils1=ChrootUtils(options)
89 
+    returnVal,chrootID=chrUtils1.createChroot()
90 
+    logger.info("Obtained chroot"+ chrootID)
91 
+    if not returnVal:
92 
+        return False
93 
+    chrUtils1.prepareChroot(chrootID)
94 
+
95 
+    tUtils.installToolChain(chrootID)
96 
+    #chrUtils1.destroyChroot(chrootID)
97 
+    '''
98 
+
99 
+    #tUtils=ToolChainUtils()
100 
+    #tUtils.buildToolChain()
101 
+
102 
+    #pkgUtils=PackageUtils()
103 
+    #pkgUtils.buildPackage("nano")
104 
+
105 
+
106 
+
107 
+
108 
+
109 
+
110 
+'''
111 
+    package_builder = BuildSystem(options.source_path,  options.spec_path,  options.rpm_path,  options.build_root, options.tools_path, options.log_path)
112 
+
113 
+    returnVal = True
114 
+    if options.clean_build:
115 
+        returnVal=package_builder.doCleanBuild()
116 
+    elif options.build_all:
117 
+        returnVal=package_builder.buildAllPackages()
118 
+    elif options.install_tool_chain:
119 
+        returnVal=package_builder.installToolchain()
120 
+    elif options.install_package:
121 
+        if (len(args)) != 1:
122 
+            parser.error("Incorrect number of arguments")
123 
+            returnVal=False
124 
+        else:
125 
+            returnVal=package_builder.installPackage(args[0],options.force_build)
126 
+    return returnVal
127 
+'''
128 
+if __name__=="__main__":
129 
+    main()
0 130 
\ No newline at end of file
support/package-builder/MiscUtils.py
History View file @ 2820c61
1 131 
new file mode 100644
... ... 
@@ -0,0 +1,36 @@
0 
+import os.path
1 
+
2 
+class MiscUtils(object):
3 
+
4 
+    @staticmethod
5 
+    def isOutdated(listInputFiles,listOutputFiles):
6 
+        thresholdTimeStamp=None
7 
+        if len(listInputFiles) == 0:
8 
+            return False
9 
+        if len(listOutputFiles) == 0:
10 
+            return True
11 
+        for f in listOutputFiles:
12 
+            t=os.path.getmtime(f)
13 
+            if thresholdTimeStamp is None:
14 
+                thresholdTimeStamp = t
15 
+            if t < thresholdTimeStamp:
16 
+                thresholdTimeStamp = t
17 
+        for f in listInputFiles:
18 
+            t=os.path.getmtime(f)
19 
+            if t > thresholdTimeStamp:
20 
+                return True
21 
+        return False
22 
+
23 
+    @staticmethod
24 
+    def getListSpecFiles(listSpecFiles,path):
25 
+        for dirEntry in os.listdir(path):
26 
+            dirEntryPath = os.path.join(path, dirEntry)
27 
+            if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(".spec"):
28 
+                listSpecFiles.append(dirEntryPath)
29 
+            elif os.path.isdir(dirEntryPath):
30 
+                MiscUtils.getListSpecFiles(listSpecFiles,dirEntryPath)
31 
+
32 
+if __name__=="__main__":
33 
+    listInputFiles=["SpecParser.py","Logger.py"]
34 
+    listOutputFiles=["Main.py"]
35 
+    print MiscUtils.isOutdated(listInputFiles, listOutputFiles)
0 36 
\ No newline at end of file
support/package-builder/PackageBuildDataGenerator.py
History View file @ 2820c61
1 37 
new file mode 100644
... ... 
@@ -0,0 +1,286 @@
0 
+from Logger import Logger
1 
+from constants import constants
2 
+from sets import Set
3 
+import copy
4 
+
5 
+
6 
+class PackageBuildDataGenerator(object):
7 
+
8 
+    cycleCount=0
9 
+
10 
+    def __init__(self,logName=None,logPath=None):
11 
+        if logName is None:
12 
+            logName = "PackageBuildDataGenerator"
13 
+        if logPath is None:
14 
+            logPath = constants.logPath
15 
+        self.logName=logName
16 
+        self.logPath=logPath
17 
+        self.logger=Logger.getLogger(logName,logPath)
18 
+        self.__mapCyclesToPackageList={}
19 
+        self.__mapPackageToCycle={}
20 
+        self.__buildDependencyGraph={}
21 
+        self.__runTimeDependencyGraph={}
22 
+        self.__sortedPackageList=[]
23 
+        self.__sortedBuildDependencyGraph={}
24 
+
25 
+    def getPackageBuildData(self,listPackages):
26 
+        if not self.__readDependencyGraphAndCyclesForGivenPackages(listPackages):
27 
+            return False,None,None,None
28 
+
29 
+        self.logger.info("sorted build order")
30 
+        self.logger.info(self.__sortedBuildDependencyGraph)
31 
+        #self.logger.info("Map cycles to package list")
32 
+        #self.logger.info(self.__mapCyclesToPackageList)
33 
+
34 
+        if not self.__getSortedBuildOrderListForGivenPackages(listPackages):
35 
+            return False,None,None,None
36 
+
37 
+        return True,self.__mapCyclesToPackageList,self.__mapPackageToCycle,self.__sortedPackageList
38 
+
39 
+    def __getSortedBuildOrderListForGivenPackages(self,listPackages):
40 
+        #should not include toolchain packages
41 
+
42 
+        alreadyProcessedPackages=[]
43 
+        sList=[]
44 
+        listPackagesCopy=self.__buildDependencyGraph.keys()#listPackages[:]
45 
+        sortedListIndex = 0
46 
+
47 
+        listPkgsRequiredToBuildToolchain=["linux-api-headers", "glibc","glibc-devel",  "zlib","zlib-devel",  "file",
48 
+            "binutils","binutils-devel",  "gmp","gmp-devel", "mpfr", "mpfr-devel", "mpc",
49 
+            "libgcc","libgcc-devel","libstdc++","libstdc++-devel","libgomp","libgomp-devel","gcc",
50 
+            "pkg-config", "ncurses", "bash", "bzip2", "sed","ncurses-devel","procps-ng","coreutils", "m4","grep",
51 
+            "readline", "diffutils","gawk", "findutils", "gettext", "gzip","make",  "patch",
52 
+            "util-linux", "tar", "xz","libtool", "flex",  "bison",
53 
+            "readline-devel", "lua","lua-devel","popt","popt-devel","nspr","sqlite-autoconf","nss","nss-devel",
54 
+            "bzip2-devel","elfutils-libelf","elfutils","elfutils-libelf-devel","elfutils-devel",
55 
+            "expat","libffi","libpipeline", "gdbm","perl","texinfo","autoconf","automake",
56 
+            "openssl","openssl-devel","python2","python2-libs","python2-devel","rpm",
57 
+            "groff", "man-db", "man-pages","cpio"]
58 
+        '''
59 
+        toolChainPackagesCount=0
60 
+        toolChainPackagelist=[]
61 
+        for p in listPkgsRequiredToBuildToolchain:
62 
+            basePkg=constants.specData.getSpecName(p)
63 
+            if basePkg in listPackagesCopy:
64 
+                listPackagesCopy.remove(basePkg)
65 
+                toolChainPackagelist.append(basePkg)
66 
+                toolChainPackagesCount = toolChainPackagesCount + 1
67 
+        '''
68 
+        sListLenBackUp=0
69 
+        while listPackagesCopy:
70 
+            pkg = None
71 
+            lenIndex=len(sList)
72 
+            for  i in range(lenIndex):
73 
+                if sList[i] in alreadyProcessedPackages:
74 
+                    continue
75 
+                pkg = sList[i]
76 
+                sortedListIndex = i
77 
+                break
78 
+
79 
+
80 
+            if pkg is None:
81 
+                pkg = listPackagesCopy.pop()
82 
+                sortedListIndex = len(sList)
83 
+
84 
+            rbrList=[]
85 
+            for p in self.__runTimeDependencyGraph[pkg]:
86 
+                if p not in rbrList:
87 
+                    rbrList.append(p)
88 
+            rbrList.append(pkg)
89 
+            brList=[]
90 
+
91 
+            for p in rbrList:
92 
+                basePkg=constants.specData.getSpecName(p)
93 
+                for bPkg in self.__sortedBuildDependencyGraph[basePkg]:
94 
+                    if bPkg not in brList:
95 
+                        brList.append(bPkg)
96 
+
97 
+            #remove any cyclic packages in brList if they exists in sList
98 
+            circularDependentPackages=[]
99 
+            if self.__mapPackageToCycle.has_key(pkg):
100 
+                circularDependentPackages.extend(self.__mapCyclesToPackageList[self.__mapPackageToCycle[pkg]])
101 
+                circularDependentPackages.remove(pkg)
102 
+
103 
+            for p in circularDependentPackages:
104 
+                if p in sList:
105 
+                    if p in brList:
106 
+                        brList.remove(p)
107 
+
108 
+            index=sortedListIndex
109 
+            subList=[]
110 
+            if sortedListIndex > 0:
111 
+                subList=sList[:sortedListIndex]
112 
+            for p in brList:
113 
+                if  p not in subList:
114 
+                    sList.insert(index, p)
115 
+                    index = index + 1
116 
+            alreadyProcessedPackages.append(p)
117 
+
118 
+            if (len(sList)-sListLenBackUp) > 100 :
119 
+                self.logger.info("Removing duplicates in sList")
120 
+                sListCopy=[]
121 
+                for p in sList:
122 
+                    if p not in sListCopy:
123 
+                        sListCopy.append(p)
124 
+                sList = sListCopy
125 
+            else:
126 
+                sListLenBackUp=len(sList)
127 
+
128 
+        self.logger.info("Only final step remains....")
129 
+        sListCopy=[]
130 
+        for p in sList:
131 
+            #if p in toolChainPackagelist:
132 
+            #    continue
133 
+            if p not in sListCopy:
134 
+                sListCopy.append(p)
135 
+        sList = sListCopy
136 
+
137 
+
138 
+        self.logger.info("sorted list")
139 
+        self.logger.info(sList)
140 
+        self.__sortedPackageList=sList
141 
+        return True
142 
+
143 
+    def __constructBuildTimeDependencyGraph(self,package):
144 
+        if self.__buildDependencyGraph.has_key(package):
145 
+            return True
146 
+        listDependentRpmPackages=constants.specData.getBuildRequiresForPackage(package)
147 
+        listDependentPackages=[]
148 
+        for rpmPkg in listDependentRpmPackages:
149 
+            basePkg=constants.specData.getSpecName(rpmPkg)
150 
+            if basePkg not in listDependentPackages:
151 
+                listDependentPackages.append(basePkg)
152 
+
153 
+        self.__buildDependencyGraph[package]=listDependentPackages
154 
+        for basePkg in listDependentPackages:
155 
+            self.__constructBuildTimeDependencyGraph(basePkg)
156 
+            self.__constructRunTimeDependencyGraph(basePkg)
157 
+
158 
+    def __constructRunTimeDependencyGraph(self,package):
159 
+        if self.__runTimeDependencyGraph.has_key(package):
160 
+            return True
161 
+        listRpmPackages=constants.specData.getPackages(package)
162 
+        for rpmPkg in listRpmPackages:
163 
+            listDependentRpmPackages=constants.specData.getRequiresAllForPackage(rpmPkg)
164 
+            self.__runTimeDependencyGraph[rpmPkg]=listDependentRpmPackages[:]
165 
+        for rpmPkg in listRpmPackages:
166 
+            for pkg in self.__runTimeDependencyGraph[rpmPkg]:
167 
+                self.__constructRunTimeDependencyGraph(pkg)
168 
+                self.__constructBuildTimeDependencyGraph(pkg)
169 
+
170 
+    def __readDependencyGraphAndCyclesForGivenPackages(self,listPackages):
171 
+        for pkg in listPackages:
172 
+            self.__constructBuildTimeDependencyGraph(pkg)
173 
+            self.__constructRunTimeDependencyGraph(pkg)
174 
+
175 
+        packagesToBUild=self.__buildDependencyGraph.keys()
176 
+        for pkg in packagesToBUild:
177 
+            sortedPackageList,circularDependentPackages = self.topologicalSortPackages(self.__buildDependencyGraph,pkg)
178 
+            if len(circularDependentPackages) > 0 :
179 
+                self.logger.error("Found circular dependency")
180 
+                self.logger.error(circularDependentPackages)
181 
+                return False
182 
+            self.__sortedBuildDependencyGraph[pkg]=sortedPackageList
183 
+        sortedPackageList,circularDependentPackages = self.topologicalSortPackages(self.__runTimeDependencyGraph)
184 
+        if len(circularDependentPackages) > 0 :
185 
+            self.__findCircularDependencies(circularDependentPackages)
186 
+        return True
187 
+
188 
+    def topologicalSortPackages(self, dependentPackages1, package=None):
189 
+        noDepPackages = Set()
190 
+        sortedPackageList = []
191 
+        dependentOfPackage = dict()
192 
+
193 
+        dependentPackages={}
194 
+        if package is None:
195 
+            dependentPackages=copy.deepcopy(dependentPackages1)
196 
+        else:
197 
+            listDepPkgs= Set()
198 
+            listDepPkgs.add(package)
199 
+            while listDepPkgs:
200 
+                pkg = listDepPkgs.pop()
201 
+                if dependentPackages.has_key(pkg):
202 
+                    continue
203 
+                dependentPackages[pkg]=dependentPackages1[pkg][:]
204 
+                for depPkg in dependentPackages1[pkg]:
205 
+                    listDepPkgs.add(depPkg)
206 
+        #Find packages with no dependencies and generate dependentof_package edge list
207 
+        for pkg in dependentPackages:
208 
+            if len(dependentPackages[pkg]) == 0:
209 
+                noDepPackages.add(pkg)
210 
+            else:
211 
+                for depPkg in dependentPackages[pkg]:
212 
+                    if not dependentOfPackage.has_key(depPkg):
213 
+                        dependentOfPackage[depPkg]=[pkg]
214 
+                    else:
215 
+                        if pkg not in dependentOfPackage[depPkg]:
216 
+                            dependentOfPackage[depPkg].append(pkg)
217 
+
218 
+        while noDepPackages:
219 
+            pkg = noDepPackages.pop()
220 
+            sortedPackageList.append(pkg)
221 
+            if dependentOfPackage.get(pkg) is not None:
222 
+                for childPkg in list(dependentOfPackage.get(pkg)):
223 
+                    dependentOfPackage.get(pkg).remove(childPkg)
224 
+                    dependentPackages[childPkg].remove(pkg)
225 
+                    if len(dependentPackages[childPkg])==0:
226 
+                        noDepPackages.add(childPkg)
227 
+
228 
+        circularDependentGraph={}
229 
+        listCircularPkg = dependentPackages.keys()
230 
+        for pkg in listCircularPkg:
231 
+            if len(dependentPackages[pkg]) != 0:
232 
+                circularDependentGraph[pkg]=dependentPackages[pkg]
233 
+
234 
+        #return (non-circular dependent package in sorted order and circular dependent package list in a dependencyGraph)
235 
+        return sortedPackageList,circularDependentGraph
236 
+
237 
+    def __findCircularDependencies(self,cyclicDependentGraph):
238 
+        if len(cyclicDependentGraph) == 0:
239 
+            return True
240 
+        #step1: construct dependency map from dependency graph
241 
+        constructDependencyMap={}
242 
+        listNodes=cyclicDependentGraph.keys()
243 
+        for node in listNodes:
244 
+            tmpDepNodeList=[]
245 
+            tmpDepNodeList.append(node)
246 
+            depNodeList=[]
247 
+            while len(tmpDepNodeList)!=0:
248 
+                currentNode = tmpDepNodeList.pop()
249 
+                addDepNodeList = cyclicDependentGraph[currentNode]
250 
+                depNodeList.append(currentNode)
251 
+                for depNode in addDepNodeList:
252 
+                    if depNode in depNodeList:
253 
+                        continue
254 
+                    else:
255 
+                        if depNode not in tmpDepNodeList:
256 
+                            tmpDepNodeList.append(depNode)
257 
+            depNodeList.remove(node)
258 
+            constructDependencyMap[node]=depNodeList
259 
+        self.logger.info("Dependency Map:")
260 
+        self.logger.info(constructDependencyMap)
261 
+
262 
+        #find cycles in dependency map
263 
+        for node in listNodes:
264 
+            listDepPkg=constructDependencyMap[node]
265 
+            cycPkgs=[]
266 
+            if not self.__mapPackageToCycle.has_key(node):
267 
+                for depPkg in listDepPkg:
268 
+                    x = constructDependencyMap[depPkg]
269 
+                    if node in x:
270 
+                        cycPkgs.append(depPkg)
271 
+
272 
+                if len(cycPkgs) != 0:
273 
+                    cycPkgs.append(node)
274 
+                    cycleName="cycle"+str(PackageBuildDataGenerator.cycleCount)
275 
+                    PackageBuildDataGenerator.cycleCount=PackageBuildDataGenerator.cycleCount+1
276 
+                    for x in cycPkgs:
277 
+                        self.__mapPackageToCycle[x]=cycleName
278 
+                    self.__mapCyclesToPackageList[cycleName]=cycPkgs
279 
+                    self.logger.info("New Circular dependency found")
280 
+                    self.logger.info(cycPkgs)
281 
+        return True
282 
+
283 
+
284 
+
285 
+
0 286 
\ No newline at end of file
support/package-builder/PackageBuilder.py
History View file @ 2820c61
1 287 
new file mode 100644
... ... 
@@ -0,0 +1,152 @@
0 
+from PackageUtils import PackageUtils
1 
+from Logger import Logger
2 
+from ChrootUtils import ChrootUtils
3 
+from ToolChainUtils import ToolChainUtils
4 
+from CommandUtils import CommandUtils
5 
+import os.path
6 
+from constants import constants
7 
+
8 
+class PackageBuilder(object):
9 
+
10 
+    #logger=None
11 
+
12 
+    #@staticmethod
13 
+    #def setLog(logPath):
14 
+    #   PackageBuilder.logger=Logger.getLogger(logPath+"/PackageBuilder")
15 
+
16 
+    def __init__(self,mapPackageToCycles,listAvailableCyclicPackages,logName=None,logPath=None):
17 
+        if logName is None:
18 
+            logName = "PackageBuilder"
19 
+        if logPath is None:
20 
+            logPath = constants.logPath
21 
+        self.logName=logName
22 
+        self.logPath=logPath
23 
+        self.logger=Logger.getLogger(logName,logPath)
24 
+        self.runInChrootCommand="./run-in-chroot.sh"
25 
+        self.mapPackageToCycles = mapPackageToCycles
26 
+        self.listAvailableCyclicPackages = listAvailableCyclicPackages
27 
+        self.listNodepsPackages = ["glibc","gmp","zlib","file","binutils","mpfr","mpc","gcc","ncurses","util-linux","groff","perl","texinfo","rpm","openssl","go"]
28 
+
29 
+    #assumes tool chain is already built
30 
+    def prepareBuildRoot(self):
31 
+        chrUtils = ChrootUtils(self.logName,self.logPath)
32 
+        returnVal,chrootID = chrUtils.createChroot()
33 
+        if not returnVal:
34 
+            self.logger.error("Unable to build tool chain.")
35 
+            chrUtils.destroyChroot(chrootID)
36 
+            return False
37 
+        tUtils=ToolChainUtils(self.logName,self.logPath)
38 
+        returnVal = tUtils.installToolChain(chrootID)
39 
+        return returnVal,chrootID
40 
+
41 
+    def findPackageNameFromRPMFile(self,rpmfile):
42 
+        rpmfile=os.path.basename(rpmfile)
43 
+        releaseindex=rpmfile.rfind("-")
44 
+        if releaseindex == -1:
45 
+            self.logger.error("Invalid rpm file:"+rpmfile)
46 
+            return None
47 
+        versionindex=rpmfile[0:releaseindex].rfind("-")
48 
+        if versionindex == -1:
49 
+            self.logger.error("Invalid rpm file:"+rpmfile)
50 
+            return None
51 
+        packageName=rpmfile[0:versionindex]
52 
+        return packageName
53 
+
54 
+    def findInstalledPackages(self,chrootID):
55 
+        cmdUtils = CommandUtils()
56 
+        listInstalledRPMs=cmdUtils.find_installed_rpm_packages(self.runInChrootCommand+" "+chrootID)
57 
+        listInstalledPackages=[]
58 
+        for installedRPM in listInstalledRPMs:
59 
+            packageName=self.findPackageNameFromRPMFile(installedRPM)
60 
+            if packageName is not None:
61 
+                listInstalledPackages.append(packageName)
62 
+        return listInstalledPackages
63 
+
64 
+    def buildPackageThreadAPI(self,package,outputMap, threadName):
65 
+        returnVal=self.buildPackage(package)
66 
+        outputMap[threadName]=returnVal
67 
+
68 
+    def buildPackage(self,package):
69 
+        #should initialize a logger based on package name
70 
+        chrUtils = ChrootUtils(self.logName,self.logPath)
71 
+        returnVal,chrootID = self.prepareBuildRoot()
72 
+        if not returnVal:
73 
+            return False
74 
+
75 
+        destLogPath=constants.logPath+"/build-"+package
76 
+        if not os.path.isdir(destLogPath):
77 
+            cmdUtils = CommandUtils()
78 
+            cmdUtils.run_command("mkdir -p "+destLogPath)
79 
+
80 
+        listInstalledPackages=self.findInstalledPackages(chrootID)
81 
+        self.logger.info("List of installed packages")
82 
+        self.logger.info(listInstalledPackages)
83 
+        returnVal,listDependentPackages=self.findBuildTimeRequiredPackages(package)
84 
+        if not returnVal:
85 
+            chrUtils.destroyChroot(chrootID)
86 
+            self.logger.error ("Failed during building the package"+package)
87 
+            return False
88 
+
89 
+        if len(listDependentPackages) != 0:
90 
+            self.logger.info("Installing the build time dependent packages......")
91 
+            for pkg in listDependentPackages:
92 
+                returnVal = self.installPackage(pkg,chrootID,destLogPath,listInstalledPackages)
93 
+                if not returnVal:
94 
+                    self.logger.error("Failed while installing the build time dependent package"+pkg)
95 
+                    chrUtils.destroyChroot(chrootID)
96 
+                    return False
97 
+            self.logger.info("Finished installing the build time dependent packages......")
98 
+
99 
+        pkgUtils = PackageUtils(self.logName,self.logPath)
100 
+        returnVal = pkgUtils.buildRPMSForGivenPackage(package,chrootID,destLogPath)
101 
+        if not returnVal:
102 
+            self.logger.error("Failed while building the package"+package)
103 
+            chrUtils.destroyChroot(chrootID)
104 
+            return False
105 
+        self.logger.info("Successfully built the package:"+package)
106 
+        chrUtils.destroyChroot(chrootID)
107 
+        return True
108 
+
109 
+    def findRunTimeRequiredRPMPackages(self,rpmPackage):
110 
+        listRequiredPackages=constants.specData.getRequiresForPackage(rpmPackage)
111 
+        return True,listRequiredPackages
112 
+
113 
+    def findBuildTimeRequiredPackages(self,package):
114 
+        listRequiredPackages=constants.specData.getBuildRequiresForPackage(package)
115 
+        return True,listRequiredPackages
116 
+
117 
+    def installPackage(self,package,chrootID,destLogPath,listInstalledPackages):
118 
+        #if toolchain package called this method, preventing from installing again
119 
+        if package in listInstalledPackages:
120 
+            return True
121 
+        returnVal = self.installDependentRunTimePackages(package,chrootID,destLogPath,listInstalledPackages)
122 
+        if not returnVal:
123 
+            return False
124 
+        pkgUtils = PackageUtils(self.logName,self.logPath)
125 
+        noDeps=False
126 
+        if self.mapPackageToCycles.has_key(package):
127 
+            noDeps = True
128 
+        if package in self.listNodepsPackages:
129 
+            noDeps=True
130 
+        returnVal = pkgUtils.installRPM(package,chrootID,noDeps,destLogPath)
131 
+        if not returnVal:
132 
+            self.logger.error("Stop installing package"+package)
133 
+            return False
134 
+        listInstalledPackages.append(package)
135 
+        self.logger.info("Installed the package:"+package)
136 
+        return True
137 
+
138 
+    def installDependentRunTimePackages(self,package,chrootID,destLogPath,listInstalledPackages):
139 
+        returnVal,listRunTimeDependentPackages=self.findRunTimeRequiredRPMPackages(package)
140 
+        if not returnVal:
141 
+            return False
142 
+        if len(listRunTimeDependentPackages) != 0:
143 
+            for pkg in listRunTimeDependentPackages:
144 
+                if self.mapPackageToCycles.has_key(pkg) and pkg not in self.listAvailableCyclicPackages:
145 
+                    continue
146 
+                if pkg in listInstalledPackages:
147 
+                    continue
148 
+                returnVal = self.installPackage(pkg,chrootID,destLogPath,listInstalledPackages)
149 
+                if not returnVal:
150 
+                    return False
151 
+        return True
support/package-builder/PackageManager.py
History View file @ 2820c61
0 152 
new file mode 100644
... ... 
@@ -0,0 +1,228 @@
0 
+from PackageBuildDataGenerator import PackageBuildDataGenerator
1 
+from Logger import Logger
2 
+import threading
3 
+from constants import constants
4 
+import time
5 
+from PackageBuilder import PackageBuilder
6 
+import os
7 
+from PackageUtils import PackageUtils
8 
+
9 
+class PackageManager(object):
10 
+
11 
+    def __init__(self,logName=None,logPath=None):
12 
+        if logName is None:
13 
+            logName = "PackageManager"
14 
+        if logPath is None:
15 
+            logPath = constants.logPath
16 
+        self.logName=logName
17 
+        self.logPath=logPath
18 
+        self.logger=Logger.getLogger(logName,logPath)
19 
+        self.mapCyclesToPackageList={}
20 
+        self.mapPackageToCycle={}
21 
+        self.sortedPackageList=[]
22 
+        self.listOfPackagesAlreadyBuilt = []
23 
+        self.listThreads={}
24 
+        self.mapOutputThread={}
25 
+        self.mapThreadsLaunchTime={}
26 
+        self.listAvailableCyclicPackages=[]
27 
+        self.listPackagesToBuild=[]
28 
+
29 
+    def readPackageBuildData(self, listPackages):
30 
+        pkgBuildDataGen = PackageBuildDataGenerator(self.logName,self.logPath)
31 
+        returnVal,self.mapCyclesToPackageList,self.mapPackageToCycle,self.sortedPackageList = pkgBuildDataGen.getPackageBuildData(listPackages)
32 
+
33 
+        if not returnVal:
34 
+            self.logger.error("unable to get sorted list")
35 
+            return False
36 
+
37 
+        return True
38 
+
39 
+    def getRequiredPackages(self,package):
40 
+        listRequiredRPMPackages=[]
41 
+        listRequiredRPMPackages.extend(constants.specData.getBuildRequiresForPackage(package))
42 
+        listRequiredRPMPackages.extend(constants.specData.getRequiresAllForPackage(package))
43 
+
44 
+        listRequiredPackages=[]
45 
+        for pkg in listRequiredRPMPackages:
46 
+            basePkg=constants.specData.getSpecName(pkg)
47 
+            if basePkg not in listRequiredPackages:
48 
+                listRequiredPackages.append(basePkg)
49 
+
50 
+        return listRequiredPackages
51 
+
52 
+
53 
+    def readAlreadyAvailablePackages(self):
54 
+        listAvailablePackages=[]
55 
+        listRPMFiles=[]
56 
+        listDirectorys=[]
57 
+        listDirectorys.append(constants.rpmPath)
58 
+
59 
+        while len(listDirectorys) > 0:
60 
+            dirPath=listDirectorys.pop()
61 
+            for dirEntry in os.listdir(dirPath):
62 
+                dirEntryPath = os.path.join(dirPath, dirEntry)
63 
+                if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(".rpm"):
64 
+                    listRPMFiles.append(dirEntryPath)
65 
+                elif os.path.isdir(dirEntryPath):
66 
+                    listDirectorys.append(dirEntryPath)
67 
+        pkgUtils = PackageUtils(self.logName,self.logPath)
68 
+        for rpmfile in listRPMFiles:
69 
+            package = pkgUtils.findPackageNameFromRPMFile(rpmfile)
70 
+            listAvailablePackages.append(package)
71 
+        return listAvailablePackages
72 
+
73 
+    def calculateParams(self,listPackages):
74 
+        self.listThreads.clear()
75 
+        self.mapOutputThread.clear()
76 
+        self.mapThreadsLaunchTime.clear()
77 
+        self.listAvailableCyclicPackages=[]
78 
+        self.mapCyclesToPackageList.clear()
79 
+        self.mapPackageToCycle.clear()
80 
+        self.sortedPackageList=[]
81 
+        self.listPackagesToBuild=[]
82 
+
83 
+        if not self.readPackageBuildData(listPackages):
84 
+            return False
85 
+
86 
+        self.listOfPackagesAlreadyBuilt = self.readAlreadyAvailablePackages()
87 
+
88 
+        self.listPackagesToBuild=self.sortedPackageList[:]
89 
+        for pkg in self.sortedPackageList:
90 
+            if pkg in self.listOfPackagesAlreadyBuilt:
91 
+                self.listPackagesToBuild.remove(pkg)
92 
+
93 
+        self.logger.info(self.listPackagesToBuild)
94 
+        self.logger.info(listPackages)
95 
+
96 
+        return True
97 
+
98 
+    def checkIfAnyThreadsAreCompleted(self):
99 
+        readyToLaunchMoreThreads = False
100 
+        listThreadsObjToRemove=[]
101 
+        for t in self.listThreads:
102 
+            self.logger.info("Checking thread "+t)
103 
+            #check if any thread is completed. If completed, we can start more threads.
104 
+            if self.mapOutputThread.has_key(t):
105 
+                output = self.mapOutputThread[t]
106 
+                self.logger.info("Output of thread "+t+" "+str(output))
107 
+                if not output:
108 
+                    self.logger.error("Thread "+ t+" is failed ")
109 
+                    #kill remaining Threads
110 
+                    return False,False
111 
+                else:
112 
+                    readyToLaunchMoreThreads=True
113 
+                    self.listPackagesToBuild.remove(t)
114 
+                    self.listOfPackagesAlreadyBuilt.append(t)
115 
+                    listThreadsObjToRemove.append(t)
116 
+                    if self.mapPackageToCycle.has_key(t):
117 
+                        self.listAvailableCyclicPackages.append(t)
118 
+
119 
+        if not readyToLaunchMoreThreads:
120 
+            return True,False
121 
+
122 
+        for k in listThreadsObjToRemove:
123 
+            self.listThreads.pop(k)
124 
+
125 
+        return True,True
126 
+
127 
+    def checkIfAnyThreadsAreHanged(self):
128 
+        currentTime = time.time()
129 
+        listThreadsHanged=[]
130 
+        for t in self.listThreads:
131 
+            self.logger.info("Checking thread "+t)
132 
+            if not self.mapOutputThread.has_key(t):
133 
+                self.logger.info("Calculating running time for thread "+t)
134 
+                launchTime = self.mapThreadsLaunchTime[t]
135 
+                if (currentTime - launchTime) > 3600.0:
136 
+                    listThreadsHanged.append(t)
137 
+
138 
+        if len(listThreadsHanged) > 0:
139 
+            self.logger.info("Looks like following threads are hanged")
140 
+            self.logger.info(listThreadsHanged)
141 
+            #kill all threads
142 
+            return False
143 
+
144 
+        return True
145 
+
146 
+    def waitTillNewThreadsCanBeSpawned(self):
147 
+        if len(self.listThreads) == 0:
148 
+            return True
149 
+        returnVal = False
150 
+        while True:
151 
+            sucess,Tfail = self.checkIfAnyThreadsAreCompleted()
152 
+            if not sucess:
153 
+                break
154 
+            if sucess and Tfail:
155 
+                returnVal = True
156 
+                break
157 
+            if not self.checkIfAnyThreadsAreHanged():
158 
+                break
159 
+            self.logger.info("Sleeping for 30 seconds")
160 
+            time.sleep(30)
161 
+        return returnVal
162 
+
163 
+    def buildPackages (self, listPackages):
164 
+        returnVal=self.calculateParams(listPackages)
165 
+        if not returnVal:
166 
+            self.logger.error("Unable to set paramaters. Terminating the package manager.")
167 
+            return False
168 
+        returnVal = True
169 
+        while len(self.listPackagesToBuild) > 0:
170 
+            #Free some threads to launch next threads
171 
+            if not self.waitTillNewThreadsCanBeSpawned():
172 
+                returnVal = False
173 
+                break
174 
+
175 
+            listOfPackagesCanBeBuild=self.findNextPackageToBuild()
176 
+            if len(listOfPackagesCanBeBuild) == 0 and len(self.listPackagesToBuild) != 0:
177 
+                self.logger.info("Waiting for current threads to complete to launch building new packages")
178 
+
179 
+            for pkg in listOfPackagesCanBeBuild:
180 
+                currentTime = time.time()
181 
+                pkgBuilder = PackageBuilder(self.mapPackageToCycle,self.listAvailableCyclicPackages,"build-"+pkg,constants.logPath)
182 
+                t = threading.Thread(target=pkgBuilder.buildPackageThreadAPI,args=(pkg,self.mapOutputThread,pkg))
183 
+                self.listThreads[pkg]=t
184 
+                self.mapThreadsLaunchTime[pkg]=currentTime
185 
+                self.logger.info("Launching thread for package:"+pkg)
186 
+                t.start()
187 
+                self.logger.info("Started the thread for "+pkg)
188 
+
189 
+            if len(self.listThreads) == 0 and len(self.listPackagesToBuild) != 0:
190 
+                self.logger.error("Following packages are waiting for unknown package")
191 
+                self.logger.error(self.listPackagesToBuild)
192 
+                self.logger.error("Unexpected error")
193 
+                returnVal = False
194 
+                break
195 
+
196 
+        if not returnVal:
197 
+            self.logger.error("Failed and exited gracefully")
198 
+            return False
199 
+
200 
+        self.logger.info( "Successfully built all the packages")
201 
+        return True
202 
+
203 
+    def findNextPackageToBuild(self):
204 
+        listOfPackagesNextToBuild=[]
205 
+        self.logger.info("Checking for next possible packages to build")
206 
+        for pkg in self.listPackagesToBuild:
207 
+            if self.listThreads.has_key(pkg):
208 
+                continue
209 
+            listRequiredPackages=self.getRequiredPackages(pkg)
210 
+            canBuild=True
211 
+            self.logger.info("Required packages for "+ pkg + " are:")
212 
+            self.logger.info(listRequiredPackages)
213 
+            for reqPkg in listRequiredPackages:
214 
+                if reqPkg not in self.listOfPackagesAlreadyBuilt:
215 
+                    canBuild=False
216 
+                    self.logger.info(reqPkg+" is not available. So we cannot build "+ pkg +" at this moment.")
217 
+                    break
218 
+            if canBuild:
219 
+                listOfPackagesNextToBuild.append(pkg)
220 
+        return listOfPackagesNextToBuild
221 
+
222 
+
223 
+
224 
+
225 
+
226 
+
227 
+
support/package-builder/PackageUtils.py
History View file @ 2820c61
0 228 
new file mode 100644
... ... 
@@ -0,0 +1,141 @@
0 
+from CommandUtils import CommandUtils
1 
+from Logger import Logger
2 
+import os
3 
+import shutil
4 
+from constants import constants
5 
+
6 
+
7 
+class PackageUtils(object):
8 
+
9 
+    def __init__(self,logName=None,logPath=None):
10 
+        if logName is None:
11 
+            self.logName = "PackageUtils"
12 
+        if logPath is None:
13 
+            logPath = constants.logPath
14 
+        self.logName=logName
15 
+        self.logPath=logPath
16 
+        self.logger=Logger.getLogger(logName,logPath)
17 
+        self.runInChrootCommand="./run-in-chroot.sh"
18 
+
19 
+
20 
+    def getRPMDestDir(self,rpmName,rpmDir):
21 
+        arch=""
22 
+        if rpmName.find("x86_64") != -1:
23 
+            arch='x86_64'
24 
+        elif rpmName.find("noarch") != -1:
25 
+            arch="noarch"
26 
+        #else: Todo throw an exeption
27 
+        rpmDestDir=rpmDir+"/"+arch
28 
+        return rpmDestDir
29 
+
30 
+    def copyRPM(self,rpmFile,destDir):
31 
+        cmdUtils = CommandUtils()
32 
+        rpmName=os.path.basename(rpmFile)
33 
+        rpmDestDir=self.getRPMDestDir(rpmName,destDir)
34 
+        if not os.path.isdir(rpmDestDir):
35 
+            cmdUtils.run_command("mkdir -p "+rpmDestDir)
36 
+        rpmDestPath=rpmDestDir+"/"+rpmName
37 
+        shutil.copyfile(rpmFile,  rpmDestPath)
38 
+        return rpmDestPath
39 
+
40 
+    def installRPM(self,package,chrootID,noDeps=False,destLogPath=None):
41 
+        rpmfile=self.findRPMFileForGivenPackage(package)
42 
+        if rpmfile is None:
43 
+            self.logger.error("unexpected error")
44 
+            self.logger.error("Stopping installing package:"+package)
45 
+            return False
46 
+
47 
+        rpmDestFile = self.copyRPM(rpmfile, chrootID+constants.topDirPath+"/RPMS")
48 
+        rpmFile=rpmDestFile.replace(chrootID,"")
49 
+        chrootCmd=self.runInChrootCommand+" "+chrootID
50 
+        logFile=chrootID+constants.topDirPath+"/LOGS"+"/"+package+".completed"
51 
+
52 
+        cmdUtils = CommandUtils()
53 
+        returnVal = cmdUtils.installRPM(rpmFile,logFile,chrootCmd,noDeps)
54 
+        if destLogPath is not None:
55 
+            shutil.copy2(logFile, destLogPath)
56 
+        if not returnVal:
57 
+            self.logger.error("Installing " + rpmFile+" rpm is failed")
58 
+            return False
59 
+        return True
60 
+
61 
+    def copySourcesTobuildroot(self,listSourceFiles,package,destDir):
62 
+        cmdUtils = CommandUtils()
63 
+        for source in listSourceFiles:
64 
+            sourcePath = cmdUtils.find_file(source,constants.sourcePath)
65 
+            if sourcePath is None or len(sourcePath) == 0:
66 
+                sourcePath = cmdUtils.find_file(source,constants.specPath)
67 
+            if sourcePath is None or len(sourcePath) == 0:
68 
+                self.logger.error("Missing source: "+source+". Cannot find sources for package: "+package)
69 
+                self.logger.error("Stopping building toolchain")
70 
+                return False
71 
+            if len(sourcePath) > 1:
72 
+                self.logger.error("Multiple sources found: Unable to determine one. ")
73 
+                self.logger.error("Stopping building toolchain")
74 
+                return False
75 
+            self.logger.info("Source path :" + source + " Source filename: " + sourcePath[0])
76 
+            shutil.copyfile(sourcePath[0],  destDir+source)
77 
+
78 
+    def buildRPMSForGivenPackage(self,package, chrootID,destLogPath=None):
79 
+        self.logger.info("Building package......"+package)
80 
+        #self.adjust_gcc_specs(package)
81 
+
82 
+        listSourcesFiles = constants.specData.getSources(package)
83 
+        listPatchFiles =  constants.specData.getPatches(package)
84 
+        specFile = constants.specData.getSpecFile(package)
85 
+        specName = constants.specData.getSpecName(package) + ".spec"
86 
+
87 
+        chrootSourcePath=chrootID+constants.topDirPath+"/SOURCES/"
88 
+        chrootSpecPath=constants.topDirPath+"/SPECS/"
89 
+        chrootLogsFilePath=chrootID+constants.topDirPath+"/LOGS/"+package+".log"
90 
+        chrootCmd=self.runInChrootCommand+" "+chrootID
91 
+        shutil.copyfile(specFile, chrootID+chrootSpecPath+specName )
92 
+
93 
+        self.copySourcesTobuildroot(listSourcesFiles,package,chrootSourcePath)
94 
+        self.copySourcesTobuildroot(listPatchFiles,package,chrootSourcePath)
95 
+
96 
+        returnVal,listRPMFiles = self.buildRPM(chrootSpecPath+"/"+specName,chrootLogsFilePath, chrootCmd)
97 
+        if destLogPath is not None:
98 
+            shutil.copy2(chrootLogsFilePath, destLogPath)
99 
+
100 
+        if not returnVal:
101 
+            return False
102 
+
103 
+        for rpmFile in listRPMFiles:
104 
+            self.copyRPM(chrootID+"/"+rpmFile, constants.rpmPath)
105 
+        return True
106 
+
107 
+    def buildRPM(self,specFile,logFile,chrootCmd):
108 
+        cmdUtils = CommandUtils()
109 
+        returnVal,listRPMFiles = cmdUtils.buildRPM(specFile,logFile,chrootCmd)
110 
+        if not returnVal:
111 
+            self.logger.error("Building rpm is failed "+specFile)
112 
+            return False,None
113 
+        return True,listRPMFiles
114 
+
115 
+    def findRPMFileForGivenPackage(self,package):
116 
+        cmdUtils = CommandUtils()
117 
+        version = constants.specData.getVersion(package)
118 
+        release = constants.specData.getRelease(package)
119 
+        listFoundRPMFiles = cmdUtils.find_file(package+"-"+version+"-"+release+"*.rpm",constants.rpmPath)
120 
+        if len(listFoundRPMFiles) == 1 :
121 
+            return listFoundRPMFiles[0]
122 
+        if len(listFoundRPMFiles) == 0 :
123 
+            return None
124 
+        if len(listFoundRPMFiles) > 1 :
125 
+            self.logger.error("Unable to determine the rpm file for package. Found multiple rpm files for given package in rpm directory")
126 
+            return None
127 
+        return None
128 
+
129 
+    def findPackageNameFromRPMFile(self,rpmfile):
130 
+        rpmfile=os.path.basename(rpmfile)
131 
+        releaseindex=rpmfile.rfind("-")
132 
+        if releaseindex == -1:
133 
+            self.logger.error("Invalid rpm file:"+rpmfile)
134 
+            return None
135 
+        versionindex=rpmfile[0:releaseindex].rfind("-")
136 
+        if versionindex == -1:
137 
+            self.logger.error("Invalid rpm file:"+rpmfile)
138 
+            return None
139 
+        packageName=rpmfile[0:versionindex]
140 
+        return packageName
support/package-builder/SpecData.py
History View file @ 2820c61
0 141 
new file mode 100644
... ... 
@@ -0,0 +1,173 @@
0 
+from SpecUtils import Specutils
1 
+import os
2 
+from Logger import Logger
3 
+
4 
+class SerializableSpecObject(object):
5 
+    def __init__(self):
6 
+        self.listPackages=[]
7 
+        self.name=""
8 
+        self.version=""
9 
+        self.release=""
10 
+        self.buildRequirePackages=[]
11 
+        self.installRequiresAllPackages=[]
12 
+        self.installRequiresPackages={}
13 
+        self.specFile=""
14 
+        self.listSources=[]
15 
+        self.listPatches=[]
16 
+
17 
+class SerializableSpecObjectsUtils(object):
18 
+
19 
+    def __init__(self,logPath):
20 
+        self.mapSerializableSpecObjects={}
21 
+        self.mapPackageToSpec={}
22 
+        self.logger=Logger.getLogger("Serializable Spec objects", logPath )
23 
+
24 
+    def readSpecsAndConvertToSerializableObjects(self,specFilesPath):
25 
+        listSpecFiles=[]
26 
+        self.getListSpecFiles(listSpecFiles,specFilesPath)
27 
+        for specFile in listSpecFiles:
28 
+            spec=Specutils(specFile)
29 
+            specName=os.path.basename(specFile)
30 
+            specName=specName.replace(".spec","")
31 
+            specObj=SerializableSpecObject()
32 
+            specObj.name=specName
33 
+            specObj.buildRequirePackages=spec.getBuildRequiresAllPackages()
34 
+            specObj.installRequiresAllPackages=spec.getRequiresAllPackages()
35 
+            specObj.listPackages=spec.getPackageNames()
36 
+            specObj.specFile=specFile
37 
+            specObj.version=spec.getVersion()
38 
+            specObj.release=spec.getRelease()
39 
+            specObj.listSources=spec.getSourceNames()
40 
+            specObj.listPatches=spec.getPatchNames()
41 
+            for specPkg in specObj.listPackages:
42 
+                specObj.installRequiresPackages[specPkg]=spec.getRequires(specPkg)
43 
+                self.mapPackageToSpec[specPkg]=specName
44 
+            self.mapSerializableSpecObjects[specName]=specObj
45 
+
46 
+    def getListSpecFiles(self,listSpecFiles,path):
47 
+        for dirEntry in os.listdir(path):
48 
+            dirEntryPath = os.path.join(path, dirEntry)
49 
+            if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(".spec"):
50 
+                listSpecFiles.append(dirEntryPath)
51 
+            elif os.path.isdir(dirEntryPath):
52 
+                self.getListSpecFiles(listSpecFiles,dirEntryPath)
53 
+
54 
+    def getBuildRequiresForPackage(self, package):
55 
+        specName=""
56 
+        if self.mapPackageToSpec.has_key(package):
57 
+            specName=self.mapPackageToSpec[package]
58 
+        if self.mapSerializableSpecObjects.has_key(specName):
59 
+            return self.mapSerializableSpecObjects[specName].buildRequirePackages
60 
+        return None
61 
+
62 
+    def getRequiresAllForPackage(self, package):
63 
+        specName=""
64 
+        if self.mapPackageToSpec.has_key(package):
65 
+            specName=self.mapPackageToSpec[package]
66 
+        if self.mapSerializableSpecObjects.has_key(specName):
67 
+            return self.mapSerializableSpecObjects[specName].installRequiresAllPackages
68 
+        return None
69 
+
70 
+    def getRequiresForPackage(self, package):
71 
+        specName=""
72 
+        if self.mapPackageToSpec.has_key(package):
73 
+            specName=self.mapPackageToSpec[package]
74 
+        if self.mapSerializableSpecObjects.has_key(specName):
75 
+            if self.mapSerializableSpecObjects[specName].installRequiresPackages.has_key(package):
76 
+                return self.mapSerializableSpecObjects[specName].installRequiresPackages[package]
77 
+        return None
78 
+
79 
+    def getRelease(self, package):
80 
+        specName=""
81 
+        if self.mapPackageToSpec.has_key(package):
82 
+            specName=self.mapPackageToSpec[package]
83 
+        if self.mapSerializableSpecObjects.has_key(specName):
84 
+            return self.mapSerializableSpecObjects[specName].release
85 
+        return None
86 
+
87 
+    def getVersion(self, package):
88 
+        specName=""
89 
+        if self.mapPackageToSpec.has_key(package):
90 
+            specName=self.mapPackageToSpec[package]
91 
+        if self.mapSerializableSpecObjects.has_key(specName):
92 
+            return self.mapSerializableSpecObjects[specName].version
93 
+        return None
94 
+
95 
+    def getSpecFile(self, package):
96 
+        specName=""
97 
+        if self.mapPackageToSpec.has_key(package):
98 
+            specName=self.mapPackageToSpec[package]
99 
+        if self.mapSerializableSpecObjects.has_key(specName):
100 
+            return self.mapSerializableSpecObjects[specName].specFile
101 
+        return None
102 
+
103 
+    def getSpecName(self, package):
104 
+        specName=""
105 
+        if self.mapPackageToSpec.has_key(package):
106 
+            specName=self.mapPackageToSpec[package]
107 
+        if self.mapSerializableSpecObjects.has_key(specName):
108 
+            return self.mapSerializableSpecObjects[specName].name
109 
+        return None
110 
+
111 
+    def getPatches(self, package):
112 
+        specName=""
113 
+        if self.mapPackageToSpec.has_key(package):
114 
+            specName=self.mapPackageToSpec[package]
115 
+        if self.mapSerializableSpecObjects.has_key(specName):
116 
+            return self.mapSerializableSpecObjects[specName].listPatches
117 
+        return None
118 
+
119 
+    def getSources(self, package):
120 
+        specName=""
121 
+        if self.mapPackageToSpec.has_key(package):
122 
+            specName=self.mapPackageToSpec[package]
123 
+        if self.mapSerializableSpecObjects.has_key(specName):
124 
+            return self.mapSerializableSpecObjects[specName].listSources
125 
+        return None
126 
+
127 
+    def getPackages(self, package):
128 
+        specName=""
129 
+        if self.mapPackageToSpec.has_key(package):
130 
+            specName=self.mapPackageToSpec[package]
131 
+        if self.mapSerializableSpecObjects.has_key(specName):
132 
+            return self.mapSerializableSpecObjects[specName].listPackages
133 
+        return None
134 
+
135 
+    def isValidPackage(self,package):
136 
+        if self.mapPackageToSpec.has_key(package):
137 
+            return True
138 
+        return False
139 
+
140 
+    def printAllObjects(self):
141 
+        listSpecs=self.mapSerializableSpecObjects.keys()
142 
+        for spec in listSpecs:
143 
+            specObj=self.mapSerializableSpecObjects[spec]
144 
+            self.logger.info("-----------Spec:"+specObj.name+"--------------")
145 
+            self.logger.info("Version:"+specObj.version)
146 
+            self.logger.info("Release:"+specObj.release)
147 
+            self.logger.info("SpecFile:"+specObj.specFile)
148 
+            self.logger.info(" ")
149 
+            self.logger.info("Source Files")
150 
+            self.logger.info(specObj.listSources)
151 
+            self.logger.info(" ")
152 
+            self.logger.info("Patch Files")
153 
+            self.logger.info(specObj.listPatches)
154 
+            self.logger.info(" ")
155 
+            self.logger.info(" ")
156 
+            self.logger.info("List RPM packages")
157 
+            self.logger.info(specObj.listPackages)
158 
+            self.logger.info(" ")
159 
+            self.logger.info(" ")
160 
+            self.logger.info("Build require packages")
161 
+            self.logger.info(specObj.buildRequirePackages)
162 
+            self.logger.info(" ")
163 
+            self.logger.info(" ")
164 
+            self.logger.info("install require packages")
165 
+            self.logger.info(specObj.installRequiresAllPackages)
166 
+            self.logger.info(" ")
167 
+            self.logger.info(specObj.installRequiresPackages)
168 
+            self.logger.info("------------------------------------------------")
169 
+
170 
+
171 
+
172 
+
0 173 
\ No newline at end of file
support/package-builder/SpecParser.py
History View file @ 2820c61
1 174 
new file mode 100644
... ... 
@@ -0,0 +1,292 @@
0 
+import re
1 
+from StringUtils import StringUtils
2 
+from SpecStructures import *
3 
+
4 
+class SpecParser(object):
5 
+    def __init__(self):
6 
+        self.cleanMacro=rpmMacro().setName("clean")
7 
+        self.prepMacro=rpmMacro().setName("prep")
8 
+        self.buildMacro=rpmMacro().setName("build")
9 
+        self.installMacro=rpmMacro().setName("install")
10 
+        self.changelogMacro=rpmMacro().setName("changelog")
11 
+        self.checkMacro=rpmMacro().setName("check")
12 
+        self.packages={}
13 
+        self.specAdditionalContent=""
14 
+
15 
+
16 
+    def readPkgNameFromPackageMacro(self,data,basePkgName=None):
17 
+        data=" ".join(data.split())
18 
+        pkgHeaderName=data.split(" ")
19 
+        lenpkgHeaderName = len(pkgHeaderName)
20 
+
21 
+        if (lenpkgHeaderName >= 3 and pkgHeaderName[1] != "-n"):
22 
+            lenpkgHeaderName = 1
23 
+        if (lenpkgHeaderName == 2 or lenpkgHeaderName == 1 ) and basePkgName is None :
24 
+            print "Invalid basePkgName"
25 
+            return False,None
26 
+        if lenpkgHeaderName == 3 :
27 
+            return True,pkgHeaderName[2]
28 
+        if lenpkgHeaderName == 2 :
29 
+            return True,basePkgName + "-"+pkgHeaderName[1]
30 
+        if lenpkgHeaderName == 1:
31 
+            return True, basePkgName
32 
+
33 
+    def parseSpecFile(self,specfile):
34 
+        self.createDefaultPackage()
35 
+        currentPkg="default"
36 
+        specFile = open(specfile)
37 
+        lines = specFile.readlines()
38 
+        totalLines=len(lines)
39 
+        i=0
40 
+        while i < totalLines:
41 
+            line = lines[i].strip()
42 
+            if self.isSpecMacro(line):
43 
+                macro,i=self.readMacroFromFile(i, lines)
44 
+                self.updateMacro(macro)
45 
+            elif self.isPackageMacro(line):
46 
+                defaultpkg = self.packages.get('default')
47 
+                returnVal,packageName=self.readPkgNameFromPackageMacro(line, defaultpkg.name)
48 
+                packageName=defaultpkg.decodeContents(packageName)
49 
+                if not returnVal:
50 
+                    return False
51 
+                if re.search('^'+'%package',line) :
52 
+                    pkg = Package(defaultpkg)
53 
+                    pkg.name=packageName
54 
+                    currentPkg=packageName
55 
+                    self.packages[pkg.name]=pkg
56 
+                else:
57 
+                    if defaultpkg.name == packageName :
58 
+                        packageName = 'default'
59 
+                    if not self.packages.has_key(packageName):
60 
+                        return False
61 
+                    macro,i=self.readMacroFromFile(i, lines)
62 
+                    self.packages[packageName].updatePackageMacro(macro)
63 
+            elif self.isPackageHeaders(line):
64 
+                self.readPackageHeaders(line, self.packages[currentPkg])
65 
+            else:
66 
+                self.specAdditionalContent+=line+"\n"
67 
+            i=i+1
68 
+        specFile.close()
69 
+
70 
+    def createDefaultPackage(self):
71 
+        pkg = Package()
72 
+        self.packages["default"]=pkg
73 
+
74 
+    def readMacroFromFile(self,currentPos,lines):
75 
+        macro = rpmMacro()
76 
+        line = lines[currentPos]
77 
+        macro.position = currentPos
78 
+        macro.endposition=currentPos
79 
+        endPos=len(lines)
80 
+        line = " ".join(line.split())
81 
+        flagindex = line.find(" ")
82 
+        if flagindex != -1:
83 
+            macro.macroFlag=line[flagindex+1:]
84 
+            macro.macroName=line[:flagindex]
85 
+        else:
86 
+            macro.macroName=line
87 
+
88 
+        if currentPos+1 < len(lines) and self.isMacro(lines[currentPos+1]):
89 
+            return macro,currentPos
90 
+
91 
+        for j in range(currentPos+1,endPos):
92 
+            content = lines[j]
93 
+            if j+1 < endPos and self.isMacro(lines[j+1]):
94 
+                return macro,j
95 
+            macro.content += content +'\n'
96 
+            macro.endposition=j
97 
+        return macro,endPos
98 
+
99 
+
100 
+    def updateMacro(self,macro):
101 
+        if macro.macroName == "%clean":
102 
+            self.cleanMacro=macro
103 
+            return True
104 
+        if macro.macroName == "%prep":
105 
+            self.prepMacro=macro
106 
+            return True
107 
+        if macro.macroName == "%build":
108 
+            self.buildMacro=macro
109 
+            return True
110 
+        if macro.macroName == "%install":
111 
+            self.installMacro=macro
112 
+            return True
113 
+        if macro.macroName == "%changelog":
114 
+            self.changelogMacro=macro
115 
+            return True
116 
+        if macro.macroName == "%check":
117 
+            self.checkMacro=macro
118 
+            return True
119 
+        return False
120 
+
121 
+    def isMacro(self,line):
122 
+        return self.isPackageMacro(line) or self.isSpecMacro(line)
123 
+
124 
+    def isSpecMacro(self,line):
125 
+        if re.search('^'+'%clean',line) :
126 
+            return True
127 
+        elif re.search('^'+'%prep',line) :
128 
+            return True
129 
+        elif re.search('^'+'%build',line) :
130 
+            return True
131 
+        elif re.search('^'+'%install',line) :
132 
+            return True
133 
+        elif re.search('^'+'%changelog',line) :
134 
+            return True
135 
+        elif re.search('^'+'%check',line) :
136 
+            return True
137 
+        return False
138 
+
139 
+    def isPackageMacro(self,line):
140 
+        line=line.strip()
141 
+
142 
+        if re.search('^'+'%post',line) :
143 
+            return True
144 
+        elif re.search('^'+'%postun',line) :
145 
+            return True
146 
+        elif re.search('^'+'%files',line) :
147 
+            return True
148 
+        elif re.search('^'+'%description',line) :
149 
+            return True
150 
+        elif re.search('^'+'%package',line) :
151 
+            return True
152 
+        return False
153 
+
154 
+    def isPackageHeaders(self,line):
155 
+        if re.search('^'+'summary:',line,flags=re.IGNORECASE) :
156 
+            return True
157 
+        elif re.search('^'+'name:',line,flags=re.IGNORECASE) :
158 
+            return True
159 
+        elif re.search('^'+'group:',line,flags=re.IGNORECASE) :
160 
+            return True
161 
+        elif re.search('^'+'license:',line,flags=re.IGNORECASE) :
162 
+            return True
163 
+        elif re.search('^'+'version:',line,flags=re.IGNORECASE) :
164 
+            return True
165 
+        elif re.search('^'+'release:',line,flags=re.IGNORECASE) :
166 
+            return True
167 
+        elif re.search('^'+'distribution:',line,flags=re.IGNORECASE) :
168 
+            return True
169 
+        elif re.search('^'+'requires:',line,flags=re.IGNORECASE) :
170 
+            return True
171 
+        elif re.search('^'+'provides:',line,flags=re.IGNORECASE) :
172 
+            return True
173 
+        elif re.search('^'+'obsoletes:',line,flags=re.IGNORECASE) :
174 
+            return True
175 
+        elif re.search('^'+'conflicts:',line,flags=re.IGNORECASE) :
176 
+            return True
177 
+        elif re.search('^'+'source[0-9]*:',line,flags=re.IGNORECASE) :
178 
+            return True
179 
+        elif re.search('^'+'patch[0-9]*:',line,flags=re.IGNORECASE) :
180 
+            return True
181 
+        elif re.search('^'+'buildrequires:',line,flags=re.IGNORECASE) :
182 
+            return True
183 
+        elif re.search('^'+'buildprovides:',line,flags=re.IGNORECASE) :
184 
+            return True
185 
+        elif re.search('^'+'buildarch:',line,flags=re.IGNORECASE) :
186 
+            return True
187 
+        return False
188 
+
189 
+    def readHeader(self,line):
190 
+        headerSplitIndex=line.find(":")
191 
+        if(headerSplitIndex+1 == len(line) ):
192 
+            print line
193 
+            print "Error:Invalid header"
194 
+            return False, None,None
195 
+        headerName=line[0:headerSplitIndex].lower()
196 
+        headerContent=line[headerSplitIndex+1:].strip()
197 
+        return True,headerName,headerContent
198 
+
199 
+
200 
+    def readDependentPackageData(self,line):
201 
+        strUtils = StringUtils()
202 
+        listPackages=line.split(",")
203 
+        listdependentpkgs=[]
204 
+        for line in listPackages:
205 
+            line=strUtils.getStringInBrackets(line)
206 
+            listContents=line.split()
207 
+            totalContents = len(listContents)
208 
+            i=0
209 
+            while i < totalContents:
210 
+                dpkg = dependentPackageData()
211 
+                compare=None
212 
+                if i+2 < len(listContents):
213 
+                    if listContents[i+1] == ">=":
214 
+                        compare="gte"
215 
+                    elif listContents[i+1] == "<=":
216 
+                        compare="lte"
217 
+                    elif listContents[i+1] == "==":
218 
+                        compare="eq"
219 
+                    elif listContents[i+1] == "<":
220 
+                        compare="lt"
221 
+                    elif listContents[i+1] == ">":
222 
+                        compare="gt"
223 
+                    elif listContents[i+1] == "=":
224 
+                        compare="eq"
225 
+
226 
+                if compare is not None:
227 
+                    dpkg.package=listContents[i]
228 
+                    dpkg.compare=compare
229 
+                    dpkg.version=listContents[i+2]
230 
+                    i=i+3
231 
+                else:
232 
+                    dpkg.package=listContents[i]
233 
+                    i=i+1
234 
+                listdependentpkgs.append(dpkg)
235 
+        return listdependentpkgs
236 
+
237 
+    def readPackageHeaders(self,line,pkg):
238 
+        returnVal,headerName,headerContent=self.readHeader(line)
239 
+        if not returnVal:
240 
+            return False
241 
+
242 
+        headerContent=pkg.decodeContents(headerContent)
243 
+        if headerName == 'summary':
244 
+            pkg.summary=headerContent
245 
+            return True
246 
+        if headerName == 'name':
247 
+            pkg.name=headerContent
248 
+            return True
249 
+        if headerName == 'group':
250 
+            pkg.group=headerContent
251 
+            return True
252 
+        if headerName == 'license':
253 
+            pkg.license=headerContent
254 
+            return True
255 
+        if headerName == 'version':
256 
+            pkg.version=headerContent
257 
+            return True
258 
+        if headerName == 'buildarch':
259 
+            pkg.buildarch=headerContent
260 
+            return True
261 
+        if headerName == 'release':
262 
+            pkg.release=headerContent
263 
+            return True
264 
+        if headerName == 'distribution':
265 
+            pkg.distribution=headerContent
266 
+            return True
267 
+        if headerName.find('source') != -1:
268 
+            pkg.sources.append(headerContent)
269 
+            return True
270 
+        if headerName.find('patch') != -1:
271 
+            pkg.patches.append(headerContent)
272 
+            return True
273 
+        if headerName == 'requires' or headerName == 'provides' or headerName == 'obsoletes' or headerName == 'conflicts' or headerName == 'buildrequires' or headerName == 'buildprovides':
274 
+            dpkg=self.readDependentPackageData(headerContent)
275 
+            if dpkg is None:
276 
+                return False
277 
+            if headerName == 'requires':
278 
+                pkg.requires.extend(dpkg)
279 
+            if headerName == 'provides':
280 
+                pkg.provides.extend(dpkg)
281 
+            if headerName == 'obsoletes':
282 
+                pkg.obsoletes.extend(dpkg)
283 
+            if headerName == 'conflicts':
284 
+                pkg.conflicts.extend(dpkg)
285 
+            if headerName == 'buildrequires':
286 
+                pkg.buildrequires.extend(dpkg)
287 
+            if headerName == 'buildprovides':
288 
+                pkg.buildprovides.extend(dpkg)
289 
+
290 
+            return True
291 
+        return False
support/package-builder/SpecStructures.py
History View file @ 2820c61
0 292 
new file mode 100644
... ... 
@@ -0,0 +1,91 @@
0 
+
1 
+class rpmMacro(object):
2 
+
3 
+    def __init__(self):
4 
+        self.macroName=""
5 
+        self.macroFlag=""
6 
+        self.content=""
7 
+        self.position=-1
8 
+        self.endposition=-1
9 
+
10 
+    def setName(self,name):
11 
+        self.macroName=name
12 
+
13 
+    def displayMacro(self):
14 
+        print "Macro:\n", self.macroName, " ",self.macroFlag," ",self.position," ",self.endposition
15 
+        print self.content
16 
+
17 
+class dependentPackageData(object):
18 
+
19 
+    def __init__(self):
20 
+        self.package=""
21 
+        self.version=""
22 
+        self.compare=""
23 
+
24 
+class Package(object):
25 
+    def __init__(self, basePkg=None):
26 
+        self.summary=""
27 
+        self.name=""
28 
+        self.group=""
29 
+        self.license=""
30 
+        self.version=""
31 
+        self.release=""
32 
+        self.buildarch="x86_64"
33 
+        self.distribution=""
34 
+        self.basePkgName=""
35 
+
36 
+        self.sources=[]
37 
+        self.patches=[]
38 
+        self.buildrequires=[]
39 
+        self.buildprovides=[]
40 
+
41 
+
42 
+        self.requires=[]
43 
+        self.provides=[]
44 
+        self.obsoletes=[]
45 
+        self.conflicts=[]
46 
+
47 
+        self.descriptionMacro= None
48 
+        self.postMacro=None
49 
+        self.postunMacro=None
50 
+        self.filesMacro=None
51 
+        self.packageMacro=None
52 
+
53 
+        if basePkg is not None:
54 
+            self.basePkgName=basePkg.name
55 
+            self.group=basePkg.group
56 
+            self.license=basePkg.license
57 
+            self.version=basePkg.version
58 
+            self.buildarch=basePkg.buildarch
59 
+            self.release=basePkg.release
60 
+            self.distribution=basePkg.distribution
61 
+
62 
+    def decodeContents(self,content):
63 
+        if content.find("%{name}") != -1:
64 
+            if self.basePkgName == "":
65 
+                content = content.replace('%{name}',self.name)
66 
+            else:
67 
+                content = content.replace('%{name}',self.basePkgName)
68 
+
69 
+        if content.find("%{release}") != -1:
70 
+            content = content.replace('%{release}',self.release)
71 
+
72 
+        if content.find("%{version}") != -1:
73 
+            content = content.replace('%{version}',self.version)
74 
+
75 
+        return content
76 
+
77 
+    def updatePackageMacro(self,macro):
78 
+        if macro.macroName == "%post":
79 
+            self.postMacro=macro
80 
+            return True
81 
+        if macro.macroName == "%postun":
82 
+            self.postunMacro=macro
83 
+            return True
84 
+        if macro.macroName == "%files":
85 
+            self.filesMacro=macro
86 
+            return True
87 
+        if macro.macroName == "%description":
88 
+            self.descriptionMacro = macro
89 
+            return True
90 
+        return False
support/package-builder/SpecUtils.py
History View file @ 2820c61
0 91 
new file mode 100644
... ... 
@@ -0,0 +1,184 @@
0 
+from SpecParser import SpecParser
1 
+from StringUtils import StringUtils
2 
+import os
3 
+
4 
+class Specutils(object):
5 
+
6 
+    def __init__(self,specfile):
7 
+        self.specfile=""
8 
+        self.spec = SpecParser()
9 
+        if self.isSpecFile(specfile):
10 
+            self.specfile=specfile
11 
+            self.spec.parseSpecFile(self.specfile)
12 
+
13 
+    def isSpecFile(self,specfile):
14 
+        if os.path.isfile(specfile) and specfile[-5:] == ".spec":
15 
+            return True
16 
+        return False
17 
+
18 
+    def getSourceNames(self):
19 
+        sourceNames=[]
20 
+        strUtils = StringUtils()
21 
+        pkg = self.spec.packages.get('default')
22 
+        if pkg is None:
23 
+            return None
24 
+        for source in pkg.sources:
25 
+            sourceName=strUtils.getFileNameFromURL(source)
26 
+            sourceNames.append(sourceName)
27 
+        return sourceNames
28 
+
29 
+    def getPatchNames(self):
30 
+        patchNames=[]
31 
+        strUtils = StringUtils()
32 
+        pkg = self.spec.packages.get('default')
33 
+        if pkg is None:
34 
+            return None
35 
+        for patch in pkg.patches:
36 
+            patchName=strUtils.getFileNameFromURL(patch)
37 
+            patchNames.append(patchName)
38 
+        return patchNames
39 
+
40 
+    def getPackageNames(self):
41 
+        packageNames=[]
42 
+        for key in self.spec.packages.keys():
43 
+            pkg = self.spec.packages.get(key)
44 
+            packageNames.append(pkg.name)
45 
+        return packageNames
46 
+
47 
+    def getRPMNames(self):
48 
+        rpmNames=[]
49 
+        for key in self.spec.packages.keys():
50 
+            pkg = self.spec.packages.get(key)
51 
+            rpmName=pkg.name+"-"+pkg.version+"-"+pkg.release
52 
+            rpmNames.append(rpmName)
53 
+        return rpmNames
54 
+
55 
+    def getRPMName(self, pkgName):
56 
+        rpmName=None
57 
+        for key in self.spec.packages.keys():
58 
+            pkg = self.spec.packages.get(key)
59 
+            if pkg.name == pkgName:
60 
+                rpmName=pkg.name+"-"+pkg.version+"-"+pkg.release
61 
+                break
62 
+        return rpmName
63 
+
64 
+    def getRPMVersion(self, pkgName):
65 
+        version=None
66 
+        for key in self.spec.packages.keys():
67 
+            pkg = self.spec.packages.get(key)
68 
+            if pkg.name == pkgName:
69 
+                version=pkg.version
70 
+                break
71 
+        return version
72 
+
73 
+    def getRPMRelease(self, pkgName):
74 
+        release=None
75 
+        for key in self.spec.packages.keys():
76 
+            pkg = self.spec.packages.get(key)
77 
+            if pkg.name == pkgName:
78 
+                release=pkg.release
79 
+                break
80 
+        return release
81 
+
82 
+    def getLicense(self, pkgName):
83 
+        licenseInfo=None
84 
+        for key in self.spec.packages.keys():
85 
+            pkg = self.spec.packages.get(key)
86 
+            if pkg.name == pkgName:
87 
+                licenseInfo=pkg.license
88 
+                break
89 
+        return licenseInfo
90 
+
91 
+    def getBuildArch(self, pkgName):
92 
+        buildArch="x86_64"
93 
+        for key in self.spec.packages.keys():
94 
+            pkg = self.spec.packages.get(key)
95 
+            if pkg.name == pkgName:
96 
+                buildArch=pkg.buildarch
97 
+                break
98 
+        return buildArch
99 
+
100 
+    def getRequiresAllPackages(self):
101 
+        depedentPackages=[]
102 
+        for key in self.spec.packages.keys():
103 
+            pkg = self.spec.packages.get(key)
104 
+            for dpkg in pkg.requires:
105 
+                depedentPackages.append(dpkg.package)
106 
+        depedentPackages=list(set(depedentPackages))
107 
+        packageNames=self.getPackageNames()
108 
+        for pkgName in packageNames:
109 
+            if pkgName in depedentPackages:
110 
+                depedentPackages.remove(pkgName)
111 
+        return depedentPackages
112 
+
113 
+    def getBuildRequiresAllPackages(self):
114 
+        depedentPackages=[]
115 
+        for key in self.spec.packages.keys():
116 
+            pkg = self.spec.packages.get(key)
117 
+            for dpkg in pkg.buildrequires:
118 
+                depedentPackages.append(dpkg.package)
119 
+        depedentPackages=list(set(depedentPackages))
120 
+        packageNames=self.getPackageNames()
121 
+        for pkgName in packageNames:
122 
+            if pkgName in depedentPackages:
123 
+                depedentPackages.remove(pkgName)
124 
+        return depedentPackages
125 
+
126 
+
127 
+    def getRequires(self,pkgName):
128 
+        dependentPackages=[]
129 
+        for key in self.spec.packages.keys():
130 
+            pkg = self.spec.packages.get(key)
131 
+            if pkg.name == pkgName:
132 
+                for dpkg in pkg.requires:
133 
+                    dependentPackages.append(dpkg.package)
134 
+        return dependentPackages
135 
+
136 
+    def getBuildRequires(self,pkgName):
137 
+        dependentPackages=[]
138 
+        for key in self.spec.packages.keys():
139 
+            pkg = self.spec.packages.get(key)
140 
+            if pkg.name == pkgName:
141 
+                for dpkg in pkg.buildrequires:
142 
+                    dependentPackages.append(dpkg.package)
143 
+        return dependentPackages
144 
+
145 
+    def getProvides(self,packageName):
146 
+        depedentPackages=[]
147 
+        defaultPkgName=self.spec.packages['default'].name
148 
+        pkg = None
149 
+        if self.spec.packages.has_key(packageName):
150 
+            pkg = self.spec.packages.get(packageName)
151 
+        if defaultPkgName == packageName:
152 
+            pkg=self.spec.packages['default']
153 
+        if pkg is not None:
154 
+            for dpkg in pkg.provides:
155 
+                depedentPackages.append(dpkg.package)
156 
+        else:
157 
+            print "package not found"
158 
+        return depedentPackages
159 
+
160 
+    def getVersion(self):
161 
+        pkg = self.spec.packages.get('default')
162 
+        return pkg.version
163 
+
164 
+    def getRelease(self):
165 
+        pkg = self.spec.packages.get('default')
166 
+        return pkg.release
167 
+
168 
+def main():
169 
+    spec = Specutils("/workspace1/myrepos/photon/SPECS/docker/docker.spec")
170 
+    print "packages",spec.getPackageNames()
171 
+    print "packages",spec.getRPMNames()
172 
+    print "sources",spec.getSourceNames()
173 
+    print "patches",spec.getPatchNames()
174 
+    print "requires",spec.getRequires('libltdl-devel')
175 
+    print "requires",spec.getRequires('libtool')
176 
+
177 
+    print "provides",spec.getProvides('libtool')
178 
+    print "all-requires",spec.getRequiresAllPackages()
179 
+    print "all-build-requires",spec.getBuildRequiresAllPackages()
180 
+
181 
+if __name__ == '__main__':
182 
+    main()
183 
+
support/package-builder/StringUtils.py
History View file @ 2820c61
0 184 
new file mode 100644
... ... 
@@ -0,0 +1,14 @@
0 
+import re
1 
+
2 
+class StringUtils(object):
3 
+
4 
+    def getStringInBrackets(self, inputstring):
5 
+        inputstring=inputstring.strip()
6 
+        m = re.search(r"^\(([A-Za-z0-9_.-]+)\)",  inputstring)
7 
+        if m is None:
8 
+            return inputstring
9 
+        return m.group(1)
10 
+
11 
+    def getFileNameFromURL(self,inputstring):
12 
+        index=inputstring.rfind("/")
13 
+        return inputstring[index+1:]
0 14 
\ No newline at end of file
support/package-builder/ToolChainUtils.py
History View file @ 2820c61
1 15 
new file mode 100644
... ... 
@@ -0,0 +1,151 @@
0 
+from CommandUtils import CommandUtils
1 
+from ChrootUtils import ChrootUtils
2 
+from Logger import Logger
3 
+from PackageUtils import PackageUtils
4 
+import shutil
5 
+from constants import constants
6 
+import subprocess
7 
+
8 
+class ToolChainUtils(object):
9 
+    __built_successfull=False
10 
+
11 
+    def __init__(self,logName=None,logPath=None):
12 
+        if logName is None:
13 
+            logName = "Toolchain Utils"
14 
+        if logPath is None:
15 
+            logPath = constants.logPath
16 
+        self.logName=logName
17 
+        self.logPath=logPath
18 
+        self.logger=Logger.getLogger(logName,logPath)
19 
+        self.adjustToolChainScript = "adjust-tool-chain.sh"
20 
+        self.localegenScript = "./locale-gen.sh"
21 
+        self.localegenConfig = "./locale-gen.conf"
22 
+        self.prepareBuildRootCmd="./prepare-build-root.sh"
23 
+
24 
+
25 
+        self.listPkgsRequiredToBuildToolchain=["linux-api-headers", "glibc","glibc-devel",  "zlib","zlib-devel",  "file",
26 
+            "binutils","binutils-devel",  "gmp","gmp-devel", "mpfr", "mpfr-devel", "mpc",
27 
+            "libgcc","libgcc-devel","libstdc++","libstdc++-devel","libgomp","libgomp-devel","gcc",
28 
+            "pkg-config", "ncurses", "bash", "bzip2", "sed","ncurses-devel","procps-ng","coreutils", "m4","grep",
29 
+            "readline", "diffutils","gawk", "findutils", "gettext", "gzip","make",  "patch",
30 
+            "util-linux", "tar", "xz","libtool", "flex",  "bison",
31 
+            "readline-devel", "lua","lua-devel","popt","popt-devel","nspr","sqlite-autoconf","nss","nss-devel",
32 
+            "bzip2-devel","elfutils-libelf","elfutils","elfutils-libelf-devel","elfutils-devel",
33 
+            "expat","libffi","libpipeline", "gdbm","perl","texinfo","autoconf","automake",
34 
+            "openssl","openssl-devel","python2","python2-libs","python2-devel","rpm",
35 
+            "groff", "man-db", "man-pages","cpio"]
36 
+
37 
+        self.listPkgsToInstallToolChain=["linux-api-headers", "glibc","glibc-devel",  "zlib","zlib-devel",  "file",
38 
+            "binutils","binutils-devel",  "gmp","gmp-devel", "mpfr", "mpfr-devel", "mpc",
39 
+            "libgcc","libgcc-devel","libstdc++","libstdc++-devel","libgomp","libgomp-devel","gcc",
40 
+            "pkg-config", "ncurses", "bash", "bzip2", "sed","procps-ng","coreutils", "m4","grep",
41 
+            "readline","diffutils","gawk", "findutils", "gettext", "gzip","make",  "patch",
42 
+            "util-linux", "tar", "xz","libtool", "flex",  "bison",
43 
+            "lua","popt","nspr","sqlite-autoconf","nss","elfutils-libelf",
44 
+            "libpipeline", "gdbm","perl","texinfo","rpm",
45 
+            "autoconf","automake", "groff", "man-db", "man-pages","elfutils","cpio"]
46 
+
47 
+
48 
+
49 
+    def prepareChroot(self,chrootID,toolsArchive=""):
50 
+        process = subprocess.Popen([self.prepareBuildRootCmd, chrootID,  constants.specPath,  constants.rpmPath, constants.toolsPath,toolsArchive])
51 
+        returnVal = process.wait()
52 
+        if returnVal != 0:
53 
+            self.logger.error("Unable to prepare chroot.")
54 
+            return False
55 
+
56 
+        cmdUtils=CommandUtils()
57 
+        cmdUtils.run_command("mkdir -p "+chrootID+constants.topDirPath+"/RPMS")
58 
+        cmdUtils.run_command("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/x86_64")
59 
+        cmdUtils.run_command("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/noarch")
60 
+        cmdUtils.run_command("mkdir -p "+chrootID+constants.topDirPath+"/SOURCES")
61 
+        cmdUtils.run_command("mkdir -p "+chrootID+constants.topDirPath+"/SPECS")
62 
+        self.logger.info("Successfully prepared chroot:"+chrootID)
63 
+        return True
64 
+
65 
+    def buildToolChain(self):
66 
+        chrUtils = ChrootUtils(self.logName,self.logPath)
67 
+        returnVal,chrootID = chrUtils.createChroot()
68 
+        if not returnVal:
69 
+            self.logger.error("Unable to build tool chain.")
70 
+            chrUtils.destroyChroot(chrootID)
71 
+            return False
72 
+        if not self.prepareChroot(chrootID):
73 
+            self.logger.error("Unable to build tool chain.")
74 
+            chrUtils.destroyChroot(chrootID)
75 
+            return False
76 
+
77 
+        pkgUtils=PackageUtils(self.logName,self.logPath)
78 
+        for package in self.listPkgsRequiredToBuildToolchain:
79 
+            rpmPkg=pkgUtils.findRPMFileForGivenPackage(package)
80 
+            if rpmPkg is None:
81 
+                #directly push them to golden repo, no test required
82 
+                returnVal = pkgUtils.buildRPMSForGivenPackage(package, chrootID)
83 
+                if not returnVal:
84 
+                    self.logger.error("Stop building toolchain")
85 
+                    chrUtils.destroyChroot(chrootID)
86 
+                    return False
87 
+            returnVal = pkgUtils.installRPM(package, chrootID, True)
88 
+            if not returnVal:
89 
+                self.logger.error("Stop building toolchain")
90 
+                chrUtils.destroyChroot(chrootID)
91 
+                return False
92 
+            if package == "glibc":
93 
+                if not self.adjustToolChain(chrootID):
94 
+                    self.logger.error("Adjusting tool chain failed. Stop building toolchain")
95 
+                    chrUtils.destroyChroot(chrootID)
96 
+                    return False
97 
+        self.logger.info("Successfully built toolchain")
98 
+        chrUtils.destroyChroot(chrootID)
99 
+
100 
+    #tool chain should be updated before calling this method
101 
+    def installToolChain(self,chrootID):
102 
+        if not self.prepareChroot(chrootID,"minimal"):
103 
+            self.logger.error("Unable to install tool chain.")
104 
+            return False
105 
+        pkgUtils= PackageUtils(self.logName,self.logPath)
106 
+        for package in self.listPkgsToInstallToolChain:
107 
+            returnVal = pkgUtils.installRPM(package, chrootID, True)
108 
+            if not returnVal:
109 
+                self.logger.error("Error while installing toolchain on chroot:"+chrootID)
110 
+                return False
111 
+            if package == "glibc":
112 
+                if not self.adjustToolChain(chrootID):
113 
+                    self.logger.error("Adjusting tool chain failed. Stop building toolchain")
114 
+                    return False
115 
+        cmdUtils=CommandUtils()
116 
+        returnVal = cmdUtils.run_command("rm -rf "+ chrootID+"/tools")
117 
+        if not returnVal:
118 
+            self.logger.error("Error while removing toolchain tar from chroot:"+chrootID)
119 
+            return False
120 
+        returnVal = cmdUtils.run_command("rm "+ chrootID+"/"+constants.topDirPath+"/RPMS/x86_64/*")
121 
+        returnVal = cmdUtils.run_command("rm "+ chrootID+"/"+constants.topDirPath+"/RPMS/noarch/*")
122 
+        self.logger.info("Installed tool chain successfully on chroot:"+chrootID)
123 
+        return True
124 
+
125 
+    def adjustToolChain(self,chrootID):
126 
+        shutil.copy2(self.adjustToolChainScript,  chrootID+"/tmp")
127 
+        shutil.copy2(self.localegenScript,  chrootID+"/sbin")
128 
+        shutil.copy2(self.localegenConfig,  chrootID+"/etc")
129 
+        cmdUtils=CommandUtils()
130 
+        logFile=constants.logPath+"/adjustToolChainScript.log"
131 
+        returnVal = cmdUtils.run_command("/tmp/"+self.adjustToolChainScript,logFile, "./run-in-chroot.sh "+chrootID)
132 
+        return returnVal
133 
+
134 
+
135 
+
136 
+
137 
+
138 
+
139 
+
140 
+
141 
+
142 
+
143 
+
144 
+
145 
+
146 
+
147 
+
148 
+
149 
+
150 
+
support/package-builder/build_package.py
History View file @ 2820c61
0 151 
deleted file mode 100755
... ... 
@@ -1,680 +0,0 @@
1 
-#! /usr/bin/python2
2 
-
3 
-from optparse import OptionParser
4 
-from Specutils import Specutils
5 
-import shutil
6 
-import subprocess
7 
-import os
8 
-import re
9 
-from sets import Set
10 
-
11 
-
12 
-def getListSpecFiles(listSpecFiles,path):
13 
-    for dir_entry in os.listdir(path):
14 
-        dir_entry_path = os.path.join(path, dir_entry)
15 
-        if os.path.isfile(dir_entry_path) and dir_entry_path.endswith(".spec"):
16 
-            listSpecFiles.append(dir_entry_path)
17 
-        elif os.path.isdir(dir_entry_path):
18 
-            getListSpecFiles(listSpecFiles,dir_entry_path)
19 
-
20 
-def getListPackages(specPath):
21 
-    list_packages=[]
22 
-    listSpecFiles=[]
23 
-    getListSpecFiles(listSpecFiles,specPath)
24 
-    for specFile in listSpecFiles:
25 
-        spec = Specutils(specFile)
26 
-        l = spec.getPackageNames()
27 
-        for pkg in l:
28 
-            list_packages.append(pkg)
29 
-    return list_packages
30 
-
31 
-class commandsUtils(object):
32 
-    def __init__(self):
33 
-        self.rpmbuild_binary = "rpmbuild"
34 
-        self.rpmbuild_buildall_option = "-ba"
35 
-        self.rpmbuild_nocheck_option = "--nocheck"
36 
-        self.rpm_binary = "rpm"
37 
-        self.install_rpm_package_options = "-Uvh"
38 
-        self.nodeps_rpm_package_options = "--nodeps"
39 
-        self.query_rpm_package_options = "-qa"
40 
-        self.force_rpm_package_options = "--force"
41 
-        self.find_command = "find"
42 
-
43 
-
44 
-    def find_file (self, filename, sourcePath):
45 
-        process = subprocess.Popen([self.find_command,  sourcePath,  "-name", filename],  stdout=subprocess.PIPE)
46 
-        returnVal = process.wait()
47 
-        result=process.communicate()[0]
48 
-        if result is None:
49 
-            return None
50 
-        return result.split()
51 
-
52 
-    def run_command(self,cmd,logfilePath=None,chrootCmd=None):
53 
-        if chrootCmd is not None:
54 
-            cmd = chrootCmd+" "+cmd
55 
-        if logfilePath is None:
56 
-            logfilePath=os.devnull
57 
-        logfile=open(logfilePath,"w")
58 
-        process = subprocess.Popen("%s" %cmd,shell=True,stdout=logfile)
59 
-        retval = process.wait()
60 
-        logfile.close()
61 
-        if retval==0:
62 
-            return True
63 
-        return False
64 
-
65 
-    def find_installed_rpm_packages(self, chrootCmd=None):
66 
-        cmd = self.rpm_binary+" "+self.query_rpm_package_options
67 
-        if chrootCmd is not None:
68 
-            cmd = chrootCmd+" "+cmd
69 
-        #process may hang if buffer is full
70 
-        process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE)
71 
-        retval = process.wait()
72 
-        result = process.communicate()[0]
73 
-        return result.split()
74 
-
75 
-    def installRPM(self, rpmfilepath,logfilePath, chrootCmd=None, withNodeps=False):
76 
-        print "Installing RPM...",rpmfilepath
77 
-        rpmInstallcmd=self.rpm_binary+" "+ self.install_rpm_package_options
78 
-
79 
-        if withNodeps:
80 
-            rpmInstallcmd+=" "+self.nodeps_rpm_package_options
81 
-
82 
-        rpmInstallcmd+=" "+rpmfilepath
83 
-        if chrootCmd is not None:
84 
-            rpmInstallcmd= chrootCmd +" "+rpmInstallcmd
85 
-        logfile=open(logfilePath,'w')
86 
-        process = subprocess.Popen("%s" %rpmInstallcmd,shell=True,stdout=logfile,stderr=logfile)
87 
-        retval = process.wait()
88 
-        logfile.close()
89 
-        if retval != 0:
90 
-            return False
91 
-        return True
92 
-
93 
-    def buildRPM(self, specfilepath, logfilePath, chrootCmd=None):
94 
-        print "Building RPM...",specfilepath
95 
-        rpmBuildcmd= self.rpmbuild_binary+" "+self.rpmbuild_buildall_option+" "+self.rpmbuild_nocheck_option
96 
-        rpmBuildcmd+=" "+specfilepath
97 
-        if chrootCmd is not None:
98 
-            rpmBuildcmd=chrootCmd +" "+rpmBuildcmd
99 
-
100 
-        logfile=open(logfilePath,'w')
101 
-        process = subprocess.Popen("%s" %rpmBuildcmd,shell=True,stdout=logfile,stderr=logfile)
102 
-        retval = process.wait()
103 
-        logfile.close()
104 
-        if retval != 0:
105 
-            print "Building rpm is failed"
106 
-            return False, None
107 
-
108 
-        #Extracting rpms created from log file
109 
-        logfile=open(logfilePath,'r')
110 
-        fileContents=logfile.readlines()
111 
-        logfile.close()
112 
-        listRpms=[]
113 
-        for i in range(0,len(fileContents)):
114 
-            if re.search("^Wrote:",fileContents[i]):
115 
-                listcontents=fileContents[i].split()
116 
-                if (len(listcontents) == 2) and listcontents[1].strip()[-4:] == ".rpm" and listcontents[1].find("/RPMS/") != -1:
117 
-                    listRpms.append(listcontents[1])
118 
-
119 
-        return True, listRpms
120 
-
121 
-class BuildSystem(object):
122 
-    def __init__(self, source_path,  spec_path,  rpm_path,  build_root, tools_path, log_path):
123 
-        self.data = []
124 
-        self.source_path = source_path
125 
-        self.spec_path = spec_path
126 
-        self.rpm_path = rpm_path
127 
-        self.tools_path = tools_path
128 
-        self.build_root = build_root
129 
-
130 
-        self.parent_path="/usr/src/photon"
131 
-        self.prepare_buildroot_command = "./prepare-build-root.sh"
132 
-        self.run_in_chroot_command = "./run-in-chroot.sh"
133 
-	# do not add '/' at the beginning. It's used for concat.
134 
-        self.adjust_tool_chain_script = "adjust-tool-chain.sh"
135 
-        self.adjust_gcc_specs_script = "adjust-gcc-specs.sh"
136 
-        self.localegen_script = "./locale-gen.sh"
137 
-        self.localegen_config = "./locale-gen.conf"
138 
-
139 
-        self.mapPackageToSpecFile={}
140 
-        self.listInstalledPackages=[]
141 
-
142 
-        self.build_root_log_path=log_path
143 
-        self.build_root_rpm_path=self.build_root+self.parent_path+"/RPMS"
144 
-        self.build_root_source_path=self.build_root+self.parent_path+"/SOURCES"
145 
-        self.build_root_spec_path=self.build_root+self.parent_path+"/SPECS"
146 
-
147 
-        self.listPkgsRequiredToBuildToolchain=["linux-api-headers", "glibc","glibc-devel",  "zlib","zlib-devel",  "file",
148 
-            "binutils","binutils-devel",  "gmp","gmp-devel", "mpfr", "mpfr-devel", "mpc",
149 
-            "libgcc","libgcc-devel","libstdc++","libstdc++-devel","libgomp","libgomp-devel","gcc",
150 
-            "pkg-config", "ncurses", "bash", "bzip2", "sed","ncurses-devel","procps-ng","coreutils", "m4","grep",
151 
-            "readline", "diffutils","gawk", "findutils", "gettext", "gzip","make",  "patch",
152 
-            "util-linux", "tar", "xz","libtool", "flex",  "bison",
153 
-            "readline-devel", "lua","lua-devel","popt","popt-devel","nspr","sqlite-autoconf","nss","nss-devel",
154 
-            "bzip2-devel","elfutils-libelf","elfutils","elfutils-libelf-devel","elfutils-devel",
155 
-            "expat","libffi","libpipeline", "gdbm","perl","texinfo","autoconf","automake",
156 
-            "openssl","openssl-devel","python2","python2-libs","python2-devel","rpm",
157 
-            "groff", "man-db", "man-pages","cpio"]
158 
-
159 
-        self.listPkgsToInstallToolChain=["linux-api-headers", "glibc","glibc-devel",  "zlib","zlib-devel",  "file",
160 
-            "binutils","binutils-devel",  "gmp","gmp-devel", "mpfr", "mpfr-devel", "mpc",
161 
-            "libgcc","libgcc-devel","libstdc++","libstdc++-devel","libgomp","libgomp-devel","gcc",
162 
-            "pkg-config", "ncurses", "bash", "bzip2", "sed","procps-ng","coreutils", "m4","grep",
163 
-            "readline","diffutils","gawk", "findutils", "gettext", "gzip","make",  "patch",
164 
-            "util-linux", "tar", "xz","libtool", "flex",  "bison",
165 
-            "lua","popt","nspr","sqlite-autoconf","nss","elfutils-libelf",
166 
-            "libpipeline", "gdbm","perl","texinfo","rpm",
167 
-            "autoconf","automake", "groff", "man-db", "man-pages","elfutils","cpio"]
168 
-
169 
-        self.list_nodeps_packages = ["glibc","gmp","zlib","file","binutils","mpfr","mpc","gcc","ncurses","util-linux","groff","perl","texinfo","rpm","openssl","go"]
170 
-        self.readPackagesInSpecFiles()
171 
-
172 
-    def prepare_build_root(self, tools_archive=""):
173 
-        process = subprocess.Popen([self.prepare_buildroot_command, self.build_root,  self.spec_path,  self.rpm_path, self.tools_path, tools_archive])
174 
-        returnVal = process.wait()
175 
-        if returnVal != 0:
176 
-            return False
177 
-        cmdUtils=commandsUtils()
178 
-        if not os.path.isdir(self.build_root_log_path):
179 
-            cmdUtils.run_command("mkdir -p "+self.build_root_log_path)
180 
-        if not os.path.isdir(self.build_root_rpm_path):
181 
-            cmdUtils.run_command("mkdir -p "+self.build_root_rpm_path)
182 
-        if not os.path.isdir(self.build_root_source_path):
183 
-            cmdUtils.run_command("mkdir -p "+self.build_root_source_path)
184 
-        if not os.path.isdir(self.build_root_spec_path):
185 
-            cmdUtils.run_command("mkdir -p "+self.build_root_spec_path)
186 
-        return True
187 
-    def adjust_tool_chain(self):
188 
-        shutil.copy2(self.adjust_tool_chain_script,  self.build_root+"/tmp/"+self.adjust_tool_chain_script)
189 
-        shutil.copy2(self.localegen_script,  self.build_root+"/sbin/locale-gen.sh")
190 
-        shutil.copy2(self.localegen_config,  self.build_root+"/etc/locale-gen.conf")
191 
-        cmdUtils=commandsUtils()
192 
-        returnVal = cmdUtils.run_command("/tmp/"+self.adjust_tool_chain_script,"/var/log/adjust_tool_chain_script.log", self.run_in_chroot_command+" "+self.build_root)
193 
-        return returnVal
194 
-
195 
-    def adjust_gcc_specs(self, package):
196 
-        opt = ""
197 
-        # linux package does not require sec gcc options
198 
-        if package == "linux" or package == "glibc":
199 
-            opt = " clean"
200 
-
201 
-        shutil.copy2(self.adjust_gcc_specs_script,  self.build_root+"/tmp/"+self.adjust_gcc_specs_script)
202 
-        cmdUtils=commandsUtils()
203 
-        returnVal = cmdUtils.run_command("/tmp/"+self.adjust_gcc_specs_script+opt,"/var/log/adjust_gcc_specs_script.log", self.run_in_chroot_command+" "+self.build_root)
204 
-        return returnVal
205 
-
206 
-    def readPackagesInSpecFiles(self):
207 
-        self.mapPackageToSpecFile.clear()
208 
-        listSpecFiles=[]
209 
-        getListSpecFiles(listSpecFiles, self.spec_path)
210 
-        for specfile in listSpecFiles:
211 
-            specparser = Specutils(specfile)
212 
-            listPackages=specparser.getPackageNames()
213 
-            for pkg in listPackages:
214 
-                self.mapPackageToSpecFile[pkg]=specfile
215 
-
216 
-    def findSpecFileForPackage(self,package):
217 
-        package=package.strip()
218 
-        if self.mapPackageToSpecFile.has_key(package):
219 
-            return self.mapPackageToSpecFile[package]
220 
-        return None
221 
-
222 
-    def findPackageNameFromRPMFile(self,rpmfile):
223 
-        rpmfile=os.path.basename(rpmfile)
224 
-        releaseindex=rpmfile.rfind("-")
225 
-        if releaseindex == -1:
226 
-            print "Invalid rpm file:",rpmfile
227 
-            return None
228 
-        versionindex=rpmfile[0:releaseindex].rfind("-")
229 
-        if versionindex == -1:
230 
-            print "Invalid rpm file:",rpmfile
231 
-            return None
232 
-        packageName=rpmfile[0:versionindex]
233 
-        return packageName
234 
-
235 
-    def findRPMFileForGivenPackage(self,package):
236 
-        cmdUtils = commandsUtils()
237 
-        specFile=self.findSpecFileForPackage(package)
238 
-        if specFile is None:
239 
-            print "Did not find spec file for package: ", package
240 
-            return None
241 
-        specparser = Specutils(specFile)
242 
-        version = specparser.getVersion()
243 
-        release = specparser.getRelease()
244 
-        listFoundRPMFiles = cmdUtils.find_file(package+"-"+version+"-"+release+"*.rpm",self.rpm_path)
245 
-        if len(listFoundRPMFiles) == 1 :
246 
-            return listFoundRPMFiles[0]
247 
-        if len(listFoundRPMFiles) == 0 :
248 
-            return None
249 
-        if len(listFoundRPMFiles) > 1 :
250 
-            print "Unable to determine the rpm file for package. Found multiple rpm files for given package in rpm directory"
251 
-            return None
252 
-        return None
253 
-
254 
-    def copySourcesTobuildroot(self,listSourceFiles,package):
255 
-        cmdUtils = commandsUtils()
256 
-        for source in listSourceFiles:
257 
-            sourcePath = cmdUtils.find_file(source,self.source_path)
258 
-            if sourcePath is None or len(sourcePath) == 0:
259 
-                sourcePath = cmdUtils.find_file(source,self.spec_path)
260 
-            if sourcePath is None or len(sourcePath) == 0:
261 
-                print "Missing source: "+source+". Cannot find sources for package: ", package
262 
-                print "Stopping building toolchain"
263 
-                return False
264 
-            if len(sourcePath) > 1:
265 
-                print "Multiple sources found: Unable to determine one. "
266 
-                print "Stopping building toolchain"
267 
-                return False
268 
-            print "Source path :" + source + " Source filename: " + sourcePath[0]
269 
-            shutil.copyfile(sourcePath[0],  self.build_root_source_path+"/"+source)
270 
-
271 
-    def buildRPMSForGivenPackage(self,package):
272 
-        print "Building package......",package
273 
-        specFile=self.findSpecFileForPackage(package)
274 
-        print "Spec file for package:",package," is ",specFile
275 
-        if specFile is None:
276 
-            print "Did not find spec file for package: ", package
277 
-            return False
278 
-        self.adjust_gcc_specs(package)
279 
-
280 
-        specName=os.path.basename(specFile)
281 
-        spec=Specutils(specFile)
282 
-        listSourceFiles=[]
283 
-        listSourceFiles.extend(spec.getSourceNames())
284 
-        listSourceFiles.extend(spec.getPatchNames())
285 
-        self.copySourcesTobuildroot(listSourceFiles,package)
286 
-        shutil.copyfile(specFile,  self.build_root_spec_path+"/"+specName)
287 
-        specChrootPath=self.build_root_spec_path.replace(self.build_root,"")
288 
-        returnVal,listRPMFiles = self.buildRPM(specChrootPath+"/"+specName,self.build_root_log_path+"/"+package+".log", self.run_in_chroot_command+" "+self.build_root)
289 
-        if not returnVal:
290 
-            return False
291 
-        for rpmFile in listRPMFiles:
292 
-            self.copyRPM(self.build_root+"/"+rpmFile, self.rpm_path)
293 
-        return True
294 
-
295 
-    def buildRPM(self,specFile,logFile,chrootCmd):
296 
-        cmdUtils = commandsUtils()
297 
-        returnVal,listRPMFiles = cmdUtils.buildRPM(specFile,logFile,chrootCmd)
298 
-        if not returnVal:
299 
-            print "Building rpm is failed",specFile
300 
-            return False,None
301 
-        return True,listRPMFiles
302 
-
303 
-    def getRPMDestDir(self,rpmName,rpmDir):
304 
-        arch=""
305 
-        if rpmName.find("x86_64") != -1:
306 
-            arch='x86_64'
307 
-        elif rpmName.find("noarch") != -1:
308 
-            arch="noarch"
309 
-        #else: Todo throw an exeption
310 
-        rpmDestDir=rpmDir+"/"+arch
311 
-        return rpmDestDir
312 
-
313 
-    def copyRPM(self,rpmFile,destDir):
314 
-        cmdUtils = commandsUtils()
315 
-        rpmName=os.path.basename(rpmFile)
316 
-        rpmDestDir=self.getRPMDestDir(rpmName,destDir)
317 
-        if not os.path.isdir(rpmDestDir):
318 
-            cmdUtils.run_command("mkdir -p "+rpmDestDir)
319 
-        rpmDestPath=rpmDestDir+"/"+rpmName
320 
-        shutil.copyfile(rpmFile,  rpmDestPath)
321 
-        return rpmDestPath
322 
-
323 
-    def installRPM(self,package):
324 
-
325 
-        rpmfile=self.findRPMFileForGivenPackage(package)
326 
-        if rpmfile is None:
327 
-            print "unexpected error"
328 
-            print "Stopping installing package",package
329 
-            return False
330 
-
331 
-        rpmDestFile = self.copyRPM(rpmfile, self.build_root+self.parent_path+"/RPMS")
332 
-        rpmFile=rpmDestFile.replace(self.build_root,"")
333 
-        chrootCmd=self.run_in_chroot_command+" "+self.build_root
334 
-        logFile=self.build_root_log_path+"/"+package+".completed"
335 
-
336 
-        cmdUtils = commandsUtils()
337 
-        nodepsOption=False
338 
-        if package in self.list_nodeps_packages:
339 
-            nodepsOption=True
340 
-        returnVal = cmdUtils.installRPM(rpmFile,logFile,chrootCmd,nodepsOption)
341 
-        if not returnVal:
342 
-            print "Installing " + rpmFile+" rpm is failed"
343 
-            return False
344 
-        self.listInstalledPackages.append(package)
345 
-        return True
346 
-
347 
-    def findInstalledPackages(self):
348 
-        cmdUtils=commandsUtils()
349 
-        listInstalledRPMs=cmdUtils.find_installed_rpm_packages(self.run_in_chroot_command+" "+self.build_root)
350 
-        listInstalledPackages=[]
351 
-        for installedRPM in listInstalledRPMs:
352 
-            packageName=self.findPackageNameFromRPMFile(installedRPM)
353 
-            if packageName is not None:
354 
-                listInstalledPackages.append(packageName)
355 
-        return listInstalledPackages
356 
-
357 
-    def buildToolchain(self):
358 
-        self.cleanBuildRoot()
359 
-        self.prepare_build_root()
360 
-        for package in self.listPkgsRequiredToBuildToolchain:
361 
-            rpmfile=self.findRPMFileForGivenPackage(package)
362 
-            if rpmfile is None :
363 
-                returnVal=self.buildRPMSForGivenPackage(package)
364 
-                if not returnVal:
365 
-                    print "Stopping building toolchain"
366 
-                    return False
367 
-            returnVal=self.installRPM(package)
368 
-            if not returnVal:
369 
-                print "Stopping building toolchain"
370 
-                return False
371 
-            if package =="glibc":
372 
-                self.adjust_tool_chain()
373 
-        print "Built toolchain succesfully"
374 
-        return True
375 
-
376 
-    def installToolchain(self):
377 
-        flagbuildToolchain=False
378 
-        for package in self.listPkgsToInstallToolChain:
379 
-            rpmfile=self.findRPMFileForGivenPackage(package)
380 
-            if rpmfile is None :
381 
-                flagbuildToolchain = True
382 
-                break
383 
-        if flagbuildToolchain:
384 
-            returnVal=self.buildToolchain()
385 
-            if not returnVal:
386 
-                print "Stopping installing toolchain"
387 
-                return False
388 
-
389 
-        self.cleanBuildRoot()
390 
-        self.prepare_build_root("minimal")
391 
-        for package in self.listPkgsToInstallToolChain:
392 
-            returnVal=self.installRPM(package)
393 
-            if not returnVal:
394 
-                print "Stopping installing toolchain"
395 
-                return False
396 
-        print "Installed toolchain succesfully"
397 
-        return True
398 
-
399 
-
400 
-    def constructDependencyGraph(self,package,mapPkgNRequiredPkgs):
401 
-        returnVal,listRequiredPackages=self.findBuildTimeRequiredPackages(package)
402 
-        if not returnVal:
403 
-             return False
404 
-        mapPkgNRequiredPkgs[package]=listRequiredPackages
405 
-        for requiredPkg in listRequiredPackages:
406 
-            if not mapPkgNRequiredPkgs.has_key(requiredPkg):
407 
-                returnVal=self.constructDependencyGraph(requiredPkg,mapPkgNRequiredPkgs)
408 
-                if not returnVal:
409 
-                    return False
410 
-        return True
411 
-
412 
-    def findBuildTimeRequiredPackages(self,package):
413 
-        specFile=self.findSpecFileForPackage(package)
414 
-        print "Spec file for package:",package," is ",specFile
415 
-        if specFile is None:
416 
-            print "Did not find spec file for package: ", package
417 
-            return False,None
418 
-        spec=Specutils(specFile)
419 
-        listRequiredPackages=[]
420 
-        listRequiredPackages=spec.getBuildRequiresAllPackages()
421 
-        return True,listRequiredPackages
422 
-
423 
-    def findRunTimeRequiredPackages(self,package):
424 
-        specFile=self.findSpecFileForPackage(package)
425 
-        print "Spec file for package:",package," is ",specFile
426 
-        if specFile is None:
427 
-            print "Did not find spec file for package: ", package
428 
-            return False,None
429 
-        spec=Specutils(specFile)
430 
-        listRequiredPackages=[]
431 
-        listRequiredPackages=spec.getRequiresAllPackages()
432 
-        return True,listRequiredPackages
433 
-
434 
-    def findRunTimeRequiredRPMPackages(self,rpmPackage):
435 
-        specFile=self.findSpecFileForPackage(rpmPackage)
436 
-        print "Spec file for package:",rpmPackage," is ",specFile
437 
-        if specFile is None:
438 
-            print "Did not find spec file for package: ", rpmPackage
439 
-            return False,None
440 
-        spec=Specutils(specFile)
441 
-        listRequiredPackages=[]
442 
-        listRequiredPackages=spec.getRequires(rpmPackage)
443 
-        return True,listRequiredPackages
444 
-
445 
-    def topological_sort_packages(self, dependent_packages):
446 
-        nodep_packages = Set()
447 
-        sorted_package_list = []
448 
-        dependentof_package = dict()
449 
-
450 
-        #Find packages with no dependencies and generate dependentof_package edge list
451 
-        for package in dependent_packages:
452 
-            if len(dependent_packages[package]) == 0:
453 
-                nodep_packages.add(package)
454 
-            else:
455 
-                for dependent_package in dependent_packages[package]:
456 
-                    if dependentof_package.get(dependent_package) is None:
457 
-                        dependentof_package[dependent_package]=[package]
458 
-                    else:
459 
-                        dependentof_package[dependent_package].append(package)
460 
-        while nodep_packages:
461 
-            package = nodep_packages.pop()
462 
-            sorted_package_list.append(package)
463 
-            if dependentof_package.get(package) is not None:
464 
-                print "dependent of: "+package+ "  are : "+ ' '.join(dependentof_package[package])
465 
-                for child_package in list(dependentof_package.get(package)):
466 
-                    dependentof_package.get(package).remove(child_package)
467 
-                    dependent_packages[child_package].remove(package)
468 
-                    print child_package +  " depends on " + ' '.join(dependent_packages[child_package])
469 
-                    if len(dependent_packages[child_package])==0:
470 
-                        nodep_packages.add(child_package)
471 
-
472 
-        return sorted_package_list
473 
-
474 
-
475 
-    def find_package_dependency_tree(self, package_name):
476 
-        print "finding dependency tree for : " + package_name
477 
-        dependent_packages={}
478 
-        returnVal=self.constructDependencyGraph(package_name, dependent_packages)
479 
-        if not returnVal:
480 
-            print "Not able to find depedendency tree for package ", package_name
481 
-            return False, None
482 
-        print dependent_packages
483 
-        sorted_package_list = self.topological_sort_packages(dependent_packages)
484 
-        return True,sorted_package_list
485 
-
486 
-    def removeToolsTar(self):
487 
-        cmdUtils=commandsUtils()
488 
-        cmdUtils.run_command("rm -rf "+self.build_root+"/tools")
489 
-        return True
490 
-
491 
-    def cleanBuildRoot(self):
492 
-        cmdUtils=commandsUtils()
493 
-        cmdUtils.run_command("./cleanup-build-root.sh "+self.build_root)
494 
-
495 
-    def buildPackage(self,package,force_build, listFailedPkgs=[]):
496 
-        # We don't need to build this package if RPM exists && !force_build
497 
-        rpmfile = None
498 
-        if not force_build:
499 
-            rpmfile=self.findRPMFileForGivenPackage(package)
500 
-        if rpmfile is not None:
501 
-            print "The given package is already built",package
502 
-            return True
503 
-
504 
-        if package in listFailedPkgs:
505 
-            print package," package is failed in previous iteration. Not building again."
506 
-            return False
507 
-        returnVal,listDependentPackages=self.find_package_dependency_tree(package)
508 
-        if not returnVal:
509 
-            print "Failed during building the package", package
510 
-            return False
511 
-        listDependentPackages.remove(package)
512 
-        listPkgsTobeBuild=[]
513 
-        returnVal,listRunTimeDependentPackages=self.findRunTimeRequiredPackages(package)
514 
-        if not returnVal:
515 
-            print "Failed during building the package", package
516 
-            return False
517 
-
518 
-        for pkg in listDependentPackages:
519 
-            rpmfile=self.findRPMFileForGivenPackage(pkg)
520 
-            if rpmfile is None :
521 
-                listPkgsTobeBuild.append(pkg)
522 
-
523 
-        for pkg in listRunTimeDependentPackages:
524 
-            rpmfile=self.findRPMFileForGivenPackage(pkg)
525 
-            if rpmfile is None :
526 
-                listPkgsTobeBuild.append(pkg)
527 
-
528 
-        if len(listPkgsTobeBuild) != 0:
529 
-            print "Dependent packages to be build",listPkgsTobeBuild
530 
-            print "Building dependent packages.........."
531 
-            for pkg in listPkgsTobeBuild:
532 
-                returnVal = self.buildPackage(pkg,force_build,listFailedPkgs)
533 
-                if not returnVal:
534 
-                    print "Failed installing package",pkg
535 
-                    print "Stop installing package",package
536 
-                    return False
537 
-            print "Finished building dependent packages"
538 
-
539 
-        # Let's build this package
540 
-        self.installToolchain()
541 
-        self.removeToolsTar()
542 
-        del self.listInstalledPackages[:]
543 
-        self.listInstalledPackages=self.findInstalledPackages()
544 
-
545 
-        returnVal,listDependentPackages=self.findBuildTimeRequiredPackages(package)
546 
-        if not returnVal:
547 
-            print "Failed during building the package", package
548 
-            return False
549 
-
550 
-        #if toolchain package called this method, preventing from installing again
551 
-        if package in self.listInstalledPackages:
552 
-            return True
553 
-
554 
-        if len(listDependentPackages) != 0:
555 
-            print "Installing the build time dependent packages......"
556 
-            for pkg in listDependentPackages:
557 
-                if pkg in self.listInstalledPackages:
558 
-                    continue
559 
-                returnVal = self.installPackage(pkg)
560 
-                if not returnVal:
561 
-                    print "Failed while installing the build time dependent package",pkg
562 
-                    return False
563 
-            print "Finished installing the build time dependent packages......"
564 
-
565 
-        returnVal=self.buildRPMSForGivenPackage(package)
566 
-        if not returnVal:
567 
-            print "Failed while building the package",package
568 
-            return False
569 
-        print "Successfully built the package:",package
570 
-        return True
571 
-
572 
-    def installPackage(self,package):
573 
-        #if toolchain package called this method, preventing from installing again
574 
-        if package in self.listInstalledPackages:
575 
-            return True
576 
-        returnVal = self.installDependentRunTimePackages(package)
577 
-        if not returnVal:
578 
-            return False
579 
-        returnVal = self.installRPM(package)
580 
-        if not returnVal:
581 
-            print "Stop installing package",package
582 
-            return False
583 
-        print "Installed the package:",package
584 
-        return True
585 
-
586 
-    def installDependentRunTimePackages(self,package):
587 
-        returnVal,listRunTimeDependentPackages=self.findRunTimeRequiredRPMPackages(package)
588 
-        if not returnVal:
589 
-            return False
590 
-        if len(listRunTimeDependentPackages) != 0:
591 
-            for pkg in listRunTimeDependentPackages:
592 
-                if pkg in self.listInstalledPackages:
593 
-                    continue
594 
-                returnVal = self.installPackage(pkg)
595 
-                if not returnVal:
596 
-                    return False
597 
-        return True
598 
-
599 
-    def doCleanBuild(self):
600 
-        cmdUtils=commandsUtils()
601 
-        cmdUtils.run_command("rm -rf "+ self.rpm_path+"/*")
602 
-        cmdUtils.run_command("mkdir "+ self.rpm_path+"/x86_64")
603 
-        cmdUtils.run_command("mkdir "+ self.rpm_path+"/noarch")
604 
-        returnVal=self.buildAllPackages()
605 
-        if not returnVal:
606 
-            print "Clean build failed."
607 
-            return False
608 
-        print "Clean buils is completed successfully"
609 
-        return True
610 
-
611 
-    def buildAllPackages(self):
612 
-        # in case previous build was terminated.
613 
-        self.cleanBuildRoot()
614 
-
615 
-        listFailedPkgs=[]
616 
-        list_packages=getListPackages(self.spec_path)
617 
-        returnVal=self.installToolchain()
618 
-        if not returnVal:
619 
-            print "Not able to build toolchain"
620 
-            print "Please fix this error and continue to build all packages"
621 
-            return False
622 
-        for pkg in list_packages:
623 
-            returnVal=self.buildPackage(pkg,False,listFailedPkgs)
624 
-            if not returnVal:
625 
-                listFailedPkgs.append(pkg)
626 
-        if len(listFailedPkgs) != 0:
627 
-            print "Some of the packages failed during clean build. Logs are locates in ", self.build_root_log_path," path."
628 
-            print "List failed Packages",listFailedPkgs
629 
-            return False
630 
-        self.cleanBuildRoot()
631 
-        print "Successfully built all packages"
632 
-        return True
633 
-
634 
-
635 
-def main():
636 
-    usage = "Usage: %prog [options] <package name>"
637 
-    parser = OptionParser(usage)
638 
-    parser.add_option("-s",  "--spec-path",  dest="spec_path",  default="../../SPECS")
639 
-    parser.add_option("-o",  "--source-path",  dest="source_path",  default="../../SOURCES")
640 
-    parser.add_option("-r",  "--rpm-path",  dest="rpm_path",  default="../../stage/RPMS")
641 
-    parser.add_option("-b",  "--build-root",  dest="build_root",  default="/mnt/photonroot")
642 
-    parser.add_option("-t",  "--install-tool-chain",  dest="install_tool_chain",  default=False,  action ="store_true")
643 
-    parser.add_option("-i",  "--install-package", dest="install_package",  default=False,  action ="store_true")
644 
-    parser.add_option("-c",  "--clean-build", dest="clean_build",  default=False,  action ="store_true")
645 
-    parser.add_option("-p",  "--tools-path", dest="tools_path",  default="../../stage")
646 
-    parser.add_option("-l",  "--log-path", dest="log_path",  default="../../stage/LOGS")
647 
-    parser.add_option("-a",  "--build-all", dest="build_all",  default=False,  action ="store_true")
648 
-    parser.add_option("-f",  "--force", dest="force_build",  default=False,  action ="store_true")
649 
-
650 
-    (options,  args) = parser.parse_args()
651 
-    #if (len(args)) != 1:
652 
-    #    parser.error("Incorrect number of arguments")
653 
-
654 
-    print "Source Path :"+options.source_path
655 
-    print "Spec Path :" + options.spec_path
656 
-    print "Rpm Path :" + options.rpm_path
657 
-    print "Tools Path :" + options.tools_path
658 
-    print "Log Path :" + options.log_path
659 
-    print "Build root :" + options.build_root
660 
-
661 
-    package_builder = BuildSystem(options.source_path,  options.spec_path,  options.rpm_path,  options.build_root, options.tools_path, options.log_path)
662 
-
663 
-    returnVal = True
664 
-    if options.clean_build:
665 
-        returnVal=package_builder.doCleanBuild()
666 
-    elif options.build_all:
667 
-        returnVal=package_builder.buildAllPackages()
668 
-    elif options.install_tool_chain:
669 
-        returnVal=package_builder.installToolchain()
670 
-    elif options.install_package:
671 
-        if (len(args)) != 1:
672 
-            parser.error("Incorrect number of arguments")
673 
-            returnVal=False
674 
-        else:
675 
-            returnVal=package_builder.buildPackage(args[0],options.force_build)
676 
-    return returnVal
677 
-
678 
-
679 
-if __name__ == '__main__':
680 
-    main()
support/package-builder/constants.py
History View file @ 2820c61
681 1 
new file mode 100644
... ... 
@@ -0,0 +1,30 @@
0 
+from SpecData import SerializableSpecObjectsUtils
1 
+
2 
+class constants(object):
3 
+    specPath=""
4 
+    sourcePath=""
5 
+    rpmPath=""
6 
+    toolsPath=""
7 
+    logPath=""
8 
+    incrementalBuild=False
9 
+    topDirPath=""
10 
+    specData=None
11 
+    testPath="test"
12 
+
13 
+
14 
+    @staticmethod
15 
+    def initialize(options):
16 
+        constants.specPath = options.specPath
17 
+        constants.sourcePath = options.sourcePath
18 
+        constants.rpmPath = options.rpmPath
19 
+        constants.toolsPath = options.toolsPath
20 
+        constants.incrementalBuild = options.incrementalBuild
21 
+        constants.topDirPath = options.topDirPath
22 
+        constants.logPath = options.logPath
23 
+        constants.specData = SerializableSpecObjectsUtils(constants.logPath)
24 
+        constants.specData.readSpecsAndConvertToSerializableObjects(constants.specPath)
25 
+
26 
+
27 
+
28 
+
29 
+
0 30 
\ No newline at end of file