support/package-builder/SpecParser.py
326d5ca8
 # pylint: disable=invalid-name,missing-docstring
2820c61a
 import re
 from StringUtils import StringUtils
4ed339b4
 from SpecStructures import dependentPackageData, Package, SpecObject
45c9260c
 from constants import constants
2820c61a
 
 class SpecParser(object):
343d89e8
 
4ed339b4
     class rpmMacro(object):
         def __init__(self):
             self.macroName = ""
             self.macroFlag = ""
             self.content = ""
             self.position = -1
             self.endposition = -1
 
a007b861
     def __init__(self, specfile, arch):
         self.arch = arch
4ed339b4
         self.cleanMacro = None
         self.prepMacro = None
         self.buildMacro = None
         self.installMacro = None
         self.changelogMacro = None
         self.checkMacro = None
87815216
         self.packages = {}
         self.specAdditionalContent = ""
         self.globalSecurityHardening = ""
         self.defs = {}
a007b861
         self.defs["_arch"] = arch
5f40784b
         self.conditionalCheckMacroEnabled = False
343d89e8
         self.macro_pattern = re.compile(r'%{(\S+?)\}')
4ed339b4
         self.specfile = specfile
5062126c
 
4ed339b4
         self._parseSpecFile()
 
     def _parseSpecFile(self):
a007b861
         self.packages["default"] = Package(self.arch)
326d5ca8
         currentPkg = "default"
4ed339b4
         with open(self.specfile) as specFile:
326d5ca8
             lines = specFile.readlines()
             totalLines = len(lines)
             i = 0
             while i < totalLines:
                 line = lines[i].strip()
                 if self._isConditionalArch(line):
a007b861
                     if self.arch != self._readConditionalArch(line):
326d5ca8
                         # skip conditional body
                         deep = 1
                         while i < totalLines and deep != 0:
                             i = i + 1
                             line = lines[i].strip()
                             if self._isConditionalMacroStart(line):
                                 deep = deep + 1
                             elif self._isConditionalMacroEnd(line):
                                 deep = deep - 1
                 elif self._isIfCondition(line):
                     if not self._isConditionTrue(line):
                         # skip conditional body
                         deep = 1
                         while i < totalLines and deep != 0:
                             i = i + 1
                             line = lines[i].strip()
                             if self._isConditionalMacroStart(line):
                                 deep = deep + 1
                             elif self._isConditionalMacroEnd(line):
                                 deep = deep - 1
                 elif self._isSpecMacro(line):
                     macro, i = self._readMacroFromFile(i, lines)
                     self._updateSpecMacro(macro)
                 elif self._isPackageMacro(line):
                     defaultpkg = self.packages.get('default')
                     returnVal, packageName = self._readPkgNameFromPackageMacro(line,
                                                                                defaultpkg.name)
                     packageName = self._replaceMacros(packageName)
                     if not returnVal:
                         return False
                     if line.startswith('%package'):
a007b861
                         pkg = Package(self.arch, defaultpkg)
326d5ca8
                         pkg.name = packageName
                         currentPkg = packageName
                         self.packages[pkg.name] = pkg
                     else:
                         if defaultpkg.name == packageName:
                             packageName = 'default'
                         macro, i = self._readMacroFromFile(i, lines)
                         if packageName not in self.packages:
                             i = i + 1
                             continue
                         self.packages[packageName].updatePackageMacro(macro)
                 elif self._isPackageHeaders(line):
                     self._readPackageHeaders(line, self.packages[currentPkg])
                 elif self._isGlobalSecurityHardening(line):
                     self._readSecurityHardening(line)
                 elif self._isChecksum(line):
                     self._readChecksum(line, self.packages[currentPkg])
8f56b626
                 elif self._isExtraBuildRequires(line):
                     self._readExtraBuildRequires(line, self.packages[currentPkg])
a007b861
                 elif self._isBuildRequiresNative(line):
                     self._readBuildRequiresNative(line, self.packages[currentPkg])
326d5ca8
                 elif self._isDefinition(line):
                     self._readDefinition(line)
                 elif self._isConditionalCheckMacro(line):
                     self.conditionalCheckMacroEnabled = True
                 elif self.conditionalCheckMacroEnabled and self._isConditionalMacroEnd(line):
                     self.conditionalCheckMacroEnabled = False
                 else:
                     self.specAdditionalContent += line + "\n"
                 i = i + 1
5062126c
 
326d5ca8
     def _readPkgNameFromPackageMacro(self, data, basePkgName=None):
87815216
         data = " ".join(data.split())
         pkgHeaderName = data.split(" ")
2820c61a
         lenpkgHeaderName = len(pkgHeaderName)
87815216
         i = 1
adf248d5
         pkgName = None
87815216
         while i < lenpkgHeaderName:
adf248d5
             if pkgHeaderName[i] == "-n" and i+1 < lenpkgHeaderName:
                 pkgName = pkgHeaderName[i+1]
                 break
             if pkgHeaderName[i].startswith('-'):
                 i = i + 2
             else:
87815216
                 pkgName = basePkgName + "-" + pkgHeaderName[i]
adf248d5
                 break
         if pkgName is None:
2820c61a
             return True, basePkgName
adf248d5
         return True, pkgName
5062126c
 
326d5ca8
     def _replaceMacros(self, string):
343d89e8
         """Replace all macros in given string with corresponding values.
 
         For example: a string '%{name}-%{version}.tar.gz' will be transformed to 'foo-2.0.tar.gz'.
 
         :return A string where all macros in given input are substituted as good as possible.
 
         """
         def _is_conditional(macro):
87815216
             return macro.startswith(("?", "!"))
343d89e8
 
         def _test_conditional(macro):
             if macro[0] == "?":
                 return True
             if macro[0] == "!":
                 return False
             raise Exception("Given string is not a conditional macro")
 
         def _is_macro_defined(macro):
aac331d9
             return (macro in self.defs.keys()) or (macro in constants.userDefinedMacros.keys()) \
                 or (macro in constants.getAdditionalMacros(self.packages["default"].name).keys())
343d89e8
 
         def _get_macro(macro):
             if macro in self.defs.keys():
                 return self.defs[macro]
             elif macro in constants.userDefinedMacros.keys():
                 return constants.userDefinedMacros[macro]
aac331d9
             elif macro in constants.getAdditionalMacros(self.packages["default"].name).keys():
                 return constants.getAdditionalMacros(self.packages["default"].name)[macro]
343d89e8
             raise Exception("Unknown macro: " + macro)
 
         def _macro_repl(match):
             macro_name = match.group(1)
             if _is_conditional(macro_name):
                 parts = macro_name[1:].split(":")
326d5ca8
                 assert parts
                 retv = ""
343d89e8
                 if _test_conditional(macro_name):  # ?
                     if _is_macro_defined(parts[0]):
                         if len(parts) == 2:
326d5ca8
                             retv = parts[1]
343d89e8
                         else:
326d5ca8
                             retv = _get_macro(parts[0])
343d89e8
                 else:  # !
                     if not _is_macro_defined(parts[0]):
                         if len(parts) == 2:
326d5ca8
                             retv = parts[1]
                 return retv
343d89e8
 
             if _is_macro_defined(macro_name):
                 return _get_macro(macro_name)
             return match.string[match.start():match.end()]
 
         #User macros
326d5ca8
         for macroName, value in constants.userDefinedMacros.items():
87815216
             macro = "%" + macroName
343d89e8
             if string.find(macro) != -1:
87815216
                 string = string.replace(macro, value)
343d89e8
         #Spec definitions
326d5ca8
         for macroName, value in self.defs.items():
87815216
             macro = "%" + macroName
343d89e8
             if string.find(macro) != -1:
87815216
                 string = string.replace(macro, value)
343d89e8
         return re.sub(self.macro_pattern, _macro_repl, string)
 
326d5ca8
     def _readMacroFromFile(self, currentPos, lines):
4ed339b4
         macro = self.rpmMacro()
2820c61a
         line = lines[currentPos]
         macro.position = currentPos
87815216
         macro.endposition = currentPos
         endPos = len(lines)
2820c61a
         line = " ".join(line.split())
         flagindex = line.find(" ")
         if flagindex != -1:
87815216
             macro.macroFlag = line[flagindex+1:]
             macro.macroName = line[:flagindex]
2820c61a
         else:
87815216
             macro.macroName = line
2820c61a
 
326d5ca8
         if currentPos + 1 < len(lines) and self._isMacro(lines[currentPos+1]):
87815216
             return macro, currentPos
5062126c
 
87815216
         for j in range(currentPos + 1, endPos):
2820c61a
             content = lines[j]
326d5ca8
             if j+1 < endPos and self._isMacro(lines[j+1]):
87815216
                 return macro, j
2820c61a
             macro.content += content +'\n'
87815216
             macro.endposition = j
         return macro, endPos
5062126c
 
326d5ca8
     def _updateSpecMacro(self, macro):
2820c61a
         if macro.macroName == "%clean":
87815216
             self.cleanMacro = macro
2820c61a
         if macro.macroName == "%prep":
87815216
             self.prepMacro = macro
2820c61a
         if macro.macroName == "%build":
87815216
             self.buildMacro = macro
2820c61a
         if macro.macroName == "%install":
87815216
             self.installMacro = macro
2820c61a
         if macro.macroName == "%changelog":
87815216
             self.changelogMacro = macro
2820c61a
         if macro.macroName == "%check":
87815216
             self.checkMacro = macro
4ed339b4
 
326d5ca8
     def _isMacro(self, line):
         return (self._isPackageMacro(line) or
                 self._isSpecMacro(line) or
                 self._isConditionalMacroStart(line) or
                 self._isConditionalMacroEnd(line))
5062126c
 
326d5ca8
     def _isConditionalArch(self, line):
87815216
         if re.search('^'+'%ifarch', line):
0f1fdc4b
             return True
         return False
5062126c
 
326d5ca8
     def _isSpecMacro(self, line):
87815216
         if line.startswith(('%clean', '%prep', '%build', '%install', '%changelog', '%check')):
2820c61a
             return True
         return False
5062126c
 
326d5ca8
     def _isPackageMacro(self, line):
87815216
         line = line.strip()
         if line.startswith(('%post', '%postun', '%files', '%description', '%package')):
2820c61a
             return True
         return False
5062126c
 
326d5ca8
     def _isPackageHeaders(self, line):
87815216
         headersPatterns = ['^summary:', '^name:', '^group:',
                            '^license:', '^version:', '^release:',
                            '^distribution:', '^requires:',
326d5ca8
                            r'^requires\((pre|post|preun|postun)\):',
87815216
                            '^provides:', '^obsoletes:', '^conflicts:',
                            '^url:', '^source[0-9]*:', '^patch[0-9]*:',
                            '^buildrequires:', '^buildprovides:',
                            '^buildarch:']
         if any([re.search(r, line, flags=re.IGNORECASE) for r in headersPatterns]):
2820c61a
             return True
         return False
 
326d5ca8
     def _isGlobalSecurityHardening(self, line):
87815216
         if re.search('^%global *security_hardening', line, flags=re.IGNORECASE):
cb4e8710
             return True
         return False
 
8f56b626
     def _isExtraBuildRequires(self, line):
         if re.search('^%define *extrabuildrequires', line, flags=re.IGNORECASE):
             return True
         return False
 
a007b861
     def _isBuildRequiresNative(self, line):
         if re.search('^%define *buildrequiresnative', line, flags=re.IGNORECASE):
             return True
         return False
 
326d5ca8
     def _isChecksum(self, line):
87815216
         if re.search('^%define *sha1', line, flags=re.IGNORECASE):
3cc43c92
             return True
         return False
 
326d5ca8
     def _isDefinition(self, line):
87815216
         if line.startswith(('%define', '%global')):
fba234bc
             return True
         return False
 
326d5ca8
     def _readConditionalArch(self, line):
87815216
         w = line.split()
0f1fdc4b
         if len(w) == 2:
87815216
             return w[1]
0f1fdc4b
         return None
 
326d5ca8
     def _readDefinition(self, line):
87815216
         listDefines = line.split()
fba234bc
         if len(listDefines) == 3:
326d5ca8
             self.defs[listDefines[1]] = self._replaceMacros(listDefines[2])
87815216
             return True
fba234bc
         return False
 
326d5ca8
     def _readHeader(self, line):
87815216
         headerSplitIndex = line.find(":")
         if headerSplitIndex + 1 == len(line):
             print(line)
             print("Error:Invalid header")
             return False, None, None
         headerName = line[0:headerSplitIndex].lower()
         headerContent = line[headerSplitIndex + 1:].strip()
         return True, headerName, headerContent
 
326d5ca8
     def _readDependentPackageData(self, line):
2820c61a
         strUtils = StringUtils()
87815216
         listPackages = line.split(",")
         listdependentpkgs = []
2820c61a
         for line in listPackages:
87815216
             line = strUtils.getStringInConditionalBrackets(line)
             listContents = line.split()
2820c61a
             totalContents = len(listContents)
87815216
             i = 0
2820c61a
             while i < totalContents:
                 dpkg = dependentPackageData()
87815216
                 compare = None
                 packageName = listContents[i]
f5cac196
                 if listContents[i].startswith("/"):
87815216
                     provider = constants.providedBy.get(listContents[i], None)
45c9260c
                     if provider is not None:
87815216
                         packageName = provider
45c9260c
                     else:
a75dbe8f
                         raise Exception('What package does provide %s? Please modify providedBy in constants.py' % (listContents[i]))
                     i += 1
326d5ca8
                 if i + 2 < len(listContents):
f93ef2b0
                     if listContents[i+1] in (">=", "<=", "=", "<", ">"):
                         compare = listContents[i+1]
5062126c
 
2820c61a
                 if compare is not None:
87815216
                     dpkg.package = packageName
                     dpkg.compare = compare
                     dpkg.version = listContents[i+2]
                     i = i + 3
2820c61a
                 else:
87815216
                     dpkg.package = packageName
                     i = i + 1
2820c61a
                 listdependentpkgs.append(dpkg)
         return listdependentpkgs
 
326d5ca8
     def _readPackageHeaders(self, line, pkg):
         returnVal, headerName, headerContent = self._readHeader(line)
2820c61a
         if not returnVal:
             return False
 
326d5ca8
         headerContent = self._replaceMacros(headerContent)
2820c61a
         if headerName == 'summary':
87815216
             pkg.summary = headerContent
2820c61a
             return True
         if headerName == 'name':
87815216
             pkg.name = headerContent
             if pkg == self.packages["default"]:
343d89e8
                 self.defs["name"] = pkg.name
2820c61a
             return True
         if headerName == 'group':
87815216
             pkg.group = headerContent
2820c61a
             return True
         if headerName == 'license':
87815216
             pkg.license = headerContent
2820c61a
             return True
         if headerName == 'version':
87815216
             pkg.version = headerContent
             if pkg == self.packages["default"]:
343d89e8
                 self.defs["version"] = pkg.version
2820c61a
             return True
         if headerName == 'buildarch':
87815216
             pkg.buildarch = headerContent
2820c61a
             return True
         if headerName == 'release':
87815216
             pkg.release = headerContent
             if pkg == self.packages["default"]:
343d89e8
                 self.defs["release"] = pkg.release
2820c61a
             return True
         if headerName == 'distribution':
87815216
             pkg.distribution = headerContent
2820c61a
             return True
920a9773
         if headerName == 'url':
87815216
             pkg.URL = headerContent
920a9773
             return True
326d5ca8
         if 'source' in headerName:
2820c61a
             pkg.sources.append(headerContent)
             return True
326d5ca8
         if 'patch' in headerName:
2820c61a
             pkg.patches.append(headerContent)
             return True
87815216
         if (headerName.startswith('requires') or
                 headerName == 'provides' or
                 headerName == 'obsoletes' or
                 headerName == 'conflicts' or
                 headerName == 'buildrequires' or
                 headerName == 'buildprovides'):
326d5ca8
             dpkg = self._readDependentPackageData(headerContent)
2820c61a
             if dpkg is None:
                 return False
45c9260c
             if headerName.startswith('requires'):
2820c61a
                 pkg.requires.extend(dpkg)
             if headerName == 'provides':
                 pkg.provides.extend(dpkg)
             if headerName == 'obsoletes':
                 pkg.obsoletes.extend(dpkg)
             if headerName == 'conflicts':
                 pkg.conflicts.extend(dpkg)
             if headerName == 'buildrequires':
5f40784b
                 if self.conditionalCheckMacroEnabled:
                     pkg.checkbuildrequires.extend(dpkg)
                 else:
                     pkg.buildrequires.extend(dpkg)
2820c61a
             if headerName == 'buildprovides':
                 pkg.buildprovides.extend(dpkg)
5062126c
 
2820c61a
             return True
         return False
cb4e8710
 
326d5ca8
     def _readSecurityHardening(self, line):
87815216
         data = line.lower().strip()
         words = data.split(" ")
cb4e8710
         nrWords = len(words)
87815216
         if nrWords != 3:
             print("Error: Unable to parse line: " + line)
cb4e8710
             return False
04c0b406
         if words[2] != "none" and words[2] != "nonow" and \
                 words[2] != "nopie" and words[2] != "nofortify":
87815216
             print("Error: Invalid security_hardening value: " + words[2])
cb4e8710
             return False
         self.globalSecurityHardening = words[2]
87815216
         return True
3cc43c92
 
8f56b626
     def _readExtraBuildRequires(self, line, pkg):
         data = line.strip()
         words = data.split(" ", 2)
         if len(words) != 3:
             print("Error: Unable to parse line: " + line)
             return False
         dpkg = self._readDependentPackageData(words[2])
         if dpkg is None:
             return False
         pkg.extrabuildrequires.extend(dpkg)
         return True
 
a007b861
     def _readBuildRequiresNative(self, line, pkg):
         data = line.strip()
         words = data.split(" ", 2)
         if len(words) != 3:
             print("Error: Unable to parse line: " + line)
             return False
         dpkg = self._readDependentPackageData(words[2])
         if dpkg is None:
             return False
         pkg.buildrequiresnative.extend(dpkg)
         return True
 
326d5ca8
     def _readChecksum(self, line, pkg):
3cc43c92
         strUtils = StringUtils()
326d5ca8
         line = self._replaceMacros(line)
87815216
         data = line.strip()
         words = data.split()
3cc43c92
         nrWords = len(words)
87815216
         if nrWords != 3:
             print("Error: Unable to parse line: " + line)
3cc43c92
             return False
87815216
         value = words[2].split("=")
         if len(value) != 2:
             print("Error: Unable to parse line: "+line)
3cc43c92
             return False
87815216
         matchedSources = []
3cc43c92
         for source in pkg.sources:
87815216
             sourceName = strUtils.getFileNameFromURL(source)
326d5ca8
             if sourceName.startswith(value[0]):
3cc43c92
                 matchedSources.append(sourceName)
326d5ca8
         if not matchedSources:
87815216
             print("Error: Can not find match for sha1 " + value[0])
3cc43c92
             return False
326d5ca8
         if len(matchedSources) > 1:
87815216
             print("Error: Too many matched Sources:" +
                   ' '.join(matchedSources) + " for sha1 " + value[0])
3cc43c92
             return False
         pkg.checksums[sourceName] = value[1]
326d5ca8
         return True
5f40784b
 
326d5ca8
     def _isConditionalCheckMacro(self, line):
5f40784b
         data = line.strip()
         words = data.split()
         nrWords = len(words)
326d5ca8
         if nrWords != 2:
5f40784b
             return False
326d5ca8
         if words[0] != "%if" or words[1] != "%{with_check}":
5f40784b
             return False
         return True
 
326d5ca8
     def _isIfCondition(self, line):
343d89e8
         return line.startswith("%if ")
 
     # Supports only %if %{}
326d5ca8
     def _isConditionTrue(self, line):
343d89e8
         data = line.strip()
         words = data.split()
         nrWords = len(words)
         # condition like %if a > b is not supported
326d5ca8
         if nrWords != 2:
343d89e8
             return True
326d5ca8
         if self._replaceMacros(words[1]) == "0":
343d89e8
             return False
         return True
 
326d5ca8
     def _isConditionalMacroStart(self, line):
0f1fdc4b
         return line.startswith("%if")
 
326d5ca8
     def _isConditionalMacroEnd(self, line):
         return line.strip() == "%endif"
4ed339b4
 
     ########################################################################
     # SpecObject generating functions
     ########################################################################
 
     #
     # @requiresType: "build" for BuildRequires or
     #                "install" for Requires dependencies.
     def _getRequiresTypeAllPackages(self, requiresType):
         dependentPackages = []
         for pkg in self.packages.values():
             if requiresType == "build":
                 dependentPackages.extend(pkg.buildrequires)
             elif requiresType == "install":
                 dependentPackages.extend(pkg.requires)
         listDependentPackages = dependentPackages.copy()
         packageNames = self._getPackageNames()
         for pkg in self.packages.values():
             for objName in listDependentPackages:
                 if objName.package == pkg.name:
                         dependentPackages.remove(objName)
         return dependentPackages
 
     def _getCheckBuildRequiresAllPackages(self):
         dependentPackages = []
         for pkg in self.packages.values():
             dependentPackages.extend(pkg.checkbuildrequires)
         return dependentPackages
 
     def _getExtraBuildRequires(self):
         dependentPackages = []
         for pkg in self.packages.values():
             dependentPackages.extend(pkg.extrabuildrequires)
         return dependentPackages
 
a007b861
     def _getBuildRequiresNative(self):
         dependentPackages = []
         for pkg in self.packages.values():
             dependentPackages.extend(pkg.buildrequiresnative)
         return dependentPackages
 
4ed339b4
     def _getPackageNames(self):
         packageNames = []
         for pkg in self.packages.values():
             packageNames.append(pkg.name)
         return packageNames
 
     def _getSourceNames(self):
         sourceNames = []
         strUtils = StringUtils()
         pkg = self.packages.get('default')
         for source in pkg.sources:
             sourceName = strUtils.getFileNameFromURL(source)
             sourceNames.append(sourceName)
         return sourceNames
 
     def _getPatchNames(self):
         patchNames = []
         strUtils = StringUtils()
         pkg = self.packages.get('default')
         for patch in pkg.patches:
             patchName = strUtils.getFileNameFromURL(patch)
             patchNames.append(patchName)
         return patchNames
 
     def _getSourceURL(self):
         pkg = self.packages.get('default')
         if not pkg.sources:
             return None
         sourceURL = pkg.sources[0]
         if sourceURL.startswith("http") or sourceURL.startswith("ftp"):
             return sourceURL
         return None
 
     def _getRequires(self, pkgName):
         dependentPackages = []
         for pkg in self.packages.values():
             if pkg.name == pkgName:
                 dependentPackages.extend(pkg.requires)
         return dependentPackages
 
     # Convert parsed data into SpecObject
     def createSpecObject(self):
         specObj = SpecObject()
         specObj.specFile = self.specfile
         defPkg = self.packages.get('default')
         specObj.name = defPkg.name
         specObj.version = defPkg.version
         specObj.release = defPkg.release
         specObj.checksums = defPkg.checksums
         specObj.license = defPkg.license
         specObj.url = defPkg.URL
         specObj.securityHardening = self.globalSecurityHardening
         specObj.isCheckAvailable = self.checkMacro is not None
         specObj.buildRequires = self._getRequiresTypeAllPackages("build")
         specObj.installRequires = self._getRequiresTypeAllPackages("install")
         specObj.checkBuildRequires = self._getCheckBuildRequiresAllPackages()
         specObj.extraBuildRequires = self._getExtraBuildRequires()
a007b861
         specObj.buildRequiresNative = self._getBuildRequiresNative()
4ed339b4
         specObj.listPackages = self._getPackageNames()
         specObj.listSources = self._getSourceNames()
         specObj.listPatches = self._getPatchNames()
         specObj.sourceurl = self._getSourceURL()
 
         for pkg in self.packages.values():
             specObj.installRequiresPackages[pkg.name] = pkg.requires
             specObj.buildarch[pkg.name] = pkg.buildarch
             if pkg.filesMacro:
                 specObj.listRPMPackages.append(pkg.name)
 
         return specObj