#!/usr/bin/env python3
import os
import re
from StringUtils import StringUtils
from constants import constants
from SpecStructures import dependentPackageData, Package, SpecObject
strUtils = StringUtils()
class SpecParser(object):
class rpmMacro(object):
def __init__(self):
self.macroName = ""
self.macroFlag = ""
self.content = ""
self.position = -1
self.endposition = -1
def __init__(self, specfile, arch):
self.arch = arch
self.cleanMacro = None
self.prepMacro = None
self.buildMacro = None
self.installMacro = None
self.changelogMacro = None
self.checkMacro = None
self.packages = {}
self.specAdditionalContent = ""
self.globalSecurityHardening = ""
self.defs = {}
self.defs["_arch"] = arch
self.conditionalCheckMacroEnabled = False
self.macro_pattern = re.compile(r"%{(\S+?)\}")
self.specfile = specfile
self.packages["default"] = Package(self.arch)
self.currentPkg = "default"
self._parseSpecFile(self.specfile)
def _parseSpecFile(self, file):
with open(file) as specFile:
lines = specFile.readlines()
totalLines = len(lines)
i = 0
def skip_conditional_body(line):
deep = 1
nonlocal i
while i < totalLines and deep:
i += 1
line = lines[i].strip()
if self._isConditionalMacroStart(line):
deep += 1
elif self._isConditionalMacroEnd(line):
deep -= 1
while i < totalLines:
line = lines[i].strip()
if self._isConditionalArch(line):
if self.arch != self._readConditionalArch(line):
skip_conditional_body(line)
elif self._isIfCondition(line):
if not self._isConditionTrue(line, file):
skip_conditional_body(line)
elif self._isSpecMacro(line):
macro, i = self._readMacroFromFile(i, lines)
self._updateSpecMacro(macro)
elif self._isPackageMacro(line):
defaultpkg = self.packages.get("default")
returnVal, packageName = self._readPkgNameFromPackageMacro(
line, defaultpkg.name
)
packageName = self._replaceMacros(packageName)
if not returnVal:
return False
if line.startswith("%package"):
pkg = Package(self.arch, defaultpkg)
pkg.name = packageName
self.currentPkg = packageName
self.packages[pkg.name] = pkg
else:
if defaultpkg.name == packageName:
packageName = "default"
macro, i = self._readMacroFromFile(i, lines)
if packageName not in self.packages:
i += 1
continue
self.packages[packageName].updatePackageMacro(macro)
elif self._isPackageHeaders(line):
self._readPackageHeaders(
line, self.packages[self.currentPkg]
)
elif self._isGlobalSecurityHardening(line):
self._readSecurityHardening(line)
elif self._isChecksum(line):
self._readChecksum(line, self.packages[self.currentPkg])
elif self._isExtraBuildRequires(line):
self._readExtraBuildRequires(
line, self.packages[self.currentPkg]
)
elif self._isBuildRequiresNative(line):
self._readBuildRequiresNative(
line, self.packages[self.currentPkg]
)
elif self._isDefinition(line):
self._readDefinition(line)
elif self._isConditionalCheckMacro(line):
self.conditionalCheckMacroEnabled = True
elif (
self.conditionalCheckMacroEnabled
and self._isConditionalMacroEnd(line)
):
self.conditionalCheckMacroEnabled = False
elif self._isInclude(line):
include = line.split()
if len(include) == 2:
includeFile = os.path.join(
os.path.dirname(file),
self._replaceMacros(include[1]),
)
# recursive parsing
self._parseSpecFile(includeFile)
else:
self.specAdditionalContent += f"{line}\n"
i += 1
def _readPkgNameFromPackageMacro(self, data, basePkgName=None):
data = " ".join(data.split())
pkgHeaderName = data.split(" ")
lenpkgHeaderName = len(pkgHeaderName)
i = 1
pkgName = None
while i < lenpkgHeaderName:
if pkgHeaderName[i] == "-n" and i + 1 < lenpkgHeaderName:
pkgName = pkgHeaderName[i + 1]
break
if pkgHeaderName[i].startswith("-"):
i += 2
else:
pkgName = f"{basePkgName}-{pkgHeaderName[i]}"
break
if not pkgName:
return True, basePkgName
return True, pkgName
def _replaceMacros(self, string):
"""
Replace all macros in given string with corresponding values.
For example: a string '%{name}-%{version}.tar.gz' will be
transformed to 'foo-2.0.tar.gz'.
:return A string where all macros in given input are substituted
as good as possible.
"""
def _is_conditional(macro):
return macro.startswith(("?", "!"))
def _test_conditional(macro):
if macro[0] == "?":
return True
if macro[0] == "!":
return False
raise Exception("Given string is not a conditional macro")
def _is_macro_defined(macro):
return (
(macro in self.defs.keys())
or (macro in constants.userDefinedMacros.keys())
or (
macro
in constants.getAdditionalMacros(
self.packages["default"].name
).keys()
)
)
def _get_macro(macro):
if macro in self.defs.keys():
return self.defs[macro]
if macro in constants.userDefinedMacros.keys():
return constants.userDefinedMacros[macro]
if (
macro
in constants.getAdditionalMacros(
self.packages["default"].name
).keys()
):
return constants.getAdditionalMacros(
self.packages["default"].name
)[macro]
raise Exception(f"Unknown macro: {macro}")
def _macro_repl(match):
macro_name = match.group(1)
if _is_conditional(macro_name):
parts = macro_name[1:].split(":")
assert parts
retv = ""
if _test_conditional(macro_name): # ?
if _is_macro_defined(parts[0]):
if len(parts) == 2:
retv = parts[1]
else:
retv = _get_macro(parts[0])
else: # !
if _is_macro_defined(parts[0]):
if len(parts) == 2:
retv = parts[1]
return retv
if _is_macro_defined(macro_name):
return _get_macro(macro_name)
return match.string[match.start() : match.end()] # noqa: E203
# User macros
for macroName, value in constants.userDefinedMacros.items():
macro = f"%{macroName}"
if string.find(macro) != -1:
string = string.replace(macro, value)
# Spec definitions
for macroName, value in self.defs.items():
macro = f"%{macroName}"
if string.find(macro) != -1:
string = string.replace(macro, value)
return re.sub(self.macro_pattern, _macro_repl, string)
def _readMacroFromFile(self, currentPos, lines):
macro = self.rpmMacro()
line = lines[currentPos]
macro.position = currentPos
macro.endposition = currentPos
endPos = len(lines)
line = " ".join(line.split())
flagindex = line.find(" ")
if flagindex != -1:
macro.macroFlag = line[flagindex + 1 :] # noqa: E203
macro.macroName = line[:flagindex]
else:
macro.macroName = line
if currentPos + 1 < len(lines) and self._isMacro(
lines[currentPos + 1]
):
return macro, currentPos
for j in range(currentPos + 1, endPos):
content = lines[j]
if j + 1 < endPos and self._isMacro(lines[j + 1]):
return macro, j
macro.content += f"{content}\n"
macro.endposition = j
return macro, endPos
def _updateSpecMacro(self, macro):
if macro.macroName == "%clean":
self.cleanMacro = macro
if macro.macroName == "%prep":
self.prepMacro = macro
if macro.macroName == "%build":
self.buildMacro = macro
if macro.macroName == "%install":
self.installMacro = macro
if macro.macroName == "%changelog":
self.changelogMacro = macro
if macro.macroName == "%check":
self.checkMacro = macro
def _isMacro(self, line):
return (
self._isPackageMacro(line)
or self._isSpecMacro(line)
or self._isConditionalMacroStart(line)
or self._isConditionalMacroEnd(line)
)
def _isConditionalArch(self, line):
return re.search("^%ifarch", line)
def _isSpecMacro(self, line):
return line.startswith(
("%clean", "%prep", "%build", "%install", "%changelog", "%check")
)
def _isPackageMacro(self, line):
line = line.strip()
return line.startswith(
("%post", "%postun", "%files", "%description", "%package")
)
def _isPackageHeaders(self, line):
headersPatterns = [
"^summary:",
"^name:",
"^group:",
"^license:",
"^version:",
"^release:",
"^distribution:",
"^requires:",
r"^requires\((pre|post|preun|postun)\):",
"^provides:",
"^obsoletes:",
"^conflicts:",
"^url:",
"^source[0-9]*:",
"^patch[0-9]*:",
"^buildrequires:",
"^buildprovides:",
"^buildarch:",
]
return any(
[re.search(r, line, flags=re.IGNORECASE) for r in headersPatterns]
)
def _isGlobalSecurityHardening(self, line):
return re.search(
"^%global *security_hardening", line, flags=re.IGNORECASE
)
def _isExtraBuildRequires(self, line):
return re.search(
"^%define *extrabuildrequires", line, flags=re.IGNORECASE
)
def _isBuildRequiresNative(self, line):
return re.search(
"^%define *buildrequiresnative", line, flags=re.IGNORECASE
)
def _isChecksum(self, line):
w = line.split()
if len(w) != 3 or w[0] != "%define" or w[1] not in {"sha1", "sha512"}:
return False
return re.match(".*=[a-z0-9]+$", w[2])
def _isDefinition(self, line):
return line.startswith(("%define", "%global"))
def _readConditionalArch(self, line):
w = line.split()
if len(w) == 2:
return w[1]
return None
def _readDefinition(self, line):
listDefines = line.split()
if len(listDefines) == 3:
self.defs[listDefines[1]] = self._replaceMacros(listDefines[2])
return True
return False
def _readHeader(self, line):
headerSplitIndex = line.find(":")
if headerSplitIndex + 1 == len(line):
print(line, "\nError:Invalid header")
return False, None, None
headerName = line[0:headerSplitIndex].lower()
headerContent = line[headerSplitIndex + 1 :].strip() # noqa: E203
return True, headerName, headerContent
def _readDependentPackageData(self, line):
listPackages = line.split(",")
listdependentpkgs = []
for line in listPackages:
line = strUtils.getStringInConditionalBrackets(line)
listContents = line.split()
totalContents = len(listContents)
i = 0
while i < totalContents:
dpkg = dependentPackageData()
compare = None
packageName = listContents[i]
provider = constants.providedBy.get(listContents[i], None)
if listContents[i].startswith("/"):
if not provider:
raise Exception(
f"Error in {self.specfile}\n"
f"What package does provide {listContents[i]} ? "
"Please modify providedBy in constants.py"
)
packageName = provider
i += 1
elif provider:
packageName = provider
i += 2
if i + 2 < len(listContents):
if listContents[i + 1] in (">=", "<=", "=", "<", ">"):
compare = listContents[i + 1]
dpkg.package = packageName
if compare:
dpkg.compare = compare
dpkg.version = listContents[i + 2]
i += 3
else:
i += 1
listdependentpkgs.append(dpkg)
return listdependentpkgs
def _readPackageHeaders(self, line, pkg):
returnVal, headerName, headerContent = self._readHeader(line)
if not returnVal:
return False
headerContent = self._replaceMacros(headerContent)
if headerName == "summary":
pkg.summary = headerContent
return True
if headerName == "name":
pkg.name = headerContent
if pkg == self.packages["default"]:
self.defs["name"] = pkg.name
return True
if headerName == "group":
pkg.group = headerContent
return True
if headerName == "license":
pkg.license = headerContent
return True
if headerName == "version":
pkg.version = headerContent
if pkg == self.packages["default"]:
self.defs["version"] = pkg.version
return True
if headerName == "buildarch":
pkg.buildarch = headerContent
return True
if headerName == "release":
pkg.release = headerContent
if pkg == self.packages["default"]:
self.defs["release"] = pkg.release
return True
if headerName == "distribution":
pkg.distribution = headerContent
return True
if headerName == "url":
pkg.URL = headerContent
return True
if "source" in headerName:
pkg.sources.append(headerContent)
sourceNum = headerName[6:]
self.defs[f"SOURCE{sourceNum}"] = headerContent
return True
if "patch" in headerName:
pkg.patches.append(headerContent)
return True
if (
headerName.startswith("requires")
or headerName == "provides"
or headerName == "obsoletes"
or headerName == "conflicts"
or headerName == "buildrequires"
or headerName == "buildprovides"
):
dpkg = self._readDependentPackageData(headerContent)
if not dpkg:
return False
if headerName.startswith("requires"):
pkg.requires.extend(dpkg)
if headerName == "obsoletes":
pkg.obsoletes.extend(dpkg)
elif headerName == "conflicts":
pkg.conflicts.extend(dpkg)
elif headerName == "provides":
pkg.provides.extend(dpkg)
elif headerName == "buildrequires":
if self.conditionalCheckMacroEnabled:
pkg.checkbuildrequires.extend(dpkg)
else:
pkg.buildrequires.extend(dpkg)
if headerName == "buildprovides":
pkg.buildprovides.extend(dpkg)
return True
return False
def _readSecurityHardening(self, line):
data = line.lower().strip()
words = data.split()
nrWords = len(words)
if nrWords != 3:
print(f"Error: Unable to parse line: {line}")
return False
if words[2] not in {"none", "nonow", "nopie", "nofortify"}:
print(f"Error: Invalid security_hardening value: {words[2]}")
return False
self.globalSecurityHardening = words[2]
return True
def _readExtraBuildRequires(self, line, pkg):
data = line.strip()
words = data.split(" ", 2)
if len(words) != 3:
print(f"Error: Unable to parse line: {line}")
return False
dpkg = self._readDependentPackageData(words[2])
if not dpkg:
return False
pkg.extrabuildrequires.extend(dpkg)
return True
def _readBuildRequiresNative(self, line, pkg):
data = line.strip()
words = data.split(" ", 2)
if len(words) != 3:
print(f"Error: Unable to parse line: {line}")
return False
dpkg = self._readDependentPackageData(words[2])
if not dpkg:
return False
pkg.buildrequiresnative.extend(dpkg)
return True
def _readChecksum(self, line, pkg):
line = self._replaceMacros(line)
data = line.strip()
words = data.split()
nrWords = len(words)
if nrWords != 3:
print(f"Error: Unable to parse line: {line}")
return False
value = words[2].split("=")
if len(value) != 2:
print(f"Error: Unable to parse line: {line}")
return False
matchedSources = []
for source in pkg.sources:
sourceName = strUtils.getFileNameFromURL(source)
if sourceName.startswith(value[0]):
matchedSources.append(sourceName)
if not matchedSources:
print(f"Error: Can not find match for checksum {value[0]}")
return False
if len(matchedSources) > 1:
print(
"Error: Too many matched Sources:"
+ " ".join(matchedSources)
+ f" for checksum {value[0]}"
)
return False
pkg.checksums[sourceName] = {words[1]: value[1]}
return True
def _isConditionalCheckMacro(self, line):
data = line.strip()
words = data.split()
if len(words) != 2:
return False
if words[0] != "%if" or "with_check" not in words[1]:
return False
return True
def _isIfCondition(self, line):
return line.startswith("%if ")
def _isConditionTrue(self, line, spec_fn):
words = line.strip().split()
if len(words) < 2:
raise Exception(f"Bad if condition {line} in {spec_fn}")
cond = ""
for w in words[1:]:
if w in {"==", ">", ">=", "<", "<=", "!=", "||", "&&"}:
if w == "||":
cond = f"{cond} or "
elif w == "&&":
cond = f"{cond} and "
else:
cond = f"{cond} {w} "
else:
val = self._replaceMacros(w).lstrip("0")
if not val:
val = "0"
cond = f"{cond} {val}"
cond = f"({cond}) != 0"
return eval(cond)
def _isConditionalMacroStart(self, line):
return line.startswith("%if")
def _isConditionalMacroEnd(self, line):
return line.strip() == "%endif"
def _isInclude(self, line):
return line.startswith("%include")
"""
SpecObject generating functions
@requiresType: "build" for BuildRequires or
"install" for Requires dependencies.
"""
def _getRequiresTypeAllPackages(self, requiresType):
dependentPackages = []
for pkg in self.packages.values():
if requiresType == "build":
dependentPackages.extend(pkg.buildrequires)
elif requiresType == "install":
dependentPackages.extend(pkg.requires)
listDependentPackages = dependentPackages.copy()
for pkg in self.packages.values():
for objName in listDependentPackages:
if objName.package == pkg.name:
dependentPackages.remove(objName)
return dependentPackages
def _getCheckBuildRequiresAllPackages(self):
dependentPackages = []
for pkg in self.packages.values():
dependentPackages.extend(pkg.checkbuildrequires)
return dependentPackages
def _getExtraBuildRequires(self):
dependentPackages = []
for pkg in self.packages.values():
dependentPackages.extend(pkg.extrabuildrequires)
return dependentPackages
def _getBuildRequiresNative(self):
dependentPackages = []
for pkg in self.packages.values():
dependentPackages.extend(pkg.buildrequiresnative)
return dependentPackages
def _getPackageNames(self):
packageNames = []
for pkg in self.packages.values():
packageNames.append(pkg.name)
return packageNames
def _getSourceNames(self):
sourceNames = []
pkg = self.packages.get("default")
for source in pkg.sources:
sourceName = strUtils.getFileNameFromURL(source)
sourceNames.append(sourceName)
return sourceNames
def _getPatchNames(self):
patchNames = []
pkg = self.packages.get("default")
for patch in pkg.patches:
patchName = strUtils.getFileNameFromURL(patch)
patchNames.append(patchName)
return patchNames
def _getSourceURL(self):
pkg = self.packages.get("default")
if not pkg.sources:
return None
sourceURL = pkg.sources[0]
if sourceURL.startswith("http") or sourceURL.startswith("ftp"):
return sourceURL
return None
def _getRequires(self, pkgName):
dependentPackages = []
for pkg in self.packages.values():
if pkg.name == pkgName:
dependentPackages.extend(pkg.requires)
return dependentPackages
# Convert parsed data into SpecObject
def createSpecObject(self):
specObj = SpecObject()
specObj.specFile = self.specfile
defPkg = self.packages.get("default")
specObj.name = defPkg.name
specObj.version = f"{defPkg.version}-{defPkg.release}"
specObj.release = defPkg.release
specObj.checksums = defPkg.checksums
specObj.license = defPkg.license
specObj.url = defPkg.URL
specObj.securityHardening = self.globalSecurityHardening
specObj.isCheckAvailable = self.checkMacro is not None
specObj.buildRequires = self._getRequiresTypeAllPackages("build")
specObj.installRequires = self._getRequiresTypeAllPackages("install")
specObj.checkBuildRequires = self._getCheckBuildRequiresAllPackages()
specObj.extraBuildRequires = self._getExtraBuildRequires()
specObj.buildRequiresNative = self._getBuildRequiresNative()
specObj.listPackages = self._getPackageNames()
specObj.listSources = self._getSourceNames()
specObj.listPatches = self._getPatchNames()
specObj.sourceurl = self._getSourceURL()
for pkg in self.packages.values():
specObj.installRequiresPackages[pkg.name] = pkg.requires
specObj.buildarch[pkg.name] = pkg.buildarch
if pkg.filesMacro:
specObj.listRPMPackages.append(pkg.name)
return specObj