zstd is fast compression algorithm, providing high compression
ratios.Zstd supports 1-22 levels, level 22 providing highest
compression/decompression ratio.
We provide option for user to give compression macro in config.json.
By default we use zstd19 which on testing has provided >7% increase
in build time and ~30% improvement in RPMS size. One can provided a
lower level for faster but more sizeable build and vice versa.
To install default toolchain rpms in chroot we use rpm package
present of the build system. To build photon on systems with
rpm package not having support for zstd. We create a photon_builder
container image with rpm installed to successfully run & install
default toolchain rpms in chroot.
Change-Id: Ib8a4a86738c39dc374522bdc04b545d29c8ff69f
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/12009
Tested-by: gerrit-photon <photon-checkins@vmware.com>
Reviewed-by: Keerthana K <keerthanak@vmware.com>
| ... | ... |
@@ -1,7 +1,7 @@ |
| 1 | 1 |
Name: drpm |
| 2 | 2 |
Summary: A library for making, reading and applying deltarpm packages |
| 3 | 3 |
Version: 0.5.0 |
| 4 |
-Release: 2%{?dist}
|
|
| 4 |
+Release: 3%{?dist}
|
|
| 5 | 5 |
License: LGPLv2+ and BSD |
| 6 | 6 |
URL: https://github.com/rpm-software-management/%{name}
|
| 7 | 7 |
Source0: https://github.com/rpm-software-management/drpm/releases/download/0.5.0/drpm-%{version}.tar.bz2
|
| ... | ... |
@@ -36,7 +36,7 @@ mkdir build |
| 36 | 36 |
|
| 37 | 37 |
%build |
| 38 | 38 |
pushd build |
| 39 |
-cmake .. -DWITH_ZSTD:BOOL=no -DHAVE_LZLIB_DEVEL:BOOL=0 -DCMAKE_INSTALL_PREFIX=/usr |
|
| 39 |
+cmake .. -DWITH_ZSTD:BOOL=yes -DHAVE_LZLIB_DEVEL:BOOL=0 -DCMAKE_INSTALL_PREFIX=/usr |
|
| 40 | 40 |
make %{?_smp_mflags}
|
| 41 | 41 |
popd |
| 42 | 42 |
|
| ... | ... |
@@ -64,6 +64,8 @@ popd |
| 64 | 64 |
%{_lib64dir}/pkgconfig/%{name}.pc
|
| 65 | 65 |
|
| 66 | 66 |
%changelog |
| 67 |
+* Fri Nov 13 2020 Prashant S Chauhan <psinghchauha@vmware.com> 0.5.0-3 |
|
| 68 |
+- make drpm build with zstd |
|
| 67 | 69 |
* Tue Sep 29 2020 Satya Naga Vasamsetty <svasamsetty@vmware.com> 0.5.0-2 |
| 68 | 70 |
- openssl 1.1.1 |
| 69 | 71 |
* Fri Jun 26 2020 Keerthana K <keerthanak@vmware.com> 0.5.0-1 |
| ... | ... |
@@ -1,7 +1,7 @@ |
| 1 | 1 |
Summary: A free package dependency solver |
| 2 | 2 |
Name: libsolv |
| 3 | 3 |
Version: 0.6.35 |
| 4 |
-Release: 6%{?dist}
|
|
| 4 |
+Release: 7%{?dist}
|
|
| 5 | 5 |
License: BSD |
| 6 | 6 |
URL: https://github.com/openSUSE/libsolv |
| 7 | 7 |
Source0: https://github.com/openSUSE/libsolv/archive/%{name}-%{version}.tar.gz
|
| ... | ... |
@@ -24,6 +24,8 @@ BuildRequires: libdb-devel |
| 24 | 24 |
BuildRequires: cmake |
| 25 | 25 |
BuildRequires: rpm-devel |
| 26 | 26 |
BuildRequires: expat-devel |
| 27 |
+BuildRequires: zstd-devel |
|
| 28 |
+ |
|
| 27 | 29 |
%description |
| 28 | 30 |
Libsolv is a free package management library, using SAT technology to solve requests. |
| 29 | 31 |
It supports debian, rpm, archlinux and haiku style distributions. |
| ... | ... |
@@ -81,6 +83,8 @@ make %{?_smp_mflags} test
|
| 81 | 81 |
%{_mandir}/man3/*
|
| 82 | 82 |
|
| 83 | 83 |
%changelog |
| 84 |
+* Wed Dec 09 2020 Prashant S Chauhan <psinghchauha@vmware.com> 0.6.35-7 |
|
| 85 |
+- Add zstd-devel as Build Requires |
|
| 84 | 86 |
* Thu Dec 03 2020 Tapas Kundu <tkundu@vmware.com> 0.6.35-6 |
| 85 | 87 |
- Further extend choicerule filtering check |
| 86 | 88 |
- Refactor solver addchoicerules function |
| ... | ... |
@@ -932,6 +932,7 @@ def initialize_constants(): |
| 932 | 932 |
Builder.get_packages_with_build_options(configdict['photon-build-param']['pkg-build-options']) |
| 933 | 933 |
Build_Config.setCommonDir(PurePath(curDir, "common", "data")) |
| 934 | 934 |
constants.setStartSchedulerServer(configdict["photon-build-param"]['start-scheduler-server']) |
| 935 |
+ constants.setCompressionMacro(configdict["photon-build-param"]["compression-macro"]) |
|
| 935 | 936 |
constants.initialize() |
| 936 | 937 |
|
| 937 | 938 |
|
| ... | ... |
@@ -8,7 +8,8 @@ |
| 8 | 8 |
"full-package-list-file" : "build_install_options_all.json", |
| 9 | 9 |
"pkg-build-options" : "pkg_build_options.json", |
| 10 | 10 |
"photon-docker-image" : "photon:3.0", |
| 11 |
- "target": "iso" |
|
| 11 |
+ "target": "iso", |
|
| 12 |
+ "compression-macro": "w19.zstdio" |
|
| 12 | 13 |
}, |
| 13 | 14 |
"photon-path" : "", |
| 14 | 15 |
"stage-path" : "" , |
| ... | ... |
@@ -80,6 +80,12 @@ Build config file is a json format with possible parameters: |
| 80 | 80 |
Default value: [Makefile targets] |
| 81 | 81 |
Example: { "photon-build-param": { "target" : "iso" } }
|
| 82 | 82 |
|
| 83 |
+ # zstd level range from 1-22, with increase in level means more compression ratio and more time |
|
| 84 |
+ "compression-macro": |
|
| 85 |
+ Default value: "w19.zstdio" |
|
| 86 |
+ values: ["w[1-22].zstdio", "gzip9"] |
|
| 87 |
+ Example: { "photon-build-param": { "compression-macro" : "w19.zstdio" } }
|
|
| 88 |
+ |
|
| 83 | 89 |
"input-rpms-path": |
| 84 | 90 |
Default: [src-root]/inputRPMS |
| 85 | 91 |
Example: { "input-rpms-path": [input RPMS path] }
|
| ... | ... |
@@ -32,6 +32,20 @@ class PackageManager(object): |
| 32 | 32 |
if self.pkgBuildType == "container": |
| 33 | 33 |
import docker |
| 34 | 34 |
self.dockerClient = docker.from_env(version="auto") |
| 35 |
+ cmdUtils = CommandUtils() |
|
| 36 |
+ # if rpm doesnt have zstd support |
|
| 37 |
+ if cmdUtils.runCommandInShell('rpm --showrc | grep -i "rpmlib(PayloadIsZstd)"', logfn=self.logger.debug):
|
|
| 38 |
+ self.createZstdBuilderImage() |
|
| 39 |
+ |
|
| 40 |
+ def createZstdBuilderImage(self): |
|
| 41 |
+ import docker |
|
| 42 |
+ self.dockerClient = docker.from_env(version="auto") |
|
| 43 |
+ self.logger.info("creating photon builder docker image")
|
|
| 44 |
+ image = self.dockerClient.images.build(tag='photon_builder:latest', |
|
| 45 |
+ path="./support/package-builder", |
|
| 46 |
+ rm=True, |
|
| 47 |
+ dockerfile="Dockerfile.photon_builder") |
|
| 48 |
+ self.logger.debug("Created Image %s"%image)
|
|
| 35 | 49 |
|
| 36 | 50 |
def buildToolChain(self): |
| 37 | 51 |
self.logger.info("Step 1 : Building the core toolchain packages for " + constants.currentArch)
|
| ... | ... |
@@ -126,6 +126,13 @@ class ToolChainUtils(object): |
| 126 | 126 |
self.logger.debug(packages) |
| 127 | 127 |
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " + |
| 128 | 128 |
chroot.getID() +" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles) |
| 129 |
+ |
|
| 130 |
+ # if rpm doesnt has zstd support |
|
| 131 |
+ if CommandUtils.runCommandInShell('rpm --showrc | grep -i "rpmlib(PayloadIsZstd)"', logfn=self.logger.debug):
|
|
| 132 |
+ cmd = ("docker run -i -v " + constants.prevPublishRPMRepo + ":" + constants.prevPublishRPMRepo +
|
|
| 133 |
+ " -v " + constants.rpmPath + ":" + constants.rpmPath + " -v " + chroot.getID() + ":" + |
|
| 134 |
+ chroot.getID() + " photon_builder:latest " + "/bin/bash -c \"" + cmd + "\"") |
|
| 135 |
+ |
|
| 129 | 136 |
retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug) |
| 130 | 137 |
if retVal != 0: |
| 131 | 138 |
self.logger.debug("Command Executed:" + cmd)
|
| ... | ... |
@@ -442,6 +442,11 @@ class constants(object): |
| 442 | 442 |
constants.katBuild = katBuild |
| 443 | 443 |
|
| 444 | 444 |
@staticmethod |
| 445 |
+ def setCompressionMacro(compressionMacro): |
|
| 446 |
+ constants.addMacro("_source_payload", compressionMacro)
|
|
| 447 |
+ constants.addMacro("_binary_payload", compressionMacro)
|
|
| 448 |
+ |
|
| 449 |
+ @staticmethod |
|
| 445 | 450 |
def initialize(): |
| 446 | 451 |
if constants.rpmCheck: |
| 447 | 452 |
constants.testLogger = Logger.getLogger("MakeCheckTest",
|
| ... | ... |
@@ -25,10 +25,10 @@ aarch64/gawk-4.1.4-2.ph3.aarch64.rpm |
| 25 | 25 |
aarch64/gcc-7.3.0-2.ph3.aarch64.rpm |
| 26 | 26 |
aarch64/gdbm-1.13-3.ph2.aarch64.rpm |
| 27 | 27 |
aarch64/gettext-0.19.8-1.ph2.aarch64.rpm |
| 28 |
-aarch64/glibc-2.28-3.ph3.aarch64.rpm |
|
| 29 |
-aarch64/glibc-devel-2.28-3.ph3.aarch64.rpm |
|
| 30 |
-aarch64/glibc-iconv-2.28-3.ph3.aarch64.rpm |
|
| 31 |
-aarch64/glibc-tools-2.28-3.ph3.aarch64.rpm |
|
| 28 |
+aarch64/glibc-2.32-2.ph4.aarch64.rpm |
|
| 29 |
+aarch64/glibc-devel-2.32-2.ph4.aarch64.rpm |
|
| 30 |
+aarch64/glibc-iconv-2.32-2.ph4.aarch64.rpm |
|
| 31 |
+aarch64/glibc-tools-2.32-2.ph4.aarch64.rpm |
|
| 32 | 32 |
aarch64/gmp-6.1.2-3.ph3.aarch64.rpm |
| 33 | 33 |
aarch64/gmp-devel-6.1.2-3.ph3.aarch64.rpm |
| 34 | 34 |
aarch64/go-1.8.1-2.ph2.aarch64.rpm |
| ... | ... |
@@ -78,10 +78,10 @@ aarch64/python2-devel-2.7.18-1.ph3.aarch64.rpm |
| 78 | 78 |
aarch64/python-xml-2.7.18-1.ph3.aarch64.rpm |
| 79 | 79 |
aarch64/readline-7.0-2.ph2.aarch64.rpm |
| 80 | 80 |
aarch64/readline-devel-7.0-2.ph2.aarch64.rpm |
| 81 |
-aarch64/rpm-4.13.0.1-5.ph2.aarch64.rpm |
|
| 82 |
-aarch64/rpm-build-4.13.0.1-5.ph2.aarch64.rpm |
|
| 83 |
-aarch64/rpm-devel-4.13.0.1-5.ph2.aarch64.rpm |
|
| 84 |
-aarch64/rpm-libs-4.13.0.1-5.ph2.aarch64.rpm |
|
| 81 |
+aarch64/rpm-4.14.2-9.ph3.aarch64.rpm |
|
| 82 |
+aarch64/rpm-build-4.14.2-9.ph3.aarch64.rpm |
|
| 83 |
+aarch64/rpm-devel-4.14.2-9.ph3.aarch64.rpm |
|
| 84 |
+aarch64/rpm-libs-4.14.2-9.ph3.aarch64.rpm |
|
| 85 | 85 |
aarch64/sed-4.4-2.ph2.aarch64.rpm |
| 86 | 86 |
aarch64/sqlite-3.19.3-1.ph2.aarch64.rpm |
| 87 | 87 |
aarch64/sqlite-libs-3.19.3-1.ph2.aarch64.rpm |
| ... | ... |
@@ -6,8 +6,8 @@ x86_64/python2-libs-2.7.18-1.ph3.x86_64.rpm |
| 6 | 6 |
x86_64/python2-devel-2.7.18-1.ph3.x86_64.rpm |
| 7 | 7 |
x86_64/python-xml-2.7.18-1.ph3.x86_64.rpm |
| 8 | 8 |
x86_64/nspr-4.11-1.ph1.x86_64.rpm |
| 9 |
-x86_64/rpm-4.13.0.1-5.ph2.x86_64.rpm |
|
| 10 |
-x86_64/rpm-libs-4.13.0.1-5.ph2.x86_64.rpm |
|
| 9 |
+x86_64/rpm-4.14.2-9.ph3.x86_64.rpm |
|
| 10 |
+x86_64/rpm-libs-4.14.2-9.ph3.x86_64.rpm |
|
| 11 | 11 |
x86_64/binutils-devel-2.31-9.ph1.x86_64.rpm |
| 12 | 12 |
x86_64/nss-3.21-1.ph1.x86_64.rpm |
| 13 | 13 |
x86_64/mpfr-devel-4.0.1-1.ph2.x86_64.rpm |
| ... | ... |
@@ -27,7 +27,7 @@ x86_64/man-db-2.7.5-1.ph1.x86_64.rpm |
| 27 | 27 |
x86_64/elfutils-libelf-0.169-2.ph2.x86_64.rpm |
| 28 | 28 |
x86_64/gdbm-1.11-1.ph1.x86_64.rpm |
| 29 | 29 |
x86_64/autoconf-2.69-3.ph1.x86_64.rpm |
| 30 |
-x86_64/rpm-devel-4.13.0.1-5.ph2.x86_64.rpm |
|
| 30 |
+x86_64/rpm-devel-4.14.2-9.ph3.x86_64.rpm |
|
| 31 | 31 |
x86_64/file-5.24-1.ph1.x86_64.rpm |
| 32 | 32 |
x86_64/mpfr-4.0.1-1.ph2.x86_64.rpm |
| 33 | 33 |
x86_64/zlib-devel-1.2.8-2.ph1.x86_64.rpm |
| ... | ... |
@@ -36,7 +36,7 @@ x86_64/libgcc-atomic-7.3.0-2.ph2.x86_64.rpm |
| 36 | 36 |
x86_64/bison-3.0.4-1.ph1.x86_64.rpm |
| 37 | 37 |
x86_64/lua-5.3.2-1.ph1.x86_64.rpm |
| 38 | 38 |
x86_64/util-linux-2.27.1-1.ph1.x86_64.rpm |
| 39 |
-x86_64/rpm-build-4.13.0.1-5.ph2.x86_64.rpm |
|
| 39 |
+x86_64/rpm-build-4.14.2-9.ph3.x86_64.rpm |
|
| 40 | 40 |
x86_64/procps-ng-3.3.11-1.ph1.x86_64.rpm |
| 41 | 41 |
x86_64/coreutils-8.24-1.ph1.x86_64.rpm |
| 42 | 42 |
x86_64/util-linux-devel-2.27.1-1.ph1.x86_64.rpm |
| ... | ... |
@@ -59,7 +59,7 @@ x86_64/expat-2.1.0-1.ph1.x86_64.rpm |
| 59 | 59 |
x86_64/popt-1.16-1.ph1.x86_64.rpm |
| 60 | 60 |
x86_64/mpc-1.0.3-1.ph1.x86_64.rpm |
| 61 | 61 |
x86_64/bzip2-1.0.6-4.ph1.x86_64.rpm |
| 62 |
-x86_64/glibc-2.22-3.ph1.x86_64.rpm |
|
| 62 |
+x86_64/glibc-2.32-1.ph4.x86_64.rpm |
|
| 63 | 63 |
x86_64/elfutils-devel-0.169-2.ph2.x86_64.rpm |
| 64 | 64 |
x86_64/patch-2.7.5-1.ph1.x86_64.rpm |
| 65 | 65 |
x86_64/xz-5.2.2-1.ph1.x86_64.rpm |
| ... | ... |
@@ -78,7 +78,7 @@ x86_64/findutils-4.4.2-1.ph1.x86_64.rpm |
| 78 | 78 |
x86_64/libstdc++-7.3.0-2.ph2.x86_64.rpm |
| 79 | 79 |
x86_64/libpipeline-1.4.1-1.ph1.x86_64.rpm |
| 80 | 80 |
x86_64/make-4.1-1.ph1.x86_64.rpm |
| 81 |
-x86_64/glibc-devel-2.22-3.ph1.x86_64.rpm |
|
| 81 |
+x86_64/glibc-devel-2.32-1.ph4.x86_64.rpm |
|
| 82 | 82 |
x86_64/lua-devel-5.3.2-1.ph1.x86_64.rpm |
| 83 | 83 |
x86_64/ncurses-6.0-1.ph1.x86_64.rpm |
| 84 | 84 |
x86_64/texinfo-6.1-1.ph1.x86_64.rpm |