Browse code

package-builder: Use zstd as compression type

zstd is fast compression algorithm, providing high compression
ratios.Zstd supports 1-22 levels, level 22 providing highest
compression/decompression ratio.

We provide option for user to give compression macro in config.json.
By default we use zstd19 which on testing has provided >7% increase
in build time and ~30% improvement in RPMS size. One can provided a
lower level for faster but more sizeable build and vice versa.

To install default toolchain rpms in chroot we use rpm package
present of the build system. To build photon on systems with
rpm package not having support for zstd. We create a photon_builder
container image with rpm installed to successfully run & install
default toolchain rpms in chroot.

Change-Id: Ib8a4a86738c39dc374522bdc04b545d29c8ff69f
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/12009
Tested-by: gerrit-photon <photon-checkins@vmware.com>
Reviewed-by: Keerthana K <keerthanak@vmware.com>

Prashant Singh Chauhan authored on 2020/12/10 09:07:34
Showing 11 changed files
... ...
@@ -1,7 +1,7 @@
1 1
 Name:            drpm
2 2
 Summary:         A library for making, reading and applying deltarpm packages
3 3
 Version:         0.5.0
4
-Release:         2%{?dist}
4
+Release:         3%{?dist}
5 5
 License:         LGPLv2+ and BSD
6 6
 URL:             https://github.com/rpm-software-management/%{name}
7 7
 Source0:         https://github.com/rpm-software-management/drpm/releases/download/0.5.0/drpm-%{version}.tar.bz2
... ...
@@ -36,7 +36,7 @@ mkdir build
36 36
 
37 37
 %build
38 38
 pushd build
39
-cmake .. -DWITH_ZSTD:BOOL=no -DHAVE_LZLIB_DEVEL:BOOL=0 -DCMAKE_INSTALL_PREFIX=/usr
39
+cmake .. -DWITH_ZSTD:BOOL=yes -DHAVE_LZLIB_DEVEL:BOOL=0 -DCMAKE_INSTALL_PREFIX=/usr
40 40
 make %{?_smp_mflags}
41 41
 popd
42 42
 
... ...
@@ -64,6 +64,8 @@ popd
64 64
 %{_lib64dir}/pkgconfig/%{name}.pc
65 65
 
66 66
 %changelog
67
+*   Fri Nov 13 2020 Prashant S Chauhan <psinghchauha@vmware.com> 0.5.0-3
68
+-   make drpm build with zstd
67 69
 *   Tue Sep 29 2020 Satya Naga Vasamsetty <svasamsetty@vmware.com> 0.5.0-2
68 70
 -   openssl 1.1.1
69 71
 *   Fri Jun 26 2020 Keerthana K <keerthanak@vmware.com> 0.5.0-1
... ...
@@ -1,7 +1,7 @@
1 1
 Summary:        A free package dependency solver
2 2
 Name:           libsolv
3 3
 Version:        0.6.35
4
-Release:        6%{?dist}
4
+Release:        7%{?dist}
5 5
 License:        BSD
6 6
 URL:            https://github.com/openSUSE/libsolv
7 7
 Source0:        https://github.com/openSUSE/libsolv/archive/%{name}-%{version}.tar.gz
... ...
@@ -24,6 +24,8 @@ BuildRequires:  libdb-devel
24 24
 BuildRequires:  cmake
25 25
 BuildRequires:  rpm-devel
26 26
 BuildRequires:  expat-devel
27
+BuildRequires:  zstd-devel
28
+
27 29
 %description
28 30
 Libsolv is a free package management library, using SAT technology to solve requests.
29 31
 It supports debian, rpm, archlinux and haiku style distributions.
... ...
@@ -81,6 +83,8 @@ make %{?_smp_mflags} test
81 81
 %{_mandir}/man3/*
82 82
 
83 83
 %changelog
84
+*   Wed Dec 09 2020 Prashant S Chauhan <psinghchauha@vmware.com> 0.6.35-7
85
+-   Add zstd-devel as Build Requires
84 86
 *   Thu Dec 03 2020 Tapas Kundu <tkundu@vmware.com> 0.6.35-6
85 87
 -   Further extend choicerule filtering check
86 88
 -   Refactor solver addchoicerules function
... ...
@@ -932,6 +932,7 @@ def initialize_constants():
932 932
     Builder.get_packages_with_build_options(configdict['photon-build-param']['pkg-build-options'])
933 933
     Build_Config.setCommonDir(PurePath(curDir, "common", "data"))
934 934
     constants.setStartSchedulerServer(configdict["photon-build-param"]['start-scheduler-server'])
935
+    constants.setCompressionMacro(configdict["photon-build-param"]["compression-macro"])
935 936
     constants.initialize()
936 937
 
937 938
 
... ...
@@ -8,7 +8,8 @@
8 8
             "full-package-list-file" : "build_install_options_all.json",
9 9
             "pkg-build-options" : "pkg_build_options.json",
10 10
             "photon-docker-image" : "photon:3.0",
11
-            "target": "iso"
11
+            "target": "iso",
12
+            "compression-macro": "w19.zstdio"
12 13
     },
13 14
     "photon-path" : "",
14 15
     "stage-path" : "" ,
... ...
@@ -80,6 +80,12 @@ Build config file is a json format with possible parameters:
80 80
         Default value: [Makefile targets]
81 81
         Example: { "photon-build-param": { "target" : "iso" } }
82 82
 
83
+  # zstd level range from 1-22, with increase in level means more compression ratio and more time
84
+  "compression-macro":
85
+        Default value: "w19.zstdio"
86
+        values: ["w[1-22].zstdio", "gzip9"]
87
+        Example: { "photon-build-param": { "compression-macro" : "w19.zstdio" } }
88
+
83 89
 "input-rpms-path":
84 90
         Default: [src-root]/inputRPMS
85 91
         Example: { "input-rpms-path": [input RPMS path] }
86 92
new file mode 100644
... ...
@@ -0,0 +1,2 @@
0
+FROM photon:latest
1
+RUN tdnf install -y rpm
... ...
@@ -32,6 +32,20 @@ class PackageManager(object):
32 32
         if self.pkgBuildType == "container":
33 33
             import docker
34 34
             self.dockerClient = docker.from_env(version="auto")
35
+        cmdUtils = CommandUtils()
36
+        # if rpm doesnt have zstd support
37
+        if cmdUtils.runCommandInShell('rpm --showrc | grep -i "rpmlib(PayloadIsZstd)"', logfn=self.logger.debug):
38
+            self.createZstdBuilderImage()
39
+
40
+    def createZstdBuilderImage(self):
41
+        import docker
42
+        self.dockerClient = docker.from_env(version="auto")
43
+        self.logger.info("creating photon builder docker image")
44
+        image = self.dockerClient.images.build(tag='photon_builder:latest',
45
+                                       path="./support/package-builder",
46
+                                       rm=True,
47
+                                       dockerfile="Dockerfile.photon_builder")
48
+        self.logger.debug("Created Image %s"%image)
35 49
 
36 50
     def buildToolChain(self):
37 51
         self.logger.info("Step 1 : Building the core toolchain packages for " + constants.currentArch)
... ...
@@ -126,6 +126,13 @@ class ToolChainUtils(object):
126 126
         self.logger.debug(packages)
127 127
         cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
128 128
                chroot.getID() +" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles)
129
+
130
+        # if rpm doesnt has zstd support
131
+        if CommandUtils.runCommandInShell('rpm --showrc | grep -i "rpmlib(PayloadIsZstd)"', logfn=self.logger.debug):
132
+            cmd = ("docker run -i -v " + constants.prevPublishRPMRepo + ":" + constants.prevPublishRPMRepo +
133
+                   " -v " + constants.rpmPath + ":" + constants.rpmPath + " -v " + chroot.getID() + ":" +
134
+                   chroot.getID() + " photon_builder:latest " + "/bin/bash -c \"" + cmd + "\"")
135
+
129 136
         retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
130 137
         if retVal != 0:
131 138
             self.logger.debug("Command Executed:" + cmd)
... ...
@@ -442,6 +442,11 @@ class constants(object):
442 442
         constants.katBuild = katBuild
443 443
 
444 444
     @staticmethod
445
+    def setCompressionMacro(compressionMacro):
446
+        constants.addMacro("_source_payload", compressionMacro)
447
+        constants.addMacro("_binary_payload", compressionMacro)
448
+
449
+    @staticmethod
445 450
     def initialize():
446 451
         if constants.rpmCheck:
447 452
             constants.testLogger = Logger.getLogger("MakeCheckTest",
... ...
@@ -25,10 +25,10 @@ aarch64/gawk-4.1.4-2.ph3.aarch64.rpm
25 25
 aarch64/gcc-7.3.0-2.ph3.aarch64.rpm
26 26
 aarch64/gdbm-1.13-3.ph2.aarch64.rpm
27 27
 aarch64/gettext-0.19.8-1.ph2.aarch64.rpm
28
-aarch64/glibc-2.28-3.ph3.aarch64.rpm
29
-aarch64/glibc-devel-2.28-3.ph3.aarch64.rpm
30
-aarch64/glibc-iconv-2.28-3.ph3.aarch64.rpm
31
-aarch64/glibc-tools-2.28-3.ph3.aarch64.rpm
28
+aarch64/glibc-2.32-2.ph4.aarch64.rpm
29
+aarch64/glibc-devel-2.32-2.ph4.aarch64.rpm
30
+aarch64/glibc-iconv-2.32-2.ph4.aarch64.rpm
31
+aarch64/glibc-tools-2.32-2.ph4.aarch64.rpm
32 32
 aarch64/gmp-6.1.2-3.ph3.aarch64.rpm
33 33
 aarch64/gmp-devel-6.1.2-3.ph3.aarch64.rpm
34 34
 aarch64/go-1.8.1-2.ph2.aarch64.rpm
... ...
@@ -78,10 +78,10 @@ aarch64/python2-devel-2.7.18-1.ph3.aarch64.rpm
78 78
 aarch64/python-xml-2.7.18-1.ph3.aarch64.rpm
79 79
 aarch64/readline-7.0-2.ph2.aarch64.rpm
80 80
 aarch64/readline-devel-7.0-2.ph2.aarch64.rpm
81
-aarch64/rpm-4.13.0.1-5.ph2.aarch64.rpm
82
-aarch64/rpm-build-4.13.0.1-5.ph2.aarch64.rpm
83
-aarch64/rpm-devel-4.13.0.1-5.ph2.aarch64.rpm
84
-aarch64/rpm-libs-4.13.0.1-5.ph2.aarch64.rpm
81
+aarch64/rpm-4.14.2-9.ph3.aarch64.rpm
82
+aarch64/rpm-build-4.14.2-9.ph3.aarch64.rpm
83
+aarch64/rpm-devel-4.14.2-9.ph3.aarch64.rpm
84
+aarch64/rpm-libs-4.14.2-9.ph3.aarch64.rpm
85 85
 aarch64/sed-4.4-2.ph2.aarch64.rpm
86 86
 aarch64/sqlite-3.19.3-1.ph2.aarch64.rpm
87 87
 aarch64/sqlite-libs-3.19.3-1.ph2.aarch64.rpm
... ...
@@ -6,8 +6,8 @@ x86_64/python2-libs-2.7.18-1.ph3.x86_64.rpm
6 6
 x86_64/python2-devel-2.7.18-1.ph3.x86_64.rpm
7 7
 x86_64/python-xml-2.7.18-1.ph3.x86_64.rpm
8 8
 x86_64/nspr-4.11-1.ph1.x86_64.rpm
9
-x86_64/rpm-4.13.0.1-5.ph2.x86_64.rpm
10
-x86_64/rpm-libs-4.13.0.1-5.ph2.x86_64.rpm
9
+x86_64/rpm-4.14.2-9.ph3.x86_64.rpm
10
+x86_64/rpm-libs-4.14.2-9.ph3.x86_64.rpm
11 11
 x86_64/binutils-devel-2.31-9.ph1.x86_64.rpm
12 12
 x86_64/nss-3.21-1.ph1.x86_64.rpm
13 13
 x86_64/mpfr-devel-4.0.1-1.ph2.x86_64.rpm
... ...
@@ -27,7 +27,7 @@ x86_64/man-db-2.7.5-1.ph1.x86_64.rpm
27 27
 x86_64/elfutils-libelf-0.169-2.ph2.x86_64.rpm
28 28
 x86_64/gdbm-1.11-1.ph1.x86_64.rpm
29 29
 x86_64/autoconf-2.69-3.ph1.x86_64.rpm
30
-x86_64/rpm-devel-4.13.0.1-5.ph2.x86_64.rpm
30
+x86_64/rpm-devel-4.14.2-9.ph3.x86_64.rpm
31 31
 x86_64/file-5.24-1.ph1.x86_64.rpm
32 32
 x86_64/mpfr-4.0.1-1.ph2.x86_64.rpm
33 33
 x86_64/zlib-devel-1.2.8-2.ph1.x86_64.rpm
... ...
@@ -36,7 +36,7 @@ x86_64/libgcc-atomic-7.3.0-2.ph2.x86_64.rpm
36 36
 x86_64/bison-3.0.4-1.ph1.x86_64.rpm
37 37
 x86_64/lua-5.3.2-1.ph1.x86_64.rpm
38 38
 x86_64/util-linux-2.27.1-1.ph1.x86_64.rpm
39
-x86_64/rpm-build-4.13.0.1-5.ph2.x86_64.rpm
39
+x86_64/rpm-build-4.14.2-9.ph3.x86_64.rpm
40 40
 x86_64/procps-ng-3.3.11-1.ph1.x86_64.rpm
41 41
 x86_64/coreutils-8.24-1.ph1.x86_64.rpm
42 42
 x86_64/util-linux-devel-2.27.1-1.ph1.x86_64.rpm
... ...
@@ -59,7 +59,7 @@ x86_64/expat-2.1.0-1.ph1.x86_64.rpm
59 59
 x86_64/popt-1.16-1.ph1.x86_64.rpm
60 60
 x86_64/mpc-1.0.3-1.ph1.x86_64.rpm
61 61
 x86_64/bzip2-1.0.6-4.ph1.x86_64.rpm
62
-x86_64/glibc-2.22-3.ph1.x86_64.rpm
62
+x86_64/glibc-2.32-1.ph4.x86_64.rpm
63 63
 x86_64/elfutils-devel-0.169-2.ph2.x86_64.rpm
64 64
 x86_64/patch-2.7.5-1.ph1.x86_64.rpm
65 65
 x86_64/xz-5.2.2-1.ph1.x86_64.rpm
... ...
@@ -78,7 +78,7 @@ x86_64/findutils-4.4.2-1.ph1.x86_64.rpm
78 78
 x86_64/libstdc++-7.3.0-2.ph2.x86_64.rpm
79 79
 x86_64/libpipeline-1.4.1-1.ph1.x86_64.rpm
80 80
 x86_64/make-4.1-1.ph1.x86_64.rpm
81
-x86_64/glibc-devel-2.22-3.ph1.x86_64.rpm
81
+x86_64/glibc-devel-2.32-1.ph4.x86_64.rpm
82 82
 x86_64/lua-devel-5.3.2-1.ph1.x86_64.rpm
83 83
 x86_64/ncurses-6.0-1.ph1.x86_64.rpm
84 84
 x86_64/texinfo-6.1-1.ph1.x86_64.rpm