These weights are the time taken (in minutes) to build each package on
the same machine, with THREADS=1. The packages not mentioned in this
list could be built in less than a minute.
When consuming these weights in the scheduler, we offset them by 1,
so that the minimum package weight would be 1 (which makes certain
scheduler calculations easier, as opposed to a minimum of 0).
Also, remove the obsolete file pkg_build_weights.json
Change-Id: I0239f69cc8de262580b2ab9758fb7a3a315ff03b
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/6161
Reviewed-by: Alexey Makhalov <amakhalov@vmware.com>
Tested-by: Anish Swaminathan <anishs@vmware.com>
... | ... |
@@ -1,13 +1,182 @@ |
1 | 1 |
{ |
2 |
- "ceph": 83.7, |
|
3 |
- "mesos": 75, |
|
4 |
- "linux": 41.2, |
|
5 |
- "linux-secure": 32, |
|
6 |
- "openjdk8": 21.55, |
|
7 |
- "kubernetes": 21.3, |
|
8 |
- "llvm": 20.267, |
|
9 |
- "linux-esx": 16.73, |
|
10 |
- "falco": 13.85, |
|
11 |
- "guile": 11.52, |
|
12 |
- "clang": 11.3 |
|
2 |
+ "mongodb": 155, |
|
3 |
+ "llvm": 129, |
|
4 |
+ "linux": 124, |
|
5 |
+ "linux-secure": 115, |
|
6 |
+ "linux-aws": 112, |
|
7 |
+ "clang": 109, |
|
8 |
+ "mysql": 103, |
|
9 |
+ "openjdk10": 98, |
|
10 |
+ "mariadb": 94, |
|
11 |
+ "boost": 82, |
|
12 |
+ "nodejs": 63, |
|
13 |
+ "envoy": 63, |
|
14 |
+ "lldb": 58, |
|
15 |
+ "openjdk8": 57, |
|
16 |
+ "linux-esx": 56, |
|
17 |
+ "kubernetes": 48, |
|
18 |
+ "protobuf": 41, |
|
19 |
+ "kubernetes": 39, |
|
20 |
+ "guile": 25, |
|
21 |
+ "elasticsearch": 22, |
|
22 |
+ "gdb": 18, |
|
23 |
+ "lightwave": 17, |
|
24 |
+ "libvirt": 13, |
|
25 |
+ "lapack": 12, |
|
26 |
+ "sysdig": 9, |
|
27 |
+ "ruby": 9, |
|
28 |
+ "go": 9, |
|
29 |
+ "systemd": 8, |
|
30 |
+ "syslog-ng": 8, |
|
31 |
+ "xerces-c": 7, |
|
32 |
+ "subversion": 7, |
|
33 |
+ "python-numpy": 7, |
|
34 |
+ "postgresql": 7, |
|
35 |
+ "perl": 7, |
|
36 |
+ "nss": 7, |
|
37 |
+ "kubernetes-dashboard": 7, |
|
38 |
+ "go": 7, |
|
39 |
+ "gnutls": 7, |
|
40 |
+ "glibmm": 7, |
|
41 |
+ "gettext": 7, |
|
42 |
+ "erlang": 7, |
|
43 |
+ "crash": 7, |
|
44 |
+ "systemtap": 6, |
|
45 |
+ "openvswitch": 6, |
|
46 |
+ "likewise-open": 6, |
|
47 |
+ "grub2": 6, |
|
48 |
+ "python-lxml": 5, |
|
49 |
+ "python3": 5, |
|
50 |
+ "powershell": 5, |
|
51 |
+ "icu": 5, |
|
52 |
+ "grpc": 5, |
|
53 |
+ "glib": 5, |
|
54 |
+ "xfsprogs": 4, |
|
55 |
+ "valgrind": 4, |
|
56 |
+ "thin-provisioning-tools": 4, |
|
57 |
+ "syslinux": 4, |
|
58 |
+ "python-gevent": 4, |
|
59 |
+ "open-vm-tools": 4, |
|
60 |
+ "openssl": 4, |
|
61 |
+ "openscap": 4, |
|
62 |
+ "net-snmp": 4, |
|
63 |
+ "httpd": 4, |
|
64 |
+ "git": 4, |
|
65 |
+ "docker": 4, |
|
66 |
+ "coreutils": 4, |
|
67 |
+ "cassandra": 4, |
|
68 |
+ "apache-maven": 4, |
|
69 |
+ "xml-security-c": 3, |
|
70 |
+ "vim": 3, |
|
71 |
+ "util-linux": 3, |
|
72 |
+ "strongswan": 3, |
|
73 |
+ "python2": 3, |
|
74 |
+ "mc": 3, |
|
75 |
+ "libunistring": 3, |
|
76 |
+ "libdb": 3, |
|
77 |
+ "krb5": 3, |
|
78 |
+ "gperftools": 3, |
|
79 |
+ "gobject-introspection": 3, |
|
80 |
+ "gobgp": 3, |
|
81 |
+ "cython": 3, |
|
82 |
+ "cython3": 3, |
|
83 |
+ "calico-libnetwork": 3, |
|
84 |
+ "calico-felix": 3, |
|
85 |
+ "calico-cni": 3, |
|
86 |
+ "calico": 3, |
|
87 |
+ "cairo": 3, |
|
88 |
+ "zsh": 2, |
|
89 |
+ "wavefront-proxy": 2, |
|
90 |
+ "unixODBC": 2, |
|
91 |
+ "unbound": 2, |
|
92 |
+ "tclap": 2, |
|
93 |
+ "rpm": 2, |
|
94 |
+ "openldap": 2, |
|
95 |
+ "openipmi": 2, |
|
96 |
+ "ntp": 2, |
|
97 |
+ "lttng-tools": 2, |
|
98 |
+ "libxml2": 2, |
|
99 |
+ "libnl": 2, |
|
100 |
+ "kubernetes-metrics-server": 2, |
|
101 |
+ "kubernetes-dns": 2, |
|
102 |
+ "kapacitor": 2, |
|
103 |
+ "json_spirit": 2, |
|
104 |
+ "ipxe": 2, |
|
105 |
+ "influxdb": 2, |
|
106 |
+ "heapster": 2, |
|
107 |
+ "groff": 2, |
|
108 |
+ "gnupg": 2, |
|
109 |
+ "elfutils": 2, |
|
110 |
+ "coredns": 2, |
|
111 |
+ "calico-k8s-policy": 2, |
|
112 |
+ "calico-bgp-daemon": 2, |
|
113 |
+ "bindutils": 2, |
|
114 |
+ "ansible": 2, |
|
115 |
+ "zstd": 1, |
|
116 |
+ "zeromq": 1, |
|
117 |
+ "xorriso": 1, |
|
118 |
+ "wget": 1, |
|
119 |
+ "userspace-rcu": 1, |
|
120 |
+ "trousers": 1, |
|
121 |
+ "texinfo": 1, |
|
122 |
+ "tcl": 1, |
|
123 |
+ "tar": 1, |
|
124 |
+ "swig": 1, |
|
125 |
+ "sudo": 1, |
|
126 |
+ "strace": 1, |
|
127 |
+ "sqlite": 1, |
|
128 |
+ "slang": 1, |
|
129 |
+ "rsyslog": 1, |
|
130 |
+ "redis": 1, |
|
131 |
+ "rabbitmq-server": 1, |
|
132 |
+ "python-Twisted": 1, |
|
133 |
+ "python-docutils": 1, |
|
134 |
+ "python-cryptography": 1, |
|
135 |
+ "python-botocore": 1, |
|
136 |
+ "protobuf-c": 1, |
|
137 |
+ "pmd": 1, |
|
138 |
+ "pixman": 1, |
|
139 |
+ "pcre": 1, |
|
140 |
+ "parted": 1, |
|
141 |
+ "openssh": 1, |
|
142 |
+ "ninja-build": 1, |
|
143 |
+ "nginx-ingress": 1, |
|
144 |
+ "nginx": 1, |
|
145 |
+ "mercurial": 1, |
|
146 |
+ "man-db": 1, |
|
147 |
+ "lvm2": 1, |
|
148 |
+ "lttng-ust": 1, |
|
149 |
+ "lshw": 1, |
|
150 |
+ "log4cpp": 1, |
|
151 |
+ "Linux-PAM": 1, |
|
152 |
+ "libwebp": 1, |
|
153 |
+ "libsoup": 1, |
|
154 |
+ "libjpeg-turbo": 1, |
|
155 |
+ "libgcrypt": 1, |
|
156 |
+ "libarchive": 1, |
|
157 |
+ "leveldb": 1, |
|
158 |
+ "js": 1, |
|
159 |
+ "jna": 1, |
|
160 |
+ "ipmitool": 1, |
|
161 |
+ "harfbuzz": 1, |
|
162 |
+ "haproxy": 1, |
|
163 |
+ "gnuplot": 1, |
|
164 |
+ "freetype2": 1, |
|
165 |
+ "flannel": 1, |
|
166 |
+ "findutils": 1, |
|
167 |
+ "etcd": 1, |
|
168 |
+ "edgex": 1, |
|
169 |
+ "dhcp": 1, |
|
170 |
+ "cyrus-sasl": 1, |
|
171 |
+ "curl": 1, |
|
172 |
+ "cppunit": 1, |
|
173 |
+ "consul": 1, |
|
174 |
+ "cni": 1, |
|
175 |
+ "c-ares": 1, |
|
176 |
+ "bzr": 1, |
|
177 |
+ "autogen": 1, |
|
178 |
+ "audit": 1, |
|
179 |
+ "apr": 1, |
|
180 |
+ "apparmor": 1, |
|
181 |
+ "apache-ant": 1 |
|
13 | 182 |
} |
... | ... |
@@ -5,6 +5,7 @@ from ThreadPool import ThreadPool |
5 | 5 |
from constants import constants |
6 | 6 |
from Logger import Logger |
7 | 7 |
from SpecData import SPECS |
8 |
+from StringUtils import StringUtils |
|
8 | 9 |
|
9 | 10 |
class Scheduler(object): |
10 | 11 |
|
... | ... |
@@ -117,17 +118,28 @@ class Scheduler(object): |
117 | 117 |
with open(constants.packageWeightsPath, 'r') as weightFile: |
118 | 118 |
Scheduler.pkgWeights = json.load(weightFile) |
119 | 119 |
|
120 |
+ # A package's weight is an indicator of the time required to build |
|
121 |
+ # that package, relative to other packages. These weights do not |
|
122 |
+ # take build-time/install-time dependencies into account -- they |
|
123 |
+ # are the individual build-times of the respective packages. |
|
124 |
+ # Package weights are positive integers, with a default value of 1. |
|
120 | 125 |
@staticmethod |
121 | 126 |
def _getWeight(package): |
127 |
+ # Package weights are assumed to be independent of package |
|
128 |
+ # version (i.e., in the case of multi-version packages such as |
|
129 |
+ # Go or Kubernetes, all the versions have the same weight). So |
|
130 |
+ # convert packageName-version to packageName before looking up |
|
131 |
+ # the package weight. |
|
132 |
+ package, _ = StringUtils.splitPackageNameAndVersion(package) |
|
122 | 133 |
try: |
123 |
- return float(Scheduler.pkgWeights[package]) |
|
134 |
+ return int(Scheduler.pkgWeights[package]) + 1 |
|
124 | 135 |
except KeyError: |
125 |
- return 0 |
|
136 |
+ return 1 |
|
126 | 137 |
|
127 | 138 |
@staticmethod |
128 | 139 |
def _getPriority(package): |
129 | 140 |
try: |
130 |
- return float(Scheduler.priorityMap[package]) |
|
141 |
+ return int(Scheduler.priorityMap[package]) |
|
131 | 142 |
except KeyError: |
132 | 143 |
return 0 |
133 | 144 |
|