Browse code

scheduler: Update package weights with fresh measurements

These weights are the time taken (in minutes) to build each package on
the same machine, with THREADS=1. The packages not mentioned in this
list could be built in less than a minute.

When consuming these weights in the scheduler, we offset them by 1,
so that the minimum package weight would be 1 (which makes certain
scheduler calculations easier, as opposed to a minimum of 0).

Also, remove the obsolete file pkg_build_weights.json

Change-Id: I0239f69cc8de262580b2ab9758fb7a3a315ff03b
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/6161
Reviewed-by: Alexey Makhalov <amakhalov@vmware.com>
Tested-by: Anish Swaminathan <anishs@vmware.com>

Srivatsa S. Bhat (VMware) authored on 2018/11/15 05:39:01
Showing 3 changed files
... ...
@@ -1,13 +1,182 @@
1 1
 {
2
-  "ceph": 83.7,
3
-  "mesos": 75,
4
-  "linux": 41.2,
5
-  "linux-secure": 32,
6
-  "openjdk8": 21.55,
7
-  "kubernetes": 21.3,
8
-  "llvm": 20.267,
9
-  "linux-esx": 16.73,
10
-  "falco": 13.85,
11
-  "guile": 11.52,
12
-  "clang": 11.3
2
+  "mongodb": 155,
3
+  "llvm": 129,
4
+  "linux": 124,
5
+  "linux-secure": 115,
6
+  "linux-aws": 112,
7
+  "clang": 109,
8
+  "mysql": 103,
9
+  "openjdk10": 98,
10
+  "mariadb": 94,
11
+  "boost": 82,
12
+  "nodejs": 63,
13
+  "envoy": 63,
14
+  "lldb": 58,
15
+  "openjdk8": 57,
16
+  "linux-esx": 56,
17
+  "kubernetes": 48,
18
+  "protobuf": 41,
19
+  "kubernetes": 39,
20
+  "guile": 25,
21
+  "elasticsearch": 22,
22
+  "gdb": 18,
23
+  "lightwave": 17,
24
+  "libvirt": 13,
25
+  "lapack": 12,
26
+  "sysdig": 9,
27
+  "ruby": 9,
28
+  "go": 9,
29
+  "systemd": 8,
30
+  "syslog-ng": 8,
31
+  "xerces-c": 7,
32
+  "subversion": 7,
33
+  "python-numpy": 7,
34
+  "postgresql": 7,
35
+  "perl": 7,
36
+  "nss": 7,
37
+  "kubernetes-dashboard": 7,
38
+  "go": 7,
39
+  "gnutls": 7,
40
+  "glibmm": 7,
41
+  "gettext": 7,
42
+  "erlang": 7,
43
+  "crash": 7,
44
+  "systemtap": 6,
45
+  "openvswitch": 6,
46
+  "likewise-open": 6,
47
+  "grub2": 6,
48
+  "python-lxml": 5,
49
+  "python3": 5,
50
+  "powershell": 5,
51
+  "icu": 5,
52
+  "grpc": 5,
53
+  "glib": 5,
54
+  "xfsprogs": 4,
55
+  "valgrind": 4,
56
+  "thin-provisioning-tools": 4,
57
+  "syslinux": 4,
58
+  "python-gevent": 4,
59
+  "open-vm-tools": 4,
60
+  "openssl": 4,
61
+  "openscap": 4,
62
+  "net-snmp": 4,
63
+  "httpd": 4,
64
+  "git": 4,
65
+  "docker": 4,
66
+  "coreutils": 4,
67
+  "cassandra": 4,
68
+  "apache-maven": 4,
69
+  "xml-security-c": 3,
70
+  "vim": 3,
71
+  "util-linux": 3,
72
+  "strongswan": 3,
73
+  "python2": 3,
74
+  "mc": 3,
75
+  "libunistring": 3,
76
+  "libdb": 3,
77
+  "krb5": 3,
78
+  "gperftools": 3,
79
+  "gobject-introspection": 3,
80
+  "gobgp": 3,
81
+  "cython": 3,
82
+  "cython3": 3,
83
+  "calico-libnetwork": 3,
84
+  "calico-felix": 3,
85
+  "calico-cni": 3,
86
+  "calico": 3,
87
+  "cairo": 3,
88
+  "zsh": 2,
89
+  "wavefront-proxy": 2,
90
+  "unixODBC": 2,
91
+  "unbound": 2,
92
+  "tclap": 2,
93
+  "rpm": 2,
94
+  "openldap": 2,
95
+  "openipmi": 2,
96
+  "ntp": 2,
97
+  "lttng-tools": 2,
98
+  "libxml2": 2,
99
+  "libnl": 2,
100
+  "kubernetes-metrics-server": 2,
101
+  "kubernetes-dns": 2,
102
+  "kapacitor": 2,
103
+  "json_spirit": 2,
104
+  "ipxe": 2,
105
+  "influxdb": 2,
106
+  "heapster": 2,
107
+  "groff": 2,
108
+  "gnupg": 2,
109
+  "elfutils": 2,
110
+  "coredns": 2,
111
+  "calico-k8s-policy": 2,
112
+  "calico-bgp-daemon": 2,
113
+  "bindutils": 2,
114
+  "ansible": 2,
115
+  "zstd": 1,
116
+  "zeromq": 1,
117
+  "xorriso": 1,
118
+  "wget": 1,
119
+  "userspace-rcu": 1,
120
+  "trousers": 1,
121
+  "texinfo": 1,
122
+  "tcl": 1,
123
+  "tar": 1,
124
+  "swig": 1,
125
+  "sudo": 1,
126
+  "strace": 1,
127
+  "sqlite": 1,
128
+  "slang": 1,
129
+  "rsyslog": 1,
130
+  "redis": 1,
131
+  "rabbitmq-server": 1,
132
+  "python-Twisted": 1,
133
+  "python-docutils": 1,
134
+  "python-cryptography": 1,
135
+  "python-botocore": 1,
136
+  "protobuf-c": 1,
137
+  "pmd": 1,
138
+  "pixman": 1,
139
+  "pcre": 1,
140
+  "parted": 1,
141
+  "openssh": 1,
142
+  "ninja-build": 1,
143
+  "nginx-ingress": 1,
144
+  "nginx": 1,
145
+  "mercurial": 1,
146
+  "man-db": 1,
147
+  "lvm2": 1,
148
+  "lttng-ust": 1,
149
+  "lshw": 1,
150
+  "log4cpp": 1,
151
+  "Linux-PAM": 1,
152
+  "libwebp": 1,
153
+  "libsoup": 1,
154
+  "libjpeg-turbo": 1,
155
+  "libgcrypt": 1,
156
+  "libarchive": 1,
157
+  "leveldb": 1,
158
+  "js": 1,
159
+  "jna": 1,
160
+  "ipmitool": 1,
161
+  "harfbuzz": 1,
162
+  "haproxy": 1,
163
+  "gnuplot": 1,
164
+  "freetype2": 1,
165
+  "flannel": 1,
166
+  "findutils": 1,
167
+  "etcd": 1,
168
+  "edgex": 1,
169
+  "dhcp": 1,
170
+  "cyrus-sasl": 1,
171
+  "curl": 1,
172
+  "cppunit": 1,
173
+  "consul": 1,
174
+  "cni": 1,
175
+  "c-ares": 1,
176
+  "bzr": 1,
177
+  "autogen": 1,
178
+  "audit": 1,
179
+  "apr": 1,
180
+  "apparmor": 1,
181
+  "apache-ant": 1
13 182
 }
14 183
deleted file mode 100644
... ...
@@ -1,3 +0,0 @@
1
-{
2
-    "mesos": "50"
3
-}
... ...
@@ -5,6 +5,7 @@ from ThreadPool import ThreadPool
5 5
 from constants import constants
6 6
 from Logger import Logger
7 7
 from SpecData import SPECS
8
+from StringUtils import StringUtils
8 9
 
9 10
 class Scheduler(object):
10 11
 
... ...
@@ -117,17 +118,28 @@ class Scheduler(object):
117 117
         with open(constants.packageWeightsPath, 'r') as weightFile:
118 118
             Scheduler.pkgWeights = json.load(weightFile)
119 119
 
120
+    # A package's weight is an indicator of the time required to build
121
+    # that package, relative to other packages. These weights do not
122
+    # take build-time/install-time dependencies into account -- they
123
+    # are the individual build-times of the respective packages.
124
+    # Package weights are positive integers, with a default value of 1.
120 125
     @staticmethod
121 126
     def _getWeight(package):
127
+	# Package weights are assumed to be independent of package
128
+	# version (i.e., in the case of multi-version packages such as
129
+	# Go or Kubernetes, all the versions have the same weight). So
130
+	# convert packageName-version to packageName before looking up
131
+	# the package weight.
132
+        package, _ = StringUtils.splitPackageNameAndVersion(package)
122 133
         try:
123
-            return float(Scheduler.pkgWeights[package])
134
+            return int(Scheduler.pkgWeights[package]) + 1
124 135
         except KeyError:
125
-            return 0
136
+            return 1
126 137
 
127 138
     @staticmethod
128 139
     def _getPriority(package):
129 140
         try:
130
-            return float(Scheduler.priorityMap[package])
141
+            return int(Scheduler.priorityMap[package])
131 142
         except KeyError:
132 143
             return 0
133 144