Browse code

Add `--cpus` flag to control cpu resources

This fix tries to address the proposal raised in 27921 and add
`--cpus` flag for `docker run/create`.

Basically, `--cpus` will allow user to specify a number (possibly partial)
about how many CPUs the container will use. For example, on a 2-CPU system
`--cpus 1.5` means the container will take 75% (1.5/2) of the CPU share.

This fix adds a `NanoCPUs` field to `HostConfig` since swarmkit alreay
have a concept of NanoCPUs for tasks. The `--cpus` flag will translate
the number into reused `NanoCPUs` to be consistent.

This fix adds integration tests to cover the changes.

Related docs (`docker run` and Remote APIs) have been updated.

This fix fixes 27921.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

Yong Tang authored on 2016/11/02 02:12:29
Showing 16 changed files
... ...
@@ -234,6 +234,7 @@ type Resources struct {
234 234
 	// Applicable to all platforms
235 235
 	CPUShares int64 `json:"CpuShares"` // CPU shares (relative weight vs. other containers)
236 236
 	Memory    int64 // Memory limit (in bytes)
237
+	NanoCPUs  int64 `json:"NanoCpus"` // CPU quota in units of 10<sup>-9</sup> CPUs.
237 238
 
238 239
 	// Applicable to UNIX platforms
239 240
 	CgroupParent         string // Parent cgroup.
... ...
@@ -2,7 +2,6 @@ package service
2 2
 
3 3
 import (
4 4
 	"fmt"
5
-	"math/big"
6 5
 	"strconv"
7 6
 	"strings"
8 7
 	"time"
... ...
@@ -40,33 +39,6 @@ func (m *memBytes) Value() int64 {
40 40
 	return int64(*m)
41 41
 }
42 42
 
43
-type nanoCPUs int64
44
-
45
-func (c *nanoCPUs) String() string {
46
-	return big.NewRat(c.Value(), 1e9).FloatString(3)
47
-}
48
-
49
-func (c *nanoCPUs) Set(value string) error {
50
-	cpu, ok := new(big.Rat).SetString(value)
51
-	if !ok {
52
-		return fmt.Errorf("Failed to parse %v as a rational number", value)
53
-	}
54
-	nano := cpu.Mul(cpu, big.NewRat(1e9, 1))
55
-	if !nano.IsInt() {
56
-		return fmt.Errorf("value is too precise")
57
-	}
58
-	*c = nanoCPUs(nano.Num().Int64())
59
-	return nil
60
-}
61
-
62
-func (c *nanoCPUs) Type() string {
63
-	return "NanoCPUs"
64
-}
65
-
66
-func (c *nanoCPUs) Value() int64 {
67
-	return int64(*c)
68
-}
69
-
70 43
 // PositiveDurationOpt is an option type for time.Duration that uses a pointer.
71 44
 // It bahave similarly to DurationOpt but only allows positive duration values.
72 45
 type PositiveDurationOpt struct {
... ...
@@ -156,9 +128,9 @@ type updateOptions struct {
156 156
 }
157 157
 
158 158
 type resourceOptions struct {
159
-	limitCPU      nanoCPUs
159
+	limitCPU      opts.NanoCPUs
160 160
 	limitMemBytes memBytes
161
-	resCPU        nanoCPUs
161
+	resCPU        opts.NanoCPUs
162 162
 	resMemBytes   memBytes
163 163
 }
164 164
 
... ...
@@ -6,6 +6,7 @@ import (
6 6
 	"time"
7 7
 
8 8
 	"github.com/docker/docker/api/types/container"
9
+	"github.com/docker/docker/opts"
9 10
 	"github.com/docker/docker/pkg/testutil/assert"
10 11
 )
11 12
 
... ...
@@ -21,12 +22,12 @@ func TestMemBytesSetAndValue(t *testing.T) {
21 21
 }
22 22
 
23 23
 func TestNanoCPUsString(t *testing.T) {
24
-	var cpus nanoCPUs = 6100000000
24
+	var cpus opts.NanoCPUs = 6100000000
25 25
 	assert.Equal(t, cpus.String(), "6.100")
26 26
 }
27 27
 
28 28
 func TestNanoCPUsSetAndValue(t *testing.T) {
29
-	var cpus nanoCPUs
29
+	var cpus opts.NanoCPUs
30 30
 	assert.NilError(t, cpus.Set("0.35"))
31 31
 	assert.Equal(t, cpus.Value(), int64(350000000))
32 32
 }
... ...
@@ -15,6 +15,7 @@ import (
15 15
 	"strconv"
16 16
 	"strings"
17 17
 	"syscall"
18
+	"time"
18 19
 
19 20
 	"github.com/Sirupsen/logrus"
20 21
 	"github.com/docker/docker/api/types"
... ...
@@ -110,6 +111,16 @@ func getCPUResources(config containertypes.Resources) *specs.CPU {
110 110
 		cpu.Mems = &cpuset
111 111
 	}
112 112
 
113
+	if config.NanoCPUs > 0 {
114
+		// Use the default setting of 100ms, as is specified in:
115
+		// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
116
+		//    	cpu.cfs_period_us=100ms
117
+		period := uint64(100 * time.Millisecond / time.Microsecond)
118
+		quota := uint64(config.NanoCPUs) * period / 1e9
119
+		cpu.Period = &period
120
+		cpu.Quota = &quota
121
+	}
122
+
113 123
 	if config.CPUPeriod != 0 {
114 124
 		period := uint64(config.CPUPeriod)
115 125
 		cpu.Period = &period
... ...
@@ -341,6 +352,19 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi
341 341
 	}
342 342
 
343 343
 	// cpu subsystem checks and adjustments
344
+	if resources.NanoCPUs > 0 && resources.CPUPeriod > 0 {
345
+		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Period cannot both be set")
346
+	}
347
+	if resources.NanoCPUs > 0 && resources.CPUQuota > 0 {
348
+		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Quota cannot both be set")
349
+	}
350
+	if resources.NanoCPUs > 0 && (!sysInfo.CPUCfsPeriod || !sysInfo.CPUCfsQuota) {
351
+		return warnings, fmt.Errorf("NanoCPUs can not be set, as your kernel does not support CPU cfs period/quota or the cgroup is not mounted")
352
+	}
353
+	if resources.NanoCPUs < 0 || resources.NanoCPUs > int64(sysinfo.NumCPU())*1e9 {
354
+		return warnings, fmt.Errorf("Range of Nano CPUs is from 1 to %d", int64(sysinfo.NumCPU())*1e9)
355
+	}
356
+
344 357
 	if resources.CPUShares > 0 && !sysInfo.CPUShares {
345 358
 		warnings = append(warnings, "Your kernel does not support CPU shares or the cgroup is not mounted. Shares discarded.")
346 359
 		logrus.Warn("Your kernel does not support CPU shares or the cgroup is not mounted. Shares discarded.")
... ...
@@ -103,6 +103,17 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi
103 103
 		return warnings, fmt.Errorf("Conflicting options: CPU Shares and CPU Percent cannot both be set")
104 104
 	}
105 105
 
106
+	if resources.NanoCPUs > 0 && resources.CPUPercent > 0 {
107
+		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Percent cannot both be set")
108
+	}
109
+
110
+	if resources.NanoCPUs > 0 && resources.CPUShares > 0 {
111
+		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Shares cannot both be set")
112
+	}
113
+	if resources.NanoCPUs < 0 || resources.NanoCPUs > int64(sysinfo.NumCPU())*1e9 {
114
+		return warnings, fmt.Errorf("Range of Nano CPUs is from 1 to %d", int64(sysinfo.NumCPU())*1e9)
115
+	}
116
+
106 117
 	// TODO Windows: Add more validation of resource settings not supported on Windows
107 118
 
108 119
 	if resources.BlkioWeight > 0 {
... ...
@@ -6,6 +6,7 @@ import (
6 6
 	containertypes "github.com/docker/docker/api/types/container"
7 7
 	"github.com/docker/docker/container"
8 8
 	"github.com/docker/docker/oci"
9
+	"github.com/docker/docker/pkg/sysinfo"
9 10
 	"github.com/opencontainers/runtime-spec/specs-go"
10 11
 )
11 12
 
... ...
@@ -82,6 +83,9 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
82 82
 	// @darrenstahlmsft implement these resources
83 83
 	cpuShares := uint16(c.HostConfig.CPUShares)
84 84
 	cpuPercent := uint8(c.HostConfig.CPUPercent)
85
+	if c.HostConfig.NanoCPUs > 0 {
86
+		cpuPercent = uint8(c.HostConfig.NanoCPUs * 100 / int64(sysinfo.NumCPU()) / 1e9)
87
+	}
85 88
 	memoryLimit := uint64(c.HostConfig.Memory)
86 89
 	s.Windows.Resources = &specs.WindowsResources{
87 90
 		CPU: &specs.WindowsCPUResources{
... ...
@@ -164,6 +164,7 @@ This section lists each version from latest to oldest.  Each listing includes a
164 164
 * The `hostConfig` option now accepts the fields `CpuRealtimePeriod` and `CpuRtRuntime` to allocate cpu runtime to rt tasks when `CONFIG_RT_GROUP_SCHED` is enabled in the kernel.
165 165
 * The `SecurityOptions` field within the `GET /info` response now includes `userns` if user namespaces are enabled in the daemon.
166 166
 * `GET /nodes` and `GET /node/(id or name)` now return `Addr` as part of a node's `Status`, which is the address that that node connects to the manager from.
167
+* The `HostConfig` field now includes `NanoCPUs` that represents CPU quota in units of 10<sup>-9</sup> CPUs.
167 168
 
168 169
 ### v1.24 API changes
169 170
 
... ...
@@ -302,6 +302,7 @@ Create a container
302 302
              "MemorySwap": 0,
303 303
              "MemoryReservation": 0,
304 304
              "KernelMemory": 0,
305
+             "NanoCPUs": 500000,
305 306
              "CpuPercent": 80,
306 307
              "CpuShares": 512,
307 308
              "CpuPeriod": 100000,
... ...
@@ -425,6 +426,7 @@ Create a container
425 425
           You must use this with `memory` and make the swap value larger than `memory`.
426 426
     -   **MemoryReservation** - Memory soft limit in bytes.
427 427
     -   **KernelMemory** - Kernel memory limit in bytes.
428
+    -   **NanoCPUs** - CPU quota in units of 10<sup>-9</sup> CPUs.
428 429
     -   **CpuPercent** - An integer value containing the usable percentage of the available CPUs. (Windows daemon only)
429 430
     -   **CpuShares** - An integer value containing the container's CPU Shares
430 431
           (ie. the relative weight vs other containers).
... ...
@@ -35,6 +35,7 @@ Options:
35 35
       --cpu-period int              Limit CPU CFS (Completely Fair Scheduler) period
36 36
       --cpu-quota int               Limit CPU CFS (Completely Fair Scheduler) quota
37 37
   -c, --cpu-shares int              CPU shares (relative weight)
38
+      --cpus NanoCPUs               Number of CPUs (default 0.000)
38 39
       --cpu-rt-period int           Limit the CPU real-time period in microseconds
39 40
       --cpu-rt-runtime int          Limit the CPU real-time runtime in microseconds
40 41
       --cpuset-cpus string          CPUs in which to allow execution (0-3, 0,1)
... ...
@@ -33,6 +33,7 @@ Options:
33 33
       --cpu-period int              Limit CPU CFS (Completely Fair Scheduler) period
34 34
       --cpu-quota int               Limit CPU CFS (Completely Fair Scheduler) quota
35 35
   -c, --cpu-shares int              CPU shares (relative weight)
36
+      --cpus NanoCPUs               Number of CPUs (default 0.000)
36 37
       --cpu-rt-period int           Limit the CPU real-time period in microseconds
37 38
       --cpu-rt-runtime int          Limit the CPU real-time runtime in microseconds
38 39
       --cpuset-cpus string          CPUs in which to allow execution (0-3, 0,1)
... ...
@@ -686,6 +686,7 @@ container:
686 686
 | `--memory-reservation=""`  | Memory soft limit (format: `<number>[<unit>]`). Number is a positive integer. Unit can be one of `b`, `k`, `m`, or `g`.                         |
687 687
 | `--kernel-memory=""`       | Kernel memory limit (format: `<number>[<unit>]`). Number is a positive integer. Unit can be one of `b`, `k`, `m`, or `g`. Minimum is 4M.        |
688 688
 | `-c`, `--cpu-shares=0`     | CPU shares (relative weight)                                                                                                                    |
689
+| `--cpus=0.000`             | Number of CPUs. Number is a fractional number. 0.000 means no limit.                                                                            |
689 690
 | `--cpu-period=0`           | Limit the CPU CFS (Completely Fair Scheduler) period                                                                                            |
690 691
 | `--cpuset-cpus=""`         | CPUs in which to allow execution (0-3, 0,1)                                                                                                     |
691 692
 | `--cpuset-mems=""`         | Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems.                                                     |
... ...
@@ -970,6 +971,13 @@ Examples:
970 970
 
971 971
 If there is 1 CPU, this means the container can get 50% CPU worth of run-time every 50ms.
972 972
 
973
+In addition to use `--cpu-period` and `--cpu-quota` for setting CPU period constraints,
974
+it is possible to specify `--cpus` with a float number to achieve the same purpose.
975
+For example, if there is 1 CPU, then `--cpus=0.5` will achieve the same result as
976
+setting `--cpu-period=50000` and `--cpu-quota=25000` (50% CPU).
977
+
978
+The default value for `--cpus` is `0.000`, which means there is no limit.
979
+
973 980
 For more information, see the [CFS documentation on bandwidth limiting](https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt).
974 981
 
975 982
 ### Cpuset constraint
... ...
@@ -1409,3 +1409,23 @@ func (s *DockerDaemonSuite) TestRunWithDaemonDefaultSeccompProfile(c *check.C) {
1409 1409
 	c.Assert(err, check.NotNil)
1410 1410
 	c.Assert(out, checker.Contains, "Operation not permitted")
1411 1411
 }
1412
+
1413
+func (s *DockerSuite) TestRunWithNanoCPUs(c *check.C) {
1414
+	testRequires(c, cpuCfsQuota, cpuCfsPeriod)
1415
+
1416
+	file1 := "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"
1417
+	file2 := "/sys/fs/cgroup/cpu/cpu.cfs_period_us"
1418
+	out, _ := dockerCmd(c, "run", "--cpus", "0.5", "--name", "test", "busybox", "sh", "-c", fmt.Sprintf("cat %s && cat %s", file1, file2))
1419
+	c.Assert(strings.TrimSpace(out), checker.Equals, "50000\n100000")
1420
+
1421
+	out = inspectField(c, "test", "HostConfig.NanoCpus")
1422
+	c.Assert(out, checker.Equals, "5e+08", check.Commentf("setting the Nano CPUs failed"))
1423
+	out = inspectField(c, "test", "HostConfig.CpuQuota")
1424
+	c.Assert(out, checker.Equals, "0", check.Commentf("CPU CFS quota should be 0"))
1425
+	out = inspectField(c, "test", "HostConfig.CpuPeriod")
1426
+	c.Assert(out, checker.Equals, "0", check.Commentf("CPU CFS period should be 0"))
1427
+
1428
+	out, _, err := dockerCmdWithError("run", "--cpus", "0.5", "--cpu-quota", "50000", "--cpu-period", "100000", "busybox", "sh")
1429
+	c.Assert(err, check.NotNil)
1430
+	c.Assert(out, checker.Contains, "Conflicting options: Nano CPUs and CPU Period cannot both be set")
1431
+}
... ...
@@ -19,6 +19,7 @@ docker-create - Create a new container
19 19
 [**--cpu-quota**[=*0*]]
20 20
 [**--cpu-rt-period**[=*0*]]
21 21
 [**--cpu-rt-runtime**[=*0*]]
22
+[**--cpus**[=*0.0*]]
22 23
 [**--cpuset-cpus**[=*CPUSET-CPUS*]]
23 24
 [**--cpuset-mems**[=*CPUSET-MEMS*]]
24 25
 [**--device**[=*[]*]]
... ...
@@ -154,6 +155,9 @@ two memory nodes.
154 154
 
155 155
    The sum of all runtimes across containers cannot exceed the amount allotted to the parent cgroup.
156 156
 
157
+**--cpus**=0.0
158
+   Number of CPUs. The default is *0.0*.
159
+
157 160
 **--device**=[]
158 161
    Add a host device to the container (e.g. --device=/dev/sdc:/dev/xvdc:rwm)
159 162
 
... ...
@@ -19,6 +19,7 @@ docker-run - Run a command in a new container
19 19
 [**--cpu-quota**[=*0*]]
20 20
 [**--cpu-rt-period**[=*0*]]
21 21
 [**--cpu-rt-runtime**[=*0*]]
22
+[**--cpus**[=*0.0*]]
22 23
 [**--cpuset-cpus**[=*CPUSET-CPUS*]]
23 24
 [**--cpuset-mems**[=*CPUSET-MEMS*]]
24 25
 [**-d**|**--detach**]
... ...
@@ -208,6 +209,9 @@ to the quota you specify.
208 208
 
209 209
    The sum of all runtimes across containers cannot exceed the amount allotted to the parent cgroup.
210 210
 
211
+**--cpus**=0.0
212
+   Number of CPUs. The default is *0.0* which means no limit.
213
+
211 214
 **-d**, **--detach**=*true*|*false*
212 215
    Detached mode: run the container in the background and print the new container ID. The default is *false*.
213 216
 
... ...
@@ -2,6 +2,7 @@ package opts
2 2
 
3 3
 import (
4 4
 	"fmt"
5
+	"math/big"
5 6
 	"net"
6 7
 	"regexp"
7 8
 	"strings"
... ...
@@ -319,3 +320,35 @@ func (o *FilterOpt) Type() string {
319 319
 func (o *FilterOpt) Value() filters.Args {
320 320
 	return o.filter
321 321
 }
322
+
323
+// NanoCPUs is a type for fixed point fractional number.
324
+type NanoCPUs int64
325
+
326
+// String returns the string format of the number
327
+func (c *NanoCPUs) String() string {
328
+	return big.NewRat(c.Value(), 1e9).FloatString(3)
329
+}
330
+
331
+// Set sets the value of the NanoCPU by passing a string
332
+func (c *NanoCPUs) Set(value string) error {
333
+	cpu, ok := new(big.Rat).SetString(value)
334
+	if !ok {
335
+		return fmt.Errorf("Failed to parse %v as a rational number", value)
336
+	}
337
+	nano := cpu.Mul(cpu, big.NewRat(1e9, 1))
338
+	if !nano.IsInt() {
339
+		return fmt.Errorf("value is too precise")
340
+	}
341
+	*c = NanoCPUs(nano.Num().Int64())
342
+	return nil
343
+}
344
+
345
+// Type returns the type
346
+func (c *NanoCPUs) Type() string {
347
+	return "NanoCPUs"
348
+}
349
+
350
+// Value returns the value in int64
351
+func (c *NanoCPUs) Value() int64 {
352
+	return int64(*c)
353
+}
... ...
@@ -79,6 +79,7 @@ type ContainerOptions struct {
79 79
 	cpuRealtimePeriod  int64
80 80
 	cpuRealtimeRuntime int64
81 81
 	cpuQuota           int64
82
+	cpus               opts.NanoCPUs
82 83
 	cpusetCpus         string
83 84
 	cpusetMems         string
84 85
 	blkioWeight        uint16
... ...
@@ -232,6 +233,7 @@ func AddFlags(flags *pflag.FlagSet) *ContainerOptions {
232 232
 	flags.Int64Var(&copts.cpuRealtimePeriod, "cpu-rt-period", 0, "Limit CPU real-time period in microseconds")
233 233
 	flags.Int64Var(&copts.cpuRealtimeRuntime, "cpu-rt-runtime", 0, "Limit CPU real-time runtime in microseconds")
234 234
 	flags.Int64VarP(&copts.cpuShares, "cpu-shares", "c", 0, "CPU shares (relative weight)")
235
+	flags.Var(&copts.cpus, "cpus", "Number of CPUs")
235 236
 	flags.Var(&copts.deviceReadBps, "device-read-bps", "Limit read rate (bytes per second) from a device")
236 237
 	flags.Var(&copts.deviceReadIOps, "device-read-iops", "Limit read rate (IO per second) from a device")
237 238
 	flags.Var(&copts.deviceWriteBps, "device-write-bps", "Limit write rate (bytes per second) to a device")
... ...
@@ -526,6 +528,7 @@ func Parse(flags *pflag.FlagSet, copts *ContainerOptions) (*container.Config, *c
526 526
 		MemorySwappiness:     &copts.swappiness,
527 527
 		KernelMemory:         kernelMemory,
528 528
 		OomKillDisable:       &copts.oomKillDisable,
529
+		NanoCPUs:             copts.cpus.Value(),
529 530
 		CPUPercent:           copts.cpuPercent,
530 531
 		CPUShares:            copts.cpuShares,
531 532
 		CPUPeriod:            copts.cpuPeriod,