Browse code

test-integration: support cgroup2

Usage: DOCKER_BUILD_ARGS="--build-arg CONTAINERD_COMMIT=master --build-arg RUNC_COMMIT=master" DOCKER_EXPERIMENTAL=1 TEST_SKIP_INTEGRATION_CLI=1 make test-integration

Depends on containerd master (v1.4) and runc master (v1.0.0-rc91).

Currently `TEST_SKIP_INTEGRATION_CLI=1` must be specified.

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

Akihiro Suda authored on 2020/06/03 22:37:14
Showing 4 changed files
... ...
@@ -25,6 +25,17 @@ if ! mountpoint -q /tmp; then
25 25
 	mount -t tmpfs none /tmp
26 26
 fi
27 27
 
28
+# cgroup v2: enable nesting
29
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
30
+	# move the init process (PID 1) from the root group to the /init group,
31
+	# otherwise writing subtree_control fails with EBUSY.
32
+	mkdir -p /sys/fs/cgroup/init
33
+	echo 1 > /sys/fs/cgroup/init/cgroup.procs
34
+	# enable controllers
35
+	sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
36
+		> /sys/fs/cgroup/cgroup.subtree_control
37
+fi
38
+
28 39
 if [ $# -gt 0 ]; then
29 40
 	exec "$@"
30 41
 fi
... ...
@@ -64,6 +64,13 @@ if [ "$DOCKER_EXPERIMENTAL" ]; then
64 64
 fi
65 65
 
66 66
 dockerd="dockerd"
67
+if [ -f "/sys/fs/cgroup/cgroup.controllers" ]; then
68
+	if [ -z "$TEST_SKIP_INTEGRATION_CLI" ]; then
69
+		echo >&2 '# cgroup v2 requires TEST_SKIP_INTEGRATION_CLI to be set'
70
+		exit 1
71
+	fi
72
+fi
73
+
67 74
 if [ -n "$DOCKER_ROOTLESS" ]; then
68 75
 	if [ -z "$TEST_SKIP_INTEGRATION_CLI" ]; then
69 76
 		echo >&2 '# DOCKER_ROOTLESS requires TEST_SKIP_INTEGRATION_CLI to be set'
... ...
@@ -69,6 +69,7 @@ func TestCgroupNamespacesRunPrivileged(t *testing.T) {
69 69
 	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
70 70
 	skip.If(t, testEnv.IsRemoteDaemon())
71 71
 	skip.If(t, !requirement.CgroupNamespacesEnabled())
72
+	skip.If(t, testEnv.DaemonInfo.CgroupVersion == "2", "on cgroup v2, privileged containers use private cgroupns")
72 73
 
73 74
 	// When the daemon defaults to private cgroup namespaces, privileged containers
74 75
 	// launched should not be inside their own cgroup namespaces
... ...
@@ -53,19 +53,34 @@ func TestUpdateMemory(t *testing.T) {
53 53
 	assert.Check(t, is.Equal(setMemory, inspect.HostConfig.Memory))
54 54
 	assert.Check(t, is.Equal(setMemorySwap, inspect.HostConfig.MemorySwap))
55 55
 
56
+	memoryFile := "/sys/fs/cgroup/memory/memory.limit_in_bytes"
57
+	if testEnv.DaemonInfo.CgroupVersion == "2" {
58
+		memoryFile = "/sys/fs/cgroup/memory.max"
59
+	}
56 60
 	res, err := container.Exec(ctx, client, cID,
57
-		[]string{"cat", "/sys/fs/cgroup/memory/memory.limit_in_bytes"})
61
+		[]string{"cat", memoryFile})
58 62
 	assert.NilError(t, err)
59 63
 	assert.Assert(t, is.Len(res.Stderr(), 0))
60 64
 	assert.Equal(t, 0, res.ExitCode)
61 65
 	assert.Check(t, is.Equal(strconv.FormatInt(setMemory, 10), strings.TrimSpace(res.Stdout())))
62 66
 
63
-	res, err = container.Exec(ctx, client, cID,
64
-		[]string{"cat", "/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes"})
65
-	assert.NilError(t, err)
66
-	assert.Assert(t, is.Len(res.Stderr(), 0))
67
-	assert.Equal(t, 0, res.ExitCode)
68
-	assert.Check(t, is.Equal(strconv.FormatInt(setMemorySwap, 10), strings.TrimSpace(res.Stdout())))
67
+	// see ConvertMemorySwapToCgroupV2Value() for the convention:
68
+	// https://github.com/opencontainers/runc/commit/c86be8a2c118ca7bad7bbe9eaf106c659a83940d
69
+	if testEnv.DaemonInfo.CgroupVersion == "2" {
70
+		res, err = container.Exec(ctx, client, cID,
71
+			[]string{"cat", "/sys/fs/cgroup/memory.swap.max"})
72
+		assert.NilError(t, err)
73
+		assert.Assert(t, is.Len(res.Stderr(), 0))
74
+		assert.Equal(t, 0, res.ExitCode)
75
+		assert.Check(t, is.Equal(strconv.FormatInt(setMemorySwap-setMemory, 10), strings.TrimSpace(res.Stdout())))
76
+	} else {
77
+		res, err = container.Exec(ctx, client, cID,
78
+			[]string{"cat", "/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes"})
79
+		assert.NilError(t, err)
80
+		assert.Assert(t, is.Len(res.Stderr(), 0))
81
+		assert.Equal(t, 0, res.ExitCode)
82
+		assert.Check(t, is.Equal(strconv.FormatInt(setMemorySwap, 10), strings.TrimSpace(res.Stdout())))
83
+	}
69 84
 }
70 85
 
71 86
 func TestUpdateCPUQuota(t *testing.T) {
... ...
@@ -85,24 +100,53 @@ func TestUpdateCPUQuota(t *testing.T) {
85 85
 		{desc: "a lower value", update: 10000},
86 86
 		{desc: "unset value", update: -1},
87 87
 	} {
88
-		_, err := client.ContainerUpdate(ctx, cID, containertypes.UpdateConfig{
89
-			Resources: containertypes.Resources{
90
-				CPUQuota: test.update,
91
-			},
92
-		})
93
-		assert.NilError(t, err)
88
+		if testEnv.DaemonInfo.CgroupVersion == "2" {
89
+			// On v2, specifying CPUQuota without CPUPeriod is currently broken:
90
+			// https://github.com/opencontainers/runc/issues/2456
91
+			// As a workaround we set them together.
92
+			_, err := client.ContainerUpdate(ctx, cID, containertypes.UpdateConfig{
93
+				Resources: containertypes.Resources{
94
+					CPUQuota:  test.update,
95
+					CPUPeriod: 100000,
96
+				},
97
+			})
98
+			assert.NilError(t, err)
99
+		} else {
100
+			_, err := client.ContainerUpdate(ctx, cID, containertypes.UpdateConfig{
101
+				Resources: containertypes.Resources{
102
+					CPUQuota: test.update,
103
+				},
104
+			})
105
+			assert.NilError(t, err)
106
+		}
94 107
 
95 108
 		inspect, err := client.ContainerInspect(ctx, cID)
96 109
 		assert.NilError(t, err)
97 110
 		assert.Check(t, is.Equal(test.update, inspect.HostConfig.CPUQuota))
98 111
 
99
-		res, err := container.Exec(ctx, client, cID,
100
-			[]string{"/bin/cat", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"})
101
-		assert.NilError(t, err)
102
-		assert.Assert(t, is.Len(res.Stderr(), 0))
103
-		assert.Equal(t, 0, res.ExitCode)
112
+		if testEnv.DaemonInfo.CgroupVersion == "2" {
113
+			res, err := container.Exec(ctx, client, cID,
114
+				[]string{"/bin/cat", "/sys/fs/cgroup/cpu.max"})
115
+			assert.NilError(t, err)
116
+			assert.Assert(t, is.Len(res.Stderr(), 0))
117
+			assert.Equal(t, 0, res.ExitCode)
118
+
119
+			quotaPeriodPair := strings.Fields(res.Stdout())
120
+			quota := quotaPeriodPair[0]
121
+			if test.update == -1 {
122
+				assert.Check(t, is.Equal("max", quota))
123
+			} else {
124
+				assert.Check(t, is.Equal(strconv.FormatInt(test.update, 10), quota))
125
+			}
126
+		} else {
127
+			res, err := container.Exec(ctx, client, cID,
128
+				[]string{"/bin/cat", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"})
129
+			assert.NilError(t, err)
130
+			assert.Assert(t, is.Len(res.Stderr(), 0))
131
+			assert.Equal(t, 0, res.ExitCode)
104 132
 
105
-		assert.Check(t, is.Equal(strconv.FormatInt(test.update, 10), strings.TrimSpace(res.Stdout())))
133
+			assert.Check(t, is.Equal(strconv.FormatInt(test.update, 10), strings.TrimSpace(res.Stdout())))
134
+		}
106 135
 	}
107 136
 }
108 137
 
... ...
@@ -160,7 +204,11 @@ func TestUpdatePidsLimit(t *testing.T) {
160 160
 			ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
161 161
 			defer cancel()
162 162
 
163
-			res, err := container.Exec(ctx, c, cID, []string{"cat", "/sys/fs/cgroup/pids/pids.max"})
163
+			pidsFile := "/sys/fs/cgroup/pids/pids.max"
164
+			if testEnv.DaemonInfo.CgroupVersion == "2" {
165
+				pidsFile = "/sys/fs/cgroup/pids.max"
166
+			}
167
+			res, err := container.Exec(ctx, c, cID, []string{"cat", pidsFile})
164 168
 			assert.NilError(t, err)
165 169
 			assert.Assert(t, is.Len(res.Stderr(), 0))
166 170