Browse code

Merge pull request #41016 from kolyshkin/cgroup-init

Brian Goff authored on 2020/07/17 03:26:52
Showing 8 changed files
... ...
@@ -500,8 +500,8 @@ func verifyPlatformContainerResources(resources *containertypes.Resources, sysIn
500 500
 	if resources.NanoCPUs > 0 && resources.CPUQuota > 0 {
501 501
 		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Quota cannot both be set")
502 502
 	}
503
-	if resources.NanoCPUs > 0 && (!sysInfo.CPUCfsPeriod || !sysInfo.CPUCfsQuota) {
504
-		return warnings, fmt.Errorf("NanoCPUs can not be set, as your kernel does not support CPU cfs period/quota or the cgroup is not mounted")
503
+	if resources.NanoCPUs > 0 && !sysInfo.CPUCfs {
504
+		return warnings, fmt.Errorf("NanoCPUs can not be set, as your kernel does not support CPU CFS scheduler or the cgroup is not mounted")
505 505
 	}
506 506
 	// The highest precision we could get on Linux is 0.001, by setting
507 507
 	//   cpu.cfs_period_us=1000ms
... ...
@@ -518,17 +518,14 @@ func verifyPlatformContainerResources(resources *containertypes.Resources, sysIn
518 518
 		warnings = append(warnings, "Your kernel does not support CPU shares or the cgroup is not mounted. Shares discarded.")
519 519
 		resources.CPUShares = 0
520 520
 	}
521
-	if resources.CPUPeriod > 0 && !sysInfo.CPUCfsPeriod {
522
-		warnings = append(warnings, "Your kernel does not support CPU cfs period or the cgroup is not mounted. Period discarded.")
521
+	if (resources.CPUPeriod != 0 || resources.CPUQuota != 0) && !sysInfo.CPUCfs {
522
+		warnings = append(warnings, "Your kernel does not support CPU CFS scheduler. CPU period/quota discarded.")
523 523
 		resources.CPUPeriod = 0
524
+		resources.CPUQuota = 0
524 525
 	}
525 526
 	if resources.CPUPeriod != 0 && (resources.CPUPeriod < 1000 || resources.CPUPeriod > 1000000) {
526 527
 		return warnings, fmt.Errorf("CPU cfs period can not be less than 1ms (i.e. 1000) or larger than 1s (i.e. 1000000)")
527 528
 	}
528
-	if resources.CPUQuota > 0 && !sysInfo.CPUCfsQuota {
529
-		warnings = append(warnings, "Your kernel does not support CPU cfs quota or the cgroup is not mounted. Quota discarded.")
530
-		resources.CPUQuota = 0
531
-	}
532 529
 	if resources.CPUQuota > 0 && resources.CPUQuota < 1000 {
533 530
 		return warnings, fmt.Errorf("CPU cfs quota can not be less than 1ms (i.e. 1000)")
534 531
 	}
... ...
@@ -1649,51 +1646,32 @@ func setupOOMScoreAdj(score int) error {
1649 1649
 	return err
1650 1650
 }
1651 1651
 
1652
-func (daemon *Daemon) initCgroupsPath(path string) error {
1652
+func (daemon *Daemon) initCPURtController(mnt, path string) error {
1653 1653
 	if path == "/" || path == "." {
1654 1654
 		return nil
1655 1655
 	}
1656 1656
 
1657
-	if daemon.configStore.CPURealtimePeriod == 0 && daemon.configStore.CPURealtimeRuntime == 0 {
1658
-		return nil
1659
-	}
1660
-
1661
-	if cgroups.IsCgroup2UnifiedMode() {
1662
-		return fmt.Errorf("daemon-scoped cpu-rt-period and cpu-rt-runtime are not implemented for cgroup v2")
1663
-	}
1664
-
1665 1657
 	// Recursively create cgroup to ensure that the system and all parent cgroups have values set
1666 1658
 	// for the period and runtime as this limits what the children can be set to.
1667
-	daemon.initCgroupsPath(filepath.Dir(path))
1668
-
1669
-	mnt, root, err := cgroups.FindCgroupMountpointAndRoot("", "cpu")
1670
-	if err != nil {
1659
+	if err := daemon.initCPURtController(mnt, filepath.Dir(path)); err != nil {
1671 1660
 		return err
1672 1661
 	}
1673
-	// When docker is run inside docker, the root is based of the host cgroup.
1674
-	// Should this be handled in runc/libcontainer/cgroups ?
1675
-	if strings.HasPrefix(root, "/docker/") {
1676
-		root = "/"
1677
-	}
1678 1662
 
1679
-	path = filepath.Join(mnt, root, path)
1680
-	sysInfo := daemon.RawSysInfo(true)
1681
-	if err := maybeCreateCPURealTimeFile(sysInfo.CPURealtimePeriod, daemon.configStore.CPURealtimePeriod, "cpu.rt_period_us", path); err != nil {
1663
+	path = filepath.Join(mnt, path)
1664
+	if err := os.MkdirAll(path, 0755); err != nil {
1682 1665
 		return err
1683 1666
 	}
1684
-	return maybeCreateCPURealTimeFile(sysInfo.CPURealtimeRuntime, daemon.configStore.CPURealtimeRuntime, "cpu.rt_runtime_us", path)
1667
+	if err := maybeCreateCPURealTimeFile(daemon.configStore.CPURealtimePeriod, "cpu.rt_period_us", path); err != nil {
1668
+		return err
1669
+	}
1670
+	return maybeCreateCPURealTimeFile(daemon.configStore.CPURealtimeRuntime, "cpu.rt_runtime_us", path)
1685 1671
 }
1686 1672
 
1687
-func maybeCreateCPURealTimeFile(sysinfoPresent bool, configValue int64, file string, path string) error {
1688
-	if sysinfoPresent && configValue != 0 {
1689
-		if err := os.MkdirAll(path, 0755); err != nil {
1690
-			return err
1691
-		}
1692
-		if err := ioutil.WriteFile(filepath.Join(path, file), []byte(strconv.FormatInt(configValue, 10)), 0700); err != nil {
1693
-			return err
1694
-		}
1673
+func maybeCreateCPURealTimeFile(configValue int64, file string, path string) error {
1674
+	if configValue == 0 {
1675
+		return nil
1695 1676
 	}
1696
-	return nil
1677
+	return ioutil.WriteFile(filepath.Join(path, file), []byte(strconv.FormatInt(configValue, 10)), 0700)
1697 1678
 }
1698 1679
 
1699 1680
 func (daemon *Daemon) setupSeccompProfile() error {
... ...
@@ -30,8 +30,8 @@ func (daemon *Daemon) fillPlatformInfo(v *types.Info, sysInfo *sysinfo.SysInfo)
30 30
 	v.KernelMemory = sysInfo.KernelMemory
31 31
 	v.KernelMemoryTCP = sysInfo.KernelMemoryTCP
32 32
 	v.OomKillDisable = sysInfo.OomKillDisable
33
-	v.CPUCfsPeriod = sysInfo.CPUCfsPeriod
34
-	v.CPUCfsQuota = sysInfo.CPUCfsQuota
33
+	v.CPUCfsPeriod = sysInfo.CPUCfs
34
+	v.CPUCfsQuota = sysInfo.CPUCfs
35 35
 	v.CPUShares = sysInfo.CPUShares
36 36
 	v.CPUSet = sysInfo.Cpuset
37 37
 	v.PidsLimit = sysInfo.PidsLimit
... ...
@@ -824,15 +824,32 @@ func WithCgroups(daemon *Daemon, c *container.Container) coci.SpecOpts {
824 824
 			cgroupsPath = filepath.Join(parent, c.ID)
825 825
 		}
826 826
 		s.Linux.CgroupsPath = cgroupsPath
827
+
828
+		// the rest is only needed for CPU RT controller
829
+
830
+		if daemon.configStore.CPURealtimePeriod == 0 && daemon.configStore.CPURealtimeRuntime == 0 {
831
+			return nil
832
+		}
833
+
834
+		if cgroups.IsCgroup2UnifiedMode() {
835
+			return errors.New("daemon-scoped cpu-rt-period and cpu-rt-runtime are not implemented for cgroup v2")
836
+		}
837
+
838
+		// FIXME this is very expensive way to check if cpu rt is supported
839
+		sysInfo := daemon.RawSysInfo(true)
840
+		if !sysInfo.CPURealtime {
841
+			return errors.New("daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by the kernel")
842
+		}
843
+
827 844
 		p := cgroupsPath
828 845
 		if useSystemd {
829 846
 			initPath, err := cgroups.GetInitCgroup("cpu")
830 847
 			if err != nil {
831
-				return err
848
+				return errors.Wrap(err, "unable to init CPU RT controller")
832 849
 			}
833 850
 			_, err = cgroups.GetOwnCgroup("cpu")
834 851
 			if err != nil {
835
-				return err
852
+				return errors.Wrap(err, "unable to init CPU RT controller")
836 853
 			}
837 854
 			p = filepath.Join(initPath, s.Linux.CgroupsPath)
838 855
 		}
... ...
@@ -843,8 +860,19 @@ func WithCgroups(daemon *Daemon, c *container.Container) coci.SpecOpts {
843 843
 			parentPath = filepath.Clean("/" + parentPath)
844 844
 		}
845 845
 
846
-		if err := daemon.initCgroupsPath(parentPath); err != nil {
847
-			return fmt.Errorf("linux init cgroups path: %v", err)
846
+		mnt, root, err := cgroups.FindCgroupMountpointAndRoot("", "cpu")
847
+		if err != nil {
848
+			return errors.Wrap(err, "unable to init CPU RT controller")
849
+		}
850
+		// When docker is run inside docker, the root is based of the host cgroup.
851
+		// Should this be handled in runc/libcontainer/cgroups ?
852
+		if strings.HasPrefix(root, "/docker/") {
853
+			root = "/"
854
+		}
855
+		mnt = filepath.Join(mnt, root)
856
+
857
+		if err := daemon.initCPURtController(mnt, parentPath); err != nil {
858
+			return errors.Wrap(err, "unable to init CPU RT controller")
848 859
 		}
849 860
 		return nil
850 861
 	}
... ...
@@ -90,10 +90,8 @@ func applyCPUCgroupInfoV2(info *SysInfo, controllers map[string]struct{}, _ stri
90 90
 		return warnings
91 91
 	}
92 92
 	info.CPUShares = true
93
-	info.CPUCfsPeriod = true
94
-	info.CPUCfsQuota = true
95
-	info.CPURealtimePeriod = false
96
-	info.CPURealtimeRuntime = false
93
+	info.CPUCfs = true
94
+	info.CPURealtime = false
97 95
 	return warnings
98 96
 }
99 97
 
... ...
@@ -62,17 +62,11 @@ type cgroupCPUInfo struct {
62 62
 	// Whether CPU shares is supported or not
63 63
 	CPUShares bool
64 64
 
65
-	// Whether CPU CFS(Completely Fair Scheduler) period is supported or not
66
-	CPUCfsPeriod bool
65
+	// Whether CPU CFS (Completely Fair Scheduler) is supported
66
+	CPUCfs bool
67 67
 
68
-	// Whether CPU CFS(Completely Fair Scheduler) quota is supported or not
69
-	CPUCfsQuota bool
70
-
71
-	// Whether CPU real-time period is supported or not
72
-	CPURealtimePeriod bool
73
-
74
-	// Whether CPU real-time runtime is supported or not
75
-	CPURealtimeRuntime bool
68
+	// Whether CPU real-time scheduler is supported
69
+	CPURealtime bool
76 70
 }
77 71
 
78 72
 type cgroupBlkioInfo struct {
... ...
@@ -144,27 +144,17 @@ func applyCPUCgroupInfo(info *SysInfo, cgMounts map[string]string) []string {
144 144
 
145 145
 	info.CPUShares = cgroupEnabled(mountPoint, "cpu.shares")
146 146
 	if !info.CPUShares {
147
-		warnings = append(warnings, "Your kernel does not support cgroup cpu shares")
147
+		warnings = append(warnings, "Your kernel does not support CPU shares")
148 148
 	}
149 149
 
150
-	info.CPUCfsPeriod = cgroupEnabled(mountPoint, "cpu.cfs_period_us")
151
-	if !info.CPUCfsPeriod {
152
-		warnings = append(warnings, "Your kernel does not support cgroup cfs period")
150
+	info.CPUCfs = cgroupEnabled(mountPoint, "cpu.cfs_quota_us")
151
+	if !info.CPUCfs {
152
+		warnings = append(warnings, "Your kernel does not support CPU CFS scheduler")
153 153
 	}
154 154
 
155
-	info.CPUCfsQuota = cgroupEnabled(mountPoint, "cpu.cfs_quota_us")
156
-	if !info.CPUCfsQuota {
157
-		warnings = append(warnings, "Your kernel does not support cgroup cfs quotas")
158
-	}
159
-
160
-	info.CPURealtimePeriod = cgroupEnabled(mountPoint, "cpu.rt_period_us")
161
-	if !info.CPURealtimePeriod {
162
-		warnings = append(warnings, "Your kernel does not support cgroup rt period")
163
-	}
164
-
165
-	info.CPURealtimeRuntime = cgroupEnabled(mountPoint, "cpu.rt_runtime_us")
166
-	if !info.CPURealtimeRuntime {
167
-		warnings = append(warnings, "Your kernel does not support cgroup rt runtime")
155
+	info.CPURealtime = cgroupEnabled(mountPoint, "cpu.rt_period_us")
156
+	if !info.CPURealtime {
157
+		warnings = append(warnings, "Your kernel does not support CPU realtime scheduler")
168 158
 	}
169 159
 
170 160
 	return warnings
... ...
@@ -240,46 +240,41 @@ func TestDecodeHostConfig(t *testing.T) {
240 240
 
241 241
 func TestValidateResources(t *testing.T) {
242 242
 	type resourceTest struct {
243
-		ConfigCPURealtimePeriod   int64
244
-		ConfigCPURealtimeRuntime  int64
245
-		SysInfoCPURealtimePeriod  bool
246
-		SysInfoCPURealtimeRuntime bool
247
-		ErrorExpected             bool
248
-		FailureMsg                string
243
+		ConfigCPURealtimePeriod  int64
244
+		ConfigCPURealtimeRuntime int64
245
+		SysInfoCPURealtime       bool
246
+		ErrorExpected            bool
247
+		FailureMsg               string
249 248
 	}
250 249
 
251 250
 	tests := []resourceTest{
252 251
 		{
253
-			ConfigCPURealtimePeriod:   1000,
254
-			ConfigCPURealtimeRuntime:  1000,
255
-			SysInfoCPURealtimePeriod:  true,
256
-			SysInfoCPURealtimeRuntime: true,
257
-			ErrorExpected:             false,
258
-			FailureMsg:                "Expected valid configuration",
252
+			ConfigCPURealtimePeriod:  1000,
253
+			ConfigCPURealtimeRuntime: 1000,
254
+			SysInfoCPURealtime:       true,
255
+			ErrorExpected:            false,
256
+			FailureMsg:               "Expected valid configuration",
259 257
 		},
260 258
 		{
261
-			ConfigCPURealtimePeriod:   5000,
262
-			ConfigCPURealtimeRuntime:  5000,
263
-			SysInfoCPURealtimePeriod:  false,
264
-			SysInfoCPURealtimeRuntime: true,
265
-			ErrorExpected:             true,
266
-			FailureMsg:                "Expected failure when cpu-rt-period is set but kernel doesn't support it",
259
+			ConfigCPURealtimePeriod:  5000,
260
+			ConfigCPURealtimeRuntime: 5000,
261
+			SysInfoCPURealtime:       false,
262
+			ErrorExpected:            true,
263
+			FailureMsg:               "Expected failure when cpu-rt-period is set but kernel doesn't support it",
267 264
 		},
268 265
 		{
269
-			ConfigCPURealtimePeriod:   5000,
270
-			ConfigCPURealtimeRuntime:  5000,
271
-			SysInfoCPURealtimePeriod:  true,
272
-			SysInfoCPURealtimeRuntime: false,
273
-			ErrorExpected:             true,
274
-			FailureMsg:                "Expected failure when cpu-rt-runtime is set but kernel doesn't support it",
266
+			ConfigCPURealtimePeriod:  5000,
267
+			ConfigCPURealtimeRuntime: 5000,
268
+			SysInfoCPURealtime:       false,
269
+			ErrorExpected:            true,
270
+			FailureMsg:               "Expected failure when cpu-rt-runtime is set but kernel doesn't support it",
275 271
 		},
276 272
 		{
277
-			ConfigCPURealtimePeriod:   5000,
278
-			ConfigCPURealtimeRuntime:  10000,
279
-			SysInfoCPURealtimePeriod:  true,
280
-			SysInfoCPURealtimeRuntime: false,
281
-			ErrorExpected:             true,
282
-			FailureMsg:                "Expected failure when cpu-rt-runtime is greater than cpu-rt-period",
273
+			ConfigCPURealtimePeriod:  5000,
274
+			ConfigCPURealtimeRuntime: 10000,
275
+			SysInfoCPURealtime:       true,
276
+			ErrorExpected:            true,
277
+			FailureMsg:               "Expected failure when cpu-rt-runtime is greater than cpu-rt-period",
283 278
 		},
284 279
 	}
285 280
 
... ...
@@ -289,8 +284,7 @@ func TestValidateResources(t *testing.T) {
289 289
 		hc.Resources.CPURealtimeRuntime = rt.ConfigCPURealtimeRuntime
290 290
 
291 291
 		var si sysinfo.SysInfo
292
-		si.CPURealtimePeriod = rt.SysInfoCPURealtimePeriod
293
-		si.CPURealtimeRuntime = rt.SysInfoCPURealtimeRuntime
292
+		si.CPURealtime = rt.SysInfoCPURealtime
294 293
 
295 294
 		if err := validateResources(&hc, &si); (err != nil) != rt.ErrorExpected {
296 295
 			t.Fatal(rt.FailureMsg, err)
... ...
@@ -85,12 +85,8 @@ func validateResources(hc *container.HostConfig, si *sysinfo.SysInfo) error {
85 85
 		return nil
86 86
 	}
87 87
 
88
-	if hc.Resources.CPURealtimePeriod > 0 && !si.CPURealtimePeriod {
89
-		return fmt.Errorf("Your kernel does not support cgroup cpu real-time period")
90
-	}
91
-
92
-	if hc.Resources.CPURealtimeRuntime > 0 && !si.CPURealtimeRuntime {
93
-		return fmt.Errorf("Your kernel does not support cgroup cpu real-time runtime")
88
+	if (hc.Resources.CPURealtimePeriod != 0 || hc.Resources.CPURealtimeRuntime != 0) && !si.CPURealtime {
89
+		return fmt.Errorf("Your kernel does not support CPU real-time scheduler")
94 90
 	}
95 91
 
96 92
 	if hc.Resources.CPURealtimePeriod != 0 && hc.Resources.CPURealtimeRuntime != 0 && hc.Resources.CPURealtimeRuntime > hc.Resources.CPURealtimePeriod {