Browse code

Adding support for memory swap settings for services

With integration tests

Relevant Swarmkit PR: https://github.com/docker/swarmkit/pull/2816
(updated the vendored version of Swarkit to that)

Signed-off-by: Jean Rouge <rougej+github@gmail.com>

Updated for latest master, fixed bitrot.

Signed-off-by: Drew Erny <derny@mirantis.com>

Jean Rouge authored on 2019/03/15 01:44:32
Showing 11 changed files
... ...
@@ -57,6 +57,18 @@ keywords: "API, Docker, rcli, REST, documentation"
57 57
 * `POST /containers/create` no longer supports configuring a container-wide MAC address
58 58
   via the container's `Config.MacAddress` field. A container's MAC address can now only 
59 59
   be configured via endpoint settings when connecting to a network.
60
+* `GET /services` now returns `SwapBytes` and `MemorySwappiness` fields as part
61
+  of the `Resource` requirements.
62
+* `GET /services/{id}` now returns `SwapBytes` and `MemorySwappiness` fields as
63
+  part of the `Resource` requirements.
64
+* `POST /services/create` now accepts `SwapBytes` and `MemorySwappiness` fields
65
+  as part of the `Resource` requirements.
66
+* `POST /services/{id}/update` now accepts `SwapBytes` and `MemorySwappiness`
67
+  fields as part of the `Resource` requirements.
68
+* `GET /tasks` now  returns `SwapBytes` and `MemorySwappiness` fields as part
69
+  of the `Resource` requirements.
70
+* `GET /tasks/{id}` now  returns `SwapBytes` and `MemorySwappiness` fields as
71
+  part of the `Resource` requirements.
60 72
 
61 73
 ## v1.51 API changes
62 74
 
... ...
@@ -4287,6 +4287,29 @@ definitions:
4287 4287
           Reservations:
4288 4288
             description: "Define resources reservation."
4289 4289
             $ref: "#/definitions/ResourceObject"
4290
+          SwapBytes:
4291
+            description: |
4292
+              Amount of swap in bytes - can only be used together with a memory limit.
4293
+              If not specified, the default behaviour is to grant a swap space twice
4294
+              as big as the memory limit.
4295
+              Set to -1 to enable unlimited swap.
4296
+            type: "integer"
4297
+            format: "int64"
4298
+            minimum: -1
4299
+            x-nullable: true
4300
+            x-omitempty: true
4301
+          MemorySwappiness:
4302
+            description: |
4303
+              Tune the service's containers' memory swappiness (0 to 100).
4304
+              If not specified, defaults to the containers' OS' default, generally 60,
4305
+              or whatever value was predefined in the image.
4306
+              Set to -1 to unset a previously set value.
4307
+            type: "integer"
4308
+            format: "int64"
4309
+            minimum: -1
4310
+            maximum: 100
4311
+            x-nullable: true
4312
+            x-omitempty: true
4290 4313
       RestartPolicy:
4291 4314
         description: |
4292 4315
           Specification for the restart policy which applies to containers
... ...
@@ -4287,6 +4287,29 @@ definitions:
4287 4287
           Reservations:
4288 4288
             description: "Define resources reservation."
4289 4289
             $ref: "#/definitions/ResourceObject"
4290
+          SwapBytes:
4291
+            description: |
4292
+              Amount of swap in bytes - can only be used together with a memory limit.
4293
+              If not specified, the default behaviour is to grant a swap space twice
4294
+              as big as the memory limit.
4295
+              Set to -1 to enable unlimited swap.
4296
+            type: "integer"
4297
+            format: "int64"
4298
+            minimum: -1
4299
+            x-nullable: true
4300
+            x-omitempty: true
4301
+          MemorySwappiness:
4302
+            description: |
4303
+              Tune the service's containers' memory swappiness (0 to 100).
4304
+              If not specified, defaults to the containers' OS' default, generally 60,
4305
+              or whatever value was predefined in the image.
4306
+              Set to -1 to unset a previously set value.
4307
+            type: "integer"
4308
+            format: "int64"
4309
+            minimum: -1
4310
+            maximum: 100
4311
+            x-nullable: true
4312
+            x-omitempty: true
4290 4313
       RestartPolicy:
4291 4314
         description: |
4292 4315
           Specification for the restart policy which applies to containers
... ...
@@ -138,6 +138,17 @@ type DiscreteGenericResource struct {
138 138
 type ResourceRequirements struct {
139 139
 	Limits       *Limit     `json:",omitempty"`
140 140
 	Reservations *Resources `json:",omitempty"`
141
+
142
+	// Amount of swap in bytes - can only be used together with a memory limit
143
+	// -1 means unlimited
144
+	// a null pointer keeps the default behaviour of granting twice the memory
145
+	// amount in swap
146
+	SwapBytes *int64 `json:"SwapBytes,omitzero"`
147
+
148
+	// Tune container memory swappiness (0 to 100) - if not specified, defaults
149
+	// to the container OS's default - generally 60, or the value predefined in
150
+	// the image; set to -1 to unset a previously set value
151
+	MemorySwappiness *int64 `json:MemorySwappiness,omitzero"`
141 152
 }
142 153
 
143 154
 // Placement represents orchestration parameters.
... ...
@@ -178,13 +178,18 @@ func ServiceSpecToGRPC(s types.ServiceSpec) (swarmapi.ServiceSpec, error) {
178 178
 		taskNetworks = append(taskNetworks, netConfig)
179 179
 	}
180 180
 
181
+	resources, err := resourcesToGRPC(s.TaskTemplate.Resources)
182
+	if err != nil {
183
+		return swarmapi.ServiceSpec{}, err
184
+	}
185
+
181 186
 	spec := swarmapi.ServiceSpec{
182 187
 		Annotations: swarmapi.Annotations{
183 188
 			Name:   name,
184 189
 			Labels: s.Labels,
185 190
 		},
186 191
 		Task: swarmapi.TaskSpec{
187
-			Resources:   resourcesToGRPC(s.TaskTemplate.Resources),
192
+			Resources:   resources,
188 193
 			LogDriver:   driverToGRPC(s.TaskTemplate.LogDriver),
189 194
 			Networks:    taskNetworks,
190 195
 			ForceUpdate: s.TaskTemplate.ForceUpdate,
... ...
@@ -439,11 +444,20 @@ func resourcesFromGRPC(ts *swarmapi.TaskSpec) *types.ResourceRequirements {
439 439
 				GenericResources: GenericResourcesFromGRPC(res.Reservations.Generic),
440 440
 			}
441 441
 		}
442
+		resources.SwapBytes = int64PointerFromGRPC(res.SwapBytes)
443
+		resources.MemorySwappiness = int64PointerFromGRPC(res.MemorySwappiness)
442 444
 	}
443 445
 
444 446
 	return resources
445 447
 }
446 448
 
449
+func int64PointerFromGRPC(v *gogotypes.Int64Value) *int64 {
450
+	if v == nil {
451
+		return nil
452
+	}
453
+	return &v.Value
454
+}
455
+
447 456
 // GenericResourcesToGRPC converts a GenericResource to a GRPC GenericResource
448 457
 func GenericResourcesToGRPC(genericRes []types.GenericResource) []*swarmapi.GenericResource {
449 458
 	var generic []*swarmapi.GenericResource
... ...
@@ -462,7 +476,7 @@ func GenericResourcesToGRPC(genericRes []types.GenericResource) []*swarmapi.Gene
462 462
 	return generic
463 463
 }
464 464
 
465
-func resourcesToGRPC(res *types.ResourceRequirements) *swarmapi.ResourceRequirements {
465
+func resourcesToGRPC(res *types.ResourceRequirements) (*swarmapi.ResourceRequirements, error) {
466 466
 	var reqs *swarmapi.ResourceRequirements
467 467
 	if res != nil {
468 468
 		reqs = &swarmapi.ResourceRequirements{}
... ...
@@ -480,8 +494,21 @@ func resourcesToGRPC(res *types.ResourceRequirements) *swarmapi.ResourceRequirem
480 480
 				Generic:     GenericResourcesToGRPC(res.Reservations.GenericResources),
481 481
 			}
482 482
 		}
483
+		reqs.SwapBytes = int64PointerToGRPC(res.SwapBytes)
484
+		reqs.MemorySwappiness = int64PointerToGRPC(res.MemorySwappiness)
485
+
486
+		if reqs.SwapBytes != nil && (reqs.Limits == nil || reqs.Limits.MemoryBytes == 0) {
487
+			return nil, errors.New("memory swap provided, but no memory-limit was set")
488
+		}
489
+	}
490
+	return reqs, nil
491
+}
492
+
493
+func int64PointerToGRPC(v *int64) *gogotypes.Int64Value {
494
+	if v == nil {
495
+		return nil
483 496
 	}
484
-	return reqs
497
+	return &gogotypes.Int64Value{Value: *v}
485 498
 }
486 499
 
487 500
 func restartPolicyFromGRPC(p *swarmapi.RestartPolicy) *types.RestartPolicy {
... ...
@@ -527,6 +527,20 @@ func (c *containerConfig) resources() container.Resources {
527 527
 
528 528
 	if r.Limits.MemoryBytes > 0 {
529 529
 		resources.Memory = r.Limits.MemoryBytes
530
+
531
+		if r.SwapBytes != nil {
532
+			if swapBytes := r.SwapBytes.Value; swapBytes == -1 {
533
+				// means unlimited
534
+				resources.MemorySwap = -1
535
+			} else if swapBytes >= 0 {
536
+				// resources.MemorySwap is actually the sum of the memory + the swap
537
+				resources.MemorySwap = resources.Memory + swapBytes
538
+			}
539
+		}
540
+	}
541
+
542
+	if r.MemorySwappiness != nil {
543
+		resources.MemorySwappiness = &r.MemorySwappiness.Value
530 544
 	}
531 545
 
532 546
 	if r.Limits.NanoCPUs > 0 {
... ...
@@ -77,6 +77,14 @@ func adjustForAPIVersion(cliVersion string, service *serviceWithLegacy) {
77 77
 	if cliVersion == "" {
78 78
 		return
79 79
 	}
80
+
81
+	if versions.LessThan(cliVersion, "1.52") {
82
+		if service.TaskTemplate.Resources != nil {
83
+			service.TaskTemplate.Resources.SwapBytes = nil
84
+			service.TaskTemplate.Resources.MemorySwappiness = nil
85
+		}
86
+	}
87
+
80 88
 	if versions.LessThan(cliVersion, "1.46") {
81 89
 		if service.TaskTemplate.ContainerSpec != nil {
82 90
 			for i, mount := range service.TaskTemplate.ContainerSpec.Mounts {
... ...
@@ -11,6 +11,9 @@ import (
11 11
 
12 12
 func TestAdjustForAPIVersion(t *testing.T) {
13 13
 	expectedSysctls := map[string]string{"foo": "bar"}
14
+	swapBytes := int64(12)
15
+	memorySwappiness := int64(28)
16
+
14 17
 	// testing the negative -- does this leave everything else alone? -- is
15 18
 	// prohibitively time-consuming to write, because it would need an object
16 19
 	// with literally every field filled in.
... ...
@@ -66,6 +69,8 @@ func TestAdjustForAPIVersion(t *testing.T) {
66 66
 				Limits: &swarm.Limit{
67 67
 					Pids: 300,
68 68
 				},
69
+				SwapBytes:        &swapBytes,
70
+				MemorySwappiness: &memorySwappiness,
69 71
 			},
70 72
 		},
71 73
 	}
... ...
@@ -73,6 +78,16 @@ func TestAdjustForAPIVersion(t *testing.T) {
73 73
 	spec := &serviceWithLegacy{
74 74
 		ServiceSpec: serviceSpec,
75 75
 	}
76
+
77
+	adjustForAPIVersion("1.52", spec)
78
+	if spec.TaskTemplate.Resources.MemorySwappiness == nil {
79
+		t.Error("SwapBytes was stripped from spec")
80
+	}
81
+
82
+	if spec.TaskTemplate.Resources.SwapBytes == nil {
83
+		t.Error("MemorySwappiness was stripped from spec")
84
+	}
85
+
76 86
 	adjustForAPIVersion("1.46", spec)
77 87
 	if !reflect.DeepEqual(
78 88
 		spec.TaskTemplate.ContainerSpec.Mounts[0].TmpfsOptions.Options,
... ...
@@ -142,4 +157,11 @@ func TestAdjustForAPIVersion(t *testing.T) {
142 142
 	if len(spec.TaskTemplate.ContainerSpec.Ulimits) != 0 {
143 143
 		t.Error("Ulimits were not stripped from spec")
144 144
 	}
145
+	if spec.TaskTemplate.Resources.MemorySwappiness != nil {
146
+		t.Error("SwapBytes was not stripped from spec")
147
+	}
148
+
149
+	if spec.TaskTemplate.Resources.SwapBytes != nil {
150
+		t.Error("MemorySwappiness was not stripped from spec")
151
+	}
145 152
 }
... ...
@@ -200,6 +200,21 @@ func ServiceWithPidsLimit(limit int64) ServiceSpecOpt {
200 200
 	}
201 201
 }
202 202
 
203
+func ServiceWithMemorySwap(swap *int64, memoryLimit int64) ServiceSpecOpt {
204
+	return func(spec *swarmtypes.ServiceSpec) {
205
+		ensureResources(spec)
206
+		spec.TaskTemplate.Resources.SwapBytes = swap
207
+		spec.TaskTemplate.Resources.Limits.MemoryBytes = memoryLimit
208
+	}
209
+}
210
+
211
+func ServiceWithMemorySwappiness(swappiness *int64) ServiceSpecOpt {
212
+	return func(spec *swarmtypes.ServiceSpec) {
213
+		ensureResources(spec)
214
+		spec.TaskTemplate.Resources.MemorySwappiness = swappiness
215
+	}
216
+}
217
+
203 218
 // GetRunningTasks gets the list of running tasks for a service
204 219
 func GetRunningTasks(ctx context.Context, t *testing.T, c client.ServiceAPIClient, serviceID string) []swarmtypes.Task {
205 220
 	t.Helper()
... ...
@@ -8,6 +8,7 @@ import (
8 8
 	"time"
9 9
 
10 10
 	cerrdefs "github.com/containerd/errdefs"
11
+	"github.com/docker/go-units"
11 12
 	"github.com/moby/moby/api/types/container"
12 13
 	swarmtypes "github.com/moby/moby/api/types/swarm"
13 14
 	"github.com/moby/moby/client"
... ...
@@ -458,3 +459,162 @@ func TestCreateServiceCapabilities(t *testing.T) {
458 458
 	assert.DeepEqual(t, service.Spec.TaskTemplate.ContainerSpec.CapabilityAdd, capAdd)
459 459
 	assert.DeepEqual(t, service.Spec.TaskTemplate.ContainerSpec.CapabilityDrop, capDrop)
460 460
 }
461
+
462
+func TestCreateServiceMemorySwap(t *testing.T) {
463
+	ctx := setupTest(t)
464
+	d := swarm.NewSwarm(ctx, t, testEnv)
465
+	defer d.Stop(t)
466
+	apiClient := d.NewClientT(t)
467
+	defer apiClient.Close()
468
+
469
+	toPtr := func(v int64) *int64 { return &v }
470
+	tests := []struct {
471
+		testName string
472
+
473
+		swapSpec  *int64
474
+		limitSpec int64
475
+
476
+		// as reported by Docker
477
+		expectedDockerSwap int64
478
+	}{
479
+		{
480
+			testName: "default",
481
+		},
482
+		{
483
+			testName:           "memory-limit and memory-swap",
484
+			swapSpec:           toPtr(1 * units.MiB),
485
+			limitSpec:          20 * units.MiB,
486
+			expectedDockerSwap: 21 * units.MiB,
487
+		},
488
+		{
489
+			testName:           "memory-limit alone - should default to twice as much swap",
490
+			limitSpec:          20 * units.MiB,
491
+			expectedDockerSwap: 40 * units.MiB,
492
+		},
493
+		{
494
+			testName:           "memory-limit and zero memory-swap",
495
+			swapSpec:           toPtr(0),
496
+			limitSpec:          20 * units.MiB,
497
+			expectedDockerSwap: 20 * units.MiB,
498
+		},
499
+		{
500
+			testName:           "memory-limit and unlimited memory-swap",
501
+			swapSpec:           toPtr(-1),
502
+			limitSpec:          20 * units.MiB,
503
+			expectedDockerSwap: -1,
504
+		},
505
+	}
506
+
507
+	for _, testCase := range tests {
508
+		t.Run("service create with "+testCase.testName, func(t *testing.T) {
509
+			serviceID := swarm.CreateService(
510
+				ctx, t, d,
511
+				swarm.ServiceWithMemorySwap(testCase.swapSpec, testCase.limitSpec),
512
+			)
513
+			poll.WaitOn(t, swarm.RunningTasksCount(ctx, apiClient, serviceID, 1))
514
+
515
+			service, _, err := apiClient.ServiceInspectWithRaw(ctx, serviceID, client.ServiceInspectOptions{})
516
+			assert.NilError(t, err)
517
+
518
+			filter := make(client.Filters)
519
+			filter.Add("service", serviceID)
520
+			tasks, err := apiClient.TaskList(ctx, client.TaskListOptions{
521
+				Filters: filter,
522
+			})
523
+			assert.NilError(t, err)
524
+			assert.Check(t, is.Equal(len(tasks), 1))
525
+			task := tasks[0]
526
+
527
+			if testCase.swapSpec == nil {
528
+				assert.Check(t, is.Nil(task.Spec.Resources.SwapBytes))
529
+				assert.Check(t, is.Nil(service.Spec.TaskTemplate.Resources.SwapBytes))
530
+			} else {
531
+				assert.Equal(t, *testCase.swapSpec, *task.Spec.Resources.SwapBytes)
532
+				assert.Equal(t, *testCase.swapSpec, *service.Spec.TaskTemplate.Resources.SwapBytes)
533
+			}
534
+
535
+			// if the host supports it (see https://github.com/moby/moby/blob/v17.03.2-ce/daemon/daemon_unix.go#L290-L294)
536
+			// then check that the swap option is set on the container, and properly reported by the group FS as well
537
+			if testEnv.DaemonInfo.SwapLimit {
538
+				ctnr, err := apiClient.ContainerInspect(ctx, task.Status.ContainerStatus.ContainerID)
539
+				assert.NilError(t, err)
540
+				assert.Equal(t, testCase.expectedDockerSwap, ctnr.HostConfig.Resources.MemorySwap)
541
+			}
542
+		})
543
+	}
544
+
545
+	t.Run("cannot create a service with a memory swap option without setting a memory limit", func(t *testing.T) {
546
+		serviceOpts := func(spec *swarmtypes.ServiceSpec) {
547
+			if spec.TaskTemplate.Resources == nil {
548
+				spec.TaskTemplate.Resources = &swarmtypes.ResourceRequirements{}
549
+			}
550
+			spec.TaskTemplate.Resources.SwapBytes = toPtr(10 * units.MiB)
551
+		}
552
+
553
+		spec := swarm.CreateServiceSpec(t, serviceOpts)
554
+		_, err := apiClient.ServiceCreate(context.Background(), spec, client.ServiceCreateOptions{})
555
+
556
+		assert.ErrorContains(t, err, "memory swap provided, but no memory-limit was set")
557
+	})
558
+}
559
+
560
+func TestCreateServiceMemorySwappiness(t *testing.T) {
561
+	ctx := setupTest(t)
562
+	d := swarm.NewSwarm(ctx, t, testEnv)
563
+	defer d.Stop(t)
564
+	apiClient := d.NewClientT(t)
565
+	defer apiClient.Close()
566
+
567
+	toPtr := func(v int64) *int64 { return &v }
568
+
569
+	tests := []struct {
570
+		testName       string
571
+		swappinessSpec *int64
572
+	}{
573
+		{testName: "default"},
574
+		{testName: "zero memory-swappiness", swappinessSpec: toPtr(0)},
575
+		{testName: "memory-swappiness", swappinessSpec: toPtr(28)},
576
+	}
577
+
578
+	for _, testCase := range tests {
579
+		t.Run("service create with "+testCase.testName, func(t *testing.T) {
580
+			serviceID := swarm.CreateService(
581
+				ctx, t, d,
582
+				swarm.ServiceWithMemorySwappiness(testCase.swappinessSpec),
583
+			)
584
+			poll.WaitOn(t, swarm.RunningTasksCount(ctx, apiClient, serviceID, 1))
585
+
586
+			filter := make(client.Filters)
587
+			filter.Add("service", serviceID)
588
+			tasks, err := apiClient.TaskList(ctx, client.TaskListOptions{
589
+				Filters: filter,
590
+			})
591
+			assert.NilError(t, err)
592
+			assert.Check(t, is.Equal(len(tasks), 1))
593
+			task := tasks[0]
594
+
595
+			service, _, err := apiClient.ServiceInspectWithRaw(ctx, serviceID, client.ServiceInspectOptions{})
596
+			assert.NilError(t, err)
597
+
598
+			// An earlier version of this test also inspected the container
599
+			// created by Swarm to ensure that MemorySwappiness was set on its
600
+			// HostConfig. However, on systems that do not support
601
+			// MemorySwappiness (the Github Actions platform is one, in late
602
+			// 2025), that field in the HostConfig is nilled out, and a warning
603
+			// is returned. Swarm doesn't do anything with the warning, so the
604
+			// setting is silently ignored. Getting the raw SysInfo can show if
605
+			// MemorySwappiness is supported, but that field is not present in
606
+			// a regular Info API call, and so is not part of
607
+			// testEnv.DaemonInfo and cannot be checked (easily) here. So,
608
+			// ultimately, we'll skip that check in the integration test.
609
+
610
+			if testCase.swappinessSpec == nil {
611
+				assert.Check(t, is.Nil(task.Spec.Resources.MemorySwappiness))
612
+				assert.Check(t, is.Nil(service.Spec.TaskTemplate.Resources.MemorySwappiness))
613
+			} else {
614
+				assert.Equal(t, *testCase.swappinessSpec, *task.Spec.Resources.MemorySwappiness)
615
+				assert.Equal(t, *testCase.swappinessSpec, *service.Spec.TaskTemplate.Resources.MemorySwappiness)
616
+			}
617
+		})
618
+	}
619
+}
... ...
@@ -138,6 +138,17 @@ type DiscreteGenericResource struct {
138 138
 type ResourceRequirements struct {
139 139
 	Limits       *Limit     `json:",omitempty"`
140 140
 	Reservations *Resources `json:",omitempty"`
141
+
142
+	// Amount of swap in bytes - can only be used together with a memory limit
143
+	// -1 means unlimited
144
+	// a null pointer keeps the default behaviour of granting twice the memory
145
+	// amount in swap
146
+	SwapBytes *int64 `json:"SwapBytes,omitzero"`
147
+
148
+	// Tune container memory swappiness (0 to 100) - if not specified, defaults
149
+	// to the container OS's default - generally 60, or the value predefined in
150
+	// the image; set to -1 to unset a previously set value
151
+	MemorySwappiness *int64 `json:MemorySwappiness,omitzero"`
141 152
 }
142 153
 
143 154
 // Placement represents orchestration parameters.