Browse code

Use rslave propagation for mounts from daemon root

By default, if a user requests a bind mount it uses private propagation.
When the source path is a path within the daemon root this, along with
some other propagation values that the user can use, causes issues when
the daemon tries to remove a mountpoint because a container will then
have a private reference to that mount which prevents removal.

Unmouting with MNT_DETATCH can help this scenario on newer kernels, but
ultimately this is just covering up the problem and doesn't actually
free up the underlying resources until all references are destroyed.

This change does essentially 2 things:

1. Change the default propagation when unspecified to `rslave` when the
source path is within the daemon root path or a parent of the daemon
root (because everything is using rbinds).
2. Creates a validation error on create when the user tries to specify
an unacceptable propagation mode for these paths...
basically the only two acceptable modes are `rslave` and `rshared`.

In cases where we have used the new default propagation but the
underlying filesystem is not setup to handle it (fs must hvae at least
rshared propagation) instead of erroring out like we normally would,
this falls back to the old default mode of `private`, which preserves
backwards compatibility.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Brian Goff authored on 2018/01/19 06:55:27
Showing 6 changed files
... ...
@@ -604,7 +604,8 @@ func setMounts(daemon *Daemon, s *specs.Spec, c *container.Container, mounts []c
604 604
 		//
605 605
 		// For private volumes any root propagation value should work.
606 606
 		pFlag := mountPropagationMap[m.Propagation]
607
-		if pFlag == mount.SHARED || pFlag == mount.RSHARED {
607
+		switch pFlag {
608
+		case mount.SHARED, mount.RSHARED:
608 609
 			if err := ensureShared(m.Source); err != nil {
609 610
 				return err
610 611
 			}
... ...
@@ -612,13 +613,34 @@ func setMounts(daemon *Daemon, s *specs.Spec, c *container.Container, mounts []c
612 612
 			if rootpg != mount.SHARED && rootpg != mount.RSHARED {
613 613
 				s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.SHARED]
614 614
 			}
615
-		} else if pFlag == mount.SLAVE || pFlag == mount.RSLAVE {
615
+		case mount.SLAVE, mount.RSLAVE:
616
+			var fallback bool
616 617
 			if err := ensureSharedOrSlave(m.Source); err != nil {
617
-				return err
618
+				// For backwards compatability purposes, treat mounts from the daemon root
619
+				// as special since we automatically add rslave propagation to these mounts
620
+				// when the user did not set anything, so we should fallback to the old
621
+				// behavior which is to use private propagation which is normally the
622
+				// default.
623
+				if !strings.HasPrefix(m.Source, daemon.root) && !strings.HasPrefix(daemon.root, m.Source) {
624
+					return err
625
+				}
626
+
627
+				cm, ok := c.MountPoints[m.Destination]
628
+				if !ok {
629
+					return err
630
+				}
631
+				if cm.Spec.BindOptions != nil && cm.Spec.BindOptions.Propagation != "" {
632
+					// This means the user explicitly set a propagation, do not fallback in that case.
633
+					return err
634
+				}
635
+				fallback = true
636
+				logrus.WithField("container", c.ID).WithField("source", m.Source).Warn("Falling back to default propagation for bind source in daemon root")
618 637
 			}
619
-			rootpg := mountPropagationMap[s.Linux.RootfsPropagation]
620
-			if rootpg != mount.SHARED && rootpg != mount.RSHARED && rootpg != mount.SLAVE && rootpg != mount.RSLAVE {
621
-				s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.RSLAVE]
638
+			if !fallback {
639
+				rootpg := mountPropagationMap[s.Linux.RootfsPropagation]
640
+				if rootpg != mount.SHARED && rootpg != mount.RSHARED && rootpg != mount.SLAVE && rootpg != mount.RSLAVE {
641
+					s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.RSLAVE]
642
+				}
622 643
 			}
623 644
 		}
624 645
 
... ...
@@ -10,6 +10,7 @@ import (
10 10
 
11 11
 	"github.com/docker/docker/api/types"
12 12
 	containertypes "github.com/docker/docker/api/types/container"
13
+	"github.com/docker/docker/api/types/mount"
13 14
 	mounttypes "github.com/docker/docker/api/types/mount"
14 15
 	"github.com/docker/docker/container"
15 16
 	"github.com/docker/docker/errdefs"
... ...
@@ -146,6 +147,13 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo
146 146
 		if err != nil {
147 147
 			return err
148 148
 		}
149
+		needsSlavePropagation, err := daemon.validateBindDaemonRoot(bind.Spec)
150
+		if err != nil {
151
+			return err
152
+		}
153
+		if needsSlavePropagation {
154
+			bind.Propagation = mount.PropagationRSlave
155
+		}
149 156
 
150 157
 		// #10618
151 158
 		_, tmpfsExists := hostConfig.Tmpfs[bind.Destination]
... ...
@@ -178,6 +186,13 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo
178 178
 		if err != nil {
179 179
 			return errdefs.InvalidParameter(err)
180 180
 		}
181
+		needsSlavePropagation, err := daemon.validateBindDaemonRoot(mp.Spec)
182
+		if err != nil {
183
+			return err
184
+		}
185
+		if needsSlavePropagation {
186
+			mp.Propagation = mount.PropagationRSlave
187
+		}
181 188
 
182 189
 		if binds[mp.Destination] {
183 190
 			return duplicateMountPointError(cfg.Target)
184 191
new file mode 100644
... ...
@@ -0,0 +1,36 @@
0
+package daemon
1
+
2
+import (
3
+	"strings"
4
+
5
+	"github.com/docker/docker/api/types/mount"
6
+	"github.com/docker/docker/errdefs"
7
+	"github.com/pkg/errors"
8
+)
9
+
10
+// validateBindDaemonRoot ensures that if a given mountpoint's source is within
11
+// the daemon root path, that the propagation is setup to prevent a container
12
+// from holding private refereneces to a mount within the daemon root, which
13
+// can cause issues when the daemon attempts to remove the mountpoint.
14
+func (daemon *Daemon) validateBindDaemonRoot(m mount.Mount) (bool, error) {
15
+	if m.Type != mount.TypeBind {
16
+		return false, nil
17
+	}
18
+
19
+	// check if the source is within the daemon root, or if the daemon root is within the source
20
+	if !strings.HasPrefix(m.Source, daemon.root) && !strings.HasPrefix(daemon.root, m.Source) {
21
+		return false, nil
22
+	}
23
+
24
+	if m.BindOptions == nil {
25
+		return true, nil
26
+	}
27
+
28
+	switch m.BindOptions.Propagation {
29
+	case mount.PropagationRSlave, mount.PropagationRShared, "":
30
+		return m.BindOptions.Propagation == "", nil
31
+	default:
32
+	}
33
+
34
+	return false, errdefs.InvalidParameter(errors.Errorf(`invalid mount config: must use either propagation mode "rslave" or "rshared" when mount source is within the daemon root, daemon root: %q, bind mount source: %q, propagation: %q`, daemon.root, m.Source, m.BindOptions.Propagation))
35
+}
0 36
new file mode 100644
... ...
@@ -0,0 +1,56 @@
0
+package daemon
1
+
2
+import (
3
+	"path/filepath"
4
+	"testing"
5
+
6
+	"github.com/docker/docker/api/types/mount"
7
+)
8
+
9
+func TestBindDaemonRoot(t *testing.T) {
10
+	t.Parallel()
11
+	d := &Daemon{root: "/a/b/c/daemon"}
12
+	for _, test := range []struct {
13
+		desc      string
14
+		opts      *mount.BindOptions
15
+		needsProp bool
16
+		err       bool
17
+	}{
18
+		{desc: "nil propagation settings", opts: nil, needsProp: true, err: false},
19
+		{desc: "empty propagation settings", opts: &mount.BindOptions{}, needsProp: true, err: false},
20
+		{desc: "private propagation", opts: &mount.BindOptions{Propagation: mount.PropagationPrivate}, err: true},
21
+		{desc: "rprivate propagation", opts: &mount.BindOptions{Propagation: mount.PropagationRPrivate}, err: true},
22
+		{desc: "slave propagation", opts: &mount.BindOptions{Propagation: mount.PropagationSlave}, err: true},
23
+		{desc: "rslave propagation", opts: &mount.BindOptions{Propagation: mount.PropagationRSlave}, err: false, needsProp: false},
24
+		{desc: "shared propagation", opts: &mount.BindOptions{Propagation: mount.PropagationShared}, err: true},
25
+		{desc: "rshared propagation", opts: &mount.BindOptions{Propagation: mount.PropagationRSlave}, err: false, needsProp: false},
26
+	} {
27
+		t.Run(test.desc, func(t *testing.T) {
28
+			test := test
29
+			for desc, source := range map[string]string{
30
+				"source is root":    d.root,
31
+				"source is subpath": filepath.Join(d.root, "a", "b"),
32
+				"source is parent":  filepath.Dir(d.root),
33
+				"source is /":       "/",
34
+			} {
35
+				t.Run(desc, func(t *testing.T) {
36
+					mount := mount.Mount{
37
+						Type:        mount.TypeBind,
38
+						Source:      source,
39
+						BindOptions: test.opts,
40
+					}
41
+					needsProp, err := d.validateBindDaemonRoot(mount)
42
+					if (err != nil) != test.err {
43
+						t.Fatalf("expected err=%v, got: %v", test.err, err)
44
+					}
45
+					if test.err {
46
+						return
47
+					}
48
+					if test.needsProp != needsProp {
49
+						t.Fatalf("expected needsProp=%v, got: %v", test.needsProp, needsProp)
50
+					}
51
+				})
52
+			}
53
+		})
54
+	}
55
+}
... ...
@@ -3,6 +3,7 @@ package daemon
3 3
 import (
4 4
 	"sort"
5 5
 
6
+	"github.com/docker/docker/api/types/mount"
6 7
 	"github.com/docker/docker/container"
7 8
 	"github.com/docker/docker/pkg/idtools"
8 9
 	"github.com/docker/docker/volume"
... ...
@@ -44,3 +45,7 @@ func (daemon *Daemon) setupMounts(c *container.Container) ([]container.Mount, er
44 44
 func setBindModeIfNull(bind *volume.MountPoint) {
45 45
 	return
46 46
 }
47
+
48
+func (daemon *Daemon) validateBindDaemonRoot(m mount.Mount) (bool, error) {
49
+	return false, nil
50
+}
... ...
@@ -4,6 +4,7 @@ import (
4 4
 	"bytes"
5 5
 	"context"
6 6
 	"fmt"
7
+	"path/filepath"
7 8
 	"testing"
8 9
 
9 10
 	"github.com/docker/docker/api/types"
... ...
@@ -12,6 +13,7 @@ import (
12 12
 	"github.com/docker/docker/api/types/network"
13 13
 	"github.com/docker/docker/client"
14 14
 	"github.com/docker/docker/integration-cli/daemon"
15
+	"github.com/docker/docker/integration/util/request"
15 16
 	"github.com/docker/docker/pkg/stdcopy"
16 17
 	"github.com/docker/docker/pkg/system"
17 18
 	"github.com/gotestyourself/gotestyourself/fs"
... ...
@@ -51,10 +53,10 @@ func TestContainerShmNoLeak(t *testing.T) {
51 51
 	hc := container.HostConfig{
52 52
 		Mounts: []mount.Mount{
53 53
 			{
54
-				Type:        mount.TypeBind,
55
-				Source:      d.Root,
56
-				Target:      "/testdaemonroot",
57
-				BindOptions: &mount.BindOptions{Propagation: mount.PropagationRPrivate}},
54
+				Type:   mount.TypeBind,
55
+				Source: d.Root,
56
+				Target: "/testdaemonroot",
57
+			},
58 58
 		},
59 59
 	}
60 60
 	cfg.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("mount | grep testdaemonroot | grep containers | grep %s", ctr.ID)}
... ...
@@ -141,3 +143,129 @@ func TestContainerNetworkMountsNoChown(t *testing.T) {
141 141
 	require.NoError(t, err)
142 142
 	assert.Equal(t, uint32(0), statT.UID(), "bind mounted network file should not change ownership from root")
143 143
 }
144
+
145
+func TestMountDaemonRoot(t *testing.T) {
146
+	t.Parallel()
147
+
148
+	client := request.NewAPIClient(t)
149
+	ctx := context.Background()
150
+	info, err := client.Info(ctx)
151
+	if err != nil {
152
+		t.Fatal(err)
153
+	}
154
+
155
+	for _, test := range []struct {
156
+		desc        string
157
+		propagation mount.Propagation
158
+		expected    mount.Propagation
159
+	}{
160
+		{
161
+			desc:        "default",
162
+			propagation: "",
163
+			expected:    mount.PropagationRSlave,
164
+		},
165
+		{
166
+			desc:        "private",
167
+			propagation: mount.PropagationPrivate,
168
+		},
169
+		{
170
+			desc:        "rprivate",
171
+			propagation: mount.PropagationRPrivate,
172
+		},
173
+		{
174
+			desc:        "slave",
175
+			propagation: mount.PropagationSlave,
176
+		},
177
+		{
178
+			desc:        "rslave",
179
+			propagation: mount.PropagationRSlave,
180
+			expected:    mount.PropagationRSlave,
181
+		},
182
+		{
183
+			desc:        "shared",
184
+			propagation: mount.PropagationShared,
185
+		},
186
+		{
187
+			desc:        "rshared",
188
+			propagation: mount.PropagationRShared,
189
+			expected:    mount.PropagationRShared,
190
+		},
191
+	} {
192
+		t.Run(test.desc, func(t *testing.T) {
193
+			test := test
194
+			t.Parallel()
195
+
196
+			propagationSpec := fmt.Sprintf(":%s", test.propagation)
197
+			if test.propagation == "" {
198
+				propagationSpec = ""
199
+			}
200
+			bindSpecRoot := info.DockerRootDir + ":" + "/foo" + propagationSpec
201
+			bindSpecSub := filepath.Join(info.DockerRootDir, "containers") + ":/foo" + propagationSpec
202
+
203
+			for name, hc := range map[string]*container.HostConfig{
204
+				"bind root":    {Binds: []string{bindSpecRoot}},
205
+				"bind subpath": {Binds: []string{bindSpecSub}},
206
+				"mount root": {
207
+					Mounts: []mount.Mount{
208
+						{
209
+							Type:        mount.TypeBind,
210
+							Source:      info.DockerRootDir,
211
+							Target:      "/foo",
212
+							BindOptions: &mount.BindOptions{Propagation: test.propagation},
213
+						},
214
+					},
215
+				},
216
+				"mount subpath": {
217
+					Mounts: []mount.Mount{
218
+						{
219
+							Type:        mount.TypeBind,
220
+							Source:      filepath.Join(info.DockerRootDir, "containers"),
221
+							Target:      "/foo",
222
+							BindOptions: &mount.BindOptions{Propagation: test.propagation},
223
+						},
224
+					},
225
+				},
226
+			} {
227
+				t.Run(name, func(t *testing.T) {
228
+					hc := hc
229
+					t.Parallel()
230
+
231
+					c, err := client.ContainerCreate(ctx, &container.Config{
232
+						Image: "busybox",
233
+						Cmd:   []string{"true"},
234
+					}, hc, nil, "")
235
+
236
+					if err != nil {
237
+						if test.expected != "" {
238
+							t.Fatal(err)
239
+						}
240
+						// expected an error, so this is ok and should not continue
241
+						return
242
+					}
243
+					if test.expected == "" {
244
+						t.Fatal("expected create to fail")
245
+					}
246
+
247
+					defer func() {
248
+						if err := client.ContainerRemove(ctx, c.ID, types.ContainerRemoveOptions{Force: true}); err != nil {
249
+							panic(err)
250
+						}
251
+					}()
252
+
253
+					inspect, err := client.ContainerInspect(ctx, c.ID)
254
+					if err != nil {
255
+						t.Fatal(err)
256
+					}
257
+					if len(inspect.Mounts) != 1 {
258
+						t.Fatalf("unexpected number of mounts: %+v", inspect.Mounts)
259
+					}
260
+
261
+					m := inspect.Mounts[0]
262
+					if m.Propagation != test.expected {
263
+						t.Fatalf("got unexpected propagation mode, expected %q, got: %v", test.expected, m.Propagation)
264
+					}
265
+				})
266
+			}
267
+		})
268
+	}
269
+}