Browse code

Merge pull request #35829 from cpuguy83/no_private_mount_for_plugins

Perform plugin mounts in the runtime

Sebastiaan van Stijn authored on 2018/02/21 20:28:13
Showing 20 changed files
... ...
@@ -23,7 +23,7 @@ func newPluginDriver(name string, pl plugingetter.CompatPlugin, config Options)
23 23
 	home := config.Root
24 24
 	if !pl.IsV1() {
25 25
 		if p, ok := pl.(*v2.Plugin); ok {
26
-			if p.PropagatedMount != "" {
26
+			if p.PluginObj.Config.PropagatedMount != "" {
27 27
 				home = p.PluginObj.Config.PropagatedMount
28 28
 			}
29 29
 		}
... ...
@@ -4,7 +4,6 @@ import (
4 4
 	"errors"
5 5
 	"fmt"
6 6
 	"io"
7
-	"path/filepath"
8 7
 
9 8
 	"github.com/docker/docker/pkg/archive"
10 9
 	"github.com/docker/docker/pkg/containerfs"
... ...
@@ -143,7 +142,7 @@ func (d *graphDriverProxy) Get(id, mountLabel string) (containerfs.ContainerFS,
143 143
 	if ret.Err != "" {
144 144
 		err = errors.New(ret.Err)
145 145
 	}
146
-	return containerfs.NewLocalContainerFS(filepath.Join(d.p.BasePath(), ret.Dir)), err
146
+	return containerfs.NewLocalContainerFS(d.p.ScopedPath(ret.Dir)), err
147 147
 }
148 148
 
149 149
 func (d *graphDriverProxy) Put(id string) error {
... ...
@@ -3,7 +3,7 @@ package logger // import "github.com/docker/docker/daemon/logger"
3 3
 import (
4 4
 	"io"
5 5
 	"os"
6
-	"strings"
6
+	"path/filepath"
7 7
 	"sync"
8 8
 	"time"
9 9
 
... ...
@@ -19,7 +19,6 @@ type pluginAdapter struct {
19 19
 	driverName   string
20 20
 	id           string
21 21
 	plugin       logPlugin
22
-	basePath     string
23 22
 	fifoPath     string
24 23
 	capabilities Capability
25 24
 	logInfo      Info
... ...
@@ -58,7 +57,7 @@ func (a *pluginAdapter) Close() error {
58 58
 	a.mu.Lock()
59 59
 	defer a.mu.Unlock()
60 60
 
61
-	if err := a.plugin.StopLogging(strings.TrimPrefix(a.fifoPath, a.basePath)); err != nil {
61
+	if err := a.plugin.StopLogging(filepath.Join("/", "run", "docker", "logging", a.id)); err != nil {
62 62
 		return err
63 63
 	}
64 64
 
... ...
@@ -5,7 +5,6 @@ import (
5 5
 	"io"
6 6
 	"os"
7 7
 	"path/filepath"
8
-	"strings"
9 8
 
10 9
 	"github.com/docker/docker/api/types/plugins/logdriver"
11 10
 	getter "github.com/docker/docker/pkg/plugingetter"
... ...
@@ -39,18 +38,20 @@ func getPlugin(name string, mode int) (Creator, error) {
39 39
 	}
40 40
 
41 41
 	d := &logPluginProxy{p.Client()}
42
-	return makePluginCreator(name, d, p.BasePath()), nil
42
+	return makePluginCreator(name, d, p.ScopedPath), nil
43 43
 }
44 44
 
45
-func makePluginCreator(name string, l *logPluginProxy, basePath string) Creator {
45
+func makePluginCreator(name string, l *logPluginProxy, scopePath func(s string) string) Creator {
46 46
 	return func(logCtx Info) (logger Logger, err error) {
47 47
 		defer func() {
48 48
 			if err != nil {
49 49
 				pluginGetter.Get(name, extName, getter.Release)
50 50
 			}
51 51
 		}()
52
-		root := filepath.Join(basePath, "run", "docker", "logging")
53
-		if err := os.MkdirAll(root, 0700); err != nil {
52
+
53
+		unscopedPath := filepath.Join("/", "run", "docker", "logging")
54
+		logRoot := scopePath(unscopedPath)
55
+		if err := os.MkdirAll(logRoot, 0700); err != nil {
54 56
 			return nil, err
55 57
 		}
56 58
 
... ...
@@ -59,8 +60,7 @@ func makePluginCreator(name string, l *logPluginProxy, basePath string) Creator
59 59
 			driverName: name,
60 60
 			id:         id,
61 61
 			plugin:     l,
62
-			basePath:   basePath,
63
-			fifoPath:   filepath.Join(root, id),
62
+			fifoPath:   filepath.Join(logRoot, id),
64 63
 			logInfo:    logCtx,
65 64
 		}
66 65
 
... ...
@@ -77,7 +77,7 @@ func makePluginCreator(name string, l *logPluginProxy, basePath string) Creator
77 77
 		a.stream = stream
78 78
 		a.enc = logdriver.NewLogEntryEncoder(a.stream)
79 79
 
80
-		if err := l.StartLogging(strings.TrimPrefix(a.fifoPath, basePath), logCtx); err != nil {
80
+		if err := l.StartLogging(filepath.Join(unscopedPath, id), logCtx); err != nil {
81 81
 			return nil, errors.Wrapf(err, "error creating logger")
82 82
 		}
83 83
 
... ...
@@ -1,10 +1,8 @@
1 1
 package daemon // import "github.com/docker/docker/daemon"
2 2
 
3 3
 import (
4
-	"path/filepath"
5 4
 	"sync"
6 5
 
7
-	"github.com/docker/docker/pkg/mount"
8 6
 	"github.com/docker/docker/pkg/plugingetter"
9 7
 	metrics "github.com/docker/go-metrics"
10 8
 	"github.com/pkg/errors"
... ...
@@ -132,18 +130,6 @@ func (d *Daemon) cleanupMetricsPlugins() {
132 132
 	}
133 133
 }
134 134
 
135
-type metricsPlugin struct {
136
-	plugingetter.CompatPlugin
137
-}
138
-
139
-func (p metricsPlugin) sock() string {
140
-	return "metrics.sock"
141
-}
142
-
143
-func (p metricsPlugin) sockBase() string {
144
-	return filepath.Join(p.BasePath(), "run", "docker")
145
-}
146
-
147 135
 func pluginStartMetricsCollection(p plugingetter.CompatPlugin) error {
148 136
 	type metricsPluginResponse struct {
149 137
 		Err string
... ...
@@ -162,12 +148,4 @@ func pluginStopMetricsCollection(p plugingetter.CompatPlugin) {
162 162
 	if err := p.Client().Call(metricsPluginType+".StopMetrics", nil, nil); err != nil {
163 163
 		logrus.WithError(err).WithField("name", p.Name()).Error("error stopping metrics collector")
164 164
 	}
165
-
166
-	mp := metricsPlugin{p}
167
-	sockPath := filepath.Join(mp.sockBase(), mp.sock())
168
-	if err := mount.Unmount(sockPath); err != nil {
169
-		if mounted, _ := mount.Mounted(sockPath); mounted {
170
-			logrus.WithError(err).WithField("name", p.Name()).WithField("socket", sockPath).Error("error unmounting metrics socket for plugin")
171
-		}
172
-	}
173 165
 }
... ...
@@ -5,13 +5,13 @@ package daemon // import "github.com/docker/docker/daemon"
5 5
 import (
6 6
 	"net"
7 7
 	"net/http"
8
-	"os"
9 8
 	"path/filepath"
10 9
 
11
-	"github.com/docker/docker/pkg/mount"
12 10
 	"github.com/docker/docker/pkg/plugingetter"
13 11
 	"github.com/docker/docker/pkg/plugins"
12
+	"github.com/docker/docker/plugin"
14 13
 	metrics "github.com/docker/go-metrics"
14
+	specs "github.com/opencontainers/runtime-spec/specs-go"
15 15
 	"github.com/pkg/errors"
16 16
 	"github.com/sirupsen/logrus"
17 17
 	"golang.org/x/sys/unix"
... ...
@@ -34,52 +34,22 @@ func (daemon *Daemon) listenMetricsSock() (string, error) {
34 34
 	return path, nil
35 35
 }
36 36
 
37
-func registerMetricsPluginCallback(getter plugingetter.PluginGetter, sockPath string) {
38
-	getter.Handle(metricsPluginType, func(name string, client *plugins.Client) {
37
+func registerMetricsPluginCallback(store *plugin.Store, sockPath string) {
38
+	store.RegisterRuntimeOpt(metricsPluginType, func(s *specs.Spec) {
39
+		f := plugin.WithSpecMounts([]specs.Mount{
40
+			{Type: "bind", Source: sockPath, Destination: "/run/docker/metrics.sock", Options: []string{"bind", "ro"}},
41
+		})
42
+		f(s)
43
+	})
44
+	store.Handle(metricsPluginType, func(name string, client *plugins.Client) {
39 45
 		// Use lookup since nothing in the system can really reference it, no need
40 46
 		// to protect against removal
41
-		p, err := getter.Get(name, metricsPluginType, plugingetter.Lookup)
47
+		p, err := store.Get(name, metricsPluginType, plugingetter.Lookup)
42 48
 		if err != nil {
43 49
 			return
44 50
 		}
45 51
 
46
-		mp := metricsPlugin{p}
47
-		sockBase := mp.sockBase()
48
-		if err := os.MkdirAll(sockBase, 0755); err != nil {
49
-			logrus.WithError(err).WithField("name", name).WithField("path", sockBase).Error("error creating metrics plugin base path")
50
-			return
51
-		}
52
-
53
-		defer func() {
54
-			if err != nil {
55
-				os.RemoveAll(sockBase)
56
-			}
57
-		}()
58
-
59
-		pluginSockPath := filepath.Join(sockBase, mp.sock())
60
-		_, err = os.Stat(pluginSockPath)
61
-		if err == nil {
62
-			mount.Unmount(pluginSockPath)
63
-		} else {
64
-			logrus.WithField("path", pluginSockPath).Debugf("creating plugin socket")
65
-			f, err := os.OpenFile(pluginSockPath, os.O_CREATE, 0600)
66
-			if err != nil {
67
-				return
68
-			}
69
-			f.Close()
70
-		}
71
-
72
-		if err := mount.Mount(sockPath, pluginSockPath, "none", "bind,ro"); err != nil {
73
-			logrus.WithError(err).WithField("name", name).Error("could not mount metrics socket to plugin")
74
-			return
75
-		}
76
-
77 52
 		if err := pluginStartMetricsCollection(p); err != nil {
78
-			if err := mount.Unmount(pluginSockPath); err != nil {
79
-				if mounted, _ := mount.Mounted(pluginSockPath); mounted {
80
-					logrus.WithError(err).WithField("sock_path", pluginSockPath).Error("error unmounting metrics socket from plugin during cleanup")
81
-				}
82
-			}
83 53
 			logrus.WithError(err).WithField("name", name).Error("error while initializing metrics plugin")
84 54
 		}
85 55
 	})
... ...
@@ -3,8 +3,6 @@
3 3
 package main
4 4
 
5 5
 import (
6
-	"os"
7
-	"path/filepath"
8 6
 	"strings"
9 7
 
10 8
 	"github.com/docker/docker/integration-cli/checker"
... ...
@@ -199,12 +197,6 @@ func (s *DockerDaemonSuite) TestVolumePlugin(c *check.C) {
199 199
 	if err != nil {
200 200
 		c.Fatalf("Could not install plugin: %v %s", err, out)
201 201
 	}
202
-	pluginID, err := s.d.Cmd("plugin", "inspect", "-f", "{{.Id}}", pName)
203
-	pluginID = strings.TrimSpace(pluginID)
204
-	if err != nil {
205
-		c.Fatalf("Could not retrieve plugin ID: %v %s", err, pluginID)
206
-	}
207
-	mountpointPrefix := filepath.Join(s.d.RootDir(), "plugins", pluginID, "rootfs")
208 202
 	defer func() {
209 203
 		if out, err := s.d.Cmd("plugin", "disable", pName); err != nil {
210 204
 			c.Fatalf("Could not disable plugin: %v %s", err, out)
... ...
@@ -213,11 +205,6 @@ func (s *DockerDaemonSuite) TestVolumePlugin(c *check.C) {
213 213
 		if out, err := s.d.Cmd("plugin", "remove", pName); err != nil {
214 214
 			c.Fatalf("Could not remove plugin: %v %s", err, out)
215 215
 		}
216
-
217
-		exists, err := existsMountpointWithPrefix(mountpointPrefix)
218
-		c.Assert(err, checker.IsNil)
219
-		c.Assert(exists, checker.Equals, false)
220
-
221 216
 	}()
222 217
 
223 218
 	out, err = s.d.Cmd("volume", "create", "-d", pName, volName)
... ...
@@ -237,21 +224,11 @@ func (s *DockerDaemonSuite) TestVolumePlugin(c *check.C) {
237 237
 	c.Assert(out, checker.Contains, volName)
238 238
 	c.Assert(out, checker.Contains, pName)
239 239
 
240
-	mountPoint, err := s.d.Cmd("volume", "inspect", volName, "--format", "{{.Mountpoint}}")
241
-	if err != nil {
242
-		c.Fatalf("Could not inspect volume: %v %s", err, mountPoint)
243
-	}
244
-	mountPoint = strings.TrimSpace(mountPoint)
245
-
246 240
 	out, err = s.d.Cmd("run", "--rm", "-v", volName+":"+destDir, "busybox", "touch", destDir+destFile)
247 241
 	c.Assert(err, checker.IsNil, check.Commentf(out))
248
-	path := filepath.Join(s.d.RootDir(), "plugins", pluginID, "rootfs", mountPoint, destFile)
249
-	_, err = os.Lstat(path)
250
-	c.Assert(err, checker.IsNil)
251 242
 
252
-	exists, err := existsMountpointWithPrefix(mountpointPrefix)
253
-	c.Assert(err, checker.IsNil)
254
-	c.Assert(exists, checker.Equals, true)
243
+	out, err = s.d.Cmd("run", "--rm", "-v", volName+":"+destDir, "busybox", "ls", destDir+destFile)
244
+	c.Assert(err, checker.IsNil, check.Commentf(out))
255 245
 }
256 246
 
257 247
 func (s *DockerDaemonSuite) TestGraphdriverPlugin(c *check.C) {
... ...
@@ -17,7 +17,7 @@ const (
17 17
 type CompatPlugin interface {
18 18
 	Client() *plugins.Client
19 19
 	Name() string
20
-	BasePath() string
20
+	ScopedPath(string) string
21 21
 	IsV1() bool
22 22
 }
23 23
 
... ...
@@ -2,8 +2,8 @@
2 2
 
3 3
 package plugins // import "github.com/docker/docker/pkg/plugins"
4 4
 
5
-// BasePath returns the path to which all paths returned by the plugin are relative to.
6
-// For v1 plugins, this always returns the host's root directory.
7
-func (p *Plugin) BasePath() string {
8
-	return "/"
5
+// ScopedPath returns the path scoped to the plugin's rootfs.
6
+// For v1 plugins, this always returns the path unchanged as v1 plugins run directly on the host.
7
+func (p *Plugin) ScopedPath(s string) string {
8
+	return s
9 9
 }
... ...
@@ -1,8 +1,7 @@
1 1
 package plugins // import "github.com/docker/docker/pkg/plugins"
2 2
 
3
-// BasePath returns the path to which all paths returned by the plugin are relative to.
4
-// For Windows v1 plugins, this returns an empty string, since the plugin is already aware
5
-// of the absolute path of the mount.
6
-func (p *Plugin) BasePath() string {
7
-	return ""
3
+// ScopedPath returns the path scoped to the plugin's rootfs.
4
+// For v1 plugins, this always returns the path unchanged as v1 plugins run directly on the host.
5
+func (p *Plugin) ScopedPath(s string) string {
6
+	return s
8 7
 }
... ...
@@ -5,12 +5,14 @@ import (
5 5
 
6 6
 	"github.com/docker/docker/pkg/plugins"
7 7
 	"github.com/docker/docker/plugin/v2"
8
+	specs "github.com/opencontainers/runtime-spec/specs-go"
8 9
 )
9 10
 
10 11
 // Store manages the plugin inventory in memory and on-disk
11 12
 type Store struct {
12 13
 	sync.RWMutex
13
-	plugins map[string]*v2.Plugin
14
+	plugins  map[string]*v2.Plugin
15
+	specOpts map[string][]SpecOpt
14 16
 	/* handlers are necessary for transition path of legacy plugins
15 17
 	 * to the new model. Legacy plugins use Handle() for registering an
16 18
 	 * activation callback.*/
... ...
@@ -21,10 +23,14 @@ type Store struct {
21 21
 func NewStore() *Store {
22 22
 	return &Store{
23 23
 		plugins:  make(map[string]*v2.Plugin),
24
+		specOpts: make(map[string][]SpecOpt),
24 25
 		handlers: make(map[string][]func(string, *plugins.Client)),
25 26
 	}
26 27
 }
27 28
 
29
+// SpecOpt is used for subsystems that need to modify the runtime spec of a plugin
30
+type SpecOpt func(*specs.Spec)
31
+
28 32
 // CreateOpt is used to configure specific plugin details when created
29 33
 type CreateOpt func(p *v2.Plugin)
30 34
 
... ...
@@ -35,3 +41,10 @@ func WithSwarmService(id string) CreateOpt {
35 35
 		p.SwarmServiceID = id
36 36
 	}
37 37
 }
38
+
39
+// WithSpecMounts is a SpecOpt which appends the provided mounts to the runtime spec
40
+func WithSpecMounts(mounts []specs.Mount) SpecOpt {
41
+	return func(s *specs.Spec) {
42
+		s.Mounts = append(s.Mounts, mounts...)
43
+	}
44
+}
... ...
@@ -112,11 +112,6 @@ func NewManager(config ManagerConfig) (*Manager, error) {
112 112
 			return nil, errors.Wrapf(err, "failed to mkdir %v", dirName)
113 113
 		}
114 114
 	}
115
-
116
-	if err := setupRoot(manager.config.Root); err != nil {
117
-		return nil, err
118
-	}
119
-
120 115
 	var err error
121 116
 	manager.executor, err = config.CreateExecutor(manager)
122 117
 	if err != nil {
... ...
@@ -151,16 +146,6 @@ func (pm *Manager) HandleExitEvent(id string) error {
151 151
 
152 152
 	os.RemoveAll(filepath.Join(pm.config.ExecRoot, id))
153 153
 
154
-	if p.PropagatedMount != "" {
155
-		if err := mount.Unmount(p.PropagatedMount); err != nil {
156
-			logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err)
157
-		}
158
-		propRoot := filepath.Join(filepath.Dir(p.Rootfs), "propagated-mount")
159
-		if err := mount.Unmount(propRoot); err != nil {
160
-			logrus.Warn("Could not unmount %s: %v", propRoot, err)
161
-		}
162
-	}
163
-
164 154
 	pm.mu.RLock()
165 155
 	c := pm.cMap[p]
166 156
 	if c.exitChan != nil {
... ...
@@ -171,6 +156,10 @@ func (pm *Manager) HandleExitEvent(id string) error {
171 171
 
172 172
 	if restart {
173 173
 		pm.enable(p, c, true)
174
+	} else {
175
+		if err := mount.RecursiveUnmount(filepath.Join(pm.config.Root, id)); err != nil {
176
+			return errors.Wrap(err, "error cleaning up plugin mounts")
177
+		}
174 178
 	}
175 179
 	return nil
176 180
 }
... ...
@@ -239,28 +228,17 @@ func (pm *Manager) reload() error { // todo: restore
239 239
 						// check if we need to migrate an older propagated mount from before
240 240
 						// these mounts were stored outside the plugin rootfs
241 241
 						if _, err := os.Stat(propRoot); os.IsNotExist(err) {
242
-							if _, err := os.Stat(p.PropagatedMount); err == nil {
243
-								// make sure nothing is mounted here
244
-								// don't care about errors
245
-								mount.Unmount(p.PropagatedMount)
246
-								if err := os.Rename(p.PropagatedMount, propRoot); err != nil {
242
+							rootfsProp := filepath.Join(p.Rootfs, p.PluginObj.Config.PropagatedMount)
243
+							if _, err := os.Stat(rootfsProp); err == nil {
244
+								if err := os.Rename(rootfsProp, propRoot); err != nil {
247 245
 									logrus.WithError(err).WithField("dir", propRoot).Error("error migrating propagated mount storage")
248 246
 								}
249
-								if err := os.MkdirAll(p.PropagatedMount, 0755); err != nil {
250
-									logrus.WithError(err).WithField("dir", p.PropagatedMount).Error("error migrating propagated mount storage")
251
-								}
252 247
 							}
253 248
 						}
254 249
 
255 250
 						if err := os.MkdirAll(propRoot, 0755); err != nil {
256 251
 							logrus.Errorf("failed to create PropagatedMount directory at %s: %v", propRoot, err)
257 252
 						}
258
-						// TODO: sanitize PropagatedMount and prevent breakout
259
-						p.PropagatedMount = filepath.Join(p.Rootfs, p.PluginObj.Config.PropagatedMount)
260
-						if err := os.MkdirAll(p.PropagatedMount, 0755); err != nil {
261
-							logrus.Errorf("failed to create PropagatedMount directory at %s: %v", p.PropagatedMount, err)
262
-							return
263
-						}
264 253
 					}
265 254
 				}
266 255
 			}
... ...
@@ -22,7 +22,7 @@ import (
22 22
 	"golang.org/x/sys/unix"
23 23
 )
24 24
 
25
-func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) (err error) {
25
+func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error {
26 26
 	p.Rootfs = filepath.Join(pm.config.Root, p.PluginObj.ID, "rootfs")
27 27
 	if p.IsEnabled() && !force {
28 28
 		return errors.Wrap(enabledError(p.Name()), "plugin already enabled")
... ...
@@ -40,20 +40,16 @@ func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) (err error) {
40 40
 	pm.mu.Unlock()
41 41
 
42 42
 	var propRoot string
43
-	if p.PropagatedMount != "" {
43
+	if p.PluginObj.Config.PropagatedMount != "" {
44 44
 		propRoot = filepath.Join(filepath.Dir(p.Rootfs), "propagated-mount")
45 45
 
46
-		if err = os.MkdirAll(propRoot, 0755); err != nil {
46
+		if err := os.MkdirAll(propRoot, 0755); err != nil {
47 47
 			logrus.Errorf("failed to create PropagatedMount directory at %s: %v", propRoot, err)
48 48
 		}
49 49
 
50
-		if err = mount.MakeRShared(propRoot); err != nil {
50
+		if err := mount.MakeRShared(propRoot); err != nil {
51 51
 			return errors.Wrap(err, "error setting up propagated mount dir")
52 52
 		}
53
-
54
-		if err = mount.Mount(propRoot, p.PropagatedMount, "none", "rbind"); err != nil {
55
-			return errors.Wrap(err, "error creating mount for propagated mount")
56
-		}
57 53
 	}
58 54
 
59 55
 	rootFS := containerfs.NewLocalContainerFS(filepath.Join(pm.config.Root, p.PluginObj.ID, rootFSFileName))
... ...
@@ -63,16 +59,12 @@ func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) (err error) {
63 63
 
64 64
 	stdout, stderr := makeLoggerStreams(p.GetID())
65 65
 	if err := pm.executor.Create(p.GetID(), *spec, stdout, stderr); err != nil {
66
-		if p.PropagatedMount != "" {
67
-			if err := mount.Unmount(p.PropagatedMount); err != nil {
68
-				logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err)
69
-			}
66
+		if p.PluginObj.Config.PropagatedMount != "" {
70 67
 			if err := mount.Unmount(propRoot); err != nil {
71 68
 				logrus.Warnf("Could not unmount %s: %v", propRoot, err)
72 69
 			}
73 70
 		}
74 71
 	}
75
-
76 72
 	return pm.pluginPostStart(p, c)
77 73
 }
78 74
 
... ...
@@ -167,13 +159,6 @@ func shutdownPlugin(p *v2.Plugin, c *controller, executor Executor) {
167 167
 	}
168 168
 }
169 169
 
170
-func setupRoot(root string) error {
171
-	if err := mount.MakePrivate(root); err != nil {
172
-		return errors.Wrap(err, "error setting plugin manager root to private")
173
-	}
174
-	return nil
175
-}
176
-
177 170
 func (pm *Manager) disable(p *v2.Plugin, c *controller) error {
178 171
 	if !p.IsEnabled() {
179 172
 		return errors.Wrap(errDisabled(p.Name()), "plugin is already disabled")
... ...
@@ -202,7 +187,9 @@ func (pm *Manager) Shutdown() {
202 202
 			shutdownPlugin(p, c, pm.executor)
203 203
 		}
204 204
 	}
205
-	mount.Unmount(pm.config.Root)
205
+	if err := mount.RecursiveUnmount(pm.config.Root); err != nil {
206
+		logrus.WithError(err).Warn("error cleaning up plugin mounts")
207
+	}
206 208
 }
207 209
 
208 210
 func (pm *Manager) upgradePlugin(p *v2.Plugin, configDigest digest.Digest, blobsums []digest.Digest, tmpRootFSDir string, privileges *types.PluginPrivileges) (err error) {
... ...
@@ -26,5 +26,3 @@ func (pm *Manager) restore(p *v2.Plugin) error {
26 26
 // Shutdown plugins
27 27
 func (pm *Manager) Shutdown() {
28 28
 }
29
-
30
-func setupRoot(root string) error { return nil }
... ...
@@ -9,6 +9,7 @@ import (
9 9
 	"github.com/docker/docker/pkg/plugingetter"
10 10
 	"github.com/docker/docker/pkg/plugins"
11 11
 	"github.com/docker/docker/plugin/v2"
12
+	specs "github.com/opencontainers/runtime-spec/specs-go"
12 13
 	"github.com/pkg/errors"
13 14
 	"github.com/sirupsen/logrus"
14 15
 )
... ...
@@ -64,6 +65,10 @@ func (ps *Store) GetAll() map[string]*v2.Plugin {
64 64
 func (ps *Store) SetAll(plugins map[string]*v2.Plugin) {
65 65
 	ps.Lock()
66 66
 	defer ps.Unlock()
67
+
68
+	for _, p := range plugins {
69
+		ps.setSpecOpts(p)
70
+	}
67 71
 	ps.plugins = plugins
68 72
 }
69 73
 
... ...
@@ -90,6 +95,22 @@ func (ps *Store) SetState(p *v2.Plugin, state bool) {
90 90
 	p.PluginObj.Enabled = state
91 91
 }
92 92
 
93
+func (ps *Store) setSpecOpts(p *v2.Plugin) {
94
+	var specOpts []SpecOpt
95
+	for _, typ := range p.GetTypes() {
96
+		opts, ok := ps.specOpts[typ.String()]
97
+		if ok {
98
+			specOpts = append(specOpts, opts...)
99
+		}
100
+	}
101
+
102
+	p.SetSpecOptModifier(func(s *specs.Spec) {
103
+		for _, o := range specOpts {
104
+			o(s)
105
+		}
106
+	})
107
+}
108
+
93 109
 // Add adds a plugin to memory and plugindb.
94 110
 // An error will be returned if there is a collision.
95 111
 func (ps *Store) Add(p *v2.Plugin) error {
... ...
@@ -99,6 +120,9 @@ func (ps *Store) Add(p *v2.Plugin) error {
99 99
 	if v, exist := ps.plugins[p.GetID()]; exist {
100 100
 		return fmt.Errorf("plugin %q has the same ID %s as %q", p.Name(), p.GetID(), v.Name())
101 101
 	}
102
+
103
+	ps.setSpecOpts(p)
104
+
102 105
 	ps.plugins[p.GetID()] = p
103 106
 	return nil
104 107
 }
... ...
@@ -182,20 +206,24 @@ func (ps *Store) GetAllByCap(capability string) ([]plugingetter.CompatPlugin, er
182 182
 	return result, nil
183 183
 }
184 184
 
185
+func pluginType(cap string) string {
186
+	return fmt.Sprintf("docker.%s/%s", strings.ToLower(cap), defaultAPIVersion)
187
+}
188
+
185 189
 // Handle sets a callback for a given capability. It is only used by network
186 190
 // and ipam drivers during plugin registration. The callback registers the
187 191
 // driver with the subsystem (network, ipam).
188 192
 func (ps *Store) Handle(capability string, callback func(string, *plugins.Client)) {
189
-	pluginType := fmt.Sprintf("docker.%s/%s", strings.ToLower(capability), defaultAPIVersion)
193
+	typ := pluginType(capability)
190 194
 
191 195
 	// Register callback with new plugin model.
192 196
 	ps.Lock()
193
-	handlers, ok := ps.handlers[pluginType]
197
+	handlers, ok := ps.handlers[typ]
194 198
 	if !ok {
195 199
 		handlers = []func(string, *plugins.Client){}
196 200
 	}
197 201
 	handlers = append(handlers, callback)
198
-	ps.handlers[pluginType] = handlers
202
+	ps.handlers[typ] = handlers
199 203
 	ps.Unlock()
200 204
 
201 205
 	// Register callback with legacy plugin model.
... ...
@@ -204,6 +232,15 @@ func (ps *Store) Handle(capability string, callback func(string, *plugins.Client
204 204
 	}
205 205
 }
206 206
 
207
+// RegisterRuntimeOpt stores a list of SpecOpts for the provided capability.
208
+// These options are applied to the runtime spec before a plugin is started for the specified capability.
209
+func (ps *Store) RegisterRuntimeOpt(cap string, opts ...SpecOpt) {
210
+	ps.Lock()
211
+	defer ps.Unlock()
212
+	typ := pluginType(cap)
213
+	ps.specOpts[typ] = append(ps.specOpts[typ], opts...)
214
+}
215
+
207 216
 // CallHandler calls the registered callback. It is invoked during plugin enable.
208 217
 func (ps *Store) CallHandler(p *v2.Plugin) {
209 218
 	for _, typ := range p.GetTypes() {
... ...
@@ -2,6 +2,7 @@ package v2 // import "github.com/docker/docker/plugin/v2"
2 2
 
3 3
 import (
4 4
 	"fmt"
5
+	"path/filepath"
5 6
 	"strings"
6 7
 	"sync"
7 8
 
... ...
@@ -9,20 +10,22 @@ import (
9 9
 	"github.com/docker/docker/pkg/plugingetter"
10 10
 	"github.com/docker/docker/pkg/plugins"
11 11
 	"github.com/opencontainers/go-digest"
12
+	specs "github.com/opencontainers/runtime-spec/specs-go"
12 13
 )
13 14
 
14 15
 // Plugin represents an individual plugin.
15 16
 type Plugin struct {
16
-	mu              sync.RWMutex
17
-	PluginObj       types.Plugin `json:"plugin"` // todo: embed struct
18
-	pClient         *plugins.Client
19
-	refCount        int
20
-	PropagatedMount string // TODO: make private
21
-	Rootfs          string // TODO: make private
17
+	mu        sync.RWMutex
18
+	PluginObj types.Plugin `json:"plugin"` // todo: embed struct
19
+	pClient   *plugins.Client
20
+	refCount  int
21
+	Rootfs    string // TODO: make private
22 22
 
23 23
 	Config   digest.Digest
24 24
 	Blobsums []digest.Digest
25 25
 
26
+	modifyRuntimeSpec func(*specs.Spec)
27
+
26 28
 	SwarmServiceID string
27 29
 }
28 30
 
... ...
@@ -37,10 +40,13 @@ func (e ErrInadequateCapability) Error() string {
37 37
 	return fmt.Sprintf("plugin does not provide %q capability", e.cap)
38 38
 }
39 39
 
40
-// BasePath returns the path to which all paths returned by the plugin are relative to.
41
-// For Plugin objects this returns the host path of the plugin container's rootfs.
42
-func (p *Plugin) BasePath() string {
43
-	return p.Rootfs
40
+// ScopedPath returns the path scoped to the plugin rootfs
41
+func (p *Plugin) ScopedPath(s string) string {
42
+	if p.PluginObj.Config.PropagatedMount != "" && strings.HasPrefix(s, p.PluginObj.Config.PropagatedMount) {
43
+		// re-scope to the propagated mount path on the host
44
+		return filepath.Join(filepath.Dir(p.Rootfs), "propagated-mount", strings.TrimPrefix(s, p.PluginObj.Config.PropagatedMount))
45
+	}
46
+	return filepath.Join(p.Rootfs, s)
44 47
 }
45 48
 
46 49
 // Client returns the plugin client.
... ...
@@ -250,3 +256,11 @@ func (p *Plugin) Acquire() {
250 250
 func (p *Plugin) Release() {
251 251
 	p.AddRefCount(plugingetter.Release)
252 252
 }
253
+
254
+// SetSpecOptModifier sets the function to use to modify the the generated
255
+// runtime spec.
256
+func (p *Plugin) SetSpecOptModifier(f func(*specs.Spec)) {
257
+	p.mu.Lock()
258
+	p.modifyRuntimeSpec = f
259
+	p.mu.Unlock()
260
+}
... ...
@@ -4,6 +4,7 @@ import (
4 4
 	"os"
5 5
 	"path/filepath"
6 6
 	"runtime"
7
+	"sort"
7 8
 	"strings"
8 9
 
9 10
 	"github.com/docker/docker/api/types"
... ...
@@ -16,6 +17,7 @@ import (
16 16
 // InitSpec creates an OCI spec from the plugin's config.
17 17
 func (p *Plugin) InitSpec(execRoot string) (*specs.Spec, error) {
18 18
 	s := oci.DefaultSpec()
19
+
19 20
 	s.Root = &specs.Root{
20 21
 		Path:     p.Rootfs,
21 22
 		Readonly: false, // TODO: all plugins should be readonly? settable in config?
... ...
@@ -31,6 +33,17 @@ func (p *Plugin) InitSpec(execRoot string) (*specs.Spec, error) {
31 31
 		return nil, errors.WithStack(err)
32 32
 	}
33 33
 
34
+	if p.PluginObj.Config.PropagatedMount != "" {
35
+		pRoot := filepath.Join(filepath.Dir(p.Rootfs), "propagated-mount")
36
+		s.Mounts = append(s.Mounts, specs.Mount{
37
+			Source:      pRoot,
38
+			Destination: p.PluginObj.Config.PropagatedMount,
39
+			Type:        "bind",
40
+			Options:     []string{"rbind", "rw", "rshared"},
41
+		})
42
+		s.Linux.RootfsPropagation = "rshared"
43
+	}
44
+
34 45
 	mounts := append(p.PluginObj.Config.Mounts, types.PluginMount{
35 46
 		Source:      &execRoot,
36 47
 		Destination: defaultPluginRuntimeDestination,
... ...
@@ -88,11 +101,6 @@ func (p *Plugin) InitSpec(execRoot string) (*specs.Spec, error) {
88 88
 		}
89 89
 	}
90 90
 
91
-	if p.PluginObj.Config.PropagatedMount != "" {
92
-		p.PropagatedMount = filepath.Join(p.Rootfs, p.PluginObj.Config.PropagatedMount)
93
-		s.Linux.RootfsPropagation = "rshared"
94
-	}
95
-
96 91
 	if p.PluginObj.Config.Linux.AllowAllDevices {
97 92
 		s.Linux.Resources.Devices = []specs.LinuxDeviceCgroup{{Allow: true, Access: "rwm"}}
98 93
 	}
... ...
@@ -126,5 +134,13 @@ func (p *Plugin) InitSpec(execRoot string) (*specs.Spec, error) {
126 126
 	caps.Inheritable = append(caps.Inheritable, p.PluginObj.Config.Linux.Capabilities...)
127 127
 	caps.Effective = append(caps.Effective, p.PluginObj.Config.Linux.Capabilities...)
128 128
 
129
+	if p.modifyRuntimeSpec != nil {
130
+		p.modifyRuntimeSpec(&s)
131
+	}
132
+
133
+	sort.Slice(s.Mounts, func(i, j int) bool {
134
+		return s.Mounts[i].Destination < s.Mounts[j].Destination
135
+	})
136
+
129 137
 	return &s, nil
130 138
 }
... ...
@@ -2,7 +2,6 @@ package volumedrivers // import "github.com/docker/docker/volume/drivers"
2 2
 
3 3
 import (
4 4
 	"errors"
5
-	"path/filepath"
6 5
 	"strings"
7 6
 	"time"
8 7
 
... ...
@@ -16,7 +15,7 @@ var (
16 16
 
17 17
 type volumeDriverAdapter struct {
18 18
 	name         string
19
-	baseHostPath string
19
+	scopePath    func(s string) string
20 20
 	capabilities *volume.Capability
21 21
 	proxy        *volumeDriverProxy
22 22
 }
... ...
@@ -30,10 +29,10 @@ func (a *volumeDriverAdapter) Create(name string, opts map[string]string) (volum
30 30
 		return nil, err
31 31
 	}
32 32
 	return &volumeAdapter{
33
-		proxy:        a.proxy,
34
-		name:         name,
35
-		driverName:   a.name,
36
-		baseHostPath: a.baseHostPath,
33
+		proxy:      a.proxy,
34
+		name:       name,
35
+		driverName: a.name,
36
+		scopePath:  a.scopePath,
37 37
 	}, nil
38 38
 }
39 39
 
... ...
@@ -41,13 +40,6 @@ func (a *volumeDriverAdapter) Remove(v volume.Volume) error {
41 41
 	return a.proxy.Remove(v.Name())
42 42
 }
43 43
 
44
-func hostPath(baseHostPath, path string) string {
45
-	if baseHostPath != "" {
46
-		path = filepath.Join(baseHostPath, path)
47
-	}
48
-	return path
49
-}
50
-
51 44
 func (a *volumeDriverAdapter) List() ([]volume.Volume, error) {
52 45
 	ls, err := a.proxy.List()
53 46
 	if err != nil {
... ...
@@ -57,11 +49,11 @@ func (a *volumeDriverAdapter) List() ([]volume.Volume, error) {
57 57
 	var out []volume.Volume
58 58
 	for _, vp := range ls {
59 59
 		out = append(out, &volumeAdapter{
60
-			proxy:        a.proxy,
61
-			name:         vp.Name,
62
-			baseHostPath: a.baseHostPath,
63
-			driverName:   a.name,
64
-			eMount:       hostPath(a.baseHostPath, vp.Mountpoint),
60
+			proxy:      a.proxy,
61
+			name:       vp.Name,
62
+			scopePath:  a.scopePath,
63
+			driverName: a.name,
64
+			eMount:     a.scopePath(vp.Mountpoint),
65 65
 		})
66 66
 	}
67 67
 	return out, nil
... ...
@@ -79,13 +71,13 @@ func (a *volumeDriverAdapter) Get(name string) (volume.Volume, error) {
79 79
 	}
80 80
 
81 81
 	return &volumeAdapter{
82
-		proxy:        a.proxy,
83
-		name:         v.Name,
84
-		driverName:   a.Name(),
85
-		eMount:       v.Mountpoint,
86
-		createdAt:    v.CreatedAt,
87
-		status:       v.Status,
88
-		baseHostPath: a.baseHostPath,
82
+		proxy:      a.proxy,
83
+		name:       v.Name,
84
+		driverName: a.Name(),
85
+		eMount:     v.Mountpoint,
86
+		createdAt:  v.CreatedAt,
87
+		status:     v.Status,
88
+		scopePath:  a.scopePath,
89 89
 	}, nil
90 90
 }
91 91
 
... ...
@@ -122,13 +114,13 @@ func (a *volumeDriverAdapter) getCapabilities() volume.Capability {
122 122
 }
123 123
 
124 124
 type volumeAdapter struct {
125
-	proxy        *volumeDriverProxy
126
-	name         string
127
-	baseHostPath string
128
-	driverName   string
129
-	eMount       string    // ephemeral host volume path
130
-	createdAt    time.Time // time the directory was created
131
-	status       map[string]interface{}
125
+	proxy      *volumeDriverProxy
126
+	name       string
127
+	scopePath  func(string) string
128
+	driverName string
129
+	eMount     string    // ephemeral host volume path
130
+	createdAt  time.Time // time the directory was created
131
+	status     map[string]interface{}
132 132
 }
133 133
 
134 134
 type proxyVolume struct {
... ...
@@ -149,7 +141,7 @@ func (a *volumeAdapter) DriverName() string {
149 149
 func (a *volumeAdapter) Path() string {
150 150
 	if len(a.eMount) == 0 {
151 151
 		mountpoint, _ := a.proxy.Path(a.name)
152
-		a.eMount = hostPath(a.baseHostPath, mountpoint)
152
+		a.eMount = a.scopePath(mountpoint)
153 153
 	}
154 154
 	return a.eMount
155 155
 }
... ...
@@ -160,7 +152,7 @@ func (a *volumeAdapter) CachedPath() string {
160 160
 
161 161
 func (a *volumeAdapter) Mount(id string) (string, error) {
162 162
 	mountpoint, err := a.proxy.Mount(a.name, id)
163
-	a.eMount = hostPath(a.baseHostPath, mountpoint)
163
+	a.eMount = a.scopePath(mountpoint)
164 164
 	return a.eMount, err
165 165
 }
166 166
 
... ...
@@ -25,9 +25,9 @@ var drivers = &driverExtpoint{
25 25
 const extName = "VolumeDriver"
26 26
 
27 27
 // NewVolumeDriver returns a driver has the given name mapped on the given client.
28
-func NewVolumeDriver(name string, baseHostPath string, c client) volume.Driver {
28
+func NewVolumeDriver(name string, scopePath func(string) string, c client) volume.Driver {
29 29
 	proxy := &volumeDriverProxy{c}
30
-	return &volumeDriverAdapter{name: name, baseHostPath: baseHostPath, proxy: proxy}
30
+	return &volumeDriverAdapter{name: name, scopePath: scopePath, proxy: proxy}
31 31
 }
32 32
 
33 33
 // volumeDriver defines the available functions that volume plugins must implement.
... ...
@@ -129,7 +129,7 @@ func lookup(name string, mode int) (volume.Driver, error) {
129 129
 			return nil, errors.Wrap(err, "error looking up volume plugin "+name)
130 130
 		}
131 131
 
132
-		d := NewVolumeDriver(p.Name(), p.BasePath(), p.Client())
132
+		d := NewVolumeDriver(p.Name(), p.ScopedPath, p.Client())
133 133
 		if err := validateDriver(d); err != nil {
134 134
 			if mode > 0 {
135 135
 				// Undo any reference count changes from the initial `Get`
... ...
@@ -224,7 +224,7 @@ func GetAllDrivers() ([]volume.Driver, error) {
224 224
 			continue
225 225
 		}
226 226
 
227
-		ext := NewVolumeDriver(name, p.BasePath(), p.Client())
227
+		ext := NewVolumeDriver(name, p.ScopedPath, p.Client())
228 228
 		if p.IsV1() {
229 229
 			drivers.extensions[name] = ext
230 230
 		}
... ...
@@ -178,8 +178,8 @@ func (p *fakePlugin) IsV1() bool {
178 178
 	return false
179 179
 }
180 180
 
181
-func (p *fakePlugin) BasePath() string {
182
-	return ""
181
+func (p *fakePlugin) ScopedPath(s string) string {
182
+	return s
183 183
 }
184 184
 
185 185
 type fakePluginGetter struct {