Browse code

Use runtime spec modifier for metrics plugin hook

Currently the metrics plugin uses a really hackish host mount with
propagated mounts to get the metrics socket into a plugin after the
plugin is alreay running.
This approach ends up leaking mounts which requires setting the plugin
manager root to private, which causes some other issues.

With this change, plugin subsystems can register a set of modifiers to
apply to the plugin's runtime spec before the plugin is ever started.
This will help to generalize some of the customization work that needs
to happen for various plugin subsystems (and future ones).

Specifically it lets the metrics plugin subsystem append a mount to the
runtime spec to mount the metrics socket in the plugin's mount namespace
rather than the host's and prevetns any leaking due to this mount.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Brian Goff authored on 2017/12/14 05:24:51
Showing 6 changed files
... ...
@@ -1,10 +1,8 @@
1 1
 package daemon
2 2
 
3 3
 import (
4
-	"path/filepath"
5 4
 	"sync"
6 5
 
7
-	"github.com/docker/docker/pkg/mount"
8 6
 	"github.com/docker/docker/pkg/plugingetter"
9 7
 	metrics "github.com/docker/go-metrics"
10 8
 	"github.com/pkg/errors"
... ...
@@ -132,18 +130,6 @@ func (d *Daemon) cleanupMetricsPlugins() {
132 132
 	}
133 133
 }
134 134
 
135
-type metricsPlugin struct {
136
-	plugingetter.CompatPlugin
137
-}
138
-
139
-func (p metricsPlugin) sock() string {
140
-	return "metrics.sock"
141
-}
142
-
143
-func (p metricsPlugin) sockBase() string {
144
-	return filepath.Join(p.BasePath(), "run", "docker")
145
-}
146
-
147 135
 func pluginStartMetricsCollection(p plugingetter.CompatPlugin) error {
148 136
 	type metricsPluginResponse struct {
149 137
 		Err string
... ...
@@ -162,12 +148,4 @@ func pluginStopMetricsCollection(p plugingetter.CompatPlugin) {
162 162
 	if err := p.Client().Call(metricsPluginType+".StopMetrics", nil, nil); err != nil {
163 163
 		logrus.WithError(err).WithField("name", p.Name()).Error("error stopping metrics collector")
164 164
 	}
165
-
166
-	mp := metricsPlugin{p}
167
-	sockPath := filepath.Join(mp.sockBase(), mp.sock())
168
-	if err := mount.Unmount(sockPath); err != nil {
169
-		if mounted, _ := mount.Mounted(sockPath); mounted {
170
-			logrus.WithError(err).WithField("name", p.Name()).WithField("socket", sockPath).Error("error unmounting metrics socket for plugin")
171
-		}
172
-	}
173 165
 }
... ...
@@ -5,13 +5,13 @@ package daemon
5 5
 import (
6 6
 	"net"
7 7
 	"net/http"
8
-	"os"
9 8
 	"path/filepath"
10 9
 
11
-	"github.com/docker/docker/pkg/mount"
12 10
 	"github.com/docker/docker/pkg/plugingetter"
13 11
 	"github.com/docker/docker/pkg/plugins"
12
+	"github.com/docker/docker/plugin"
14 13
 	metrics "github.com/docker/go-metrics"
14
+	specs "github.com/opencontainers/runtime-spec/specs-go"
15 15
 	"github.com/pkg/errors"
16 16
 	"github.com/sirupsen/logrus"
17 17
 	"golang.org/x/sys/unix"
... ...
@@ -34,52 +34,22 @@ func (daemon *Daemon) listenMetricsSock() (string, error) {
34 34
 	return path, nil
35 35
 }
36 36
 
37
-func registerMetricsPluginCallback(getter plugingetter.PluginGetter, sockPath string) {
38
-	getter.Handle(metricsPluginType, func(name string, client *plugins.Client) {
37
+func registerMetricsPluginCallback(store *plugin.Store, sockPath string) {
38
+	store.RegisterRuntimeOpt(metricsPluginType, func(s *specs.Spec) {
39
+		f := plugin.WithSpecMounts([]specs.Mount{
40
+			{Type: "bind", Source: sockPath, Destination: "/run/docker/metrics.sock", Options: []string{"bind", "ro"}},
41
+		})
42
+		f(s)
43
+	})
44
+	store.Handle(metricsPluginType, func(name string, client *plugins.Client) {
39 45
 		// Use lookup since nothing in the system can really reference it, no need
40 46
 		// to protect against removal
41
-		p, err := getter.Get(name, metricsPluginType, plugingetter.Lookup)
47
+		p, err := store.Get(name, metricsPluginType, plugingetter.Lookup)
42 48
 		if err != nil {
43 49
 			return
44 50
 		}
45 51
 
46
-		mp := metricsPlugin{p}
47
-		sockBase := mp.sockBase()
48
-		if err := os.MkdirAll(sockBase, 0755); err != nil {
49
-			logrus.WithError(err).WithField("name", name).WithField("path", sockBase).Error("error creating metrics plugin base path")
50
-			return
51
-		}
52
-
53
-		defer func() {
54
-			if err != nil {
55
-				os.RemoveAll(sockBase)
56
-			}
57
-		}()
58
-
59
-		pluginSockPath := filepath.Join(sockBase, mp.sock())
60
-		_, err = os.Stat(pluginSockPath)
61
-		if err == nil {
62
-			mount.Unmount(pluginSockPath)
63
-		} else {
64
-			logrus.WithField("path", pluginSockPath).Debugf("creating plugin socket")
65
-			f, err := os.OpenFile(pluginSockPath, os.O_CREATE, 0600)
66
-			if err != nil {
67
-				return
68
-			}
69
-			f.Close()
70
-		}
71
-
72
-		if err := mount.Mount(sockPath, pluginSockPath, "none", "bind,ro"); err != nil {
73
-			logrus.WithError(err).WithField("name", name).Error("could not mount metrics socket to plugin")
74
-			return
75
-		}
76
-
77 52
 		if err := pluginStartMetricsCollection(p); err != nil {
78
-			if err := mount.Unmount(pluginSockPath); err != nil {
79
-				if mounted, _ := mount.Mounted(pluginSockPath); mounted {
80
-					logrus.WithError(err).WithField("sock_path", pluginSockPath).Error("error unmounting metrics socket from plugin during cleanup")
81
-				}
82
-			}
83 53
 			logrus.WithError(err).WithField("name", name).Error("error while initializing metrics plugin")
84 54
 		}
85 55
 	})
... ...
@@ -5,12 +5,14 @@ import (
5 5
 
6 6
 	"github.com/docker/docker/pkg/plugins"
7 7
 	"github.com/docker/docker/plugin/v2"
8
+	specs "github.com/opencontainers/runtime-spec/specs-go"
8 9
 )
9 10
 
10 11
 // Store manages the plugin inventory in memory and on-disk
11 12
 type Store struct {
12 13
 	sync.RWMutex
13
-	plugins map[string]*v2.Plugin
14
+	plugins  map[string]*v2.Plugin
15
+	specOpts map[string][]SpecOpt
14 16
 	/* handlers are necessary for transition path of legacy plugins
15 17
 	 * to the new model. Legacy plugins use Handle() for registering an
16 18
 	 * activation callback.*/
... ...
@@ -21,10 +23,14 @@ type Store struct {
21 21
 func NewStore() *Store {
22 22
 	return &Store{
23 23
 		plugins:  make(map[string]*v2.Plugin),
24
+		specOpts: make(map[string][]SpecOpt),
24 25
 		handlers: make(map[string][]func(string, *plugins.Client)),
25 26
 	}
26 27
 }
27 28
 
29
+// SpecOpt is used for subsystems that need to modify the runtime spec of a plugin
30
+type SpecOpt func(*specs.Spec)
31
+
28 32
 // CreateOpt is used to configure specific plugin details when created
29 33
 type CreateOpt func(p *v2.Plugin)
30 34
 
... ...
@@ -35,3 +41,10 @@ func WithSwarmService(id string) CreateOpt {
35 35
 		p.SwarmServiceID = id
36 36
 	}
37 37
 }
38
+
39
+// WithSpecMounts is a SpecOpt which appends the provided mounts to the runtime spec
40
+func WithSpecMounts(mounts []specs.Mount) SpecOpt {
41
+	return func(s *specs.Spec) {
42
+		s.Mounts = append(s.Mounts, mounts...)
43
+	}
44
+}
... ...
@@ -9,6 +9,7 @@ import (
9 9
 	"github.com/docker/docker/pkg/plugingetter"
10 10
 	"github.com/docker/docker/pkg/plugins"
11 11
 	"github.com/docker/docker/plugin/v2"
12
+	specs "github.com/opencontainers/runtime-spec/specs-go"
12 13
 	"github.com/pkg/errors"
13 14
 	"github.com/sirupsen/logrus"
14 15
 )
... ...
@@ -64,6 +65,10 @@ func (ps *Store) GetAll() map[string]*v2.Plugin {
64 64
 func (ps *Store) SetAll(plugins map[string]*v2.Plugin) {
65 65
 	ps.Lock()
66 66
 	defer ps.Unlock()
67
+
68
+	for _, p := range plugins {
69
+		ps.setSpecOpts(p)
70
+	}
67 71
 	ps.plugins = plugins
68 72
 }
69 73
 
... ...
@@ -90,6 +95,22 @@ func (ps *Store) SetState(p *v2.Plugin, state bool) {
90 90
 	p.PluginObj.Enabled = state
91 91
 }
92 92
 
93
+func (ps *Store) setSpecOpts(p *v2.Plugin) {
94
+	var specOpts []SpecOpt
95
+	for _, typ := range p.GetTypes() {
96
+		opts, ok := ps.specOpts[typ.String()]
97
+		if ok {
98
+			specOpts = append(specOpts, opts...)
99
+		}
100
+	}
101
+
102
+	p.SetSpecOptModifier(func(s *specs.Spec) {
103
+		for _, o := range specOpts {
104
+			o(s)
105
+		}
106
+	})
107
+}
108
+
93 109
 // Add adds a plugin to memory and plugindb.
94 110
 // An error will be returned if there is a collision.
95 111
 func (ps *Store) Add(p *v2.Plugin) error {
... ...
@@ -99,6 +120,9 @@ func (ps *Store) Add(p *v2.Plugin) error {
99 99
 	if v, exist := ps.plugins[p.GetID()]; exist {
100 100
 		return fmt.Errorf("plugin %q has the same ID %s as %q", p.Name(), p.GetID(), v.Name())
101 101
 	}
102
+
103
+	ps.setSpecOpts(p)
104
+
102 105
 	ps.plugins[p.GetID()] = p
103 106
 	return nil
104 107
 }
... ...
@@ -182,20 +206,24 @@ func (ps *Store) GetAllByCap(capability string) ([]plugingetter.CompatPlugin, er
182 182
 	return result, nil
183 183
 }
184 184
 
185
+func pluginType(cap string) string {
186
+	return fmt.Sprintf("docker.%s/%s", strings.ToLower(cap), defaultAPIVersion)
187
+}
188
+
185 189
 // Handle sets a callback for a given capability. It is only used by network
186 190
 // and ipam drivers during plugin registration. The callback registers the
187 191
 // driver with the subsystem (network, ipam).
188 192
 func (ps *Store) Handle(capability string, callback func(string, *plugins.Client)) {
189
-	pluginType := fmt.Sprintf("docker.%s/%s", strings.ToLower(capability), defaultAPIVersion)
193
+	typ := pluginType(capability)
190 194
 
191 195
 	// Register callback with new plugin model.
192 196
 	ps.Lock()
193
-	handlers, ok := ps.handlers[pluginType]
197
+	handlers, ok := ps.handlers[typ]
194 198
 	if !ok {
195 199
 		handlers = []func(string, *plugins.Client){}
196 200
 	}
197 201
 	handlers = append(handlers, callback)
198
-	ps.handlers[pluginType] = handlers
202
+	ps.handlers[typ] = handlers
199 203
 	ps.Unlock()
200 204
 
201 205
 	// Register callback with legacy plugin model.
... ...
@@ -204,6 +232,15 @@ func (ps *Store) Handle(capability string, callback func(string, *plugins.Client
204 204
 	}
205 205
 }
206 206
 
207
+// RegisterRuntimeOpt stores a list of SpecOpts for the provided capability.
208
+// These options are applied to the runtime spec before a plugin is started for the specified capability.
209
+func (ps *Store) RegisterRuntimeOpt(cap string, opts ...SpecOpt) {
210
+	ps.Lock()
211
+	defer ps.Unlock()
212
+	typ := pluginType(cap)
213
+	ps.specOpts[typ] = append(ps.specOpts[typ], opts...)
214
+}
215
+
207 216
 // CallHandler calls the registered callback. It is invoked during plugin enable.
208 217
 func (ps *Store) CallHandler(p *v2.Plugin) {
209 218
 	for _, typ := range p.GetTypes() {
... ...
@@ -9,6 +9,7 @@ import (
9 9
 	"github.com/docker/docker/pkg/plugingetter"
10 10
 	"github.com/docker/docker/pkg/plugins"
11 11
 	"github.com/opencontainers/go-digest"
12
+	specs "github.com/opencontainers/runtime-spec/specs-go"
12 13
 )
13 14
 
14 15
 // Plugin represents an individual plugin.
... ...
@@ -23,6 +24,8 @@ type Plugin struct {
23 23
 	Config   digest.Digest
24 24
 	Blobsums []digest.Digest
25 25
 
26
+	modifyRuntimeSpec func(*specs.Spec)
27
+
26 28
 	SwarmServiceID string
27 29
 }
28 30
 
... ...
@@ -250,3 +253,11 @@ func (p *Plugin) Acquire() {
250 250
 func (p *Plugin) Release() {
251 251
 	p.AddRefCount(plugingetter.Release)
252 252
 }
253
+
254
+// SetSpecOptModifier sets the function to use to modify the the generated
255
+// runtime spec.
256
+func (p *Plugin) SetSpecOptModifier(f func(*specs.Spec)) {
257
+	p.mu.Lock()
258
+	p.modifyRuntimeSpec = f
259
+	p.mu.Unlock()
260
+}
... ...
@@ -16,6 +16,7 @@ import (
16 16
 // InitSpec creates an OCI spec from the plugin's config.
17 17
 func (p *Plugin) InitSpec(execRoot string) (*specs.Spec, error) {
18 18
 	s := oci.DefaultSpec()
19
+
19 20
 	s.Root = &specs.Root{
20 21
 		Path:     p.Rootfs,
21 22
 		Readonly: false, // TODO: all plugins should be readonly? settable in config?
... ...
@@ -126,5 +127,9 @@ func (p *Plugin) InitSpec(execRoot string) (*specs.Spec, error) {
126 126
 	caps.Inheritable = append(caps.Inheritable, p.PluginObj.Config.Linux.Capabilities...)
127 127
 	caps.Effective = append(caps.Effective, p.PluginObj.Config.Linux.Capabilities...)
128 128
 
129
+	if p.modifyRuntimeSpec != nil {
130
+		p.modifyRuntimeSpec(&s)
131
+	}
132
+
129 133
 	return &s, nil
130 134
 }