Browse code

Fixup some issues with plugin refcounting

In some circumstances we were not properly releasing plugin references,
leading to failures in removing a plugin with no way to recover other
than restarting the daemon.

1. If volume create fails (in the driver)
2. If a driver validation fails (should be rare)
3. If trying to get a plugin that does not match the passed in capability

Ideally the test for 1 and 2 would just be a unit test, however the
plugin interfaces are too complicated as `plugingetter` relies on
github.com/pkg/plugin/Client (a concrete type), which will require
spinning up services from within the unit test... it just wouldn't be a
unit test at this point.
I attempted to refactor this a bit, but since both libnetwork and
swarmkit are reliant on `plugingetter` as well, this would not work.
This really requires a re-write of the lower-level plugin management to
decouple these pieces.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Brian Goff authored on 2017/10/21 04:39:47
Showing 10 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1 @@
0
+package cmd
0 1
new file mode 100644
... ...
@@ -0,0 +1,23 @@
0
+package main
1
+
2
+import (
3
+	"net"
4
+	"net/http"
5
+)
6
+
7
+func main() {
8
+	l, err := net.Listen("unix", "/run/docker/plugins/plugin.sock")
9
+	if err != nil {
10
+		panic(err)
11
+	}
12
+
13
+	mux := http.NewServeMux()
14
+	server := http.Server{
15
+		Addr:    l.Addr().String(),
16
+		Handler: http.NewServeMux(),
17
+	}
18
+	mux.HandleFunc("/VolumeDriver.Create", func(w http.ResponseWriter, r *http.Request) {
19
+		http.Error(w, "error during create", http.StatusInternalServerError)
20
+	})
21
+	server.Serve(l)
22
+}
0 23
new file mode 100644
... ...
@@ -0,0 +1 @@
0
+package main
0 1
new file mode 100644
... ...
@@ -0,0 +1,51 @@
0
+// +build linux
1
+
2
+package volume
3
+
4
+import (
5
+	"context"
6
+	"testing"
7
+
8
+	"github.com/docker/docker/api/types"
9
+	"github.com/docker/docker/api/types/volume"
10
+	"github.com/docker/docker/integration-cli/daemon"
11
+)
12
+
13
+// TestCreateDerefOnError ensures that if a volume create fails, that the plugin is dereferenced
14
+// Normally 1 volume == 1 reference to a plugin, which prevents a plugin from being removed.
15
+// If the volume create fails, we should make sure to dereference the plugin.
16
+func TestCreateDerefOnError(t *testing.T) {
17
+	t.Parallel()
18
+
19
+	d := daemon.New(t, "", dockerdBinary, daemon.Config{})
20
+	d.Start(t)
21
+	defer d.Stop(t)
22
+
23
+	c, err := d.NewClient()
24
+	if err != nil {
25
+		t.Fatal(err)
26
+	}
27
+
28
+	pName := "testderef"
29
+	createPlugin(t, c, pName, "create-error", asVolumeDriver)
30
+
31
+	if err := c.PluginEnable(context.Background(), pName, types.PluginEnableOptions{Timeout: 30}); err != nil {
32
+		t.Fatal(err)
33
+	}
34
+
35
+	_, err = c.VolumeCreate(context.Background(), volume.VolumesCreateBody{
36
+		Driver: pName,
37
+		Name:   "fake",
38
+	})
39
+	if err == nil {
40
+		t.Fatal("volume create should have failed")
41
+	}
42
+
43
+	if err := c.PluginDisable(context.Background(), pName, types.PluginDisableOptions{}); err != nil {
44
+		t.Fatal(err)
45
+	}
46
+
47
+	if err := c.PluginRemove(context.Background(), pName, types.PluginRemoveOptions{}); err != nil {
48
+		t.Fatal(err)
49
+	}
50
+}
0 51
new file mode 100644
... ...
@@ -0,0 +1,69 @@
0
+package volume
1
+
2
+import (
3
+	"context"
4
+	"os"
5
+	"os/exec"
6
+	"path/filepath"
7
+	"testing"
8
+	"time"
9
+
10
+	"github.com/docker/docker/api/types"
11
+	"github.com/docker/docker/integration-cli/fixtures/plugin"
12
+	"github.com/docker/docker/pkg/locker"
13
+	"github.com/pkg/errors"
14
+)
15
+
16
+const dockerdBinary = "dockerd"
17
+
18
+var pluginBuildLock = locker.New()
19
+
20
+func ensurePlugin(t *testing.T, name string) string {
21
+	pluginBuildLock.Lock(name)
22
+	defer pluginBuildLock.Unlock(name)
23
+
24
+	installPath := filepath.Join(os.Getenv("GOPATH"), "bin", name)
25
+	if _, err := os.Stat(installPath); err == nil {
26
+		return installPath
27
+	}
28
+
29
+	goBin, err := exec.LookPath("go")
30
+	if err != nil {
31
+		t.Fatal(err)
32
+	}
33
+
34
+	cmd := exec.Command(goBin, "build", "-o", installPath, "./"+filepath.Join("cmd", name))
35
+	cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
36
+	if out, err := cmd.CombinedOutput(); err != nil {
37
+		t.Fatal(errors.Wrapf(err, "error building basic plugin bin: %s", string(out)))
38
+	}
39
+
40
+	return installPath
41
+}
42
+
43
+func asVolumeDriver(cfg *plugin.Config) {
44
+	cfg.Interface.Types = []types.PluginInterfaceType{
45
+		{Capability: "volumedriver", Prefix: "docker", Version: "1.0"},
46
+	}
47
+}
48
+
49
+func withSockPath(name string) func(*plugin.Config) {
50
+	return func(cfg *plugin.Config) {
51
+		cfg.Interface.Socket = name
52
+	}
53
+}
54
+
55
+func createPlugin(t *testing.T, client plugin.CreateClient, alias, bin string, opts ...plugin.CreateOpt) {
56
+	pluginBin := ensurePlugin(t, bin)
57
+
58
+	opts = append(opts, withSockPath("plugin.sock"))
59
+	opts = append(opts, plugin.WithBinary(pluginBin))
60
+
61
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
62
+	err := plugin.Create(ctx, client, alias, opts...)
63
+	cancel()
64
+
65
+	if err != nil {
66
+		t.Fatal(err)
67
+	}
68
+}
... ...
@@ -1,6 +1,8 @@
1 1
 package plugingetter
2 2
 
3
-import "github.com/docker/docker/pkg/plugins"
3
+import (
4
+	"github.com/docker/docker/pkg/plugins"
5
+)
4 6
 
5 7
 const (
6 8
 	// Lookup doesn't update RefCount
... ...
@@ -115,10 +115,15 @@ func (ps *Store) Get(name, capability string, mode int) (plugingetter.CompatPlug
115 115
 	if ps != nil {
116 116
 		p, err := ps.GetV2Plugin(name)
117 117
 		if err == nil {
118
-			p.AddRefCount(mode)
119 118
 			if p.IsEnabled() {
120
-				return p.FilterByCap(capability)
119
+				fp, err := p.FilterByCap(capability)
120
+				if err != nil {
121
+					return nil, err
122
+				}
123
+				p.AddRefCount(mode)
124
+				return fp, nil
121 125
 			}
126
+
122 127
 			// Plugin was found but it is disabled, so we should not fall back to legacy plugins
123 128
 			// but we should error out right away
124 129
 			return nil, errDisabled(name)
... ...
@@ -4,6 +4,7 @@ import (
4 4
 	"testing"
5 5
 
6 6
 	"github.com/docker/docker/api/types"
7
+	"github.com/docker/docker/pkg/plugingetter"
7 8
 	"github.com/docker/docker/plugin/v2"
8 9
 )
9 10
 
... ...
@@ -31,3 +32,33 @@ func TestFilterByCapPos(t *testing.T) {
31 31
 		t.Fatalf("expected no error, got %v", err)
32 32
 	}
33 33
 }
34
+
35
+func TestStoreGetPluginNotMatchCapRefs(t *testing.T) {
36
+	s := NewStore()
37
+	p := v2.Plugin{PluginObj: types.Plugin{Name: "test:latest"}}
38
+
39
+	iType := types.PluginInterfaceType{Capability: "whatever", Prefix: "docker", Version: "1.0"}
40
+	i := types.PluginConfigInterface{Socket: "plugins.sock", Types: []types.PluginInterfaceType{iType}}
41
+	p.PluginObj.Config.Interface = i
42
+
43
+	if err := s.Add(&p); err != nil {
44
+		t.Fatal(err)
45
+	}
46
+
47
+	if _, err := s.Get("test", "volumedriver", plugingetter.Acquire); err == nil {
48
+		t.Fatal("exepcted error when getting plugin that doesn't match the passed in capability")
49
+	}
50
+
51
+	if refs := p.GetRefCount(); refs != 0 {
52
+		t.Fatalf("reference count should be 0, got: %d", refs)
53
+	}
54
+
55
+	p.PluginObj.Enabled = true
56
+	if _, err := s.Get("test", "volumedriver", plugingetter.Acquire); err == nil {
57
+		t.Fatal("exepcted error when getting plugin that doesn't match the passed in capability")
58
+	}
59
+
60
+	if refs := p.GetRefCount(); refs != 0 {
61
+		t.Fatalf("reference count should be 0, got: %d", refs)
62
+	}
63
+}
... ...
@@ -11,6 +11,7 @@ import (
11 11
 	getter "github.com/docker/docker/pkg/plugingetter"
12 12
 	"github.com/docker/docker/volume"
13 13
 	"github.com/pkg/errors"
14
+	"github.com/sirupsen/logrus"
14 15
 )
15 16
 
16 17
 // currently created by hand. generation tool would generate this like:
... ...
@@ -130,6 +131,12 @@ func lookup(name string, mode int) (volume.Driver, error) {
130 130
 
131 131
 		d := NewVolumeDriver(p.Name(), p.BasePath(), p.Client())
132 132
 		if err := validateDriver(d); err != nil {
133
+			if mode > 0 {
134
+				// Undo any reference count changes from the initial `Get`
135
+				if _, err := drivers.plugingetter.Get(name, extName, mode*-1); err != nil {
136
+					logrus.WithError(err).WithField("action", "validate-driver").WithField("plugin", name).Error("error releasing reference to plugin")
137
+				}
138
+			}
133 139
 			return nil, err
134 140
 		}
135 141
 
... ...
@@ -169,9 +176,9 @@ func CreateDriver(name string) (volume.Driver, error) {
169 169
 	return lookup(name, getter.Acquire)
170 170
 }
171 171
 
172
-// RemoveDriver returns a volume driver by its name and decrements RefCount..
172
+// ReleaseDriver returns a volume driver by its name and decrements RefCount..
173 173
 // If the driver is empty, it looks for the local driver.
174
-func RemoveDriver(name string) (volume.Driver, error) {
174
+func ReleaseDriver(name string) (volume.Driver, error) {
175 175
 	if name == "" {
176 176
 		name = volume.DefaultDriverName
177 177
 	}
... ...
@@ -145,7 +145,7 @@ func (s *VolumeStore) Purge(name string) {
145 145
 	s.globalLock.Lock()
146 146
 	v, exists := s.names[name]
147 147
 	if exists {
148
-		if _, err := volumedrivers.RemoveDriver(v.DriverName()); err != nil {
148
+		if _, err := volumedrivers.ReleaseDriver(v.DriverName()); err != nil {
149 149
 			logrus.Errorf("Error dereferencing volume driver: %v", err)
150 150
 		}
151 151
 	}
... ...
@@ -409,6 +409,9 @@ func (s *VolumeStore) create(name, driverName string, opts, labels map[string]st
409 409
 	}
410 410
 	v, err = vd.Create(name, opts)
411 411
 	if err != nil {
412
+		if _, err := volumedrivers.ReleaseDriver(driverName); err != nil {
413
+			logrus.WithError(err).WithField("driver", driverName).Error("Error releasing reference to volume driver")
414
+		}
412 415
 		return nil, err
413 416
 	}
414 417
 	s.globalLock.Lock()