In some circumstances we were not properly releasing plugin references,
leading to failures in removing a plugin with no way to recover other
than restarting the daemon.
1. If volume create fails (in the driver)
2. If a driver validation fails (should be rare)
3. If trying to get a plugin that does not match the passed in capability
Ideally the test for 1 and 2 would just be a unit test, however the
plugin interfaces are too complicated as `plugingetter` relies on
github.com/pkg/plugin/Client (a concrete type), which will require
spinning up services from within the unit test... it just wouldn't be a
unit test at this point.
I attempted to refactor this a bit, but since both libnetwork and
swarmkit are reliant on `plugingetter` as well, this would not work.
This really requires a re-write of the lower-level plugin management to
decouple these pieces.
Signed-off-by: Brian Goff <cpuguy83@gmail.com>
0 | 1 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,23 @@ |
0 |
+package main |
|
1 |
+ |
|
2 |
+import ( |
|
3 |
+ "net" |
|
4 |
+ "net/http" |
|
5 |
+) |
|
6 |
+ |
|
7 |
+func main() { |
|
8 |
+ l, err := net.Listen("unix", "/run/docker/plugins/plugin.sock") |
|
9 |
+ if err != nil { |
|
10 |
+ panic(err) |
|
11 |
+ } |
|
12 |
+ |
|
13 |
+ mux := http.NewServeMux() |
|
14 |
+ server := http.Server{ |
|
15 |
+ Addr: l.Addr().String(), |
|
16 |
+ Handler: http.NewServeMux(), |
|
17 |
+ } |
|
18 |
+ mux.HandleFunc("/VolumeDriver.Create", func(w http.ResponseWriter, r *http.Request) { |
|
19 |
+ http.Error(w, "error during create", http.StatusInternalServerError) |
|
20 |
+ }) |
|
21 |
+ server.Serve(l) |
|
22 |
+} |
0 | 1 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,51 @@ |
0 |
+// +build linux |
|
1 |
+ |
|
2 |
+package volume |
|
3 |
+ |
|
4 |
+import ( |
|
5 |
+ "context" |
|
6 |
+ "testing" |
|
7 |
+ |
|
8 |
+ "github.com/docker/docker/api/types" |
|
9 |
+ "github.com/docker/docker/api/types/volume" |
|
10 |
+ "github.com/docker/docker/integration-cli/daemon" |
|
11 |
+) |
|
12 |
+ |
|
13 |
+// TestCreateDerefOnError ensures that if a volume create fails, that the plugin is dereferenced |
|
14 |
+// Normally 1 volume == 1 reference to a plugin, which prevents a plugin from being removed. |
|
15 |
+// If the volume create fails, we should make sure to dereference the plugin. |
|
16 |
+func TestCreateDerefOnError(t *testing.T) { |
|
17 |
+ t.Parallel() |
|
18 |
+ |
|
19 |
+ d := daemon.New(t, "", dockerdBinary, daemon.Config{}) |
|
20 |
+ d.Start(t) |
|
21 |
+ defer d.Stop(t) |
|
22 |
+ |
|
23 |
+ c, err := d.NewClient() |
|
24 |
+ if err != nil { |
|
25 |
+ t.Fatal(err) |
|
26 |
+ } |
|
27 |
+ |
|
28 |
+ pName := "testderef" |
|
29 |
+ createPlugin(t, c, pName, "create-error", asVolumeDriver) |
|
30 |
+ |
|
31 |
+ if err := c.PluginEnable(context.Background(), pName, types.PluginEnableOptions{Timeout: 30}); err != nil { |
|
32 |
+ t.Fatal(err) |
|
33 |
+ } |
|
34 |
+ |
|
35 |
+ _, err = c.VolumeCreate(context.Background(), volume.VolumesCreateBody{ |
|
36 |
+ Driver: pName, |
|
37 |
+ Name: "fake", |
|
38 |
+ }) |
|
39 |
+ if err == nil { |
|
40 |
+ t.Fatal("volume create should have failed") |
|
41 |
+ } |
|
42 |
+ |
|
43 |
+ if err := c.PluginDisable(context.Background(), pName, types.PluginDisableOptions{}); err != nil { |
|
44 |
+ t.Fatal(err) |
|
45 |
+ } |
|
46 |
+ |
|
47 |
+ if err := c.PluginRemove(context.Background(), pName, types.PluginRemoveOptions{}); err != nil { |
|
48 |
+ t.Fatal(err) |
|
49 |
+ } |
|
50 |
+} |
0 | 51 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,69 @@ |
0 |
+package volume |
|
1 |
+ |
|
2 |
+import ( |
|
3 |
+ "context" |
|
4 |
+ "os" |
|
5 |
+ "os/exec" |
|
6 |
+ "path/filepath" |
|
7 |
+ "testing" |
|
8 |
+ "time" |
|
9 |
+ |
|
10 |
+ "github.com/docker/docker/api/types" |
|
11 |
+ "github.com/docker/docker/integration-cli/fixtures/plugin" |
|
12 |
+ "github.com/docker/docker/pkg/locker" |
|
13 |
+ "github.com/pkg/errors" |
|
14 |
+) |
|
15 |
+ |
|
16 |
+const dockerdBinary = "dockerd" |
|
17 |
+ |
|
18 |
+var pluginBuildLock = locker.New() |
|
19 |
+ |
|
20 |
+func ensurePlugin(t *testing.T, name string) string { |
|
21 |
+ pluginBuildLock.Lock(name) |
|
22 |
+ defer pluginBuildLock.Unlock(name) |
|
23 |
+ |
|
24 |
+ installPath := filepath.Join(os.Getenv("GOPATH"), "bin", name) |
|
25 |
+ if _, err := os.Stat(installPath); err == nil { |
|
26 |
+ return installPath |
|
27 |
+ } |
|
28 |
+ |
|
29 |
+ goBin, err := exec.LookPath("go") |
|
30 |
+ if err != nil { |
|
31 |
+ t.Fatal(err) |
|
32 |
+ } |
|
33 |
+ |
|
34 |
+ cmd := exec.Command(goBin, "build", "-o", installPath, "./"+filepath.Join("cmd", name)) |
|
35 |
+ cmd.Env = append(cmd.Env, "CGO_ENABLED=0") |
|
36 |
+ if out, err := cmd.CombinedOutput(); err != nil { |
|
37 |
+ t.Fatal(errors.Wrapf(err, "error building basic plugin bin: %s", string(out))) |
|
38 |
+ } |
|
39 |
+ |
|
40 |
+ return installPath |
|
41 |
+} |
|
42 |
+ |
|
43 |
+func asVolumeDriver(cfg *plugin.Config) { |
|
44 |
+ cfg.Interface.Types = []types.PluginInterfaceType{ |
|
45 |
+ {Capability: "volumedriver", Prefix: "docker", Version: "1.0"}, |
|
46 |
+ } |
|
47 |
+} |
|
48 |
+ |
|
49 |
+func withSockPath(name string) func(*plugin.Config) { |
|
50 |
+ return func(cfg *plugin.Config) { |
|
51 |
+ cfg.Interface.Socket = name |
|
52 |
+ } |
|
53 |
+} |
|
54 |
+ |
|
55 |
+func createPlugin(t *testing.T, client plugin.CreateClient, alias, bin string, opts ...plugin.CreateOpt) { |
|
56 |
+ pluginBin := ensurePlugin(t, bin) |
|
57 |
+ |
|
58 |
+ opts = append(opts, withSockPath("plugin.sock")) |
|
59 |
+ opts = append(opts, plugin.WithBinary(pluginBin)) |
|
60 |
+ |
|
61 |
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) |
|
62 |
+ err := plugin.Create(ctx, client, alias, opts...) |
|
63 |
+ cancel() |
|
64 |
+ |
|
65 |
+ if err != nil { |
|
66 |
+ t.Fatal(err) |
|
67 |
+ } |
|
68 |
+} |
... | ... |
@@ -115,10 +115,15 @@ func (ps *Store) Get(name, capability string, mode int) (plugingetter.CompatPlug |
115 | 115 |
if ps != nil { |
116 | 116 |
p, err := ps.GetV2Plugin(name) |
117 | 117 |
if err == nil { |
118 |
- p.AddRefCount(mode) |
|
119 | 118 |
if p.IsEnabled() { |
120 |
- return p.FilterByCap(capability) |
|
119 |
+ fp, err := p.FilterByCap(capability) |
|
120 |
+ if err != nil { |
|
121 |
+ return nil, err |
|
122 |
+ } |
|
123 |
+ p.AddRefCount(mode) |
|
124 |
+ return fp, nil |
|
121 | 125 |
} |
126 |
+ |
|
122 | 127 |
// Plugin was found but it is disabled, so we should not fall back to legacy plugins |
123 | 128 |
// but we should error out right away |
124 | 129 |
return nil, errDisabled(name) |
... | ... |
@@ -4,6 +4,7 @@ import ( |
4 | 4 |
"testing" |
5 | 5 |
|
6 | 6 |
"github.com/docker/docker/api/types" |
7 |
+ "github.com/docker/docker/pkg/plugingetter" |
|
7 | 8 |
"github.com/docker/docker/plugin/v2" |
8 | 9 |
) |
9 | 10 |
|
... | ... |
@@ -31,3 +32,33 @@ func TestFilterByCapPos(t *testing.T) { |
31 | 31 |
t.Fatalf("expected no error, got %v", err) |
32 | 32 |
} |
33 | 33 |
} |
34 |
+ |
|
35 |
+func TestStoreGetPluginNotMatchCapRefs(t *testing.T) { |
|
36 |
+ s := NewStore() |
|
37 |
+ p := v2.Plugin{PluginObj: types.Plugin{Name: "test:latest"}} |
|
38 |
+ |
|
39 |
+ iType := types.PluginInterfaceType{Capability: "whatever", Prefix: "docker", Version: "1.0"} |
|
40 |
+ i := types.PluginConfigInterface{Socket: "plugins.sock", Types: []types.PluginInterfaceType{iType}} |
|
41 |
+ p.PluginObj.Config.Interface = i |
|
42 |
+ |
|
43 |
+ if err := s.Add(&p); err != nil { |
|
44 |
+ t.Fatal(err) |
|
45 |
+ } |
|
46 |
+ |
|
47 |
+ if _, err := s.Get("test", "volumedriver", plugingetter.Acquire); err == nil { |
|
48 |
+ t.Fatal("exepcted error when getting plugin that doesn't match the passed in capability") |
|
49 |
+ } |
|
50 |
+ |
|
51 |
+ if refs := p.GetRefCount(); refs != 0 { |
|
52 |
+ t.Fatalf("reference count should be 0, got: %d", refs) |
|
53 |
+ } |
|
54 |
+ |
|
55 |
+ p.PluginObj.Enabled = true |
|
56 |
+ if _, err := s.Get("test", "volumedriver", plugingetter.Acquire); err == nil { |
|
57 |
+ t.Fatal("exepcted error when getting plugin that doesn't match the passed in capability") |
|
58 |
+ } |
|
59 |
+ |
|
60 |
+ if refs := p.GetRefCount(); refs != 0 { |
|
61 |
+ t.Fatalf("reference count should be 0, got: %d", refs) |
|
62 |
+ } |
|
63 |
+} |
... | ... |
@@ -11,6 +11,7 @@ import ( |
11 | 11 |
getter "github.com/docker/docker/pkg/plugingetter" |
12 | 12 |
"github.com/docker/docker/volume" |
13 | 13 |
"github.com/pkg/errors" |
14 |
+ "github.com/sirupsen/logrus" |
|
14 | 15 |
) |
15 | 16 |
|
16 | 17 |
// currently created by hand. generation tool would generate this like: |
... | ... |
@@ -130,6 +131,12 @@ func lookup(name string, mode int) (volume.Driver, error) { |
130 | 130 |
|
131 | 131 |
d := NewVolumeDriver(p.Name(), p.BasePath(), p.Client()) |
132 | 132 |
if err := validateDriver(d); err != nil { |
133 |
+ if mode > 0 { |
|
134 |
+ // Undo any reference count changes from the initial `Get` |
|
135 |
+ if _, err := drivers.plugingetter.Get(name, extName, mode*-1); err != nil { |
|
136 |
+ logrus.WithError(err).WithField("action", "validate-driver").WithField("plugin", name).Error("error releasing reference to plugin") |
|
137 |
+ } |
|
138 |
+ } |
|
133 | 139 |
return nil, err |
134 | 140 |
} |
135 | 141 |
|
... | ... |
@@ -169,9 +176,9 @@ func CreateDriver(name string) (volume.Driver, error) { |
169 | 169 |
return lookup(name, getter.Acquire) |
170 | 170 |
} |
171 | 171 |
|
172 |
-// RemoveDriver returns a volume driver by its name and decrements RefCount.. |
|
172 |
+// ReleaseDriver returns a volume driver by its name and decrements RefCount.. |
|
173 | 173 |
// If the driver is empty, it looks for the local driver. |
174 |
-func RemoveDriver(name string) (volume.Driver, error) { |
|
174 |
+func ReleaseDriver(name string) (volume.Driver, error) { |
|
175 | 175 |
if name == "" { |
176 | 176 |
name = volume.DefaultDriverName |
177 | 177 |
} |
... | ... |
@@ -145,7 +145,7 @@ func (s *VolumeStore) Purge(name string) { |
145 | 145 |
s.globalLock.Lock() |
146 | 146 |
v, exists := s.names[name] |
147 | 147 |
if exists { |
148 |
- if _, err := volumedrivers.RemoveDriver(v.DriverName()); err != nil { |
|
148 |
+ if _, err := volumedrivers.ReleaseDriver(v.DriverName()); err != nil { |
|
149 | 149 |
logrus.Errorf("Error dereferencing volume driver: %v", err) |
150 | 150 |
} |
151 | 151 |
} |
... | ... |
@@ -409,6 +409,9 @@ func (s *VolumeStore) create(name, driverName string, opts, labels map[string]st |
409 | 409 |
} |
410 | 410 |
v, err = vd.Create(name, opts) |
411 | 411 |
if err != nil { |
412 |
+ if _, err := volumedrivers.ReleaseDriver(driverName); err != nil { |
|
413 |
+ logrus.WithError(err).WithField("driver", driverName).Error("Error releasing reference to volume driver") |
|
414 |
+ } |
|
412 | 415 |
return nil, err |
413 | 416 |
} |
414 | 417 |
s.globalLock.Lock() |