In some circumstances we were not properly releasing plugin references,
leading to failures in removing a plugin with no way to recover other
than restarting the daemon.
1. If volume create fails (in the driver)
2. If a driver validation fails (should be rare)
3. If trying to get a plugin that does not match the passed in capability
Ideally the test for 1 and 2 would just be a unit test, however the
plugin interfaces are too complicated as `plugingetter` relies on
github.com/pkg/plugin/Client (a concrete type), which will require
spinning up services from within the unit test... it just wouldn't be a
unit test at this point.
I attempted to refactor this a bit, but since both libnetwork and
swarmkit are reliant on `plugingetter` as well, this would not work.
This really requires a re-write of the lower-level plugin management to
decouple these pieces.
Signed-off-by: Brian Goff <cpuguy83@gmail.com>
| 0 | 1 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,23 @@ |
| 0 |
+package main |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "net" |
|
| 4 |
+ "net/http" |
|
| 5 |
+) |
|
| 6 |
+ |
|
| 7 |
+func main() {
|
|
| 8 |
+ l, err := net.Listen("unix", "/run/docker/plugins/plugin.sock")
|
|
| 9 |
+ if err != nil {
|
|
| 10 |
+ panic(err) |
|
| 11 |
+ } |
|
| 12 |
+ |
|
| 13 |
+ mux := http.NewServeMux() |
|
| 14 |
+ server := http.Server{
|
|
| 15 |
+ Addr: l.Addr().String(), |
|
| 16 |
+ Handler: http.NewServeMux(), |
|
| 17 |
+ } |
|
| 18 |
+ mux.HandleFunc("/VolumeDriver.Create", func(w http.ResponseWriter, r *http.Request) {
|
|
| 19 |
+ http.Error(w, "error during create", http.StatusInternalServerError) |
|
| 20 |
+ }) |
|
| 21 |
+ server.Serve(l) |
|
| 22 |
+} |
| 0 | 1 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,51 @@ |
| 0 |
+// +build linux |
|
| 1 |
+ |
|
| 2 |
+package volume |
|
| 3 |
+ |
|
| 4 |
+import ( |
|
| 5 |
+ "context" |
|
| 6 |
+ "testing" |
|
| 7 |
+ |
|
| 8 |
+ "github.com/docker/docker/api/types" |
|
| 9 |
+ "github.com/docker/docker/api/types/volume" |
|
| 10 |
+ "github.com/docker/docker/integration-cli/daemon" |
|
| 11 |
+) |
|
| 12 |
+ |
|
| 13 |
+// TestCreateDerefOnError ensures that if a volume create fails, that the plugin is dereferenced |
|
| 14 |
+// Normally 1 volume == 1 reference to a plugin, which prevents a plugin from being removed. |
|
| 15 |
+// If the volume create fails, we should make sure to dereference the plugin. |
|
| 16 |
+func TestCreateDerefOnError(t *testing.T) {
|
|
| 17 |
+ t.Parallel() |
|
| 18 |
+ |
|
| 19 |
+ d := daemon.New(t, "", dockerdBinary, daemon.Config{})
|
|
| 20 |
+ d.Start(t) |
|
| 21 |
+ defer d.Stop(t) |
|
| 22 |
+ |
|
| 23 |
+ c, err := d.NewClient() |
|
| 24 |
+ if err != nil {
|
|
| 25 |
+ t.Fatal(err) |
|
| 26 |
+ } |
|
| 27 |
+ |
|
| 28 |
+ pName := "testderef" |
|
| 29 |
+ createPlugin(t, c, pName, "create-error", asVolumeDriver) |
|
| 30 |
+ |
|
| 31 |
+ if err := c.PluginEnable(context.Background(), pName, types.PluginEnableOptions{Timeout: 30}); err != nil {
|
|
| 32 |
+ t.Fatal(err) |
|
| 33 |
+ } |
|
| 34 |
+ |
|
| 35 |
+ _, err = c.VolumeCreate(context.Background(), volume.VolumesCreateBody{
|
|
| 36 |
+ Driver: pName, |
|
| 37 |
+ Name: "fake", |
|
| 38 |
+ }) |
|
| 39 |
+ if err == nil {
|
|
| 40 |
+ t.Fatal("volume create should have failed")
|
|
| 41 |
+ } |
|
| 42 |
+ |
|
| 43 |
+ if err := c.PluginDisable(context.Background(), pName, types.PluginDisableOptions{}); err != nil {
|
|
| 44 |
+ t.Fatal(err) |
|
| 45 |
+ } |
|
| 46 |
+ |
|
| 47 |
+ if err := c.PluginRemove(context.Background(), pName, types.PluginRemoveOptions{}); err != nil {
|
|
| 48 |
+ t.Fatal(err) |
|
| 49 |
+ } |
|
| 50 |
+} |
| 0 | 51 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,69 @@ |
| 0 |
+package volume |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ "os" |
|
| 5 |
+ "os/exec" |
|
| 6 |
+ "path/filepath" |
|
| 7 |
+ "testing" |
|
| 8 |
+ "time" |
|
| 9 |
+ |
|
| 10 |
+ "github.com/docker/docker/api/types" |
|
| 11 |
+ "github.com/docker/docker/integration-cli/fixtures/plugin" |
|
| 12 |
+ "github.com/docker/docker/pkg/locker" |
|
| 13 |
+ "github.com/pkg/errors" |
|
| 14 |
+) |
|
| 15 |
+ |
|
| 16 |
+const dockerdBinary = "dockerd" |
|
| 17 |
+ |
|
| 18 |
+var pluginBuildLock = locker.New() |
|
| 19 |
+ |
|
| 20 |
+func ensurePlugin(t *testing.T, name string) string {
|
|
| 21 |
+ pluginBuildLock.Lock(name) |
|
| 22 |
+ defer pluginBuildLock.Unlock(name) |
|
| 23 |
+ |
|
| 24 |
+ installPath := filepath.Join(os.Getenv("GOPATH"), "bin", name)
|
|
| 25 |
+ if _, err := os.Stat(installPath); err == nil {
|
|
| 26 |
+ return installPath |
|
| 27 |
+ } |
|
| 28 |
+ |
|
| 29 |
+ goBin, err := exec.LookPath("go")
|
|
| 30 |
+ if err != nil {
|
|
| 31 |
+ t.Fatal(err) |
|
| 32 |
+ } |
|
| 33 |
+ |
|
| 34 |
+ cmd := exec.Command(goBin, "build", "-o", installPath, "./"+filepath.Join("cmd", name))
|
|
| 35 |
+ cmd.Env = append(cmd.Env, "CGO_ENABLED=0") |
|
| 36 |
+ if out, err := cmd.CombinedOutput(); err != nil {
|
|
| 37 |
+ t.Fatal(errors.Wrapf(err, "error building basic plugin bin: %s", string(out))) |
|
| 38 |
+ } |
|
| 39 |
+ |
|
| 40 |
+ return installPath |
|
| 41 |
+} |
|
| 42 |
+ |
|
| 43 |
+func asVolumeDriver(cfg *plugin.Config) {
|
|
| 44 |
+ cfg.Interface.Types = []types.PluginInterfaceType{
|
|
| 45 |
+ {Capability: "volumedriver", Prefix: "docker", Version: "1.0"},
|
|
| 46 |
+ } |
|
| 47 |
+} |
|
| 48 |
+ |
|
| 49 |
+func withSockPath(name string) func(*plugin.Config) {
|
|
| 50 |
+ return func(cfg *plugin.Config) {
|
|
| 51 |
+ cfg.Interface.Socket = name |
|
| 52 |
+ } |
|
| 53 |
+} |
|
| 54 |
+ |
|
| 55 |
+func createPlugin(t *testing.T, client plugin.CreateClient, alias, bin string, opts ...plugin.CreateOpt) {
|
|
| 56 |
+ pluginBin := ensurePlugin(t, bin) |
|
| 57 |
+ |
|
| 58 |
+ opts = append(opts, withSockPath("plugin.sock"))
|
|
| 59 |
+ opts = append(opts, plugin.WithBinary(pluginBin)) |
|
| 60 |
+ |
|
| 61 |
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) |
|
| 62 |
+ err := plugin.Create(ctx, client, alias, opts...) |
|
| 63 |
+ cancel() |
|
| 64 |
+ |
|
| 65 |
+ if err != nil {
|
|
| 66 |
+ t.Fatal(err) |
|
| 67 |
+ } |
|
| 68 |
+} |
| ... | ... |
@@ -115,10 +115,15 @@ func (ps *Store) Get(name, capability string, mode int) (plugingetter.CompatPlug |
| 115 | 115 |
if ps != nil {
|
| 116 | 116 |
p, err := ps.GetV2Plugin(name) |
| 117 | 117 |
if err == nil {
|
| 118 |
- p.AddRefCount(mode) |
|
| 119 | 118 |
if p.IsEnabled() {
|
| 120 |
- return p.FilterByCap(capability) |
|
| 119 |
+ fp, err := p.FilterByCap(capability) |
|
| 120 |
+ if err != nil {
|
|
| 121 |
+ return nil, err |
|
| 122 |
+ } |
|
| 123 |
+ p.AddRefCount(mode) |
|
| 124 |
+ return fp, nil |
|
| 121 | 125 |
} |
| 126 |
+ |
|
| 122 | 127 |
// Plugin was found but it is disabled, so we should not fall back to legacy plugins |
| 123 | 128 |
// but we should error out right away |
| 124 | 129 |
return nil, errDisabled(name) |
| ... | ... |
@@ -4,6 +4,7 @@ import ( |
| 4 | 4 |
"testing" |
| 5 | 5 |
|
| 6 | 6 |
"github.com/docker/docker/api/types" |
| 7 |
+ "github.com/docker/docker/pkg/plugingetter" |
|
| 7 | 8 |
"github.com/docker/docker/plugin/v2" |
| 8 | 9 |
) |
| 9 | 10 |
|
| ... | ... |
@@ -31,3 +32,33 @@ func TestFilterByCapPos(t *testing.T) {
|
| 31 | 31 |
t.Fatalf("expected no error, got %v", err)
|
| 32 | 32 |
} |
| 33 | 33 |
} |
| 34 |
+ |
|
| 35 |
+func TestStoreGetPluginNotMatchCapRefs(t *testing.T) {
|
|
| 36 |
+ s := NewStore() |
|
| 37 |
+ p := v2.Plugin{PluginObj: types.Plugin{Name: "test:latest"}}
|
|
| 38 |
+ |
|
| 39 |
+ iType := types.PluginInterfaceType{Capability: "whatever", Prefix: "docker", Version: "1.0"}
|
|
| 40 |
+ i := types.PluginConfigInterface{Socket: "plugins.sock", Types: []types.PluginInterfaceType{iType}}
|
|
| 41 |
+ p.PluginObj.Config.Interface = i |
|
| 42 |
+ |
|
| 43 |
+ if err := s.Add(&p); err != nil {
|
|
| 44 |
+ t.Fatal(err) |
|
| 45 |
+ } |
|
| 46 |
+ |
|
| 47 |
+ if _, err := s.Get("test", "volumedriver", plugingetter.Acquire); err == nil {
|
|
| 48 |
+ t.Fatal("exepcted error when getting plugin that doesn't match the passed in capability")
|
|
| 49 |
+ } |
|
| 50 |
+ |
|
| 51 |
+ if refs := p.GetRefCount(); refs != 0 {
|
|
| 52 |
+ t.Fatalf("reference count should be 0, got: %d", refs)
|
|
| 53 |
+ } |
|
| 54 |
+ |
|
| 55 |
+ p.PluginObj.Enabled = true |
|
| 56 |
+ if _, err := s.Get("test", "volumedriver", plugingetter.Acquire); err == nil {
|
|
| 57 |
+ t.Fatal("exepcted error when getting plugin that doesn't match the passed in capability")
|
|
| 58 |
+ } |
|
| 59 |
+ |
|
| 60 |
+ if refs := p.GetRefCount(); refs != 0 {
|
|
| 61 |
+ t.Fatalf("reference count should be 0, got: %d", refs)
|
|
| 62 |
+ } |
|
| 63 |
+} |
| ... | ... |
@@ -11,6 +11,7 @@ import ( |
| 11 | 11 |
getter "github.com/docker/docker/pkg/plugingetter" |
| 12 | 12 |
"github.com/docker/docker/volume" |
| 13 | 13 |
"github.com/pkg/errors" |
| 14 |
+ "github.com/sirupsen/logrus" |
|
| 14 | 15 |
) |
| 15 | 16 |
|
| 16 | 17 |
// currently created by hand. generation tool would generate this like: |
| ... | ... |
@@ -130,6 +131,12 @@ func lookup(name string, mode int) (volume.Driver, error) {
|
| 130 | 130 |
|
| 131 | 131 |
d := NewVolumeDriver(p.Name(), p.BasePath(), p.Client()) |
| 132 | 132 |
if err := validateDriver(d); err != nil {
|
| 133 |
+ if mode > 0 {
|
|
| 134 |
+ // Undo any reference count changes from the initial `Get` |
|
| 135 |
+ if _, err := drivers.plugingetter.Get(name, extName, mode*-1); err != nil {
|
|
| 136 |
+ logrus.WithError(err).WithField("action", "validate-driver").WithField("plugin", name).Error("error releasing reference to plugin")
|
|
| 137 |
+ } |
|
| 138 |
+ } |
|
| 133 | 139 |
return nil, err |
| 134 | 140 |
} |
| 135 | 141 |
|
| ... | ... |
@@ -169,9 +176,9 @@ func CreateDriver(name string) (volume.Driver, error) {
|
| 169 | 169 |
return lookup(name, getter.Acquire) |
| 170 | 170 |
} |
| 171 | 171 |
|
| 172 |
-// RemoveDriver returns a volume driver by its name and decrements RefCount.. |
|
| 172 |
+// ReleaseDriver returns a volume driver by its name and decrements RefCount.. |
|
| 173 | 173 |
// If the driver is empty, it looks for the local driver. |
| 174 |
-func RemoveDriver(name string) (volume.Driver, error) {
|
|
| 174 |
+func ReleaseDriver(name string) (volume.Driver, error) {
|
|
| 175 | 175 |
if name == "" {
|
| 176 | 176 |
name = volume.DefaultDriverName |
| 177 | 177 |
} |
| ... | ... |
@@ -145,7 +145,7 @@ func (s *VolumeStore) Purge(name string) {
|
| 145 | 145 |
s.globalLock.Lock() |
| 146 | 146 |
v, exists := s.names[name] |
| 147 | 147 |
if exists {
|
| 148 |
- if _, err := volumedrivers.RemoveDriver(v.DriverName()); err != nil {
|
|
| 148 |
+ if _, err := volumedrivers.ReleaseDriver(v.DriverName()); err != nil {
|
|
| 149 | 149 |
logrus.Errorf("Error dereferencing volume driver: %v", err)
|
| 150 | 150 |
} |
| 151 | 151 |
} |
| ... | ... |
@@ -409,6 +409,9 @@ func (s *VolumeStore) create(name, driverName string, opts, labels map[string]st |
| 409 | 409 |
} |
| 410 | 410 |
v, err = vd.Create(name, opts) |
| 411 | 411 |
if err != nil {
|
| 412 |
+ if _, err := volumedrivers.ReleaseDriver(driverName); err != nil {
|
|
| 413 |
+ logrus.WithError(err).WithField("driver", driverName).Error("Error releasing reference to volume driver")
|
|
| 414 |
+ } |
|
| 412 | 415 |
return nil, err |
| 413 | 416 |
} |
| 414 | 417 |
s.globalLock.Lock() |