Browse code

Add support for volume scopes

This is similar to network scopes where a volume can either be `local`
or `global`. A `global` volume is one that exists across the entire
cluster where as a `local` volume exists on a single engine.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Brian Goff authored on 2016/04/12 00:17:52
Showing 13 changed files
... ...
@@ -745,7 +745,9 @@ func configureVolumes(config *Config, rootUID, rootGID int) (*store.VolumeStore,
745 745
 		return nil, err
746 746
 	}
747 747
 
748
-	volumedrivers.Register(volumesDriver, volumesDriver.Name())
748
+	if !volumedrivers.Register(volumesDriver, volumesDriver.Name()) {
749
+		return nil, fmt.Errorf("local volume driver could not be registered")
750
+	}
749 751
 	return store.New(config.Root)
750 752
 }
751 753
 
... ...
@@ -27,11 +27,13 @@ func volumeToAPIType(v volume.Volume) *types.Volume {
27 27
 		Name:   v.Name(),
28 28
 		Driver: v.DriverName(),
29 29
 	}
30
-	if v, ok := v.(interface {
31
-		Labels() map[string]string
32
-	}); ok {
30
+	if v, ok := v.(volume.LabeledVolume); ok {
33 31
 		tv.Labels = v.Labels()
34 32
 	}
33
+
34
+	if v, ok := v.(volume.ScopedVolume); ok {
35
+		tv.Scope = v.Scope()
36
+	}
35 37
 	return tv
36 38
 }
37 39
 
... ...
@@ -20,6 +20,7 @@ documentation](plugins.md) for more information.
20 20
 ### 1.12.0
21 21
 
22 22
 - Add `Status` field to `VolumeDriver.Get` response ([#21006](https://github.com/docker/docker/pull/21006#))
23
+- Add `VolumeDriver.Capabilities` to get capabilities of the volume driver([#22077](https://github.com/docker/docker/pull/22077))
23 24
 
24 25
 ### 1.10.0
25 26
 
... ...
@@ -236,3 +237,29 @@ Get the list of volumes registered with the plugin.
236 236
 ```
237 237
 
238 238
 Respond with a string error if an error occurred.
239
+
240
+### /VolumeDriver.Capabilities
241
+
242
+**Request**:
243
+```json
244
+{}
245
+```
246
+
247
+Get the list of capabilities the driver supports.
248
+The driver is not required to implement this endpoint, however in such cases
249
+the default values will be taken.
250
+
251
+**Response**:
252
+```json
253
+{
254
+  "Capabilities": {
255
+    "Scope": "global"
256
+  }
257
+}
258
+```
259
+
260
+Supported scopes are `global` and `local`. Any other value in `Scope` will be
261
+ignored and assumed to be `local`. Scope allows cluster managers to handle the
262
+volume differently, for instance with a scope of `global`, the cluster manager
263
+knows it only needs to create the volume once instead of on every engine. More
264
+capabilities may be added in the future.
... ...
@@ -16,6 +16,7 @@ import (
16 16
 	"time"
17 17
 
18 18
 	"github.com/docker/docker/pkg/integration/checker"
19
+	"github.com/docker/docker/volume"
19 20
 	"github.com/docker/engine-api/types"
20 21
 	"github.com/go-check/check"
21 22
 )
... ...
@@ -35,6 +36,7 @@ type eventCounter struct {
35 35
 	paths       int
36 36
 	lists       int
37 37
 	gets        int
38
+	caps        int
38 39
 }
39 40
 
40 41
 type DockerExternalVolumeSuite struct {
... ...
@@ -225,6 +227,18 @@ func (s *DockerExternalVolumeSuite) SetUpSuite(c *check.C) {
225 225
 		send(w, nil)
226 226
 	})
227 227
 
228
+	mux.HandleFunc("/VolumeDriver.Capabilities", func(w http.ResponseWriter, r *http.Request) {
229
+		s.ec.caps++
230
+
231
+		_, err := read(r.Body)
232
+		if err != nil {
233
+			send(w, err)
234
+			return
235
+		}
236
+
237
+		send(w, `{"Capabilities": { "Scope": "global" }}`)
238
+	})
239
+
228 240
 	err := os.MkdirAll("/etc/docker/plugins", 0755)
229 241
 	c.Assert(err, checker.IsNil)
230 242
 
... ...
@@ -491,3 +505,18 @@ func (s *DockerExternalVolumeSuite) TestExternalVolumeDriverMountID(c *check.C)
491 491
 	c.Assert(err, checker.IsNil, check.Commentf(out))
492 492
 	c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "")
493 493
 }
494
+
495
+// Check that VolumeDriver.Capabilities gets called, and only called once
496
+func (s *DockerExternalVolumeSuite) TestExternalVolumeDriverCapabilities(c *check.C) {
497
+	c.Assert(s.d.Start(), checker.IsNil)
498
+	c.Assert(s.ec.caps, checker.Equals, 0)
499
+
500
+	for i := 0; i < 3; i++ {
501
+		out, err := s.d.Cmd("volume", "create", "-d", "test-external-volume-driver", "--name", fmt.Sprintf("test%d", i))
502
+		c.Assert(err, checker.IsNil, check.Commentf(out))
503
+		c.Assert(s.ec.caps, checker.Equals, 1)
504
+		out, err = s.d.Cmd("volume", "inspect", "--format={{.Scope}}", fmt.Sprintf("test%d", i))
505
+		c.Assert(err, checker.IsNil)
506
+		c.Assert(strings.TrimSpace(out), checker.Equals, volume.GlobalScope)
507
+	}
508
+}
... ...
@@ -78,7 +78,7 @@ func main() {
78 78
 
79 79
 	errorOut("parser error", generatedTempl.Execute(&buf, analysis))
80 80
 	src, err := format.Source(buf.Bytes())
81
-	errorOut("error formating generated source:\n"+buf.String(), err)
81
+	errorOut("error formatting generated source:\n"+buf.String(), err)
82 82
 	errorOut("error writing file", ioutil.WriteFile(*outputFile, src, 0644))
83 83
 }
84 84
 
... ...
@@ -1,14 +1,22 @@
1 1
 package volumedrivers
2 2
 
3 3
 import (
4
-	"fmt"
4
+	"errors"
5
+	"strings"
5 6
 
7
+	"github.com/Sirupsen/logrus"
6 8
 	"github.com/docker/docker/volume"
7 9
 )
8 10
 
11
+var (
12
+	errInvalidScope = errors.New("invalid scope")
13
+	errNoSuchVolume = errors.New("no such volume")
14
+)
15
+
9 16
 type volumeDriverAdapter struct {
10
-	name  string
11
-	proxy *volumeDriverProxy
17
+	name         string
18
+	capabilities *volume.Capability
19
+	proxy        *volumeDriverProxy
12 20
 }
13 21
 
14 22
 func (a *volumeDriverAdapter) Name() string {
... ...
@@ -56,7 +64,7 @@ func (a *volumeDriverAdapter) Get(name string) (volume.Volume, error) {
56 56
 
57 57
 	// plugin may have returned no volume and no error
58 58
 	if v == nil {
59
-		return nil, fmt.Errorf("no such volume")
59
+		return nil, errNoSuchVolume
60 60
 	}
61 61
 
62 62
 	return &volumeAdapter{
... ...
@@ -68,6 +76,38 @@ func (a *volumeDriverAdapter) Get(name string) (volume.Volume, error) {
68 68
 	}, nil
69 69
 }
70 70
 
71
+func (a *volumeDriverAdapter) Scope() string {
72
+	cap := a.getCapabilities()
73
+	return cap.Scope
74
+}
75
+
76
+func (a *volumeDriverAdapter) getCapabilities() volume.Capability {
77
+	if a.capabilities != nil {
78
+		return *a.capabilities
79
+	}
80
+	cap, err := a.proxy.Capabilities()
81
+	if err != nil {
82
+		// `GetCapabilities` is a not a required endpoint.
83
+		// On error assume it's a local-only driver
84
+		logrus.Warnf("Volume driver %s returned an error while trying to query it's capabilities, using default capabilties: %v", a.name, err)
85
+		return volume.Capability{Scope: volume.LocalScope}
86
+	}
87
+
88
+	// don't spam the warn log below just because the plugin didn't provide a scope
89
+	if len(cap.Scope) == 0 {
90
+		cap.Scope = volume.LocalScope
91
+	}
92
+
93
+	cap.Scope = strings.ToLower(cap.Scope)
94
+	if cap.Scope != volume.LocalScope && cap.Scope != volume.GlobalScope {
95
+		logrus.Warnf("Volume driver %q returned an invalid scope: %q", a.Name(), cap.Scope)
96
+		cap.Scope = volume.LocalScope
97
+	}
98
+
99
+	a.capabilities = &cap
100
+	return cap
101
+}
102
+
71 103
 type volumeAdapter struct {
72 104
 	proxy      *volumeDriverProxy
73 105
 	name       string
... ...
@@ -42,6 +42,8 @@ type volumeDriver interface {
42 42
 	List() (volumes []*proxyVolume, err error)
43 43
 	// Get retrieves the volume with the requested name
44 44
 	Get(name string) (volume *proxyVolume, err error)
45
+	// Capabilities gets the list of capabilities of the driver
46
+	Capabilities() (capabilities volume.Capability, err error)
45 47
 }
46 48
 
47 49
 type driverExtpoint struct {
... ...
@@ -64,6 +66,11 @@ func Register(extension volume.Driver, name string) bool {
64 64
 	if exists {
65 65
 		return false
66 66
 	}
67
+
68
+	if err := validateDriver(extension); err != nil {
69
+		return false
70
+	}
71
+
67 72
 	drivers.extensions[name] = extension
68 73
 	return true
69 74
 }
... ...
@@ -107,10 +114,22 @@ func Lookup(name string) (volume.Driver, error) {
107 107
 	}
108 108
 
109 109
 	d := NewVolumeDriver(name, pl.Client)
110
+	if err := validateDriver(d); err != nil {
111
+		return nil, err
112
+	}
113
+
110 114
 	drivers.extensions[name] = d
111 115
 	return d, nil
112 116
 }
113 117
 
118
+func validateDriver(vd volume.Driver) error {
119
+	scope := vd.Scope()
120
+	if scope != volume.LocalScope && scope != volume.GlobalScope {
121
+		return fmt.Errorf("Driver %q provided an invalid capability scope: %s", vd.Name(), scope)
122
+	}
123
+	return nil
124
+}
125
+
114 126
 // GetDriver returns a volume driver by its name.
115 127
 // If the driver is empty, it looks for the local driver.
116 128
 func GetDriver(name string) (volume.Driver, error) {
... ...
@@ -2,7 +2,10 @@
2 2
 
3 3
 package volumedrivers
4 4
 
5
-import "errors"
5
+import (
6
+	"errors"
7
+	"github.com/docker/docker/volume"
8
+)
6 9
 
7 10
 type client interface {
8 11
 	Call(string, interface{}, interface{}) error
... ...
@@ -209,3 +212,30 @@ func (pp *volumeDriverProxy) Get(name string) (volume *proxyVolume, err error) {
209 209
 
210 210
 	return
211 211
 }
212
+
213
+type volumeDriverProxyCapabilitiesRequest struct {
214
+}
215
+
216
+type volumeDriverProxyCapabilitiesResponse struct {
217
+	Capabilities volume.Capability
218
+	Err          string
219
+}
220
+
221
+func (pp *volumeDriverProxy) Capabilities() (capabilities volume.Capability, err error) {
222
+	var (
223
+		req volumeDriverProxyCapabilitiesRequest
224
+		ret volumeDriverProxyCapabilitiesResponse
225
+	)
226
+
227
+	if err = pp.Call("VolumeDriver.Capabilities", req, &ret); err != nil {
228
+		return
229
+	}
230
+
231
+	capabilities = ret.Capabilities
232
+
233
+	if ret.Err != "" {
234
+		err = errors.New(ret.Err)
235
+	}
236
+
237
+	return
238
+}
... ...
@@ -52,6 +52,11 @@ func TestVolumeRequestError(t *testing.T) {
52 52
 		fmt.Fprintln(w, `{"Err": "Cannot get volume"}`)
53 53
 	})
54 54
 
55
+	mux.HandleFunc("/VolumeDriver.Capabilities", func(w http.ResponseWriter, r *http.Request) {
56
+		w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json")
57
+		http.Error(w, "error", 500)
58
+	})
59
+
55 60
 	u, _ := url.Parse(server.URL)
56 61
 	client, err := plugins.NewClient("tcp://"+u.Host, tlsconfig.Options{InsecureSkipVerify: true})
57 62
 	if err != nil {
... ...
@@ -119,4 +124,9 @@ func TestVolumeRequestError(t *testing.T) {
119 119
 	if !strings.Contains(err.Error(), "Cannot get volume") {
120 120
 		t.Fatalf("Unexpected error: %v\n", err)
121 121
 	}
122
+
123
+	_, err = driver.Capabilities()
124
+	if err == nil {
125
+		t.Fatal(err)
126
+	}
122 127
 }
... ...
@@ -248,6 +248,11 @@ func (r *Root) Get(name string) (volume.Volume, error) {
248 248
 	return v, nil
249 249
 }
250 250
 
251
+// Scope returns the local volume scope
252
+func (r *Root) Scope() string {
253
+	return volume.LocalScope
254
+}
255
+
251 256
 func (r *Root) validateName(name string) error {
252 257
 	if !volumeNameRegex.MatchString(name) {
253 258
 		return validationError{fmt.Errorf("%q includes invalid characters for a local volume name, only %q are allowed", name, utils.RestrictedNameChars)}
... ...
@@ -25,15 +25,29 @@ type volumeMetadata struct {
25 25
 	Labels map[string]string
26 26
 }
27 27
 
28
-type volumeWithLabels struct {
28
+type volumeWrapper struct {
29 29
 	volume.Volume
30 30
 	labels map[string]string
31
+	scope  string
31 32
 }
32 33
 
33
-func (v volumeWithLabels) Labels() map[string]string {
34
+func (v volumeWrapper) Labels() map[string]string {
34 35
 	return v.labels
35 36
 }
36 37
 
38
+func (v volumeWrapper) Scope() string {
39
+	return v.scope
40
+}
41
+
42
+func (v volumeWrapper) CachedPath() string {
43
+	if vv, ok := v.Volume.(interface {
44
+		CachedPath() string
45
+	}); ok {
46
+		return vv.CachedPath()
47
+	}
48
+	return v.Volume.Path()
49
+}
50
+
37 51
 // New initializes a VolumeStore to keep
38 52
 // reference counting of volumes in the system.
39 53
 func New(rootPath string) (*VolumeStore, error) {
... ...
@@ -166,6 +180,10 @@ func (s *VolumeStore) list() ([]volume.Volume, []string, error) {
166 166
 				chVols <- vols{driverName: d.Name(), err: &OpErr{Err: err, Name: d.Name(), Op: "list"}}
167 167
 				return
168 168
 			}
169
+			for i, v := range vs {
170
+				vs[i] = volumeWrapper{v, s.labels[v.Name()], d.Scope()}
171
+			}
172
+
169 173
 			chVols <- vols{vols: vs}
170 174
 		}(vd)
171 175
 	}
... ...
@@ -291,7 +309,7 @@ func (s *VolumeStore) create(name, driverName string, opts, labels map[string]st
291 291
 		}
292 292
 	}
293 293
 
294
-	return volumeWithLabels{v, labels}, nil
294
+	return volumeWrapper{v, labels, vd.Scope()}, nil
295 295
 }
296 296
 
297 297
 // GetWithRef gets a volume with the given name from the passed in driver and stores the ref
... ...
@@ -313,10 +331,8 @@ func (s *VolumeStore) GetWithRef(name, driverName, ref string) (volume.Volume, e
313 313
 	}
314 314
 
315 315
 	s.setNamed(v, ref)
316
-	if labels, ok := s.labels[name]; ok {
317
-		return volumeWithLabels{v, labels}, nil
318
-	}
319
-	return v, nil
316
+
317
+	return volumeWrapper{v, s.labels[name], vd.Scope()}, nil
320 318
 }
321 319
 
322 320
 // Get looks if a volume with the given name exists and returns it if so
... ...
@@ -376,7 +392,7 @@ func (s *VolumeStore) getVolume(name string) (volume.Volume, error) {
376 376
 		if err != nil {
377 377
 			return nil, err
378 378
 		}
379
-		return volumeWithLabels{vol, labels}, nil
379
+		return volumeWrapper{vol, labels, vd.Scope()}, nil
380 380
 	}
381 381
 
382 382
 	logrus.Debugf("Probing all drivers for volume with name: %s", name)
... ...
@@ -391,7 +407,7 @@ func (s *VolumeStore) getVolume(name string) (volume.Volume, error) {
391 391
 			continue
392 392
 		}
393 393
 
394
-		return volumeWithLabels{v, labels}, nil
394
+		return volumeWrapper{v, labels, d.Scope()}, nil
395 395
 	}
396 396
 	return nil, errNoSuchVolume
397 397
 }
... ...
@@ -412,7 +428,7 @@ func (s *VolumeStore) Remove(v volume.Volume) error {
412 412
 	}
413 413
 
414 414
 	logrus.Debugf("Removing volume reference: driver %s, name %s", v.DriverName(), name)
415
-	vol := withoutLabels(v)
415
+	vol := unwrapVolume(v)
416 416
 	if err := vd.Remove(vol); err != nil {
417 417
 		return &OpErr{Err: err, Name: name, Op: "remove"}
418 418
 	}
... ...
@@ -465,6 +481,9 @@ func (s *VolumeStore) FilterByDriver(name string) ([]volume.Volume, error) {
465 465
 	if err != nil {
466 466
 		return nil, &OpErr{Err: err, Name: name, Op: "list"}
467 467
 	}
468
+	for i, v := range ls {
469
+		ls[i] = volumeWrapper{v, s.labels[v.Name()], vd.Scope()}
470
+	}
468 471
 	return ls, nil
469 472
 }
470 473
 
... ...
@@ -497,8 +516,8 @@ func (s *VolumeStore) filter(vols []volume.Volume, f filterFunc) []volume.Volume
497 497
 	return ls
498 498
 }
499 499
 
500
-func withoutLabels(v volume.Volume) volume.Volume {
501
-	if vol, ok := v.(volumeWithLabels); ok {
500
+func unwrapVolume(v volume.Volume) volume.Volume {
501
+	if vol, ok := v.(volumeWrapper); ok {
502 502
 		return vol.Volume
503 503
 	}
504 504
 
... ...
@@ -109,3 +109,8 @@ func (d *FakeDriver) Get(name string) (volume.Volume, error) {
109 109
 	}
110 110
 	return nil, fmt.Errorf("no such volume")
111 111
 }
112
+
113
+// Scope returns the local scope
114
+func (*FakeDriver) Scope() string {
115
+	return "local"
116
+}
... ...
@@ -13,7 +13,14 @@ import (
13 13
 
14 14
 // DefaultDriverName is the driver name used for the driver
15 15
 // implemented in the local package.
16
-const DefaultDriverName string = "local"
16
+const DefaultDriverName = "local"
17
+
18
+// Scopes define if a volume has is cluster-wide (global) or local only.
19
+// Scopes are returned by the volume driver when it is queried for capabilities and then set on a volume
20
+const (
21
+	LocalScope  = "local"
22
+	GlobalScope = "global"
23
+)
17 24
 
18 25
 // Driver is for creating and removing volumes.
19 26
 type Driver interface {
... ...
@@ -27,6 +34,18 @@ type Driver interface {
27 27
 	List() ([]Volume, error)
28 28
 	// Get retrieves the volume with the requested name
29 29
 	Get(name string) (Volume, error)
30
+	// Scope returns the scope of the driver (e.g. `golbal` or `local`).
31
+	// Scope determines how the driver is handled at a cluster level
32
+	Scope() string
33
+}
34
+
35
+// Capability defines a set of capabilities that a driver is able to handle.
36
+type Capability struct {
37
+	// Scope is the scope of the driver, `global` or `local`
38
+	// A `global` scope indicates that the driver manages volumes across the cluster
39
+	// A `local` scope indicates that the driver only manages volumes resources local to the host
40
+	// Scope is declared by the driver
41
+	Scope string
30 42
 }
31 43
 
32 44
 // Volume is a place to store data. It is backed by a specific driver, and can be mounted.
... ...
@@ -46,6 +65,18 @@ type Volume interface {
46 46
 	Status() map[string]interface{}
47 47
 }
48 48
 
49
+// LabeledVolume wraps a Volume with user-defined labels
50
+type LabeledVolume interface {
51
+	Labels() map[string]string
52
+	Volume
53
+}
54
+
55
+// ScopedVolume wraps a volume with a cluster scope (e.g., `local` or `global`)
56
+type ScopedVolume interface {
57
+	Scope() string
58
+	Volume
59
+}
60
+
49 61
 // MountPoint is the intersection point between a volume and a container. It
50 62
 // specifies which volume is to be used and where inside a container it should
51 63
 // be mounted.