Browse code

Let graphdrivers declare diff stream fidelity

This allows graphdrivers to declare that they can reproduce the original
diff stream for a layer. If they do so, the layer store will not use
tar-split processing, but will still verify the digest on layer export.
This makes it easier to experiment with non-default diff formats.

Signed-off-by: Alfred Landrum <alfred.landrum@docker.com>

Alfred Landrum authored on 2017/03/21 03:38:17
Showing 6 changed files
... ...
@@ -112,6 +112,23 @@ type Driver interface {
112 112
 	DiffDriver
113 113
 }
114 114
 
115
+// Capabilities defines a list of capabilities a driver may implement.
116
+// These capabilities are not required; however, they do determine how a
117
+// graphdriver can be used.
118
+type Capabilities struct {
119
+	// Flags that this driver is capable of reproducing exactly equivalent
120
+	// diffs for read-only layers. If set, clients can rely on the driver
121
+	// for consistent tar streams, and avoid extra processing to account
122
+	// for potential differences (eg: the layer store's use of tar-split).
123
+	ReproducesExactDiffs bool
124
+}
125
+
126
+// CapabilityDriver is the interface for layered file system drivers that
127
+// can report on their Capabilities.
128
+type CapabilityDriver interface {
129
+	Capabilities() Capabilities
130
+}
131
+
115 132
 // DiffGetterDriver is the interface for layered file system drivers that
116 133
 // provide a specialized function for getting file contents for tar-split.
117 134
 type DiffGetterDriver interface {
... ...
@@ -38,6 +38,6 @@ func newPluginDriver(name string, pl plugingetter.CompatPlugin, config Options)
38 38
 			}
39 39
 		}
40 40
 	}
41
-	proxy := &graphDriverProxy{name, pl}
41
+	proxy := &graphDriverProxy{name, pl, Capabilities{}}
42 42
 	return proxy, proxy.Init(filepath.Join(home, name), config.DriverOptions, config.UIDMaps, config.GIDMaps)
43 43
 }
... ...
@@ -9,11 +9,13 @@ import (
9 9
 	"github.com/docker/docker/pkg/archive"
10 10
 	"github.com/docker/docker/pkg/idtools"
11 11
 	"github.com/docker/docker/pkg/plugingetter"
12
+	"github.com/docker/docker/pkg/plugins"
12 13
 )
13 14
 
14 15
 type graphDriverProxy struct {
15 16
 	name string
16 17
 	p    plugingetter.CompatPlugin
18
+	caps Capabilities
17 19
 }
18 20
 
19 21
 type graphDriverRequest struct {
... ...
@@ -24,13 +26,14 @@ type graphDriverRequest struct {
24 24
 }
25 25
 
26 26
 type graphDriverResponse struct {
27
-	Err      string            `json:",omitempty"`
28
-	Dir      string            `json:",omitempty"`
29
-	Exists   bool              `json:",omitempty"`
30
-	Status   [][2]string       `json:",omitempty"`
31
-	Changes  []archive.Change  `json:",omitempty"`
32
-	Size     int64             `json:",omitempty"`
33
-	Metadata map[string]string `json:",omitempty"`
27
+	Err          string            `json:",omitempty"`
28
+	Dir          string            `json:",omitempty"`
29
+	Exists       bool              `json:",omitempty"`
30
+	Status       [][2]string       `json:",omitempty"`
31
+	Changes      []archive.Change  `json:",omitempty"`
32
+	Size         int64             `json:",omitempty"`
33
+	Metadata     map[string]string `json:",omitempty"`
34
+	Capabilities Capabilities      `json:",omitempty"`
34 35
 }
35 36
 
36 37
 type graphDriverInitRequest struct {
... ...
@@ -60,13 +63,33 @@ func (d *graphDriverProxy) Init(home string, opts []string, uidMaps, gidMaps []i
60 60
 	if ret.Err != "" {
61 61
 		return errors.New(ret.Err)
62 62
 	}
63
+	caps, err := d.fetchCaps()
64
+	if err != nil {
65
+		return err
66
+	}
67
+	d.caps = caps
63 68
 	return nil
64 69
 }
65 70
 
71
+func (d *graphDriverProxy) fetchCaps() (Capabilities, error) {
72
+	args := &graphDriverRequest{}
73
+	var ret graphDriverResponse
74
+	if err := d.p.Client().Call("GraphDriver.Capabilities", args, &ret); err != nil {
75
+		if !plugins.IsNotFound(err) {
76
+			return Capabilities{}, err
77
+		}
78
+	}
79
+	return ret.Capabilities, nil
80
+}
81
+
66 82
 func (d *graphDriverProxy) String() string {
67 83
 	return d.name
68 84
 }
69 85
 
86
+func (d *graphDriverProxy) Capabilities() Capabilities {
87
+	return d.caps
88
+}
89
+
70 90
 func (d *graphDriverProxy) CreateReadWrite(id, parent string, opts *CreateOpts) error {
71 91
 	return d.create("GraphDriver.CreateReadWrite", id, parent, opts)
72 92
 }
... ...
@@ -84,6 +84,29 @@ The request also includes a list of UID and GID mappings, structed as follows:
84 84
 Respond with a non-empty string error if an error occurred.
85 85
 
86 86
 
87
+### /GraphDriver.Capabilities
88
+
89
+**Request**:
90
+```json
91
+{}
92
+```
93
+
94
+Get behavioral characteristics of the graph driver. If a plugin does not handle
95
+this request, the engine will use default values for all capabilities.
96
+
97
+**Response**:
98
+```json
99
+{
100
+  "ReproducesExactDiffs": false,
101
+}
102
+```
103
+
104
+Respond with values of capabilities:
105
+
106
+* **ReproducesExactDiffs** Defaults to false. Flags that this driver is capable
107
+of reproducing exactly equivalent diffs for read-only filesystem layers.
108
+
109
+
87 110
 ### /GraphDriver.Create
88 111
 
89 112
 **Request**:
... ...
@@ -34,6 +34,8 @@ type layerStore struct {
34 34
 
35 35
 	mounts map[string]*mountedLayer
36 36
 	mountL sync.Mutex
37
+
38
+	useTarSplit bool
37 39
 }
38 40
 
39 41
 // StoreOptions are the options used to create a new Store instance
... ...
@@ -74,11 +76,17 @@ func NewStoreFromOptions(options StoreOptions) (Store, error) {
74 74
 // metadata store and graph driver. The metadata store will be used to restore
75 75
 // the Store.
76 76
 func NewStoreFromGraphDriver(store MetadataStore, driver graphdriver.Driver) (Store, error) {
77
+	caps := graphdriver.Capabilities{}
78
+	if capDriver, ok := driver.(graphdriver.CapabilityDriver); ok {
79
+		caps = capDriver.Capabilities()
80
+	}
81
+
77 82
 	ls := &layerStore{
78
-		store:    store,
79
-		driver:   driver,
80
-		layerMap: map[ChainID]*roLayer{},
81
-		mounts:   map[string]*mountedLayer{},
83
+		store:       store,
84
+		driver:      driver,
85
+		layerMap:    map[ChainID]*roLayer{},
86
+		mounts:      map[string]*mountedLayer{},
87
+		useTarSplit: !caps.ReproducesExactDiffs,
82 88
 	}
83 89
 
84 90
 	ids, mounts, err := store.List()
... ...
@@ -207,18 +215,21 @@ func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent stri
207 207
 	digester := digest.Canonical.Digester()
208 208
 	tr := io.TeeReader(ts, digester.Hash())
209 209
 
210
-	tsw, err := tx.TarSplitWriter(true)
211
-	if err != nil {
212
-		return err
213
-	}
214
-	metaPacker := storage.NewJSONPacker(tsw)
215
-	defer tsw.Close()
210
+	rdr := tr
211
+	if ls.useTarSplit {
212
+		tsw, err := tx.TarSplitWriter(true)
213
+		if err != nil {
214
+			return err
215
+		}
216
+		metaPacker := storage.NewJSONPacker(tsw)
217
+		defer tsw.Close()
216 218
 
217
-	// we're passing nil here for the file putter, because the ApplyDiff will
218
-	// handle the extraction of the archive
219
-	rdr, err := asm.NewInputTarStream(tr, metaPacker, nil)
220
-	if err != nil {
221
-		return err
219
+		// we're passing nil here for the file putter, because the ApplyDiff will
220
+		// handle the extraction of the archive
221
+		rdr, err = asm.NewInputTarStream(tr, metaPacker, nil)
222
+		if err != nil {
223
+			return err
224
+		}
222 225
 	}
223 226
 
224 227
 	applySize, err := ls.driver.ApplyDiff(layer.cacheID, parent, rdr)
... ...
@@ -640,6 +651,34 @@ func (ls *layerStore) initMount(graphID, parent, mountLabel string, initFunc Mou
640 640
 	return initID, nil
641 641
 }
642 642
 
643
+func (ls *layerStore) getTarStream(rl *roLayer) (io.ReadCloser, error) {
644
+	if !ls.useTarSplit {
645
+		var parentCacheID string
646
+		if rl.parent != nil {
647
+			parentCacheID = rl.parent.cacheID
648
+		}
649
+
650
+		return ls.driver.Diff(rl.cacheID, parentCacheID)
651
+	}
652
+
653
+	r, err := ls.store.TarSplitReader(rl.chainID)
654
+	if err != nil {
655
+		return nil, err
656
+	}
657
+
658
+	pr, pw := io.Pipe()
659
+	go func() {
660
+		err := ls.assembleTarTo(rl.cacheID, r, nil, pw)
661
+		if err != nil {
662
+			pw.CloseWithError(err)
663
+		} else {
664
+			pw.Close()
665
+		}
666
+	}()
667
+
668
+	return pr, nil
669
+}
670
+
643 671
 func (ls *layerStore) assembleTarTo(graphID string, metadata io.ReadCloser, size *int64, w io.Writer) error {
644 672
 	diffDriver, ok := ls.driver.(graphdriver.DiffGetterDriver)
645 673
 	if !ok {
... ...
@@ -24,25 +24,16 @@ type roLayer struct {
24 24
 // TarStream for roLayer guarantees that the data that is produced is the exact
25 25
 // data that the layer was registered with.
26 26
 func (rl *roLayer) TarStream() (io.ReadCloser, error) {
27
-	r, err := rl.layerStore.store.TarSplitReader(rl.chainID)
27
+	rc, err := rl.layerStore.getTarStream(rl)
28 28
 	if err != nil {
29 29
 		return nil, err
30 30
 	}
31 31
 
32
-	pr, pw := io.Pipe()
33
-	go func() {
34
-		err := rl.layerStore.assembleTarTo(rl.cacheID, r, nil, pw)
35
-		if err != nil {
36
-			pw.CloseWithError(err)
37
-		} else {
38
-			pw.Close()
39
-		}
40
-	}()
41
-	rc, err := newVerifiedReadCloser(pr, digest.Digest(rl.diffID))
32
+	vrc, err := newVerifiedReadCloser(rc, digest.Digest(rl.diffID))
42 33
 	if err != nil {
43 34
 		return nil, err
44 35
 	}
45
-	return rc, nil
36
+	return vrc, nil
46 37
 }
47 38
 
48 39
 // TarStreamFrom does not make any guarantees to the correctness of the produced