This allows graphdrivers to declare that they can reproduce the original
diff stream for a layer. If they do so, the layer store will not use
tar-split processing, but will still verify the digest on layer export.
This makes it easier to experiment with non-default diff formats.
Signed-off-by: Alfred Landrum <alfred.landrum@docker.com>
| ... | ... |
@@ -112,6 +112,23 @@ type Driver interface {
|
| 112 | 112 |
DiffDriver |
| 113 | 113 |
} |
| 114 | 114 |
|
| 115 |
+// Capabilities defines a list of capabilities a driver may implement. |
|
| 116 |
+// These capabilities are not required; however, they do determine how a |
|
| 117 |
+// graphdriver can be used. |
|
| 118 |
+type Capabilities struct {
|
|
| 119 |
+ // Flags that this driver is capable of reproducing exactly equivalent |
|
| 120 |
+ // diffs for read-only layers. If set, clients can rely on the driver |
|
| 121 |
+ // for consistent tar streams, and avoid extra processing to account |
|
| 122 |
+ // for potential differences (eg: the layer store's use of tar-split). |
|
| 123 |
+ ReproducesExactDiffs bool |
|
| 124 |
+} |
|
| 125 |
+ |
|
| 126 |
+// CapabilityDriver is the interface for layered file system drivers that |
|
| 127 |
+// can report on their Capabilities. |
|
| 128 |
+type CapabilityDriver interface {
|
|
| 129 |
+ Capabilities() Capabilities |
|
| 130 |
+} |
|
| 131 |
+ |
|
| 115 | 132 |
// DiffGetterDriver is the interface for layered file system drivers that |
| 116 | 133 |
// provide a specialized function for getting file contents for tar-split. |
| 117 | 134 |
type DiffGetterDriver interface {
|
| ... | ... |
@@ -38,6 +38,6 @@ func newPluginDriver(name string, pl plugingetter.CompatPlugin, config Options) |
| 38 | 38 |
} |
| 39 | 39 |
} |
| 40 | 40 |
} |
| 41 |
- proxy := &graphDriverProxy{name, pl}
|
|
| 41 |
+ proxy := &graphDriverProxy{name, pl, Capabilities{}}
|
|
| 42 | 42 |
return proxy, proxy.Init(filepath.Join(home, name), config.DriverOptions, config.UIDMaps, config.GIDMaps) |
| 43 | 43 |
} |
| ... | ... |
@@ -9,11 +9,13 @@ import ( |
| 9 | 9 |
"github.com/docker/docker/pkg/archive" |
| 10 | 10 |
"github.com/docker/docker/pkg/idtools" |
| 11 | 11 |
"github.com/docker/docker/pkg/plugingetter" |
| 12 |
+ "github.com/docker/docker/pkg/plugins" |
|
| 12 | 13 |
) |
| 13 | 14 |
|
| 14 | 15 |
type graphDriverProxy struct {
|
| 15 | 16 |
name string |
| 16 | 17 |
p plugingetter.CompatPlugin |
| 18 |
+ caps Capabilities |
|
| 17 | 19 |
} |
| 18 | 20 |
|
| 19 | 21 |
type graphDriverRequest struct {
|
| ... | ... |
@@ -24,13 +26,14 @@ type graphDriverRequest struct {
|
| 24 | 24 |
} |
| 25 | 25 |
|
| 26 | 26 |
type graphDriverResponse struct {
|
| 27 |
- Err string `json:",omitempty"` |
|
| 28 |
- Dir string `json:",omitempty"` |
|
| 29 |
- Exists bool `json:",omitempty"` |
|
| 30 |
- Status [][2]string `json:",omitempty"` |
|
| 31 |
- Changes []archive.Change `json:",omitempty"` |
|
| 32 |
- Size int64 `json:",omitempty"` |
|
| 33 |
- Metadata map[string]string `json:",omitempty"` |
|
| 27 |
+ Err string `json:",omitempty"` |
|
| 28 |
+ Dir string `json:",omitempty"` |
|
| 29 |
+ Exists bool `json:",omitempty"` |
|
| 30 |
+ Status [][2]string `json:",omitempty"` |
|
| 31 |
+ Changes []archive.Change `json:",omitempty"` |
|
| 32 |
+ Size int64 `json:",omitempty"` |
|
| 33 |
+ Metadata map[string]string `json:",omitempty"` |
|
| 34 |
+ Capabilities Capabilities `json:",omitempty"` |
|
| 34 | 35 |
} |
| 35 | 36 |
|
| 36 | 37 |
type graphDriverInitRequest struct {
|
| ... | ... |
@@ -60,13 +63,33 @@ func (d *graphDriverProxy) Init(home string, opts []string, uidMaps, gidMaps []i |
| 60 | 60 |
if ret.Err != "" {
|
| 61 | 61 |
return errors.New(ret.Err) |
| 62 | 62 |
} |
| 63 |
+ caps, err := d.fetchCaps() |
|
| 64 |
+ if err != nil {
|
|
| 65 |
+ return err |
|
| 66 |
+ } |
|
| 67 |
+ d.caps = caps |
|
| 63 | 68 |
return nil |
| 64 | 69 |
} |
| 65 | 70 |
|
| 71 |
+func (d *graphDriverProxy) fetchCaps() (Capabilities, error) {
|
|
| 72 |
+ args := &graphDriverRequest{}
|
|
| 73 |
+ var ret graphDriverResponse |
|
| 74 |
+ if err := d.p.Client().Call("GraphDriver.Capabilities", args, &ret); err != nil {
|
|
| 75 |
+ if !plugins.IsNotFound(err) {
|
|
| 76 |
+ return Capabilities{}, err
|
|
| 77 |
+ } |
|
| 78 |
+ } |
|
| 79 |
+ return ret.Capabilities, nil |
|
| 80 |
+} |
|
| 81 |
+ |
|
| 66 | 82 |
func (d *graphDriverProxy) String() string {
|
| 67 | 83 |
return d.name |
| 68 | 84 |
} |
| 69 | 85 |
|
| 86 |
+func (d *graphDriverProxy) Capabilities() Capabilities {
|
|
| 87 |
+ return d.caps |
|
| 88 |
+} |
|
| 89 |
+ |
|
| 70 | 90 |
func (d *graphDriverProxy) CreateReadWrite(id, parent string, opts *CreateOpts) error {
|
| 71 | 91 |
return d.create("GraphDriver.CreateReadWrite", id, parent, opts)
|
| 72 | 92 |
} |
| ... | ... |
@@ -84,6 +84,29 @@ The request also includes a list of UID and GID mappings, structed as follows: |
| 84 | 84 |
Respond with a non-empty string error if an error occurred. |
| 85 | 85 |
|
| 86 | 86 |
|
| 87 |
+### /GraphDriver.Capabilities |
|
| 88 |
+ |
|
| 89 |
+**Request**: |
|
| 90 |
+```json |
|
| 91 |
+{}
|
|
| 92 |
+``` |
|
| 93 |
+ |
|
| 94 |
+Get behavioral characteristics of the graph driver. If a plugin does not handle |
|
| 95 |
+this request, the engine will use default values for all capabilities. |
|
| 96 |
+ |
|
| 97 |
+**Response**: |
|
| 98 |
+```json |
|
| 99 |
+{
|
|
| 100 |
+ "ReproducesExactDiffs": false, |
|
| 101 |
+} |
|
| 102 |
+``` |
|
| 103 |
+ |
|
| 104 |
+Respond with values of capabilities: |
|
| 105 |
+ |
|
| 106 |
+* **ReproducesExactDiffs** Defaults to false. Flags that this driver is capable |
|
| 107 |
+of reproducing exactly equivalent diffs for read-only filesystem layers. |
|
| 108 |
+ |
|
| 109 |
+ |
|
| 87 | 110 |
### /GraphDriver.Create |
| 88 | 111 |
|
| 89 | 112 |
**Request**: |
| ... | ... |
@@ -34,6 +34,8 @@ type layerStore struct {
|
| 34 | 34 |
|
| 35 | 35 |
mounts map[string]*mountedLayer |
| 36 | 36 |
mountL sync.Mutex |
| 37 |
+ |
|
| 38 |
+ useTarSplit bool |
|
| 37 | 39 |
} |
| 38 | 40 |
|
| 39 | 41 |
// StoreOptions are the options used to create a new Store instance |
| ... | ... |
@@ -74,11 +76,17 @@ func NewStoreFromOptions(options StoreOptions) (Store, error) {
|
| 74 | 74 |
// metadata store and graph driver. The metadata store will be used to restore |
| 75 | 75 |
// the Store. |
| 76 | 76 |
func NewStoreFromGraphDriver(store MetadataStore, driver graphdriver.Driver) (Store, error) {
|
| 77 |
+ caps := graphdriver.Capabilities{}
|
|
| 78 |
+ if capDriver, ok := driver.(graphdriver.CapabilityDriver); ok {
|
|
| 79 |
+ caps = capDriver.Capabilities() |
|
| 80 |
+ } |
|
| 81 |
+ |
|
| 77 | 82 |
ls := &layerStore{
|
| 78 |
- store: store, |
|
| 79 |
- driver: driver, |
|
| 80 |
- layerMap: map[ChainID]*roLayer{},
|
|
| 81 |
- mounts: map[string]*mountedLayer{},
|
|
| 83 |
+ store: store, |
|
| 84 |
+ driver: driver, |
|
| 85 |
+ layerMap: map[ChainID]*roLayer{},
|
|
| 86 |
+ mounts: map[string]*mountedLayer{},
|
|
| 87 |
+ useTarSplit: !caps.ReproducesExactDiffs, |
|
| 82 | 88 |
} |
| 83 | 89 |
|
| 84 | 90 |
ids, mounts, err := store.List() |
| ... | ... |
@@ -207,18 +215,21 @@ func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent stri |
| 207 | 207 |
digester := digest.Canonical.Digester() |
| 208 | 208 |
tr := io.TeeReader(ts, digester.Hash()) |
| 209 | 209 |
|
| 210 |
- tsw, err := tx.TarSplitWriter(true) |
|
| 211 |
- if err != nil {
|
|
| 212 |
- return err |
|
| 213 |
- } |
|
| 214 |
- metaPacker := storage.NewJSONPacker(tsw) |
|
| 215 |
- defer tsw.Close() |
|
| 210 |
+ rdr := tr |
|
| 211 |
+ if ls.useTarSplit {
|
|
| 212 |
+ tsw, err := tx.TarSplitWriter(true) |
|
| 213 |
+ if err != nil {
|
|
| 214 |
+ return err |
|
| 215 |
+ } |
|
| 216 |
+ metaPacker := storage.NewJSONPacker(tsw) |
|
| 217 |
+ defer tsw.Close() |
|
| 216 | 218 |
|
| 217 |
- // we're passing nil here for the file putter, because the ApplyDiff will |
|
| 218 |
- // handle the extraction of the archive |
|
| 219 |
- rdr, err := asm.NewInputTarStream(tr, metaPacker, nil) |
|
| 220 |
- if err != nil {
|
|
| 221 |
- return err |
|
| 219 |
+ // we're passing nil here for the file putter, because the ApplyDiff will |
|
| 220 |
+ // handle the extraction of the archive |
|
| 221 |
+ rdr, err = asm.NewInputTarStream(tr, metaPacker, nil) |
|
| 222 |
+ if err != nil {
|
|
| 223 |
+ return err |
|
| 224 |
+ } |
|
| 222 | 225 |
} |
| 223 | 226 |
|
| 224 | 227 |
applySize, err := ls.driver.ApplyDiff(layer.cacheID, parent, rdr) |
| ... | ... |
@@ -640,6 +651,34 @@ func (ls *layerStore) initMount(graphID, parent, mountLabel string, initFunc Mou |
| 640 | 640 |
return initID, nil |
| 641 | 641 |
} |
| 642 | 642 |
|
| 643 |
+func (ls *layerStore) getTarStream(rl *roLayer) (io.ReadCloser, error) {
|
|
| 644 |
+ if !ls.useTarSplit {
|
|
| 645 |
+ var parentCacheID string |
|
| 646 |
+ if rl.parent != nil {
|
|
| 647 |
+ parentCacheID = rl.parent.cacheID |
|
| 648 |
+ } |
|
| 649 |
+ |
|
| 650 |
+ return ls.driver.Diff(rl.cacheID, parentCacheID) |
|
| 651 |
+ } |
|
| 652 |
+ |
|
| 653 |
+ r, err := ls.store.TarSplitReader(rl.chainID) |
|
| 654 |
+ if err != nil {
|
|
| 655 |
+ return nil, err |
|
| 656 |
+ } |
|
| 657 |
+ |
|
| 658 |
+ pr, pw := io.Pipe() |
|
| 659 |
+ go func() {
|
|
| 660 |
+ err := ls.assembleTarTo(rl.cacheID, r, nil, pw) |
|
| 661 |
+ if err != nil {
|
|
| 662 |
+ pw.CloseWithError(err) |
|
| 663 |
+ } else {
|
|
| 664 |
+ pw.Close() |
|
| 665 |
+ } |
|
| 666 |
+ }() |
|
| 667 |
+ |
|
| 668 |
+ return pr, nil |
|
| 669 |
+} |
|
| 670 |
+ |
|
| 643 | 671 |
func (ls *layerStore) assembleTarTo(graphID string, metadata io.ReadCloser, size *int64, w io.Writer) error {
|
| 644 | 672 |
diffDriver, ok := ls.driver.(graphdriver.DiffGetterDriver) |
| 645 | 673 |
if !ok {
|
| ... | ... |
@@ -24,25 +24,16 @@ type roLayer struct {
|
| 24 | 24 |
// TarStream for roLayer guarantees that the data that is produced is the exact |
| 25 | 25 |
// data that the layer was registered with. |
| 26 | 26 |
func (rl *roLayer) TarStream() (io.ReadCloser, error) {
|
| 27 |
- r, err := rl.layerStore.store.TarSplitReader(rl.chainID) |
|
| 27 |
+ rc, err := rl.layerStore.getTarStream(rl) |
|
| 28 | 28 |
if err != nil {
|
| 29 | 29 |
return nil, err |
| 30 | 30 |
} |
| 31 | 31 |
|
| 32 |
- pr, pw := io.Pipe() |
|
| 33 |
- go func() {
|
|
| 34 |
- err := rl.layerStore.assembleTarTo(rl.cacheID, r, nil, pw) |
|
| 35 |
- if err != nil {
|
|
| 36 |
- pw.CloseWithError(err) |
|
| 37 |
- } else {
|
|
| 38 |
- pw.Close() |
|
| 39 |
- } |
|
| 40 |
- }() |
|
| 41 |
- rc, err := newVerifiedReadCloser(pr, digest.Digest(rl.diffID)) |
|
| 32 |
+ vrc, err := newVerifiedReadCloser(rc, digest.Digest(rl.diffID)) |
|
| 42 | 33 |
if err != nil {
|
| 43 | 34 |
return nil, err |
| 44 | 35 |
} |
| 45 |
- return rc, nil |
|
| 36 |
+ return vrc, nil |
|
| 46 | 37 |
} |
| 47 | 38 |
|
| 48 | 39 |
// TarStreamFrom does not make any guarantees to the correctness of the produced |