Browse code

Abstract distribution interfaces from image specific types

Move configurations into a single file.
Abstract download manager in pull config.
Add supports for schema2 only and schema2 type checking.
Add interface for providing push layers.
Abstract image store to generically handle configurations.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>
(cherry picked from commit 3c7676a057a4c0103895f793e407dc6736df139a)

Derek McGowan authored on 2016/12/17 04:19:05
Showing 13 changed files
... ...
@@ -74,7 +74,7 @@ func runPull(dockerCli *command.DockerCli, opts pullOptions) error {
74 74
 		err = imagePullPrivileged(ctx, dockerCli, authConfig, distributionRef.String(), requestPrivilege, opts.all)
75 75
 	}
76 76
 	if err != nil {
77
-		if strings.Contains(err.Error(), "target is a plugin") {
77
+		if strings.Contains(err.Error(), "target is plugin") {
78 78
 			return errors.New(err.Error() + " - Use `docker plugin install`")
79 79
 		}
80 80
 		return err
... ...
@@ -89,15 +89,18 @@ func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference.
89 89
 	}()
90 90
 
91 91
 	imagePullConfig := &distribution.ImagePullConfig{
92
-		MetaHeaders:      metaHeaders,
93
-		AuthConfig:       authConfig,
94
-		ProgressOutput:   progress.ChanOutput(progressChan),
95
-		RegistryService:  daemon.RegistryService,
96
-		ImageEventLogger: daemon.LogImageEvent,
97
-		MetadataStore:    daemon.distributionMetadataStore,
98
-		ImageStore:       daemon.imageStore,
99
-		ReferenceStore:   daemon.referenceStore,
100
-		DownloadManager:  daemon.downloadManager,
92
+		Config: distribution.Config{
93
+			MetaHeaders:      metaHeaders,
94
+			AuthConfig:       authConfig,
95
+			ProgressOutput:   progress.ChanOutput(progressChan),
96
+			RegistryService:  daemon.RegistryService,
97
+			ImageEventLogger: daemon.LogImageEvent,
98
+			MetadataStore:    daemon.distributionMetadataStore,
99
+			ImageStore:       distribution.NewImageConfigStoreFromStore(daemon.imageStore),
100
+			ReferenceStore:   daemon.referenceStore,
101
+		},
102
+		DownloadManager: daemon.downloadManager,
103
+		Schema2Types:    distribution.ImageTypes,
101 104
 	}
102 105
 
103 106
 	err := distribution.Pull(ctx, ref, imagePullConfig)
... ...
@@ -3,6 +3,7 @@ package daemon
3 3
 import (
4 4
 	"io"
5 5
 
6
+	"github.com/docker/distribution/manifest/schema2"
6 7
 	"github.com/docker/docker/api/types"
7 8
 	"github.com/docker/docker/distribution"
8 9
 	"github.com/docker/docker/pkg/progress"
... ...
@@ -38,17 +39,20 @@ func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHead
38 38
 	}()
39 39
 
40 40
 	imagePushConfig := &distribution.ImagePushConfig{
41
-		MetaHeaders:      metaHeaders,
42
-		AuthConfig:       authConfig,
43
-		ProgressOutput:   progress.ChanOutput(progressChan),
44
-		RegistryService:  daemon.RegistryService,
45
-		ImageEventLogger: daemon.LogImageEvent,
46
-		MetadataStore:    daemon.distributionMetadataStore,
47
-		LayerStore:       daemon.layerStore,
48
-		ImageStore:       daemon.imageStore,
49
-		ReferenceStore:   daemon.referenceStore,
50
-		TrustKey:         daemon.trustKey,
51
-		UploadManager:    daemon.uploadManager,
41
+		Config: distribution.Config{
42
+			MetaHeaders:      metaHeaders,
43
+			AuthConfig:       authConfig,
44
+			ProgressOutput:   progress.ChanOutput(progressChan),
45
+			RegistryService:  daemon.RegistryService,
46
+			ImageEventLogger: daemon.LogImageEvent,
47
+			MetadataStore:    daemon.distributionMetadataStore,
48
+			ImageStore:       distribution.NewImageConfigStoreFromStore(daemon.imageStore),
49
+			ReferenceStore:   daemon.referenceStore,
50
+		},
51
+		ConfigMediaType: schema2.MediaTypeImageConfig,
52
+		LayerStore:      distribution.NewLayerProviderFromStore(daemon.layerStore),
53
+		TrustKey:        daemon.trustKey,
54
+		UploadManager:   daemon.uploadManager,
52 55
 	}
53 56
 
54 57
 	err = distribution.Push(ctx, ref, imagePushConfig)
55 58
new file mode 100644
... ...
@@ -0,0 +1,233 @@
0
+package distribution
1
+
2
+import (
3
+	"encoding/json"
4
+	"fmt"
5
+	"io"
6
+	"runtime"
7
+
8
+	"github.com/docker/distribution"
9
+	"github.com/docker/distribution/digest"
10
+	"github.com/docker/distribution/manifest/schema2"
11
+	"github.com/docker/docker/api/types"
12
+	"github.com/docker/docker/distribution/metadata"
13
+	"github.com/docker/docker/distribution/xfer"
14
+	"github.com/docker/docker/image"
15
+	"github.com/docker/docker/layer"
16
+	"github.com/docker/docker/pkg/progress"
17
+	"github.com/docker/docker/reference"
18
+	"github.com/docker/docker/registry"
19
+	"github.com/docker/libtrust"
20
+	"golang.org/x/net/context"
21
+)
22
+
23
+// Config stores configuration for communicating
24
+// with a registry.
25
+type Config struct {
26
+	// MetaHeaders stores HTTP headers with metadata about the image
27
+	MetaHeaders map[string][]string
28
+	// AuthConfig holds authentication credentials for authenticating with
29
+	// the registry.
30
+	AuthConfig *types.AuthConfig
31
+	// ProgressOutput is the interface for showing the status of the pull
32
+	// operation.
33
+	ProgressOutput progress.Output
34
+	// RegistryService is the registry service to use for TLS configuration
35
+	// and endpoint lookup.
36
+	RegistryService registry.Service
37
+	// ImageEventLogger notifies events for a given image
38
+	ImageEventLogger func(id, name, action string)
39
+	// MetadataStore is the storage backend for distribution-specific
40
+	// metadata.
41
+	MetadataStore metadata.Store
42
+	// ImageStore manages images.
43
+	ImageStore ImageConfigStore
44
+	// ReferenceStore manages tags. This value is optional, when excluded
45
+	// content will not be tagged.
46
+	ReferenceStore reference.Store
47
+	// RequireSchema2 ensures that only schema2 manifests are used.
48
+	RequireSchema2 bool
49
+}
50
+
51
+// ImagePullConfig stores pull configuration.
52
+type ImagePullConfig struct {
53
+	Config
54
+
55
+	// DownloadManager manages concurrent pulls.
56
+	DownloadManager RootFSDownloadManager
57
+	// Schema2Types is the valid schema2 configuration types allowed
58
+	// by the pull operation.
59
+	Schema2Types []string
60
+}
61
+
62
+// ImagePushConfig stores push configuration.
63
+type ImagePushConfig struct {
64
+	Config
65
+
66
+	// ConfigMediaType is the configuration media type for
67
+	// schema2 manifests.
68
+	ConfigMediaType string
69
+	// LayerStore manages layers.
70
+	LayerStore PushLayerProvider
71
+	// TrustKey is the private key for legacy signatures. This is typically
72
+	// an ephemeral key, since these signatures are no longer verified.
73
+	TrustKey libtrust.PrivateKey
74
+	// UploadManager dispatches uploads.
75
+	UploadManager *xfer.LayerUploadManager
76
+}
77
+
78
+// ImageConfigStore handles storing and getting image configurations
79
+// by digest. Allows getting an image configurations rootfs from the
80
+// configuration.
81
+type ImageConfigStore interface {
82
+	Put([]byte) (digest.Digest, error)
83
+	Get(digest.Digest) ([]byte, error)
84
+	RootFSFromConfig([]byte) (*image.RootFS, error)
85
+}
86
+
87
+// PushLayerProvider provides layers to be pushed by ChainID.
88
+type PushLayerProvider interface {
89
+	Get(layer.ChainID) (PushLayer, error)
90
+}
91
+
92
+// PushLayer is a pushable layer with metadata about the layer
93
+// and access to the content of the layer.
94
+type PushLayer interface {
95
+	ChainID() layer.ChainID
96
+	DiffID() layer.DiffID
97
+	Parent() PushLayer
98
+	Open() (io.ReadCloser, error)
99
+	Size() (int64, error)
100
+	MediaType() string
101
+	Release()
102
+}
103
+
104
+// RootFSDownloadManager handles downloading of the rootfs
105
+type RootFSDownloadManager interface {
106
+	// Download downloads the layers into the given initial rootfs and
107
+	// returns the final rootfs.
108
+	// Given progress output to track download progress
109
+	// Returns function to release download resources
110
+	Download(ctx context.Context, initialRootFS image.RootFS, layers []xfer.DownloadDescriptor, progressOutput progress.Output) (image.RootFS, func(), error)
111
+}
112
+
113
+type imageConfigStore struct {
114
+	image.Store
115
+}
116
+
117
+// NewImageConfigStoreFromStore returns an ImageConfigStore backed
118
+// by an image.Store for container images.
119
+func NewImageConfigStoreFromStore(is image.Store) ImageConfigStore {
120
+	return &imageConfigStore{
121
+		Store: is,
122
+	}
123
+}
124
+
125
+func (s *imageConfigStore) Put(c []byte) (digest.Digest, error) {
126
+	id, err := s.Store.Create(c)
127
+	return digest.Digest(id), err
128
+}
129
+
130
+func (s *imageConfigStore) Get(d digest.Digest) ([]byte, error) {
131
+	img, err := s.Store.Get(image.IDFromDigest(d))
132
+	if err != nil {
133
+		return nil, err
134
+	}
135
+	return img.RawJSON(), nil
136
+}
137
+
138
+func (s *imageConfigStore) RootFSFromConfig(c []byte) (*image.RootFS, error) {
139
+	var unmarshalledConfig image.Image
140
+	if err := json.Unmarshal(c, &unmarshalledConfig); err != nil {
141
+		return nil, err
142
+	}
143
+
144
+	// fail immediately on windows
145
+	if runtime.GOOS == "windows" && unmarshalledConfig.OS == "linux" {
146
+		return nil, fmt.Errorf("image operating system %q cannot be used on this platform", unmarshalledConfig.OS)
147
+	}
148
+
149
+	return unmarshalledConfig.RootFS, nil
150
+}
151
+
152
+type storeLayerProvider struct {
153
+	ls layer.Store
154
+}
155
+
156
+// NewLayerProviderFromStore returns a layer provider backed by
157
+// an instance of LayerStore. Only getting layers as gzipped
158
+// tars is supported.
159
+func NewLayerProviderFromStore(ls layer.Store) PushLayerProvider {
160
+	return &storeLayerProvider{
161
+		ls: ls,
162
+	}
163
+}
164
+
165
+func (p *storeLayerProvider) Get(lid layer.ChainID) (PushLayer, error) {
166
+	if lid == "" {
167
+		return &storeLayer{
168
+			Layer: layer.EmptyLayer,
169
+		}, nil
170
+	}
171
+	l, err := p.ls.Get(lid)
172
+	if err != nil {
173
+		return nil, err
174
+	}
175
+
176
+	sl := storeLayer{
177
+		Layer: l,
178
+		ls:    p.ls,
179
+	}
180
+	if d, ok := l.(distribution.Describable); ok {
181
+		return &describableStoreLayer{
182
+			storeLayer:  sl,
183
+			describable: d,
184
+		}, nil
185
+	}
186
+
187
+	return &sl, nil
188
+}
189
+
190
+type storeLayer struct {
191
+	layer.Layer
192
+	ls layer.Store
193
+}
194
+
195
+func (l *storeLayer) Parent() PushLayer {
196
+	p := l.Layer.Parent()
197
+	if p == nil {
198
+		return nil
199
+	}
200
+	return &storeLayer{
201
+		Layer: p,
202
+		ls:    l.ls,
203
+	}
204
+}
205
+
206
+func (l *storeLayer) Open() (io.ReadCloser, error) {
207
+	return l.Layer.TarStream()
208
+}
209
+
210
+func (l *storeLayer) Size() (int64, error) {
211
+	return l.Layer.DiffSize()
212
+}
213
+
214
+func (l *storeLayer) MediaType() string {
215
+	// layer store always returns uncompressed tars
216
+	return schema2.MediaTypeUncompressedLayer
217
+}
218
+
219
+func (l *storeLayer) Release() {
220
+	if l.ls != nil {
221
+		layer.ReleaseAndLog(l.ls, l.Layer)
222
+	}
223
+}
224
+
225
+type describableStoreLayer struct {
226
+	storeLayer
227
+	describable distribution.Describable
228
+}
229
+
230
+func (l *describableStoreLayer) Descriptor() distribution.Descriptor {
231
+	return l.describable.Descriptor()
232
+}
... ...
@@ -6,42 +6,13 @@ import (
6 6
 	"github.com/Sirupsen/logrus"
7 7
 	"github.com/docker/distribution/digest"
8 8
 	"github.com/docker/docker/api"
9
-	"github.com/docker/docker/api/types"
10 9
 	"github.com/docker/docker/distribution/metadata"
11
-	"github.com/docker/docker/distribution/xfer"
12
-	"github.com/docker/docker/image"
13 10
 	"github.com/docker/docker/pkg/progress"
14 11
 	"github.com/docker/docker/reference"
15 12
 	"github.com/docker/docker/registry"
16 13
 	"golang.org/x/net/context"
17 14
 )
18 15
 
19
-// ImagePullConfig stores pull configuration.
20
-type ImagePullConfig struct {
21
-	// MetaHeaders stores HTTP headers with metadata about the image
22
-	MetaHeaders map[string][]string
23
-	// AuthConfig holds authentication credentials for authenticating with
24
-	// the registry.
25
-	AuthConfig *types.AuthConfig
26
-	// ProgressOutput is the interface for showing the status of the pull
27
-	// operation.
28
-	ProgressOutput progress.Output
29
-	// RegistryService is the registry service to use for TLS configuration
30
-	// and endpoint lookup.
31
-	RegistryService registry.Service
32
-	// ImageEventLogger notifies events for a given image
33
-	ImageEventLogger func(id, name, action string)
34
-	// MetadataStore is the storage backend for distribution-specific
35
-	// metadata.
36
-	MetadataStore metadata.Store
37
-	// ImageStore manages images.
38
-	ImageStore image.Store
39
-	// ReferenceStore manages tags.
40
-	ReferenceStore reference.Store
41
-	// DownloadManager manages concurrent pulls.
42
-	DownloadManager *xfer.LayerDownloadManager
43
-}
44
-
45 16
 // Puller is an interface that abstracts pulling for different API versions.
46 17
 type Puller interface {
47 18
 	// Pull tries to pull the image referenced by `tag`
... ...
@@ -117,6 +88,10 @@ func Pull(ctx context.Context, ref reference.Named, imagePullConfig *ImagePullCo
117 117
 		confirmedTLSRegistries = make(map[string]struct{})
118 118
 	)
119 119
 	for _, endpoint := range endpoints {
120
+		if imagePullConfig.RequireSchema2 && endpoint.Version == registry.APIVersion1 {
121
+			continue
122
+		}
123
+
120 124
 		if confirmedV2 && endpoint.Version == registry.APIVersion1 {
121 125
 			logrus.Debugf("Skipping v1 endpoint %s because v2 registry was detected", endpoint.URL)
122 126
 			continue
... ...
@@ -243,13 +243,15 @@ func (p *v1Puller) pullImage(ctx context.Context, v1ID, endpoint string, localNa
243 243
 		return err
244 244
 	}
245 245
 
246
-	imageID, err := p.config.ImageStore.Create(config)
246
+	imageID, err := p.config.ImageStore.Put(config)
247 247
 	if err != nil {
248 248
 		return err
249 249
 	}
250 250
 
251
-	if err := p.config.ReferenceStore.AddTag(localNameRef, imageID.Digest(), true); err != nil {
252
-		return err
251
+	if p.config.ReferenceStore != nil {
252
+		if err := p.config.ReferenceStore.AddTag(localNameRef, imageID, true); err != nil {
253
+			return err
254
+		}
253 255
 	}
254 256
 
255 257
 	return nil
... ...
@@ -33,9 +33,8 @@ import (
33 33
 )
34 34
 
35 35
 var (
36
-	errRootFSMismatch  = errors.New("layers from manifest don't match image configuration")
37
-	errMediaTypePlugin = errors.New("target is a plugin")
38
-	errRootFSInvalid   = errors.New("invalid rootfs in image configuration")
36
+	errRootFSMismatch = errors.New("layers from manifest don't match image configuration")
37
+	errRootFSInvalid  = errors.New("invalid rootfs in image configuration")
39 38
 )
40 39
 
41 40
 // ImageConfigPullError is an error pulling the image config blob
... ...
@@ -355,8 +354,19 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdat
355 355
 	}
356 356
 
357 357
 	if m, ok := manifest.(*schema2.DeserializedManifest); ok {
358
-		if m.Manifest.Config.MediaType == schema2.MediaTypePluginConfig {
359
-			return false, errMediaTypePlugin
358
+		var allowedMediatype bool
359
+		for _, t := range p.config.Schema2Types {
360
+			if m.Manifest.Config.MediaType == t {
361
+				allowedMediatype = true
362
+				break
363
+			}
364
+		}
365
+		if !allowedMediatype {
366
+			configClass := mediaTypeClasses[m.Manifest.Config.MediaType]
367
+			if configClass == "" {
368
+				configClass = "unknown"
369
+			}
370
+			return false, fmt.Errorf("target is %s", configClass)
360 371
 		}
361 372
 	}
362 373
 
... ...
@@ -374,6 +384,9 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdat
374 374
 
375 375
 	switch v := manifest.(type) {
376 376
 	case *schema1.SignedManifest:
377
+		if p.config.RequireSchema2 {
378
+			return false, fmt.Errorf("invalid manifest: not schema2")
379
+		}
377 380
 		id, manifestDigest, err = p.pullSchema1(ctx, ref, v)
378 381
 		if err != nil {
379 382
 			return false, err
... ...
@@ -394,25 +407,27 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdat
394 394
 
395 395
 	progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
396 396
 
397
-	oldTagID, err := p.config.ReferenceStore.Get(ref)
398
-	if err == nil {
399
-		if oldTagID == id {
400
-			return false, addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id)
401
-		}
402
-	} else if err != reference.ErrDoesNotExist {
403
-		return false, err
404
-	}
405
-
406
-	if canonical, ok := ref.(reference.Canonical); ok {
407
-		if err = p.config.ReferenceStore.AddDigest(canonical, id, true); err != nil {
408
-			return false, err
409
-		}
410
-	} else {
411
-		if err = addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id); err != nil {
397
+	if p.config.ReferenceStore != nil {
398
+		oldTagID, err := p.config.ReferenceStore.Get(ref)
399
+		if err == nil {
400
+			if oldTagID == id {
401
+				return false, addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id)
402
+			}
403
+		} else if err != reference.ErrDoesNotExist {
412 404
 			return false, err
413 405
 		}
414
-		if err = p.config.ReferenceStore.AddTag(ref, id, true); err != nil {
415
-			return false, err
406
+
407
+		if canonical, ok := ref.(reference.Canonical); ok {
408
+			if err = p.config.ReferenceStore.AddDigest(canonical, id, true); err != nil {
409
+				return false, err
410
+			}
411
+		} else {
412
+			if err = addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id); err != nil {
413
+				return false, err
414
+			}
415
+			if err = p.config.ReferenceStore.AddTag(ref, id, true); err != nil {
416
+				return false, err
417
+			}
416 418
 		}
417 419
 	}
418 420
 	return true, nil
... ...
@@ -481,14 +496,14 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverif
481 481
 		return "", "", err
482 482
 	}
483 483
 
484
-	imageID, err := p.config.ImageStore.Create(config)
484
+	imageID, err := p.config.ImageStore.Put(config)
485 485
 	if err != nil {
486 486
 		return "", "", err
487 487
 	}
488 488
 
489 489
 	manifestDigest = digest.FromBytes(unverifiedManifest.Canonical)
490 490
 
491
-	return imageID.Digest(), manifestDigest, nil
491
+	return imageID, manifestDigest, nil
492 492
 }
493 493
 
494 494
 func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest) (id digest.Digest, manifestDigest digest.Digest, err error) {
... ...
@@ -498,7 +513,7 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
498 498
 	}
499 499
 
500 500
 	target := mfst.Target()
501
-	if _, err := p.config.ImageStore.Get(image.IDFromDigest(target.Digest)); err == nil {
501
+	if _, err := p.config.ImageStore.Get(target.Digest); err == nil {
502 502
 		// If the image already exists locally, no need to pull
503 503
 		// anything.
504 504
 		return target.Digest, manifestDigest, nil
... ...
@@ -537,9 +552,9 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
537 537
 	}()
538 538
 
539 539
 	var (
540
-		configJSON         []byte       // raw serialized image config
541
-		unmarshalledConfig image.Image  // deserialized image config
542
-		downloadRootFS     image.RootFS // rootFS to use for registering layers.
540
+		configJSON       []byte        // raw serialized image config
541
+		downloadedRootFS *image.RootFS // rootFS from registered layers
542
+		configRootFS     *image.RootFS // rootFS from configuration
543 543
 	)
544 544
 
545 545
 	// https://github.com/docker/docker/issues/24766 - Err on the side of caution,
... ...
@@ -551,84 +566,87 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
551 551
 	// check to block Windows images being pulled on Linux is implemented, it
552 552
 	// may be necessary to perform the same type of serialisation.
553 553
 	if runtime.GOOS == "windows" {
554
-		configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan)
554
+		configJSON, configRootFS, err = receiveConfig(p.config.ImageStore, configChan, errChan)
555 555
 		if err != nil {
556 556
 			return "", "", err
557 557
 		}
558 558
 
559
-		if unmarshalledConfig.RootFS == nil {
559
+		if configRootFS == nil {
560 560
 			return "", "", errRootFSInvalid
561 561
 		}
562
-
563
-		if unmarshalledConfig.OS == "linux" {
564
-			return "", "", fmt.Errorf("image operating system %q cannot be used on this platform", unmarshalledConfig.OS)
565
-		}
566 562
 	}
567 563
 
568
-	downloadRootFS = *image.NewRootFS()
569
-
570
-	rootFS, release, err := p.config.DownloadManager.Download(ctx, downloadRootFS, descriptors, p.config.ProgressOutput)
571
-	if err != nil {
572
-		if configJSON != nil {
573
-			// Already received the config
574
-			return "", "", err
575
-		}
576
-		select {
577
-		case err = <-errChan:
578
-			return "", "", err
579
-		default:
580
-			cancel()
564
+	if p.config.DownloadManager != nil {
565
+		downloadRootFS := *image.NewRootFS()
566
+		rootFS, release, err := p.config.DownloadManager.Download(ctx, downloadRootFS, descriptors, p.config.ProgressOutput)
567
+		if err != nil {
568
+			if configJSON != nil {
569
+				// Already received the config
570
+				return "", "", err
571
+			}
581 572
 			select {
582
-			case <-configChan:
583
-			case <-errChan:
573
+			case err = <-errChan:
574
+				return "", "", err
575
+			default:
576
+				cancel()
577
+				select {
578
+				case <-configChan:
579
+				case <-errChan:
580
+				}
581
+				return "", "", err
584 582
 			}
585
-			return "", "", err
586 583
 		}
584
+		if release != nil {
585
+			defer release()
586
+		}
587
+
588
+		downloadedRootFS = &rootFS
587 589
 	}
588
-	defer release()
589 590
 
590 591
 	if configJSON == nil {
591
-		configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan)
592
+		configJSON, configRootFS, err = receiveConfig(p.config.ImageStore, configChan, errChan)
592 593
 		if err != nil {
593 594
 			return "", "", err
594 595
 		}
595 596
 
596
-		if unmarshalledConfig.RootFS == nil {
597
+		if configRootFS == nil {
597 598
 			return "", "", errRootFSInvalid
598 599
 		}
599 600
 	}
600 601
 
601
-	// The DiffIDs returned in rootFS MUST match those in the config.
602
-	// Otherwise the image config could be referencing layers that aren't
603
-	// included in the manifest.
604
-	if len(rootFS.DiffIDs) != len(unmarshalledConfig.RootFS.DiffIDs) {
605
-		return "", "", errRootFSMismatch
606
-	}
607
-
608
-	for i := range rootFS.DiffIDs {
609
-		if rootFS.DiffIDs[i] != unmarshalledConfig.RootFS.DiffIDs[i] {
602
+	if downloadedRootFS != nil {
603
+		// The DiffIDs returned in rootFS MUST match those in the config.
604
+		// Otherwise the image config could be referencing layers that aren't
605
+		// included in the manifest.
606
+		if len(downloadedRootFS.DiffIDs) != len(configRootFS.DiffIDs) {
610 607
 			return "", "", errRootFSMismatch
611 608
 		}
609
+
610
+		for i := range downloadedRootFS.DiffIDs {
611
+			if downloadedRootFS.DiffIDs[i] != configRootFS.DiffIDs[i] {
612
+				return "", "", errRootFSMismatch
613
+			}
614
+		}
612 615
 	}
613 616
 
614
-	imageID, err := p.config.ImageStore.Create(configJSON)
617
+	imageID, err := p.config.ImageStore.Put(configJSON)
615 618
 	if err != nil {
616 619
 		return "", "", err
617 620
 	}
618 621
 
619
-	return imageID.Digest(), manifestDigest, nil
622
+	return imageID, manifestDigest, nil
620 623
 }
621 624
 
622
-func receiveConfig(configChan <-chan []byte, errChan <-chan error) ([]byte, image.Image, error) {
625
+func receiveConfig(s ImageConfigStore, configChan <-chan []byte, errChan <-chan error) ([]byte, *image.RootFS, error) {
623 626
 	select {
624 627
 	case configJSON := <-configChan:
625
-		var unmarshalledConfig image.Image
626
-		if err := json.Unmarshal(configJSON, &unmarshalledConfig); err != nil {
627
-			return nil, image.Image{}, err
628
+		rootfs, err := s.RootFSFromConfig(configJSON)
629
+		if err != nil {
630
+			return nil, nil, err
628 631
 		}
629
-		return configJSON, unmarshalledConfig, nil
632
+		return configJSON, rootfs, nil
630 633
 	case err := <-errChan:
631
-		return nil, image.Image{}, err
634
+		return nil, nil, err
632 635
 		// Don't need a case for ctx.Done in the select because cancellation
633 636
 		// will trigger an error in p.pullSchema2ImageConfig.
634 637
 	}
... ...
@@ -7,49 +7,13 @@ import (
7 7
 	"io"
8 8
 
9 9
 	"github.com/Sirupsen/logrus"
10
-	"github.com/docker/docker/api/types"
11 10
 	"github.com/docker/docker/distribution/metadata"
12
-	"github.com/docker/docker/distribution/xfer"
13
-	"github.com/docker/docker/image"
14
-	"github.com/docker/docker/layer"
15 11
 	"github.com/docker/docker/pkg/progress"
16 12
 	"github.com/docker/docker/reference"
17 13
 	"github.com/docker/docker/registry"
18
-	"github.com/docker/libtrust"
19 14
 	"golang.org/x/net/context"
20 15
 )
21 16
 
22
-// ImagePushConfig stores push configuration.
23
-type ImagePushConfig struct {
24
-	// MetaHeaders store HTTP headers with metadata about the image
25
-	MetaHeaders map[string][]string
26
-	// AuthConfig holds authentication credentials for authenticating with
27
-	// the registry.
28
-	AuthConfig *types.AuthConfig
29
-	// ProgressOutput is the interface for showing the status of the push
30
-	// operation.
31
-	ProgressOutput progress.Output
32
-	// RegistryService is the registry service to use for TLS configuration
33
-	// and endpoint lookup.
34
-	RegistryService registry.Service
35
-	// ImageEventLogger notifies events for a given image
36
-	ImageEventLogger func(id, name, action string)
37
-	// MetadataStore is the storage backend for distribution-specific
38
-	// metadata.
39
-	MetadataStore metadata.Store
40
-	// LayerStore manages layers.
41
-	LayerStore layer.Store
42
-	// ImageStore manages images.
43
-	ImageStore image.Store
44
-	// ReferenceStore manages tags.
45
-	ReferenceStore reference.Store
46
-	// TrustKey is the private key for legacy signatures. This is typically
47
-	// an ephemeral key, since these signatures are no longer verified.
48
-	TrustKey libtrust.PrivateKey
49
-	// UploadManager dispatches uploads.
50
-	UploadManager *xfer.LayerUploadManager
51
-}
52
-
53 17
 // Pusher is an interface that abstracts pushing for different API versions.
54 18
 type Pusher interface {
55 19
 	// Push tries to push the image configured at the creation of Pusher.
... ...
@@ -127,6 +91,9 @@ func Push(ctx context.Context, ref reference.Named, imagePushConfig *ImagePushCo
127 127
 	)
128 128
 
129 129
 	for _, endpoint := range endpoints {
130
+		if imagePushConfig.RequireSchema2 && endpoint.Version == registry.APIVersion1 {
131
+			continue
132
+		}
130 133
 		if confirmedV2 && endpoint.Version == registry.APIVersion1 {
131 134
 			logrus.Debugf("Skipping v1 endpoint %s because v2 registry was detected", endpoint.URL)
132 135
 			continue
... ...
@@ -137,7 +137,7 @@ func newV1DependencyImage(l layer.Layer, parent *v1DependencyImage) (*v1Dependen
137 137
 }
138 138
 
139 139
 // Retrieve the all the images to be uploaded in the correct order
140
-func (p *v1Pusher) getImageList() (imageList []v1Image, tagsByImage map[image.ID][]string, referencedLayers []layer.Layer, err error) {
140
+func (p *v1Pusher) getImageList() (imageList []v1Image, tagsByImage map[image.ID][]string, referencedLayers []PushLayer, err error) {
141 141
 	tagsByImage = make(map[image.ID][]string)
142 142
 
143 143
 	// Ignore digest references
... ...
@@ -202,24 +202,30 @@ func (p *v1Pusher) getImageList() (imageList []v1Image, tagsByImage map[image.ID
202 202
 	return
203 203
 }
204 204
 
205
-func (p *v1Pusher) imageListForTag(imgID image.ID, dependenciesSeen map[layer.ChainID]*v1DependencyImage, referencedLayers *[]layer.Layer) (imageListForThisTag []v1Image, err error) {
206
-	img, err := p.config.ImageStore.Get(imgID)
205
+func (p *v1Pusher) imageListForTag(imgID image.ID, dependenciesSeen map[layer.ChainID]*v1DependencyImage, referencedLayers *[]PushLayer) (imageListForThisTag []v1Image, err error) {
206
+	ics, ok := p.config.ImageStore.(*imageConfigStore)
207
+	if !ok {
208
+		return nil, fmt.Errorf("only image store images supported for v1 push")
209
+	}
210
+	img, err := ics.Store.Get(imgID)
207 211
 	if err != nil {
208 212
 		return nil, err
209 213
 	}
210 214
 
211 215
 	topLayerID := img.RootFS.ChainID()
212 216
 
213
-	var l layer.Layer
214
-	if topLayerID == "" {
215
-		l = layer.EmptyLayer
216
-	} else {
217
-		l, err = p.config.LayerStore.Get(topLayerID)
218
-		*referencedLayers = append(*referencedLayers, l)
219
-		if err != nil {
220
-			return nil, fmt.Errorf("failed to get top layer from image: %v", err)
221
-		}
217
+	pl, err := p.config.LayerStore.Get(topLayerID)
218
+	*referencedLayers = append(*referencedLayers, pl)
219
+	if err != nil {
220
+		return nil, fmt.Errorf("failed to get top layer from image: %v", err)
221
+	}
222
+
223
+	// V1 push is deprecated, only support existing layerstore layers
224
+	lsl, ok := pl.(*storeLayer)
225
+	if !ok {
226
+		return nil, fmt.Errorf("only layer store layers supported for v1 push")
222 227
 	}
228
+	l := lsl.Layer
223 229
 
224 230
 	dependencyImages, parent, err := generateDependencyImages(l.Parent(), dependenciesSeen)
225 231
 	if err != nil {
... ...
@@ -371,7 +377,7 @@ func (p *v1Pusher) pushRepository(ctx context.Context) error {
371 371
 	imgList, tags, referencedLayers, err := p.getImageList()
372 372
 	defer func() {
373 373
 		for _, l := range referencedLayers {
374
-			p.config.LayerStore.Release(l)
374
+			l.Release()
375 375
 		}
376 376
 	}()
377 377
 	if err != nil {
... ...
@@ -20,7 +20,6 @@ import (
20 20
 	"github.com/docker/distribution/registry/client"
21 21
 	"github.com/docker/docker/distribution/metadata"
22 22
 	"github.com/docker/docker/distribution/xfer"
23
-	"github.com/docker/docker/image"
24 23
 	"github.com/docker/docker/layer"
25 24
 	"github.com/docker/docker/pkg/ioutils"
26 25
 	"github.com/docker/docker/pkg/progress"
... ...
@@ -123,23 +122,21 @@ func (p *v2Pusher) pushV2Repository(ctx context.Context) (err error) {
123 123
 func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, id digest.Digest) error {
124 124
 	logrus.Debugf("Pushing repository: %s", ref.String())
125 125
 
126
-	img, err := p.config.ImageStore.Get(image.IDFromDigest(id))
126
+	imgConfig, err := p.config.ImageStore.Get(id)
127 127
 	if err != nil {
128 128
 		return fmt.Errorf("could not find image from tag %s: %v", ref.String(), err)
129 129
 	}
130 130
 
131
-	var l layer.Layer
131
+	rootfs, err := p.config.ImageStore.RootFSFromConfig(imgConfig)
132
+	if err != nil {
133
+		return fmt.Errorf("unable to get rootfs for image %s: %s", ref.String(), err)
134
+	}
132 135
 
133
-	topLayerID := img.RootFS.ChainID()
134
-	if topLayerID == "" {
135
-		l = layer.EmptyLayer
136
-	} else {
137
-		l, err = p.config.LayerStore.Get(topLayerID)
138
-		if err != nil {
139
-			return fmt.Errorf("failed to get top layer from image: %v", err)
140
-		}
141
-		defer layer.ReleaseAndLog(p.config.LayerStore, l)
136
+	l, err := p.config.LayerStore.Get(rootfs.ChainID())
137
+	if err != nil {
138
+		return fmt.Errorf("failed to get top layer from image: %v", err)
142 139
 	}
140
+	defer l.Release()
143 141
 
144 142
 	hmacKey, err := metadata.ComputeV2MetadataHMACKey(p.config.AuthConfig)
145 143
 	if err != nil {
... ...
@@ -158,7 +155,7 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, id
158 158
 	}
159 159
 
160 160
 	// Loop bounds condition is to avoid pushing the base layer on Windows.
161
-	for i := 0; i < len(img.RootFS.DiffIDs); i++ {
161
+	for i := 0; i < len(rootfs.DiffIDs); i++ {
162 162
 		descriptor := descriptorTemplate
163 163
 		descriptor.layer = l
164 164
 		descriptor.checkedDigests = make(map[digest.Digest]struct{})
... ...
@@ -172,7 +169,7 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, id
172 172
 	}
173 173
 
174 174
 	// Try schema2 first
175
-	builder := schema2.NewManifestBuilder(p.repo.Blobs(ctx), img.RawJSON())
175
+	builder := schema2.NewManifestBuilder(p.repo.Blobs(ctx), p.config.ConfigMediaType, imgConfig)
176 176
 	manifest, err := manifestFromBuilder(ctx, builder, descriptors)
177 177
 	if err != nil {
178 178
 		return err
... ...
@@ -185,7 +182,7 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, id
185 185
 
186 186
 	putOptions := []distribution.ManifestServiceOption{distribution.WithTag(ref.Tag())}
187 187
 	if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
188
-		if runtime.GOOS == "windows" {
188
+		if runtime.GOOS == "windows" || p.config.TrustKey == nil || p.config.RequireSchema2 {
189 189
 			logrus.Warnf("failed to upload schema2 manifest: %v", err)
190 190
 			return err
191 191
 		}
... ...
@@ -196,7 +193,7 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, id
196 196
 		if err != nil {
197 197
 			return err
198 198
 		}
199
-		builder = schema1.NewConfigManifestBuilder(p.repo.Blobs(ctx), p.config.TrustKey, manifestRef, img.RawJSON())
199
+		builder = schema1.NewConfigManifestBuilder(p.repo.Blobs(ctx), p.config.TrustKey, manifestRef, imgConfig)
200 200
 		manifest, err = manifestFromBuilder(ctx, builder, descriptors)
201 201
 		if err != nil {
202 202
 			return err
... ...
@@ -246,7 +243,7 @@ func manifestFromBuilder(ctx context.Context, builder distribution.ManifestBuild
246 246
 }
247 247
 
248 248
 type v2PushDescriptor struct {
249
-	layer             layer.Layer
249
+	layer             PushLayer
250 250
 	v2MetadataService metadata.V2MetadataService
251 251
 	hmacKey           []byte
252 252
 	repoInfo          reference.Named
... ...
@@ -425,26 +422,32 @@ func (pd *v2PushDescriptor) uploadUsingSession(
425 425
 	diffID layer.DiffID,
426 426
 	layerUpload distribution.BlobWriter,
427 427
 ) (distribution.Descriptor, error) {
428
-	arch, err := pd.layer.TarStream()
429
-	if err != nil {
430
-		return distribution.Descriptor{}, xfer.DoNotRetry{Err: err}
431
-	}
432
-
433
-	// don't care if this fails; best effort
434
-	size, _ := pd.layer.DiffSize()
435
-
436
-	reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, arch), progressOutput, size, pd.ID(), "Pushing")
437
-	compressedReader, compressionDone := compress(reader)
438
-	defer func() {
428
+	var reader io.ReadCloser
429
+
430
+	contentReader, err := pd.layer.Open()
431
+	size, _ := pd.layer.Size()
432
+
433
+	reader = progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, contentReader), progressOutput, size, pd.ID(), "Pushing")
434
+
435
+	switch m := pd.layer.MediaType(); m {
436
+	case schema2.MediaTypeUncompressedLayer:
437
+		compressedReader, compressionDone := compress(reader)
438
+		defer func(closer io.Closer) {
439
+			closer.Close()
440
+			<-compressionDone
441
+		}(reader)
442
+		reader = compressedReader
443
+	case schema2.MediaTypeLayer:
444
+	default:
439 445
 		reader.Close()
440
-		<-compressionDone
441
-	}()
446
+		return distribution.Descriptor{}, fmt.Errorf("unsupported layer media type %s", m)
447
+	}
442 448
 
443 449
 	digester := digest.Canonical.New()
444
-	tee := io.TeeReader(compressedReader, digester.Hash())
450
+	tee := io.TeeReader(reader, digester.Hash())
445 451
 
446 452
 	nn, err := layerUpload.ReadFrom(tee)
447
-	compressedReader.Close()
453
+	reader.Close()
448 454
 	if err != nil {
449 455
 		return distribution.Descriptor{}, retryOnError(err)
450 456
 	}
... ...
@@ -568,8 +571,8 @@ attempts:
568 568
 // repository and whether the check shall be done also with digests mapped to different repositories. The
569 569
 // decision is based on layer size. The smaller the layer, the fewer attempts shall be made because the cost
570 570
 // of upload does not outweigh a latency.
571
-func getMaxMountAndExistenceCheckAttempts(layer layer.Layer) (maxMountAttempts, maxExistenceCheckAttempts int, checkOtherRepositories bool) {
572
-	size, err := layer.DiffSize()
571
+func getMaxMountAndExistenceCheckAttempts(layer PushLayer) (maxMountAttempts, maxExistenceCheckAttempts int, checkOtherRepositories bool) {
572
+	size, err := layer.Size()
573 573
 	switch {
574 574
 	// big blob
575 575
 	case size > middleLayerMaximumSize:
... ...
@@ -387,9 +387,11 @@ func TestLayerAlreadyExists(t *testing.T) {
387 387
 		ctx := context.Background()
388 388
 		ms := &mockV2MetadataService{}
389 389
 		pd := &v2PushDescriptor{
390
-			hmacKey:           []byte(tc.hmacKey),
391
-			repoInfo:          repoInfo,
392
-			layer:             layer.EmptyLayer,
390
+			hmacKey:  []byte(tc.hmacKey),
391
+			repoInfo: repoInfo,
392
+			layer: &storeLayer{
393
+				Layer: layer.EmptyLayer,
394
+			},
393 395
 			repo:              repo,
394 396
 			v2MetadataService: ms,
395 397
 			pushState:         &pushState{remoteLayers: make(map[layer.DiffID]distribution.Descriptor)},
... ...
@@ -7,6 +7,7 @@ import (
7 7
 	"time"
8 8
 
9 9
 	"github.com/docker/distribution"
10
+	"github.com/docker/distribution/manifest/schema2"
10 11
 	distreference "github.com/docker/distribution/reference"
11 12
 	"github.com/docker/distribution/registry/client"
12 13
 	"github.com/docker/distribution/registry/client/auth"
... ...
@@ -18,6 +19,34 @@ import (
18 18
 	"golang.org/x/net/context"
19 19
 )
20 20
 
21
+// ImageTypes represents the schema2 config types for images
22
+var ImageTypes = []string{
23
+	schema2.MediaTypeImageConfig,
24
+	// Handle unexpected values from https://github.com/docker/distribution/issues/1621
25
+	"application/octet-stream",
26
+	// Treat defaulted values as images, newer types cannot be implied
27
+	"",
28
+}
29
+
30
+// PluginTypes represents the schema2 config types for plugins
31
+var PluginTypes = []string{
32
+	schema2.MediaTypePluginConfig,
33
+}
34
+
35
+var mediaTypeClasses map[string]string
36
+
37
+func init() {
38
+	// initialize media type classes with all know types for
39
+	// plugin
40
+	mediaTypeClasses = map[string]string{}
41
+	for _, t := range ImageTypes {
42
+		mediaTypeClasses[t] = "image"
43
+	}
44
+	for _, t := range PluginTypes {
45
+		mediaTypeClasses[t] = "plugin"
46
+	}
47
+}
48
+
21 49
 // NewV2Repository returns a repository (v2 only). It creates an HTTP transport
22 50
 // providing timeout settings and authentication support, and also verifies the
23 51
 // remote API version.
... ...
@@ -70,10 +70,13 @@ func testTokenPassThru(t *testing.T, ts *httptest.Server) {
70 70
 		Official: false,
71 71
 	}
72 72
 	imagePullConfig := &ImagePullConfig{
73
-		MetaHeaders: http.Header{},
74
-		AuthConfig: &types.AuthConfig{
75
-			RegistryToken: secretRegistryToken,
73
+		Config: Config{
74
+			MetaHeaders: http.Header{},
75
+			AuthConfig: &types.AuthConfig{
76
+				RegistryToken: secretRegistryToken,
77
+			},
76 78
 		},
79
+		Schema2Types: ImageTypes,
77 80
 	}
78 81
 	puller, err := newPuller(endpoint, repoInfo, imagePullConfig)
79 82
 	if err != nil {