Browse code

Add schema2 pull support

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

Aaron Lehmann authored on 2015/12/12 08:24:12
Showing 2 changed files
... ...
@@ -13,6 +13,7 @@ import (
13 13
 	"github.com/docker/distribution"
14 14
 	"github.com/docker/distribution/digest"
15 15
 	"github.com/docker/distribution/manifest/schema1"
16
+	"github.com/docker/distribution/manifest/schema2"
16 17
 	"github.com/docker/distribution/registry/api/errcode"
17 18
 	"github.com/docker/distribution/registry/client"
18 19
 	"github.com/docker/docker/distribution/metadata"
... ...
@@ -28,6 +29,8 @@ import (
28 28
 	"golang.org/x/net/context"
29 29
 )
30 30
 
31
+var errRootFSMismatch = errors.New("layers from manifest don't match image configuration")
32
+
31 33
 type v2Puller struct {
32 34
 	blobSumService *metadata.BlobSumService
33 35
 	endpoint       registry.APIEndpoint
... ...
@@ -235,11 +238,19 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdat
235 235
 	logrus.Debugf("Pulling ref from V2 registry: %s", ref.String())
236 236
 	progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+p.repo.Name())
237 237
 
238
-	var imageID image.ID
238
+	var (
239
+		imageID        image.ID
240
+		manifestDigest digest.Digest
241
+	)
239 242
 
240 243
 	switch v := manifest.(type) {
241 244
 	case *schema1.SignedManifest:
242
-		imageID, err = p.pullSchema1(ctx, ref, v)
245
+		imageID, manifestDigest, err = p.pullSchema1(ctx, ref, v)
246
+		if err != nil {
247
+			return false, err
248
+		}
249
+	case *schema2.DeserializedManifest:
250
+		imageID, manifestDigest, err = p.pullSchema2(ctx, ref, v)
243 251
 		if err != nil {
244 252
 			return false, err
245 253
 		}
... ...
@@ -247,9 +258,15 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdat
247 247
 		return false, errors.New("unsupported manifest format")
248 248
 	}
249 249
 
250
+	progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
251
+
250 252
 	oldTagImageID, err := p.config.ReferenceStore.Get(ref)
251
-	if err == nil && oldTagImageID == imageID {
252
-		return false, nil
253
+	if err == nil {
254
+		if oldTagImageID == imageID {
255
+			return false, nil
256
+		}
257
+	} else if err != reference.ErrDoesNotExist {
258
+		return false, err
253 259
 	}
254 260
 
255 261
 	if canonical, ok := ref.(reference.Canonical); ok {
... ...
@@ -263,23 +280,23 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdat
263 263
 	return true, nil
264 264
 }
265 265
 
266
-func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverifiedManifest *schema1.SignedManifest) (imageID image.ID, err error) {
266
+func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverifiedManifest *schema1.SignedManifest) (imageID image.ID, manifestDigest digest.Digest, err error) {
267 267
 	var verifiedManifest *schema1.Manifest
268
-	verifiedManifest, err = verifyManifest(unverifiedManifest, ref)
268
+	verifiedManifest, err = verifySchema1Manifest(unverifiedManifest, ref)
269 269
 	if err != nil {
270
-		return "", err
270
+		return "", "", err
271 271
 	}
272 272
 
273 273
 	rootFS := image.NewRootFS()
274 274
 
275 275
 	if err := detectBaseLayer(p.config.ImageStore, verifiedManifest, rootFS); err != nil {
276
-		return "", err
276
+		return "", "", err
277 277
 	}
278 278
 
279 279
 	// remove duplicate layers and check parent chain validity
280 280
 	err = fixManifestLayers(verifiedManifest)
281 281
 	if err != nil {
282
-		return "", err
282
+		return "", "", err
283 283
 	}
284 284
 
285 285
 	var descriptors []xfer.DownloadDescriptor
... ...
@@ -296,12 +313,12 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverif
296 296
 			ThrowAway bool `json:"throwaway,omitempty"`
297 297
 		}
298 298
 		if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
299
-			return "", err
299
+			return "", "", err
300 300
 		}
301 301
 
302 302
 		h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
303 303
 		if err != nil {
304
-			return "", err
304
+			return "", "", err
305 305
 		}
306 306
 		history = append(history, h)
307 307
 
... ...
@@ -320,30 +337,161 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverif
320 320
 
321 321
 	resultRootFS, release, err := p.config.DownloadManager.Download(ctx, *rootFS, descriptors, p.config.ProgressOutput)
322 322
 	if err != nil {
323
-		return "", err
323
+		return "", "", err
324 324
 	}
325 325
 	defer release()
326 326
 
327 327
 	config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), &resultRootFS, history)
328 328
 	if err != nil {
329
-		return "", err
329
+		return "", "", err
330 330
 	}
331 331
 
332 332
 	imageID, err = p.config.ImageStore.Create(config)
333 333
 	if err != nil {
334
-		return "", err
334
+		return "", "", err
335
+	}
336
+
337
+	manifestDigest = digest.FromBytes(unverifiedManifest.Canonical)
338
+
339
+	return imageID, manifestDigest, nil
340
+}
341
+
342
+func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest) (imageID image.ID, manifestDigest digest.Digest, err error) {
343
+	_, canonical, err := mfst.Payload()
344
+	if err != nil {
345
+		return "", "", err
346
+	}
347
+
348
+	// If pull by digest, then verify the manifest digest.
349
+	if digested, isDigested := ref.(reference.Canonical); isDigested {
350
+		verifier, err := digest.NewDigestVerifier(digested.Digest())
351
+		if err != nil {
352
+			return "", "", err
353
+		}
354
+		if _, err := verifier.Write(canonical); err != nil {
355
+			return "", "", err
356
+		}
357
+		if !verifier.Verified() {
358
+			err := fmt.Errorf("manifest verification failed for digest %s", digested.Digest())
359
+			logrus.Error(err)
360
+			return "", "", err
361
+		}
362
+		manifestDigest = digested.Digest()
363
+	} else {
364
+		manifestDigest = digest.FromBytes(canonical)
365
+	}
366
+
367
+	target := mfst.Target()
368
+	imageID = image.ID(target.Digest)
369
+	if _, err := p.config.ImageStore.Get(imageID); err == nil {
370
+		// If the image already exists locally, no need to pull
371
+		// anything.
372
+		return imageID, manifestDigest, nil
373
+	}
374
+
375
+	configChan := make(chan []byte, 1)
376
+	errChan := make(chan error, 1)
377
+	var cancel func()
378
+	ctx, cancel = context.WithCancel(ctx)
379
+
380
+	// Pull the image config
381
+	go func() {
382
+		configJSON, err := p.pullSchema2ImageConfig(ctx, target.Digest)
383
+		if err != nil {
384
+			errChan <- err
385
+			cancel()
386
+			return
387
+		}
388
+		configChan <- configJSON
389
+	}()
390
+
391
+	var descriptors []xfer.DownloadDescriptor
392
+
393
+	// Note that the order of this loop is in the direction of bottom-most
394
+	// to top-most, so that the downloads slice gets ordered correctly.
395
+	for _, d := range mfst.References() {
396
+		layerDescriptor := &v2LayerDescriptor{
397
+			digest:         d.Digest,
398
+			repo:           p.repo,
399
+			blobSumService: p.blobSumService,
400
+		}
401
+
402
+		descriptors = append(descriptors, layerDescriptor)
403
+	}
404
+
405
+	rootFS, release, err := p.config.DownloadManager.Download(ctx, *image.NewRootFS(), descriptors, p.config.ProgressOutput)
406
+	if err != nil {
407
+		select {
408
+		case err = <-errChan:
409
+			return "", "", err
410
+		default:
411
+			cancel()
412
+			select {
413
+			case <-configChan:
414
+			case <-errChan:
415
+			}
416
+			return "", "", err
417
+		}
418
+	}
419
+	defer release()
420
+
421
+	var configJSON []byte
422
+	select {
423
+	case configJSON = <-configChan:
424
+	case err = <-errChan:
425
+		return "", "", err
426
+		// Don't need a case for ctx.Done in the select because cancellation
427
+		// will trigger an error in p.pullSchema2ImageConfig.
428
+	}
429
+
430
+	// The DiffIDs returned in rootFS MUST match those in the config.
431
+	// Otherwise the image config could be referencing layers that aren't
432
+	// included in the manifest.
433
+	var unmarshalledConfig image.Image
434
+	if err = json.Unmarshal(configJSON, &unmarshalledConfig); err != nil {
435
+		return "", "", err
436
+	}
437
+
438
+	if len(rootFS.DiffIDs) != len(unmarshalledConfig.RootFS.DiffIDs) {
439
+		return "", "", errRootFSMismatch
440
+	}
441
+
442
+	for i := range rootFS.DiffIDs {
443
+		if rootFS.DiffIDs[i] != unmarshalledConfig.RootFS.DiffIDs[i] {
444
+			return "", "", errRootFSMismatch
445
+		}
446
+	}
447
+
448
+	imageID, err = p.config.ImageStore.Create(configJSON)
449
+	if err != nil {
450
+		return "", "", err
335 451
 	}
336 452
 
337
-	manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo)
453
+	return imageID, manifestDigest, nil
454
+}
455
+
456
+func (p *v2Puller) pullSchema2ImageConfig(ctx context.Context, dgst digest.Digest) (configJSON []byte, err error) {
457
+	blobs := p.repo.Blobs(ctx)
458
+	configJSON, err = blobs.Get(ctx, dgst)
338 459
 	if err != nil {
339
-		return "", err
460
+		return nil, err
340 461
 	}
341 462
 
342
-	if manifestDigest != "" {
343
-		progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
463
+	// Verify image config digest
464
+	verifier, err := digest.NewDigestVerifier(dgst)
465
+	if err != nil {
466
+		return nil, err
467
+	}
468
+	if _, err := verifier.Write(configJSON); err != nil {
469
+		return nil, err
470
+	}
471
+	if !verifier.Verified() {
472
+		err := fmt.Errorf("image config verification failed for digest %s", dgst)
473
+		logrus.Error(err)
474
+		return nil, err
344 475
 	}
345 476
 
346
-	return imageID, nil
477
+	return configJSON, nil
347 478
 }
348 479
 
349 480
 // allowV1Fallback checks if the error is a possible reason to fallback to v1
... ...
@@ -367,7 +515,7 @@ func allowV1Fallback(err error) error {
367 367
 	return err
368 368
 }
369 369
 
370
-func verifyManifest(signedManifest *schema1.SignedManifest, ref reference.Named) (m *schema1.Manifest, err error) {
370
+func verifySchema1Manifest(signedManifest *schema1.SignedManifest, ref reference.Named) (m *schema1.Manifest, err error) {
371 371
 	// If pull by digest, then verify the manifest digest. NOTE: It is
372 372
 	// important to do this first, before any other content validation. If the
373 373
 	// digest cannot be verified, don't even bother with those other things.
... ...
@@ -123,7 +123,7 @@ func TestValidateManifest(t *testing.T) {
123 123
 		t.Fatal("error unmarshaling manifest:", err)
124 124
 	}
125 125
 
126
-	verifiedManifest, err := verifyManifest(&goodSignedManifest, expectedDigest)
126
+	verifiedManifest, err := verifySchema1Manifest(&goodSignedManifest, expectedDigest)
127 127
 	if err != nil {
128 128
 		t.Fatal("validateManifest failed:", err)
129 129
 	}
... ...
@@ -145,7 +145,7 @@ func TestValidateManifest(t *testing.T) {
145 145
 		t.Fatal("error unmarshaling manifest:", err)
146 146
 	}
147 147
 
148
-	verifiedManifest, err = verifyManifest(&extraDataSignedManifest, expectedDigest)
148
+	verifiedManifest, err = verifySchema1Manifest(&extraDataSignedManifest, expectedDigest)
149 149
 	if err != nil {
150 150
 		t.Fatal("validateManifest failed:", err)
151 151
 	}
... ...
@@ -167,7 +167,7 @@ func TestValidateManifest(t *testing.T) {
167 167
 		t.Fatal("error unmarshaling manifest:", err)
168 168
 	}
169 169
 
170
-	verifiedManifest, err = verifyManifest(&badSignedManifest, expectedDigest)
170
+	verifiedManifest, err = verifySchema1Manifest(&badSignedManifest, expectedDigest)
171 171
 	if err == nil || !strings.HasPrefix(err.Error(), "image verification failed for digest") {
172 172
 		t.Fatal("expected validateManifest to fail with digest error")
173 173
 	}