Browse code

Update code to compile against new manifest interface

Also, digest.FromBytes no longer returns an error.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

Aaron Lehmann authored on 2015/12/09 04:14:02
Showing 13 changed files
... ...
@@ -14,6 +14,7 @@ import (
14 14
 	"github.com/docker/distribution/digest"
15 15
 	"github.com/docker/distribution/manifest/schema1"
16 16
 	"github.com/docker/distribution/registry/api/errcode"
17
+	"github.com/docker/distribution/registry/client"
17 18
 	"github.com/docker/docker/distribution/metadata"
18 19
 	"github.com/docker/docker/distribution/xfer"
19 20
 	"github.com/docker/docker/image"
... ...
@@ -61,18 +62,12 @@ func (p *v2Puller) Pull(ctx context.Context, ref reference.Named) (err error) {
61 61
 func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named) (err error) {
62 62
 	var layersDownloaded bool
63 63
 	if !reference.IsNameOnly(ref) {
64
-		var err error
65 64
 		layersDownloaded, err = p.pullV2Tag(ctx, ref)
66 65
 		if err != nil {
67 66
 			return err
68 67
 		}
69 68
 	} else {
70
-		manSvc, err := p.repo.Manifests(ctx)
71
-		if err != nil {
72
-			return err
73
-		}
74
-
75
-		tags, err := manSvc.Tags()
69
+		tags, err := p.repo.Tags(ctx).All(ctx)
76 70
 		if err != nil {
77 71
 			// If this repository doesn't exist on V2, we should
78 72
 			// permit a fallback to V1.
... ...
@@ -84,8 +79,6 @@ func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named) (e
84 84
 		// error later on.
85 85
 		p.confirmedV2 = true
86 86
 
87
-		// This probably becomes a lot nicer after the manifest
88
-		// refactor...
89 87
 		for _, tag := range tags {
90 88
 			tagRef, err := reference.WithTag(ref, tag)
91 89
 			if err != nil {
... ...
@@ -203,58 +196,92 @@ func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) {
203 203
 }
204 204
 
205 205
 func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdated bool, err error) {
206
-	tagOrDigest := ""
206
+	manSvc, err := p.repo.Manifests(ctx)
207
+	if err != nil {
208
+		return false, err
209
+	}
210
+
211
+	var (
212
+		manifest    distribution.Manifest
213
+		tagOrDigest string // Used for logging/progress only
214
+	)
207 215
 	if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
216
+		// NOTE: not using TagService.Get, since it uses HEAD requests
217
+		// against the manifests endpoint, which are not supported by
218
+		// all registry versions.
219
+		manifest, err = manSvc.Get(ctx, "", client.WithTag(tagged.Tag()))
220
+		if err != nil {
221
+			return false, allowV1Fallback(err)
222
+		}
208 223
 		tagOrDigest = tagged.Tag()
209
-	} else if digested, isCanonical := ref.(reference.Canonical); isCanonical {
224
+	} else if digested, isDigested := ref.(reference.Canonical); isDigested {
225
+		manifest, err = manSvc.Get(ctx, digested.Digest())
226
+		if err != nil {
227
+			return false, err
228
+		}
210 229
 		tagOrDigest = digested.Digest().String()
211 230
 	} else {
212 231
 		return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", ref.String())
213 232
 	}
214 233
 
215
-	logrus.Debugf("Pulling ref from V2 registry: %s:%s", ref.FullName(), tagOrDigest)
234
+	if manifest == nil {
235
+		return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
236
+	}
216 237
 
217
-	manSvc, err := p.repo.Manifests(ctx)
218
-	if err != nil {
219
-		return false, err
238
+	// If manSvc.Get succeeded, we can be confident that the registry on
239
+	// the other side speaks the v2 protocol.
240
+	p.confirmedV2 = true
241
+
242
+	logrus.Debugf("Pulling ref from V2 registry: %s", ref.String())
243
+	progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+p.repo.Name())
244
+
245
+	var imageID image.ID
246
+
247
+	switch v := manifest.(type) {
248
+	case *schema1.SignedManifest:
249
+		imageID, err = p.pullSchema1(ctx, ref, v)
250
+		if err != nil {
251
+			return false, err
252
+		}
253
+	default:
254
+		return false, errors.New("unsupported manifest format")
220 255
 	}
221 256
 
222
-	unverifiedManifest, err := manSvc.GetByTag(tagOrDigest)
223
-	if err != nil {
224
-		// If this manifest did not exist, we should allow a possible
225
-		// fallback to the v1 protocol, because dual-version setups may
226
-		// not host all manifests with the v2 protocol. We may also get
227
-		// a "not authorized" error if the manifest doesn't exist.
228
-		return false, allowV1Fallback(err)
257
+	oldTagImageID, err := p.config.ReferenceStore.Get(ref)
258
+	if err == nil && oldTagImageID == imageID {
259
+		return false, nil
229 260
 	}
230
-	if unverifiedManifest == nil {
231
-		return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
261
+
262
+	if canonical, ok := ref.(reference.Canonical); ok {
263
+		if err = p.config.ReferenceStore.AddDigest(canonical, imageID, true); err != nil {
264
+			return false, err
265
+		}
266
+	} else if err = p.config.ReferenceStore.AddTag(ref, imageID, true); err != nil {
267
+		return false, err
232 268
 	}
233 269
 
234
-	// If GetByTag succeeded, we can be confident that the registry on
235
-	// the other side speaks the v2 protocol.
236
-	p.confirmedV2 = true
270
+	return true, nil
271
+}
237 272
 
273
+func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverifiedManifest *schema1.SignedManifest) (imageID image.ID, err error) {
238 274
 	var verifiedManifest *schema1.Manifest
239 275
 	verifiedManifest, err = verifyManifest(unverifiedManifest, ref)
240 276
 	if err != nil {
241
-		return false, err
277
+		return "", err
242 278
 	}
243 279
 
244 280
 	rootFS := image.NewRootFS()
245 281
 
246 282
 	if err := detectBaseLayer(p.config.ImageStore, verifiedManifest, rootFS); err != nil {
247
-		return false, err
283
+		return "", err
248 284
 	}
249 285
 
250 286
 	// remove duplicate layers and check parent chain validity
251 287
 	err = fixManifestLayers(verifiedManifest)
252 288
 	if err != nil {
253
-		return false, err
289
+		return "", err
254 290
 	}
255 291
 
256
-	progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+p.repo.Name())
257
-
258 292
 	var descriptors []xfer.DownloadDescriptor
259 293
 
260 294
 	// Image history converted to the new format
... ...
@@ -269,12 +296,12 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdat
269 269
 			ThrowAway bool `json:"throwaway,omitempty"`
270 270
 		}
271 271
 		if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
272
-			return false, err
272
+			return "", err
273 273
 		}
274 274
 
275 275
 		h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
276 276
 		if err != nil {
277
-			return false, err
277
+			return "", err
278 278
 		}
279 279
 		history = append(history, h)
280 280
 
... ...
@@ -293,43 +320,30 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdat
293 293
 
294 294
 	resultRootFS, release, err := p.config.DownloadManager.Download(ctx, *rootFS, descriptors, p.config.ProgressOutput)
295 295
 	if err != nil {
296
-		return false, err
296
+		return "", err
297 297
 	}
298 298
 	defer release()
299 299
 
300 300
 	config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), &resultRootFS, history)
301 301
 	if err != nil {
302
-		return false, err
302
+		return "", err
303 303
 	}
304 304
 
305
-	imageID, err := p.config.ImageStore.Create(config)
305
+	imageID, err = p.config.ImageStore.Create(config)
306 306
 	if err != nil {
307
-		return false, err
307
+		return "", err
308 308
 	}
309 309
 
310 310
 	manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo)
311 311
 	if err != nil {
312
-		return false, err
312
+		return "", err
313 313
 	}
314 314
 
315 315
 	if manifestDigest != "" {
316 316
 		progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
317 317
 	}
318 318
 
319
-	oldTagImageID, err := p.config.ReferenceStore.Get(ref)
320
-	if err == nil && oldTagImageID == imageID {
321
-		return false, nil
322
-	}
323
-
324
-	if canonical, ok := ref.(reference.Canonical); ok {
325
-		if err = p.config.ReferenceStore.AddDigest(canonical, imageID, true); err != nil {
326
-			return false, err
327
-		}
328
-	} else if err = p.config.ReferenceStore.AddTag(ref, imageID, true); err != nil {
329
-		return false, err
330
-	}
331
-
332
-	return true, nil
319
+	return imageID, nil
333 320
 }
334 321
 
335 322
 // allowV1Fallback checks if the error is a possible reason to fallback to v1
... ...
@@ -362,13 +376,7 @@ func verifyManifest(signedManifest *schema1.SignedManifest, ref reference.Named)
362 362
 		if err != nil {
363 363
 			return nil, err
364 364
 		}
365
-		payload, err := signedManifest.Payload()
366
-		if err != nil {
367
-			// If this failed, the signatures section was corrupted
368
-			// or missing. Treat the entire manifest as the payload.
369
-			payload = signedManifest.Raw
370
-		}
371
-		if _, err := verifier.Write(payload); err != nil {
365
+		if _, err := verifier.Write(signedManifest.Canonical); err != nil {
372 366
 			return nil, err
373 367
 		}
374 368
 		if !verifier.Verified() {
... ...
@@ -376,15 +384,8 @@ func verifyManifest(signedManifest *schema1.SignedManifest, ref reference.Named)
376 376
 			logrus.Error(err)
377 377
 			return nil, err
378 378
 		}
379
-
380
-		var verifiedManifest schema1.Manifest
381
-		if err = json.Unmarshal(payload, &verifiedManifest); err != nil {
382
-			return nil, err
383
-		}
384
-		m = &verifiedManifest
385
-	} else {
386
-		m = &signedManifest.Manifest
387 379
 	}
380
+	m = &signedManifest.Manifest
388 381
 
389 382
 	if m.SchemaVersion != 1 {
390 383
 		return nil, fmt.Errorf("unsupported schema version %d for %q", m.SchemaVersion, ref.String())
... ...
@@ -194,7 +194,9 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, association reference.Associat
194 194
 	if err != nil {
195 195
 		return err
196 196
 	}
197
-	return manSvc.Put(signed)
197
+	_, err = manSvc.Put(ctx, signed)
198
+	// FIXME create a tag
199
+	return err
198 200
 }
199 201
 
200 202
 type v2PushDescriptor struct {
... ...
@@ -370,10 +372,7 @@ func CreateV2Manifest(name, tag string, img *image.Image, fsLayers map[layer.Dif
370 370
 		if !present {
371 371
 			return nil, fmt.Errorf("missing layer in CreateV2Manifest: %s", diffID.String())
372 372
 		}
373
-		dgst, err := digest.FromBytes([]byte(fsLayer.Hex() + " " + parent))
374
-		if err != nil {
375
-			return nil, err
376
-		}
373
+		dgst := digest.FromBytes([]byte(fsLayer.Hex() + " " + parent))
377 374
 		v1ID := dgst.Hex()
378 375
 
379 376
 		v1Compatibility := v1Compatibility{
... ...
@@ -414,11 +413,8 @@ func CreateV2Manifest(name, tag string, img *image.Image, fsLayers map[layer.Dif
414 414
 		return nil, fmt.Errorf("missing layer in CreateV2Manifest: %s", diffID.String())
415 415
 	}
416 416
 
417
-	dgst, err := digest.FromBytes([]byte(fsLayer.Hex() + " " + parent + " " + string(img.RawJSON())))
418
-	if err != nil {
419
-		return nil, err
420
-	}
421 417
 	fsLayerList[0] = schema1.FSLayer{BlobSum: fsLayer}
418
+	dgst := digest.FromBytes([]byte(fsLayer.Hex() + " " + parent + " " + string(img.RawJSON())))
422 419
 
423 420
 	// Top-level v1compatibility string should be a modified version of the
424 421
 	// image config.
... ...
@@ -8,7 +8,6 @@ import (
8 8
 	"strings"
9 9
 	"time"
10 10
 
11
-	"github.com/Sirupsen/logrus"
12 11
 	"github.com/docker/distribution"
13 12
 	"github.com/docker/distribution/digest"
14 13
 	"github.com/docker/distribution/manifest/schema1"
... ...
@@ -126,17 +125,7 @@ func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, end
126 126
 }
127 127
 
128 128
 func digestFromManifest(m *schema1.SignedManifest, name reference.Named) (digest.Digest, int, error) {
129
-	payload, err := m.Payload()
130
-	if err != nil {
131
-		// If this failed, the signatures section was corrupted
132
-		// or missing. Treat the entire manifest as the payload.
133
-		payload = m.Raw
134
-	}
135
-	manifestDigest, err := digest.FromBytes(payload)
136
-	if err != nil {
137
-		logrus.Infof("Could not compute manifest digest for %s:%s : %v", name.Name(), m.Tag, err)
138
-	}
139
-	return manifestDigest, len(payload), nil
129
+	return digest.FromBytes(m.Canonical), len(m.Canonical), nil
140 130
 }
141 131
 
142 132
 type existingTokenHandler struct {
... ...
@@ -65,12 +65,7 @@ func createChainIDFromParent(parent layer.ChainID, dgsts ...layer.DiffID) layer.
65 65
 		return createChainIDFromParent(layer.ChainID(dgsts[0]), dgsts[1:]...)
66 66
 	}
67 67
 	// H = "H(n-1) SHA256(n)"
68
-	dgst, err := digest.FromBytes([]byte(string(parent) + " " + string(dgsts[0])))
69
-	if err != nil {
70
-		// Digest calculation is not expected to throw an error,
71
-		// any error at this point is a program error
72
-		panic(err)
73
-	}
68
+	dgst := digest.FromBytes([]byte(string(parent) + " " + string(dgsts[0])))
74 69
 	return createChainIDFromParent(layer.ChainID(dgst), dgsts[1:]...)
75 70
 }
76 71
 
... ...
@@ -92,11 +87,7 @@ func (ls *mockLayerStore) Register(reader io.Reader, parentID layer.ChainID) (la
92 92
 	if err != nil {
93 93
 		return nil, err
94 94
 	}
95
-	diffID, err := digest.FromBytes(l.layerData.Bytes())
96
-	if err != nil {
97
-		return nil, err
98
-	}
99
-	l.diffID = layer.DiffID(diffID)
95
+	l.diffID = layer.DiffID(digest.FromBytes(l.layerData.Bytes()))
100 96
 	l.chainID = createChainIDFromParent(parentID, l.diffID)
101 97
 
102 98
 	ls.layers[l.chainID] = l
... ...
@@ -62,7 +62,7 @@ func (u *mockUploadDescriptor) Upload(ctx context.Context, progressOutput progre
62 62
 
63 63
 	// For the mock implementation, use SHA256(DiffID) as the returned
64 64
 	// digest.
65
-	return digest.FromBytes([]byte(u.diffID.String()))
65
+	return digest.FromBytes([]byte(u.diffID.String())), nil
66 66
 }
67 67
 
68 68
 func uploadDescriptors(currentUploads *int32) []UploadDescriptor {
... ...
@@ -101,11 +101,7 @@ func (s *fs) get(id ID) ([]byte, error) {
101 101
 	}
102 102
 
103 103
 	// todo: maybe optional
104
-	validated, err := digest.FromBytes(content)
105
-	if err != nil {
106
-		return nil, err
107
-	}
108
-	if ID(validated) != id {
104
+	if ID(digest.FromBytes(content)) != id {
109 105
 		return nil, fmt.Errorf("failed to verify image: %v", id)
110 106
 	}
111 107
 
... ...
@@ -121,11 +117,7 @@ func (s *fs) Set(data []byte) (ID, error) {
121 121
 		return "", fmt.Errorf("Invalid empty data")
122 122
 	}
123 123
 
124
-	dgst, err := digest.FromBytes(data)
125
-	if err != nil {
126
-		return "", err
127
-	}
128
-	id := ID(dgst)
124
+	id := ID(digest.FromBytes(data))
129 125
 	filePath := s.contentFile(id)
130 126
 	tempFilePath := s.contentFile(id) + ".tmp"
131 127
 	if err := ioutil.WriteFile(tempFilePath, data, 0600); err != nil {
... ...
@@ -67,10 +67,7 @@ func TestFSInvalidSet(t *testing.T) {
67 67
 		t.Fatal(err)
68 68
 	}
69 69
 
70
-	id, err := digest.FromBytes([]byte("foobar"))
71
-	if err != nil {
72
-		t.Fatal(err)
73
-	}
70
+	id := digest.FromBytes([]byte("foobar"))
74 71
 	err = os.Mkdir(filepath.Join(tmpdir, contentDirName, string(id.Algorithm()), id.Hex()), 0700)
75 72
 	if err != nil {
76 73
 		t.Fatal(err)
... ...
@@ -160,11 +157,7 @@ func testMetadataGetSet(t *testing.T, store StoreBackend) {
160 160
 		t.Fatal("Expected error for getting metadata for unknown key")
161 161
 	}
162 162
 
163
-	id3, err := digest.FromBytes([]byte("baz"))
164
-	if err != nil {
165
-		t.Fatal(err)
166
-	}
167
-
163
+	id3 := digest.FromBytes([]byte("baz"))
168 164
 	err = store.SetMetadata(ID(id3), "tkey", []byte("tval"))
169 165
 	if err == nil {
170 166
 		t.Fatal("Expected error for setting metadata for unknown ID.")
... ...
@@ -27,7 +27,7 @@ func (r *RootFS) BaseLayerID() string {
27 27
 
28 28
 // ChainID returns the ChainID for the top layer in RootFS.
29 29
 func (r *RootFS) ChainID() layer.ChainID {
30
-	baseDiffID, _ := digest.FromBytes([]byte(r.BaseLayerID())) // can never error
30
+	baseDiffID := digest.FromBytes([]byte(r.BaseLayerID()))
31 31
 	return layer.CreateChainID(append([]layer.DiffID{layer.DiffID(baseDiffID)}, r.DiffIDs...))
32 32
 }
33 33
 
... ...
@@ -63,7 +63,7 @@ func CreateID(v1Image image.V1Image, layerID layer.ChainID, parent digest.Digest
63 63
 	}
64 64
 	logrus.Debugf("CreateV1ID %s", configJSON)
65 65
 
66
-	return digest.FromBytes(configJSON)
66
+	return digest.FromBytes(configJSON), nil
67 67
 }
68 68
 
69 69
 // MakeConfigFromV1Config creates an image config from the legacy V1 config format.
... ...
@@ -15,12 +15,8 @@ import (
15 15
 
16 16
 func randomLayerID(seed int64) ChainID {
17 17
 	r := rand.New(rand.NewSource(seed))
18
-	dgst, err := digest.FromBytes([]byte(fmt.Sprintf("%d", r.Int63())))
19
-	if err != nil {
20
-		panic(err)
21
-	}
22 18
 
23
-	return ChainID(dgst)
19
+	return ChainID(digest.FromBytes([]byte(fmt.Sprintf("%d", r.Int63()))))
24 20
 }
25 21
 
26 22
 func newFileMetadataStore(t *testing.T) (*fileMetadataStore, string, func()) {
... ...
@@ -233,12 +233,7 @@ func createChainIDFromParent(parent ChainID, dgsts ...DiffID) ChainID {
233 233
 		return createChainIDFromParent(ChainID(dgsts[0]), dgsts[1:]...)
234 234
 	}
235 235
 	// H = "H(n-1) SHA256(n)"
236
-	dgst, err := digest.FromBytes([]byte(string(parent) + " " + string(dgsts[0])))
237
-	if err != nil {
238
-		// Digest calculation is not expected to throw an error,
239
-		// any error at this point is a program error
240
-		panic(err)
241
-	}
236
+	dgst := digest.FromBytes([]byte(string(parent) + " " + string(dgsts[0])))
242 237
 	return createChainIDFromParent(ChainID(dgst), dgsts[1:]...)
243 238
 }
244 239
 
... ...
@@ -548,10 +548,7 @@ func TestTarStreamStability(t *testing.T) {
548 548
 }
549 549
 
550 550
 func assertLayerDiff(t *testing.T, expected []byte, layer Layer) {
551
-	expectedDigest, err := digest.FromBytes(expected)
552
-	if err != nil {
553
-		t.Fatal(err)
554
-	}
551
+	expectedDigest := digest.FromBytes(expected)
555 552
 
556 553
 	if digest.Digest(layer.DiffID()) != expectedDigest {
557 554
 		t.Fatalf("Mismatched diff id for %s, got %s, expected %s", layer.ChainID(), layer.DiffID(), expected)
... ...
@@ -573,10 +570,7 @@ func assertLayerDiff(t *testing.T, expected []byte, layer Layer) {
573 573
 		t.Fatalf("Mismatched tar stream size for %s, got %d, expected %d", layer.ChainID(), len(actual), len(expected))
574 574
 	}
575 575
 
576
-	actualDigest, err := digest.FromBytes(actual)
577
-	if err != nil {
578
-		t.Fatal(err)
579
-	}
576
+	actualDigest := digest.FromBytes(actual)
580 577
 
581 578
 	if actualDigest != expectedDigest {
582 579
 		logByteDiff(t, actual, expected)
... ...
@@ -37,10 +37,7 @@ func GetLayerPath(s Store, layer ChainID) (string, error) {
37 37
 
38 38
 func (ls *layerStore) RegisterDiffID(graphID string, size int64) (Layer, error) {
39 39
 	var err error // this is used for cleanup in existingLayer case
40
-	diffID, err := digest.FromBytes([]byte(graphID))
41
-	if err != nil {
42
-		return nil, err
43
-	}
40
+	diffID := digest.FromBytes([]byte(graphID))
44 41
 
45 42
 	// Create new roLayer
46 43
 	layer := &roLayer{