Browse code

Use ocischema package instead of custom handler

Previously we were re-using schema2.DeserializedManifest to handle oci
manifests. The issue lies in the fact that distribution started
validating the media type string during json deserialization. This
change broke our usage of that type.

Instead distribution now provides direct support for oci schemas, so use
that instead of our custom handlers.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>
(cherry picked from commit e443512ce4799380941374ef64fc30edc989650e)
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

Brian Goff authored on 2019/10/11 06:33:15
Showing 4 changed files
1 1
deleted file mode 100644
... ...
@@ -1,29 +0,0 @@
1
-package distribution
2
-
3
-import (
4
-	"fmt"
5
-
6
-	"github.com/docker/distribution"
7
-	"github.com/docker/distribution/manifest/schema2"
8
-	digest "github.com/opencontainers/go-digest"
9
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
10
-)
11
-
12
-func init() {
13
-	// TODO: Remove this registration if distribution is included with OCI support
14
-
15
-	ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
16
-		m := new(schema2.DeserializedManifest)
17
-		err := m.UnmarshalJSON(b)
18
-		if err != nil {
19
-			return nil, distribution.Descriptor{}, err
20
-		}
21
-
22
-		dgst := digest.FromBytes(b)
23
-		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: ocispec.MediaTypeImageManifest}, err
24
-	}
25
-	err := distribution.RegisterManifestSchema(ocispec.MediaTypeImageManifest, ocischemaFunc)
26
-	if err != nil {
27
-		panic(fmt.Sprintf("Unable to register manifest: %s", err))
28
-	}
29
-}
... ...
@@ -14,6 +14,7 @@ import (
14 14
 	"github.com/containerd/containerd/platforms"
15 15
 	"github.com/docker/distribution"
16 16
 	"github.com/docker/distribution/manifest/manifestlist"
17
+	"github.com/docker/distribution/manifest/ocischema"
17 18
 	"github.com/docker/distribution/manifest/schema1"
18 19
 	"github.com/docker/distribution/manifest/schema2"
19 20
 	"github.com/docker/distribution/reference"
... ...
@@ -410,6 +411,11 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, platform
410 410
 		if err != nil {
411 411
 			return false, err
412 412
 		}
413
+	case *ocischema.DeserializedManifest:
414
+		id, manifestDigest, err = p.pullOCI(ctx, ref, v, platform)
415
+		if err != nil {
416
+			return false, err
417
+		}
413 418
 	case *manifestlist.DeserializedManifestList:
414 419
 		id, manifestDigest, err = p.pullManifestList(ctx, ref, v, platform)
415 420
 		if err != nil {
... ...
@@ -557,24 +563,18 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unv
557 557
 	return imageID, manifestDigest, nil
558 558
 }
559 559
 
560
-func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
561
-	manifestDigest, err = schema2ManifestDigest(ref, mfst)
562
-	if err != nil {
563
-		return "", "", err
564
-	}
565
-
566
-	target := mfst.Target()
560
+func (p *v2Puller) pullSchema2Layers(ctx context.Context, target distribution.Descriptor, layers []distribution.Descriptor, platform *specs.Platform) (id digest.Digest, err error) {
567 561
 	if _, err := p.config.ImageStore.Get(target.Digest); err == nil {
568 562
 		// If the image already exists locally, no need to pull
569 563
 		// anything.
570
-		return target.Digest, manifestDigest, nil
564
+		return target.Digest, nil
571 565
 	}
572 566
 
573 567
 	var descriptors []xfer.DownloadDescriptor
574 568
 
575 569
 	// Note that the order of this loop is in the direction of bottom-most
576 570
 	// to top-most, so that the downloads slice gets ordered correctly.
577
-	for _, d := range mfst.Layers {
571
+	for _, d := range layers {
578 572
 		layerDescriptor := &v2LayerDescriptor{
579 573
 			digest:            d.Digest,
580 574
 			repo:              p.repo,
... ...
@@ -629,23 +629,23 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
629 629
 	if runtime.GOOS == "windows" {
630 630
 		configJSON, configRootFS, configPlatform, err = receiveConfig(p.config.ImageStore, configChan, configErrChan)
631 631
 		if err != nil {
632
-			return "", "", err
632
+			return "", err
633 633
 		}
634 634
 		if configRootFS == nil {
635
-			return "", "", errRootFSInvalid
635
+			return "", errRootFSInvalid
636 636
 		}
637 637
 		if err := checkImageCompatibility(configPlatform.OS, configPlatform.OSVersion); err != nil {
638
-			return "", "", err
638
+			return "", err
639 639
 		}
640 640
 
641 641
 		if len(descriptors) != len(configRootFS.DiffIDs) {
642
-			return "", "", errRootFSMismatch
642
+			return "", errRootFSMismatch
643 643
 		}
644 644
 		if platform == nil {
645 645
 			// Early bath if the requested OS doesn't match that of the configuration.
646 646
 			// This avoids doing the download, only to potentially fail later.
647 647
 			if !system.IsOSSupported(configPlatform.OS) {
648
-				return "", "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, layerStoreOS)
648
+				return "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, layerStoreOS)
649 649
 			}
650 650
 			layerStoreOS = configPlatform.OS
651 651
 		}
... ...
@@ -692,14 +692,14 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
692 692
 			case <-downloadsDone:
693 693
 			case <-layerErrChan:
694 694
 			}
695
-			return "", "", err
695
+			return "", err
696 696
 		}
697 697
 	}
698 698
 
699 699
 	select {
700 700
 	case <-downloadsDone:
701 701
 	case err = <-layerErrChan:
702
-		return "", "", err
702
+		return "", err
703 703
 	}
704 704
 
705 705
 	if release != nil {
... ...
@@ -711,22 +711,40 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
711 711
 		// Otherwise the image config could be referencing layers that aren't
712 712
 		// included in the manifest.
713 713
 		if len(downloadedRootFS.DiffIDs) != len(configRootFS.DiffIDs) {
714
-			return "", "", errRootFSMismatch
714
+			return "", errRootFSMismatch
715 715
 		}
716 716
 
717 717
 		for i := range downloadedRootFS.DiffIDs {
718 718
 			if downloadedRootFS.DiffIDs[i] != configRootFS.DiffIDs[i] {
719
-				return "", "", errRootFSMismatch
719
+				return "", errRootFSMismatch
720 720
 			}
721 721
 		}
722 722
 	}
723 723
 
724 724
 	imageID, err := p.config.ImageStore.Put(configJSON)
725 725
 	if err != nil {
726
+		return "", err
727
+	}
728
+
729
+	return imageID, nil
730
+}
731
+
732
+func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
733
+	manifestDigest, err = schema2ManifestDigest(ref, mfst)
734
+	if err != nil {
726 735
 		return "", "", err
727 736
 	}
737
+	id, err = p.pullSchema2Layers(ctx, mfst.Target(), mfst.Layers, platform)
738
+	return id, manifestDigest, err
739
+}
728 740
 
729
-	return imageID, manifestDigest, nil
741
+func (p *v2Puller) pullOCI(ctx context.Context, ref reference.Named, mfst *ocischema.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
742
+	manifestDigest, err = schema2ManifestDigest(ref, mfst)
743
+	if err != nil {
744
+		return "", "", err
745
+	}
746
+	id, err = p.pullSchema2Layers(ctx, mfst.Target(), mfst.Layers, platform)
747
+	return id, manifestDigest, err
730 748
 }
731 749
 
732 750
 func receiveConfig(s ImageConfigStore, configChan <-chan []byte, errChan <-chan error) ([]byte, *image.RootFS, *specs.Platform, error) {
... ...
@@ -811,6 +829,12 @@ func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mf
811 811
 		if err != nil {
812 812
 			return "", "", err
813 813
 		}
814
+	case *ocischema.DeserializedManifest:
815
+		platform := toOCIPlatform(manifestMatches[0].Platform)
816
+		id, _, err = p.pullOCI(ctx, manifestRef, v, &platform)
817
+		if err != nil {
818
+			return "", "", err
819
+		}
814 820
 	default:
815 821
 		return "", "", errors.New("unsupported manifest format")
816 822
 	}
817 823
new file mode 100644
... ...
@@ -0,0 +1,107 @@
0
+package ocischema
1
+
2
+import (
3
+	"context"
4
+	"errors"
5
+
6
+	"github.com/docker/distribution"
7
+	"github.com/docker/distribution/manifest"
8
+	"github.com/opencontainers/go-digest"
9
+	"github.com/opencontainers/image-spec/specs-go/v1"
10
+)
11
+
12
+// Builder is a type for constructing manifests.
13
+type Builder struct {
14
+	// bs is a BlobService used to publish the configuration blob.
15
+	bs distribution.BlobService
16
+
17
+	// configJSON references
18
+	configJSON []byte
19
+
20
+	// layers is a list of layer descriptors that gets built by successive
21
+	// calls to AppendReference.
22
+	layers []distribution.Descriptor
23
+
24
+	// Annotations contains arbitrary metadata relating to the targeted content.
25
+	annotations map[string]string
26
+
27
+	// For testing purposes
28
+	mediaType string
29
+}
30
+
31
+// NewManifestBuilder is used to build new manifests for the current schema
32
+// version. It takes a BlobService so it can publish the configuration blob
33
+// as part of the Build process, and annotations.
34
+func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder {
35
+	mb := &Builder{
36
+		bs:          bs,
37
+		configJSON:  make([]byte, len(configJSON)),
38
+		annotations: annotations,
39
+		mediaType:   v1.MediaTypeImageManifest,
40
+	}
41
+	copy(mb.configJSON, configJSON)
42
+
43
+	return mb
44
+}
45
+
46
+// SetMediaType assigns the passed mediatype or error if the mediatype is not a
47
+// valid media type for oci image manifests currently: "" or "application/vnd.oci.image.manifest.v1+json"
48
+func (mb *Builder) SetMediaType(mediaType string) error {
49
+	if mediaType != "" && mediaType != v1.MediaTypeImageManifest {
50
+		return errors.New("invalid media type for OCI image manifest")
51
+	}
52
+
53
+	mb.mediaType = mediaType
54
+	return nil
55
+}
56
+
57
+// Build produces a final manifest from the given references.
58
+func (mb *Builder) Build(ctx context.Context) (distribution.Manifest, error) {
59
+	m := Manifest{
60
+		Versioned: manifest.Versioned{
61
+			SchemaVersion: 2,
62
+			MediaType:     mb.mediaType,
63
+		},
64
+		Layers:      make([]distribution.Descriptor, len(mb.layers)),
65
+		Annotations: mb.annotations,
66
+	}
67
+	copy(m.Layers, mb.layers)
68
+
69
+	configDigest := digest.FromBytes(mb.configJSON)
70
+
71
+	var err error
72
+	m.Config, err = mb.bs.Stat(ctx, configDigest)
73
+	switch err {
74
+	case nil:
75
+		// Override MediaType, since Put always replaces the specified media
76
+		// type with application/octet-stream in the descriptor it returns.
77
+		m.Config.MediaType = v1.MediaTypeImageConfig
78
+		return FromStruct(m)
79
+	case distribution.ErrBlobUnknown:
80
+		// nop
81
+	default:
82
+		return nil, err
83
+	}
84
+
85
+	// Add config to the blob store
86
+	m.Config, err = mb.bs.Put(ctx, v1.MediaTypeImageConfig, mb.configJSON)
87
+	// Override MediaType, since Put always replaces the specified media
88
+	// type with application/octet-stream in the descriptor it returns.
89
+	m.Config.MediaType = v1.MediaTypeImageConfig
90
+	if err != nil {
91
+		return nil, err
92
+	}
93
+
94
+	return FromStruct(m)
95
+}
96
+
97
+// AppendReference adds a reference to the current ManifestBuilder.
98
+func (mb *Builder) AppendReference(d distribution.Describable) error {
99
+	mb.layers = append(mb.layers, d.Descriptor())
100
+	return nil
101
+}
102
+
103
+// References returns the current references added to this builder.
104
+func (mb *Builder) References() []distribution.Descriptor {
105
+	return mb.layers
106
+}
0 107
new file mode 100644
... ...
@@ -0,0 +1,124 @@
0
+package ocischema
1
+
2
+import (
3
+	"encoding/json"
4
+	"errors"
5
+	"fmt"
6
+
7
+	"github.com/docker/distribution"
8
+	"github.com/docker/distribution/manifest"
9
+	"github.com/opencontainers/go-digest"
10
+	"github.com/opencontainers/image-spec/specs-go/v1"
11
+)
12
+
13
+var (
14
+	// SchemaVersion provides a pre-initialized version structure for this
15
+	// packages version of the manifest.
16
+	SchemaVersion = manifest.Versioned{
17
+		SchemaVersion: 2, // historical value here.. does not pertain to OCI or docker version
18
+		MediaType:     v1.MediaTypeImageManifest,
19
+	}
20
+)
21
+
22
+func init() {
23
+	ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
24
+		m := new(DeserializedManifest)
25
+		err := m.UnmarshalJSON(b)
26
+		if err != nil {
27
+			return nil, distribution.Descriptor{}, err
28
+		}
29
+
30
+		dgst := digest.FromBytes(b)
31
+		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageManifest}, err
32
+	}
33
+	err := distribution.RegisterManifestSchema(v1.MediaTypeImageManifest, ocischemaFunc)
34
+	if err != nil {
35
+		panic(fmt.Sprintf("Unable to register manifest: %s", err))
36
+	}
37
+}
38
+
39
+// Manifest defines a ocischema manifest.
40
+type Manifest struct {
41
+	manifest.Versioned
42
+
43
+	// Config references the image configuration as a blob.
44
+	Config distribution.Descriptor `json:"config"`
45
+
46
+	// Layers lists descriptors for the layers referenced by the
47
+	// configuration.
48
+	Layers []distribution.Descriptor `json:"layers"`
49
+
50
+	// Annotations contains arbitrary metadata for the image manifest.
51
+	Annotations map[string]string `json:"annotations,omitempty"`
52
+}
53
+
54
+// References returns the descriptors of this manifests references.
55
+func (m Manifest) References() []distribution.Descriptor {
56
+	references := make([]distribution.Descriptor, 0, 1+len(m.Layers))
57
+	references = append(references, m.Config)
58
+	references = append(references, m.Layers...)
59
+	return references
60
+}
61
+
62
+// Target returns the target of this manifest.
63
+func (m Manifest) Target() distribution.Descriptor {
64
+	return m.Config
65
+}
66
+
67
+// DeserializedManifest wraps Manifest with a copy of the original JSON.
68
+// It satisfies the distribution.Manifest interface.
69
+type DeserializedManifest struct {
70
+	Manifest
71
+
72
+	// canonical is the canonical byte representation of the Manifest.
73
+	canonical []byte
74
+}
75
+
76
+// FromStruct takes a Manifest structure, marshals it to JSON, and returns a
77
+// DeserializedManifest which contains the manifest and its JSON representation.
78
+func FromStruct(m Manifest) (*DeserializedManifest, error) {
79
+	var deserialized DeserializedManifest
80
+	deserialized.Manifest = m
81
+
82
+	var err error
83
+	deserialized.canonical, err = json.MarshalIndent(&m, "", "   ")
84
+	return &deserialized, err
85
+}
86
+
87
+// UnmarshalJSON populates a new Manifest struct from JSON data.
88
+func (m *DeserializedManifest) UnmarshalJSON(b []byte) error {
89
+	m.canonical = make([]byte, len(b))
90
+	// store manifest in canonical
91
+	copy(m.canonical, b)
92
+
93
+	// Unmarshal canonical JSON into Manifest object
94
+	var manifest Manifest
95
+	if err := json.Unmarshal(m.canonical, &manifest); err != nil {
96
+		return err
97
+	}
98
+
99
+	if manifest.MediaType != "" && manifest.MediaType != v1.MediaTypeImageManifest {
100
+		return fmt.Errorf("if present, mediaType in manifest should be '%s' not '%s'",
101
+			v1.MediaTypeImageManifest, manifest.MediaType)
102
+	}
103
+
104
+	m.Manifest = manifest
105
+
106
+	return nil
107
+}
108
+
109
+// MarshalJSON returns the contents of canonical. If canonical is empty,
110
+// marshals the inner contents.
111
+func (m *DeserializedManifest) MarshalJSON() ([]byte, error) {
112
+	if len(m.canonical) > 0 {
113
+		return m.canonical, nil
114
+	}
115
+
116
+	return nil, errors.New("JSON representation not initialized in DeserializedManifest")
117
+}
118
+
119
+// Payload returns the raw content of the manifest. The contents can be used to
120
+// calculate the content identifier.
121
+func (m DeserializedManifest) Payload() (string, []byte, error) {
122
+	return v1.MediaTypeImageManifest, m.canonical, nil
123
+}