Browse code

Merge pull request #405 from thaJeztah/19.03_backport_oci_regression

[19.03 backport] Use ocischema package instead of custom handler

Andrew Hsu authored on 2019/10/29 02:50:08
Showing 4 changed files
1 1
deleted file mode 100644
... ...
@@ -1,29 +0,0 @@
1
-package distribution
2
-
3
-import (
4
-	"fmt"
5
-
6
-	"github.com/docker/distribution"
7
-	"github.com/docker/distribution/manifest/schema2"
8
-	digest "github.com/opencontainers/go-digest"
9
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
10
-)
11
-
12
-func init() {
13
-	// TODO: Remove this registration if distribution is included with OCI support
14
-
15
-	ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
16
-		m := new(schema2.DeserializedManifest)
17
-		err := m.UnmarshalJSON(b)
18
-		if err != nil {
19
-			return nil, distribution.Descriptor{}, err
20
-		}
21
-
22
-		dgst := digest.FromBytes(b)
23
-		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: ocispec.MediaTypeImageManifest}, err
24
-	}
25
-	err := distribution.RegisterManifestSchema(ocispec.MediaTypeImageManifest, ocischemaFunc)
26
-	if err != nil {
27
-		panic(fmt.Sprintf("Unable to register manifest: %s", err))
28
-	}
29
-}
... ...
@@ -14,6 +14,7 @@ import (
14 14
 	"github.com/containerd/containerd/platforms"
15 15
 	"github.com/docker/distribution"
16 16
 	"github.com/docker/distribution/manifest/manifestlist"
17
+	"github.com/docker/distribution/manifest/ocischema"
17 18
 	"github.com/docker/distribution/manifest/schema1"
18 19
 	"github.com/docker/distribution/manifest/schema2"
19 20
 	"github.com/docker/distribution/reference"
... ...
@@ -410,6 +411,11 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, platform
410 410
 		if err != nil {
411 411
 			return false, err
412 412
 		}
413
+	case *ocischema.DeserializedManifest:
414
+		id, manifestDigest, err = p.pullOCI(ctx, ref, v, platform)
415
+		if err != nil {
416
+			return false, err
417
+		}
413 418
 	case *manifestlist.DeserializedManifestList:
414 419
 		id, manifestDigest, err = p.pullManifestList(ctx, ref, v, platform)
415 420
 		if err != nil {
... ...
@@ -557,24 +563,18 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unv
557 557
 	return imageID, manifestDigest, nil
558 558
 }
559 559
 
560
-func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
561
-	manifestDigest, err = schema2ManifestDigest(ref, mfst)
562
-	if err != nil {
563
-		return "", "", err
564
-	}
565
-
566
-	target := mfst.Target()
560
+func (p *v2Puller) pullSchema2Layers(ctx context.Context, target distribution.Descriptor, layers []distribution.Descriptor, platform *specs.Platform) (id digest.Digest, err error) {
567 561
 	if _, err := p.config.ImageStore.Get(target.Digest); err == nil {
568 562
 		// If the image already exists locally, no need to pull
569 563
 		// anything.
570
-		return target.Digest, manifestDigest, nil
564
+		return target.Digest, nil
571 565
 	}
572 566
 
573 567
 	var descriptors []xfer.DownloadDescriptor
574 568
 
575 569
 	// Note that the order of this loop is in the direction of bottom-most
576 570
 	// to top-most, so that the downloads slice gets ordered correctly.
577
-	for _, d := range mfst.Layers {
571
+	for _, d := range layers {
578 572
 		layerDescriptor := &v2LayerDescriptor{
579 573
 			digest:            d.Digest,
580 574
 			repo:              p.repo,
... ...
@@ -629,23 +629,23 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
629 629
 	if runtime.GOOS == "windows" {
630 630
 		configJSON, configRootFS, configPlatform, err = receiveConfig(p.config.ImageStore, configChan, configErrChan)
631 631
 		if err != nil {
632
-			return "", "", err
632
+			return "", err
633 633
 		}
634 634
 		if configRootFS == nil {
635
-			return "", "", errRootFSInvalid
635
+			return "", errRootFSInvalid
636 636
 		}
637 637
 		if err := checkImageCompatibility(configPlatform.OS, configPlatform.OSVersion); err != nil {
638
-			return "", "", err
638
+			return "", err
639 639
 		}
640 640
 
641 641
 		if len(descriptors) != len(configRootFS.DiffIDs) {
642
-			return "", "", errRootFSMismatch
642
+			return "", errRootFSMismatch
643 643
 		}
644 644
 		if platform == nil {
645 645
 			// Early bath if the requested OS doesn't match that of the configuration.
646 646
 			// This avoids doing the download, only to potentially fail later.
647 647
 			if !system.IsOSSupported(configPlatform.OS) {
648
-				return "", "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, layerStoreOS)
648
+				return "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, layerStoreOS)
649 649
 			}
650 650
 			layerStoreOS = configPlatform.OS
651 651
 		}
... ...
@@ -692,14 +692,14 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
692 692
 			case <-downloadsDone:
693 693
 			case <-layerErrChan:
694 694
 			}
695
-			return "", "", err
695
+			return "", err
696 696
 		}
697 697
 	}
698 698
 
699 699
 	select {
700 700
 	case <-downloadsDone:
701 701
 	case err = <-layerErrChan:
702
-		return "", "", err
702
+		return "", err
703 703
 	}
704 704
 
705 705
 	if release != nil {
... ...
@@ -711,22 +711,40 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
711 711
 		// Otherwise the image config could be referencing layers that aren't
712 712
 		// included in the manifest.
713 713
 		if len(downloadedRootFS.DiffIDs) != len(configRootFS.DiffIDs) {
714
-			return "", "", errRootFSMismatch
714
+			return "", errRootFSMismatch
715 715
 		}
716 716
 
717 717
 		for i := range downloadedRootFS.DiffIDs {
718 718
 			if downloadedRootFS.DiffIDs[i] != configRootFS.DiffIDs[i] {
719
-				return "", "", errRootFSMismatch
719
+				return "", errRootFSMismatch
720 720
 			}
721 721
 		}
722 722
 	}
723 723
 
724 724
 	imageID, err := p.config.ImageStore.Put(configJSON)
725 725
 	if err != nil {
726
+		return "", err
727
+	}
728
+
729
+	return imageID, nil
730
+}
731
+
732
+func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
733
+	manifestDigest, err = schema2ManifestDigest(ref, mfst)
734
+	if err != nil {
726 735
 		return "", "", err
727 736
 	}
737
+	id, err = p.pullSchema2Layers(ctx, mfst.Target(), mfst.Layers, platform)
738
+	return id, manifestDigest, err
739
+}
728 740
 
729
-	return imageID, manifestDigest, nil
741
+func (p *v2Puller) pullOCI(ctx context.Context, ref reference.Named, mfst *ocischema.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
742
+	manifestDigest, err = schema2ManifestDigest(ref, mfst)
743
+	if err != nil {
744
+		return "", "", err
745
+	}
746
+	id, err = p.pullSchema2Layers(ctx, mfst.Target(), mfst.Layers, platform)
747
+	return id, manifestDigest, err
730 748
 }
731 749
 
732 750
 func receiveConfig(s ImageConfigStore, configChan <-chan []byte, errChan <-chan error) ([]byte, *image.RootFS, *specs.Platform, error) {
... ...
@@ -811,6 +829,12 @@ func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mf
811 811
 		if err != nil {
812 812
 			return "", "", err
813 813
 		}
814
+	case *ocischema.DeserializedManifest:
815
+		platform := toOCIPlatform(manifestMatches[0].Platform)
816
+		id, _, err = p.pullOCI(ctx, manifestRef, v, &platform)
817
+		if err != nil {
818
+			return "", "", err
819
+		}
814 820
 	default:
815 821
 		return "", "", errors.New("unsupported manifest format")
816 822
 	}
817 823
new file mode 100644
... ...
@@ -0,0 +1,107 @@
0
+package ocischema
1
+
2
+import (
3
+	"context"
4
+	"errors"
5
+
6
+	"github.com/docker/distribution"
7
+	"github.com/docker/distribution/manifest"
8
+	"github.com/opencontainers/go-digest"
9
+	"github.com/opencontainers/image-spec/specs-go/v1"
10
+)
11
+
12
+// Builder is a type for constructing manifests.
13
+type Builder struct {
14
+	// bs is a BlobService used to publish the configuration blob.
15
+	bs distribution.BlobService
16
+
17
+	// configJSON references
18
+	configJSON []byte
19
+
20
+	// layers is a list of layer descriptors that gets built by successive
21
+	// calls to AppendReference.
22
+	layers []distribution.Descriptor
23
+
24
+	// Annotations contains arbitrary metadata relating to the targeted content.
25
+	annotations map[string]string
26
+
27
+	// For testing purposes
28
+	mediaType string
29
+}
30
+
31
+// NewManifestBuilder is used to build new manifests for the current schema
32
+// version. It takes a BlobService so it can publish the configuration blob
33
+// as part of the Build process, and annotations.
34
+func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder {
35
+	mb := &Builder{
36
+		bs:          bs,
37
+		configJSON:  make([]byte, len(configJSON)),
38
+		annotations: annotations,
39
+		mediaType:   v1.MediaTypeImageManifest,
40
+	}
41
+	copy(mb.configJSON, configJSON)
42
+
43
+	return mb
44
+}
45
+
46
+// SetMediaType assigns the passed mediatype or error if the mediatype is not a
47
+// valid media type for oci image manifests currently: "" or "application/vnd.oci.image.manifest.v1+json"
48
+func (mb *Builder) SetMediaType(mediaType string) error {
49
+	if mediaType != "" && mediaType != v1.MediaTypeImageManifest {
50
+		return errors.New("invalid media type for OCI image manifest")
51
+	}
52
+
53
+	mb.mediaType = mediaType
54
+	return nil
55
+}
56
+
57
+// Build produces a final manifest from the given references.
58
+func (mb *Builder) Build(ctx context.Context) (distribution.Manifest, error) {
59
+	m := Manifest{
60
+		Versioned: manifest.Versioned{
61
+			SchemaVersion: 2,
62
+			MediaType:     mb.mediaType,
63
+		},
64
+		Layers:      make([]distribution.Descriptor, len(mb.layers)),
65
+		Annotations: mb.annotations,
66
+	}
67
+	copy(m.Layers, mb.layers)
68
+
69
+	configDigest := digest.FromBytes(mb.configJSON)
70
+
71
+	var err error
72
+	m.Config, err = mb.bs.Stat(ctx, configDigest)
73
+	switch err {
74
+	case nil:
75
+		// Override MediaType, since Put always replaces the specified media
76
+		// type with application/octet-stream in the descriptor it returns.
77
+		m.Config.MediaType = v1.MediaTypeImageConfig
78
+		return FromStruct(m)
79
+	case distribution.ErrBlobUnknown:
80
+		// nop
81
+	default:
82
+		return nil, err
83
+	}
84
+
85
+	// Add config to the blob store
86
+	m.Config, err = mb.bs.Put(ctx, v1.MediaTypeImageConfig, mb.configJSON)
87
+	// Override MediaType, since Put always replaces the specified media
88
+	// type with application/octet-stream in the descriptor it returns.
89
+	m.Config.MediaType = v1.MediaTypeImageConfig
90
+	if err != nil {
91
+		return nil, err
92
+	}
93
+
94
+	return FromStruct(m)
95
+}
96
+
97
+// AppendReference adds a reference to the current ManifestBuilder.
98
+func (mb *Builder) AppendReference(d distribution.Describable) error {
99
+	mb.layers = append(mb.layers, d.Descriptor())
100
+	return nil
101
+}
102
+
103
+// References returns the current references added to this builder.
104
+func (mb *Builder) References() []distribution.Descriptor {
105
+	return mb.layers
106
+}
0 107
new file mode 100644
... ...
@@ -0,0 +1,124 @@
0
+package ocischema
1
+
2
+import (
3
+	"encoding/json"
4
+	"errors"
5
+	"fmt"
6
+
7
+	"github.com/docker/distribution"
8
+	"github.com/docker/distribution/manifest"
9
+	"github.com/opencontainers/go-digest"
10
+	"github.com/opencontainers/image-spec/specs-go/v1"
11
+)
12
+
13
+var (
14
+	// SchemaVersion provides a pre-initialized version structure for this
15
+	// packages version of the manifest.
16
+	SchemaVersion = manifest.Versioned{
17
+		SchemaVersion: 2, // historical value here.. does not pertain to OCI or docker version
18
+		MediaType:     v1.MediaTypeImageManifest,
19
+	}
20
+)
21
+
22
+func init() {
23
+	ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
24
+		m := new(DeserializedManifest)
25
+		err := m.UnmarshalJSON(b)
26
+		if err != nil {
27
+			return nil, distribution.Descriptor{}, err
28
+		}
29
+
30
+		dgst := digest.FromBytes(b)
31
+		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageManifest}, err
32
+	}
33
+	err := distribution.RegisterManifestSchema(v1.MediaTypeImageManifest, ocischemaFunc)
34
+	if err != nil {
35
+		panic(fmt.Sprintf("Unable to register manifest: %s", err))
36
+	}
37
+}
38
+
39
+// Manifest defines a ocischema manifest.
40
+type Manifest struct {
41
+	manifest.Versioned
42
+
43
+	// Config references the image configuration as a blob.
44
+	Config distribution.Descriptor `json:"config"`
45
+
46
+	// Layers lists descriptors for the layers referenced by the
47
+	// configuration.
48
+	Layers []distribution.Descriptor `json:"layers"`
49
+
50
+	// Annotations contains arbitrary metadata for the image manifest.
51
+	Annotations map[string]string `json:"annotations,omitempty"`
52
+}
53
+
54
+// References returns the descriptors of this manifests references.
55
+func (m Manifest) References() []distribution.Descriptor {
56
+	references := make([]distribution.Descriptor, 0, 1+len(m.Layers))
57
+	references = append(references, m.Config)
58
+	references = append(references, m.Layers...)
59
+	return references
60
+}
61
+
62
+// Target returns the target of this manifest.
63
+func (m Manifest) Target() distribution.Descriptor {
64
+	return m.Config
65
+}
66
+
67
+// DeserializedManifest wraps Manifest with a copy of the original JSON.
68
+// It satisfies the distribution.Manifest interface.
69
+type DeserializedManifest struct {
70
+	Manifest
71
+
72
+	// canonical is the canonical byte representation of the Manifest.
73
+	canonical []byte
74
+}
75
+
76
+// FromStruct takes a Manifest structure, marshals it to JSON, and returns a
77
+// DeserializedManifest which contains the manifest and its JSON representation.
78
+func FromStruct(m Manifest) (*DeserializedManifest, error) {
79
+	var deserialized DeserializedManifest
80
+	deserialized.Manifest = m
81
+
82
+	var err error
83
+	deserialized.canonical, err = json.MarshalIndent(&m, "", "   ")
84
+	return &deserialized, err
85
+}
86
+
87
+// UnmarshalJSON populates a new Manifest struct from JSON data.
88
+func (m *DeserializedManifest) UnmarshalJSON(b []byte) error {
89
+	m.canonical = make([]byte, len(b))
90
+	// store manifest in canonical
91
+	copy(m.canonical, b)
92
+
93
+	// Unmarshal canonical JSON into Manifest object
94
+	var manifest Manifest
95
+	if err := json.Unmarshal(m.canonical, &manifest); err != nil {
96
+		return err
97
+	}
98
+
99
+	if manifest.MediaType != "" && manifest.MediaType != v1.MediaTypeImageManifest {
100
+		return fmt.Errorf("if present, mediaType in manifest should be '%s' not '%s'",
101
+			v1.MediaTypeImageManifest, manifest.MediaType)
102
+	}
103
+
104
+	m.Manifest = manifest
105
+
106
+	return nil
107
+}
108
+
109
+// MarshalJSON returns the contents of canonical. If canonical is empty,
110
+// marshals the inner contents.
111
+func (m *DeserializedManifest) MarshalJSON() ([]byte, error) {
112
+	if len(m.canonical) > 0 {
113
+		return m.canonical, nil
114
+	}
115
+
116
+	return nil, errors.New("JSON representation not initialized in DeserializedManifest")
117
+}
118
+
119
+// Payload returns the raw content of the manifest. The contents can be used to
120
+// calculate the content identifier.
121
+func (m DeserializedManifest) Payload() (string, []byte, error) {
122
+	return v1.MediaTypeImageManifest, m.canonical, nil
123
+}