Also renames BlobSumService to V2MetadataService, BlobSum to
V2Metadata
Signed-off-by: Brian Bland <brian.bland@docker.com>
| ... | ... |
@@ -152,7 +152,7 @@ RUN set -x \ |
| 152 | 152 |
# both. This allows integration-cli tests to cover push/pull with both schema1 |
| 153 | 153 |
# and schema2 manifests. |
| 154 | 154 |
ENV REGISTRY_COMMIT_SCHEMA1 ec87e9b6971d831f0eff752ddb54fb64693e51cd |
| 155 |
-ENV REGISTRY_COMMIT 93d9070c8bb28414de9ec96fd38c89614acd8435 |
|
| 155 |
+ENV REGISTRY_COMMIT cb08de17d74bef86ce6c5abe8b240e282f5750be |
|
| 156 | 156 |
RUN set -x \ |
| 157 | 157 |
&& export GOPATH="$(mktemp -d)" \ |
| 158 | 158 |
&& git clone https://github.com/docker/distribution.git "$GOPATH/src/github.com/docker/distribution" \ |
| 159 | 159 |
deleted file mode 100644 |
| ... | ... |
@@ -1,137 +0,0 @@ |
| 1 |
-package metadata |
|
| 2 |
- |
|
| 3 |
-import ( |
|
| 4 |
- "encoding/json" |
|
| 5 |
- |
|
| 6 |
- "github.com/docker/distribution/digest" |
|
| 7 |
- "github.com/docker/docker/layer" |
|
| 8 |
-) |
|
| 9 |
- |
|
| 10 |
-// BlobSumService maps layer IDs to a set of known blobsums for |
|
| 11 |
-// the layer. |
|
| 12 |
-type BlobSumService struct {
|
|
| 13 |
- store Store |
|
| 14 |
-} |
|
| 15 |
- |
|
| 16 |
-// BlobSum contains the digest and source repository information for a layer. |
|
| 17 |
-type BlobSum struct {
|
|
| 18 |
- Digest digest.Digest |
|
| 19 |
- SourceRepository string |
|
| 20 |
-} |
|
| 21 |
- |
|
| 22 |
-// maxBlobSums is the number of blobsums to keep per layer DiffID. |
|
| 23 |
-const maxBlobSums = 50 |
|
| 24 |
- |
|
| 25 |
-// NewBlobSumService creates a new blobsum mapping service. |
|
| 26 |
-func NewBlobSumService(store Store) *BlobSumService {
|
|
| 27 |
- return &BlobSumService{
|
|
| 28 |
- store: store, |
|
| 29 |
- } |
|
| 30 |
-} |
|
| 31 |
- |
|
| 32 |
-func (blobserv *BlobSumService) diffIDNamespace() string {
|
|
| 33 |
- return "blobsum-storage" |
|
| 34 |
-} |
|
| 35 |
- |
|
| 36 |
-func (blobserv *BlobSumService) blobSumNamespace() string {
|
|
| 37 |
- return "blobsum-lookup" |
|
| 38 |
-} |
|
| 39 |
- |
|
| 40 |
-func (blobserv *BlobSumService) diffIDKey(diffID layer.DiffID) string {
|
|
| 41 |
- return string(digest.Digest(diffID).Algorithm()) + "/" + digest.Digest(diffID).Hex() |
|
| 42 |
-} |
|
| 43 |
- |
|
| 44 |
-func (blobserv *BlobSumService) blobSumKey(blobsum BlobSum) string {
|
|
| 45 |
- return string(blobsum.Digest.Algorithm()) + "/" + blobsum.Digest.Hex() |
|
| 46 |
-} |
|
| 47 |
- |
|
| 48 |
-// GetBlobSums finds the blobsums associated with a layer DiffID. |
|
| 49 |
-func (blobserv *BlobSumService) GetBlobSums(diffID layer.DiffID) ([]BlobSum, error) {
|
|
| 50 |
- jsonBytes, err := blobserv.store.Get(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID)) |
|
| 51 |
- if err != nil {
|
|
| 52 |
- return nil, err |
|
| 53 |
- } |
|
| 54 |
- |
|
| 55 |
- var blobsums []BlobSum |
|
| 56 |
- if err := json.Unmarshal(jsonBytes, &blobsums); err != nil {
|
|
| 57 |
- return nil, err |
|
| 58 |
- } |
|
| 59 |
- |
|
| 60 |
- return blobsums, nil |
|
| 61 |
-} |
|
| 62 |
- |
|
| 63 |
-// GetDiffID finds a layer DiffID from a blobsum hash. |
|
| 64 |
-func (blobserv *BlobSumService) GetDiffID(blobsum BlobSum) (layer.DiffID, error) {
|
|
| 65 |
- diffIDBytes, err := blobserv.store.Get(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum)) |
|
| 66 |
- if err != nil {
|
|
| 67 |
- return layer.DiffID(""), err
|
|
| 68 |
- } |
|
| 69 |
- |
|
| 70 |
- return layer.DiffID(diffIDBytes), nil |
|
| 71 |
-} |
|
| 72 |
- |
|
| 73 |
-// Add associates a blobsum with a layer DiffID. If too many blobsums are |
|
| 74 |
-// present, the oldest one is dropped. |
|
| 75 |
-func (blobserv *BlobSumService) Add(diffID layer.DiffID, blobsum BlobSum) error {
|
|
| 76 |
- oldBlobSums, err := blobserv.GetBlobSums(diffID) |
|
| 77 |
- if err != nil {
|
|
| 78 |
- oldBlobSums = nil |
|
| 79 |
- } |
|
| 80 |
- newBlobSums := make([]BlobSum, 0, len(oldBlobSums)+1) |
|
| 81 |
- |
|
| 82 |
- // Copy all other blobsums to new slice |
|
| 83 |
- for _, oldSum := range oldBlobSums {
|
|
| 84 |
- if oldSum != blobsum {
|
|
| 85 |
- newBlobSums = append(newBlobSums, oldSum) |
|
| 86 |
- } |
|
| 87 |
- } |
|
| 88 |
- |
|
| 89 |
- newBlobSums = append(newBlobSums, blobsum) |
|
| 90 |
- |
|
| 91 |
- if len(newBlobSums) > maxBlobSums {
|
|
| 92 |
- newBlobSums = newBlobSums[len(newBlobSums)-maxBlobSums:] |
|
| 93 |
- } |
|
| 94 |
- |
|
| 95 |
- jsonBytes, err := json.Marshal(newBlobSums) |
|
| 96 |
- if err != nil {
|
|
| 97 |
- return err |
|
| 98 |
- } |
|
| 99 |
- |
|
| 100 |
- err = blobserv.store.Set(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID), jsonBytes) |
|
| 101 |
- if err != nil {
|
|
| 102 |
- return err |
|
| 103 |
- } |
|
| 104 |
- |
|
| 105 |
- return blobserv.store.Set(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum), []byte(diffID)) |
|
| 106 |
-} |
|
| 107 |
- |
|
| 108 |
-// Remove unassociates a blobsum from a layer DiffID. |
|
| 109 |
-func (blobserv *BlobSumService) Remove(blobsum BlobSum) error {
|
|
| 110 |
- diffID, err := blobserv.GetDiffID(blobsum) |
|
| 111 |
- if err != nil {
|
|
| 112 |
- return err |
|
| 113 |
- } |
|
| 114 |
- oldBlobSums, err := blobserv.GetBlobSums(diffID) |
|
| 115 |
- if err != nil {
|
|
| 116 |
- oldBlobSums = nil |
|
| 117 |
- } |
|
| 118 |
- newBlobSums := make([]BlobSum, 0, len(oldBlobSums)) |
|
| 119 |
- |
|
| 120 |
- // Copy all other blobsums to new slice |
|
| 121 |
- for _, oldSum := range oldBlobSums {
|
|
| 122 |
- if oldSum != blobsum {
|
|
| 123 |
- newBlobSums = append(newBlobSums, oldSum) |
|
| 124 |
- } |
|
| 125 |
- } |
|
| 126 |
- |
|
| 127 |
- if len(newBlobSums) == 0 {
|
|
| 128 |
- return blobserv.store.Delete(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID)) |
|
| 129 |
- } |
|
| 130 |
- |
|
| 131 |
- jsonBytes, err := json.Marshal(newBlobSums) |
|
| 132 |
- if err != nil {
|
|
| 133 |
- return err |
|
| 134 |
- } |
|
| 135 |
- |
|
| 136 |
- return blobserv.store.Set(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID), jsonBytes) |
|
| 137 |
-} |
| 138 | 1 |
deleted file mode 100644 |
| ... | ... |
@@ -1,115 +0,0 @@ |
| 1 |
-package metadata |
|
| 2 |
- |
|
| 3 |
-import ( |
|
| 4 |
- "encoding/hex" |
|
| 5 |
- "io/ioutil" |
|
| 6 |
- "math/rand" |
|
| 7 |
- "os" |
|
| 8 |
- "reflect" |
|
| 9 |
- "testing" |
|
| 10 |
- |
|
| 11 |
- "github.com/docker/distribution/digest" |
|
| 12 |
- "github.com/docker/docker/layer" |
|
| 13 |
-) |
|
| 14 |
- |
|
| 15 |
-func TestBlobSumService(t *testing.T) {
|
|
| 16 |
- tmpDir, err := ioutil.TempDir("", "blobsum-storage-service-test")
|
|
| 17 |
- if err != nil {
|
|
| 18 |
- t.Fatalf("could not create temp dir: %v", err)
|
|
| 19 |
- } |
|
| 20 |
- defer os.RemoveAll(tmpDir) |
|
| 21 |
- |
|
| 22 |
- metadataStore, err := NewFSMetadataStore(tmpDir) |
|
| 23 |
- if err != nil {
|
|
| 24 |
- t.Fatalf("could not create metadata store: %v", err)
|
|
| 25 |
- } |
|
| 26 |
- blobSumService := NewBlobSumService(metadataStore) |
|
| 27 |
- |
|
| 28 |
- tooManyBlobSums := make([]BlobSum, 100) |
|
| 29 |
- for i := range tooManyBlobSums {
|
|
| 30 |
- randDigest := randomDigest() |
|
| 31 |
- tooManyBlobSums[i] = BlobSum{Digest: randDigest}
|
|
| 32 |
- } |
|
| 33 |
- |
|
| 34 |
- testVectors := []struct {
|
|
| 35 |
- diffID layer.DiffID |
|
| 36 |
- blobsums []BlobSum |
|
| 37 |
- }{
|
|
| 38 |
- {
|
|
| 39 |
- diffID: layer.DiffID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
|
|
| 40 |
- blobsums: []BlobSum{
|
|
| 41 |
- {Digest: digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937")},
|
|
| 42 |
- }, |
|
| 43 |
- }, |
|
| 44 |
- {
|
|
| 45 |
- diffID: layer.DiffID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
|
|
| 46 |
- blobsums: []BlobSum{
|
|
| 47 |
- {Digest: digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937")},
|
|
| 48 |
- {Digest: digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e")},
|
|
| 49 |
- }, |
|
| 50 |
- }, |
|
| 51 |
- {
|
|
| 52 |
- diffID: layer.DiffID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
|
|
| 53 |
- blobsums: tooManyBlobSums, |
|
| 54 |
- }, |
|
| 55 |
- } |
|
| 56 |
- |
|
| 57 |
- // Set some associations |
|
| 58 |
- for _, vec := range testVectors {
|
|
| 59 |
- for _, blobsum := range vec.blobsums {
|
|
| 60 |
- err := blobSumService.Add(vec.diffID, blobsum) |
|
| 61 |
- if err != nil {
|
|
| 62 |
- t.Fatalf("error calling Set: %v", err)
|
|
| 63 |
- } |
|
| 64 |
- } |
|
| 65 |
- } |
|
| 66 |
- |
|
| 67 |
- // Check the correct values are read back |
|
| 68 |
- for _, vec := range testVectors {
|
|
| 69 |
- blobsums, err := blobSumService.GetBlobSums(vec.diffID) |
|
| 70 |
- if err != nil {
|
|
| 71 |
- t.Fatalf("error calling Get: %v", err)
|
|
| 72 |
- } |
|
| 73 |
- expectedBlobsums := len(vec.blobsums) |
|
| 74 |
- if expectedBlobsums > 50 {
|
|
| 75 |
- expectedBlobsums = 50 |
|
| 76 |
- } |
|
| 77 |
- if !reflect.DeepEqual(blobsums, vec.blobsums[len(vec.blobsums)-expectedBlobsums:len(vec.blobsums)]) {
|
|
| 78 |
- t.Fatal("Get returned incorrect layer ID")
|
|
| 79 |
- } |
|
| 80 |
- } |
|
| 81 |
- |
|
| 82 |
- // Test GetBlobSums on a nonexistent entry |
|
| 83 |
- _, err = blobSumService.GetBlobSums(layer.DiffID("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
|
|
| 84 |
- if err == nil {
|
|
| 85 |
- t.Fatal("expected error looking up nonexistent entry")
|
|
| 86 |
- } |
|
| 87 |
- |
|
| 88 |
- // Test GetDiffID on a nonexistent entry |
|
| 89 |
- _, err = blobSumService.GetDiffID(BlobSum{Digest: digest.Digest("sha256:82379823067823853223359023576437723560923756b03560378f4497753917")})
|
|
| 90 |
- if err == nil {
|
|
| 91 |
- t.Fatal("expected error looking up nonexistent entry")
|
|
| 92 |
- } |
|
| 93 |
- |
|
| 94 |
- // Overwrite one of the entries and read it back |
|
| 95 |
- err = blobSumService.Add(testVectors[1].diffID, testVectors[0].blobsums[0]) |
|
| 96 |
- if err != nil {
|
|
| 97 |
- t.Fatalf("error calling Add: %v", err)
|
|
| 98 |
- } |
|
| 99 |
- diffID, err := blobSumService.GetDiffID(testVectors[0].blobsums[0]) |
|
| 100 |
- if err != nil {
|
|
| 101 |
- t.Fatalf("error calling GetDiffID: %v", err)
|
|
| 102 |
- } |
|
| 103 |
- if diffID != testVectors[1].diffID {
|
|
| 104 |
- t.Fatal("GetDiffID returned incorrect diffID")
|
|
| 105 |
- } |
|
| 106 |
-} |
|
| 107 |
- |
|
| 108 |
-func randomDigest() digest.Digest {
|
|
| 109 |
- b := [32]byte{}
|
|
| 110 |
- for i := 0; i < len(b); i++ {
|
|
| 111 |
- b[i] = byte(rand.Intn(256)) |
|
| 112 |
- } |
|
| 113 |
- d := hex.EncodeToString(b[:]) |
|
| 114 |
- return digest.Digest("sha256:" + d)
|
|
| 115 |
-} |
| 116 | 1 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,137 @@ |
| 0 |
+package metadata |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "encoding/json" |
|
| 4 |
+ |
|
| 5 |
+ "github.com/docker/distribution/digest" |
|
| 6 |
+ "github.com/docker/docker/layer" |
|
| 7 |
+) |
|
| 8 |
+ |
|
| 9 |
+// V2MetadataService maps layer IDs to a set of known metadata for |
|
| 10 |
+// the layer. |
|
| 11 |
+type V2MetadataService struct {
|
|
| 12 |
+ store Store |
|
| 13 |
+} |
|
| 14 |
+ |
|
| 15 |
+// V2Metadata contains the digest and source repository information for a layer. |
|
| 16 |
+type V2Metadata struct {
|
|
| 17 |
+ Digest digest.Digest |
|
| 18 |
+ SourceRepository string |
|
| 19 |
+} |
|
| 20 |
+ |
|
| 21 |
+// maxMetadata is the number of metadata entries to keep per layer DiffID. |
|
| 22 |
+const maxMetadata = 50 |
|
| 23 |
+ |
|
| 24 |
+// NewV2MetadataService creates a new diff ID to v2 metadata mapping service. |
|
| 25 |
+func NewV2MetadataService(store Store) *V2MetadataService {
|
|
| 26 |
+ return &V2MetadataService{
|
|
| 27 |
+ store: store, |
|
| 28 |
+ } |
|
| 29 |
+} |
|
| 30 |
+ |
|
| 31 |
+func (serv *V2MetadataService) diffIDNamespace() string {
|
|
| 32 |
+ return "v2metadata-by-diffid" |
|
| 33 |
+} |
|
| 34 |
+ |
|
| 35 |
+func (serv *V2MetadataService) digestNamespace() string {
|
|
| 36 |
+ return "diffid-by-digest" |
|
| 37 |
+} |
|
| 38 |
+ |
|
| 39 |
+func (serv *V2MetadataService) diffIDKey(diffID layer.DiffID) string {
|
|
| 40 |
+ return string(digest.Digest(diffID).Algorithm()) + "/" + digest.Digest(diffID).Hex() |
|
| 41 |
+} |
|
| 42 |
+ |
|
| 43 |
+func (serv *V2MetadataService) digestKey(dgst digest.Digest) string {
|
|
| 44 |
+ return string(dgst.Algorithm()) + "/" + dgst.Hex() |
|
| 45 |
+} |
|
| 46 |
+ |
|
| 47 |
+// GetMetadata finds the metadata associated with a layer DiffID. |
|
| 48 |
+func (serv *V2MetadataService) GetMetadata(diffID layer.DiffID) ([]V2Metadata, error) {
|
|
| 49 |
+ jsonBytes, err := serv.store.Get(serv.diffIDNamespace(), serv.diffIDKey(diffID)) |
|
| 50 |
+ if err != nil {
|
|
| 51 |
+ return nil, err |
|
| 52 |
+ } |
|
| 53 |
+ |
|
| 54 |
+ var metadata []V2Metadata |
|
| 55 |
+ if err := json.Unmarshal(jsonBytes, &metadata); err != nil {
|
|
| 56 |
+ return nil, err |
|
| 57 |
+ } |
|
| 58 |
+ |
|
| 59 |
+ return metadata, nil |
|
| 60 |
+} |
|
| 61 |
+ |
|
| 62 |
+// GetDiffID finds a layer DiffID from a digest. |
|
| 63 |
+func (serv *V2MetadataService) GetDiffID(dgst digest.Digest) (layer.DiffID, error) {
|
|
| 64 |
+ diffIDBytes, err := serv.store.Get(serv.digestNamespace(), serv.digestKey(dgst)) |
|
| 65 |
+ if err != nil {
|
|
| 66 |
+ return layer.DiffID(""), err
|
|
| 67 |
+ } |
|
| 68 |
+ |
|
| 69 |
+ return layer.DiffID(diffIDBytes), nil |
|
| 70 |
+} |
|
| 71 |
+ |
|
| 72 |
+// Add associates metadata with a layer DiffID. If too many metadata entries are |
|
| 73 |
+// present, the oldest one is dropped. |
|
| 74 |
+func (serv *V2MetadataService) Add(diffID layer.DiffID, metadata V2Metadata) error {
|
|
| 75 |
+ oldMetadata, err := serv.GetMetadata(diffID) |
|
| 76 |
+ if err != nil {
|
|
| 77 |
+ oldMetadata = nil |
|
| 78 |
+ } |
|
| 79 |
+ newMetadata := make([]V2Metadata, 0, len(oldMetadata)+1) |
|
| 80 |
+ |
|
| 81 |
+ // Copy all other metadata to new slice |
|
| 82 |
+ for _, oldMeta := range oldMetadata {
|
|
| 83 |
+ if oldMeta != metadata {
|
|
| 84 |
+ newMetadata = append(newMetadata, oldMeta) |
|
| 85 |
+ } |
|
| 86 |
+ } |
|
| 87 |
+ |
|
| 88 |
+ newMetadata = append(newMetadata, metadata) |
|
| 89 |
+ |
|
| 90 |
+ if len(newMetadata) > maxMetadata {
|
|
| 91 |
+ newMetadata = newMetadata[len(newMetadata)-maxMetadata:] |
|
| 92 |
+ } |
|
| 93 |
+ |
|
| 94 |
+ jsonBytes, err := json.Marshal(newMetadata) |
|
| 95 |
+ if err != nil {
|
|
| 96 |
+ return err |
|
| 97 |
+ } |
|
| 98 |
+ |
|
| 99 |
+ err = serv.store.Set(serv.diffIDNamespace(), serv.diffIDKey(diffID), jsonBytes) |
|
| 100 |
+ if err != nil {
|
|
| 101 |
+ return err |
|
| 102 |
+ } |
|
| 103 |
+ |
|
| 104 |
+ return serv.store.Set(serv.digestNamespace(), serv.digestKey(metadata.Digest), []byte(diffID)) |
|
| 105 |
+} |
|
| 106 |
+ |
|
| 107 |
+// Remove unassociates a metadata entry from a layer DiffID. |
|
| 108 |
+func (serv *V2MetadataService) Remove(metadata V2Metadata) error {
|
|
| 109 |
+ diffID, err := serv.GetDiffID(metadata.Digest) |
|
| 110 |
+ if err != nil {
|
|
| 111 |
+ return err |
|
| 112 |
+ } |
|
| 113 |
+ oldMetadata, err := serv.GetMetadata(diffID) |
|
| 114 |
+ if err != nil {
|
|
| 115 |
+ oldMetadata = nil |
|
| 116 |
+ } |
|
| 117 |
+ newMetadata := make([]V2Metadata, 0, len(oldMetadata)) |
|
| 118 |
+ |
|
| 119 |
+ // Copy all other metadata to new slice |
|
| 120 |
+ for _, oldMeta := range oldMetadata {
|
|
| 121 |
+ if oldMeta != metadata {
|
|
| 122 |
+ newMetadata = append(newMetadata, oldMeta) |
|
| 123 |
+ } |
|
| 124 |
+ } |
|
| 125 |
+ |
|
| 126 |
+ if len(newMetadata) == 0 {
|
|
| 127 |
+ return serv.store.Delete(serv.diffIDNamespace(), serv.diffIDKey(diffID)) |
|
| 128 |
+ } |
|
| 129 |
+ |
|
| 130 |
+ jsonBytes, err := json.Marshal(newMetadata) |
|
| 131 |
+ if err != nil {
|
|
| 132 |
+ return err |
|
| 133 |
+ } |
|
| 134 |
+ |
|
| 135 |
+ return serv.store.Set(serv.diffIDNamespace(), serv.diffIDKey(diffID), jsonBytes) |
|
| 136 |
+} |
| 0 | 137 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,115 @@ |
| 0 |
+package metadata |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "encoding/hex" |
|
| 4 |
+ "io/ioutil" |
|
| 5 |
+ "math/rand" |
|
| 6 |
+ "os" |
|
| 7 |
+ "reflect" |
|
| 8 |
+ "testing" |
|
| 9 |
+ |
|
| 10 |
+ "github.com/docker/distribution/digest" |
|
| 11 |
+ "github.com/docker/docker/layer" |
|
| 12 |
+) |
|
| 13 |
+ |
|
| 14 |
+func TestV2MetadataService(t *testing.T) {
|
|
| 15 |
+ tmpDir, err := ioutil.TempDir("", "blobsum-storage-service-test")
|
|
| 16 |
+ if err != nil {
|
|
| 17 |
+ t.Fatalf("could not create temp dir: %v", err)
|
|
| 18 |
+ } |
|
| 19 |
+ defer os.RemoveAll(tmpDir) |
|
| 20 |
+ |
|
| 21 |
+ metadataStore, err := NewFSMetadataStore(tmpDir) |
|
| 22 |
+ if err != nil {
|
|
| 23 |
+ t.Fatalf("could not create metadata store: %v", err)
|
|
| 24 |
+ } |
|
| 25 |
+ V2MetadataService := NewV2MetadataService(metadataStore) |
|
| 26 |
+ |
|
| 27 |
+ tooManyBlobSums := make([]V2Metadata, 100) |
|
| 28 |
+ for i := range tooManyBlobSums {
|
|
| 29 |
+ randDigest := randomDigest() |
|
| 30 |
+ tooManyBlobSums[i] = V2Metadata{Digest: randDigest}
|
|
| 31 |
+ } |
|
| 32 |
+ |
|
| 33 |
+ testVectors := []struct {
|
|
| 34 |
+ diffID layer.DiffID |
|
| 35 |
+ metadata []V2Metadata |
|
| 36 |
+ }{
|
|
| 37 |
+ {
|
|
| 38 |
+ diffID: layer.DiffID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
|
|
| 39 |
+ metadata: []V2Metadata{
|
|
| 40 |
+ {Digest: digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937")},
|
|
| 41 |
+ }, |
|
| 42 |
+ }, |
|
| 43 |
+ {
|
|
| 44 |
+ diffID: layer.DiffID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
|
|
| 45 |
+ metadata: []V2Metadata{
|
|
| 46 |
+ {Digest: digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937")},
|
|
| 47 |
+ {Digest: digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e")},
|
|
| 48 |
+ }, |
|
| 49 |
+ }, |
|
| 50 |
+ {
|
|
| 51 |
+ diffID: layer.DiffID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
|
|
| 52 |
+ metadata: tooManyBlobSums, |
|
| 53 |
+ }, |
|
| 54 |
+ } |
|
| 55 |
+ |
|
| 56 |
+ // Set some associations |
|
| 57 |
+ for _, vec := range testVectors {
|
|
| 58 |
+ for _, blobsum := range vec.metadata {
|
|
| 59 |
+ err := V2MetadataService.Add(vec.diffID, blobsum) |
|
| 60 |
+ if err != nil {
|
|
| 61 |
+ t.Fatalf("error calling Set: %v", err)
|
|
| 62 |
+ } |
|
| 63 |
+ } |
|
| 64 |
+ } |
|
| 65 |
+ |
|
| 66 |
+ // Check the correct values are read back |
|
| 67 |
+ for _, vec := range testVectors {
|
|
| 68 |
+ metadata, err := V2MetadataService.GetMetadata(vec.diffID) |
|
| 69 |
+ if err != nil {
|
|
| 70 |
+ t.Fatalf("error calling Get: %v", err)
|
|
| 71 |
+ } |
|
| 72 |
+ expectedMetadataEntries := len(vec.metadata) |
|
| 73 |
+ if expectedMetadataEntries > 50 {
|
|
| 74 |
+ expectedMetadataEntries = 50 |
|
| 75 |
+ } |
|
| 76 |
+ if !reflect.DeepEqual(metadata, vec.metadata[len(vec.metadata)-expectedMetadataEntries:len(vec.metadata)]) {
|
|
| 77 |
+ t.Fatal("Get returned incorrect layer ID")
|
|
| 78 |
+ } |
|
| 79 |
+ } |
|
| 80 |
+ |
|
| 81 |
+ // Test GetMetadata on a nonexistent entry |
|
| 82 |
+ _, err = V2MetadataService.GetMetadata(layer.DiffID("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
|
|
| 83 |
+ if err == nil {
|
|
| 84 |
+ t.Fatal("expected error looking up nonexistent entry")
|
|
| 85 |
+ } |
|
| 86 |
+ |
|
| 87 |
+ // Test GetDiffID on a nonexistent entry |
|
| 88 |
+ _, err = V2MetadataService.GetDiffID(digest.Digest("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
|
|
| 89 |
+ if err == nil {
|
|
| 90 |
+ t.Fatal("expected error looking up nonexistent entry")
|
|
| 91 |
+ } |
|
| 92 |
+ |
|
| 93 |
+ // Overwrite one of the entries and read it back |
|
| 94 |
+ err = V2MetadataService.Add(testVectors[1].diffID, testVectors[0].metadata[0]) |
|
| 95 |
+ if err != nil {
|
|
| 96 |
+ t.Fatalf("error calling Add: %v", err)
|
|
| 97 |
+ } |
|
| 98 |
+ diffID, err := V2MetadataService.GetDiffID(testVectors[0].metadata[0].Digest) |
|
| 99 |
+ if err != nil {
|
|
| 100 |
+ t.Fatalf("error calling GetDiffID: %v", err)
|
|
| 101 |
+ } |
|
| 102 |
+ if diffID != testVectors[1].diffID {
|
|
| 103 |
+ t.Fatal("GetDiffID returned incorrect diffID")
|
|
| 104 |
+ } |
|
| 105 |
+} |
|
| 106 |
+ |
|
| 107 |
+func randomDigest() digest.Digest {
|
|
| 108 |
+ b := [32]byte{}
|
|
| 109 |
+ for i := 0; i < len(b); i++ {
|
|
| 110 |
+ b[i] = byte(rand.Intn(256)) |
|
| 111 |
+ } |
|
| 112 |
+ d := hex.EncodeToString(b[:]) |
|
| 113 |
+ return digest.Digest("sha256:" + d)
|
|
| 114 |
+} |
| ... | ... |
@@ -61,10 +61,10 @@ func newPuller(endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, |
| 61 | 61 |
switch endpoint.Version {
|
| 62 | 62 |
case registry.APIVersion2: |
| 63 | 63 |
return &v2Puller{
|
| 64 |
- blobSumService: metadata.NewBlobSumService(imagePullConfig.MetadataStore), |
|
| 65 |
- endpoint: endpoint, |
|
| 66 |
- config: imagePullConfig, |
|
| 67 |
- repoInfo: repoInfo, |
|
| 64 |
+ V2MetadataService: metadata.NewV2MetadataService(imagePullConfig.MetadataStore), |
|
| 65 |
+ endpoint: endpoint, |
|
| 66 |
+ config: imagePullConfig, |
|
| 67 |
+ repoInfo: repoInfo, |
|
| 68 | 68 |
}, nil |
| 69 | 69 |
case registry.APIVersion1: |
| 70 | 70 |
return &v1Puller{
|
| ... | ... |
@@ -33,11 +33,11 @@ import ( |
| 33 | 33 |
var errRootFSMismatch = errors.New("layers from manifest don't match image configuration")
|
| 34 | 34 |
|
| 35 | 35 |
type v2Puller struct {
|
| 36 |
- blobSumService *metadata.BlobSumService |
|
| 37 |
- endpoint registry.APIEndpoint |
|
| 38 |
- config *ImagePullConfig |
|
| 39 |
- repoInfo *registry.RepositoryInfo |
|
| 40 |
- repo distribution.Repository |
|
| 36 |
+ V2MetadataService *metadata.V2MetadataService |
|
| 37 |
+ endpoint registry.APIEndpoint |
|
| 38 |
+ config *ImagePullConfig |
|
| 39 |
+ repoInfo *registry.RepositoryInfo |
|
| 40 |
+ repo distribution.Repository |
|
| 41 | 41 |
// confirmedV2 is set to true if we confirm we're talking to a v2 |
| 42 | 42 |
// registry. This is used to limit fallbacks to the v1 protocol. |
| 43 | 43 |
confirmedV2 bool |
| ... | ... |
@@ -110,10 +110,10 @@ func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named) (e |
| 110 | 110 |
} |
| 111 | 111 |
|
| 112 | 112 |
type v2LayerDescriptor struct {
|
| 113 |
- digest digest.Digest |
|
| 114 |
- repoInfo *registry.RepositoryInfo |
|
| 115 |
- repo distribution.Repository |
|
| 116 |
- blobSumService *metadata.BlobSumService |
|
| 113 |
+ digest digest.Digest |
|
| 114 |
+ repoInfo *registry.RepositoryInfo |
|
| 115 |
+ repo distribution.Repository |
|
| 116 |
+ V2MetadataService *metadata.V2MetadataService |
|
| 117 | 117 |
} |
| 118 | 118 |
|
| 119 | 119 |
func (ld *v2LayerDescriptor) Key() string {
|
| ... | ... |
@@ -125,7 +125,7 @@ func (ld *v2LayerDescriptor) ID() string {
|
| 125 | 125 |
} |
| 126 | 126 |
|
| 127 | 127 |
func (ld *v2LayerDescriptor) DiffID() (layer.DiffID, error) {
|
| 128 |
- return ld.blobSumService.GetDiffID(metadata.BlobSum{Digest: ld.digest, SourceRepository: ld.repoInfo.FullName()})
|
|
| 128 |
+ return ld.V2MetadataService.GetDiffID(ld.digest) |
|
| 129 | 129 |
} |
| 130 | 130 |
|
| 131 | 131 |
func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) {
|
| ... | ... |
@@ -197,7 +197,7 @@ func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progre |
| 197 | 197 |
|
| 198 | 198 |
func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) {
|
| 199 | 199 |
// Cache mapping from this layer's DiffID to the blobsum |
| 200 |
- ld.blobSumService.Add(diffID, metadata.BlobSum{Digest: ld.digest, SourceRepository: ld.repoInfo.FullName()})
|
|
| 200 |
+ ld.V2MetadataService.Add(diffID, metadata.V2Metadata{Digest: ld.digest, SourceRepository: ld.repoInfo.FullName()})
|
|
| 201 | 201 |
} |
| 202 | 202 |
|
| 203 | 203 |
func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdated bool, err error) {
|
| ... | ... |
@@ -334,10 +334,10 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverif |
| 334 | 334 |
} |
| 335 | 335 |
|
| 336 | 336 |
layerDescriptor := &v2LayerDescriptor{
|
| 337 |
- digest: blobSum, |
|
| 338 |
- repoInfo: p.repoInfo, |
|
| 339 |
- repo: p.repo, |
|
| 340 |
- blobSumService: p.blobSumService, |
|
| 337 |
+ digest: blobSum, |
|
| 338 |
+ repoInfo: p.repoInfo, |
|
| 339 |
+ repo: p.repo, |
|
| 340 |
+ V2MetadataService: p.V2MetadataService, |
|
| 341 | 341 |
} |
| 342 | 342 |
|
| 343 | 343 |
descriptors = append(descriptors, layerDescriptor) |
| ... | ... |
@@ -400,10 +400,10 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s |
| 400 | 400 |
// to top-most, so that the downloads slice gets ordered correctly. |
| 401 | 401 |
for _, d := range mfst.References() {
|
| 402 | 402 |
layerDescriptor := &v2LayerDescriptor{
|
| 403 |
- digest: d.Digest, |
|
| 404 |
- repo: p.repo, |
|
| 405 |
- repoInfo: p.repoInfo, |
|
| 406 |
- blobSumService: p.blobSumService, |
|
| 403 |
+ digest: d.Digest, |
|
| 404 |
+ repo: p.repo, |
|
| 405 |
+ repoInfo: p.repoInfo, |
|
| 406 |
+ V2MetadataService: p.V2MetadataService, |
|
| 407 | 407 |
} |
| 408 | 408 |
|
| 409 | 409 |
descriptors = append(descriptors, layerDescriptor) |
| ... | ... |
@@ -71,11 +71,11 @@ func NewPusher(ref reference.Named, endpoint registry.APIEndpoint, repoInfo *reg |
| 71 | 71 |
switch endpoint.Version {
|
| 72 | 72 |
case registry.APIVersion2: |
| 73 | 73 |
return &v2Pusher{
|
| 74 |
- blobSumService: metadata.NewBlobSumService(imagePushConfig.MetadataStore), |
|
| 75 |
- ref: ref, |
|
| 76 |
- endpoint: endpoint, |
|
| 77 |
- repoInfo: repoInfo, |
|
| 78 |
- config: imagePushConfig, |
|
| 74 |
+ v2MetadataService: metadata.NewV2MetadataService(imagePushConfig.MetadataStore), |
|
| 75 |
+ ref: ref, |
|
| 76 |
+ endpoint: endpoint, |
|
| 77 |
+ repoInfo: repoInfo, |
|
| 78 |
+ config: imagePushConfig, |
|
| 79 | 79 |
}, nil |
| 80 | 80 |
case registry.APIVersion1: |
| 81 | 81 |
return &v1Pusher{
|
| ... | ... |
@@ -11,6 +11,7 @@ import ( |
| 11 | 11 |
"github.com/docker/distribution/digest" |
| 12 | 12 |
"github.com/docker/distribution/manifest/schema1" |
| 13 | 13 |
"github.com/docker/distribution/manifest/schema2" |
| 14 |
+ distreference "github.com/docker/distribution/reference" |
|
| 14 | 15 |
"github.com/docker/distribution/registry/client" |
| 15 | 16 |
"github.com/docker/docker/distribution/metadata" |
| 16 | 17 |
"github.com/docker/docker/distribution/xfer" |
| ... | ... |
@@ -34,12 +35,12 @@ type PushResult struct {
|
| 34 | 34 |
} |
| 35 | 35 |
|
| 36 | 36 |
type v2Pusher struct {
|
| 37 |
- blobSumService *metadata.BlobSumService |
|
| 38 |
- ref reference.Named |
|
| 39 |
- endpoint registry.APIEndpoint |
|
| 40 |
- repoInfo *registry.RepositoryInfo |
|
| 41 |
- config *ImagePushConfig |
|
| 42 |
- repo distribution.Repository |
|
| 37 |
+ v2MetadataService *metadata.V2MetadataService |
|
| 38 |
+ ref reference.Named |
|
| 39 |
+ endpoint registry.APIEndpoint |
|
| 40 |
+ repoInfo *registry.RepositoryInfo |
|
| 41 |
+ config *ImagePushConfig |
|
| 42 |
+ repo distribution.Repository |
|
| 43 | 43 |
|
| 44 | 44 |
// pushState is state built by the Download functions. |
| 45 | 45 |
pushState pushState |
| ... | ... |
@@ -130,10 +131,10 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, ima |
| 130 | 130 |
var descriptors []xfer.UploadDescriptor |
| 131 | 131 |
|
| 132 | 132 |
descriptorTemplate := v2PushDescriptor{
|
| 133 |
- blobSumService: p.blobSumService, |
|
| 134 |
- repoInfo: p.repoInfo, |
|
| 135 |
- repo: p.repo, |
|
| 136 |
- pushState: &p.pushState, |
|
| 133 |
+ v2MetadataService: p.v2MetadataService, |
|
| 134 |
+ repoInfo: p.repoInfo, |
|
| 135 |
+ repo: p.repo, |
|
| 136 |
+ pushState: &p.pushState, |
|
| 137 | 137 |
} |
| 138 | 138 |
|
| 139 | 139 |
// Loop bounds condition is to avoid pushing the base layer on Windows. |
| ... | ... |
@@ -210,11 +211,11 @@ func manifestFromBuilder(ctx context.Context, builder distribution.ManifestBuild |
| 210 | 210 |
} |
| 211 | 211 |
|
| 212 | 212 |
type v2PushDescriptor struct {
|
| 213 |
- layer layer.Layer |
|
| 214 |
- blobSumService *metadata.BlobSumService |
|
| 215 |
- repoInfo reference.Named |
|
| 216 |
- repo distribution.Repository |
|
| 217 |
- pushState *pushState |
|
| 213 |
+ layer layer.Layer |
|
| 214 |
+ v2MetadataService *metadata.V2MetadataService |
|
| 215 |
+ repoInfo reference.Named |
|
| 216 |
+ repo distribution.Repository |
|
| 217 |
+ pushState *pushState |
|
| 218 | 218 |
} |
| 219 | 219 |
|
| 220 | 220 |
func (pd *v2PushDescriptor) Key() string {
|
| ... | ... |
@@ -242,10 +243,10 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. |
| 242 | 242 |
} |
| 243 | 243 |
pd.pushState.Unlock() |
| 244 | 244 |
|
| 245 |
- // Do we have any blobsums associated with this layer's DiffID? |
|
| 246 |
- possibleBlobsums, err := pd.blobSumService.GetBlobSums(diffID) |
|
| 245 |
+ // Do we have any metadata associated with this layer's DiffID? |
|
| 246 |
+ v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID) |
|
| 247 | 247 |
if err == nil {
|
| 248 |
- descriptor, exists, err := blobSumAlreadyExists(ctx, possibleBlobsums, pd.repoInfo, pd.repo, pd.pushState) |
|
| 248 |
+ descriptor, exists, err := layerAlreadyExists(ctx, v2Metadata, pd.repoInfo, pd.repo, pd.pushState) |
|
| 249 | 249 |
if err != nil {
|
| 250 | 250 |
progress.Update(progressOutput, pd.ID(), "Image push failed") |
| 251 | 251 |
return retryOnError(err) |
| ... | ... |
@@ -265,39 +266,69 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. |
| 265 | 265 |
// then push the blob. |
| 266 | 266 |
bs := pd.repo.Blobs(ctx) |
| 267 | 267 |
|
| 268 |
+ var mountFrom metadata.V2Metadata |
|
| 269 |
+ |
|
| 268 | 270 |
// Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload |
| 269 |
- for _, blobsum := range possibleBlobsums {
|
|
| 270 |
- sourceRepo, err := reference.ParseNamed(blobsum.SourceRepository) |
|
| 271 |
+ for _, metadata := range v2Metadata {
|
|
| 272 |
+ sourceRepo, err := reference.ParseNamed(metadata.SourceRepository) |
|
| 271 | 273 |
if err != nil {
|
| 272 | 274 |
continue |
| 273 | 275 |
} |
| 274 | 276 |
if pd.repoInfo.Hostname() == sourceRepo.Hostname() {
|
| 275 |
- logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, blobsum.Digest, sourceRepo.FullName())
|
|
| 277 |
+ logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, metadata.Digest, sourceRepo.FullName())
|
|
| 278 |
+ mountFrom = metadata |
|
| 279 |
+ break |
|
| 280 |
+ } |
|
| 281 |
+ } |
|
| 276 | 282 |
|
| 277 |
- desc, err := bs.Mount(ctx, sourceRepo.RemoteName(), blobsum.Digest) |
|
| 278 |
- if err == nil {
|
|
| 279 |
- progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", sourceRepo.RemoteName()) |
|
| 283 |
+ var createOpts []distribution.BlobCreateOption |
|
| 280 | 284 |
|
| 281 |
- pd.pushState.Lock() |
|
| 282 |
- pd.pushState.confirmedV2 = true |
|
| 283 |
- pd.pushState.remoteLayers[diffID] = desc |
|
| 284 |
- pd.pushState.Unlock() |
|
| 285 |
+ if mountFrom.SourceRepository != "" {
|
|
| 286 |
+ namedRef, err := reference.WithName(mountFrom.SourceRepository) |
|
| 287 |
+ if err != nil {
|
|
| 288 |
+ return err |
|
| 289 |
+ } |
|
| 285 | 290 |
|
| 286 |
- // Cache mapping from this layer's DiffID to the blobsum |
|
| 287 |
- if err := pd.blobSumService.Add(diffID, metadata.BlobSum{Digest: blobsum.Digest, SourceRepository: pd.repoInfo.FullName()}); err != nil {
|
|
| 288 |
- return xfer.DoNotRetry{Err: err}
|
|
| 289 |
- } |
|
| 291 |
+ // TODO (brianbland): We need to construct a reference where the Name is |
|
| 292 |
+ // only the full remote name, so clean this up when distribution has a |
|
| 293 |
+ // richer reference package |
|
| 294 |
+ remoteRef, err := distreference.WithName(namedRef.RemoteName()) |
|
| 295 |
+ if err != nil {
|
|
| 296 |
+ return err |
|
| 297 |
+ } |
|
| 290 | 298 |
|
| 291 |
- return nil |
|
| 292 |
- } |
|
| 293 |
- // Unable to mount layer from this repository, so this source mapping is no longer valid |
|
| 294 |
- logrus.Debugf("unassociating layer %s (%s) with %s", diffID, blobsum.Digest, sourceRepo.FullName())
|
|
| 295 |
- pd.blobSumService.Remove(blobsum) |
|
| 299 |
+ canonicalRef, err := distreference.WithDigest(remoteRef, mountFrom.Digest) |
|
| 300 |
+ if err != nil {
|
|
| 301 |
+ return err |
|
| 296 | 302 |
} |
| 303 |
+ |
|
| 304 |
+ createOpts = append(createOpts, client.WithMountFrom(canonicalRef)) |
|
| 297 | 305 |
} |
| 298 | 306 |
|
| 299 | 307 |
// Send the layer |
| 300 |
- layerUpload, err := bs.Create(ctx) |
|
| 308 |
+ layerUpload, err := bs.Create(ctx, createOpts...) |
|
| 309 |
+ switch err := err.(type) {
|
|
| 310 |
+ case distribution.ErrBlobMounted: |
|
| 311 |
+ progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name()) |
|
| 312 |
+ |
|
| 313 |
+ pd.pushState.Lock() |
|
| 314 |
+ pd.pushState.confirmedV2 = true |
|
| 315 |
+ pd.pushState.remoteLayers[diffID] = err.Descriptor |
|
| 316 |
+ pd.pushState.Unlock() |
|
| 317 |
+ |
|
| 318 |
+ // Cache mapping from this layer's DiffID to the blobsum |
|
| 319 |
+ if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: mountFrom.Digest, SourceRepository: pd.repoInfo.FullName()}); err != nil {
|
|
| 320 |
+ return xfer.DoNotRetry{Err: err}
|
|
| 321 |
+ } |
|
| 322 |
+ |
|
| 323 |
+ return nil |
|
| 324 |
+ } |
|
| 325 |
+ if mountFrom.SourceRepository != "" {
|
|
| 326 |
+ // unable to mount layer from this repository, so this source mapping is no longer valid |
|
| 327 |
+ logrus.Debugf("unassociating layer %s (%s) with %s", diffID, mountFrom.Digest, mountFrom.SourceRepository)
|
|
| 328 |
+ pd.v2MetadataService.Remove(mountFrom) |
|
| 329 |
+ } |
|
| 330 |
+ |
|
| 301 | 331 |
if err != nil {
|
| 302 | 332 |
return retryOnError(err) |
| 303 | 333 |
} |
| ... | ... |
@@ -333,7 +364,7 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. |
| 333 | 333 |
progress.Update(progressOutput, pd.ID(), "Pushed") |
| 334 | 334 |
|
| 335 | 335 |
// Cache mapping from this layer's DiffID to the blobsum |
| 336 |
- if err := pd.blobSumService.Add(diffID, metadata.BlobSum{Digest: pushDigest, SourceRepository: pd.repoInfo.FullName()}); err != nil {
|
|
| 336 |
+ if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: pushDigest, SourceRepository: pd.repoInfo.FullName()}); err != nil {
|
|
| 337 | 337 |
return xfer.DoNotRetry{Err: err}
|
| 338 | 338 |
} |
| 339 | 339 |
|
| ... | ... |
@@ -362,16 +393,16 @@ func (pd *v2PushDescriptor) Descriptor() distribution.Descriptor {
|
| 362 | 362 |
return pd.pushState.remoteLayers[pd.DiffID()] |
| 363 | 363 |
} |
| 364 | 364 |
|
| 365 |
-// blobSumAlreadyExists checks if the registry already know about any of the |
|
| 366 |
-// blobsums passed in the "blobsums" slice. If it finds one that the registry |
|
| 365 |
+// layerAlreadyExists checks if the registry already know about any of the |
|
| 366 |
+// metadata passed in the "metadata" slice. If it finds one that the registry |
|
| 367 | 367 |
// knows about, it returns the known digest and "true". |
| 368 |
-func blobSumAlreadyExists(ctx context.Context, blobsums []metadata.BlobSum, repoInfo reference.Named, repo distribution.Repository, pushState *pushState) (distribution.Descriptor, bool, error) {
|
|
| 369 |
- for _, blobSum := range blobsums {
|
|
| 368 |
+func layerAlreadyExists(ctx context.Context, metadata []metadata.V2Metadata, repoInfo reference.Named, repo distribution.Repository, pushState *pushState) (distribution.Descriptor, bool, error) {
|
|
| 369 |
+ for _, meta := range metadata {
|
|
| 370 | 370 |
// Only check blobsums that are known to this repository or have an unknown source |
| 371 |
- if blobSum.SourceRepository != "" && blobSum.SourceRepository != repoInfo.FullName() {
|
|
| 371 |
+ if meta.SourceRepository != "" && meta.SourceRepository != repoInfo.FullName() {
|
|
| 372 | 372 |
continue |
| 373 | 373 |
} |
| 374 |
- descriptor, err := repo.Blobs(ctx).Stat(ctx, blobSum.Digest) |
|
| 374 |
+ descriptor, err := repo.Blobs(ctx).Stat(ctx, meta.Digest) |
|
| 375 | 375 |
switch err {
|
| 376 | 376 |
case nil: |
| 377 | 377 |
descriptor.MediaType = schema2.MediaTypeLayer |
| ... | ... |
@@ -44,7 +44,7 @@ clone git github.com/boltdb/bolt v1.1.0 |
| 44 | 44 |
clone git github.com/miekg/dns d27455715200c7d3e321a1e5cadb27c9ee0b0f02 |
| 45 | 45 |
|
| 46 | 46 |
# get graph and distribution packages |
| 47 |
-clone git github.com/docker/distribution 93d9070c8bb28414de9ec96fd38c89614acd8435 |
|
| 47 |
+clone git github.com/docker/distribution cb08de17d74bef86ce6c5abe8b240e282f5750be |
|
| 48 | 48 |
clone git github.com/vbatts/tar-split v0.9.11 |
| 49 | 49 |
|
| 50 | 50 |
# get desired notary commit, might also need to be updated in Dockerfile |
| ... | ... |
@@ -166,9 +166,11 @@ func (s *DockerRegistrySuite) TestCrossRepositoryLayerPush(c *check.C) {
|
| 166 | 166 |
// ensure that layers were mounted from the first repo during push |
| 167 | 167 |
c.Assert(strings.Contains(out2, "Mounted from dockercli/busybox"), check.Equals, true) |
| 168 | 168 |
|
| 169 |
- // ensure that we can pull the cross-repo-pushed repository |
|
| 169 |
+ // ensure that we can pull and run the cross-repo-pushed repository |
|
| 170 | 170 |
dockerCmd(c, "rmi", destRepoName) |
| 171 | 171 |
dockerCmd(c, "pull", destRepoName) |
| 172 |
+ out3, _ := dockerCmd(c, "run", destRepoName, "echo", "-n", "hello world") |
|
| 173 |
+ c.Assert(out3, check.Equals, "hello world") |
|
| 172 | 174 |
} |
| 173 | 175 |
|
| 174 | 176 |
func (s *DockerSchema1RegistrySuite) TestCrossRepositoryLayerPushNotSupported(c *check.C) {
|
| ... | ... |
@@ -190,9 +192,11 @@ func (s *DockerSchema1RegistrySuite) TestCrossRepositoryLayerPushNotSupported(c |
| 190 | 190 |
// schema1 registry should not support cross-repo layer mounts, so ensure that this does not happen |
| 191 | 191 |
c.Assert(strings.Contains(out2, "Mounted from dockercli/busybox"), check.Equals, false) |
| 192 | 192 |
|
| 193 |
- // ensure that we can pull the second pushed repository |
|
| 193 |
+ // ensure that we can pull and run the second pushed repository |
|
| 194 | 194 |
dockerCmd(c, "rmi", destRepoName) |
| 195 | 195 |
dockerCmd(c, "pull", destRepoName) |
| 196 |
+ out3, _ := dockerCmd(c, "run", destRepoName, "echo", "-n", "hello world") |
|
| 197 |
+ c.Assert(out3, check.Equals, "hello world") |
|
| 196 | 198 |
} |
| 197 | 199 |
|
| 198 | 200 |
func (s *DockerTrustSuite) TestTrustedPush(c *check.C) {
|
| ... | ... |
@@ -476,8 +476,8 @@ func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metad |
| 476 | 476 |
if err == nil { // best effort
|
| 477 | 477 |
dgst, err := digest.ParseDigest(string(checksum)) |
| 478 | 478 |
if err == nil {
|
| 479 |
- blobSumService := metadata.NewBlobSumService(ms) |
|
| 480 |
- blobSumService.Add(layer.DiffID(), metadata.BlobSum{Digest: dgst})
|
|
| 479 |
+ V2MetadataService := metadata.NewV2MetadataService(ms) |
|
| 480 |
+ V2MetadataService.Add(layer.DiffID(), metadata.V2Metadata{Digest: dgst})
|
|
| 481 | 481 |
} |
| 482 | 482 |
} |
| 483 | 483 |
_, err = ls.Release(layer) |
| ... | ... |
@@ -210,19 +210,19 @@ func TestMigrateImages(t *testing.T) {
|
| 210 | 210 |
t.Fatalf("invalid register count: expected %q, got %q", expected, actual)
|
| 211 | 211 |
} |
| 212 | 212 |
|
| 213 |
- blobSumService := metadata.NewBlobSumService(ms) |
|
| 214 |
- blobsums, err := blobSumService.GetBlobSums(layer.EmptyLayer.DiffID()) |
|
| 213 |
+ v2MetadataService := metadata.NewV2MetadataService(ms) |
|
| 214 |
+ receivedMetadata, err := v2MetadataService.GetMetadata(layer.EmptyLayer.DiffID()) |
|
| 215 | 215 |
if err != nil {
|
| 216 | 216 |
t.Fatal(err) |
| 217 | 217 |
} |
| 218 | 218 |
|
| 219 |
- expectedBlobsums := []metadata.BlobSum{
|
|
| 219 |
+ expectedMetadata := []metadata.V2Metadata{
|
|
| 220 | 220 |
{Digest: digest.Digest("sha256:55dc925c23d1ed82551fd018c27ac3ee731377b6bad3963a2a4e76e753d70e57")},
|
| 221 | 221 |
{Digest: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
|
| 222 | 222 |
} |
| 223 | 223 |
|
| 224 |
- if !reflect.DeepEqual(expectedBlobsums, blobsums) {
|
|
| 225 |
- t.Fatalf("invalid blobsums: expected %q, got %q", expectedBlobsums, blobsums)
|
|
| 224 |
+ if !reflect.DeepEqual(expectedMetadata, receivedMetadata) {
|
|
| 225 |
+ t.Fatalf("invalid metadata: expected %q, got %q", expectedMetadata, receivedMetadata)
|
|
| 226 | 226 |
} |
| 227 | 227 |
|
| 228 | 228 |
} |
| ... | ... |
@@ -9,6 +9,7 @@ import ( |
| 9 | 9 |
|
| 10 | 10 |
"github.com/docker/distribution/context" |
| 11 | 11 |
"github.com/docker/distribution/digest" |
| 12 |
+ "github.com/docker/distribution/reference" |
|
| 12 | 13 |
) |
| 13 | 14 |
|
| 14 | 15 |
var ( |
| ... | ... |
@@ -40,6 +41,18 @@ func (err ErrBlobInvalidDigest) Error() string {
|
| 40 | 40 |
err.Digest, err.Reason) |
| 41 | 41 |
} |
| 42 | 42 |
|
| 43 |
+// ErrBlobMounted returned when a blob is mounted from another repository |
|
| 44 |
+// instead of initiating an upload session. |
|
| 45 |
+type ErrBlobMounted struct {
|
|
| 46 |
+ From reference.Canonical |
|
| 47 |
+ Descriptor Descriptor |
|
| 48 |
+} |
|
| 49 |
+ |
|
| 50 |
+func (err ErrBlobMounted) Error() string {
|
|
| 51 |
+ return fmt.Sprintf("blob mounted from: %v to: %v",
|
|
| 52 |
+ err.From, err.Descriptor) |
|
| 53 |
+} |
|
| 54 |
+ |
|
| 43 | 55 |
// Descriptor describes targeted content. Used in conjunction with a blob |
| 44 | 56 |
// store, a descriptor can be used to fetch, store and target any kind of |
| 45 | 57 |
// blob. The struct also describes the wire protocol format. Fields should |
| ... | ... |
@@ -151,14 +164,19 @@ type BlobIngester interface {
|
| 151 | 151 |
// returned handle can be written to and later resumed using an opaque |
| 152 | 152 |
// identifier. With this approach, one can Close and Resume a BlobWriter |
| 153 | 153 |
// multiple times until the BlobWriter is committed or cancelled. |
| 154 |
- Create(ctx context.Context) (BlobWriter, error) |
|
| 154 |
+ Create(ctx context.Context, options ...BlobCreateOption) (BlobWriter, error) |
|
| 155 | 155 |
|
| 156 | 156 |
// Resume attempts to resume a write to a blob, identified by an id. |
| 157 | 157 |
Resume(ctx context.Context, id string) (BlobWriter, error) |
| 158 |
+} |
|
| 158 | 159 |
|
| 159 |
- // Mount adds a blob to this service from another source repository, |
|
| 160 |
- // identified by a digest. |
|
| 161 |
- Mount(ctx context.Context, sourceRepo string, dgst digest.Digest) (Descriptor, error) |
|
| 160 |
+// BlobCreateOption is a general extensible function argument for blob creation |
|
| 161 |
+// methods. A BlobIngester may choose to honor any or none of the given |
|
| 162 |
+// BlobCreateOptions, which can be specific to the implementation of the |
|
| 163 |
+// BlobIngester receiving them. |
|
| 164 |
+// TODO (brianbland): unify this with ManifestServiceOption in the future |
|
| 165 |
+type BlobCreateOption interface {
|
|
| 166 |
+ Apply(interface{}) error
|
|
| 162 | 167 |
} |
| 163 | 168 |
|
| 164 | 169 |
// BlobWriter provides a handle for inserting data into a blob store. |
| ... | ... |
@@ -10,7 +10,6 @@ import ( |
| 10 | 10 |
"net/http" |
| 11 | 11 |
"net/url" |
| 12 | 12 |
"strconv" |
| 13 |
- "sync" |
|
| 14 | 13 |
"time" |
| 15 | 14 |
|
| 16 | 15 |
"github.com/docker/distribution" |
| ... | ... |
@@ -500,9 +499,6 @@ type blobs struct {
|
| 500 | 500 |
|
| 501 | 501 |
statter distribution.BlobDescriptorService |
| 502 | 502 |
distribution.BlobDeleter |
| 503 |
- |
|
| 504 |
- cacheLock sync.Mutex |
|
| 505 |
- cachedBlobUpload distribution.BlobWriter |
|
| 506 | 503 |
} |
| 507 | 504 |
|
| 508 | 505 |
func sanitizeLocation(location, base string) (string, error) {
|
| ... | ... |
@@ -576,90 +572,95 @@ func (bs *blobs) Put(ctx context.Context, mediaType string, p []byte) (distribut |
| 576 | 576 |
return writer.Commit(ctx, desc) |
| 577 | 577 |
} |
| 578 | 578 |
|
| 579 |
-func (bs *blobs) Create(ctx context.Context) (distribution.BlobWriter, error) {
|
|
| 580 |
- bs.cacheLock.Lock() |
|
| 581 |
- if bs.cachedBlobUpload != nil {
|
|
| 582 |
- upload := bs.cachedBlobUpload |
|
| 583 |
- bs.cachedBlobUpload = nil |
|
| 584 |
- bs.cacheLock.Unlock() |
|
| 585 |
- |
|
| 586 |
- return upload, nil |
|
| 579 |
+// createOptions is a collection of blob creation modifiers relevant to general |
|
| 580 |
+// blob storage intended to be configured by the BlobCreateOption.Apply method. |
|
| 581 |
+type createOptions struct {
|
|
| 582 |
+ Mount struct {
|
|
| 583 |
+ ShouldMount bool |
|
| 584 |
+ From reference.Canonical |
|
| 587 | 585 |
} |
| 588 |
- bs.cacheLock.Unlock() |
|
| 586 |
+} |
|
| 589 | 587 |
|
| 590 |
- u, err := bs.ub.BuildBlobUploadURL(bs.name) |
|
| 591 |
- if err != nil {
|
|
| 592 |
- return nil, err |
|
| 593 |
- } |
|
| 588 |
+type optionFunc func(interface{}) error
|
|
| 594 | 589 |
|
| 595 |
- resp, err := bs.client.Post(u, "", nil) |
|
| 596 |
- if err != nil {
|
|
| 597 |
- return nil, err |
|
| 598 |
- } |
|
| 599 |
- defer resp.Body.Close() |
|
| 590 |
+func (f optionFunc) Apply(v interface{}) error {
|
|
| 591 |
+ return f(v) |
|
| 592 |
+} |
|
| 600 | 593 |
|
| 601 |
- if SuccessStatus(resp.StatusCode) {
|
|
| 602 |
- // TODO(dmcgowan): Check for invalid UUID |
|
| 603 |
- uuid := resp.Header.Get("Docker-Upload-UUID")
|
|
| 604 |
- location, err := sanitizeLocation(resp.Header.Get("Location"), u)
|
|
| 594 |
+// WithMountFrom returns a BlobCreateOption which designates that the blob should be |
|
| 595 |
+// mounted from the given canonical reference. |
|
| 596 |
+func WithMountFrom(ref reference.Canonical) distribution.BlobCreateOption {
|
|
| 597 |
+ return optionFunc(func(v interface{}) error {
|
|
| 598 |
+ opts, ok := v.(*createOptions) |
|
| 599 |
+ if !ok {
|
|
| 600 |
+ return fmt.Errorf("unexpected options type: %T", v)
|
|
| 601 |
+ } |
|
| 602 |
+ |
|
| 603 |
+ opts.Mount.ShouldMount = true |
|
| 604 |
+ opts.Mount.From = ref |
|
| 605 |
+ |
|
| 606 |
+ return nil |
|
| 607 |
+ }) |
|
| 608 |
+} |
|
| 609 |
+ |
|
| 610 |
+func (bs *blobs) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) {
|
|
| 611 |
+ var opts createOptions |
|
| 612 |
+ |
|
| 613 |
+ for _, option := range options {
|
|
| 614 |
+ err := option.Apply(&opts) |
|
| 605 | 615 |
if err != nil {
|
| 606 | 616 |
return nil, err |
| 607 | 617 |
} |
| 608 |
- |
|
| 609 |
- return &httpBlobUpload{
|
|
| 610 |
- statter: bs.statter, |
|
| 611 |
- client: bs.client, |
|
| 612 |
- uuid: uuid, |
|
| 613 |
- startedAt: time.Now(), |
|
| 614 |
- location: location, |
|
| 615 |
- }, nil |
|
| 616 | 618 |
} |
| 617 |
- return nil, HandleErrorResponse(resp) |
|
| 618 |
-} |
|
| 619 | 619 |
|
| 620 |
-func (bs *blobs) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
|
|
| 621 |
- panic("not implemented")
|
|
| 622 |
-} |
|
| 620 |
+ var values []url.Values |
|
| 623 | 621 |
|
| 624 |
-func (bs *blobs) Mount(ctx context.Context, sourceRepo string, dgst digest.Digest) (distribution.Descriptor, error) {
|
|
| 625 |
- u, err := bs.ub.BuildBlobUploadURL(bs.name, url.Values{"from": {sourceRepo}, "mount": {dgst.String()}})
|
|
| 622 |
+ if opts.Mount.ShouldMount {
|
|
| 623 |
+ values = append(values, url.Values{"from": {opts.Mount.From.Name()}, "mount": {opts.Mount.From.Digest().String()}})
|
|
| 624 |
+ } |
|
| 625 |
+ |
|
| 626 |
+ u, err := bs.ub.BuildBlobUploadURL(bs.name, values...) |
|
| 626 | 627 |
if err != nil {
|
| 627 |
- return distribution.Descriptor{}, err
|
|
| 628 |
+ return nil, err |
|
| 628 | 629 |
} |
| 629 | 630 |
|
| 630 | 631 |
resp, err := bs.client.Post(u, "", nil) |
| 631 | 632 |
if err != nil {
|
| 632 |
- return distribution.Descriptor{}, err
|
|
| 633 |
+ return nil, err |
|
| 633 | 634 |
} |
| 634 | 635 |
defer resp.Body.Close() |
| 635 | 636 |
|
| 636 | 637 |
switch resp.StatusCode {
|
| 637 | 638 |
case http.StatusCreated: |
| 638 |
- return bs.Stat(ctx, dgst) |
|
| 639 |
+ desc, err := bs.statter.Stat(ctx, opts.Mount.From.Digest()) |
|
| 640 |
+ if err != nil {
|
|
| 641 |
+ return nil, err |
|
| 642 |
+ } |
|
| 643 |
+ return nil, distribution.ErrBlobMounted{From: opts.Mount.From, Descriptor: desc}
|
|
| 639 | 644 |
case http.StatusAccepted: |
| 640 |
- // Triggered a blob upload (legacy behavior), so cache the creation info |
|
| 645 |
+ // TODO(dmcgowan): Check for invalid UUID |
|
| 641 | 646 |
uuid := resp.Header.Get("Docker-Upload-UUID")
|
| 642 | 647 |
location, err := sanitizeLocation(resp.Header.Get("Location"), u)
|
| 643 | 648 |
if err != nil {
|
| 644 |
- return distribution.Descriptor{}, err
|
|
| 649 |
+ return nil, err |
|
| 645 | 650 |
} |
| 646 | 651 |
|
| 647 |
- bs.cacheLock.Lock() |
|
| 648 |
- bs.cachedBlobUpload = &httpBlobUpload{
|
|
| 652 |
+ return &httpBlobUpload{
|
|
| 649 | 653 |
statter: bs.statter, |
| 650 | 654 |
client: bs.client, |
| 651 | 655 |
uuid: uuid, |
| 652 | 656 |
startedAt: time.Now(), |
| 653 | 657 |
location: location, |
| 654 |
- } |
|
| 655 |
- bs.cacheLock.Unlock() |
|
| 656 |
- |
|
| 657 |
- return distribution.Descriptor{}, HandleErrorResponse(resp)
|
|
| 658 |
+ }, nil |
|
| 658 | 659 |
default: |
| 659 |
- return distribution.Descriptor{}, HandleErrorResponse(resp)
|
|
| 660 |
+ return nil, HandleErrorResponse(resp) |
|
| 660 | 661 |
} |
| 661 | 662 |
} |
| 662 | 663 |
|
| 664 |
+func (bs *blobs) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
|
|
| 665 |
+ panic("not implemented")
|
|
| 666 |
+} |
|
| 667 |
+ |
|
| 663 | 668 |
func (bs *blobs) Delete(ctx context.Context, dgst digest.Digest) error {
|
| 664 | 669 |
return bs.statter.Clear(ctx, dgst) |
| 665 | 670 |
} |