Tracks source repository information for each blob in the blobsum
service, which is then used to attempt to mount blobs from another
repository when pushing instead of having to re-push blobs to the same
registry.
Signed-off-by: Brian Bland <brian.bland@docker.com>
| ... | ... |
@@ -152,7 +152,7 @@ RUN set -x \ |
| 152 | 152 |
# both. This allows integration-cli tests to cover push/pull with both schema1 |
| 153 | 153 |
# and schema2 manifests. |
| 154 | 154 |
ENV REGISTRY_COMMIT_SCHEMA1 ec87e9b6971d831f0eff752ddb54fb64693e51cd |
| 155 |
-ENV REGISTRY_COMMIT a7ae88da459b98b481a245e5b1750134724ac67d |
|
| 155 |
+ENV REGISTRY_COMMIT 93d9070c8bb28414de9ec96fd38c89614acd8435 |
|
| 156 | 156 |
RUN set -x \ |
| 157 | 157 |
&& export GOPATH="$(mktemp -d)" \ |
| 158 | 158 |
&& git clone https://github.com/docker/distribution.git "$GOPATH/src/github.com/docker/distribution" \ |
| ... | ... |
@@ -13,8 +13,14 @@ type BlobSumService struct {
|
| 13 | 13 |
store Store |
| 14 | 14 |
} |
| 15 | 15 |
|
| 16 |
+// BlobSum contains the digest and source repository information for a layer. |
|
| 17 |
+type BlobSum struct {
|
|
| 18 |
+ Digest digest.Digest |
|
| 19 |
+ SourceRepository string |
|
| 20 |
+} |
|
| 21 |
+ |
|
| 16 | 22 |
// maxBlobSums is the number of blobsums to keep per layer DiffID. |
| 17 |
-const maxBlobSums = 5 |
|
| 23 |
+const maxBlobSums = 50 |
|
| 18 | 24 |
|
| 19 | 25 |
// NewBlobSumService creates a new blobsum mapping service. |
| 20 | 26 |
func NewBlobSumService(store Store) *BlobSumService {
|
| ... | ... |
@@ -35,18 +41,18 @@ func (blobserv *BlobSumService) diffIDKey(diffID layer.DiffID) string {
|
| 35 | 35 |
return string(digest.Digest(diffID).Algorithm()) + "/" + digest.Digest(diffID).Hex() |
| 36 | 36 |
} |
| 37 | 37 |
|
| 38 |
-func (blobserv *BlobSumService) blobSumKey(blobsum digest.Digest) string {
|
|
| 39 |
- return string(blobsum.Algorithm()) + "/" + blobsum.Hex() |
|
| 38 |
+func (blobserv *BlobSumService) blobSumKey(blobsum BlobSum) string {
|
|
| 39 |
+ return string(blobsum.Digest.Algorithm()) + "/" + blobsum.Digest.Hex() |
|
| 40 | 40 |
} |
| 41 | 41 |
|
| 42 | 42 |
// GetBlobSums finds the blobsums associated with a layer DiffID. |
| 43 |
-func (blobserv *BlobSumService) GetBlobSums(diffID layer.DiffID) ([]digest.Digest, error) {
|
|
| 43 |
+func (blobserv *BlobSumService) GetBlobSums(diffID layer.DiffID) ([]BlobSum, error) {
|
|
| 44 | 44 |
jsonBytes, err := blobserv.store.Get(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID)) |
| 45 | 45 |
if err != nil {
|
| 46 | 46 |
return nil, err |
| 47 | 47 |
} |
| 48 | 48 |
|
| 49 |
- var blobsums []digest.Digest |
|
| 49 |
+ var blobsums []BlobSum |
|
| 50 | 50 |
if err := json.Unmarshal(jsonBytes, &blobsums); err != nil {
|
| 51 | 51 |
return nil, err |
| 52 | 52 |
} |
| ... | ... |
@@ -55,7 +61,7 @@ func (blobserv *BlobSumService) GetBlobSums(diffID layer.DiffID) ([]digest.Diges |
| 55 | 55 |
} |
| 56 | 56 |
|
| 57 | 57 |
// GetDiffID finds a layer DiffID from a blobsum hash. |
| 58 |
-func (blobserv *BlobSumService) GetDiffID(blobsum digest.Digest) (layer.DiffID, error) {
|
|
| 58 |
+func (blobserv *BlobSumService) GetDiffID(blobsum BlobSum) (layer.DiffID, error) {
|
|
| 59 | 59 |
diffIDBytes, err := blobserv.store.Get(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum)) |
| 60 | 60 |
if err != nil {
|
| 61 | 61 |
return layer.DiffID(""), err
|
| ... | ... |
@@ -66,12 +72,12 @@ func (blobserv *BlobSumService) GetDiffID(blobsum digest.Digest) (layer.DiffID, |
| 66 | 66 |
|
| 67 | 67 |
// Add associates a blobsum with a layer DiffID. If too many blobsums are |
| 68 | 68 |
// present, the oldest one is dropped. |
| 69 |
-func (blobserv *BlobSumService) Add(diffID layer.DiffID, blobsum digest.Digest) error {
|
|
| 69 |
+func (blobserv *BlobSumService) Add(diffID layer.DiffID, blobsum BlobSum) error {
|
|
| 70 | 70 |
oldBlobSums, err := blobserv.GetBlobSums(diffID) |
| 71 | 71 |
if err != nil {
|
| 72 | 72 |
oldBlobSums = nil |
| 73 | 73 |
} |
| 74 |
- newBlobSums := make([]digest.Digest, 0, len(oldBlobSums)+1) |
|
| 74 |
+ newBlobSums := make([]BlobSum, 0, len(oldBlobSums)+1) |
|
| 75 | 75 |
|
| 76 | 76 |
// Copy all other blobsums to new slice |
| 77 | 77 |
for _, oldSum := range oldBlobSums {
|
| ... | ... |
@@ -98,3 +104,34 @@ func (blobserv *BlobSumService) Add(diffID layer.DiffID, blobsum digest.Digest) |
| 98 | 98 |
|
| 99 | 99 |
return blobserv.store.Set(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum), []byte(diffID)) |
| 100 | 100 |
} |
| 101 |
+ |
|
| 102 |
+// Remove unassociates a blobsum from a layer DiffID. |
|
| 103 |
+func (blobserv *BlobSumService) Remove(blobsum BlobSum) error {
|
|
| 104 |
+ diffID, err := blobserv.GetDiffID(blobsum) |
|
| 105 |
+ if err != nil {
|
|
| 106 |
+ return err |
|
| 107 |
+ } |
|
| 108 |
+ oldBlobSums, err := blobserv.GetBlobSums(diffID) |
|
| 109 |
+ if err != nil {
|
|
| 110 |
+ oldBlobSums = nil |
|
| 111 |
+ } |
|
| 112 |
+ newBlobSums := make([]BlobSum, 0, len(oldBlobSums)) |
|
| 113 |
+ |
|
| 114 |
+ // Copy all other blobsums to new slice |
|
| 115 |
+ for _, oldSum := range oldBlobSums {
|
|
| 116 |
+ if oldSum != blobsum {
|
|
| 117 |
+ newBlobSums = append(newBlobSums, oldSum) |
|
| 118 |
+ } |
|
| 119 |
+ } |
|
| 120 |
+ |
|
| 121 |
+ if len(newBlobSums) == 0 {
|
|
| 122 |
+ return blobserv.store.Delete(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID)) |
|
| 123 |
+ } |
|
| 124 |
+ |
|
| 125 |
+ jsonBytes, err := json.Marshal(newBlobSums) |
|
| 126 |
+ if err != nil {
|
|
| 127 |
+ return err |
|
| 128 |
+ } |
|
| 129 |
+ |
|
| 130 |
+ return blobserv.store.Set(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID), jsonBytes) |
|
| 131 |
+} |
| ... | ... |
@@ -1,7 +1,9 @@ |
| 1 | 1 |
package metadata |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
+ "encoding/hex" |
|
| 4 | 5 |
"io/ioutil" |
| 6 |
+ "math/rand" |
|
| 5 | 7 |
"os" |
| 6 | 8 |
"reflect" |
| 7 | 9 |
"testing" |
| ... | ... |
@@ -23,33 +25,32 @@ func TestBlobSumService(t *testing.T) {
|
| 23 | 23 |
} |
| 24 | 24 |
blobSumService := NewBlobSumService(metadataStore) |
| 25 | 25 |
|
| 26 |
+ tooManyBlobSums := make([]BlobSum, 100) |
|
| 27 |
+ for i := range tooManyBlobSums {
|
|
| 28 |
+ randDigest := randomDigest() |
|
| 29 |
+ tooManyBlobSums[i] = BlobSum{Digest: randDigest}
|
|
| 30 |
+ } |
|
| 31 |
+ |
|
| 26 | 32 |
testVectors := []struct {
|
| 27 | 33 |
diffID layer.DiffID |
| 28 |
- blobsums []digest.Digest |
|
| 34 |
+ blobsums []BlobSum |
|
| 29 | 35 |
}{
|
| 30 | 36 |
{
|
| 31 | 37 |
diffID: layer.DiffID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
|
| 32 |
- blobsums: []digest.Digest{
|
|
| 33 |
- digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"),
|
|
| 38 |
+ blobsums: []BlobSum{
|
|
| 39 |
+ {Digest: digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937")},
|
|
| 34 | 40 |
}, |
| 35 | 41 |
}, |
| 36 | 42 |
{
|
| 37 | 43 |
diffID: layer.DiffID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
|
| 38 |
- blobsums: []digest.Digest{
|
|
| 39 |
- digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"),
|
|
| 40 |
- digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e"),
|
|
| 44 |
+ blobsums: []BlobSum{
|
|
| 45 |
+ {Digest: digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937")},
|
|
| 46 |
+ {Digest: digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e")},
|
|
| 41 | 47 |
}, |
| 42 | 48 |
}, |
| 43 | 49 |
{
|
| 44 |
- diffID: layer.DiffID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
|
|
| 45 |
- blobsums: []digest.Digest{
|
|
| 46 |
- digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"),
|
|
| 47 |
- digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e"),
|
|
| 48 |
- digest.Digest("sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf"),
|
|
| 49 |
- digest.Digest("sha256:8902a7ca89aabbb868835260912159026637634090dd8899eee969523252236e"),
|
|
| 50 |
- digest.Digest("sha256:c84364306344ccc48532c52ff5209236273525231dddaaab53262322352883aa"),
|
|
| 51 |
- digest.Digest("sha256:aa7583bbc87532a8352bbb72520a821b3623523523a8352523a52352aaa888fe"),
|
|
| 52 |
- }, |
|
| 50 |
+ diffID: layer.DiffID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
|
|
| 51 |
+ blobsums: tooManyBlobSums, |
|
| 53 | 52 |
}, |
| 54 | 53 |
} |
| 55 | 54 |
|
| ... | ... |
@@ -70,8 +71,8 @@ func TestBlobSumService(t *testing.T) {
|
| 70 | 70 |
t.Fatalf("error calling Get: %v", err)
|
| 71 | 71 |
} |
| 72 | 72 |
expectedBlobsums := len(vec.blobsums) |
| 73 |
- if expectedBlobsums > 5 {
|
|
| 74 |
- expectedBlobsums = 5 |
|
| 73 |
+ if expectedBlobsums > 50 {
|
|
| 74 |
+ expectedBlobsums = 50 |
|
| 75 | 75 |
} |
| 76 | 76 |
if !reflect.DeepEqual(blobsums, vec.blobsums[len(vec.blobsums)-expectedBlobsums:len(vec.blobsums)]) {
|
| 77 | 77 |
t.Fatal("Get returned incorrect layer ID")
|
| ... | ... |
@@ -85,7 +86,7 @@ func TestBlobSumService(t *testing.T) {
|
| 85 | 85 |
} |
| 86 | 86 |
|
| 87 | 87 |
// Test GetDiffID on a nonexistent entry |
| 88 |
- _, err = blobSumService.GetDiffID(digest.Digest("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
|
|
| 88 |
+ _, err = blobSumService.GetDiffID(BlobSum{Digest: digest.Digest("sha256:82379823067823853223359023576437723560923756b03560378f4497753917")})
|
|
| 89 | 89 |
if err == nil {
|
| 90 | 90 |
t.Fatal("expected error looking up nonexistent entry")
|
| 91 | 91 |
} |
| ... | ... |
@@ -103,3 +104,12 @@ func TestBlobSumService(t *testing.T) {
|
| 103 | 103 |
t.Fatal("GetDiffID returned incorrect diffID")
|
| 104 | 104 |
} |
| 105 | 105 |
} |
| 106 |
+ |
|
| 107 |
+func randomDigest() digest.Digest {
|
|
| 108 |
+ b := [32]byte{}
|
|
| 109 |
+ for i := 0; i < len(b); i++ {
|
|
| 110 |
+ b[i] = byte(rand.Intn(256)) |
|
| 111 |
+ } |
|
| 112 |
+ d := hex.EncodeToString(b[:]) |
|
| 113 |
+ return digest.Digest("sha256:" + d)
|
|
| 114 |
+} |
| ... | ... |
@@ -15,6 +15,8 @@ type Store interface {
|
| 15 | 15 |
Get(namespace string, key string) ([]byte, error) |
| 16 | 16 |
// Set writes data indexed by namespace and key. |
| 17 | 17 |
Set(namespace, key string, value []byte) error |
| 18 |
+ // Delete removes data indexed by namespace and key. |
|
| 19 |
+ Delete(namespace, key string) error |
|
| 18 | 20 |
} |
| 19 | 21 |
|
| 20 | 22 |
// FSMetadataStore uses the filesystem to associate metadata with layer and |
| ... | ... |
@@ -63,3 +65,13 @@ func (store *FSMetadataStore) Set(namespace, key string, value []byte) error {
|
| 63 | 63 |
} |
| 64 | 64 |
return os.Rename(tempFilePath, path) |
| 65 | 65 |
} |
| 66 |
+ |
|
| 67 |
+// Delete removes data indexed by namespace and key. The data file named after |
|
| 68 |
+// the key, stored in the namespace's directory is deleted. |
|
| 69 |
+func (store *FSMetadataStore) Delete(namespace, key string) error {
|
|
| 70 |
+ store.Lock() |
|
| 71 |
+ defer store.Unlock() |
|
| 72 |
+ |
|
| 73 |
+ path := store.path(namespace, key) |
|
| 74 |
+ return os.Remove(path) |
|
| 75 |
+} |
| ... | ... |
@@ -111,6 +111,7 @@ func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named) (e |
| 111 | 111 |
|
| 112 | 112 |
type v2LayerDescriptor struct {
|
| 113 | 113 |
digest digest.Digest |
| 114 |
+ repoInfo *registry.RepositoryInfo |
|
| 114 | 115 |
repo distribution.Repository |
| 115 | 116 |
blobSumService *metadata.BlobSumService |
| 116 | 117 |
} |
| ... | ... |
@@ -124,7 +125,7 @@ func (ld *v2LayerDescriptor) ID() string {
|
| 124 | 124 |
} |
| 125 | 125 |
|
| 126 | 126 |
func (ld *v2LayerDescriptor) DiffID() (layer.DiffID, error) {
|
| 127 |
- return ld.blobSumService.GetDiffID(ld.digest) |
|
| 127 |
+ return ld.blobSumService.GetDiffID(metadata.BlobSum{Digest: ld.digest, SourceRepository: ld.repoInfo.FullName()})
|
|
| 128 | 128 |
} |
| 129 | 129 |
|
| 130 | 130 |
func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) {
|
| ... | ... |
@@ -196,7 +197,7 @@ func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progre |
| 196 | 196 |
|
| 197 | 197 |
func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) {
|
| 198 | 198 |
// Cache mapping from this layer's DiffID to the blobsum |
| 199 |
- ld.blobSumService.Add(diffID, ld.digest) |
|
| 199 |
+ ld.blobSumService.Add(diffID, metadata.BlobSum{Digest: ld.digest, SourceRepository: ld.repoInfo.FullName()})
|
|
| 200 | 200 |
} |
| 201 | 201 |
|
| 202 | 202 |
func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdated bool, err error) {
|
| ... | ... |
@@ -334,6 +335,7 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverif |
| 334 | 334 |
|
| 335 | 335 |
layerDescriptor := &v2LayerDescriptor{
|
| 336 | 336 |
digest: blobSum, |
| 337 |
+ repoInfo: p.repoInfo, |
|
| 337 | 338 |
repo: p.repo, |
| 338 | 339 |
blobSumService: p.blobSumService, |
| 339 | 340 |
} |
| ... | ... |
@@ -400,6 +402,7 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s |
| 400 | 400 |
layerDescriptor := &v2LayerDescriptor{
|
| 401 | 401 |
digest: d.Digest, |
| 402 | 402 |
repo: p.repo, |
| 403 |
+ repoInfo: p.repoInfo, |
|
| 403 | 404 |
blobSumService: p.blobSumService, |
| 404 | 405 |
} |
| 405 | 406 |
|
| ... | ... |
@@ -131,6 +131,7 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, ima |
| 131 | 131 |
|
| 132 | 132 |
descriptorTemplate := v2PushDescriptor{
|
| 133 | 133 |
blobSumService: p.blobSumService, |
| 134 |
+ repoInfo: p.repoInfo, |
|
| 134 | 135 |
repo: p.repo, |
| 135 | 136 |
pushState: &p.pushState, |
| 136 | 137 |
} |
| ... | ... |
@@ -211,6 +212,7 @@ func manifestFromBuilder(ctx context.Context, builder distribution.ManifestBuild |
| 211 | 211 |
type v2PushDescriptor struct {
|
| 212 | 212 |
layer layer.Layer |
| 213 | 213 |
blobSumService *metadata.BlobSumService |
| 214 |
+ repoInfo reference.Named |
|
| 214 | 215 |
repo distribution.Repository |
| 215 | 216 |
pushState *pushState |
| 216 | 217 |
} |
| ... | ... |
@@ -243,7 +245,7 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. |
| 243 | 243 |
// Do we have any blobsums associated with this layer's DiffID? |
| 244 | 244 |
possibleBlobsums, err := pd.blobSumService.GetBlobSums(diffID) |
| 245 | 245 |
if err == nil {
|
| 246 |
- descriptor, exists, err := blobSumAlreadyExists(ctx, possibleBlobsums, pd.repo, pd.pushState) |
|
| 246 |
+ descriptor, exists, err := blobSumAlreadyExists(ctx, possibleBlobsums, pd.repoInfo, pd.repo, pd.pushState) |
|
| 247 | 247 |
if err != nil {
|
| 248 | 248 |
progress.Update(progressOutput, pd.ID(), "Image push failed") |
| 249 | 249 |
return retryOnError(err) |
| ... | ... |
@@ -263,6 +265,37 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. |
| 263 | 263 |
// then push the blob. |
| 264 | 264 |
bs := pd.repo.Blobs(ctx) |
| 265 | 265 |
|
| 266 |
+ // Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload |
|
| 267 |
+ for _, blobsum := range possibleBlobsums {
|
|
| 268 |
+ sourceRepo, err := reference.ParseNamed(blobsum.SourceRepository) |
|
| 269 |
+ if err != nil {
|
|
| 270 |
+ continue |
|
| 271 |
+ } |
|
| 272 |
+ if pd.repoInfo.Hostname() == sourceRepo.Hostname() {
|
|
| 273 |
+ logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, blobsum.Digest, sourceRepo.FullName())
|
|
| 274 |
+ |
|
| 275 |
+ desc, err := bs.Mount(ctx, sourceRepo.RemoteName(), blobsum.Digest) |
|
| 276 |
+ if err == nil {
|
|
| 277 |
+ progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", sourceRepo.RemoteName()) |
|
| 278 |
+ |
|
| 279 |
+ pd.pushState.Lock() |
|
| 280 |
+ pd.pushState.confirmedV2 = true |
|
| 281 |
+ pd.pushState.remoteLayers[diffID] = desc |
|
| 282 |
+ pd.pushState.Unlock() |
|
| 283 |
+ |
|
| 284 |
+ // Cache mapping from this layer's DiffID to the blobsum |
|
| 285 |
+ if err := pd.blobSumService.Add(diffID, metadata.BlobSum{Digest: blobsum.Digest, SourceRepository: pd.repoInfo.FullName()}); err != nil {
|
|
| 286 |
+ return xfer.DoNotRetry{Err: err}
|
|
| 287 |
+ } |
|
| 288 |
+ |
|
| 289 |
+ return nil |
|
| 290 |
+ } |
|
| 291 |
+ // Unable to mount layer from this repository, so this source mapping is no longer valid |
|
| 292 |
+ logrus.Debugf("unassociating layer %s (%s) with %s", diffID, blobsum.Digest, sourceRepo.FullName())
|
|
| 293 |
+ pd.blobSumService.Remove(blobsum) |
|
| 294 |
+ } |
|
| 295 |
+ } |
|
| 296 |
+ |
|
| 266 | 297 |
// Send the layer |
| 267 | 298 |
layerUpload, err := bs.Create(ctx) |
| 268 | 299 |
if err != nil {
|
| ... | ... |
@@ -300,7 +333,7 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. |
| 300 | 300 |
progress.Update(progressOutput, pd.ID(), "Pushed") |
| 301 | 301 |
|
| 302 | 302 |
// Cache mapping from this layer's DiffID to the blobsum |
| 303 |
- if err := pd.blobSumService.Add(diffID, pushDigest); err != nil {
|
|
| 303 |
+ if err := pd.blobSumService.Add(diffID, metadata.BlobSum{Digest: pushDigest, SourceRepository: pd.repoInfo.FullName()}); err != nil {
|
|
| 304 | 304 |
return xfer.DoNotRetry{Err: err}
|
| 305 | 305 |
} |
| 306 | 306 |
|
| ... | ... |
@@ -332,9 +365,13 @@ func (pd *v2PushDescriptor) Descriptor() distribution.Descriptor {
|
| 332 | 332 |
// blobSumAlreadyExists checks if the registry already know about any of the |
| 333 | 333 |
// blobsums passed in the "blobsums" slice. If it finds one that the registry |
| 334 | 334 |
// knows about, it returns the known digest and "true". |
| 335 |
-func blobSumAlreadyExists(ctx context.Context, blobsums []digest.Digest, repo distribution.Repository, pushState *pushState) (distribution.Descriptor, bool, error) {
|
|
| 336 |
- for _, dgst := range blobsums {
|
|
| 337 |
- descriptor, err := repo.Blobs(ctx).Stat(ctx, dgst) |
|
| 335 |
+func blobSumAlreadyExists(ctx context.Context, blobsums []metadata.BlobSum, repoInfo reference.Named, repo distribution.Repository, pushState *pushState) (distribution.Descriptor, bool, error) {
|
|
| 336 |
+ for _, blobSum := range blobsums {
|
|
| 337 |
+ // Only check blobsums that are known to this repository or have an unknown source |
|
| 338 |
+ if blobSum.SourceRepository != "" && blobSum.SourceRepository != repoInfo.FullName() {
|
|
| 339 |
+ continue |
|
| 340 |
+ } |
|
| 341 |
+ descriptor, err := repo.Blobs(ctx).Stat(ctx, blobSum.Digest) |
|
| 338 | 342 |
switch err {
|
| 339 | 343 |
case nil: |
| 340 | 344 |
descriptor.MediaType = schema2.MediaTypeLayer |
| ... | ... |
@@ -44,7 +44,7 @@ clone git github.com/boltdb/bolt v1.1.0 |
| 44 | 44 |
clone git github.com/miekg/dns d27455715200c7d3e321a1e5cadb27c9ee0b0f02 |
| 45 | 45 |
|
| 46 | 46 |
# get graph and distribution packages |
| 47 |
-clone git github.com/docker/distribution a7ae88da459b98b481a245e5b1750134724ac67d |
|
| 47 |
+clone git github.com/docker/distribution 93d9070c8bb28414de9ec96fd38c89614acd8435 |
|
| 48 | 48 |
clone git github.com/vbatts/tar-split v0.9.11 |
| 49 | 49 |
|
| 50 | 50 |
# get desired notary commit, might also need to be updated in Dockerfile |
| ... | ... |
@@ -147,6 +147,54 @@ func (s *DockerSchema1RegistrySuite) TestPushEmptyLayer(c *check.C) {
|
| 147 | 147 |
testPushEmptyLayer(c) |
| 148 | 148 |
} |
| 149 | 149 |
|
| 150 |
+func (s *DockerRegistrySuite) TestCrossRepositoryLayerPush(c *check.C) {
|
|
| 151 |
+ sourceRepoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)
|
|
| 152 |
+ // tag the image to upload it to the private registry |
|
| 153 |
+ dockerCmd(c, "tag", "busybox", sourceRepoName) |
|
| 154 |
+ // push the image to the registry |
|
| 155 |
+ out1, _, err := dockerCmdWithError("push", sourceRepoName)
|
|
| 156 |
+ c.Assert(err, check.IsNil, check.Commentf("pushing the image to the private registry has failed: %s", out1))
|
|
| 157 |
+ // ensure that none of the layers were mounted from another repository during push |
|
| 158 |
+ c.Assert(strings.Contains(out1, "Mounted from"), check.Equals, false) |
|
| 159 |
+ |
|
| 160 |
+ destRepoName := fmt.Sprintf("%v/dockercli/crossrepopush", privateRegistryURL)
|
|
| 161 |
+ // retag the image to upload the same layers to another repo in the same registry |
|
| 162 |
+ dockerCmd(c, "tag", "busybox", destRepoName) |
|
| 163 |
+ // push the image to the registry |
|
| 164 |
+ out2, _, err := dockerCmdWithError("push", destRepoName)
|
|
| 165 |
+ c.Assert(err, check.IsNil, check.Commentf("pushing the image to the private registry has failed: %s", out2))
|
|
| 166 |
+ // ensure that layers were mounted from the first repo during push |
|
| 167 |
+ c.Assert(strings.Contains(out2, "Mounted from dockercli/busybox"), check.Equals, true) |
|
| 168 |
+ |
|
| 169 |
+ // ensure that we can pull the cross-repo-pushed repository |
|
| 170 |
+ dockerCmd(c, "rmi", destRepoName) |
|
| 171 |
+ dockerCmd(c, "pull", destRepoName) |
|
| 172 |
+} |
|
| 173 |
+ |
|
| 174 |
+func (s *DockerSchema1RegistrySuite) TestCrossRepositoryLayerPushNotSupported(c *check.C) {
|
|
| 175 |
+ sourceRepoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)
|
|
| 176 |
+ // tag the image to upload it to the private registry |
|
| 177 |
+ dockerCmd(c, "tag", "busybox", sourceRepoName) |
|
| 178 |
+ // push the image to the registry |
|
| 179 |
+ out1, _, err := dockerCmdWithError("push", sourceRepoName)
|
|
| 180 |
+ c.Assert(err, check.IsNil, check.Commentf("pushing the image to the private registry has failed: %s", out1))
|
|
| 181 |
+ // ensure that none of the layers were mounted from another repository during push |
|
| 182 |
+ c.Assert(strings.Contains(out1, "Mounted from"), check.Equals, false) |
|
| 183 |
+ |
|
| 184 |
+ destRepoName := fmt.Sprintf("%v/dockercli/crossrepopush", privateRegistryURL)
|
|
| 185 |
+ // retag the image to upload the same layers to another repo in the same registry |
|
| 186 |
+ dockerCmd(c, "tag", "busybox", destRepoName) |
|
| 187 |
+ // push the image to the registry |
|
| 188 |
+ out2, _, err := dockerCmdWithError("push", destRepoName)
|
|
| 189 |
+ c.Assert(err, check.IsNil, check.Commentf("pushing the image to the private registry has failed: %s", out2))
|
|
| 190 |
+ // schema1 registry should not support cross-repo layer mounts, so ensure that this does not happen |
|
| 191 |
+ c.Assert(strings.Contains(out2, "Mounted from dockercli/busybox"), check.Equals, false) |
|
| 192 |
+ |
|
| 193 |
+ // ensure that we can pull the second pushed repository |
|
| 194 |
+ dockerCmd(c, "rmi", destRepoName) |
|
| 195 |
+ dockerCmd(c, "pull", destRepoName) |
|
| 196 |
+} |
|
| 197 |
+ |
|
| 150 | 198 |
func (s *DockerTrustSuite) TestTrustedPush(c *check.C) {
|
| 151 | 199 |
repoName := fmt.Sprintf("%v/dockercli/trusted:latest", privateRegistryURL)
|
| 152 | 200 |
// tag the image and upload it to the private registry |
| ... | ... |
@@ -477,7 +477,7 @@ func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metad |
| 477 | 477 |
dgst, err := digest.ParseDigest(string(checksum)) |
| 478 | 478 |
if err == nil {
|
| 479 | 479 |
blobSumService := metadata.NewBlobSumService(ms) |
| 480 |
- blobSumService.Add(layer.DiffID(), dgst) |
|
| 480 |
+ blobSumService.Add(layer.DiffID(), metadata.BlobSum{Digest: dgst})
|
|
| 481 | 481 |
} |
| 482 | 482 |
} |
| 483 | 483 |
_, err = ls.Release(layer) |
| ... | ... |
@@ -216,9 +216,9 @@ func TestMigrateImages(t *testing.T) {
|
| 216 | 216 |
t.Fatal(err) |
| 217 | 217 |
} |
| 218 | 218 |
|
| 219 |
- expectedBlobsums := []digest.Digest{
|
|
| 220 |
- "sha256:55dc925c23d1ed82551fd018c27ac3ee731377b6bad3963a2a4e76e753d70e57", |
|
| 221 |
- "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4", |
|
| 219 |
+ expectedBlobsums := []metadata.BlobSum{
|
|
| 220 |
+ {Digest: digest.Digest("sha256:55dc925c23d1ed82551fd018c27ac3ee731377b6bad3963a2a4e76e753d70e57")},
|
|
| 221 |
+ {Digest: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
|
|
| 222 | 222 |
} |
| 223 | 223 |
|
| 224 | 224 |
if !reflect.DeepEqual(expectedBlobsums, blobsums) {
|
| ... | ... |
@@ -155,6 +155,10 @@ type BlobIngester interface {
|
| 155 | 155 |
|
| 156 | 156 |
// Resume attempts to resume a write to a blob, identified by an id. |
| 157 | 157 |
Resume(ctx context.Context, id string) (BlobWriter, error) |
| 158 |
+ |
|
| 159 |
+ // Mount adds a blob to this service from another source repository, |
|
| 160 |
+ // identified by a digest. |
|
| 161 |
+ Mount(ctx context.Context, sourceRepo string, dgst digest.Digest) (Descriptor, error) |
|
| 158 | 162 |
} |
| 159 | 163 |
|
| 160 | 164 |
// BlobWriter provides a handle for inserting data into a blob store. |
| ... | ... |
@@ -1041,6 +1041,70 @@ var routeDescriptors = []RouteDescriptor{
|
| 1041 | 1041 |
deniedResponseDescriptor, |
| 1042 | 1042 |
}, |
| 1043 | 1043 |
}, |
| 1044 |
+ {
|
|
| 1045 |
+ Name: "Mount Blob", |
|
| 1046 |
+ Description: "Mount a blob identified by the `mount` parameter from another repository.", |
|
| 1047 |
+ Headers: []ParameterDescriptor{
|
|
| 1048 |
+ hostHeader, |
|
| 1049 |
+ authHeader, |
|
| 1050 |
+ contentLengthZeroHeader, |
|
| 1051 |
+ }, |
|
| 1052 |
+ PathParameters: []ParameterDescriptor{
|
|
| 1053 |
+ nameParameterDescriptor, |
|
| 1054 |
+ }, |
|
| 1055 |
+ QueryParameters: []ParameterDescriptor{
|
|
| 1056 |
+ {
|
|
| 1057 |
+ Name: "mount", |
|
| 1058 |
+ Type: "query", |
|
| 1059 |
+ Format: "<digest>", |
|
| 1060 |
+ Regexp: digest.DigestRegexp, |
|
| 1061 |
+ Description: `Digest of blob to mount from the source repository.`, |
|
| 1062 |
+ }, |
|
| 1063 |
+ {
|
|
| 1064 |
+ Name: "from", |
|
| 1065 |
+ Type: "query", |
|
| 1066 |
+ Format: "<repository name>", |
|
| 1067 |
+ Regexp: reference.NameRegexp, |
|
| 1068 |
+ Description: `Name of the source repository.`, |
|
| 1069 |
+ }, |
|
| 1070 |
+ }, |
|
| 1071 |
+ Successes: []ResponseDescriptor{
|
|
| 1072 |
+ {
|
|
| 1073 |
+ Description: "The blob has been mounted in the repository and is available at the provided location.", |
|
| 1074 |
+ StatusCode: http.StatusCreated, |
|
| 1075 |
+ Headers: []ParameterDescriptor{
|
|
| 1076 |
+ {
|
|
| 1077 |
+ Name: "Location", |
|
| 1078 |
+ Type: "url", |
|
| 1079 |
+ Format: "<blob location>", |
|
| 1080 |
+ }, |
|
| 1081 |
+ contentLengthZeroHeader, |
|
| 1082 |
+ dockerUploadUUIDHeader, |
|
| 1083 |
+ }, |
|
| 1084 |
+ }, |
|
| 1085 |
+ }, |
|
| 1086 |
+ Failures: []ResponseDescriptor{
|
|
| 1087 |
+ {
|
|
| 1088 |
+ Name: "Invalid Name or Digest", |
|
| 1089 |
+ StatusCode: http.StatusBadRequest, |
|
| 1090 |
+ ErrorCodes: []errcode.ErrorCode{
|
|
| 1091 |
+ ErrorCodeDigestInvalid, |
|
| 1092 |
+ ErrorCodeNameInvalid, |
|
| 1093 |
+ }, |
|
| 1094 |
+ }, |
|
| 1095 |
+ {
|
|
| 1096 |
+ Name: "Not allowed", |
|
| 1097 |
+ Description: "Blob mount is not allowed because the registry is configured as a pull-through cache or for some other reason", |
|
| 1098 |
+ StatusCode: http.StatusMethodNotAllowed, |
|
| 1099 |
+ ErrorCodes: []errcode.ErrorCode{
|
|
| 1100 |
+ errcode.ErrorCodeUnsupported, |
|
| 1101 |
+ }, |
|
| 1102 |
+ }, |
|
| 1103 |
+ unauthorizedResponseDescriptor, |
|
| 1104 |
+ repositoryNotFoundResponseDescriptor, |
|
| 1105 |
+ deniedResponseDescriptor, |
|
| 1106 |
+ }, |
|
| 1107 |
+ }, |
|
| 1044 | 1108 |
}, |
| 1045 | 1109 |
}, |
| 1046 | 1110 |
}, |
| ... | ... |
@@ -108,6 +108,8 @@ type tokenHandler struct {
|
| 108 | 108 |
tokenLock sync.Mutex |
| 109 | 109 |
tokenCache string |
| 110 | 110 |
tokenExpiration time.Time |
| 111 |
+ |
|
| 112 |
+ additionalScopes map[string]struct{}
|
|
| 111 | 113 |
} |
| 112 | 114 |
|
| 113 | 115 |
// tokenScope represents the scope at which a token will be requested. |
| ... | ... |
@@ -145,6 +147,7 @@ func newTokenHandler(transport http.RoundTripper, creds CredentialStore, c clock |
| 145 | 145 |
Scope: scope, |
| 146 | 146 |
Actions: actions, |
| 147 | 147 |
}, |
| 148 |
+ additionalScopes: map[string]struct{}{},
|
|
| 148 | 149 |
} |
| 149 | 150 |
} |
| 150 | 151 |
|
| ... | ... |
@@ -160,7 +163,15 @@ func (th *tokenHandler) Scheme() string {
|
| 160 | 160 |
} |
| 161 | 161 |
|
| 162 | 162 |
func (th *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
|
| 163 |
- if err := th.refreshToken(params); err != nil {
|
|
| 163 |
+ var additionalScopes []string |
|
| 164 |
+ if fromParam := req.URL.Query().Get("from"); fromParam != "" {
|
|
| 165 |
+ additionalScopes = append(additionalScopes, tokenScope{
|
|
| 166 |
+ Resource: "repository", |
|
| 167 |
+ Scope: fromParam, |
|
| 168 |
+ Actions: []string{"pull"},
|
|
| 169 |
+ }.String()) |
|
| 170 |
+ } |
|
| 171 |
+ if err := th.refreshToken(params, additionalScopes...); err != nil {
|
|
| 164 | 172 |
return err |
| 165 | 173 |
} |
| 166 | 174 |
|
| ... | ... |
@@ -169,11 +180,18 @@ func (th *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]st |
| 169 | 169 |
return nil |
| 170 | 170 |
} |
| 171 | 171 |
|
| 172 |
-func (th *tokenHandler) refreshToken(params map[string]string) error {
|
|
| 172 |
+func (th *tokenHandler) refreshToken(params map[string]string, additionalScopes ...string) error {
|
|
| 173 | 173 |
th.tokenLock.Lock() |
| 174 | 174 |
defer th.tokenLock.Unlock() |
| 175 |
+ var addedScopes bool |
|
| 176 |
+ for _, scope := range additionalScopes {
|
|
| 177 |
+ if _, ok := th.additionalScopes[scope]; !ok {
|
|
| 178 |
+ th.additionalScopes[scope] = struct{}{}
|
|
| 179 |
+ addedScopes = true |
|
| 180 |
+ } |
|
| 181 |
+ } |
|
| 175 | 182 |
now := th.clock.Now() |
| 176 |
- if now.After(th.tokenExpiration) {
|
|
| 183 |
+ if now.After(th.tokenExpiration) || addedScopes {
|
|
| 177 | 184 |
tr, err := th.fetchToken(params) |
| 178 | 185 |
if err != nil {
|
| 179 | 186 |
return err |
| ... | ... |
@@ -223,6 +241,10 @@ func (th *tokenHandler) fetchToken(params map[string]string) (token *tokenRespon |
| 223 | 223 |
reqParams.Add("scope", scopeField)
|
| 224 | 224 |
} |
| 225 | 225 |
|
| 226 |
+ for scope := range th.additionalScopes {
|
|
| 227 |
+ reqParams.Add("scope", scope)
|
|
| 228 |
+ } |
|
| 229 |
+ |
|
| 226 | 230 |
if th.creds != nil {
|
| 227 | 231 |
username, password := th.creds.Basic(realmURL) |
| 228 | 232 |
if username != "" && password != "" {
|
| ... | ... |
@@ -10,6 +10,7 @@ import ( |
| 10 | 10 |
"net/http" |
| 11 | 11 |
"net/url" |
| 12 | 12 |
"strconv" |
| 13 |
+ "sync" |
|
| 13 | 14 |
"time" |
| 14 | 15 |
|
| 15 | 16 |
"github.com/docker/distribution" |
| ... | ... |
@@ -499,6 +500,9 @@ type blobs struct {
|
| 499 | 499 |
|
| 500 | 500 |
statter distribution.BlobDescriptorService |
| 501 | 501 |
distribution.BlobDeleter |
| 502 |
+ |
|
| 503 |
+ cacheLock sync.Mutex |
|
| 504 |
+ cachedBlobUpload distribution.BlobWriter |
|
| 502 | 505 |
} |
| 503 | 506 |
|
| 504 | 507 |
func sanitizeLocation(location, base string) (string, error) {
|
| ... | ... |
@@ -573,7 +577,20 @@ func (bs *blobs) Put(ctx context.Context, mediaType string, p []byte) (distribut |
| 573 | 573 |
} |
| 574 | 574 |
|
| 575 | 575 |
func (bs *blobs) Create(ctx context.Context) (distribution.BlobWriter, error) {
|
| 576 |
+ bs.cacheLock.Lock() |
|
| 577 |
+ if bs.cachedBlobUpload != nil {
|
|
| 578 |
+ upload := bs.cachedBlobUpload |
|
| 579 |
+ bs.cachedBlobUpload = nil |
|
| 580 |
+ bs.cacheLock.Unlock() |
|
| 581 |
+ |
|
| 582 |
+ return upload, nil |
|
| 583 |
+ } |
|
| 584 |
+ bs.cacheLock.Unlock() |
|
| 585 |
+ |
|
| 576 | 586 |
u, err := bs.ub.BuildBlobUploadURL(bs.name) |
| 587 |
+ if err != nil {
|
|
| 588 |
+ return nil, err |
|
| 589 |
+ } |
|
| 577 | 590 |
|
| 578 | 591 |
resp, err := bs.client.Post(u, "", nil) |
| 579 | 592 |
if err != nil {
|
| ... | ... |
@@ -604,6 +621,45 @@ func (bs *blobs) Resume(ctx context.Context, id string) (distribution.BlobWriter |
| 604 | 604 |
panic("not implemented")
|
| 605 | 605 |
} |
| 606 | 606 |
|
| 607 |
+func (bs *blobs) Mount(ctx context.Context, sourceRepo string, dgst digest.Digest) (distribution.Descriptor, error) {
|
|
| 608 |
+ u, err := bs.ub.BuildBlobUploadURL(bs.name, url.Values{"from": {sourceRepo}, "mount": {dgst.String()}})
|
|
| 609 |
+ if err != nil {
|
|
| 610 |
+ return distribution.Descriptor{}, err
|
|
| 611 |
+ } |
|
| 612 |
+ |
|
| 613 |
+ resp, err := bs.client.Post(u, "", nil) |
|
| 614 |
+ if err != nil {
|
|
| 615 |
+ return distribution.Descriptor{}, err
|
|
| 616 |
+ } |
|
| 617 |
+ defer resp.Body.Close() |
|
| 618 |
+ |
|
| 619 |
+ switch resp.StatusCode {
|
|
| 620 |
+ case http.StatusCreated: |
|
| 621 |
+ return bs.Stat(ctx, dgst) |
|
| 622 |
+ case http.StatusAccepted: |
|
| 623 |
+ // Triggered a blob upload (legacy behavior), so cache the creation info |
|
| 624 |
+ uuid := resp.Header.Get("Docker-Upload-UUID")
|
|
| 625 |
+ location, err := sanitizeLocation(resp.Header.Get("Location"), u)
|
|
| 626 |
+ if err != nil {
|
|
| 627 |
+ return distribution.Descriptor{}, err
|
|
| 628 |
+ } |
|
| 629 |
+ |
|
| 630 |
+ bs.cacheLock.Lock() |
|
| 631 |
+ bs.cachedBlobUpload = &httpBlobUpload{
|
|
| 632 |
+ statter: bs.statter, |
|
| 633 |
+ client: bs.client, |
|
| 634 |
+ uuid: uuid, |
|
| 635 |
+ startedAt: time.Now(), |
|
| 636 |
+ location: location, |
|
| 637 |
+ } |
|
| 638 |
+ bs.cacheLock.Unlock() |
|
| 639 |
+ |
|
| 640 |
+ return distribution.Descriptor{}, HandleErrorResponse(resp)
|
|
| 641 |
+ default: |
|
| 642 |
+ return distribution.Descriptor{}, HandleErrorResponse(resp)
|
|
| 643 |
+ } |
|
| 644 |
+} |
|
| 645 |
+ |
|
| 607 | 646 |
func (bs *blobs) Delete(ctx context.Context, dgst digest.Digest) error {
|
| 608 | 647 |
return bs.statter.Clear(ctx, dgst) |
| 609 | 648 |
} |