Browse code

api: fix "GET /distribution" endpoint ignoring mirrors

If the daemon is configured to use a mirror for the default (Docker Hub)
registry, the endpoint did not fall back to querying the upstream if the mirror
did not contain the given reference.

If the daemon is configured to use a mirror for the default (Docker Hub)
registry, did not fall back to querying the upstream if the mirror did not
contain the given reference.

For pull-through registry-mirrors, this was not a problem, as in that case the
registry would forward the request, but for other mirrors, no fallback would
happen. This was inconsistent with how "pulling" images handled this situation;
when pulling images, both the mirror and upstream would be tried.

This problem was caused by the logic used in GetRepository, which had an
optimization to only return the first registry it was successfully able to
configure (and connect to), with the assumption that the mirror either contained
all images used, or to be configured as a pull-through mirror.

This patch:

- Introduces a GetRepositories method, which returns all candidates (both
mirror(s) and upstream).
- Updates the endpoint to try all

Before this patch:

# the daemon is configured to use a mirror for Docker Hub
cat /etc/docker/daemon.json
{ "registry-mirrors": ["http://localhost:5000"]}

# start the mirror (empty registry, not configured as pull-through mirror)
docker run -d --name registry -p 127.0.0.1:5000:5000 registry:2

# querying the endpoint fails, because the image-manifest is not found in the mirror:
curl -s --unix-socket /var/run/docker.sock http://localhost/v1.43/distribution/docker.io/library/hello-world:latest/json
{
"message": "manifest unknown: manifest unknown"
}

With this patch applied:

# the daemon is configured to use a mirror for Docker Hub
cat /etc/docker/daemon.json
{ "registry-mirrors": ["http://localhost:5000"]}

# start the mirror (empty registry, not configured as pull-through mirror)
docker run -d --name registry -p 127.0.0.1:5000:5000 registry:2

# querying the endpoint succeeds (manifest is fetched from the upstream Docker Hub registry):
curl -s --unix-socket /var/run/docker.sock http://localhost/v1.43/distribution/docker.io/library/hello-world:latest/json | jq .
{
"Descriptor": {
"mediaType": "application/vnd.oci.image.index.v1+json",
"digest": "sha256:1b9844d846ce3a6a6af7013e999a373112c3c0450aca49e155ae444526a2c45e",
"size": 3849
},
"Platforms": [
{
"architecture": "amd64",
"os": "linux"
}
]
}

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

Sebastiaan van Stijn authored on 2024/01/04 21:20:35
Showing 5 changed files
... ...
@@ -11,5 +11,5 @@ import (
11 11
 // Backend is all the methods that need to be implemented
12 12
 // to provide image specific functionality.
13 13
 type Backend interface {
14
-	GetRepository(context.Context, reference.Named, *registry.AuthConfig) (distribution.Repository, error)
14
+	GetRepositories(context.Context, reference.Named, *registry.AuthConfig) ([]distribution.Repository, error)
15 15
 }
... ...
@@ -6,6 +6,7 @@ import (
6 6
 	"net/http"
7 7
 
8 8
 	"github.com/distribution/reference"
9
+	"github.com/docker/distribution"
9 10
 	"github.com/docker/distribution/manifest/manifestlist"
10 11
 	"github.com/docker/distribution/manifest/schema1"
11 12
 	"github.com/docker/distribution/manifest/schema2"
... ...
@@ -42,24 +43,50 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res
42 42
 	// For a search it is not an error if no auth was given. Ignore invalid
43 43
 	// AuthConfig to increase compatibility with the existing API.
44 44
 	authConfig, _ := registry.DecodeAuthConfig(r.Header.Get(registry.AuthHeader))
45
-	distrepo, err := s.backend.GetRepository(ctx, namedRef, authConfig)
45
+	repos, err := s.backend.GetRepositories(ctx, namedRef, authConfig)
46 46
 	if err != nil {
47 47
 		return err
48 48
 	}
49
-	blobsrvc := distrepo.Blobs(ctx)
50 49
 
50
+	// Fetch the manifest; if a mirror is configured, try the mirror first,
51
+	// but continue with upstream on failure.
52
+	//
53
+	// FIXME(thaJeztah): construct "repositories" on-demand;
54
+	// GetRepositories() will attempt to connect to all endpoints (registries),
55
+	// but we may only need the first one if it contains the manifest we're
56
+	// looking for, or if the configured mirror is a pull-through mirror.
57
+	//
58
+	// Logic for this could be implemented similar to "distribution.Pull()",
59
+	// which uses the "pullEndpoints" utility to iterate over the list
60
+	// of endpoints;
61
+	//
62
+	// - https://github.com/moby/moby/blob/12c7411b6b7314bef130cd59f1c7384a7db06d0b/distribution/pull.go#L17-L31
63
+	// - https://github.com/moby/moby/blob/12c7411b6b7314bef130cd59f1c7384a7db06d0b/distribution/pull.go#L76-L152
64
+	var lastErr error
65
+	for _, repo := range repos {
66
+		distributionInspect, err := s.fetchManifest(ctx, repo, namedRef)
67
+		if err != nil {
68
+			lastErr = err
69
+			continue
70
+		}
71
+		return httputils.WriteJSON(w, http.StatusOK, distributionInspect)
72
+	}
73
+	return lastErr
74
+}
75
+
76
+func (s *distributionRouter) fetchManifest(ctx context.Context, distrepo distribution.Repository, namedRef reference.Named) (registry.DistributionInspect, error) {
51 77
 	var distributionInspect registry.DistributionInspect
52 78
 	if canonicalRef, ok := namedRef.(reference.Canonical); !ok {
53 79
 		namedRef = reference.TagNameOnly(namedRef)
54 80
 
55 81
 		taggedRef, ok := namedRef.(reference.NamedTagged)
56 82
 		if !ok {
57
-			return errdefs.InvalidParameter(errors.Errorf("image reference not tagged: %s", image))
83
+			return registry.DistributionInspect{}, errdefs.InvalidParameter(errors.Errorf("image reference not tagged: %s", namedRef))
58 84
 		}
59 85
 
60 86
 		descriptor, err := distrepo.Tags(ctx).Get(ctx, taggedRef.Tag())
61 87
 		if err != nil {
62
-			return err
88
+			return registry.DistributionInspect{}, err
63 89
 		}
64 90
 		distributionInspect.Descriptor = ocispec.Descriptor{
65 91
 			MediaType: descriptor.MediaType,
... ...
@@ -76,7 +103,7 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res
76 76
 	// we have a digest, so we can retrieve the manifest
77 77
 	mnfstsrvc, err := distrepo.Manifests(ctx)
78 78
 	if err != nil {
79
-		return err
79
+		return registry.DistributionInspect{}, err
80 80
 	}
81 81
 	mnfst, err := mnfstsrvc.Get(ctx, distributionInspect.Descriptor.Digest)
82 82
 	if err != nil {
... ...
@@ -88,14 +115,14 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res
88 88
 			reference.ErrNameEmpty,
89 89
 			reference.ErrNameTooLong,
90 90
 			reference.ErrNameNotCanonical:
91
-			return errdefs.InvalidParameter(err)
91
+			return registry.DistributionInspect{}, errdefs.InvalidParameter(err)
92 92
 		}
93
-		return err
93
+		return registry.DistributionInspect{}, err
94 94
 	}
95 95
 
96 96
 	mediaType, payload, err := mnfst.Payload()
97 97
 	if err != nil {
98
-		return err
98
+		return registry.DistributionInspect{}, err
99 99
 	}
100 100
 	// update MediaType because registry might return something incorrect
101 101
 	distributionInspect.Descriptor.MediaType = mediaType
... ...
@@ -116,7 +143,8 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res
116 116
 			})
117 117
 		}
118 118
 	case *schema2.DeserializedManifest:
119
-		configJSON, err := blobsrvc.Get(ctx, mnfstObj.Config.Digest)
119
+		blobStore := distrepo.Blobs(ctx)
120
+		configJSON, err := blobStore.Get(ctx, mnfstObj.Config.Digest)
120 121
 		var platform ocispec.Platform
121 122
 		if err == nil {
122 123
 			err := json.Unmarshal(configJSON, &platform)
... ...
@@ -131,6 +159,5 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res
131 131
 		}
132 132
 		distributionInspect.Platforms = append(distributionInspect.Platforms, platform)
133 133
 	}
134
-
135
-	return httputils.WriteJSON(w, http.StatusOK, distributionInspect)
134
+	return distributionInspect, nil
136 135
 }
... ...
@@ -78,5 +78,6 @@ type VolumeBackend interface {
78 78
 type ImageBackend interface {
79 79
 	PullImage(ctx context.Context, ref reference.Named, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error
80 80
 	GetRepository(context.Context, reference.Named, *registry.AuthConfig) (distribution.Repository, error)
81
+	GetRepositories(context.Context, reference.Named, *registry.AuthConfig) ([]distribution.Repository, error)
81 82
 	GetImage(ctx context.Context, refOrID string, options opts.GetImageOpts) (*image.Image, error)
82 83
 }
... ...
@@ -1595,3 +1595,19 @@ func (i *imageBackend) GetRepository(ctx context.Context, ref reference.Named, a
1595 1595
 		},
1596 1596
 	})
1597 1597
 }
1598
+
1599
+// GetRepositories returns a list of repositories configured for the given
1600
+// reference. Multiple repositories can be returned if the reference is for
1601
+// the default (Docker Hub) registry and a mirror is configured, but it omits
1602
+// registries that were not reachable (pinging the /v2/ endpoint failed).
1603
+//
1604
+// It returns an error if it was unable to reach any of the registries for
1605
+// the given reference, or if the provided reference is invalid.
1606
+func (i *imageBackend) GetRepositories(ctx context.Context, ref reference.Named, authConfig *registrytypes.AuthConfig) ([]dist.Repository, error) {
1607
+	return distribution.GetRepositories(ctx, ref, &distribution.ImagePullConfig{
1608
+		Config: distribution.Config{
1609
+			AuthConfig:      authConfig,
1610
+			RegistryService: i.registryService,
1611
+		},
1612
+	})
1613
+}
... ...
@@ -3,6 +3,7 @@ package distribution
3 3
 import (
4 4
 	"context"
5 5
 
6
+	"github.com/containerd/log"
6 7
 	"github.com/distribution/reference"
7 8
 	"github.com/docker/distribution"
8 9
 	"github.com/docker/docker/errdefs"
... ...
@@ -10,6 +11,25 @@ import (
10 10
 
11 11
 // GetRepository returns a repository from the registry.
12 12
 func GetRepository(ctx context.Context, ref reference.Named, config *ImagePullConfig) (repository distribution.Repository, lastError error) {
13
+	repos, err := getRepositories(ctx, ref, config, true)
14
+	if len(repos) == 0 {
15
+		return nil, err
16
+	}
17
+	return repos[0], nil
18
+}
19
+
20
+// GetRepositories returns a list of repositories configured for the given
21
+// reference. Multiple repositories can be returned if the reference is for
22
+// the default (Docker Hub) registry and a mirror is configured, but it omits
23
+// registries that were not reachable (pinging the /v2/ endpoint failed).
24
+//
25
+// It returns an error if it was unable to reach any of the registries for
26
+// the given reference, or if the provided reference is invalid.
27
+func GetRepositories(ctx context.Context, ref reference.Named, config *ImagePullConfig) ([]distribution.Repository, error) {
28
+	return getRepositories(ctx, ref, config, false)
29
+}
30
+
31
+func getRepositories(ctx context.Context, ref reference.Named, config *ImagePullConfig, firstOnly bool) ([]distribution.Repository, error) {
13 32
 	repoInfo, err := config.RegistryService.ResolveRepository(ref)
14 33
 	if err != nil {
15 34
 		return nil, errdefs.InvalidParameter(err)
... ...
@@ -24,11 +44,24 @@ func GetRepository(ctx context.Context, ref reference.Named, config *ImagePullCo
24 24
 		return nil, err
25 25
 	}
26 26
 
27
+	var (
28
+		repositories []distribution.Repository
29
+		lastError    error
30
+	)
27 31
 	for _, endpoint := range endpoints {
28
-		repository, lastError = newRepository(ctx, repoInfo, endpoint, nil, config.AuthConfig, "pull")
29
-		if lastError == nil {
30
-			break
32
+		repo, err := newRepository(ctx, repoInfo, endpoint, nil, config.AuthConfig, "pull")
33
+		if err != nil {
34
+			log.G(ctx).WithFields(log.Fields{"endpoint": endpoint.URL.String(), "error": err}).Info("endpoint")
35
+			lastError = err
36
+			continue
31 37
 		}
38
+		repositories = append(repositories, repo)
39
+		if firstOnly {
40
+			return repositories, nil
41
+		}
42
+	}
43
+	if len(repositories) == 0 {
44
+		return nil, lastError
32 45
 	}
33
-	return repository, lastError
46
+	return repositories, nil
34 47
 }