Browse code

Pass upstream client's user agent through to registry on image pulls

Changes how the Engine interacts with Registry servers on image pull.
Previously, Engine sent a User-Agent string to the Registry server
that included only the Engine's version information. This commit
appends to that string the fields from the User-Agent sent by the
client (e.g., Compose) of the Engine. This allows Registry server
operators to understand what tools are actually generating pulls on
their registries.

Signed-off-by: Mike Goelzer <mgoelzer@docker.com>

Mike Goelzer authored on 2016/03/09 11:18:53
Showing 11 changed files
... ...
@@ -152,7 +152,7 @@ func (cli *DockerCli) getNotaryRepository(repoInfo *registry.RepositoryInfo, aut
152 152
 	}
153 153
 
154 154
 	// Skip configuration headers since request is not going to Docker daemon
155
-	modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(), http.Header{})
155
+	modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(""), http.Header{})
156 156
 	authTransport := transport.NewTransport(base, modifiers...)
157 157
 	pingClient := &http.Client{
158 158
 		Transport: authTransport,
... ...
@@ -16,6 +16,9 @@ import (
16 16
 // APIVersionKey is the client's requested API version.
17 17
 const APIVersionKey = "api-version"
18 18
 
19
+// UAStringKey is used as key type for user-agent string in net/context struct
20
+const UAStringKey = "upstream-user-agent"
21
+
19 22
 // APIFunc is an adapter to allow the use of ordinary functions as Docker API endpoints.
20 23
 // Any function that has the appropriate signature can be registered as a API endpoint (e.g. getVersion).
21 24
 type APIFunc func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error
... ...
@@ -16,6 +16,8 @@ func NewUserAgentMiddleware(versionCheck string) Middleware {
16 16
 
17 17
 	return func(handler httputils.APIFunc) httputils.APIFunc {
18 18
 		return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
19
+			ctx = context.WithValue(ctx, httputils.UAStringKey, r.Header.Get("User-Agent"))
20
+
19 21
 			if strings.Contains(r.Header.Get("User-Agent"), "Docker-Client/") {
20 22
 				userAgent := strings.Split(r.Header.Get("User-Agent"), "/")
21 23
 
... ...
@@ -7,6 +7,7 @@ import (
7 7
 	"github.com/docker/engine-api/types"
8 8
 	"github.com/docker/engine-api/types/container"
9 9
 	"github.com/docker/engine-api/types/registry"
10
+	"golang.org/x/net/context"
10 11
 )
11 12
 
12 13
 // Backend is all the methods that need to be implemented
... ...
@@ -37,7 +38,7 @@ type importExportBackend interface {
37 37
 }
38 38
 
39 39
 type registryBackend interface {
40
-	PullImage(ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
40
+	PullImage(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
41 41
 	PushImage(ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
42 42
 	SearchRegistryForImages(term string, authConfig *types.AuthConfig, metaHeaders map[string][]string) (*registry.SearchResults, error)
43 43
 }
... ...
@@ -129,7 +129,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite
129 129
 					}
130 130
 				}
131 131
 
132
-				err = s.backend.PullImage(ref, metaHeaders, authConfig, output)
132
+				err = s.backend.PullImage(ctx, ref, metaHeaders, authConfig, output)
133 133
 			}
134 134
 		}
135 135
 		// Check the error from pulling an image to make sure the request
... ...
@@ -1007,14 +1007,14 @@ func isBrokenPipe(e error) bool {
1007 1007
 
1008 1008
 // PullImage initiates a pull operation. image is the repository name to pull, and
1009 1009
 // tag may be either empty, or indicate a specific tag to pull.
1010
-func (daemon *Daemon) PullImage(ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error {
1010
+func (daemon *Daemon) PullImage(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error {
1011 1011
 	// Include a buffer so that slow client connections don't affect
1012 1012
 	// transfer performance.
1013 1013
 	progressChan := make(chan progress.Progress, 100)
1014 1014
 
1015 1015
 	writesDone := make(chan struct{})
1016 1016
 
1017
-	ctx, cancelFunc := context.WithCancel(context.Background())
1017
+	ctx, cancelFunc := context.WithCancel(ctx)
1018 1018
 
1019 1019
 	go func() {
1020 1020
 		writeDistributionProgress(cancelFunc, outStream, progressChan)
... ...
@@ -1062,7 +1062,7 @@ func (daemon *Daemon) PullOnBuild(name string, authConfigs map[string]types.Auth
1062 1062
 		pullRegistryAuth = &resolvedConfig
1063 1063
 	}
1064 1064
 
1065
-	if err := daemon.PullImage(ref, nil, pullRegistryAuth, output); err != nil {
1065
+	if err := daemon.PullImage(context.Background(), ref, nil, pullRegistryAuth, output); err != nil {
1066 1066
 		return nil, err
1067 1067
 	}
1068 1068
 	return daemon.GetImage(name)
... ...
@@ -1519,7 +1519,7 @@ func configureVolumes(config *Config, rootUID, rootGID int) (*store.VolumeStore,
1519 1519
 
1520 1520
 // AuthenticateToRegistry checks the validity of credentials in authConfig
1521 1521
 func (daemon *Daemon) AuthenticateToRegistry(authConfig *types.AuthConfig) (string, string, error) {
1522
-	return daemon.RegistryService.Auth(authConfig, dockerversion.DockerUserAgent())
1522
+	return daemon.RegistryService.Auth(authConfig, dockerversion.DockerUserAgent(""))
1523 1523
 }
1524 1524
 
1525 1525
 // SearchRegistryForImages queries the registry for images matching
... ...
@@ -1527,7 +1527,7 @@ func (daemon *Daemon) AuthenticateToRegistry(authConfig *types.AuthConfig) (stri
1527 1527
 func (daemon *Daemon) SearchRegistryForImages(term string,
1528 1528
 	authConfig *types.AuthConfig,
1529 1529
 	headers map[string][]string) (*registrytypes.SearchResults, error) {
1530
-	return daemon.RegistryService.Search(term, authConfig, dockerversion.DockerUserAgent(), headers)
1530
+	return daemon.RegistryService.Search(term, authConfig, dockerversion.DockerUserAgent(""), headers)
1531 1531
 }
1532 1532
 
1533 1533
 // IsShuttingDown tells whether the daemon is shutting down or not
... ...
@@ -49,10 +49,10 @@ func (p *v1Puller) Pull(ctx context.Context, ref reference.Named) error {
49 49
 	tr := transport.NewTransport(
50 50
 		// TODO(tiborvass): was ReceiveTimeout
51 51
 		registry.NewTransport(tlsConfig),
52
-		registry.DockerHeaders(dockerversion.DockerUserAgent(), p.config.MetaHeaders)...,
52
+		registry.DockerHeaders(dockerversion.DockerUserAgent(""), p.config.MetaHeaders)...,
53 53
 	)
54 54
 	client := registry.HTTPClient(tr)
55
-	v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(), p.config.MetaHeaders)
55
+	v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(""), p.config.MetaHeaders)
56 56
 	if err != nil {
57 57
 		logrus.Debugf("Could not get v1 endpoint: %v", err)
58 58
 		return fallbackError{err: err}
... ...
@@ -38,10 +38,10 @@ func (p *v1Pusher) Push(ctx context.Context) error {
38 38
 	tr := transport.NewTransport(
39 39
 		// TODO(tiborvass): was NoTimeout
40 40
 		registry.NewTransport(tlsConfig),
41
-		registry.DockerHeaders(dockerversion.DockerUserAgent(), p.config.MetaHeaders)...,
41
+		registry.DockerHeaders(dockerversion.DockerUserAgent(""), p.config.MetaHeaders)...,
42 42
 	)
43 43
 	client := registry.HTTPClient(tr)
44
-	v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(), p.config.MetaHeaders)
44
+	v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(""), p.config.MetaHeaders)
45 45
 	if err != nil {
46 46
 		logrus.Debugf("Could not get v1 endpoint: %v", err)
47 47
 		return fallbackError{err: err}
... ...
@@ -37,6 +37,8 @@ func (dcs dumbCredentialStore) SetRefreshToken(*url.URL, string, string) {
37 37
 // providing timeout settings and authentication support, and also verifies the
38 38
 // remote API version.
39 39
 func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *types.AuthConfig, actions ...string) (repo distribution.Repository, foundVersion bool, err error) {
40
+	upstreamUA := dockerversion.GetUserAgentFromContext(ctx)
41
+
40 42
 	repoName := repoInfo.FullName()
41 43
 	// If endpoint does not support CanonicalName, use the RemoteName instead
42 44
 	if endpoint.TrimHostname {
... ...
@@ -57,7 +59,7 @@ func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, end
57 57
 		DisableKeepAlives: true,
58 58
 	}
59 59
 
60
-	modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(), metaHeaders)
60
+	modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(upstreamUA), metaHeaders)
61 61
 	authTransport := transport.NewTransport(base, modifiers...)
62 62
 
63 63
 	challengeManager, foundVersion, err := registry.PingV2Registry(endpoint, authTransport)
... ...
@@ -1,15 +1,19 @@
1 1
 package dockerversion
2 2
 
3 3
 import (
4
+	"fmt"
4 5
 	"runtime"
5 6
 
7
+	"github.com/docker/docker/api/server/httputils"
6 8
 	"github.com/docker/docker/pkg/parsers/kernel"
7 9
 	"github.com/docker/docker/pkg/useragent"
10
+	"golang.org/x/net/context"
8 11
 )
9 12
 
10 13
 // DockerUserAgent is the User-Agent the Docker client uses to identify itself.
11
-// It is populated from version information of different components.
12
-func DockerUserAgent() string {
14
+// In accordance with RFC 7231 (5.5.3) is of the form:
15
+//    [docker client's UA] UpstreamClient([upstream client's UA])
16
+func DockerUserAgent(upstreamUA string) string {
13 17
 	httpVersion := make([]useragent.VersionInfo, 0, 6)
14 18
 	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "docker", Version: Version})
15 19
 	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "go", Version: runtime.Version()})
... ...
@@ -20,5 +24,50 @@ func DockerUserAgent() string {
20 20
 	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "os", Version: runtime.GOOS})
21 21
 	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "arch", Version: runtime.GOARCH})
22 22
 
23
-	return useragent.AppendVersions("", httpVersion...)
23
+	dockerUA := useragent.AppendVersions("", httpVersion...)
24
+	if len(upstreamUA) > 0 {
25
+		ret := insertUpstreamUserAgent(upstreamUA, dockerUA)
26
+		return ret
27
+	}
28
+	return dockerUA
29
+}
30
+
31
+// GetUserAgentFromContext returns the previously saved user-agent context stored in ctx, if one exists
32
+func GetUserAgentFromContext(ctx context.Context) string {
33
+	var upstreamUA string
34
+	if ctx != nil {
35
+		var ki interface{} = ctx.Value(httputils.UAStringKey)
36
+		if ki != nil {
37
+			upstreamUA = ctx.Value(httputils.UAStringKey).(string)
38
+		}
39
+	}
40
+	return upstreamUA
41
+}
42
+
43
+// escapeStr returns s with every rune in charsToEscape escaped by a backslash
44
+func escapeStr(s string, charsToEscape string) string {
45
+	var ret string
46
+	for _, currRune := range s {
47
+		appended := false
48
+		for _, escapeableRune := range charsToEscape {
49
+			if currRune == escapeableRune {
50
+				ret += "\\" + string(currRune)
51
+				appended = true
52
+				break
53
+			}
54
+		}
55
+		if !appended {
56
+			ret += string(currRune)
57
+		}
58
+	}
59
+	return ret
60
+}
61
+
62
+// insertUpstreamUserAgent adds the upstream client useragent to create a user-agent
63
+// string of the form:
64
+//   $dockerUA UpstreamClient($upstreamUA)
65
+func insertUpstreamUserAgent(upstreamUA string, dockerUA string) string {
66
+	charsToEscape := "();\\" //["\\", ";", "(", ")"]string
67
+	upstreamUAEscaped := escapeStr(upstreamUA, charsToEscape)
68
+	return fmt.Sprintf("%s UpstreamClient(%s)", dockerUA, upstreamUAEscaped)
24 69
 }
25 70
new file mode 100644
... ...
@@ -0,0 +1,90 @@
0
+package main
1
+
2
+import (
3
+	"fmt"
4
+	"net/http"
5
+	"regexp"
6
+
7
+	"github.com/go-check/check"
8
+)
9
+
10
+// unescapeBackslashSemicolonParens unescapes \;()
11
+func unescapeBackslashSemicolonParens(s string) string {
12
+	re := regexp.MustCompile("\\\\;")
13
+	ret := re.ReplaceAll([]byte(s), []byte(";"))
14
+
15
+	re = regexp.MustCompile("\\\\\\(")
16
+	ret = re.ReplaceAll([]byte(ret), []byte("("))
17
+
18
+	re = regexp.MustCompile("\\\\\\)")
19
+	ret = re.ReplaceAll([]byte(ret), []byte(")"))
20
+
21
+	re = regexp.MustCompile("\\\\\\\\")
22
+	ret = re.ReplaceAll([]byte(ret), []byte("\\"))
23
+
24
+	return string(ret)
25
+}
26
+
27
+func regexpCheckUA(c *check.C, ua string) {
28
+	re := regexp.MustCompile("(?P<dockerUA>.+) UpstreamClient(?P<upstreamUA>.+)")
29
+	substrArr := re.FindStringSubmatch(ua)
30
+
31
+	c.Assert(substrArr, check.HasLen, 3, check.Commentf("Expected 'UpstreamClient()' with upstream client UA"))
32
+	dockerUA := substrArr[1]
33
+	upstreamUAEscaped := substrArr[2]
34
+
35
+	// check dockerUA looks correct
36
+	reDockerUA := regexp.MustCompile("^docker/[0-9A-Za-z+]")
37
+	bMatchDockerUA := reDockerUA.MatchString(dockerUA)
38
+	c.Assert(bMatchDockerUA, check.Equals, true, check.Commentf("Docker Engine User-Agent malformed"))
39
+
40
+	// check upstreamUA looks correct
41
+	// Expecting something like:  Docker-Client/1.11.0-dev (linux)
42
+	upstreamUA := unescapeBackslashSemicolonParens(upstreamUAEscaped)
43
+	reUpstreamUA := regexp.MustCompile("^\\(Docker-Client/[0-9A-Za-z+]")
44
+	bMatchUpstreamUA := reUpstreamUA.MatchString(upstreamUA)
45
+	c.Assert(bMatchUpstreamUA, check.Equals, true, check.Commentf("(Upstream) Docker Client User-Agent malformed"))
46
+}
47
+
48
+// TestUserAgentPassThroughOnPull verifies that when an image is pulled from
49
+// a registry, the registry should see a User-Agent string of the form
50
+//   [docker engine UA] UptreamClientSTREAM-CLIENT([client UA])
51
+func (s *DockerRegistrySuite) TestUserAgentPassThroughOnPull(c *check.C) {
52
+	reg, err := newTestRegistry(c)
53
+	c.Assert(err, check.IsNil)
54
+	expectUpstreamUA := false
55
+
56
+	reg.registerHandler("/v2/", func(w http.ResponseWriter, r *http.Request) {
57
+		w.WriteHeader(404)
58
+		var ua string
59
+		for k, v := range r.Header {
60
+			if k == "User-Agent" {
61
+				ua = v[0]
62
+			}
63
+		}
64
+		c.Assert(ua, check.Not(check.Equals), "", check.Commentf("No User-Agent found in request"))
65
+		if r.URL.Path == "/v2/busybox/manifests/latest" {
66
+			if expectUpstreamUA {
67
+				regexpCheckUA(c, ua)
68
+			}
69
+		}
70
+	})
71
+
72
+	repoName := fmt.Sprintf("%s/busybox", reg.hostport)
73
+	err = s.d.Start("--insecure-registry", reg.hostport, "--disable-legacy-registry=true")
74
+	c.Assert(err, check.IsNil)
75
+
76
+	dockerfileName, cleanup, err := makefile(fmt.Sprintf("FROM %s/busybox", reg.hostport))
77
+	c.Assert(err, check.IsNil, check.Commentf("Unable to create test dockerfile"))
78
+	defer cleanup()
79
+
80
+	s.d.Cmd("build", "--file", dockerfileName, ".")
81
+
82
+	s.d.Cmd("run", repoName)
83
+	s.d.Cmd("login", "-u", "richard", "-p", "testtest", "-e", "testuser@testdomain.com", reg.hostport)
84
+	s.d.Cmd("tag", "busybox", repoName)
85
+	s.d.Cmd("push", repoName)
86
+
87
+	expectUpstreamUA = true
88
+	s.d.Cmd("pull", repoName)
89
+}