Browse code

Add daemon flag to specify public registry mirrors

Adds support for a --registry-mirror=scheme://<host>[:port]
daemon flag. The flag may be present multiple times. If
provided, mirrors are prepended to the list of endpoints used
for image pull. Note that only mirrors of the public
index.docker.io registry are supported, and image/tag resolution
is still performed via the official index.

Docker-DCO-1.1-Signed-off-by: Tim Smith <timbot@google.com> (github: timbot)

Tim Smith authored on 2014/07/19 03:48:19
Showing 10 changed files
... ...
@@ -23,6 +23,7 @@ type Config struct {
23 23
 	AutoRestart                 bool
24 24
 	Dns                         []string
25 25
 	DnsSearch                   []string
26
+	Mirrors                     []string
26 27
 	EnableIptables              bool
27 28
 	EnableIpForward             bool
28 29
 	DefaultIp                   net.IP
... ...
@@ -60,6 +61,7 @@ func (config *Config) InstallFlags() {
60 60
 	// FIXME: why the inconsistency between "hosts" and "sockets"?
61 61
 	opts.IPListVar(&config.Dns, []string{"#dns", "-dns"}, "Force Docker to use specific DNS servers")
62 62
 	opts.DnsSearchListVar(&config.DnsSearch, []string{"-dns-search"}, "Force Docker to use specific DNS search domains")
63
+	opts.MirrorListVar(&config.Mirrors, []string{"-registry-mirror"}, "Specify a preferred Docker registry mirror")
63 64
 }
64 65
 
65 66
 func GetDefaultNetworkMtu() int {
... ...
@@ -791,7 +791,7 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error)
791 791
 		return nil, err
792 792
 	}
793 793
 	log.Debugf("Creating repository list")
794
-	repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g)
794
+	repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g, config.Mirrors)
795 795
 	if err != nil {
796 796
 		return nil, fmt.Errorf("Couldn't create Tag store: %s", err)
797 797
 	}
... ...
@@ -64,6 +64,9 @@ unix://[/path/to/socket] to use.
64 64
 **-p**=""
65 65
   Path to use for daemon PID file. Default is `/var/run/docker.pid`
66 66
 
67
+**--registry-mirror=<scheme>://<host>
68
+  Prepend a registry mirror to be used for image pulls. May be specified multiple times.
69
+
67 70
 **-s**=""
68 71
   Force the Docker runtime to use a specific storage driver.
69 72
 
... ...
@@ -12,3 +12,4 @@
12 12
  - [Automatically Start Containers](host_integration/)
13 13
  - [Link via an Ambassador Container](ambassador_pattern_linking/)
14 14
  - [Increase a Boot2Docker Volume](b2d_volume_resize/)
15
+ - [Run a Local Registry Mirror](registry_mirror/)
15 16
new file mode 100644
... ...
@@ -0,0 +1,83 @@
0
+page_title: Run a local registry mirror
1
+page_description: How to set up and run a local registry mirror
2
+page_keywords: docker, registry, mirror, examples
3
+
4
+# Run a local registry mirror
5
+
6
+## Why?
7
+
8
+If you have multiple instances of Docker running in your environment
9
+(e.g., multiple physical or virtual machines, all running the Docker
10
+daemon), each time one of them requires an image that it doesn't have
11
+it will go out to the internet and fetch it from the public Docker
12
+registry. By running a local registry mirror, you can keep most of the
13
+image fetch traffic on your local network.
14
+
15
+## How does it work?
16
+
17
+The first time you request an image from your local registry mirror,
18
+it pulls the image from the public Docker registry and stores it locally
19
+before handing it back to you. On subsequent requests, the local registry
20
+mirror is able to serve the image from its own storage.
21
+
22
+## How do I set up a local registry mirror?
23
+
24
+There are two steps to set up and use a local registry mirror.
25
+
26
+### Step 1: Configure your Docker daemons to use the local registry mirror
27
+
28
+You will need to pass the `--registry-mirror` option to your Docker daemon on
29
+startup:
30
+
31
+    docker --registry-mirror=http://<my-docker-mirror-host> -d
32
+
33
+For example, if your mirror is serving on `http://10.0.0.2:5000`, you would run:
34
+
35
+    docker --registry-mirror=http://10.0.0.2:5000 -d
36
+
37
+**NOTE:**
38
+Depending on your local host setup, you may be able to add the
39
+`--registry-mirror` options to the `DOCKER_OPTS` variable in
40
+`/etc/defaults/docker`.
41
+
42
+### Step 2: Run the local registry mirror
43
+
44
+You will need to start a local registry mirror service. The
45
+[`registry` image](https://registry.hub.docker.com/_/registry/) provides this
46
+functionality. For example, to run a local registry mirror that serves on
47
+port `5000` and mirrors the content at `registry-1.docker.io`:
48
+
49
+    docker run -p 5000:5000 \
50
+        -e STANDALONE=false \
51
+        -e MIRROR_SOURCE=https://registry-1.docker.io \
52
+        -e MIRROR_SOURCE_INDEX=https://index.docker.io registry
53
+
54
+## Test it out
55
+
56
+With your mirror running, pull an image that you haven't pulled before (using
57
+`time` to time it):
58
+
59
+    $ time docker pull node:latest
60
+    Pulling repository node
61
+    [...]
62
+    
63
+    real   1m14.078s
64
+    user   0m0.176s
65
+    sys    0m0.120s
66
+
67
+Now, remove the image from your local machine:
68
+
69
+    $ docker rmi node:latest
70
+
71
+Finally, re-pull the image:
72
+
73
+    $ time docker pull node:latest
74
+    Pulling repository node
75
+    [...]
76
+    
77
+    real   0m51.376s
78
+    user   0m0.120s
79
+    sys    0m0.116s
80
+
81
+The second time around, the local registry mirror served the image from storage,
82
+avoiding a trip out to the internet to refetch it.
... ...
@@ -71,6 +71,7 @@ expect an integer, and they can only be specified once.
71 71
       --mtu=0                                    Set the containers network MTU
72 72
                                                    if no value is provided: default to the default route MTU or 1500 if no default route is available
73 73
       -p, --pidfile="/var/run/docker.pid"        Path to use for daemon PID file
74
+      --registry-mirror=[]                       Specify a preferred Docker registry mirror
74 75
       -s, --storage-driver=""                    Force the Docker runtime to use a specific storage driver
75 76
       --selinux-enabled=false                    Enable selinux support. SELinux does not presently support the BTRFS storage driver
76 77
       --storage-opt=[]                           Set storage driver options
... ...
@@ -25,6 +25,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
25 25
 		sf          = utils.NewStreamFormatter(job.GetenvBool("json"))
26 26
 		authConfig  = &registry.AuthConfig{}
27 27
 		metaHeaders map[string][]string
28
+		mirrors     []string
28 29
 	)
29 30
 	if len(job.Args) > 1 {
30 31
 		tag = job.Args[1]
... ...
@@ -64,16 +65,19 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
64 64
 	if endpoint == registry.IndexServerAddress() {
65 65
 		// If pull "index.docker.io/foo/bar", it's stored locally under "foo/bar"
66 66
 		localName = remoteName
67
+
68
+		// Use provided mirrors, if any
69
+		mirrors = s.mirrors
67 70
 	}
68 71
 
69
-	if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel")); err != nil {
72
+	if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel"), mirrors); err != nil {
70 73
 		return job.Error(err)
71 74
 	}
72 75
 
73 76
 	return engine.StatusOK
74 77
 }
75 78
 
76
-func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName, remoteName, askedTag string, sf *utils.StreamFormatter, parallel bool) error {
79
+func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName, remoteName, askedTag string, sf *utils.StreamFormatter, parallel bool, mirrors []string) error {
77 80
 	out.Write(sf.FormatStatus("", "Pulling repository %s", localName))
78 81
 
79 82
 	repoData, err := r.GetRepositoryData(remoteName)
... ...
@@ -153,17 +157,31 @@ func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName,
153 153
 			out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, localName), nil))
154 154
 			success := false
155 155
 			var lastErr error
156
-			for _, ep := range repoData.Endpoints {
157
-				out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, localName, ep), nil))
158
-				if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
159
-					// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
160
-					// As the error is also given to the output stream the user will see the error.
161
-					lastErr = err
162
-					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err), nil))
163
-					continue
156
+			if mirrors != nil {
157
+				for _, ep := range mirrors {
158
+					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, localName, ep), nil))
159
+					if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
160
+						// Don't report errors when pulling from mirrors.
161
+						log.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, localName, ep, err)
162
+						continue
163
+					}
164
+					success = true
165
+					break
166
+				}
167
+			}
168
+			if !success {
169
+				for _, ep := range repoData.Endpoints {
170
+					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, localName, ep), nil))
171
+					if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
172
+						// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
173
+						// As the error is also given to the output stream the user will see the error.
174
+						lastErr = err
175
+						out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err), nil))
176
+						continue
177
+					}
178
+					success = true
179
+					break
164 180
 				}
165
-				success = true
166
-				break
167 181
 			}
168 182
 			if !success {
169 183
 				err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, localName, lastErr)
... ...
@@ -20,6 +20,7 @@ const DEFAULTTAG = "latest"
20 20
 type TagStore struct {
21 21
 	path         string
22 22
 	graph        *Graph
23
+	mirrors      []string
23 24
 	Repositories map[string]Repository
24 25
 	sync.Mutex
25 26
 	// FIXME: move push/pull-related fields
... ...
@@ -48,7 +49,7 @@ func (r Repository) Contains(u Repository) bool {
48 48
 	return true
49 49
 }
50 50
 
51
-func NewTagStore(path string, graph *Graph) (*TagStore, error) {
51
+func NewTagStore(path string, graph *Graph, mirrors []string) (*TagStore, error) {
52 52
 	abspath, err := filepath.Abs(path)
53 53
 	if err != nil {
54 54
 		return nil, err
... ...
@@ -56,6 +57,7 @@ func NewTagStore(path string, graph *Graph) (*TagStore, error) {
56 56
 	store := &TagStore{
57 57
 		path:         abspath,
58 58
 		graph:        graph,
59
+		mirrors:      mirrors,
59 60
 		Repositories: make(map[string]Repository),
60 61
 		pullingPool:  make(map[string]chan struct{}),
61 62
 		pushingPool:  make(map[string]chan struct{}),
... ...
@@ -52,7 +52,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore {
52 52
 	if err != nil {
53 53
 		t.Fatal(err)
54 54
 	}
55
-	store, err := NewTagStore(path.Join(root, "tags"), graph)
55
+	store, err := NewTagStore(path.Join(root, "tags"), graph, nil)
56 56
 	if err != nil {
57 57
 		t.Fatal(err)
58 58
 	}
... ...
@@ -3,6 +3,7 @@ package opts
3 3
 import (
4 4
 	"fmt"
5 5
 	"net"
6
+	"net/url"
6 7
 	"os"
7 8
 	"path/filepath"
8 9
 	"regexp"
... ...
@@ -33,6 +34,10 @@ func IPVar(value *net.IP, names []string, defaultValue, usage string) {
33 33
 	flag.Var(NewIpOpt(value, defaultValue), names, usage)
34 34
 }
35 35
 
36
+func MirrorListVar(values *[]string, names []string, usage string) {
37
+	flag.Var(newListOptsRef(values, ValidateMirror), names, usage)
38
+}
39
+
36 40
 // ListOpts type
37 41
 type ListOpts struct {
38 42
 	values    *[]string
... ...
@@ -190,3 +195,21 @@ func validateDomain(val string) (string, error) {
190 190
 	}
191 191
 	return "", fmt.Errorf("%s is not a valid domain", val)
192 192
 }
193
+
194
+// Validates an HTTP(S) registry mirror
195
+func ValidateMirror(val string) (string, error) {
196
+	uri, err := url.Parse(val)
197
+	if err != nil {
198
+		return "", fmt.Errorf("%s is not a valid URI", val)
199
+	}
200
+
201
+	if uri.Scheme != "http" && uri.Scheme != "https" {
202
+		return "", fmt.Errorf("Unsupported scheme %s", uri.Scheme)
203
+	}
204
+
205
+	if uri.Path != "" || uri.RawQuery != "" || uri.Fragment != "" {
206
+		return "", fmt.Errorf("Unsupported path/query/fragment at end of the URI")
207
+	}
208
+
209
+	return fmt.Sprintf("%s://%s/v1/", uri.Scheme, uri.Host), nil
210
+}