Browse code

Inroduce SWARM --data-path-addr flag

This new flag will allow the configuration of an interface that
can be used for data path traffic to be isolated from control
plane traffic. This flag is simply percolated down to libnetwork
and will be used by all the global scope drivers (today overlay)

Negative test added for invalid flag arguments

Signed-off-by: Flavio Crisciani <flavio.crisciani@docker.com>

Flavio Crisciani authored on 2017/04/15 08:54:17
Showing 13 changed files
... ...
@@ -132,6 +132,7 @@ type ExternalCA struct {
132 132
 type InitRequest struct {
133 133
 	ListenAddr       string
134 134
 	AdvertiseAddr    string
135
+	DataPathAddr     string
135 136
 	ForceNewCluster  bool
136 137
 	Spec             Spec
137 138
 	AutoLockManagers bool
... ...
@@ -142,6 +143,7 @@ type InitRequest struct {
142 142
 type JoinRequest struct {
143 143
 	ListenAddr    string
144 144
 	AdvertiseAddr string
145
+	DataPathAddr  string
145 146
 	RemoteAddrs   []string
146 147
 	JoinToken     string // accept by secret
147 148
 	Availability  NodeAvailability
... ...
@@ -19,6 +19,7 @@ type initOptions struct {
19 19
 	listenAddr NodeAddrOption
20 20
 	// Not a NodeAddrOption because it has no default port.
21 21
 	advertiseAddr   string
22
+	dataPathAddr    string
22 23
 	forceNewCluster bool
23 24
 	availability    string
24 25
 }
... ...
@@ -40,6 +41,7 @@ func newInitCommand(dockerCli command.Cli) *cobra.Command {
40 40
 	flags := cmd.Flags()
41 41
 	flags.Var(&opts.listenAddr, flagListenAddr, "Listen address (format: <ip|interface>[:port])")
42 42
 	flags.StringVar(&opts.advertiseAddr, flagAdvertiseAddr, "", "Advertised address (format: <ip|interface>[:port])")
43
+	flags.StringVar(&opts.dataPathAddr, flagDataPathAddr, "", "Address or interface to use for data path traffic (format: <ip|interface>)")
43 44
 	flags.BoolVar(&opts.forceNewCluster, "force-new-cluster", false, "Force create a new cluster from current state")
44 45
 	flags.BoolVar(&opts.autolock, flagAutolock, false, "Enable manager autolocking (requiring an unlock key to start a stopped manager)")
45 46
 	flags.StringVar(&opts.availability, flagAvailability, "active", `Availability of the node ("active"|"pause"|"drain")`)
... ...
@@ -54,6 +56,7 @@ func runInit(dockerCli command.Cli, flags *pflag.FlagSet, opts initOptions) erro
54 54
 	req := swarm.InitRequest{
55 55
 		ListenAddr:       opts.listenAddr.String(),
56 56
 		AdvertiseAddr:    opts.advertiseAddr,
57
+		DataPathAddr:     opts.dataPathAddr,
57 58
 		ForceNewCluster:  opts.forceNewCluster,
58 59
 		Spec:             opts.swarmOptions.ToSpec(flags),
59 60
 		AutoLockManagers: opts.swarmOptions.autolock,
... ...
@@ -19,6 +19,7 @@ type joinOptions struct {
19 19
 	listenAddr NodeAddrOption
20 20
 	// Not a NodeAddrOption because it has no default port.
21 21
 	advertiseAddr string
22
+	dataPathAddr  string
22 23
 	token         string
23 24
 	availability  string
24 25
 }
... ...
@@ -41,6 +42,7 @@ func newJoinCommand(dockerCli command.Cli) *cobra.Command {
41 41
 	flags := cmd.Flags()
42 42
 	flags.Var(&opts.listenAddr, flagListenAddr, "Listen address (format: <ip|interface>[:port])")
43 43
 	flags.StringVar(&opts.advertiseAddr, flagAdvertiseAddr, "", "Advertised address (format: <ip|interface>[:port])")
44
+	flags.StringVar(&opts.dataPathAddr, flagDataPathAddr, "", "Address or interface to use for data path traffic (format: <ip|interface>)")
44 45
 	flags.StringVar(&opts.token, flagToken, "", "Token for entry into the swarm")
45 46
 	flags.StringVar(&opts.availability, flagAvailability, "active", `Availability of the node ("active"|"pause"|"drain")`)
46 47
 	return cmd
... ...
@@ -54,6 +56,7 @@ func runJoin(dockerCli command.Cli, flags *pflag.FlagSet, opts joinOptions) erro
54 54
 		JoinToken:     opts.token,
55 55
 		ListenAddr:    opts.listenAddr.String(),
56 56
 		AdvertiseAddr: opts.advertiseAddr,
57
+		DataPathAddr:  opts.dataPathAddr,
57 58
 		RemoteAddrs:   []string{opts.remote},
58 59
 	}
59 60
 	if flags.Changed(flagAvailability) {
... ...
@@ -19,6 +19,7 @@ const (
19 19
 	flagDispatcherHeartbeat = "dispatcher-heartbeat"
20 20
 	flagListenAddr          = "listen-addr"
21 21
 	flagAdvertiseAddr       = "advertise-addr"
22
+	flagDataPathAddr        = "data-path-addr"
22 23
 	flagQuiet               = "quiet"
23 24
 	flagRotate              = "rotate"
24 25
 	flagToken               = "token"
... ...
@@ -3301,7 +3301,7 @@ _docker_swarm_init() {
3301 3301
 
3302 3302
 	case "$cur" in
3303 3303
 		-*)
3304
-			COMPREPLY=( $( compgen -W "--advertise-addr --autolock --availability --cert-expiry --dispatcher-heartbeat --external-ca --force-new-cluster --help --listen-addr --max-snapshots --snapshot-interval --task-history-limit" -- "$cur" ) )
3304
+			COMPREPLY=( $( compgen -W "--advertise-addr --data-path-addr --autolock --availability --cert-expiry --dispatcher-heartbeat --external-ca --force-new-cluster --help --listen-addr --max-snapshots --snapshot-interval --task-history-limit" -- "$cur" ) )
3305 3305
 			;;
3306 3306
 	esac
3307 3307
 }
... ...
@@ -3337,7 +3337,7 @@ _docker_swarm_join() {
3337 3337
 
3338 3338
 	case "$cur" in
3339 3339
 		-*)
3340
-			COMPREPLY=( $( compgen -W "--advertise-addr --availability --help --listen-addr --token" -- "$cur" ) )
3340
+			COMPREPLY=( $( compgen -W "--advertise-addr --data-path-addr --availability --help --listen-addr --token" -- "$cur" ) )
3341 3341
 			;;
3342 3342
 		*:)
3343 3343
 			COMPREPLY=( $( compgen -W "2377" -- "${cur##*:}" ) )
... ...
@@ -2267,6 +2267,7 @@ __docker_swarm_subcommand() {
2267 2267
             _arguments $(__docker_arguments) \
2268 2268
                 $opts_help \
2269 2269
                 "($help)--advertise-addr=[Advertised address]:ip\:port: " \
2270
+                "($help)--data-path-addr=[Data path IP or interface]:ip " \
2270 2271
                 "($help)--autolock[Enable manager autolocking]" \
2271 2272
                 "($help)--availability=[Availability of the node]:availability:(active drain pause)" \
2272 2273
                 "($help)--cert-expiry=[Validity period for node certificates]:duration: " \
... ...
@@ -2282,6 +2283,7 @@ __docker_swarm_subcommand() {
2282 2282
             _arguments $(__docker_arguments) -A '-*' \
2283 2283
                 $opts_help \
2284 2284
                 "($help)--advertise-addr=[Advertised address]:ip\:port: " \
2285
+                "($help)--data-path-addr=[Data path IP or interface]:ip " \
2285 2286
                 "($help)--availability=[Availability of the node]:availability:(active drain pause)" \
2286 2287
                 "($help)--listen-addr=[Listen address]:ip\:port: " \
2287 2288
                 "($help)--token=[Token for entry into the swarm]:secret: " \
... ...
@@ -270,6 +270,16 @@ func (c *Cluster) GetAdvertiseAddress() string {
270 270
 	return c.currentNodeState().actualLocalAddr
271 271
 }
272 272
 
273
+// GetDataPathAddress returns the address to be used for the data path traffic, if specified.
274
+func (c *Cluster) GetDataPathAddress() string {
275
+	c.mu.RLock()
276
+	defer c.mu.RUnlock()
277
+	if c.nr != nil {
278
+		return c.nr.config.DataPathAddr
279
+	}
280
+	return ""
281
+}
282
+
273 283
 // GetRemoteAddress returns a known advertise address of a remote manager if
274 284
 // available.
275 285
 // todo: change to array/connect with info
... ...
@@ -10,8 +10,10 @@ var (
10 10
 	errNoSuchInterface         = errors.New("no such interface")
11 11
 	errNoIP                    = errors.New("could not find the system's IP address")
12 12
 	errMustSpecifyListenAddr   = errors.New("must specify a listening address because the address to advertise is not recognized as a system address, and a system's IP address to use could not be uniquely identified")
13
+	errBadNetworkIdentifier    = errors.New("must specify a valid IP address or interface name")
13 14
 	errBadListenAddr           = errors.New("listen address must be an IP address or network interface (with optional port number)")
14 15
 	errBadAdvertiseAddr        = errors.New("advertise address must be a non-zero IP address or network interface (with optional port number)")
16
+	errBadDataPathAddr         = errors.New("data path address must be a non-zero IP address or network interface (without a port number)")
15 17
 	errBadDefaultAdvertiseAddr = errors.New("default advertise address must be a non-zero IP address or network interface (without a port number)")
16 18
 )
17 19
 
... ...
@@ -20,23 +22,17 @@ func resolveListenAddr(specifiedAddr string) (string, string, error) {
20 20
 	if err != nil {
21 21
 		return "", "", fmt.Errorf("could not parse listen address %s", specifiedAddr)
22 22
 	}
23
-
24 23
 	// Does the host component match any of the interface names on the
25 24
 	// system? If so, use the address from that interface.
26
-	interfaceAddr, err := resolveInterfaceAddr(specifiedHost)
27
-	if err == nil {
28
-		return interfaceAddr.String(), specifiedPort, nil
29
-	}
30
-	if err != errNoSuchInterface {
25
+	specifiedIP, err := resolveInputIPAddr(specifiedHost, true)
26
+	if err != nil {
27
+		if err == errBadNetworkIdentifier {
28
+			err = errBadListenAddr
29
+		}
31 30
 		return "", "", err
32 31
 	}
33 32
 
34
-	// If it's not an interface, it must be an IP (for now)
35
-	if net.ParseIP(specifiedHost) == nil {
36
-		return "", "", errBadListenAddr
37
-	}
38
-
39
-	return specifiedHost, specifiedPort, nil
33
+	return specifiedIP.String(), specifiedPort, nil
40 34
 }
41 35
 
42 36
 func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (string, string, error) {
... ...
@@ -57,43 +53,32 @@ func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (st
57 57
 			advertiseHost = advertiseAddr
58 58
 			advertisePort = listenAddrPort
59 59
 		}
60
-
61 60
 		// Does the host component match any of the interface names on the
62 61
 		// system? If so, use the address from that interface.
63
-		interfaceAddr, err := resolveInterfaceAddr(advertiseHost)
64
-		if err == nil {
65
-			return interfaceAddr.String(), advertisePort, nil
66
-		}
67
-		if err != errNoSuchInterface {
62
+		advertiseIP, err := resolveInputIPAddr(advertiseHost, false)
63
+		if err != nil {
64
+			if err == errBadNetworkIdentifier {
65
+				err = errBadAdvertiseAddr
66
+			}
68 67
 			return "", "", err
69 68
 		}
70 69
 
71
-		// If it's not an interface, it must be an IP (for now)
72
-		if ip := net.ParseIP(advertiseHost); ip == nil || ip.IsUnspecified() {
73
-			return "", "", errBadAdvertiseAddr
74
-		}
75
-
76
-		return advertiseHost, advertisePort, nil
70
+		return advertiseIP.String(), advertisePort, nil
77 71
 	}
78 72
 
79 73
 	if c.config.DefaultAdvertiseAddr != "" {
80 74
 		// Does the default advertise address component match any of the
81 75
 		// interface names on the system? If so, use the address from
82 76
 		// that interface.
83
-		interfaceAddr, err := resolveInterfaceAddr(c.config.DefaultAdvertiseAddr)
84
-		if err == nil {
85
-			return interfaceAddr.String(), listenAddrPort, nil
86
-		}
87
-		if err != errNoSuchInterface {
77
+		defaultAdvertiseIP, err := resolveInputIPAddr(c.config.DefaultAdvertiseAddr, false)
78
+		if err != nil {
79
+			if err == errBadNetworkIdentifier {
80
+				err = errBadDefaultAdvertiseAddr
81
+			}
88 82
 			return "", "", err
89 83
 		}
90 84
 
91
-		// If it's not an interface, it must be an IP (for now)
92
-		if ip := net.ParseIP(c.config.DefaultAdvertiseAddr); ip == nil || ip.IsUnspecified() {
93
-			return "", "", errBadDefaultAdvertiseAddr
94
-		}
95
-
96
-		return c.config.DefaultAdvertiseAddr, listenAddrPort, nil
85
+		return defaultAdvertiseIP.String(), listenAddrPort, nil
97 86
 	}
98 87
 
99 88
 	systemAddr, err := c.resolveSystemAddr()
... ...
@@ -103,6 +88,22 @@ func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (st
103 103
 	return systemAddr.String(), listenAddrPort, nil
104 104
 }
105 105
 
106
+func resolveDataPathAddr(dataPathAddr string) (string, error) {
107
+	if dataPathAddr == "" {
108
+		// dataPathAddr is not defined
109
+		return "", nil
110
+	}
111
+	// If a data path flag is specified try to resolve the IP address.
112
+	dataPathIP, err := resolveInputIPAddr(dataPathAddr, false)
113
+	if err != nil {
114
+		if err == errBadNetworkIdentifier {
115
+			err = errBadDataPathAddr
116
+		}
117
+		return "", err
118
+	}
119
+	return dataPathIP.String(), nil
120
+}
121
+
106 122
 func resolveInterfaceAddr(specifiedInterface string) (net.IP, error) {
107 123
 	// Use a specific interface's IP address.
108 124
 	intf, err := net.InterfaceByName(specifiedInterface)
... ...
@@ -149,6 +150,30 @@ func resolveInterfaceAddr(specifiedInterface string) (net.IP, error) {
149 149
 	return interfaceAddr6, nil
150 150
 }
151 151
 
152
+// resolveInputIPAddr tries to resolve the IP address from the string passed as input
153
+// - tries to match the string as an interface name, if so returns the IP address associated with it
154
+// - on failure of previous step tries to parse the string as an IP address itself
155
+//	 if succeeds returns the IP address
156
+func resolveInputIPAddr(input string, isUnspecifiedValid bool) (net.IP, error) {
157
+	// Try to see if it is an interface name
158
+	interfaceAddr, err := resolveInterfaceAddr(input)
159
+	if err == nil {
160
+		return interfaceAddr, nil
161
+	}
162
+	// String matched interface but there is a potential ambiguity to be resolved
163
+	if err != errNoSuchInterface {
164
+		return nil, err
165
+	}
166
+
167
+	// String is not an interface check if it is a valid IP
168
+	if ip := net.ParseIP(input); ip != nil && (isUnspecifiedValid || !ip.IsUnspecified()) {
169
+		return ip, nil
170
+	}
171
+
172
+	// Not valid IP found
173
+	return nil, errBadNetworkIdentifier
174
+}
175
+
152 176
 func (c *Cluster) resolveSystemAddrViaSubnetCheck() (net.IP, error) {
153 177
 	// Use the system's only IP address, or fail if there are
154 178
 	// multiple addresses to choose from. Skip interfaces which
... ...
@@ -46,7 +46,10 @@ type nodeStartConfig struct {
46 46
 	ListenAddr string
47 47
 	// AdvertiseAddr is the address other nodes should connect to,
48 48
 	// including a port.
49
-	AdvertiseAddr   string
49
+	AdvertiseAddr string
50
+	// DataPathAddr is the address that has to be used for the data path
51
+	DataPathAddr string
52
+
50 53
 	joinAddr        string
51 54
 	forceNewCluster bool
52 55
 	joinToken       string
... ...
@@ -54,6 +54,11 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {
54 54
 		return "", err
55 55
 	}
56 56
 
57
+	dataPathAddr, err := resolveDataPathAddr(req.DataPathAddr)
58
+	if err != nil {
59
+		return "", err
60
+	}
61
+
57 62
 	localAddr := listenHost
58 63
 
59 64
 	// If the local address is undetermined, the advertise address
... ...
@@ -93,6 +98,7 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {
93 93
 		LocalAddr:       localAddr,
94 94
 		ListenAddr:      net.JoinHostPort(listenHost, listenPort),
95 95
 		AdvertiseAddr:   net.JoinHostPort(advertiseHost, advertisePort),
96
+		DataPathAddr:    dataPathAddr,
96 97
 		availability:    req.Availability,
97 98
 	})
98 99
 	if err != nil {
... ...
@@ -155,12 +161,18 @@ func (c *Cluster) Join(req types.JoinRequest) error {
155 155
 		}
156 156
 	}
157 157
 
158
+	dataPathAddr, err := resolveDataPathAddr(req.DataPathAddr)
159
+	if err != nil {
160
+		return err
161
+	}
162
+
158 163
 	clearPersistentState(c.root)
159 164
 
160 165
 	nr, err := c.newNodeRunner(nodeStartConfig{
161 166
 		RemoteAddr:    req.RemoteAddrs[0],
162 167
 		ListenAddr:    net.JoinHostPort(listenHost, listenPort),
163 168
 		AdvertiseAddr: advertiseAddr,
169
+		DataPathAddr:  dataPathAddr,
164 170
 		joinAddr:      req.RemoteAddrs[0],
165 171
 		joinToken:     req.JoinToken,
166 172
 		availability:  req.Availability,
... ...
@@ -25,6 +25,7 @@ Options:
25 25
       --autolock                        Enable manager autolocking (requiring an unlock key to start a stopped manager)
26 26
       --availability string             Availability of the node ("active"|"pause"|"drain") (default "active")
27 27
       --cert-expiry duration            Validity period for node certificates (ns|us|ms|s|m|h) (default 2160h0m0s)
28
+      --data-path-addr string           Address or interface to use for data path traffic (format: <ip|interface>)
28 29
       --dispatcher-heartbeat duration   Dispatcher heartbeat period (ns|us|ms|s|m|h) (default 5s)
29 30
       --external-ca external-ca         Specifications of one or more certificate signing endpoints
30 31
       --force-new-cluster               Force create a new cluster from current state
... ...
@@ -118,6 +119,15 @@ for example `--advertise-addr eth0:2377`.
118 118
 Specifying a port is optional. If the value is a bare IP address or interface
119 119
 name, the default port 2377 will be used.
120 120
 
121
+### `--data-path-addr`
122
+
123
+This flag specifies the address that global scope network drivers will publish towards
124
+other nodes in order to reach the containers running on this node.
125
+Using this parameter it is then possible to separate the container's data traffic from the
126
+management traffic of the cluster.
127
+If unspecified, Docker will use the same IP address or interface that is used for the
128
+advertise address.
129
+
121 130
 ### `--task-history-limit`
122 131
 
123 132
 This flag sets up task history retention limit.
... ...
@@ -23,6 +23,7 @@ Join a swarm as a node and/or manager
23 23
 Options:
24 24
       --advertise-addr string   Advertised address (format: <ip|interface>[:port])
25 25
       --availability string     Availability of the node ("active"|"pause"|"drain") (default "active")
26
+      --data-path-addr string   Address or interface to use for data path traffic (format: <ip|interface>)
26 27
       --help                    Print usage
27 28
       --listen-addr node-addr   Listen address (format: <ip|interface>[:port]) (default 0.0.0.0:2377)
28 29
       --token string            Token for entry into the swarm
... ...
@@ -95,6 +96,15 @@ name, the default port 2377 will be used.
95 95
 
96 96
 This flag is generally not necessary when joining an existing swarm.
97 97
 
98
+### `--data-path-addr`
99
+
100
+This flag specifies the address that global scope network drivers will publish towards
101
+other nodes in order to reach the containers running on this node.
102
+Using this parameter it is then possible to separate the container's data traffic from the
103
+management traffic of the cluster.
104
+If unspecified, Docker will use the same IP address or interface that is used for the
105
+advertise address.
106
+
98 107
 ### `--token string`
99 108
 
100 109
 Secret value required for nodes to join the swarm
... ...
@@ -1932,3 +1932,15 @@ func (s *DockerSwarmSuite) TestSwarmServiceLsFilterMode(c *check.C) {
1932 1932
 	c.Assert(out, checker.Contains, "top1")
1933 1933
 	c.Assert(out, checker.Not(checker.Contains), "top2")
1934 1934
 }
1935
+
1936
+func (s *DockerSwarmSuite) TestSwarmInitUnspecifiedDataPathAddr(c *check.C) {
1937
+	d := s.AddDaemon(c, false, false)
1938
+
1939
+	out, err := d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0")
1940
+	c.Assert(err, checker.NotNil)
1941
+	c.Assert(out, checker.Contains, "data path address must be a non-zero IP")
1942
+
1943
+	out, err = d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0:2000")
1944
+	c.Assert(err, checker.NotNil)
1945
+	c.Assert(out, checker.Contains, "data path address must be a non-zero IP")
1946
+}