Browse code

update libnetwork to improve scalabiltiy of bridge network isolation rules

* libnetwork#2121: Retry other external DNS servers on ServFail
* libnetwork#2125: Fix README flag and expose orphan network peers
* libnetwork#2126: Adding goreport card
* libnetwork#2130: Modify awk to use cut in check_ip_overlap
* libnetwork#2117: [Carry 1534] Improve scalabiltiy of bridge network isolation rules

Full changes: https://github.com/docker/libnetwork/compare/2bf63300c52f5ea61989f85c732f00097d746530...5c1218c956c99f3365711974e300087810c31379

Signed-off-by: Akihiro Suda <suda.akihiro@lab.ntt.co.jp>

Akihiro Suda authored on 2018/04/03 16:50:00
Showing 8 changed files
... ...
@@ -3,7 +3,7 @@
3 3
 # LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When
4 4
 # updating the binary version, consider updating github.com/docker/libnetwork
5 5
 # in vendor.conf accordingly
6
-LIBNETWORK_COMMIT=1b91bc94094ecfdae41daa465cc0c8df37dfb3dd
6
+LIBNETWORK_COMMIT=5c1218c956c99f3365711974e300087810c31379
7 7
 
8 8
 install_proxy() {
9 9
 	case "$1" in
... ...
@@ -32,7 +32,7 @@ github.com/tonistiigi/fsutil dea3a0da73aee887fc02142d995be764106ac5e2
32 32
 #get libnetwork packages
33 33
 
34 34
 # When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly
35
-github.com/docker/libnetwork 2bf63300c52f5ea61989f85c732f00097d746530
35
+github.com/docker/libnetwork 5c1218c956c99f3365711974e300087810c31379
36 36
 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
37 37
 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
38 38
 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
... ...
@@ -1,6 +1,6 @@
1 1
 # libnetwork - networking for containers
2 2
 
3
-[![Circle CI](https://circleci.com/gh/docker/libnetwork/tree/master.svg?style=svg)](https://circleci.com/gh/docker/libnetwork/tree/master) [![Coverage Status](https://coveralls.io/repos/docker/libnetwork/badge.svg)](https://coveralls.io/r/docker/libnetwork) [![GoDoc](https://godoc.org/github.com/docker/libnetwork?status.svg)](https://godoc.org/github.com/docker/libnetwork)
3
+[![Circle CI](https://circleci.com/gh/docker/libnetwork/tree/master.svg?style=svg)](https://circleci.com/gh/docker/libnetwork/tree/master) [![Coverage Status](https://coveralls.io/repos/docker/libnetwork/badge.svg)](https://coveralls.io/r/docker/libnetwork) [![GoDoc](https://godoc.org/github.com/docker/libnetwork?status.svg)](https://godoc.org/github.com/docker/libnetwork) [![Go Report Card](https://goreportcard.com/badge/github.com/docker/libnetwork)](https://goreportcard.com/report/github.com/docker/libnetwork)
4 4
 
5 5
 Libnetwork provides a native Go implementation for connecting containers
6 6
 
... ...
@@ -137,15 +137,16 @@ type bridgeNetwork struct {
137 137
 }
138 138
 
139 139
 type driver struct {
140
-	config         *configuration
141
-	network        *bridgeNetwork
142
-	natChain       *iptables.ChainInfo
143
-	filterChain    *iptables.ChainInfo
144
-	isolationChain *iptables.ChainInfo
145
-	networks       map[string]*bridgeNetwork
146
-	store          datastore.DataStore
147
-	nlh            *netlink.Handle
148
-	configNetwork  sync.Mutex
140
+	config          *configuration
141
+	network         *bridgeNetwork
142
+	natChain        *iptables.ChainInfo
143
+	filterChain     *iptables.ChainInfo
144
+	isolationChain1 *iptables.ChainInfo
145
+	isolationChain2 *iptables.ChainInfo
146
+	networks        map[string]*bridgeNetwork
147
+	store           datastore.DataStore
148
+	nlh             *netlink.Handle
149
+	configNetwork   sync.Mutex
149 150
 	sync.Mutex
150 151
 }
151 152
 
... ...
@@ -266,15 +267,15 @@ func (n *bridgeNetwork) registerIptCleanFunc(clean iptableCleanFunc) {
266 266
 	n.iptCleanFuncs = append(n.iptCleanFuncs, clean)
267 267
 }
268 268
 
269
-func (n *bridgeNetwork) getDriverChains() (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
269
+func (n *bridgeNetwork) getDriverChains() (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
270 270
 	n.Lock()
271 271
 	defer n.Unlock()
272 272
 
273 273
 	if n.driver == nil {
274
-		return nil, nil, nil, types.BadRequestErrorf("no driver found")
274
+		return nil, nil, nil, nil, types.BadRequestErrorf("no driver found")
275 275
 	}
276 276
 
277
-	return n.driver.natChain, n.driver.filterChain, n.driver.isolationChain, nil
277
+	return n.driver.natChain, n.driver.filterChain, n.driver.isolationChain1, n.driver.isolationChain2, nil
278 278
 }
279 279
 
280 280
 func (n *bridgeNetwork) getNetworkBridgeName() string {
... ...
@@ -311,33 +312,18 @@ func (n *bridgeNetwork) isolateNetwork(others []*bridgeNetwork, enable bool) err
311 311
 		return nil
312 312
 	}
313 313
 
314
-	// Install the rules to isolate this networks against each of the other networks
315
-	for _, o := range others {
316
-		o.Lock()
317
-		otherConfig := o.config
318
-		o.Unlock()
319
-
320
-		if otherConfig.Internal {
321
-			continue
322
-		}
323
-
324
-		if thisConfig.BridgeName != otherConfig.BridgeName {
325
-			if err := setINC(thisConfig.BridgeName, otherConfig.BridgeName, enable); err != nil {
326
-				return err
327
-			}
328
-		}
329
-	}
330
-
331
-	return nil
314
+	// Install the rules to isolate this network against each of the other networks
315
+	return setINC(thisConfig.BridgeName, enable)
332 316
 }
333 317
 
334 318
 func (d *driver) configure(option map[string]interface{}) error {
335 319
 	var (
336
-		config         *configuration
337
-		err            error
338
-		natChain       *iptables.ChainInfo
339
-		filterChain    *iptables.ChainInfo
340
-		isolationChain *iptables.ChainInfo
320
+		config          *configuration
321
+		err             error
322
+		natChain        *iptables.ChainInfo
323
+		filterChain     *iptables.ChainInfo
324
+		isolationChain1 *iptables.ChainInfo
325
+		isolationChain2 *iptables.ChainInfo
341 326
 	)
342 327
 
343 328
 	genericData, ok := option[netlabel.GenericData]
... ...
@@ -365,7 +351,7 @@ func (d *driver) configure(option map[string]interface{}) error {
365 365
 			}
366 366
 		}
367 367
 		removeIPChains()
368
-		natChain, filterChain, isolationChain, err = setupIPChains(config)
368
+		natChain, filterChain, isolationChain1, isolationChain2, err = setupIPChains(config)
369 369
 		if err != nil {
370 370
 			return err
371 371
 		}
... ...
@@ -384,7 +370,8 @@ func (d *driver) configure(option map[string]interface{}) error {
384 384
 	d.Lock()
385 385
 	d.natChain = natChain
386 386
 	d.filterChain = filterChain
387
-	d.isolationChain = isolationChain
387
+	d.isolationChain1 = isolationChain1
388
+	d.isolationChain2 = isolationChain2
388 389
 	d.config = config
389 390
 	d.Unlock()
390 391
 
... ...
@@ -12,52 +12,85 @@ import (
12 12
 
13 13
 // DockerChain: DOCKER iptable chain name
14 14
 const (
15
-	DockerChain    = "DOCKER"
16
-	IsolationChain = "DOCKER-ISOLATION"
15
+	DockerChain = "DOCKER"
16
+	// Isolation between bridge networks is achieved in two stages by means
17
+	// of the following two chains in the filter table. The first chain matches
18
+	// on the source interface being a bridge network's bridge and the
19
+	// destination being a different interface. A positive match leads to the
20
+	// second isolation chain. No match returns to the parent chain. The second
21
+	// isolation chain matches on destination interface being a bridge network's
22
+	// bridge. A positive match identifies a packet originated from one bridge
23
+	// network's bridge destined to another bridge network's bridge and will
24
+	// result in the packet being dropped. No match returns to the parent chain.
25
+	IsolationChain1 = "DOCKER-ISOLATION-STAGE-1"
26
+	IsolationChain2 = "DOCKER-ISOLATION-STAGE-2"
17 27
 )
18 28
 
19
-func setupIPChains(config *configuration) (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
29
+func setupIPChains(config *configuration) (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
20 30
 	// Sanity check.
21 31
 	if config.EnableIPTables == false {
22
-		return nil, nil, nil, errors.New("cannot create new chains, EnableIPTable is disabled")
32
+		return nil, nil, nil, nil, errors.New("cannot create new chains, EnableIPTable is disabled")
23 33
 	}
24 34
 
25 35
 	hairpinMode := !config.EnableUserlandProxy
26 36
 
27 37
 	natChain, err := iptables.NewChain(DockerChain, iptables.Nat, hairpinMode)
28 38
 	if err != nil {
29
-		return nil, nil, nil, fmt.Errorf("failed to create NAT chain: %v", err)
39
+		return nil, nil, nil, nil, fmt.Errorf("failed to create NAT chain %s: %v", DockerChain, err)
30 40
 	}
31 41
 	defer func() {
32 42
 		if err != nil {
33 43
 			if err := iptables.RemoveExistingChain(DockerChain, iptables.Nat); err != nil {
34
-				logrus.Warnf("failed on removing iptables NAT chain on cleanup: %v", err)
44
+				logrus.Warnf("failed on removing iptables NAT chain %s on cleanup: %v", DockerChain, err)
35 45
 			}
36 46
 		}
37 47
 	}()
38 48
 
39 49
 	filterChain, err := iptables.NewChain(DockerChain, iptables.Filter, false)
40 50
 	if err != nil {
41
-		return nil, nil, nil, fmt.Errorf("failed to create FILTER chain: %v", err)
51
+		return nil, nil, nil, nil, fmt.Errorf("failed to create FILTER chain %s: %v", DockerChain, err)
42 52
 	}
43 53
 	defer func() {
44 54
 		if err != nil {
45 55
 			if err := iptables.RemoveExistingChain(DockerChain, iptables.Filter); err != nil {
46
-				logrus.Warnf("failed on removing iptables FILTER chain on cleanup: %v", err)
56
+				logrus.Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", DockerChain, err)
47 57
 			}
48 58
 		}
49 59
 	}()
50 60
 
51
-	isolationChain, err := iptables.NewChain(IsolationChain, iptables.Filter, false)
61
+	isolationChain1, err := iptables.NewChain(IsolationChain1, iptables.Filter, false)
52 62
 	if err != nil {
53
-		return nil, nil, nil, fmt.Errorf("failed to create FILTER isolation chain: %v", err)
63
+		return nil, nil, nil, nil, fmt.Errorf("failed to create FILTER isolation chain: %v", err)
54 64
 	}
65
+	defer func() {
66
+		if err != nil {
67
+			if err := iptables.RemoveExistingChain(IsolationChain1, iptables.Filter); err != nil {
68
+				logrus.Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", IsolationChain1, err)
69
+			}
70
+		}
71
+	}()
72
+
73
+	isolationChain2, err := iptables.NewChain(IsolationChain2, iptables.Filter, false)
74
+	if err != nil {
75
+		return nil, nil, nil, nil, fmt.Errorf("failed to create FILTER isolation chain: %v", err)
76
+	}
77
+	defer func() {
78
+		if err != nil {
79
+			if err := iptables.RemoveExistingChain(IsolationChain2, iptables.Filter); err != nil {
80
+				logrus.Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", IsolationChain2, err)
81
+			}
82
+		}
83
+	}()
55 84
 
56
-	if err := iptables.AddReturnRule(IsolationChain); err != nil {
57
-		return nil, nil, nil, err
85
+	if err := iptables.AddReturnRule(IsolationChain1); err != nil {
86
+		return nil, nil, nil, nil, err
58 87
 	}
59 88
 
60
-	return natChain, filterChain, isolationChain, nil
89
+	if err := iptables.AddReturnRule(IsolationChain2); err != nil {
90
+		return nil, nil, nil, nil, err
91
+	}
92
+
93
+	return natChain, filterChain, isolationChain1, isolationChain2, nil
61 94
 }
62 95
 
63 96
 func (n *bridgeNetwork) setupIPTables(config *networkConfiguration, i *bridgeInterface) error {
... ...
@@ -94,7 +127,7 @@ func (n *bridgeNetwork) setupIPTables(config *networkConfiguration, i *bridgeInt
94 94
 		n.registerIptCleanFunc(func() error {
95 95
 			return setupIPTablesInternal(config.BridgeName, maskedAddrv4, config.EnableICC, config.EnableIPMasquerade, hairpinMode, false)
96 96
 		})
97
-		natChain, filterChain, _, err := n.getDriverChains()
97
+		natChain, filterChain, _, _, err := n.getDriverChains()
98 98
 		if err != nil {
99 99
 			return fmt.Errorf("Failed to setup IP tables, cannot acquire chain info %s", err.Error())
100 100
 		}
... ...
@@ -117,7 +150,7 @@ func (n *bridgeNetwork) setupIPTables(config *networkConfiguration, i *bridgeInt
117 117
 	}
118 118
 
119 119
 	d.Lock()
120
-	err = iptables.EnsureJumpRule("FORWARD", IsolationChain)
120
+	err = iptables.EnsureJumpRule("FORWARD", IsolationChain1)
121 121
 	d.Unlock()
122 122
 	if err != nil {
123 123
 		return err
... ...
@@ -245,42 +278,56 @@ func setIcc(bridgeIface string, iccEnable, insert bool) error {
245 245
 	return nil
246 246
 }
247 247
 
248
-// Control Inter Network Communication. Install/remove only if it is not/is present.
249
-func setINC(iface1, iface2 string, enable bool) error {
248
+// Control Inter Network Communication. Install[Remove] only if it is [not] present.
249
+func setINC(iface string, enable bool) error {
250 250
 	var (
251
-		table = iptables.Filter
252
-		chain = IsolationChain
253
-		args  = [2][]string{{"-i", iface1, "-o", iface2, "-j", "DROP"}, {"-i", iface2, "-o", iface1, "-j", "DROP"}}
251
+		action    = iptables.Insert
252
+		actionMsg = "add"
253
+		chains    = []string{IsolationChain1, IsolationChain2}
254
+		rules     = [][]string{
255
+			{"-i", iface, "!", "-o", iface, "-j", IsolationChain2},
256
+			{"-o", iface, "-j", "DROP"},
257
+		}
254 258
 	)
255 259
 
256
-	if enable {
257
-		for i := 0; i < 2; i++ {
258
-			if iptables.Exists(table, chain, args[i]...) {
259
-				continue
260
-			}
261
-			if err := iptables.RawCombinedOutput(append([]string{"-I", chain}, args[i]...)...); err != nil {
262
-				return fmt.Errorf("unable to add inter-network communication rule: %v", err)
263
-			}
264
-		}
265
-	} else {
266
-		for i := 0; i < 2; i++ {
267
-			if !iptables.Exists(table, chain, args[i]...) {
268
-				continue
269
-			}
270
-			if err := iptables.RawCombinedOutput(append([]string{"-D", chain}, args[i]...)...); err != nil {
271
-				return fmt.Errorf("unable to remove inter-network communication rule: %v", err)
260
+	if !enable {
261
+		action = iptables.Delete
262
+		actionMsg = "remove"
263
+	}
264
+
265
+	for i, chain := range chains {
266
+		if err := iptables.ProgramRule(iptables.Filter, chain, action, rules[i]); err != nil {
267
+			msg := fmt.Sprintf("unable to %s inter-network communication rule: %v", actionMsg, err)
268
+			if enable {
269
+				if i == 1 {
270
+					// Rollback the rule installed on first chain
271
+					if err2 := iptables.ProgramRule(iptables.Filter, chains[0], iptables.Delete, rules[0]); err2 != nil {
272
+						logrus.Warn("Failed to rollback iptables rule after failure (%v): %v", err, err2)
273
+					}
274
+				}
275
+				return fmt.Errorf(msg)
272 276
 			}
277
+			logrus.Warn(msg)
273 278
 		}
274 279
 	}
275 280
 
276 281
 	return nil
277 282
 }
278 283
 
284
+// Obsolete chain from previous docker versions
285
+const oldIsolationChain = "DOCKER-ISOLATION"
286
+
279 287
 func removeIPChains() {
288
+	// Remove obsolete rules from default chains
289
+	iptables.ProgramRule(iptables.Filter, "FORWARD", iptables.Delete, []string{"-j", oldIsolationChain})
290
+
291
+	// Remove chains
280 292
 	for _, chainInfo := range []iptables.ChainInfo{
281 293
 		{Name: DockerChain, Table: iptables.Nat},
282 294
 		{Name: DockerChain, Table: iptables.Filter},
283
-		{Name: IsolationChain, Table: iptables.Filter},
295
+		{Name: IsolationChain1, Table: iptables.Filter},
296
+		{Name: IsolationChain2, Table: iptables.Filter},
297
+		{Name: oldIsolationChain, Table: iptables.Filter},
284 298
 	} {
285 299
 		if err := chainInfo.Remove(); err != nil {
286 300
 			logrus.Warnf("Failed to remove existing iptables entries in table %s chain %s : %v", chainInfo.Table, chainInfo.Name, err)
... ...
@@ -290,8 +337,8 @@ func removeIPChains() {
290 290
 
291 291
 func setupInternalNetworkRules(bridgeIface string, addr net.Addr, icc, insert bool) error {
292 292
 	var (
293
-		inDropRule  = iptRule{table: iptables.Filter, chain: IsolationChain, args: []string{"-i", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}}
294
-		outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain, args: []string{"-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}}
293
+		inDropRule  = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{"-i", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}}
294
+		outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{"-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}}
295 295
 	)
296 296
 	if err := programChainRule(inDropRule, "DROP INCOMING", insert); err != nil {
297 297
 		return err
... ...
@@ -313,7 +313,7 @@ func (nDB *NetworkDB) Peers(nid string) []PeerInfo {
313 313
 		} else {
314 314
 			// Added for testing purposes, this condition should never happen else mean that the network list
315 315
 			// is out of sync with the node list
316
-			peers = append(peers, PeerInfo{})
316
+			peers = append(peers, PeerInfo{Name: nodeName, IP: "unknown"})
317 317
 		}
318 318
 	}
319 319
 	return peers
... ...
@@ -84,7 +84,11 @@ func dbPeers(ctx interface{}, w http.ResponseWriter, r *http.Request) {
84 84
 		peers := nDB.Peers(r.Form["nid"][0])
85 85
 		rsp := &diagnostic.TableObj{Length: len(peers)}
86 86
 		for i, peerInfo := range peers {
87
-			rsp.Elements = append(rsp.Elements, &diagnostic.PeerEntryObj{Index: i, Name: peerInfo.Name, IP: peerInfo.IP})
87
+			if peerInfo.IP == "unknown" {
88
+				rsp.Elements = append(rsp.Elements, &diagnostic.PeerEntryObj{Index: i, Name: "orphan-" + peerInfo.Name, IP: peerInfo.IP})
89
+			} else {
90
+				rsp.Elements = append(rsp.Elements, &diagnostic.PeerEntryObj{Index: i, Name: peerInfo.Name, IP: peerInfo.IP})
91
+			}
88 92
 		}
89 93
 		log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("network peers done")
90 94
 		diagnostic.HTTPReply(w, diagnostic.CommandSucceed(rsp), json)
... ...
@@ -452,7 +452,6 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {
452 452
 				logrus.Warnf("[resolver] connect failed: %s", err)
453 453
 				continue
454 454
 			}
455
-
456 455
 			queryType := dns.TypeToString[query.Question[0].Qtype]
457 456
 			logrus.Debugf("[resolver] query %s (%s) from %s, forwarding to %s:%s", name, queryType,
458 457
 				extConn.LocalAddr().String(), proto, extDNS.IPStr)
... ...
@@ -492,6 +491,11 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {
492 492
 			}
493 493
 			r.forwardQueryEnd()
494 494
 			if resp != nil {
495
+				if resp.Rcode == dns.RcodeServerFailure {
496
+					// for Server Failure response, continue to the next external DNS server
497
+					logrus.Debugf("[resolver] external DNS %s:%s responded with ServFail for %q", proto, extDNS.IPStr, name)
498
+					continue
499
+				}
495 500
 				answers := 0
496 501
 				for _, rr := range resp.Answer {
497 502
 					h := rr.Header()
... ...
@@ -511,10 +515,10 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {
511 511
 				if resp.Answer == nil || answers == 0 {
512 512
 					logrus.Debugf("[resolver] external DNS %s:%s did not return any %s records for %q", proto, extDNS.IPStr, queryType, name)
513 513
 				}
514
+				resp.Compress = true
514 515
 			} else {
515 516
 				logrus.Debugf("[resolver] external DNS %s:%s returned empty response for %q", proto, extDNS.IPStr, name)
516 517
 			}
517
-			resp.Compress = true
518 518
 			break
519 519
 		}
520 520
 		if resp == nil {