Browse code

Fix issue for `--fixed-cidr` when bridge has multiple addresses

This fix tries to address the issue raised in:
https://github.com/docker/docker/issues/26341
where multiple addresses in a bridge may cause `--fixed-cidr` to
not have the correct addresses.

The issue is that `netutils.ElectInterfaceAddresses(bridgeName)`
only returns the first IPv4 address.

This fix changes `ElectInterfaceAddresses()` and `addresses()`
so that all IPv4 addresses are returned. This will allow the
possibility of selectively choose the address needed.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

Yong Tang authored on 2016/09/17 14:40:44
Showing 11 changed files
... ...
@@ -507,11 +507,11 @@ func encodeData(data interface{}) (*bytes.Buffer, error) {
507 507
 }
508 508
 
509 509
 func ipamOption(bridgeName string) libnetwork.NetworkOption {
510
-	if nw, _, err := netutils.ElectInterfaceAddresses(bridgeName); err == nil {
511
-		ipamV4Conf := &libnetwork.IpamConf{PreferredPool: nw.String()}
512
-		hip, _ := types.GetHostPartIP(nw.IP, nw.Mask)
510
+	if nws, _, err := netutils.ElectInterfaceAddresses(bridgeName); err == nil {
511
+		ipamV4Conf := &libnetwork.IpamConf{PreferredPool: nws[0].String()}
512
+		hip, _ := types.GetHostPartIP(nws[0].IP, nws[0].Mask)
513 513
 		if hip.IsGlobalUnicast() {
514
-			ipamV4Conf.Gateway = nw.IP.String()
514
+			ipamV4Conf.Gateway = nws[0].IP.String()
515 515
 		}
516 516
 		return libnetwork.NetworkOptionIpam("default", "", []*libnetwork.IpamConf{ipamV4Conf}, nil, nil)
517 517
 	}
... ...
@@ -169,13 +169,13 @@ func compareBindings(a, b []types.PortBinding) bool {
169 169
 
170 170
 func getIPv4Data(t *testing.T, iface string) []driverapi.IPAMData {
171 171
 	ipd := driverapi.IPAMData{AddressSpace: "full"}
172
-	nw, _, err := netutils.ElectInterfaceAddresses(iface)
172
+	nws, _, err := netutils.ElectInterfaceAddresses(iface)
173 173
 	if err != nil {
174 174
 		t.Fatal(err)
175 175
 	}
176
-	ipd.Pool = nw
176
+	ipd.Pool = nws[0]
177 177
 	// Set network gateway to X.X.X.1
178
-	ipd.Gateway = types.GetIPNetCopy(nw)
178
+	ipd.Gateway = types.GetIPNetCopy(nws[0])
179 179
 	ipd.Gateway.IP[len(ipd.Gateway.IP)-1] = 1
180 180
 	return []driverapi.IPAMData{ipd}
181 181
 }
... ...
@@ -1054,12 +1054,12 @@ func TestCreateWithExistingBridge(t *testing.T) {
1054 1054
 		t.Fatalf("Failed to getNetwork(%s): %v", brName, err)
1055 1055
 	}
1056 1056
 
1057
-	addr4, _, err := nw.bridge.addresses()
1057
+	addrs4, _, err := nw.bridge.addresses()
1058 1058
 	if err != nil {
1059 1059
 		t.Fatalf("Failed to get the bridge network's address: %v", err)
1060 1060
 	}
1061 1061
 
1062
-	if !addr4.IP.Equal(ip) {
1062
+	if !addrs4[0].IP.Equal(ip) {
1063 1063
 		t.Fatal("Creating bridge network with existing bridge interface unexpectedly modified the IP address of the bridge")
1064 1064
 	}
1065 1065
 
... ...
@@ -52,23 +52,22 @@ func (i *bridgeInterface) exists() bool {
52 52
 	return i.Link != nil
53 53
 }
54 54
 
55
-// addresses returns a single IPv4 address and all IPv6 addresses for the
56
-// bridge interface.
57
-func (i *bridgeInterface) addresses() (netlink.Addr, []netlink.Addr, error) {
55
+// addresses returns all IPv4 addresses and all IPv6 addresses for the bridge interface.
56
+func (i *bridgeInterface) addresses() ([]netlink.Addr, []netlink.Addr, error) {
58 57
 	v4addr, err := i.nlh.AddrList(i.Link, netlink.FAMILY_V4)
59 58
 	if err != nil {
60
-		return netlink.Addr{}, nil, fmt.Errorf("Failed to retrieve V4 addresses: %v", err)
59
+		return nil, nil, fmt.Errorf("Failed to retrieve V4 addresses: %v", err)
61 60
 	}
62 61
 
63 62
 	v6addr, err := i.nlh.AddrList(i.Link, netlink.FAMILY_V6)
64 63
 	if err != nil {
65
-		return netlink.Addr{}, nil, fmt.Errorf("Failed to retrieve V6 addresses: %v", err)
64
+		return nil, nil, fmt.Errorf("Failed to retrieve V6 addresses: %v", err)
66 65
 	}
67 66
 
68 67
 	if len(v4addr) == 0 {
69
-		return netlink.Addr{}, v6addr, nil
68
+		return nil, v6addr, nil
70 69
 	}
71
-	return v4addr[0], v6addr, nil
70
+	return v4addr, v6addr, nil
72 71
 }
73 72
 
74 73
 func (i *bridgeInterface) programIPv6Address() error {
... ...
@@ -37,12 +37,12 @@ func TestAddressesEmptyInterface(t *testing.T) {
37 37
 		t.Fatalf("newInterface() failed: %v", err)
38 38
 	}
39 39
 
40
-	addrv4, addrsv6, err := inf.addresses()
40
+	addrsv4, addrsv6, err := inf.addresses()
41 41
 	if err != nil {
42 42
 		t.Fatalf("Failed to get addresses of default interface: %v", err)
43 43
 	}
44
-	if expected := (netlink.Addr{}); addrv4 != expected {
45
-		t.Fatalf("Default interface has unexpected IPv4: %s", addrv4)
44
+	if len(addrsv4) != 0 {
45
+		t.Fatalf("Default interface has unexpected IPv4: %s", addrsv4)
46 46
 	}
47 47
 	if len(addrsv6) != 0 {
48 48
 		t.Fatalf("Default interface has unexpected IPv6: %v", addrsv6)
... ...
@@ -3,6 +3,7 @@ package bridge
3 3
 import (
4 4
 	"fmt"
5 5
 	"io/ioutil"
6
+	"net"
6 7
 	"path/filepath"
7 8
 
8 9
 	log "github.com/Sirupsen/logrus"
... ...
@@ -10,12 +11,28 @@ import (
10 10
 	"github.com/vishvananda/netlink"
11 11
 )
12 12
 
13
+func selectIPv4Address(addresses []netlink.Addr, selector *net.IPNet) (netlink.Addr, error) {
14
+	if len(addresses) == 0 {
15
+		return netlink.Addr{}, fmt.Errorf("unable to select an address as the address pool is empty")
16
+	}
17
+	if selector != nil {
18
+		for _, addr := range addresses {
19
+			if selector.Contains(addr.IP) {
20
+				return addr, nil
21
+			}
22
+		}
23
+	}
24
+	return addresses[0], nil
25
+}
26
+
13 27
 func setupBridgeIPv4(config *networkConfiguration, i *bridgeInterface) error {
14
-	addrv4, _, err := i.addresses()
28
+	addrv4List, _, err := i.addresses()
15 29
 	if err != nil {
16 30
 		return fmt.Errorf("failed to retrieve bridge interface addresses: %v", err)
17 31
 	}
18 32
 
33
+	addrv4, _ := selectIPv4Address(addrv4List, config.AddressIPv4)
34
+
19 35
 	if !types.CompareIPNet(addrv4.IPNet, config.AddressIPv4) {
20 36
 		if addrv4.IPNet != nil {
21 37
 			if err := i.nlh.AddrDel(i.Link, &addrv4); err != nil {
... ...
@@ -11,12 +11,14 @@ import (
11 11
 )
12 12
 
13 13
 func setupVerifyAndReconcile(config *networkConfiguration, i *bridgeInterface) error {
14
-	// Fetch a single IPv4 and a slice of IPv6 addresses from the bridge.
15
-	addrv4, addrsv6, err := i.addresses()
14
+	// Fetch a slice of IPv4 addresses and a slice of IPv6 addresses from the bridge.
15
+	addrsv4, addrsv6, err := i.addresses()
16 16
 	if err != nil {
17 17
 		return fmt.Errorf("Failed to verify ip addresses: %v", err)
18 18
 	}
19 19
 
20
+	addrv4, _ := selectIPv4Address(addrsv4, config.AddressIPv4)
21
+
20 22
 	// Verify that the bridge does have an IPv4 address.
21 23
 	if addrv4.IPNet == nil {
22 24
 		return &ErrNoIPAddr{}
... ...
@@ -7,10 +7,12 @@ import (
7 7
 )
8 8
 
9 9
 // ElectInterfaceAddresses looks for an interface on the OS with the specified name
10
-// and returns its IPv4 and IPv6 addresses in CIDR form. If the interface does not exist,
11
-// it chooses from a predifined list the first IPv4 address which does not conflict
12
-// with other interfaces on the system.
13
-func ElectInterfaceAddresses(name string) (*net.IPNet, []*net.IPNet, error) {
10
+// and returns returns all its IPv4 and IPv6 addresses in CIDR notation.
11
+// If a failure in retrieving the addresses or no IPv4 address is found, an error is returned.
12
+// If the interface does not exist, it chooses from a predefined
13
+// list the first IPv4 address which does not conflict with other
14
+// interfaces on the system.
15
+func ElectInterfaceAddresses(name string) ([]*net.IPNet, []*net.IPNet, error) {
14 16
 	return nil, nil, types.NotImplementedErrorf("not supported on freebsd")
15 17
 }
16 18
 
... ...
@@ -62,15 +62,15 @@ func GenerateIfaceName(nlh *netlink.Handle, prefix string, len int) (string, err
62 62
 }
63 63
 
64 64
 // ElectInterfaceAddresses looks for an interface on the OS with the
65
-// specified name and returns its IPv4 and IPv6 addresses in CIDR
66
-// form. If the interface does not exist, it chooses from a predefined
65
+// specified name and returns returns all its IPv4 and IPv6 addresses in CIDR notation.
66
+// If a failure in retrieving the addresses or no IPv4 address is found, an error is returned.
67
+// If the interface does not exist, it chooses from a predefined
67 68
 // list the first IPv4 address which does not conflict with other
68 69
 // interfaces on the system.
69
-func ElectInterfaceAddresses(name string) (*net.IPNet, []*net.IPNet, error) {
70
+func ElectInterfaceAddresses(name string) ([]*net.IPNet, []*net.IPNet, error) {
70 71
 	var (
71
-		v4Net  *net.IPNet
72
+		v4Nets []*net.IPNet
72 73
 		v6Nets []*net.IPNet
73
-		err    error
74 74
 	)
75 75
 
76 76
 	defer osl.InitOSContext()()
... ...
@@ -85,23 +85,24 @@ func ElectInterfaceAddresses(name string) (*net.IPNet, []*net.IPNet, error) {
85 85
 		if err != nil {
86 86
 			return nil, nil, err
87 87
 		}
88
-		if len(v4addr) > 0 {
89
-			v4Net = v4addr[0].IPNet
88
+		for _, nlAddr := range v4addr {
89
+			v4Nets = append(v4Nets, nlAddr.IPNet)
90 90
 		}
91 91
 		for _, nlAddr := range v6addr {
92 92
 			v6Nets = append(v6Nets, nlAddr.IPNet)
93 93
 		}
94 94
 	}
95 95
 
96
-	if link == nil || v4Net == nil {
96
+	if link == nil || len(v4Nets) == 0 {
97 97
 		// Choose from predefined broad networks
98
-		v4Net, err = FindAvailableNetwork(ipamutils.PredefinedBroadNetworks)
98
+		v4Net, err := FindAvailableNetwork(ipamutils.PredefinedBroadNetworks)
99 99
 		if err != nil {
100 100
 			return nil, nil, err
101 101
 		}
102
+		v4Nets = append(v4Nets, v4Net)
102 103
 	}
103 104
 
104
-	return v4Net, v6Nets, nil
105
+	return v4Nets, v6Nets, nil
105 106
 }
106 107
 
107 108
 // FindAvailableNetwork returns a network from the passed list which does not
... ...
@@ -22,10 +22,12 @@ func CheckRouteOverlaps(toCheck *net.IPNet) error {
22 22
 }
23 23
 
24 24
 // ElectInterfaceAddresses looks for an interface on the OS with the specified name
25
-// and returns its IPv4 and IPv6 addresses in CIDR form. If the interface does not exist,
26
-// it chooses from a predifined list the first IPv4 address which does not conflict
27
-// with other interfaces on the system.
28
-func ElectInterfaceAddresses(name string) (*net.IPNet, []*net.IPNet, error) {
25
+// and returns returns all its IPv4 and IPv6 addresses in CIDR notation.
26
+// If a failure in retrieving the addresses or no IPv4 address is found, an error is returned.
27
+// If the interface does not exist, it chooses from a predefined
28
+// list the first IPv4 address which does not conflict with other
29
+// interfaces on the system.
30
+func ElectInterfaceAddresses(name string) ([]*net.IPNet, []*net.IPNet, error) {
29 31
 	var (
30 32
 		v4Net *net.IPNet
31 33
 	)
... ...
@@ -63,7 +65,7 @@ func ElectInterfaceAddresses(name string) (*net.IPNet, []*net.IPNet, error) {
63 63
 			return nil, nil, err
64 64
 		}
65 65
 	}
66
-	return v4Net, nil, nil
66
+	return []*net.IPNet{v4Net}, nil, nil
67 67
 }
68 68
 
69 69
 // FindAvailableNetwork returns a network from the passed list which does not
... ...
@@ -5,6 +5,7 @@ package netutils
5 5
 import (
6 6
 	"bytes"
7 7
 	"net"
8
+	"sort"
8 9
 	"testing"
9 10
 
10 11
 	"github.com/docker/libnetwork/ipamutils"
... ...
@@ -265,6 +266,43 @@ func TestNetworkRequest(t *testing.T) {
265 265
 	}
266 266
 }
267 267
 
268
+func TestElectInterfaceAddressMultipleAddresses(t *testing.T) {
269
+	defer testutils.SetupTestOSContext(t)()
270
+	ipamutils.InitNetworks()
271
+
272
+	nws := []string{"172.101.202.254/16", "172.102.202.254/16"}
273
+	createInterface(t, "test", nws...)
274
+
275
+	ipv4NwList, ipv6NwList, err := ElectInterfaceAddresses("test")
276
+	if err != nil {
277
+		t.Fatal(err)
278
+	}
279
+
280
+	if len(ipv4NwList) == 0 {
281
+		t.Fatalf("unexpected empty ipv4 network addresses")
282
+	}
283
+
284
+	if len(ipv6NwList) == 0 {
285
+		t.Fatalf("unexpected empty ipv6 network addresses")
286
+	}
287
+
288
+	nwList := []string{}
289
+	for _, ipv4Nw := range ipv4NwList {
290
+		nwList = append(nwList, ipv4Nw.String())
291
+	}
292
+	sort.Strings(nws)
293
+	sort.Strings(nwList)
294
+
295
+	if len(nws) != len(nwList) {
296
+		t.Fatalf("expected %v. got %v", nws, nwList)
297
+	}
298
+	for i, nw := range nws {
299
+		if nw != nwList[i] {
300
+			t.Fatalf("expected %v. got %v", nw, nwList[i])
301
+		}
302
+	}
303
+}
304
+
268 305
 func TestElectInterfaceAddress(t *testing.T) {
269 306
 	defer testutils.SetupTestOSContext(t)()
270 307
 	ipamutils.InitNetworks()
... ...
@@ -277,37 +315,43 @@ func TestElectInterfaceAddress(t *testing.T) {
277 277
 		t.Fatal(err)
278 278
 	}
279 279
 
280
-	if ipv4Nw == nil {
280
+	if len(ipv4Nw) == 0 {
281 281
 		t.Fatalf("unexpected empty ipv4 network addresses")
282 282
 	}
283 283
 
284 284
 	if len(ipv6Nw) == 0 {
285
-		t.Fatalf("unexpected empty ipv4 network addresses")
285
+		t.Fatalf("unexpected empty ipv6 network addresses")
286 286
 	}
287 287
 
288
-	if nws != ipv4Nw.String() {
289
-		t.Fatalf("expected %s. got %s", nws, ipv4Nw)
288
+	if nws != ipv4Nw[0].String() {
289
+		t.Fatalf("expected %s. got %s", nws, ipv4Nw[0])
290 290
 	}
291 291
 }
292 292
 
293
-func createInterface(t *testing.T, name, nw string) {
293
+func createInterface(t *testing.T, name string, nws ...string) {
294 294
 	// Add interface
295 295
 	link := &netlink.Bridge{
296 296
 		LinkAttrs: netlink.LinkAttrs{
297 297
 			Name: "test",
298 298
 		},
299 299
 	}
300
-	bip, err := types.ParseCIDR(nw)
301
-	if err != nil {
302
-		t.Fatal(err)
300
+	bips := []*net.IPNet{}
301
+	for _, nw := range nws {
302
+		bip, err := types.ParseCIDR(nw)
303
+		if err != nil {
304
+			t.Fatal(err)
305
+		}
306
+		bips = append(bips, bip)
303 307
 	}
304
-	if err = netlink.LinkAdd(link); err != nil {
308
+	if err := netlink.LinkAdd(link); err != nil {
305 309
 		t.Fatalf("Failed to create interface via netlink: %v", err)
306 310
 	}
307
-	if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: bip}); err != nil {
308
-		t.Fatal(err)
311
+	for _, bip := range bips {
312
+		if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: bip}); err != nil {
313
+			t.Fatal(err)
314
+		}
309 315
 	}
310
-	if err = netlink.LinkSetUp(link); err != nil {
316
+	if err := netlink.LinkSetUp(link); err != nil {
311 317
 		t.Fatal(err)
312 318
 	}
313 319
 }
... ...
@@ -7,10 +7,12 @@ import (
7 7
 )
8 8
 
9 9
 // ElectInterfaceAddresses looks for an interface on the OS with the specified name
10
-// and returns its IPv4 and IPv6 addresses in CIDR form. If the interface does not exist,
11
-// it chooses from a predifined list the first IPv4 address which does not conflict
12
-// with other interfaces on the system.
13
-func ElectInterfaceAddresses(name string) (*net.IPNet, []*net.IPNet, error) {
10
+// and returns returns all its IPv4 and IPv6 addresses in CIDR notation.
11
+// If a failure in retrieving the addresses or no IPv4 address is found, an error is returned.
12
+// If the interface does not exist, it chooses from a predefined
13
+// list the first IPv4 address which does not conflict with other
14
+// interfaces on the system.
15
+func ElectInterfaceAddresses(name string) ([]*net.IPNet, []*net.IPNet, error) {
14 16
 	return nil, nil, types.NotImplementedErrorf("not supported on windows")
15 17
 }
16 18