Browse code

libnet/d/bridge: Don't set container's gateway when network is internal

So far, internal networks were only isolated from the host by iptables
DROP rules. As a consequence, outbound connections from containers would
timeout instead of being "rejected" through an immediate ICMP dest/port
unreachable, a TCP RST or a failing `connect` syscall.

This was visible when internal containers were trying to resolve a
domain that don't match any container on the same network (be it a truly
"external" domain, or a container that don't exist/is dead). In that
case, the embedded resolver would try to forward DNS queries for the
different values of resolv.conf `search` option, making DNS resolution
slow to return an error, and the slowness being exacerbated by some libc
implementations.

This change makes `connect` syscall to return ENETUNREACH, and thus
solves the broader issue of failing fast when external connections are
attempted.

Signed-off-by: Albin Kerouanton <albinker@gmail.com>

Albin Kerouanton authored on 2023/10/09 18:59:13
Showing 5 changed files
... ...
@@ -31,6 +31,7 @@ import (
31 31
 	"github.com/vishvananda/netlink"
32 32
 	"golang.org/x/sys/unix"
33 33
 	"gotest.tools/v3/assert"
34
+	is "gotest.tools/v3/assert/cmp"
34 35
 	"gotest.tools/v3/icmd"
35 36
 )
36 37
 
... ...
@@ -1611,7 +1612,7 @@ func (s *DockerCLINetworkSuite) TestDockerNetworkInternalMode(c *testing.T) {
1611 1611
 	assert.Assert(c, waitRun("second") == nil)
1612 1612
 	out, _, err := dockerCmdWithError("exec", "first", "ping", "-W", "4", "-c", "1", "8.8.8.8")
1613 1613
 	assert.ErrorContains(c, err, "")
1614
-	assert.Assert(c, strings.Contains(out, "100% packet loss"))
1614
+	assert.Assert(c, is.Contains(out, "Network is unreachable"))
1615 1615
 	_, _, err = dockerCmdWithError("exec", "second", "ping", "-c", "1", "first")
1616 1616
 	assert.NilError(c, err)
1617 1617
 }
... ...
@@ -1223,14 +1223,16 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
1223 1223
 		return err
1224 1224
 	}
1225 1225
 
1226
-	err = jinfo.SetGateway(network.bridge.gatewayIPv4)
1227
-	if err != nil {
1228
-		return err
1229
-	}
1226
+	if !network.config.Internal {
1227
+		err = jinfo.SetGateway(network.bridge.gatewayIPv4)
1228
+		if err != nil {
1229
+			return err
1230
+		}
1230 1231
 
1231
-	err = jinfo.SetGatewayIPv6(network.bridge.gatewayIPv6)
1232
-	if err != nil {
1233
-		return err
1232
+		err = jinfo.SetGatewayIPv6(network.bridge.gatewayIPv6)
1233
+		if err != nil {
1234
+			return err
1235
+		}
1234 1236
 	}
1235 1237
 
1236 1238
 	return nil
... ...
@@ -28,6 +28,14 @@ func selectIPv4Address(addresses []netlink.Addr, selector *net.IPNet) (netlink.A
28 28
 }
29 29
 
30 30
 func setupBridgeIPv4(config *networkConfiguration, i *bridgeInterface) error {
31
+	// TODO(aker): the bridge driver panics if its bridgeIPv4 field isn't set. Once bridge subnet and bridge IP address
32
+	//             are decoupled, we should assign it only when it's really needed.
33
+	i.bridgeIPv4 = config.AddressIPv4
34
+
35
+	if config.Internal {
36
+		return nil
37
+	}
38
+
31 39
 	if !config.InhibitIPv4 {
32 40
 		addrv4List, _, err := i.addresses()
33 41
 		if err != nil {
... ...
@@ -49,8 +57,7 @@ func setupBridgeIPv4(config *networkConfiguration, i *bridgeInterface) error {
49 49
 		}
50 50
 	}
51 51
 
52
-	// Store bridge network and default gateway
53
-	i.bridgeIPv4 = config.AddressIPv4
52
+	// Store the default gateway
54 53
 	i.gatewayIPv4 = config.AddressIPv4.IP
55 54
 
56 55
 	return nil
... ...
@@ -60,6 +67,9 @@ func setupGatewayIPv4(config *networkConfiguration, i *bridgeInterface) error {
60 60
 	if !i.bridgeIPv4.Contains(config.DefaultGatewayIPv4) {
61 61
 		return &ErrInvalidGateway{}
62 62
 	}
63
+	if config.Internal {
64
+		return types.InvalidParameterErrorf("no gateway can be set on an internal bridge network")
65
+	}
63 66
 
64 67
 	// Store requested default gateway
65 68
 	i.gatewayIPv4 = config.DefaultGatewayIPv4
... ...
@@ -11,6 +11,8 @@ import (
11 11
 	"github.com/vishvananda/netlink"
12 12
 )
13 13
 
14
+// setupVerifyAndReconcile checks what IP addresses the given i interface has and ensures that they match the passed
15
+// network config. It also removes any extra unicast IPv6 addresses found.
14 16
 func setupVerifyAndReconcile(config *networkConfiguration, i *bridgeInterface) error {
15 17
 	// Fetch a slice of IPv4 addresses and a slice of IPv6 addresses from the bridge.
16 18
 	addrsv4, addrsv6, err := i.addresses()
... ...
@@ -21,18 +23,18 @@ func setupVerifyAndReconcile(config *networkConfiguration, i *bridgeInterface) e
21 21
 	addrv4, _ := selectIPv4Address(addrsv4, config.AddressIPv4)
22 22
 
23 23
 	// Verify that the bridge does have an IPv4 address.
24
-	if addrv4.IPNet == nil {
24
+	if !config.Internal && addrv4.IPNet == nil {
25 25
 		return &ErrNoIPAddr{}
26 26
 	}
27 27
 
28 28
 	// Verify that the bridge IPv4 address matches the requested configuration.
29
-	if config.AddressIPv4 != nil && !addrv4.IP.Equal(config.AddressIPv4.IP) {
29
+	if config.AddressIPv4 != nil && addrv4.IPNet != nil && !addrv4.IP.Equal(config.AddressIPv4.IP) {
30 30
 		return &IPv4AddrNoMatchError{IP: addrv4.IP, CfgIP: config.AddressIPv4.IP}
31 31
 	}
32 32
 
33 33
 	// Verify that one of the bridge IPv6 addresses matches the requested
34 34
 	// configuration.
35
-	if config.EnableIPv6 && !findIPv6Address(netlink.Addr{IPNet: bridgeIPv6}, addrsv6) {
35
+	if config.EnableIPv6 && !config.Internal && !findIPv6Address(netlink.Addr{IPNet: bridgeIPv6}, addrsv6) {
36 36
 		return (*IPv6AddrNoMatchError)(bridgeIPv6)
37 37
 	}
38 38
 
... ...
@@ -30,6 +30,9 @@ const (
30 30
 func (sb *Sandbox) startResolver(restore bool) {
31 31
 	sb.resolverOnce.Do(func() {
32 32
 		var err error
33
+		// The embedded resolver is always started with proxyDNS set as true, even when the sandbox is only attached to
34
+		// an internal network. This way, it's the driver responsibility to make sure `connect` syscall fails fast when
35
+		// no external connectivity is available (eg. by not setting a default gateway).
33 36
 		sb.resolver = NewResolver(resolverIPSandbox, true, sb)
34 37
 		defer func() {
35 38
 			if err != nil {