So far, internal networks were only isolated from the host by iptables
DROP rules. As a consequence, outbound connections from containers would
timeout instead of being "rejected" through an immediate ICMP dest/port
unreachable, a TCP RST or a failing `connect` syscall.
This was visible when internal containers were trying to resolve a
domain that don't match any container on the same network (be it a truly
"external" domain, or a container that don't exist/is dead). In that
case, the embedded resolver would try to forward DNS queries for the
different values of resolv.conf `search` option, making DNS resolution
slow to return an error, and the slowness being exacerbated by some libc
implementations.
This change makes `connect` syscall to return ENETUNREACH, and thus
solves the broader issue of failing fast when external connections are
attempted.
Signed-off-by: Albin Kerouanton <albinker@gmail.com>
| ... | ... |
@@ -31,6 +31,7 @@ import ( |
| 31 | 31 |
"github.com/vishvananda/netlink" |
| 32 | 32 |
"golang.org/x/sys/unix" |
| 33 | 33 |
"gotest.tools/v3/assert" |
| 34 |
+ is "gotest.tools/v3/assert/cmp" |
|
| 34 | 35 |
"gotest.tools/v3/icmd" |
| 35 | 36 |
) |
| 36 | 37 |
|
| ... | ... |
@@ -1611,7 +1612,7 @@ func (s *DockerCLINetworkSuite) TestDockerNetworkInternalMode(c *testing.T) {
|
| 1611 | 1611 |
assert.Assert(c, waitRun("second") == nil)
|
| 1612 | 1612 |
out, _, err := dockerCmdWithError("exec", "first", "ping", "-W", "4", "-c", "1", "8.8.8.8")
|
| 1613 | 1613 |
assert.ErrorContains(c, err, "") |
| 1614 |
- assert.Assert(c, strings.Contains(out, "100% packet loss")) |
|
| 1614 |
+ assert.Assert(c, is.Contains(out, "Network is unreachable")) |
|
| 1615 | 1615 |
_, _, err = dockerCmdWithError("exec", "second", "ping", "-c", "1", "first")
|
| 1616 | 1616 |
assert.NilError(c, err) |
| 1617 | 1617 |
} |
| ... | ... |
@@ -1223,14 +1223,16 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, |
| 1223 | 1223 |
return err |
| 1224 | 1224 |
} |
| 1225 | 1225 |
|
| 1226 |
- err = jinfo.SetGateway(network.bridge.gatewayIPv4) |
|
| 1227 |
- if err != nil {
|
|
| 1228 |
- return err |
|
| 1229 |
- } |
|
| 1226 |
+ if !network.config.Internal {
|
|
| 1227 |
+ err = jinfo.SetGateway(network.bridge.gatewayIPv4) |
|
| 1228 |
+ if err != nil {
|
|
| 1229 |
+ return err |
|
| 1230 |
+ } |
|
| 1230 | 1231 |
|
| 1231 |
- err = jinfo.SetGatewayIPv6(network.bridge.gatewayIPv6) |
|
| 1232 |
- if err != nil {
|
|
| 1233 |
- return err |
|
| 1232 |
+ err = jinfo.SetGatewayIPv6(network.bridge.gatewayIPv6) |
|
| 1233 |
+ if err != nil {
|
|
| 1234 |
+ return err |
|
| 1235 |
+ } |
|
| 1234 | 1236 |
} |
| 1235 | 1237 |
|
| 1236 | 1238 |
return nil |
| ... | ... |
@@ -28,6 +28,14 @@ func selectIPv4Address(addresses []netlink.Addr, selector *net.IPNet) (netlink.A |
| 28 | 28 |
} |
| 29 | 29 |
|
| 30 | 30 |
func setupBridgeIPv4(config *networkConfiguration, i *bridgeInterface) error {
|
| 31 |
+ // TODO(aker): the bridge driver panics if its bridgeIPv4 field isn't set. Once bridge subnet and bridge IP address |
|
| 32 |
+ // are decoupled, we should assign it only when it's really needed. |
|
| 33 |
+ i.bridgeIPv4 = config.AddressIPv4 |
|
| 34 |
+ |
|
| 35 |
+ if config.Internal {
|
|
| 36 |
+ return nil |
|
| 37 |
+ } |
|
| 38 |
+ |
|
| 31 | 39 |
if !config.InhibitIPv4 {
|
| 32 | 40 |
addrv4List, _, err := i.addresses() |
| 33 | 41 |
if err != nil {
|
| ... | ... |
@@ -49,8 +57,7 @@ func setupBridgeIPv4(config *networkConfiguration, i *bridgeInterface) error {
|
| 49 | 49 |
} |
| 50 | 50 |
} |
| 51 | 51 |
|
| 52 |
- // Store bridge network and default gateway |
|
| 53 |
- i.bridgeIPv4 = config.AddressIPv4 |
|
| 52 |
+ // Store the default gateway |
|
| 54 | 53 |
i.gatewayIPv4 = config.AddressIPv4.IP |
| 55 | 54 |
|
| 56 | 55 |
return nil |
| ... | ... |
@@ -60,6 +67,9 @@ func setupGatewayIPv4(config *networkConfiguration, i *bridgeInterface) error {
|
| 60 | 60 |
if !i.bridgeIPv4.Contains(config.DefaultGatewayIPv4) {
|
| 61 | 61 |
return &ErrInvalidGateway{}
|
| 62 | 62 |
} |
| 63 |
+ if config.Internal {
|
|
| 64 |
+ return types.InvalidParameterErrorf("no gateway can be set on an internal bridge network")
|
|
| 65 |
+ } |
|
| 63 | 66 |
|
| 64 | 67 |
// Store requested default gateway |
| 65 | 68 |
i.gatewayIPv4 = config.DefaultGatewayIPv4 |
| ... | ... |
@@ -11,6 +11,8 @@ import ( |
| 11 | 11 |
"github.com/vishvananda/netlink" |
| 12 | 12 |
) |
| 13 | 13 |
|
| 14 |
+// setupVerifyAndReconcile checks what IP addresses the given i interface has and ensures that they match the passed |
|
| 15 |
+// network config. It also removes any extra unicast IPv6 addresses found. |
|
| 14 | 16 |
func setupVerifyAndReconcile(config *networkConfiguration, i *bridgeInterface) error {
|
| 15 | 17 |
// Fetch a slice of IPv4 addresses and a slice of IPv6 addresses from the bridge. |
| 16 | 18 |
addrsv4, addrsv6, err := i.addresses() |
| ... | ... |
@@ -21,18 +23,18 @@ func setupVerifyAndReconcile(config *networkConfiguration, i *bridgeInterface) e |
| 21 | 21 |
addrv4, _ := selectIPv4Address(addrsv4, config.AddressIPv4) |
| 22 | 22 |
|
| 23 | 23 |
// Verify that the bridge does have an IPv4 address. |
| 24 |
- if addrv4.IPNet == nil {
|
|
| 24 |
+ if !config.Internal && addrv4.IPNet == nil {
|
|
| 25 | 25 |
return &ErrNoIPAddr{}
|
| 26 | 26 |
} |
| 27 | 27 |
|
| 28 | 28 |
// Verify that the bridge IPv4 address matches the requested configuration. |
| 29 |
- if config.AddressIPv4 != nil && !addrv4.IP.Equal(config.AddressIPv4.IP) {
|
|
| 29 |
+ if config.AddressIPv4 != nil && addrv4.IPNet != nil && !addrv4.IP.Equal(config.AddressIPv4.IP) {
|
|
| 30 | 30 |
return &IPv4AddrNoMatchError{IP: addrv4.IP, CfgIP: config.AddressIPv4.IP}
|
| 31 | 31 |
} |
| 32 | 32 |
|
| 33 | 33 |
// Verify that one of the bridge IPv6 addresses matches the requested |
| 34 | 34 |
// configuration. |
| 35 |
- if config.EnableIPv6 && !findIPv6Address(netlink.Addr{IPNet: bridgeIPv6}, addrsv6) {
|
|
| 35 |
+ if config.EnableIPv6 && !config.Internal && !findIPv6Address(netlink.Addr{IPNet: bridgeIPv6}, addrsv6) {
|
|
| 36 | 36 |
return (*IPv6AddrNoMatchError)(bridgeIPv6) |
| 37 | 37 |
} |
| 38 | 38 |
|
| ... | ... |
@@ -30,6 +30,9 @@ const ( |
| 30 | 30 |
func (sb *Sandbox) startResolver(restore bool) {
|
| 31 | 31 |
sb.resolverOnce.Do(func() {
|
| 32 | 32 |
var err error |
| 33 |
+ // The embedded resolver is always started with proxyDNS set as true, even when the sandbox is only attached to |
|
| 34 |
+ // an internal network. This way, it's the driver responsibility to make sure `connect` syscall fails fast when |
|
| 35 |
+ // no external connectivity is available (eg. by not setting a default gateway). |
|
| 33 | 36 |
sb.resolver = NewResolver(resolverIPSandbox, true, sb) |
| 34 | 37 |
defer func() {
|
| 35 | 38 |
if err != nil {
|