Browse code

Fix IPv6 Port Forwarding for the Bridge Driver

1. Allocate either a IPv4 and/or IPv6 Port Binding (HostIP, HostPort, ContainerIP,
ContainerPort) based on the input and system parameters
2. Update the userland proxy as well as dummy proxy (inside port mapper) to
specifically listen on either the IPv4 or IPv6 network

Signed-off-by: Arko Dasgupta <arko.dasgupta@docker.com>

Arko Dasgupta authored on 2020/12/14 08:56:30
Showing 10 changed files
... ...
@@ -8,6 +8,16 @@ import (
8 8
 	"github.com/ishidawataru/sctp"
9 9
 )
10 10
 
11
+// ipVersion refers to IP version - v4 or v6
12
+type ipVersion string
13
+
14
+const (
15
+	// IPv4 is version 4
16
+	ipv4 ipVersion = "4"
17
+	// IPv4 is version 6
18
+	ipv6 ipVersion = "6"
19
+)
20
+
11 21
 // Proxy defines the behavior of a proxy. It forwards traffic back and forth
12 22
 // between two endpoints : the frontend and the backend.
13 23
 // It can be used to do software port-mapping between two addresses.
... ...
@@ -19,7 +19,12 @@ type SCTPProxy struct {
19 19
 
20 20
 // NewSCTPProxy creates a new SCTPProxy.
21 21
 func NewSCTPProxy(frontendAddr, backendAddr *sctp.SCTPAddr) (*SCTPProxy, error) {
22
-	listener, err := sctp.ListenSCTP("sctp", frontendAddr)
22
+	// detect version of hostIP to bind only to correct version
23
+	ipVersion := ipv4
24
+	if frontendAddr.IPAddrs[0].IP.To4() == nil {
25
+		ipVersion = ipv6
26
+	}
27
+	listener, err := sctp.ListenSCTP("sctp"+string(ipVersion), frontendAddr)
23 28
 	if err != nil {
24 29
 		return nil, err
25 30
 	}
... ...
@@ -17,7 +17,12 @@ type TCPProxy struct {
17 17
 
18 18
 // NewTCPProxy creates a new TCPProxy.
19 19
 func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) {
20
-	listener, err := net.ListenTCP("tcp", frontendAddr)
20
+	// detect version of hostIP to bind only to correct version
21
+	ipVersion := ipv4
22
+	if frontendAddr.IP.To4() == nil {
23
+		ipVersion = ipv6
24
+	}
25
+	listener, err := net.ListenTCP("tcp"+string(ipVersion), frontendAddr)
21 26
 	if err != nil {
22 27
 		return nil, err
23 28
 	}
... ...
@@ -55,7 +55,12 @@ type UDPProxy struct {
55 55
 
56 56
 // NewUDPProxy creates a new UDPProxy.
57 57
 func NewUDPProxy(frontendAddr, backendAddr *net.UDPAddr) (*UDPProxy, error) {
58
-	listener, err := net.ListenUDP("udp", frontendAddr)
58
+	// detect version of hostIP to bind only to correct version
59
+	ipVersion := ipv4
60
+	if frontendAddr.IP.To4() == nil {
61
+		ipVersion = ipv6
62
+	}
63
+	listener, err := net.ListenUDP("udp"+string(ipVersion), frontendAddr)
59 64
 	if err != nil {
60 65
 		return nil, err
61 66
 	}
... ...
@@ -11,71 +11,113 @@ import (
11 11
 	"github.com/sirupsen/logrus"
12 12
 )
13 13
 
14
-var (
15
-	defaultBindingIP   = net.IPv4(0, 0, 0, 0)
16
-	defaultBindingIPV6 = net.ParseIP("::")
17
-)
18
-
19 14
 func (n *bridgeNetwork) allocatePorts(ep *bridgeEndpoint, reqDefBindIP net.IP, ulPxyEnabled bool) ([]types.PortBinding, error) {
20 15
 	if ep.extConnConfig == nil || ep.extConnConfig.PortBindings == nil {
21 16
 		return nil, nil
22 17
 	}
23 18
 
24
-	defHostIP := defaultBindingIP
19
+	defHostIP := net.IPv4zero // 0.0.0.0
25 20
 	if reqDefBindIP != nil {
26 21
 		defHostIP = reqDefBindIP
27 22
 	}
28 23
 
29
-	// IPv4 port binding including user land proxy
30
-	pb, err := n.allocatePortsInternal(ep.extConnConfig.PortBindings, ep.addr.IP, defHostIP, ulPxyEnabled)
31
-	if err != nil {
32
-		return nil, err
24
+	var containerIPv6 net.IP
25
+	if ep.addrv6 != nil {
26
+		containerIPv6 = ep.addrv6.IP
33 27
 	}
34 28
 
35
-	// IPv6 port binding excluding user land proxy
36
-	if n.driver.config.EnableIP6Tables && ep.addrv6 != nil {
37
-		// TODO IPv6 custom default binding IP
38
-		pbv6, err := n.allocatePortsInternal(ep.extConnConfig.PortBindings, ep.addrv6.IP, defaultBindingIPV6, false)
39
-		if err != nil {
40
-			// ensure we clear the previous allocated IPv4 ports
41
-			n.releasePortsInternal(pb)
42
-			return nil, err
43
-		}
44
-
45
-		pb = append(pb, pbv6...)
29
+	pb, err := n.allocatePortsInternal(ep.extConnConfig.PortBindings, ep.addr.IP, containerIPv6, defHostIP, ulPxyEnabled)
30
+	if err != nil {
31
+		return nil, err
46 32
 	}
47 33
 	return pb, nil
48 34
 }
49 35
 
50
-func (n *bridgeNetwork) allocatePortsInternal(bindings []types.PortBinding, containerIP, defHostIP net.IP, ulPxyEnabled bool) ([]types.PortBinding, error) {
36
+func (n *bridgeNetwork) allocatePortsInternal(bindings []types.PortBinding, containerIPv4, containerIPv6, defHostIP net.IP, ulPxyEnabled bool) ([]types.PortBinding, error) {
51 37
 	bs := make([]types.PortBinding, 0, len(bindings))
52 38
 	for _, c := range bindings {
53
-		b := c.GetCopy()
54
-		if err := n.allocatePort(&b, containerIP, defHostIP, ulPxyEnabled); err != nil {
55
-			// On allocation failure, release previously allocated ports. On cleanup error, just log a warning message
56
-			if cuErr := n.releasePortsInternal(bs); cuErr != nil {
57
-				logrus.Warnf("Upon allocation failure for %v, failed to clear previously allocated port bindings: %v", b, cuErr)
39
+		bIPv4 := c.GetCopy()
40
+		bIPv6 := c.GetCopy()
41
+		// Allocate IPv4 Port mappings
42
+		if ok := n.validatePortBindingIPv4(&bIPv4, containerIPv4, defHostIP); ok {
43
+			if err := n.allocatePort(&bIPv4, ulPxyEnabled); err != nil {
44
+				// On allocation failure, release previously allocated ports. On cleanup error, just log a warning message
45
+				if cuErr := n.releasePortsInternal(bs); cuErr != nil {
46
+					logrus.Warnf("allocation failure for %v, failed to clear previously allocated ipv4 port bindings: %v", bIPv4, cuErr)
47
+				}
48
+				return nil, err
58 49
 			}
59
-			return nil, err
50
+			bs = append(bs, bIPv4)
51
+		}
52
+		// Allocate IPv6 Port mappings
53
+		if ok := n.validatePortBindingIPv6(&bIPv6, containerIPv6, defHostIP); ok {
54
+			if err := n.allocatePort(&bIPv6, ulPxyEnabled); err != nil {
55
+				// On allocation failure, release previously allocated ports. On cleanup error, just log a warning message
56
+				if cuErr := n.releasePortsInternal(bs); cuErr != nil {
57
+					logrus.Warnf("allocation failure for %v, failed to clear previously allocated ipv6 port bindings: %v", bIPv6, cuErr)
58
+				}
59
+				return nil, err
60
+			}
61
+			bs = append(bs, bIPv6)
60 62
 		}
61
-		bs = append(bs, b)
62 63
 	}
63 64
 	return bs, nil
64 65
 }
65 66
 
66
-func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHostIP net.IP, ulPxyEnabled bool) error {
67
-	var (
68
-		host net.Addr
69
-		err  error
70
-	)
71
-
72
-	// Store the container interface address in the operational binding
73
-	bnd.IP = containerIP
74
-
67
+// validatePortBindingIPv4 validates the port binding, populates the missing Host IP field and returns true
68
+// if this is a valid IPv4 binding, else returns false
69
+func (n *bridgeNetwork) validatePortBindingIPv4(bnd *types.PortBinding, containerIPv4, defHostIP net.IP) bool {
70
+	//Return early if there is a valid Host IP, but its not a IPv6 address
71
+	if len(bnd.HostIP) > 0 && bnd.HostIP.To4() == nil {
72
+		return false
73
+	}
75 74
 	// Adjust the host address in the operational binding
76 75
 	if len(bnd.HostIP) == 0 {
76
+		// Return early if the default binding address is an IPv6 address
77
+		if defHostIP.To4() == nil {
78
+			return false
79
+		}
77 80
 		bnd.HostIP = defHostIP
78 81
 	}
82
+	bnd.IP = containerIPv4
83
+	return true
84
+
85
+}
86
+
87
+// validatePortBindingIPv6 validates the port binding, populates the missing Host IP field and returns true
88
+// if this is a valid IP6v binding, else returns false
89
+func (n *bridgeNetwork) validatePortBindingIPv6(bnd *types.PortBinding, containerIPv6, defHostIP net.IP) bool {
90
+	// Return early if there is no IPv6 container endpoint
91
+	if containerIPv6 == nil {
92
+		return false
93
+	}
94
+	// Return early if there is a valid Host IP, which is a IPv4 address
95
+	if len(bnd.HostIP) > 0 && bnd.HostIP.To4() != nil {
96
+		return false
97
+	}
98
+
99
+	// Setup a binding to  "::" if Host IP is empty and the default binding IP is 0.0.0.0
100
+	if len(bnd.HostIP) == 0 {
101
+		if defHostIP.Equal(net.IPv4zero) {
102
+			bnd.HostIP = net.IPv6zero
103
+			// If the default binding IP is an IPv6 address, use it
104
+		} else if defHostIP.To4() == nil {
105
+			bnd.HostIP = defHostIP
106
+			// Return false if default binding ip is an IPv4 address
107
+		} else {
108
+			return false
109
+		}
110
+	}
111
+	bnd.IP = containerIPv6
112
+	return true
113
+
114
+}
115
+
116
+func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, ulPxyEnabled bool) error {
117
+	var (
118
+		host net.Addr
119
+		err  error
120
+	)
79 121
 
80 122
 	// Adjust HostPortEnd if this is not a range.
81 123
 	if bnd.HostPortEnd == 0 {
... ...
@@ -90,7 +132,7 @@ func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHos
90 90
 
91 91
 	portmapper := n.portMapper
92 92
 
93
-	if containerIP.To4() == nil {
93
+	if bnd.IP.To4() == nil {
94 94
 		portmapper = n.portMapperV6
95 95
 	}
96 96
 
... ...
@@ -95,3 +95,74 @@ func TestPortMappingConfig(t *testing.T) {
95 95
 		t.Fatal(err)
96 96
 	}
97 97
 }
98
+
99
+func TestPortMappingV6Config(t *testing.T) {
100
+	defer testutils.SetupTestOSContext(t)()
101
+	d := newDriver()
102
+
103
+	config := &configuration{
104
+		EnableIPTables:  true,
105
+		EnableIP6Tables: true,
106
+	}
107
+	genericOption := make(map[string]interface{})
108
+	genericOption[netlabel.GenericData] = config
109
+
110
+	if err := d.configure(genericOption); err != nil {
111
+		t.Fatalf("Failed to setup driver config: %v", err)
112
+	}
113
+
114
+	portBindings := []types.PortBinding{
115
+		{Proto: types.UDP, Port: uint16(400), HostPort: uint16(54000)},
116
+		{Proto: types.TCP, Port: uint16(500), HostPort: uint16(65000)},
117
+		{Proto: types.SCTP, Port: uint16(500), HostPort: uint16(65000)},
118
+	}
119
+
120
+	sbOptions := make(map[string]interface{})
121
+	sbOptions[netlabel.PortMap] = portBindings
122
+	netConfig := &networkConfiguration{
123
+		BridgeName: DefaultBridgeName,
124
+		EnableIPv6: true,
125
+	}
126
+	netOptions := make(map[string]interface{})
127
+	netOptions[netlabel.GenericData] = netConfig
128
+
129
+	ipdList := getIPv4Data(t, "")
130
+	err := d.CreateNetwork("dummy", netOptions, nil, ipdList, nil)
131
+	if err != nil {
132
+		t.Fatalf("Failed to create bridge: %v", err)
133
+	}
134
+
135
+	te := newTestEndpoint(ipdList[0].Pool, 11)
136
+	err = d.CreateEndpoint("dummy", "ep1", te.Interface(), nil)
137
+	if err != nil {
138
+		t.Fatalf("Failed to create the endpoint: %s", err.Error())
139
+	}
140
+
141
+	if err = d.Join("dummy", "ep1", "sbox", te, sbOptions); err != nil {
142
+		t.Fatalf("Failed to join the endpoint: %v", err)
143
+	}
144
+
145
+	if err = d.ProgramExternalConnectivity("dummy", "ep1", sbOptions); err != nil {
146
+		t.Fatalf("Failed to program external connectivity: %v", err)
147
+	}
148
+
149
+	network, ok := d.networks["dummy"]
150
+	if !ok {
151
+		t.Fatalf("Cannot find network %s inside driver", "dummy")
152
+	}
153
+	ep, _ := network.endpoints["ep1"]
154
+	if len(ep.portMapping) != 6 {
155
+		t.Fatalf("Failed to store the port bindings into the sandbox info. Found: %v", ep.portMapping)
156
+	}
157
+
158
+	// release host mapped ports
159
+	err = d.Leave("dummy", "ep1")
160
+	if err != nil {
161
+		t.Fatal(err)
162
+	}
163
+
164
+	err = d.RevokeExternalConnectivity("dummy", "ep1")
165
+	if err != nil {
166
+		t.Fatal(err)
167
+	}
168
+}
... ...
@@ -199,7 +199,7 @@ func TestBridge(t *testing.T) {
199 199
 	if !ok {
200 200
 		t.Fatalf("Unexpected format for port mapping in endpoint operational data")
201 201
 	}
202
-	if len(pm) != 5 {
202
+	if len(pm) != 10 {
203 203
 		t.Fatalf("Incomplete data for port mapping in endpoint operational data: %d", len(pm))
204 204
 	}
205 205
 }
... ...
@@ -151,20 +151,16 @@ func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart,
151 151
 	}
152 152
 
153 153
 	containerIP, containerPort := getIPAndPort(m.container)
154
-	if pm.checkIP(hostIP) {
155
-		if err := pm.AppendForwardingTableEntry(m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort); err != nil {
156
-			return nil, err
157
-		}
154
+	if err := pm.AppendForwardingTableEntry(m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort); err != nil {
155
+		return nil, err
158 156
 	}
159 157
 
160 158
 	cleanup := func() error {
161 159
 		// need to undo the iptables rules before we return
162 160
 		m.userlandProxy.Stop()
163
-		if pm.checkIP(hostIP) {
164
-			pm.DeleteForwardingTableEntry(m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort)
165
-			if err := pm.Allocator.ReleasePort(hostIP, m.proto, allocatedHostPort); err != nil {
166
-				return err
167
-			}
161
+		pm.DeleteForwardingTableEntry(m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort)
162
+		if err := pm.Allocator.ReleasePort(hostIP, m.proto, allocatedHostPort); err != nil {
163
+			return err
168 164
 		}
169 165
 
170 166
 		return nil
... ...
@@ -44,11 +44,3 @@ func (pm *PortMapper) forward(action iptables.Action, proto string, sourceIP net
44 44
 	}
45 45
 	return pm.chain.Forward(action, sourceIP, sourcePort, proto, containerIP, containerPort, pm.bridgeName)
46 46
 }
47
-
48
-// checkIP checks if IP is valid and matching to chain version
49
-func (pm *PortMapper) checkIP(ip net.IP) bool {
50
-	if pm.chain == nil || pm.chain.IPTable.Version == iptables.IPv4 {
51
-		return ip.To4() != nil
52
-	}
53
-	return ip.To16() != nil
54
-}
... ...
@@ -19,6 +19,16 @@ type userlandProxy interface {
19 19
 	Stop() error
20 20
 }
21 21
 
22
+// ipVersion refers to IP version - v4 or v6
23
+type ipVersion string
24
+
25
+const (
26
+	// IPv4 is version 4
27
+	ipv4 ipVersion = "4"
28
+	// IPv4 is version 6
29
+	ipv6 ipVersion = "6"
30
+)
31
+
22 32
 // proxyCommand wraps an exec.Cmd to run the userland TCP and UDP
23 33
 // proxies as separate processes.
24 34
 type proxyCommand struct {
... ...
@@ -77,21 +87,27 @@ func (p *proxyCommand) Stop() error {
77 77
 // port allocations on bound port, because without userland proxy we using
78 78
 // iptables rules and not net.Listen
79 79
 type dummyProxy struct {
80
-	listener io.Closer
81
-	addr     net.Addr
80
+	listener  io.Closer
81
+	addr      net.Addr
82
+	ipVersion ipVersion
82 83
 }
83 84
 
84 85
 func newDummyProxy(proto string, hostIP net.IP, hostPort int) (userlandProxy, error) {
86
+	// detect version of hostIP to bind only to correct version
87
+	version := ipv4
88
+	if hostIP.To4() == nil {
89
+		version = ipv6
90
+	}
85 91
 	switch proto {
86 92
 	case "tcp":
87 93
 		addr := &net.TCPAddr{IP: hostIP, Port: hostPort}
88
-		return &dummyProxy{addr: addr}, nil
94
+		return &dummyProxy{addr: addr, ipVersion: version}, nil
89 95
 	case "udp":
90 96
 		addr := &net.UDPAddr{IP: hostIP, Port: hostPort}
91
-		return &dummyProxy{addr: addr}, nil
97
+		return &dummyProxy{addr: addr, ipVersion: version}, nil
92 98
 	case "sctp":
93 99
 		addr := &sctp.SCTPAddr{IPAddrs: []net.IPAddr{{IP: hostIP}}, Port: hostPort}
94
-		return &dummyProxy{addr: addr}, nil
100
+		return &dummyProxy{addr: addr, ipVersion: version}, nil
95 101
 	default:
96 102
 		return nil, fmt.Errorf("Unknown addr type: %s", proto)
97 103
 	}
... ...
@@ -100,19 +116,19 @@ func newDummyProxy(proto string, hostIP net.IP, hostPort int) (userlandProxy, er
100 100
 func (p *dummyProxy) Start() error {
101 101
 	switch addr := p.addr.(type) {
102 102
 	case *net.TCPAddr:
103
-		l, err := net.ListenTCP("tcp", addr)
103
+		l, err := net.ListenTCP("tcp"+string(p.ipVersion), addr)
104 104
 		if err != nil {
105 105
 			return err
106 106
 		}
107 107
 		p.listener = l
108 108
 	case *net.UDPAddr:
109
-		l, err := net.ListenUDP("udp", addr)
109
+		l, err := net.ListenUDP("udp"+string(p.ipVersion), addr)
110 110
 		if err != nil {
111 111
 			return err
112 112
 		}
113 113
 		p.listener = l
114 114
 	case *sctp.SCTPAddr:
115
-		l, err := sctp.ListenSCTP("sctp", addr)
115
+		l, err := sctp.ListenSCTP("sctp"+string(p.ipVersion), addr)
116 116
 		if err != nil {
117 117
 			return err
118 118
 		}