Browse code

Restrict portallocator to Docker allocated ports

Port allocation status is stored in a global map: a port detected in use will remain as such for the lifetime of the daemon. Change the behavior to only mark as allocated ports which are claimed by Docker itself (which we can trust to properly remove from the allocation map once released). Ports allocated by other applications will always be retried to account for the eventually of the port having been released.

Docker-DCO-1.1-Signed-off-by: Arnaud Porterie <icecrime@gmail.com> (github: icecrime)

Arnaud Porterie authored on 2014/06/20 06:33:51
Showing 4 changed files
... ...
@@ -11,7 +11,6 @@ import (
11 11
 	"github.com/docker/libcontainer/netlink"
12 12
 	"github.com/dotcloud/docker/daemon/networkdriver"
13 13
 	"github.com/dotcloud/docker/daemon/networkdriver/ipallocator"
14
-	"github.com/dotcloud/docker/daemon/networkdriver/portallocator"
15 14
 	"github.com/dotcloud/docker/daemon/networkdriver/portmapper"
16 15
 	"github.com/dotcloud/docker/engine"
17 16
 	"github.com/dotcloud/docker/pkg/iptables"
... ...
@@ -354,9 +353,6 @@ func Release(job *engine.Job) engine.Status {
354 354
 	var (
355 355
 		id                 = job.Args[0]
356 356
 		containerInterface = currentInterfaces.Get(id)
357
-		ip                 net.IP
358
-		port               int
359
-		proto              string
360 357
 	)
361 358
 
362 359
 	if containerInterface == nil {
... ...
@@ -367,22 +363,6 @@ func Release(job *engine.Job) engine.Status {
367 367
 		if err := portmapper.Unmap(nat); err != nil {
368 368
 			log.Printf("Unable to unmap port %s: %s", nat, err)
369 369
 		}
370
-
371
-		// this is host mappings
372
-		switch a := nat.(type) {
373
-		case *net.TCPAddr:
374
-			proto = "tcp"
375
-			ip = a.IP
376
-			port = a.Port
377
-		case *net.UDPAddr:
378
-			proto = "udp"
379
-			ip = a.IP
380
-			port = a.Port
381
-		}
382
-
383
-		if err := portallocator.ReleasePort(ip, proto, port); err != nil {
384
-			log.Printf("Unable to release port %s", nat)
385
-		}
386 370
 	}
387 371
 
388 372
 	if err := ipallocator.ReleaseIP(bridgeNetwork, &containerInterface.IP); err != nil {
... ...
@@ -399,7 +379,7 @@ func AllocatePort(job *engine.Job) engine.Status {
399 399
 		ip            = defaultBindingIP
400 400
 		id            = job.Args[0]
401 401
 		hostIP        = job.Getenv("HostIP")
402
-		origHostPort  = job.GetenvInt("HostPort")
402
+		hostPort      = job.GetenvInt("HostPort")
403 403
 		containerPort = job.GetenvInt("ContainerPort")
404 404
 		proto         = job.Getenv("Proto")
405 405
 		network       = currentInterfaces.Get(id)
... ...
@@ -409,11 +389,16 @@ func AllocatePort(job *engine.Job) engine.Status {
409 409
 		ip = net.ParseIP(hostIP)
410 410
 	}
411 411
 
412
-	var (
413
-		hostPort  int
414
-		container net.Addr
415
-		host      net.Addr
416
-	)
412
+	// host ip, proto, and host port
413
+	var container net.Addr
414
+	switch proto {
415
+	case "tcp":
416
+		container = &net.TCPAddr{IP: network.IP, Port: containerPort}
417
+	case "udp":
418
+		container = &net.UDPAddr{IP: network.IP, Port: containerPort}
419
+	default:
420
+		return job.Errorf("unsupported address type %s", proto)
421
+	}
417 422
 
418 423
 	/*
419 424
 	 Try up to 10 times to get a port that's not already allocated.
... ...
@@ -421,27 +406,22 @@ func AllocatePort(job *engine.Job) engine.Status {
421 421
 	 In the event of failure to bind, return the error that portmapper.Map
422 422
 	 yields.
423 423
 	*/
424
-	for i := 0; i < 10; i++ {
425
-		// host ip, proto, and host port
426
-		hostPort, err = portallocator.RequestPort(ip, proto, origHostPort)
427 424
 
428
-		if err != nil {
429
-			return job.Error(err)
430
-		}
431
-
432
-		if proto == "tcp" {
433
-			host = &net.TCPAddr{IP: ip, Port: hostPort}
434
-			container = &net.TCPAddr{IP: network.IP, Port: containerPort}
435
-		} else {
436
-			host = &net.UDPAddr{IP: ip, Port: hostPort}
437
-			container = &net.UDPAddr{IP: network.IP, Port: containerPort}
425
+	var host net.Addr
426
+	for i := 0; i < 10; i++ {
427
+		if host, err = portmapper.Map(container, ip, hostPort); err == nil {
428
+			break
438 429
 		}
439 430
 
440
-		if err = portmapper.Map(container, ip, hostPort); err == nil {
431
+		// There is no point in immediately retrying to map an explicitely
432
+		// chosen port.
433
+		if hostPort != 0 {
434
+			job.Logf("Failed to bind %s for container address %s", host.String(), container.String())
441 435
 			break
442 436
 		}
443 437
 
444
-		job.Logf("Failed to bind %s:%d for container address %s:%d. Trying another port.", ip.String(), hostPort, network.IP.String(), containerPort)
438
+		// Automatically chosen 'free' port failed to bind: move on the next.
439
+		job.Logf("Failed to bind %s for container address %s. Trying another port.", host.String(), container.String())
445 440
 	}
446 441
 
447 442
 	if err != nil {
... ...
@@ -451,12 +431,18 @@ func AllocatePort(job *engine.Job) engine.Status {
451 451
 	network.PortMappings = append(network.PortMappings, host)
452 452
 
453 453
 	out := engine.Env{}
454
-	out.Set("HostIP", ip.String())
455
-	out.SetInt("HostPort", hostPort)
456
-
454
+	switch netAddr := host.(type) {
455
+	case *net.TCPAddr:
456
+		out.Set("HostIP", netAddr.IP.String())
457
+		out.SetInt("HostPort", netAddr.Port)
458
+	case *net.UDPAddr:
459
+		out.Set("HostIP", netAddr.IP.String())
460
+		out.SetInt("HostPort", netAddr.Port)
461
+	}
457 462
 	if _, err := out.WriteTo(job.Stdout); err != nil {
458 463
 		return job.Error(err)
459 464
 	}
465
+
460 466
 	return engine.StatusOK
461 467
 }
462 468
 
463 469
new file mode 100644
... ...
@@ -0,0 +1,106 @@
0
+package bridge
1
+
2
+import (
3
+	"fmt"
4
+	"net"
5
+	"strconv"
6
+	"testing"
7
+
8
+	"github.com/dotcloud/docker/engine"
9
+)
10
+
11
+func findFreePort(t *testing.T) int {
12
+	l, err := net.Listen("tcp", ":0")
13
+	if err != nil {
14
+		t.Fatal("Failed to find a free port")
15
+	}
16
+	defer l.Close()
17
+
18
+	result, err := net.ResolveTCPAddr("tcp", l.Addr().String())
19
+	if err != nil {
20
+		t.Fatal("Failed to resolve address to identify free port")
21
+	}
22
+	return result.Port
23
+}
24
+
25
+func newPortAllocationJob(eng *engine.Engine, port int) (job *engine.Job) {
26
+	strPort := strconv.Itoa(port)
27
+
28
+	job = eng.Job("allocate_port", "container_id")
29
+	job.Setenv("HostIP", "127.0.0.1")
30
+	job.Setenv("HostPort", strPort)
31
+	job.Setenv("Proto", "tcp")
32
+	job.Setenv("ContainerPort", strPort)
33
+	return
34
+}
35
+
36
+func TestAllocatePortDetection(t *testing.T) {
37
+	eng := engine.New()
38
+	eng.Logging = false
39
+
40
+	freePort := findFreePort(t)
41
+
42
+	// Init driver
43
+	job := eng.Job("initdriver")
44
+	if res := InitDriver(job); res != engine.StatusOK {
45
+		t.Fatal("Failed to initialize network driver")
46
+	}
47
+
48
+	// Allocate interface
49
+	job = eng.Job("allocate_interface", "container_id")
50
+	if res := Allocate(job); res != engine.StatusOK {
51
+		t.Fatal("Failed to allocate network interface")
52
+	}
53
+
54
+	// Allocate same port twice, expect failure on second call
55
+	job = newPortAllocationJob(eng, freePort)
56
+	if res := AllocatePort(job); res != engine.StatusOK {
57
+		t.Fatal("Failed to find a free port to allocate")
58
+	}
59
+	if res := AllocatePort(job); res == engine.StatusOK {
60
+		t.Fatal("Duplicate port allocation granted by AllocatePort")
61
+	}
62
+}
63
+
64
+func TestAllocatePortReclaim(t *testing.T) {
65
+	eng := engine.New()
66
+	eng.Logging = false
67
+
68
+	freePort := findFreePort(t)
69
+
70
+	// Init driver
71
+	job := eng.Job("initdriver")
72
+	if res := InitDriver(job); res != engine.StatusOK {
73
+		t.Fatal("Failed to initialize network driver")
74
+	}
75
+
76
+	// Allocate interface
77
+	job = eng.Job("allocate_interface", "container_id")
78
+	if res := Allocate(job); res != engine.StatusOK {
79
+		t.Fatal("Failed to allocate network interface")
80
+	}
81
+
82
+	// Occupy port
83
+	listenAddr := fmt.Sprintf(":%d", freePort)
84
+	tcpListenAddr, err := net.ResolveTCPAddr("tcp", listenAddr)
85
+	if err != nil {
86
+		t.Fatalf("Failed to resolve TCP address '%s'", listenAddr)
87
+	}
88
+
89
+	l, err := net.ListenTCP("tcp", tcpListenAddr)
90
+	if err != nil {
91
+		t.Fatalf("Fail to listen on port %d", freePort)
92
+	}
93
+
94
+	// Allocate port, expect failure
95
+	job = newPortAllocationJob(eng, freePort)
96
+	if res := AllocatePort(job); res == engine.StatusOK {
97
+		t.Fatal("Successfully allocated currently used port")
98
+	}
99
+
100
+	// Reclaim port, retry allocation
101
+	l.Close()
102
+	if res := AllocatePort(job); res != engine.StatusOK {
103
+		t.Fatal("Failed to allocate previously reclaimed port")
104
+	}
105
+}
... ...
@@ -3,10 +3,12 @@ package portmapper
3 3
 import (
4 4
 	"errors"
5 5
 	"fmt"
6
-	"github.com/dotcloud/docker/pkg/iptables"
7
-	"github.com/dotcloud/docker/pkg/proxy"
8 6
 	"net"
9 7
 	"sync"
8
+
9
+	"github.com/dotcloud/docker/daemon/networkdriver/portallocator"
10
+	"github.com/dotcloud/docker/pkg/iptables"
11
+	"github.com/dotcloud/docker/pkg/proxy"
10 12
 )
11 13
 
12 14
 type mapping struct {
... ...
@@ -31,47 +33,87 @@ var (
31 31
 	ErrPortNotMapped             = errors.New("port is not mapped")
32 32
 )
33 33
 
34
+type genericAddr struct {
35
+	IP   net.IP
36
+	Port int
37
+}
38
+
39
+func (g *genericAddr) Network() string {
40
+	return ""
41
+}
42
+
43
+func (g *genericAddr) String() string {
44
+	return fmt.Sprintf("%s:%d", g.IP.String(), g.Port)
45
+}
46
+
34 47
 func SetIptablesChain(c *iptables.Chain) {
35 48
 	chain = c
36 49
 }
37 50
 
38
-func Map(container net.Addr, hostIP net.IP, hostPort int) error {
51
+func Map(container net.Addr, hostIP net.IP, hostPort int) (net.Addr, error) {
39 52
 	lock.Lock()
40 53
 	defer lock.Unlock()
41 54
 
42
-	var m *mapping
55
+	var (
56
+		m                 *mapping
57
+		err               error
58
+		proto             string
59
+		allocatedHostPort int
60
+	)
61
+
43 62
 	switch container.(type) {
44 63
 	case *net.TCPAddr:
64
+		proto = "tcp"
65
+		if allocatedHostPort, err = portallocator.RequestPort(hostIP, proto, hostPort); err != nil {
66
+			return &net.TCPAddr{IP: hostIP, Port: hostPort}, err
67
+		}
45 68
 		m = &mapping{
46
-			proto:     "tcp",
47
-			host:      &net.TCPAddr{IP: hostIP, Port: hostPort},
69
+			proto:     proto,
70
+			host:      &net.TCPAddr{IP: hostIP, Port: allocatedHostPort},
48 71
 			container: container,
49 72
 		}
50 73
 	case *net.UDPAddr:
74
+		proto = "udp"
75
+		if allocatedHostPort, err = portallocator.RequestPort(hostIP, proto, hostPort); err != nil {
76
+			return &net.UDPAddr{IP: hostIP, Port: hostPort}, err
77
+		}
51 78
 		m = &mapping{
52
-			proto:     "udp",
53
-			host:      &net.UDPAddr{IP: hostIP, Port: hostPort},
79
+			proto:     proto,
80
+			host:      &net.UDPAddr{IP: hostIP, Port: allocatedHostPort},
54 81
 			container: container,
55 82
 		}
56 83
 	default:
57
-		return ErrUnknownBackendAddressType
84
+		// Always return a proper net.Addr for proper reporting.
85
+		return &genericAddr{IP: hostIP, Port: hostPort}, ErrUnknownBackendAddressType
58 86
 	}
59 87
 
88
+	// When binding fails:
89
+	//  - for a specifically requested port: it might be locked by some other
90
+	//    process, so we want to allow for an ulterior retry
91
+	//  - for an automatically allocated port: it falls in the Docker range of
92
+	//    ports, so we'll just remember it as used and try the next free one
93
+	defer func() {
94
+		if err != nil && hostPort != 0 {
95
+			portallocator.ReleasePort(hostIP, proto, allocatedHostPort)
96
+		}
97
+	}()
98
+
60 99
 	key := getKey(m.host)
61 100
 	if _, exists := currentMappings[key]; exists {
62
-		return ErrPortMappedForIP
101
+		err = ErrPortMappedForIP
102
+		return m.host, err
63 103
 	}
64 104
 
65 105
 	containerIP, containerPort := getIPAndPort(m.container)
66
-	if err := forward(iptables.Add, m.proto, hostIP, hostPort, containerIP.String(), containerPort); err != nil {
67
-		return err
106
+	if err := forward(iptables.Add, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort); err != nil {
107
+		return m.host, err
68 108
 	}
69 109
 
70 110
 	p, err := newProxy(m.host, m.container)
71 111
 	if err != nil {
72 112
 		// need to undo the iptables rules before we reutrn
73
-		forward(iptables.Delete, m.proto, hostIP, hostPort, containerIP.String(), containerPort)
74
-		return err
113
+		forward(iptables.Delete, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort)
114
+		return m.host, err
75 115
 	}
76 116
 
77 117
 	m.userlandProxy = p
... ...
@@ -79,7 +121,7 @@ func Map(container net.Addr, hostIP net.IP, hostPort int) error {
79 79
 
80 80
 	go p.Run()
81 81
 
82
-	return nil
82
+	return m.host, nil
83 83
 }
84 84
 
85 85
 func Unmap(host net.Addr) error {
... ...
@@ -100,6 +142,18 @@ func Unmap(host net.Addr) error {
100 100
 	if err := forward(iptables.Delete, data.proto, hostIP, hostPort, containerIP.String(), containerPort); err != nil {
101 101
 		return err
102 102
 	}
103
+
104
+	switch a := host.(type) {
105
+	case *net.TCPAddr:
106
+		if err := portallocator.ReleasePort(a.IP, "tcp", a.Port); err != nil {
107
+			return err
108
+		}
109
+	case *net.UDPAddr:
110
+		if err := portallocator.ReleasePort(a.IP, "udp", a.Port); err != nil {
111
+			return err
112
+		}
113
+	}
114
+
103 115
 	return nil
104 116
 }
105 117
 
... ...
@@ -44,20 +44,36 @@ func TestMapPorts(t *testing.T) {
44 44
 	srcAddr1 := &net.TCPAddr{Port: 1080, IP: net.ParseIP("172.16.0.1")}
45 45
 	srcAddr2 := &net.TCPAddr{Port: 1080, IP: net.ParseIP("172.16.0.2")}
46 46
 
47
-	if err := Map(srcAddr1, dstIp1, 80); err != nil {
47
+	addrEqual := func(addr1, addr2 net.Addr) bool {
48
+		return (addr1.Network() == addr2.Network()) && (addr1.String() == addr2.String())
49
+	}
50
+
51
+	if host, err := Map(srcAddr1, dstIp1, 80); err != nil {
48 52
 		t.Fatalf("Failed to allocate port: %s", err)
53
+	} else if !addrEqual(dstAddr1, host) {
54
+		t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
55
+			dstAddr1.String(), dstAddr1.Network(), host.String(), host.Network())
49 56
 	}
50 57
 
51
-	if Map(srcAddr1, dstIp1, 80) == nil {
58
+	if host, err := Map(srcAddr1, dstIp1, 80); err == nil {
52 59
 		t.Fatalf("Port is in use - mapping should have failed")
60
+	} else if !addrEqual(dstAddr1, host) {
61
+		t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
62
+			dstAddr1.String(), dstAddr1.Network(), host.String(), host.Network())
53 63
 	}
54 64
 
55
-	if Map(srcAddr2, dstIp1, 80) == nil {
65
+	if host, err := Map(srcAddr2, dstIp1, 80); err == nil {
56 66
 		t.Fatalf("Port is in use - mapping should have failed")
67
+	} else if !addrEqual(dstAddr1, host) {
68
+		t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
69
+			dstAddr1.String(), dstAddr1.Network(), host.String(), host.Network())
57 70
 	}
58 71
 
59
-	if err := Map(srcAddr2, dstIp2, 80); err != nil {
72
+	if host, err := Map(srcAddr2, dstIp2, 80); err != nil {
60 73
 		t.Fatalf("Failed to allocate port: %s", err)
74
+	} else if !addrEqual(dstAddr2, host) {
75
+		t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
76
+			dstAddr1.String(), dstAddr1.Network(), host.String(), host.Network())
61 77
 	}
62 78
 
63 79
 	if Unmap(dstAddr1) != nil {