Browse code

Adding libnetwork support to publish on custom host port ranges. See https://github.com/docker/docker/pull/12927 for docker portion.

Signed-off-by: Don Kjer <don.kjer@gmail.com>

Don Kjer authored on 2015/06/12 10:19:42
Showing 6 changed files
... ...
@@ -57,6 +57,11 @@ func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHos
57 57
 		bnd.HostIP = defHostIP
58 58
 	}
59 59
 
60
+	// Adjust HostPortEnd if this is not a range.
61
+	if bnd.HostPortEnd == 0 {
62
+		bnd.HostPortEnd = bnd.HostPort
63
+	}
64
+
60 65
 	// Construct the container side transport address
61 66
 	container, err := bnd.ContainerAddr()
62 67
 	if err != nil {
... ...
@@ -65,12 +70,12 @@ func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHos
65 65
 
66 66
 	// Try up to maxAllocatePortAttempts times to get a port that's not already allocated.
67 67
 	for i := 0; i < maxAllocatePortAttempts; i++ {
68
-		if host, err = n.portMapper.Map(container, bnd.HostIP, int(bnd.HostPort), ulPxyEnabled); err == nil {
68
+		if host, err = n.portMapper.MapRange(container, bnd.HostIP, int(bnd.HostPort), int(bnd.HostPortEnd), ulPxyEnabled); err == nil {
69 69
 			break
70 70
 		}
71 71
 		// There is no point in immediately retrying to map an explicitly chosen port.
72 72
 		if bnd.HostPort != 0 {
73
-			logrus.Warnf("Failed to allocate and map port %d: %s", bnd.HostPort, err)
73
+			logrus.Warnf("Failed to allocate and map port %d-%d: %s", bnd.HostPort, bnd.HostPortEnd, err)
74 74
 			break
75 75
 		}
76 76
 		logrus.Warnf("Failed to allocate and map port: %s, retry: %d", err, i+1)
... ...
@@ -89,9 +89,11 @@ func getEmptyGenericOption() map[string]interface{} {
89 89
 
90 90
 func getPortMapping() []types.PortBinding {
91 91
 	return []types.PortBinding{
92
-		types.PortBinding{Proto: types.TCP, Port: uint16(230), HostPort: uint16(23000)},
93
-		types.PortBinding{Proto: types.UDP, Port: uint16(200), HostPort: uint16(22000)},
94
-		types.PortBinding{Proto: types.TCP, Port: uint16(120), HostPort: uint16(12000)},
92
+		{Proto: types.TCP, Port: uint16(230), HostPort: uint16(23000)},
93
+		{Proto: types.UDP, Port: uint16(200), HostPort: uint16(22000)},
94
+		{Proto: types.TCP, Port: uint16(120), HostPort: uint16(12000)},
95
+		{Proto: types.TCP, Port: uint16(320), HostPort: uint16(32000), HostPortEnd: uint16(32999)},
96
+		{Proto: types.UDP, Port: uint16(420), HostPort: uint16(42000), HostPortEnd: uint16(42001)},
95 97
 	}
96 98
 }
97 99
 
... ...
@@ -279,7 +281,7 @@ func TestBridge(t *testing.T) {
279 279
 	if !ok {
280 280
 		t.Fatalf("Unexpected format for port mapping in endpoint operational data")
281 281
 	}
282
-	if len(pm) != 3 {
282
+	if len(pm) != 5 {
283 283
 		t.Fatalf("Incomplete data for port mapping in endpoint operational data: %d", len(pm))
284 284
 	}
285 285
 
... ...
@@ -70,10 +70,15 @@ type (
70 70
 		Begin int
71 71
 		End   int
72 72
 	}
73
+	portRange struct {
74
+		begin int
75
+		end   int
76
+		last  int
77
+	}
73 78
 	portMap struct {
74
-		p          map[int]struct{}
75
-		begin, end int
76
-		last       int
79
+		p            map[int]struct{}
80
+		defaultRange string
81
+		portRanges   map[string]*portRange
77 82
 	}
78 83
 	protoMap map[string]*portMap
79 84
 )
... ...
@@ -123,8 +128,17 @@ func getDynamicPortRange() (start int, end int, err error) {
123 123
 
124 124
 // RequestPort requests new port from global ports pool for specified ip and proto.
125 125
 // If port is 0 it returns first free port. Otherwise it checks port availability
126
-// in pool and return that port or error if port is already busy.
126
+// in proto's pool and returns that port or error if port is already busy.
127 127
 func (p *PortAllocator) RequestPort(ip net.IP, proto string, port int) (int, error) {
128
+	return p.RequestPortInRange(ip, proto, port, port)
129
+}
130
+
131
+// RequestPortInRange requests new port from global ports pool for specified ip and proto.
132
+// If portStart and portEnd are 0 it returns the first free port in the default ephemeral range.
133
+// If portStart != portEnd it returns the first free port in the requested range.
134
+// Otherwise (portStart == portEnd) it checks port availability in the requested proto's port-pool
135
+// and returns that port or error if port is already busy.
136
+func (p *PortAllocator) RequestPortInRange(ip net.IP, proto string, portStart, portEnd int) (int, error) {
128 137
 	p.mutex.Lock()
129 138
 	defer p.mutex.Unlock()
130 139
 
... ...
@@ -146,15 +160,15 @@ func (p *PortAllocator) RequestPort(ip net.IP, proto string, port int) (int, err
146 146
 		p.ipMap[ipstr] = protomap
147 147
 	}
148 148
 	mapping := protomap[proto]
149
-	if port > 0 {
150
-		if _, ok := mapping.p[port]; !ok {
151
-			mapping.p[port] = struct{}{}
152
-			return port, nil
149
+	if portStart > 0 && portStart == portEnd {
150
+		if _, ok := mapping.p[portStart]; !ok {
151
+			mapping.p[portStart] = struct{}{}
152
+			return portStart, nil
153 153
 		}
154
-		return 0, newErrPortAlreadyAllocated(ipstr, port)
154
+		return 0, newErrPortAlreadyAllocated(ipstr, portStart)
155 155
 	}
156 156
 
157
-	port, err := mapping.findPort()
157
+	port, err := mapping.findPort(portStart, portEnd)
158 158
 	if err != nil {
159 159
 		return 0, err
160 160
 	}
... ...
@@ -178,12 +192,15 @@ func (p *PortAllocator) ReleasePort(ip net.IP, proto string, port int) error {
178 178
 }
179 179
 
180 180
 func (p *PortAllocator) newPortMap() *portMap {
181
-	return &portMap{
182
-		p:     map[int]struct{}{},
183
-		begin: p.Begin,
184
-		end:   p.End,
185
-		last:  p.End,
181
+	defaultKey := getRangeKey(p.Begin, p.End)
182
+	pm := &portMap{
183
+		p:            map[int]struct{}{},
184
+		defaultRange: defaultKey,
185
+		portRanges: map[string]*portRange{
186
+			defaultKey: newPortRange(p.Begin, p.End),
187
+		},
186 188
 	}
189
+	return pm
187 190
 }
188 191
 
189 192
 // ReleaseAll releases all ports for all ips.
... ...
@@ -194,17 +211,58 @@ func (p *PortAllocator) ReleaseAll() error {
194 194
 	return nil
195 195
 }
196 196
 
197
-func (pm *portMap) findPort() (int, error) {
198
-	port := pm.last
199
-	for i := 0; i <= pm.end-pm.begin; i++ {
197
+func getRangeKey(portStart, portEnd int) string {
198
+	return fmt.Sprintf("%d-%d", portStart, portEnd)
199
+}
200
+
201
+func newPortRange(portStart, portEnd int) *portRange {
202
+	return &portRange{
203
+		begin: portStart,
204
+		end:   portEnd,
205
+		last:  portEnd,
206
+	}
207
+}
208
+
209
+func (pm *portMap) getPortRange(portStart, portEnd int) (*portRange, error) {
210
+	var key string
211
+	if portStart == 0 && portEnd == 0 {
212
+		key = pm.defaultRange
213
+	} else {
214
+		key = getRangeKey(portStart, portEnd)
215
+		if portStart == portEnd ||
216
+			portStart == 0 || portEnd == 0 ||
217
+			portEnd < portStart {
218
+			return nil, fmt.Errorf("invalid port range: %s", key)
219
+		}
220
+	}
221
+
222
+	// Return existing port range, if already known.
223
+	if pr, exists := pm.portRanges[key]; exists {
224
+		return pr, nil
225
+	}
226
+
227
+	// Otherwise create a new port range.
228
+	pr := newPortRange(portStart, portEnd)
229
+	pm.portRanges[key] = pr
230
+	return pr, nil
231
+}
232
+
233
+func (pm *portMap) findPort(portStart, portEnd int) (int, error) {
234
+	pr, err := pm.getPortRange(portStart, portEnd)
235
+	if err != nil {
236
+		return 0, err
237
+	}
238
+	port := pr.last
239
+
240
+	for i := 0; i <= pr.end-pr.begin; i++ {
200 241
 		port++
201
-		if port > pm.end {
202
-			port = pm.begin
242
+		if port > pr.end {
243
+			port = pr.begin
203 244
 		}
204 245
 
205 246
 		if _, ok := pm.p[port]; !ok {
206 247
 			pm.p[port] = struct{}{}
207
-			pm.last = port
248
+			pr.last = port
208 249
 			return port, nil
209 250
 		}
210 251
 	}
... ...
@@ -236,6 +236,72 @@ func TestPortAllocation(t *testing.T) {
236 236
 	}
237 237
 }
238 238
 
239
+func TestPortAllocationWithCustomRange(t *testing.T) {
240
+	p := Get()
241
+	defer resetPortAllocator()
242
+
243
+	start, end := 8081, 8082
244
+	specificPort := 8000
245
+
246
+	//get an ephemeral port.
247
+	port1, err := p.RequestPortInRange(defaultIP, "tcp", 0, 0)
248
+	if err != nil {
249
+		t.Fatal(err)
250
+	}
251
+
252
+	//request invalid ranges
253
+	if _, err := p.RequestPortInRange(defaultIP, "tcp", 0, end); err == nil {
254
+		t.Fatalf("Expected error for invalid range %d-%d", 0, end)
255
+	}
256
+	if _, err := p.RequestPortInRange(defaultIP, "tcp", start, 0); err == nil {
257
+		t.Fatalf("Expected error for invalid range %d-%d", 0, end)
258
+	}
259
+	if _, err := p.RequestPortInRange(defaultIP, "tcp", 8081, 8080); err == nil {
260
+		t.Fatalf("Expected error for invalid range %d-%d", 0, end)
261
+	}
262
+
263
+	//request a single port
264
+	port, err := p.RequestPortInRange(defaultIP, "tcp", specificPort, specificPort)
265
+	if err != nil {
266
+		t.Fatal(err)
267
+	}
268
+	if port != specificPort {
269
+		t.Fatalf("Expected port %d, got %d", specificPort, port)
270
+	}
271
+
272
+	//get a port from the range
273
+	port2, err := p.RequestPortInRange(defaultIP, "tcp", start, end)
274
+	if err != nil {
275
+		t.Fatal(err)
276
+	}
277
+	if port2 < start || port2 > end {
278
+		t.Fatalf("Expected a port between %d and %d, got %d", start, end, port2)
279
+	}
280
+	//get another ephemeral port (should be > port1)
281
+	port3, err := p.RequestPortInRange(defaultIP, "tcp", 0, 0)
282
+	if err != nil {
283
+		t.Fatal(err)
284
+	}
285
+	if port3 < port1 {
286
+		t.Fatalf("Expected new port > %d in the ephemeral range, got %d", port1, port3)
287
+	}
288
+	//get another (and in this case the only other) port from the range
289
+	port4, err := p.RequestPortInRange(defaultIP, "tcp", start, end)
290
+	if err != nil {
291
+		t.Fatal(err)
292
+	}
293
+	if port4 < start || port4 > end {
294
+		t.Fatalf("Expected a port between %d and %d, got %d", start, end, port4)
295
+	}
296
+	if port4 == port2 {
297
+		t.Fatal("Allocated the same port from a custom range")
298
+	}
299
+	//request 3rd port from the range of 2
300
+	if _, err := p.RequestPortInRange(defaultIP, "tcp", start, end); err != ErrAllPortsAllocated {
301
+		t.Fatalf("Expected error %s got %s", ErrAllPortsAllocated, err)
302
+	}
303
+}
304
+
239 305
 func TestNoDuplicateBPR(t *testing.T) {
240 306
 	p := Get()
241 307
 	defer resetPortAllocator()
... ...
@@ -62,6 +62,11 @@ func (pm *PortMapper) SetIptablesChain(c *iptables.ChainInfo, bridgeName string)
62 62
 
63 63
 // Map maps the specified container transport address to the host's network address and transport port
64 64
 func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, useProxy bool) (host net.Addr, err error) {
65
+	return pm.MapRange(container, hostIP, hostPort, hostPort, useProxy)
66
+}
67
+
68
+// MapRange maps the specified container transport address to the host's network address and transport port range
69
+func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, hostPortEnd int, useProxy bool) (host net.Addr, err error) {
65 70
 	pm.lock.Lock()
66 71
 	defer pm.lock.Unlock()
67 72
 
... ...
@@ -74,7 +79,7 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, usePr
74 74
 	switch container.(type) {
75 75
 	case *net.TCPAddr:
76 76
 		proto = "tcp"
77
-		if allocatedHostPort, err = pm.Allocator.RequestPort(hostIP, proto, hostPort); err != nil {
77
+		if allocatedHostPort, err = pm.Allocator.RequestPortInRange(hostIP, proto, hostPortStart, hostPortEnd); err != nil {
78 78
 			return nil, err
79 79
 		}
80 80
 
... ...
@@ -91,7 +96,7 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, usePr
91 91
 		}
92 92
 	case *net.UDPAddr:
93 93
 		proto = "udp"
94
-		if allocatedHostPort, err = pm.Allocator.RequestPort(hostIP, proto, hostPort); err != nil {
94
+		if allocatedHostPort, err = pm.Allocator.RequestPortInRange(hostIP, proto, hostPortStart, hostPortEnd); err != nil {
95 95
 			return nil, err
96 96
 		}
97 97
 
... ...
@@ -24,11 +24,12 @@ func (t *TransportPort) GetCopy() TransportPort {
24 24
 
25 25
 // PortBinding represent a port binding between the container and the host
26 26
 type PortBinding struct {
27
-	Proto    Protocol
28
-	IP       net.IP
29
-	Port     uint16
30
-	HostIP   net.IP
31
-	HostPort uint16
27
+	Proto       Protocol
28
+	IP          net.IP
29
+	Port        uint16
30
+	HostIP      net.IP
31
+	HostPort    uint16
32
+	HostPortEnd uint16
32 33
 }
33 34
 
34 35
 // HostAddr returns the host side transport address
... ...
@@ -58,11 +59,12 @@ func (p PortBinding) ContainerAddr() (net.Addr, error) {
58 58
 // GetCopy returns a copy of this PortBinding structure instance
59 59
 func (p *PortBinding) GetCopy() PortBinding {
60 60
 	return PortBinding{
61
-		Proto:    p.Proto,
62
-		IP:       GetIPCopy(p.IP),
63
-		Port:     p.Port,
64
-		HostIP:   GetIPCopy(p.HostIP),
65
-		HostPort: p.HostPort,
61
+		Proto:       p.Proto,
62
+		IP:          GetIPCopy(p.IP),
63
+		Port:        p.Port,
64
+		HostIP:      GetIPCopy(p.HostIP),
65
+		HostPort:    p.HostPort,
66
+		HostPortEnd: p.HostPortEnd,
66 67
 	}
67 68
 }
68 69
 
... ...
@@ -76,7 +78,8 @@ func (p *PortBinding) Equal(o *PortBinding) bool {
76 76
 		return false
77 77
 	}
78 78
 
79
-	if p.Proto != o.Proto || p.Port != o.Port || p.HostPort != o.HostPort {
79
+	if p.Proto != o.Proto || p.Port != o.Port ||
80
+		p.HostPort != o.HostPort || p.HostPortEnd != o.HostPortEnd {
80 81
 		return false
81 82
 	}
82 83