Browse code

Integrated the network allocator into Docker. A networking environment is assigned to each container upon Start and released whenever the container exits.

Andrea Luzzardi authored on 2013/02/26 07:06:22
Showing 5 changed files
... ...
@@ -33,9 +33,12 @@ type Container struct {
33 33
 
34 34
 	Config     *Config
35 35
 	Filesystem *Filesystem
36
-	Network    *NetworkInterface
37 36
 	State      *State
38 37
 
38
+	network          *NetworkInterface
39
+	networkAllocator *NetworkAllocator
40
+	NetworkConfig    *NetworkConfig
41
+
39 42
 	SysInitPath   string
40 43
 	lxcConfigPath string
41 44
 	cmd           *exec.Cmd
... ...
@@ -56,16 +59,23 @@ type Config struct {
56 56
 	OpenStdin bool // Open stdin
57 57
 }
58 58
 
59
-func createContainer(id string, root string, command string, args []string, layers []string, config *Config) (*Container, error) {
59
+type NetworkConfig struct {
60
+	IpAddress   string
61
+	IpPrefixLen int
62
+}
63
+
64
+func createContainer(id string, root string, command string, args []string, layers []string, config *Config, netAllocator *NetworkAllocator) (*Container, error) {
60 65
 	container := &Container{
61
-		Id:         id,
62
-		Root:       root,
63
-		Created:    time.Now(),
64
-		Path:       command,
65
-		Args:       args,
66
-		Config:     config,
67
-		Filesystem: newFilesystem(path.Join(root, "rootfs"), path.Join(root, "rw"), layers),
68
-		State:      newState(),
66
+		Id:               id,
67
+		Root:             root,
68
+		Created:          time.Now(),
69
+		Path:             command,
70
+		Args:             args,
71
+		Config:           config,
72
+		Filesystem:       newFilesystem(path.Join(root, "rootfs"), path.Join(root, "rw"), layers),
73
+		State:            newState(),
74
+		networkAllocator: netAllocator,
75
+		NetworkConfig:    &NetworkConfig{},
69 76
 
70 77
 		SysInitPath:   sysInitPath,
71 78
 		lxcConfigPath: path.Join(root, "config.lxc"),
... ...
@@ -88,27 +98,25 @@ func createContainer(id string, root string, command string, args []string, laye
88 88
 	if err := container.Filesystem.createMountPoints(); err != nil {
89 89
 		return nil, err
90 90
 	}
91
-	var err error
92
-	if container.Network, err = allocateNetwork(); err != nil {
93
-		return nil, err
94
-	}
95 91
 	if err := container.save(); err != nil {
96 92
 		return nil, err
97 93
 	}
98 94
 	return container, nil
99 95
 }
100 96
 
101
-func loadContainer(containerPath string) (*Container, error) {
97
+func loadContainer(containerPath string, netAllocator *NetworkAllocator) (*Container, error) {
102 98
 	data, err := ioutil.ReadFile(path.Join(containerPath, "config.json"))
103 99
 	if err != nil {
104 100
 		return nil, err
105 101
 	}
106 102
 	container := &Container{
107
-		stdout:        newWriteBroadcaster(),
108
-		stderr:        newWriteBroadcaster(),
109
-		stdoutLog:     new(bytes.Buffer),
110
-		stderrLog:     new(bytes.Buffer),
111
-		lxcConfigPath: path.Join(containerPath, "config.lxc"),
103
+		stdout:           newWriteBroadcaster(),
104
+		stderr:           newWriteBroadcaster(),
105
+		stdoutLog:        new(bytes.Buffer),
106
+		stderrLog:        new(bytes.Buffer),
107
+		lxcConfigPath:    path.Join(containerPath, "config.lxc"),
108
+		networkAllocator: netAllocator,
109
+		NetworkConfig:    &NetworkConfig{},
112 110
 	}
113 111
 	if err := json.Unmarshal(data, container); err != nil {
114 112
 		return nil, err
... ...
@@ -268,6 +276,9 @@ func (container *Container) Start() error {
268 268
 	if err := container.Filesystem.EnsureMounted(); err != nil {
269 269
 		return err
270 270
 	}
271
+	if err := container.allocateNetwork(); err != nil {
272
+		return err
273
+	}
271 274
 	if err := container.generateLXCConfig(); err != nil {
272 275
 		return err
273 276
 	}
... ...
@@ -279,7 +290,7 @@ func (container *Container) Start() error {
279 279
 	}
280 280
 
281 281
 	// Networking
282
-	params = append(params, "-g", container.Network.Gateway.String())
282
+	params = append(params, "-g", container.network.Gateway.String())
283 283
 
284 284
 	// User
285 285
 	if container.Config.User != "" {
... ...
@@ -356,12 +367,33 @@ func (container *Container) StderrLog() io.Reader {
356 356
 	return strings.NewReader(container.stderrLog.String())
357 357
 }
358 358
 
359
+func (container *Container) allocateNetwork() error {
360
+	iface, err := container.networkAllocator.Allocate()
361
+	if err != nil {
362
+		return err
363
+	}
364
+	container.network = iface
365
+	container.NetworkConfig.IpAddress = iface.IPNet.IP.String()
366
+	container.NetworkConfig.IpPrefixLen, _ = iface.IPNet.Mask.Size()
367
+	return nil
368
+}
369
+
370
+func (container *Container) releaseNetwork() error {
371
+	err := container.networkAllocator.Release(container.network)
372
+	container.network = nil
373
+	container.NetworkConfig = &NetworkConfig{}
374
+	return err
375
+}
376
+
359 377
 func (container *Container) monitor() {
360 378
 	// Wait for the program to exit
361 379
 	container.cmd.Wait()
362 380
 	exitCode := container.cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
363 381
 
364 382
 	// Cleanup
383
+	if err := container.releaseNetwork(); err != nil {
384
+		log.Printf("%v: Failed to release network: %v", container.Id, err)
385
+	}
365 386
 	container.stdout.Close()
366 387
 	container.stderr.Close()
367 388
 	if err := container.Filesystem.Umount(); err != nil {
... ...
@@ -429,7 +461,6 @@ func (container *Container) Restart() error {
429 429
 }
430 430
 
431 431
 func (container *Container) Wait() {
432
-
433 432
 	for container.State.Running {
434 433
 		container.State.wait()
435 434
 	}
... ...
@@ -11,9 +11,10 @@ import (
11 11
 )
12 12
 
13 13
 type Docker struct {
14
-	root       string
15
-	repository string
16
-	containers *list.List
14
+	root             string
15
+	repository       string
16
+	containers       *list.List
17
+	networkAllocator *NetworkAllocator
17 18
 }
18 19
 
19 20
 func (docker *Docker) List() []*Container {
... ...
@@ -51,7 +52,7 @@ func (docker *Docker) Create(id string, command string, args []string, layers []
51 51
 		return nil, fmt.Errorf("Container %v already exists", id)
52 52
 	}
53 53
 	root := path.Join(docker.repository, id)
54
-	container, err := createContainer(id, root, command, args, layers, config)
54
+	container, err := createContainer(id, root, command, args, layers, config, docker.networkAllocator)
55 55
 	if err != nil {
56 56
 		return nil, err
57 57
 	}
... ...
@@ -86,7 +87,7 @@ func (docker *Docker) restore() error {
86 86
 		return err
87 87
 	}
88 88
 	for _, v := range dir {
89
-		container, err := loadContainer(path.Join(docker.repository, v.Name()))
89
+		container, err := loadContainer(path.Join(docker.repository, v.Name()), docker.networkAllocator)
90 90
 		if err != nil {
91 91
 			log.Printf("Failed to load container %v: %v", v.Name(), err)
92 92
 			continue
... ...
@@ -101,10 +102,15 @@ func New() (*Docker, error) {
101 101
 }
102 102
 
103 103
 func NewFromDirectory(root string) (*Docker, error) {
104
+	alloc, err := newNetworkAllocator(networkBridgeIface)
105
+	if err != nil {
106
+		return nil, err
107
+	}
104 108
 	docker := &Docker{
105
-		root:       root,
106
-		repository: path.Join(root, "containers"),
107
-		containers: list.New(),
109
+		root:             root,
110
+		repository:       path.Join(root, "containers"),
111
+		containers:       list.New(),
112
+		networkAllocator: alloc,
108 113
 	}
109 114
 
110 115
 	if err := os.MkdirAll(docker.repository, 0700); err != nil && !os.IsExist(err) {
... ...
@@ -19,7 +19,7 @@ lxc.network.flags = up
19 19
 lxc.network.link = lxcbr0
20 20
 lxc.network.name = eth0
21 21
 lxc.network.mtu = 1500
22
-lxc.network.ipv4 = {{.Network.IpAddress}}/{{.Network.IpPrefixLen}}
22
+lxc.network.ipv4 = {{.NetworkConfig.IpAddress}}/{{.NetworkConfig.IpPrefixLen}}
23 23
 
24 24
 # root filesystem
25 25
 {{$ROOTFS := .Filesystem.RootFS}}
... ...
@@ -5,7 +5,6 @@ import (
5 5
 	"encoding/binary"
6 6
 	"errors"
7 7
 	"fmt"
8
-	"math/rand"
9 8
 	"net"
10 9
 )
11 10
 
... ...
@@ -14,11 +13,12 @@ const (
14 14
 )
15 15
 
16 16
 type NetworkInterface struct {
17
-	IpAddress   string
18
-	IpPrefixLen int
19
-	Gateway     net.IP
17
+	IPNet   net.IPNet
18
+	Gateway net.IP
20 19
 }
21 20
 
21
+// IP utils
22
+
22 23
 func networkRange(network *net.IPNet) (net.IP, net.IP) {
23 24
 	netIP := network.IP.To4()
24 25
 	firstIP := netIP.Mask(network.Mask)
... ...
@@ -51,10 +51,11 @@ func intToIp(n int32) (net.IP, error) {
51 51
 }
52 52
 
53 53
 func networkSize(mask net.IPMask) (int32, error) {
54
+	m := net.IPv4Mask(0, 0, 0, 0)
54 55
 	for i := 0; i < net.IPv4len; i++ {
55
-		mask[i] = ^mask[i]
56
+		m[i] = ^mask[i]
56 57
 	}
57
-	buf := bytes.NewBuffer(mask)
58
+	buf := bytes.NewBuffer(m)
58 59
 	var n int32
59 60
 	if err := binary.Read(buf, binary.BigEndian, &n); err != nil {
60 61
 		return 0, err
... ...
@@ -62,21 +63,7 @@ func networkSize(mask net.IPMask) (int32, error) {
62 62
 	return n + 1, nil
63 63
 }
64 64
 
65
-func allocateIPAddress(network *net.IPNet) (net.IP, error) {
66
-	ip, _ := networkRange(network)
67
-	netSize, err := networkSize(network.Mask)
68
-	if err != nil {
69
-		return net.IP{}, err
70
-	}
71
-	numIp, err := ipToInt(ip)
72
-	if err != nil {
73
-		return net.IP{}, err
74
-	}
75
-	numIp += rand.Int31n(netSize)
76
-	return intToIp(numIp)
77
-}
78
-
79
-func getBridgeAddr(name string) (net.Addr, error) {
65
+func getIfaceAddr(name string) (net.Addr, error) {
80 66
 	iface, err := net.InterfaceByName(name)
81 67
 	if err != nil {
82 68
 		return nil, err
... ...
@@ -101,31 +88,31 @@ func getBridgeAddr(name string) (net.Addr, error) {
101 101
 	return addrs4[0], nil
102 102
 }
103 103
 
104
-func allocateNetwork() (*NetworkInterface, error) {
105
-	bridgeAddr, err := getBridgeAddr(networkBridgeIface)
104
+// Network allocator
105
+func newNetworkAllocator(iface string) (*NetworkAllocator, error) {
106
+	addr, err := getIfaceAddr(iface)
106 107
 	if err != nil {
107 108
 		return nil, err
108 109
 	}
109
-	bridge := bridgeAddr.(*net.IPNet)
110
-	ipPrefixLen, _ := bridge.Mask.Size()
111
-	ip, err := allocateIPAddress(bridge)
112
-	if err != nil {
113
-		return nil, err
110
+	network := addr.(*net.IPNet)
111
+
112
+	alloc := &NetworkAllocator{
113
+		iface: iface,
114
+		net:   network,
114 115
 	}
115
-	iface := &NetworkInterface{
116
-		IpAddress:   ip.String(),
117
-		IpPrefixLen: ipPrefixLen,
118
-		Gateway:     bridge.IP,
116
+	if err := alloc.populateFromNetwork(network); err != nil {
117
+		return nil, err
119 118
 	}
120
-	return iface, nil
119
+	return alloc, nil
121 120
 }
122 121
 
123 122
 type NetworkAllocator struct {
124 123
 	iface string
124
+	net   *net.IPNet
125 125
 	queue chan (net.IP)
126 126
 }
127 127
 
128
-func (alloc *NetworkAllocator) Acquire() (net.IP, error) {
128
+func (alloc *NetworkAllocator) acquireIP() (net.IP, error) {
129 129
 	select {
130 130
 	case ip := <-alloc.queue:
131 131
 		return ip, nil
... ...
@@ -135,7 +122,7 @@ func (alloc *NetworkAllocator) Acquire() (net.IP, error) {
135 135
 	return net.IP{}, nil
136 136
 }
137 137
 
138
-func (alloc *NetworkAllocator) Release(ip net.IP) error {
138
+func (alloc *NetworkAllocator) releaseIP(ip net.IP) error {
139 139
 	select {
140 140
 	case alloc.queue <- ip:
141 141
 		return nil
... ...
@@ -145,7 +132,7 @@ func (alloc *NetworkAllocator) Release(ip net.IP) error {
145 145
 	return nil
146 146
 }
147 147
 
148
-func (alloc *NetworkAllocator) PopulateFromNetwork(network *net.IPNet) error {
148
+func (alloc *NetworkAllocator) populateFromNetwork(network *net.IPNet) error {
149 149
 	firstIP, _ := networkRange(network)
150 150
 	size, err := networkSize(network.Mask)
151 151
 	if err != nil {
... ...
@@ -168,16 +155,24 @@ func (alloc *NetworkAllocator) PopulateFromNetwork(network *net.IPNet) error {
168 168
 		if ip.Equal(network.IP) {
169 169
 			continue
170 170
 		}
171
-		alloc.Release(ip)
171
+		alloc.releaseIP(ip)
172 172
 	}
173 173
 	return nil
174 174
 }
175 175
 
176
-func (alloc *NetworkAllocator) PopulateFromInterface(iface string) error {
177
-	addr, err := getBridgeAddr(iface)
176
+func (alloc *NetworkAllocator) Allocate() (*NetworkInterface, error) {
177
+	// ipPrefixLen, _ := alloc.net.Mask.Size()
178
+	ip, err := alloc.acquireIP()
178 179
 	if err != nil {
179
-		return err
180
+		return nil, err
180 181
 	}
181
-	network := addr.(*net.IPNet)
182
-	return alloc.PopulateFromNetwork(network)
182
+	iface := &NetworkInterface{
183
+		IPNet:   net.IPNet{ip, alloc.net.Mask},
184
+		Gateway: alloc.net.IP,
185
+	}
186
+	return iface, nil
187
+}
188
+
189
+func (alloc *NetworkAllocator) Release(iface *NetworkInterface) error {
190
+	return alloc.releaseIP(iface.IPNet.IP)
183 191
 }
... ...
@@ -103,22 +103,22 @@ func TestConversion(t *testing.T) {
103 103
 func TestNetworkAllocator(t *testing.T) {
104 104
 	alloc := NetworkAllocator{}
105 105
 	_, n, _ := net.ParseCIDR("127.0.0.1/29")
106
-	alloc.PopulateFromNetwork(n)
106
+	alloc.populateFromNetwork(n)
107 107
 	var lastIP net.IP
108 108
 	for i := 0; i < 5; i++ {
109
-		ip, err := alloc.Acquire()
109
+		ip, err := alloc.acquireIP()
110 110
 		if err != nil {
111 111
 			t.Fatal(err)
112 112
 		}
113 113
 		lastIP = ip
114 114
 	}
115
-	ip, err := alloc.Acquire()
115
+	ip, err := alloc.acquireIP()
116 116
 	if err == nil {
117 117
 		t.Fatal("There shouldn't be any IP addresses at this point")
118 118
 	}
119 119
 	// Release 1 IP
120
-	alloc.Release(lastIP)
121
-	ip, err = alloc.Acquire()
120
+	alloc.releaseIP(lastIP)
121
+	ip, err = alloc.acquireIP()
122 122
 	if err != nil {
123 123
 		t.Fatal(err)
124 124
 	}