Browse code

libnetwork: Add garbage collection trigger

When the daemon is going down trigger immediate
garbage collection of libnetwork resources deleted
like namespace path since there will be no way to
remove them when the daemon restarts.

Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>

Jana Radhakrishnan authored on 2015/06/06 07:02:56
Showing 16 changed files
... ...
@@ -996,6 +996,11 @@ func (daemon *Daemon) Shutdown() error {
996 996
 			}
997 997
 		}
998 998
 		group.Wait()
999
+
1000
+		// trigger libnetwork GC only if it's initialized
1001
+		if daemon.netController != nil {
1002
+			daemon.netController.GC()
1003
+		}
999 1004
 	}
1000 1005
 
1001 1006
 	return nil
... ...
@@ -55,8 +55,8 @@ clone hg code.google.com/p/go.net 84a4013f96e0
55 55
 clone hg code.google.com/p/gosqlite 74691fb6f837
56 56
 
57 57
 #get libnetwork packages
58
-clone git github.com/docker/libnetwork f72ad20491e8c46d9664da3f32a0eddb301e7c8d
59
-clone git github.com/vishvananda/netns 008d17ae001344769b031375bdb38a86219154c6
58
+clone git github.com/docker/libnetwork ace6b576239cd671d1645d82520b16791737f60d
59
+clone git github.com/vishvananda/netns 5478c060110032f972e86a1f844fdb9a2f008f2c
60 60
 clone git github.com/vishvananda/netlink 8eb64238879fed52fd51c5b30ad20b928fb4c36c
61 61
 
62 62
 # get distribution packages
... ...
@@ -1174,6 +1174,7 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithContainerRunning(t *check.C) {
1174 1174
 	if out, err := s.d.Cmd("run", "-ti", "-d", "--name", "test", "busybox"); err != nil {
1175 1175
 		t.Fatal(out, err)
1176 1176
 	}
1177
+
1177 1178
 	if err := s.d.Restart(); err != nil {
1178 1179
 		t.Fatal(err)
1179 1180
 	}
... ...
@@ -1182,3 +1183,41 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithContainerRunning(t *check.C) {
1182 1182
 		t.Fatal(out, err)
1183 1183
 	}
1184 1184
 }
1185
+
1186
+func (s *DockerDaemonSuite) TestDaemonRestartCleanupNetns(c *check.C) {
1187
+	if err := s.d.StartWithBusybox(); err != nil {
1188
+		c.Fatal(err)
1189
+	}
1190
+	out, err := s.d.Cmd("run", "--name", "netns", "-d", "busybox", "top")
1191
+	if err != nil {
1192
+		c.Fatal(out, err)
1193
+	}
1194
+	if out, err := s.d.Cmd("stop", "netns"); err != nil {
1195
+		c.Fatal(out, err)
1196
+	}
1197
+
1198
+	// Construct netns file name from container id
1199
+	out = strings.TrimSpace(out)
1200
+	nsFile := out[:12]
1201
+
1202
+	// Test if the file still exists
1203
+	out, _, err = runCommandWithOutput(exec.Command("stat", "-c", "%n", "/var/run/docker/netns/"+nsFile))
1204
+	out = strings.TrimSpace(out)
1205
+	c.Assert(err, check.IsNil, check.Commentf("Output: %s", out))
1206
+	c.Assert(out, check.Equals, "/var/run/docker/netns/"+nsFile, check.Commentf("Output: %s", out))
1207
+
1208
+	// Remove the container and restart the daemon
1209
+	if out, err := s.d.Cmd("rm", "netns"); err != nil {
1210
+		c.Fatal(out, err)
1211
+	}
1212
+
1213
+	if err := s.d.Restart(); err != nil {
1214
+		c.Fatal(err)
1215
+	}
1216
+
1217
+	// Test again and see now the netns file does not exist
1218
+	out, _, err = runCommandWithOutput(exec.Command("stat", "-c", "%n", "/var/run/docker/netns/"+nsFile))
1219
+	out = strings.TrimSpace(out)
1220
+	c.Assert(err, check.Not(check.IsNil), check.Commentf("Output: %s", out))
1221
+	// c.Assert(out, check.Equals, "", check.Commentf("Output: %s", out))
1222
+}
... ...
@@ -79,7 +79,7 @@
79 79
 		},
80 80
 		{
81 81
 			"ImportPath": "github.com/vishvananda/netns",
82
-			"Rev": "008d17ae001344769b031375bdb38a86219154c6"
82
+			"Rev": "5478c060110032f972e86a1f844fdb9a2f008f2c"
83 83
 		}
84 84
 	]
85 85
 }
... ...
@@ -76,6 +76,9 @@ type NetworkController interface {
76 76
 
77 77
 	// NetworkByID returns the Network which has the passed id. If not found, the error ErrNoSuchNetwork is returned.
78 78
 	NetworkByID(id string) (Network, error)
79
+
80
+	// GC triggers immediate garbage collection of resources which are garbage collected.
81
+	GC()
79 82
 }
80 83
 
81 84
 // NetworkWalker is a client provided function which will be used to walk the Networks.
... ...
@@ -299,3 +302,7 @@ func (c *controller) loadDriver(networkType string) (driverapi.Driver, error) {
299 299
 	}
300 300
 	return d, nil
301 301
 }
302
+
303
+func (c *controller) GC() {
304
+	sandbox.GC()
305
+}
... ...
@@ -24,6 +24,7 @@ var (
24 24
 	gpmLock          sync.Mutex
25 25
 	gpmWg            sync.WaitGroup
26 26
 	gpmCleanupPeriod = 60 * time.Second
27
+	gpmChan          = make(chan chan struct{})
27 28
 )
28 29
 
29 30
 // The networkNamespace type is the linux implementation of the Sandbox
... ...
@@ -55,7 +56,18 @@ func removeUnusedPaths() {
55 55
 	period := gpmCleanupPeriod
56 56
 	gpmLock.Unlock()
57 57
 
58
-	for range time.Tick(period) {
58
+	ticker := time.NewTicker(period)
59
+	for {
60
+		var (
61
+			gc   chan struct{}
62
+			gcOk bool
63
+		)
64
+
65
+		select {
66
+		case <-ticker.C:
67
+		case gc, gcOk = <-gpmChan:
68
+		}
69
+
59 70
 		gpmLock.Lock()
60 71
 		pathList := make([]string, 0, len(garbagePathMap))
61 72
 		for path := range garbagePathMap {
... ...
@@ -70,6 +82,9 @@ func removeUnusedPaths() {
70 70
 		}
71 71
 
72 72
 		gpmWg.Done()
73
+		if gcOk {
74
+			close(gc)
75
+		}
73 76
 	}
74 77
 }
75 78
 
... ...
@@ -85,6 +100,18 @@ func removeFromGarbagePaths(path string) {
85 85
 	gpmLock.Unlock()
86 86
 }
87 87
 
88
+// GC triggers garbage collection of namespace path right away
89
+// and waits for it.
90
+func GC() {
91
+	waitGC := make(chan struct{})
92
+
93
+	// Trigger GC now
94
+	gpmChan <- waitGC
95
+
96
+	// wait for gc to complete
97
+	<-waitGC
98
+}
99
+
88 100
 // GenerateKey generates a sandbox key based on the passed
89 101
 // container id.
90 102
 func GenerateKey(containerID string) string {
... ...
@@ -144,9 +144,16 @@ func verifySandbox(t *testing.T, s Sandbox) {
144 144
 	}
145 145
 }
146 146
 
147
-func verifyCleanup(t *testing.T, s Sandbox) {
148
-	time.Sleep(time.Duration(gpmCleanupPeriod * 2))
147
+func verifyCleanup(t *testing.T, s Sandbox, wait bool) {
148
+	if wait {
149
+		time.Sleep(time.Duration(gpmCleanupPeriod * 2))
150
+	}
151
+
149 152
 	if _, err := os.Stat(s.Key()); err == nil {
150
-		t.Fatalf("The sandbox path %s is not getting cleanup event after twice the cleanup period", s.Key())
153
+		if wait {
154
+			t.Fatalf("The sandbox path %s is not getting cleaned up even after twice the cleanup period", s.Key())
155
+		} else {
156
+			t.Fatalf("The sandbox path %s is not cleaned up after running gc", s.Key())
157
+		}
151 158
 	}
152 159
 }
... ...
@@ -54,7 +54,7 @@ func TestSandboxCreate(t *testing.T) {
54 54
 
55 55
 	verifySandbox(t, s)
56 56
 	s.Destroy()
57
-	verifyCleanup(t, s)
57
+	verifyCleanup(t, s, true)
58 58
 }
59 59
 
60 60
 func TestSandboxCreateTwice(t *testing.T) {
... ...
@@ -77,6 +77,23 @@ func TestSandboxCreateTwice(t *testing.T) {
77 77
 	s.Destroy()
78 78
 }
79 79
 
80
+func TestSandboxGC(t *testing.T) {
81
+	key, err := newKey(t)
82
+	if err != nil {
83
+		t.Fatalf("Failed to obtain a key: %v", err)
84
+	}
85
+
86
+	s, err := NewSandbox(key, true)
87
+	if err != nil {
88
+		t.Fatalf("Failed to create a new sandbox: %v", err)
89
+	}
90
+
91
+	s.Destroy()
92
+
93
+	GC()
94
+	verifyCleanup(t, s, false)
95
+}
96
+
80 97
 func TestInterfaceEqual(t *testing.T) {
81 98
 	list := getInterfaceList()
82 99
 
... ...
@@ -12,6 +12,7 @@ import (
12 12
 	"fmt"
13 13
 	"syscall"
14 14
 )
15
+
15 16
 // NsHandle is a handle to a network namespace. It can be cast directly
16 17
 // to an int and used as a file descriptor.
17 18
 type NsHandle int
... ...
@@ -1,3 +1,5 @@
1
+// +build linux
2
+
1 3
 package netns
2 4
 
3 5
 import (
... ...
@@ -1,3 +1,5 @@
1
+// +build linux,386
2
+
1 3
 package netns
2 4
 
3 5
 const (
4 6
deleted file mode 100644
... ...
@@ -1,5 +0,0 @@
1
-package netns
2
-
3
-const (
4
-	SYS_SETNS = 308
5
-)
6 1
new file mode 100644
... ...
@@ -0,0 +1,7 @@
0
+// +build linux,amd64
1
+
2
+package netns
3
+
4
+const (
5
+	SYS_SETNS = 308
6
+)
... ...
@@ -1,3 +1,5 @@
1
+// +build linux,arm
2
+
1 3
 package netns
2 4
 
3 5
 const (
4 6
new file mode 100644
... ...
@@ -0,0 +1,7 @@
0
+// +build linux,ppc64le
1
+
2
+package netns
3
+
4
+const (
5
+	SYS_SETNS = 350
6
+)
... ...
@@ -10,26 +10,26 @@ var (
10 10
 	ErrNotImplemented = errors.New("not implemented")
11 11
 )
12 12
 
13
-func Set(ns Namespace) (err error) {
13
+func Set(ns NsHandle) (err error) {
14 14
 	return ErrNotImplemented
15 15
 }
16 16
 
17
-func New() (ns Namespace, err error) {
17
+func New() (ns NsHandle, err error) {
18 18
 	return -1, ErrNotImplemented
19 19
 }
20 20
 
21
-func Get() (Namespace, error) {
21
+func Get() (NsHandle, error) {
22 22
 	return -1, ErrNotImplemented
23 23
 }
24 24
 
25
-func GetFromName(name string) (Namespace, error) {
25
+func GetFromName(name string) (NsHandle, error) {
26 26
 	return -1, ErrNotImplemented
27 27
 }
28 28
 
29
-func GetFromPid(pid int) (Namespace, error) {
29
+func GetFromPid(pid int) (NsHandle, error) {
30 30
 	return -1, ErrNotImplemented
31 31
 }
32 32
 
33
-func GetFromDocker(id string) (Namespace, error) {
33
+func GetFromDocker(id string) (NsHandle, error) {
34 34
 	return -1, ErrNotImplemented
35 35
 }