Before this, if `forceRemove` is set the container data will be removed
no matter what, including if there are issues with removing container
on-disk state (rw layer, container root).
In practice this causes a lot of issues with leaked data sitting on
disk that users are not able to clean up themselves.
This is particularly a problem while the `EBUSY` errors on remove are so
prevalent. So for now let's not keep this behavior.
Signed-off-by: Brian Goff <cpuguy83@gmail.com>
... | ... |
@@ -12,6 +12,7 @@ import ( |
12 | 12 |
"github.com/docker/docker/api/types" |
13 | 13 |
"github.com/docker/docker/container" |
14 | 14 |
"github.com/docker/docker/layer" |
15 |
+ "github.com/docker/docker/pkg/system" |
|
15 | 16 |
volumestore "github.com/docker/docker/volume/store" |
16 | 17 |
"github.com/pkg/errors" |
17 | 18 |
) |
... | ... |
@@ -111,37 +112,30 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, forceRemo |
111 | 111 |
logrus.Errorf("Error saving dying container to disk: %v", err) |
112 | 112 |
} |
113 | 113 |
|
114 |
- // If force removal is required, delete container from various |
|
115 |
- // indexes even if removal failed. |
|
116 |
- defer func() { |
|
117 |
- if err == nil || forceRemove { |
|
118 |
- daemon.nameIndex.Delete(container.ID) |
|
119 |
- daemon.linkIndex.delete(container) |
|
120 |
- selinuxFreeLxcContexts(container.ProcessLabel) |
|
121 |
- daemon.idIndex.Delete(container.ID) |
|
122 |
- daemon.containers.Delete(container.ID) |
|
123 |
- if e := daemon.removeMountPoints(container, removeVolume); e != nil { |
|
124 |
- logrus.Error(e) |
|
125 |
- } |
|
126 |
- daemon.LogContainerEvent(container, "destroy") |
|
127 |
- stateCtr.del(container.ID) |
|
128 |
- } |
|
129 |
- }() |
|
130 |
- |
|
131 |
- if err = os.RemoveAll(container.Root); err != nil { |
|
132 |
- return fmt.Errorf("Unable to remove filesystem for %v: %v", container.ID, err) |
|
133 |
- } |
|
134 |
- |
|
135 | 114 |
// When container creation fails and `RWLayer` has not been created yet, we |
136 | 115 |
// do not call `ReleaseRWLayer` |
137 | 116 |
if container.RWLayer != nil { |
138 | 117 |
metadata, err := daemon.layerStore.ReleaseRWLayer(container.RWLayer) |
139 | 118 |
layer.LogReleaseMetadata(metadata) |
140 | 119 |
if err != nil && err != layer.ErrMountDoesNotExist { |
141 |
- return fmt.Errorf("Driver %s failed to remove root filesystem %s: %s", daemon.GraphDriverName(), container.ID, err) |
|
120 |
+ return errors.Wrapf(err, "driver %q failed to remove root filesystem for %s", daemon.GraphDriverName(), container.ID) |
|
142 | 121 |
} |
143 | 122 |
} |
144 | 123 |
|
124 |
+ if err := system.EnsureRemoveAll(container.Root); err != nil { |
|
125 |
+ return errors.Wrapf(err, "unable to remove filesystem for %s", container.ID) |
|
126 |
+ } |
|
127 |
+ |
|
128 |
+ daemon.nameIndex.Delete(container.ID) |
|
129 |
+ daemon.linkIndex.delete(container) |
|
130 |
+ selinuxFreeLxcContexts(container.ProcessLabel) |
|
131 |
+ daemon.idIndex.Delete(container.ID) |
|
132 |
+ daemon.containers.Delete(container.ID) |
|
133 |
+ if e := daemon.removeMountPoints(container, removeVolume); e != nil { |
|
134 |
+ logrus.Error(e) |
|
135 |
+ } |
|
136 |
+ stateCtr.del(container.ID) |
|
137 |
+ daemon.LogContainerEvent(container, "destroy") |
|
145 | 138 |
return nil |
146 | 139 |
} |
147 | 140 |
|
... | ... |
@@ -46,6 +46,7 @@ import ( |
46 | 46 |
"github.com/docker/docker/pkg/idtools" |
47 | 47 |
"github.com/docker/docker/pkg/locker" |
48 | 48 |
mountpk "github.com/docker/docker/pkg/mount" |
49 |
+ "github.com/docker/docker/pkg/system" |
|
49 | 50 |
|
50 | 51 |
rsystem "github.com/opencontainers/runc/libcontainer/system" |
51 | 52 |
"github.com/opencontainers/selinux/go-selinux/label" |
... | ... |
@@ -319,13 +320,13 @@ func (a *Driver) Remove(id string) error { |
319 | 319 |
} |
320 | 320 |
return err |
321 | 321 |
} |
322 |
- defer os.RemoveAll(tmpMntPath) |
|
322 |
+ defer system.EnsureRemoveAll(tmpMntPath) |
|
323 | 323 |
|
324 | 324 |
tmpDiffpath := path.Join(a.diffPath(), fmt.Sprintf("%s-removing", id)) |
325 | 325 |
if err := os.Rename(a.getDiffPath(id), tmpDiffpath); err != nil && !os.IsNotExist(err) { |
326 | 326 |
return err |
327 | 327 |
} |
328 |
- defer os.RemoveAll(tmpDiffpath) |
|
328 |
+ defer system.EnsureRemoveAll(tmpDiffpath) |
|
329 | 329 |
|
330 | 330 |
// Remove the layers file for the id |
331 | 331 |
if err := os.Remove(path.Join(a.rootPath(), "layers", id)); err != nil && !os.IsNotExist(err) { |
... | ... |
@@ -27,6 +27,7 @@ import ( |
27 | 27 |
"github.com/docker/docker/pkg/idtools" |
28 | 28 |
"github.com/docker/docker/pkg/mount" |
29 | 29 |
"github.com/docker/docker/pkg/parsers" |
30 |
+ "github.com/docker/docker/pkg/system" |
|
30 | 31 |
"github.com/docker/go-units" |
31 | 32 |
"github.com/opencontainers/selinux/go-selinux/label" |
32 | 33 |
) |
... | ... |
@@ -535,7 +536,7 @@ func (d *Driver) Remove(id string) error { |
535 | 535 |
if err := subvolDelete(d.subvolumesDir(), id); err != nil { |
536 | 536 |
return err |
537 | 537 |
} |
538 |
- if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) { |
|
538 |
+ if err := system.EnsureRemoveAll(dir); err != nil { |
|
539 | 539 |
return err |
540 | 540 |
} |
541 | 541 |
if err := d.subvolRescanQuota(); err != nil { |
... | ... |
@@ -16,6 +16,7 @@ import ( |
16 | 16 |
"github.com/docker/docker/pkg/idtools" |
17 | 17 |
"github.com/docker/docker/pkg/locker" |
18 | 18 |
"github.com/docker/docker/pkg/mount" |
19 |
+ "github.com/docker/docker/pkg/system" |
|
19 | 20 |
units "github.com/docker/go-units" |
20 | 21 |
) |
21 | 22 |
|
... | ... |
@@ -160,7 +161,7 @@ func (d *Driver) Remove(id string) error { |
160 | 160 |
} |
161 | 161 |
|
162 | 162 |
mp := path.Join(d.home, "mnt", id) |
163 |
- if err := os.RemoveAll(mp); err != nil && !os.IsNotExist(err) { |
|
163 |
+ if err := system.EnsureRemoveAll(mp); err != nil { |
|
164 | 164 |
return err |
165 | 165 |
} |
166 | 166 |
|
... | ... |
@@ -21,6 +21,7 @@ import ( |
21 | 21 |
"github.com/docker/docker/pkg/idtools" |
22 | 22 |
"github.com/docker/docker/pkg/locker" |
23 | 23 |
"github.com/docker/docker/pkg/mount" |
24 |
+ "github.com/docker/docker/pkg/system" |
|
24 | 25 |
"github.com/opencontainers/selinux/go-selinux/label" |
25 | 26 |
) |
26 | 27 |
|
... | ... |
@@ -339,10 +340,7 @@ func (d *Driver) dir(id string) string { |
339 | 339 |
func (d *Driver) Remove(id string) error { |
340 | 340 |
d.locker.Lock(id) |
341 | 341 |
defer d.locker.Unlock(id) |
342 |
- if err := os.RemoveAll(d.dir(id)); err != nil && !os.IsNotExist(err) { |
|
343 |
- return err |
|
344 |
- } |
|
345 |
- return nil |
|
342 |
+ return system.EnsureRemoveAll(d.dir(id)) |
|
346 | 343 |
} |
347 | 344 |
|
348 | 345 |
// Get creates and mounts the required file system for the given id and returns the mount path. |
... | ... |
@@ -31,6 +31,7 @@ import ( |
31 | 31 |
"github.com/docker/docker/pkg/mount" |
32 | 32 |
"github.com/docker/docker/pkg/parsers" |
33 | 33 |
"github.com/docker/docker/pkg/parsers/kernel" |
34 |
+ "github.com/docker/docker/pkg/system" |
|
34 | 35 |
units "github.com/docker/go-units" |
35 | 36 |
|
36 | 37 |
"github.com/opencontainers/selinux/go-selinux/label" |
... | ... |
@@ -464,7 +465,7 @@ func (d *Driver) Remove(id string) error { |
464 | 464 |
} |
465 | 465 |
} |
466 | 466 |
|
467 |
- if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) { |
|
467 |
+ if err := system.EnsureRemoveAll(dir); err != nil && !os.IsNotExist(err) { |
|
468 | 468 |
return err |
469 | 469 |
} |
470 | 470 |
return nil |
... | ... |
@@ -8,6 +8,7 @@ import ( |
8 | 8 |
"github.com/docker/docker/daemon/graphdriver" |
9 | 9 |
"github.com/docker/docker/pkg/chrootarchive" |
10 | 10 |
"github.com/docker/docker/pkg/idtools" |
11 |
+ "github.com/docker/docker/pkg/system" |
|
11 | 12 |
|
12 | 13 |
"github.com/opencontainers/selinux/go-selinux/label" |
13 | 14 |
) |
... | ... |
@@ -114,7 +115,7 @@ func (d *Driver) dir(id string) string { |
114 | 114 |
|
115 | 115 |
// Remove deletes the content from the directory for a given id. |
116 | 116 |
func (d *Driver) Remove(id string) error { |
117 |
- if err := os.RemoveAll(d.dir(id)); err != nil && !os.IsNotExist(err) { |
|
117 |
+ if err := system.EnsureRemoveAll(d.dir(id)); err != nil { |
|
118 | 118 |
return err |
119 | 119 |
} |
120 | 120 |
return nil |
... | ... |
@@ -41,17 +41,6 @@ func (daemon *Daemon) StateChanged(id string, e libcontainerd.StateInfo) error { |
41 | 41 |
daemon.updateHealthMonitor(c) |
42 | 42 |
daemon.LogContainerEvent(c, "oom") |
43 | 43 |
case libcontainerd.StateExit: |
44 |
- // if container's AutoRemove flag is set, remove it after clean up |
|
45 |
- autoRemove := func() { |
|
46 |
- c.Lock() |
|
47 |
- ar := c.HostConfig.AutoRemove |
|
48 |
- c.Unlock() |
|
49 |
- if ar { |
|
50 |
- if err := daemon.ContainerRm(c.ID, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil { |
|
51 |
- logrus.Errorf("can't remove container %s: %v", c.ID, err) |
|
52 |
- } |
|
53 |
- } |
|
54 |
- } |
|
55 | 44 |
|
56 | 45 |
c.Lock() |
57 | 46 |
c.StreamConfig.Wait() |
... | ... |
@@ -63,7 +52,7 @@ func (daemon *Daemon) StateChanged(id string, e libcontainerd.StateInfo) error { |
63 | 63 |
c.SetRestarting(platformConstructExitStatus(e)) |
64 | 64 |
} else { |
65 | 65 |
c.SetStopped(platformConstructExitStatus(e)) |
66 |
- defer autoRemove() |
|
66 |
+ defer daemon.autoRemove(c) |
|
67 | 67 |
} |
68 | 68 |
|
69 | 69 |
// cancel healthcheck here, they will be automatically |
... | ... |
@@ -85,7 +74,7 @@ func (daemon *Daemon) StateChanged(id string, e libcontainerd.StateInfo) error { |
85 | 85 |
} |
86 | 86 |
if err != nil { |
87 | 87 |
c.SetStopped(platformConstructExitStatus(e)) |
88 |
- defer autoRemove() |
|
88 |
+ defer daemon.autoRemove(c) |
|
89 | 89 |
if err != restartmanager.ErrRestartCanceled { |
90 | 90 |
logrus.Errorf("restartmanger wait error: %+v", err) |
91 | 91 |
} |
... | ... |
@@ -153,3 +142,24 @@ func (daemon *Daemon) StateChanged(id string, e libcontainerd.StateInfo) error { |
153 | 153 |
} |
154 | 154 |
return nil |
155 | 155 |
} |
156 |
+ |
|
157 |
+func (daemon *Daemon) autoRemove(c *container.Container) { |
|
158 |
+ c.Lock() |
|
159 |
+ ar := c.HostConfig.AutoRemove |
|
160 |
+ c.Unlock() |
|
161 |
+ if !ar { |
|
162 |
+ return |
|
163 |
+ } |
|
164 |
+ |
|
165 |
+ var err error |
|
166 |
+ if err = daemon.ContainerRm(c.ID, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err == nil { |
|
167 |
+ return |
|
168 |
+ } |
|
169 |
+ if c := daemon.containers.Get(c.ID); c == nil { |
|
170 |
+ return |
|
171 |
+ } |
|
172 |
+ |
|
173 |
+ if err != nil { |
|
174 |
+ logrus.WithError(err).WithField("container", c.ID).Error("error removing container") |
|
175 |
+ } |
|
176 |
+} |
... | ... |
@@ -1,5 +1,10 @@ |
1 | 1 |
package mount |
2 | 2 |
|
3 |
+import ( |
|
4 |
+ "sort" |
|
5 |
+ "strings" |
|
6 |
+) |
|
7 |
+ |
|
3 | 8 |
// GetMounts retrieves a list of mounts for the current running process. |
4 | 9 |
func GetMounts() ([]*Info, error) { |
5 | 10 |
return parseMountTable() |
... | ... |
@@ -53,3 +58,29 @@ func Unmount(target string) error { |
53 | 53 |
} |
54 | 54 |
return unmount(target, mntDetach) |
55 | 55 |
} |
56 |
+ |
|
57 |
+// RecursiveUnmount unmounts the target and all mounts underneath, starting with |
|
58 |
+// the deepsest mount first. |
|
59 |
+func RecursiveUnmount(target string) error { |
|
60 |
+ mounts, err := GetMounts() |
|
61 |
+ if err != nil { |
|
62 |
+ return err |
|
63 |
+ } |
|
64 |
+ |
|
65 |
+ // Make the deepest mount be first |
|
66 |
+ sort.Sort(sort.Reverse(byMountpoint(mounts))) |
|
67 |
+ |
|
68 |
+ for i, m := range mounts { |
|
69 |
+ if !strings.HasPrefix(m.Mountpoint, target) { |
|
70 |
+ continue |
|
71 |
+ } |
|
72 |
+ if err := Unmount(m.Mountpoint); err != nil && i == len(mounts)-1 { |
|
73 |
+ if mounted, err := Mounted(m.Mountpoint); err != nil || mounted { |
|
74 |
+ return err |
|
75 |
+ } |
|
76 |
+ // Ignore errors for submounts and continue trying to unmount others |
|
77 |
+ // The final unmount should fail if there ane any submounts remaining |
|
78 |
+ } |
|
79 |
+ } |
|
80 |
+ return nil |
|
81 |
+} |
... | ... |
@@ -38,3 +38,17 @@ type Info struct { |
38 | 38 |
// VfsOpts represents per super block options. |
39 | 39 |
VfsOpts string |
40 | 40 |
} |
41 |
+ |
|
42 |
+type byMountpoint []*Info |
|
43 |
+ |
|
44 |
+func (by byMountpoint) Len() int { |
|
45 |
+ return len(by) |
|
46 |
+} |
|
47 |
+ |
|
48 |
+func (by byMountpoint) Less(i, j int) bool { |
|
49 |
+ return by[i].Mountpoint < by[j].Mountpoint |
|
50 |
+} |
|
51 |
+ |
|
52 |
+func (by byMountpoint) Swap(i, j int) { |
|
53 |
+ by[i], by[j] = by[j], by[i] |
|
54 |
+} |
41 | 55 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,80 @@ |
0 |
+package system |
|
1 |
+ |
|
2 |
+import ( |
|
3 |
+ "os" |
|
4 |
+ "syscall" |
|
5 |
+ "time" |
|
6 |
+ |
|
7 |
+ "github.com/docker/docker/pkg/mount" |
|
8 |
+ "github.com/pkg/errors" |
|
9 |
+) |
|
10 |
+ |
|
11 |
+// EnsureRemoveAll wraps `os.RemoveAll` to check for specific errors that can |
|
12 |
+// often be remedied. |
|
13 |
+// Only use `EnsureRemoveAll` if you really want to make every effort to remove |
|
14 |
+// a directory. |
|
15 |
+// |
|
16 |
+// Because of the way `os.Remove` (and by extension `os.RemoveAll`) works, there |
|
17 |
+// can be a race between reading directory entries and then actually attempting |
|
18 |
+// to remove everything in the directory. |
|
19 |
+// These types of errors do not need to be returned since it's ok for the dir to |
|
20 |
+// be gone we can just retry the remove operation. |
|
21 |
+// |
|
22 |
+// This should not return a `os.ErrNotExist` kind of error under any cirucmstances |
|
23 |
+func EnsureRemoveAll(dir string) error { |
|
24 |
+ notExistErr := make(map[string]bool) |
|
25 |
+ |
|
26 |
+ // track retries |
|
27 |
+ exitOnErr := make(map[string]int) |
|
28 |
+ maxRetry := 5 |
|
29 |
+ |
|
30 |
+ // Attempt to unmount anything beneath this dir first |
|
31 |
+ mount.RecursiveUnmount(dir) |
|
32 |
+ |
|
33 |
+ for { |
|
34 |
+ err := os.RemoveAll(dir) |
|
35 |
+ if err == nil { |
|
36 |
+ return err |
|
37 |
+ } |
|
38 |
+ |
|
39 |
+ pe, ok := err.(*os.PathError) |
|
40 |
+ if !ok { |
|
41 |
+ return err |
|
42 |
+ } |
|
43 |
+ |
|
44 |
+ if os.IsNotExist(err) { |
|
45 |
+ if notExistErr[pe.Path] { |
|
46 |
+ return err |
|
47 |
+ } |
|
48 |
+ notExistErr[pe.Path] = true |
|
49 |
+ |
|
50 |
+ // There is a race where some subdir can be removed but after the parent |
|
51 |
+ // dir entries have been read. |
|
52 |
+ // So the path could be from `os.Remove(subdir)` |
|
53 |
+ // If the reported non-existent path is not the passed in `dir` we |
|
54 |
+ // should just retry, but otherwise return with no error. |
|
55 |
+ if pe.Path == dir { |
|
56 |
+ return nil |
|
57 |
+ } |
|
58 |
+ continue |
|
59 |
+ } |
|
60 |
+ |
|
61 |
+ if pe.Err != syscall.EBUSY { |
|
62 |
+ return err |
|
63 |
+ } |
|
64 |
+ |
|
65 |
+ if mounted, _ := mount.Mounted(pe.Path); mounted { |
|
66 |
+ if e := mount.Unmount(pe.Path); e != nil { |
|
67 |
+ if mounted, _ := mount.Mounted(pe.Path); mounted { |
|
68 |
+ return errors.Wrapf(e, "error while removing %s", dir) |
|
69 |
+ } |
|
70 |
+ } |
|
71 |
+ } |
|
72 |
+ |
|
73 |
+ if exitOnErr[pe.Path] == maxRetry { |
|
74 |
+ return err |
|
75 |
+ } |
|
76 |
+ exitOnErr[pe.Path]++ |
|
77 |
+ time.Sleep(100 * time.Millisecond) |
|
78 |
+ } |
|
79 |
+} |
0 | 80 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,84 @@ |
0 |
+package system |
|
1 |
+ |
|
2 |
+import ( |
|
3 |
+ "io/ioutil" |
|
4 |
+ "os" |
|
5 |
+ "path/filepath" |
|
6 |
+ "runtime" |
|
7 |
+ "testing" |
|
8 |
+ "time" |
|
9 |
+ |
|
10 |
+ "github.com/docker/docker/pkg/mount" |
|
11 |
+) |
|
12 |
+ |
|
13 |
+func TestEnsureRemoveAllNotExist(t *testing.T) { |
|
14 |
+ // should never return an error for a non-existent path |
|
15 |
+ if err := EnsureRemoveAll("/non/existent/path"); err != nil { |
|
16 |
+ t.Fatal(err) |
|
17 |
+ } |
|
18 |
+} |
|
19 |
+ |
|
20 |
+func TestEnsureRemoveAllWithDir(t *testing.T) { |
|
21 |
+ dir, err := ioutil.TempDir("", "test-ensure-removeall-with-dir") |
|
22 |
+ if err != nil { |
|
23 |
+ t.Fatal(err) |
|
24 |
+ } |
|
25 |
+ if err := EnsureRemoveAll(dir); err != nil { |
|
26 |
+ t.Fatal(err) |
|
27 |
+ } |
|
28 |
+} |
|
29 |
+ |
|
30 |
+func TestEnsureRemoveAllWithFile(t *testing.T) { |
|
31 |
+ tmp, err := ioutil.TempFile("", "test-ensure-removeall-with-dir") |
|
32 |
+ if err != nil { |
|
33 |
+ t.Fatal(err) |
|
34 |
+ } |
|
35 |
+ tmp.Close() |
|
36 |
+ if err := EnsureRemoveAll(tmp.Name()); err != nil { |
|
37 |
+ t.Fatal(err) |
|
38 |
+ } |
|
39 |
+} |
|
40 |
+ |
|
41 |
+func TestEnsureRemoveAllWithMount(t *testing.T) { |
|
42 |
+ if runtime.GOOS == "windows" { |
|
43 |
+ t.Skip("mount not supported on Windows") |
|
44 |
+ } |
|
45 |
+ |
|
46 |
+ dir1, err := ioutil.TempDir("", "test-ensure-removeall-with-dir1") |
|
47 |
+ if err != nil { |
|
48 |
+ t.Fatal(err) |
|
49 |
+ } |
|
50 |
+ dir2, err := ioutil.TempDir("", "test-ensure-removeall-with-dir2") |
|
51 |
+ if err != nil { |
|
52 |
+ t.Fatal(err) |
|
53 |
+ } |
|
54 |
+ defer os.RemoveAll(dir2) |
|
55 |
+ |
|
56 |
+ bindDir := filepath.Join(dir1, "bind") |
|
57 |
+ if err := os.MkdirAll(bindDir, 0755); err != nil { |
|
58 |
+ t.Fatal(err) |
|
59 |
+ } |
|
60 |
+ |
|
61 |
+ if err := mount.Mount(dir2, bindDir, "none", "bind"); err != nil { |
|
62 |
+ t.Fatal(err) |
|
63 |
+ } |
|
64 |
+ |
|
65 |
+ done := make(chan struct{}) |
|
66 |
+ go func() { |
|
67 |
+ err = EnsureRemoveAll(dir1) |
|
68 |
+ close(done) |
|
69 |
+ }() |
|
70 |
+ |
|
71 |
+ select { |
|
72 |
+ case <-done: |
|
73 |
+ if err != nil { |
|
74 |
+ t.Fatal(err) |
|
75 |
+ } |
|
76 |
+ case <-time.After(5 * time.Second): |
|
77 |
+ t.Fatal("timeout waiting for EnsureRemoveAll to finish") |
|
78 |
+ } |
|
79 |
+ |
|
80 |
+ if _, err := os.Stat(dir1); !os.IsNotExist(err) { |
|
81 |
+ t.Fatalf("expected %q to not exist", dir1) |
|
82 |
+ } |
|
83 |
+} |