- Print the mount table as in /proc/self/mountinfo
- Do not exit prematurely when one of the ipc mounts doesn't exist.
- Do not exit prematurely when one of the ipc mounts cannot be unmounted.
- Add a unit test to see if the cleanup really works.
- Use syscall.MNT_DETACH to cleanup mounts after a crash.
- Unmount IPC mounts when the daemon unregisters an old running container.
Signed-off-by: David Calavera <david.calavera@gmail.com>
| ... | ... |
@@ -296,7 +296,7 @@ func (container *Container) Start() (err error) {
|
| 296 | 296 |
return err |
| 297 | 297 |
} |
| 298 | 298 |
|
| 299 |
- if !(container.hostConfig.IpcMode.IsContainer() || container.hostConfig.IpcMode.IsHost()) {
|
|
| 299 |
+ if !container.hostConfig.IpcMode.IsContainer() && !container.hostConfig.IpcMode.IsHost() {
|
|
| 300 | 300 |
if err := container.setupIpcDirs(); err != nil {
|
| 301 | 301 |
return err |
| 302 | 302 |
} |
| ... | ... |
@@ -366,11 +366,11 @@ func (container *Container) cleanup() {
|
| 366 | 366 |
container.releaseNetwork() |
| 367 | 367 |
|
| 368 | 368 |
if err := container.unmountIpcMounts(); err != nil {
|
| 369 |
- logrus.Errorf("%v: Failed to umount ipc filesystems: %v", container.ID, err)
|
|
| 369 |
+ logrus.Errorf("%s: Failed to umount ipc filesystems: %v", container.ID, err)
|
|
| 370 | 370 |
} |
| 371 | 371 |
|
| 372 | 372 |
if err := container.Unmount(); err != nil {
|
| 373 |
- logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err)
|
|
| 373 |
+ logrus.Errorf("%s: Failed to umount filesystem: %v", container.ID, err)
|
|
| 374 | 374 |
} |
| 375 | 375 |
|
| 376 | 376 |
for _, eConfig := range container.execCommands.s {
|
| ... | ... |
@@ -1242,10 +1242,10 @@ func (container *Container) removeMountPoints(rm bool) error {
|
| 1242 | 1242 |
} |
| 1243 | 1243 |
|
| 1244 | 1244 |
func (container *Container) shmPath() (string, error) {
|
| 1245 |
- return container.GetRootResourcePath("shm")
|
|
| 1245 |
+ return container.getRootResourcePath("shm")
|
|
| 1246 | 1246 |
} |
| 1247 | 1247 |
func (container *Container) mqueuePath() (string, error) {
|
| 1248 |
- return container.GetRootResourcePath("mqueue")
|
|
| 1248 |
+ return container.getRootResourcePath("mqueue")
|
|
| 1249 | 1249 |
} |
| 1250 | 1250 |
|
| 1251 | 1251 |
func (container *Container) setupIpcDirs() error {
|
| ... | ... |
@@ -1258,7 +1258,7 @@ func (container *Container) setupIpcDirs() error {
|
| 1258 | 1258 |
return err |
| 1259 | 1259 |
} |
| 1260 | 1260 |
|
| 1261 |
- if err := syscall.Mount("shm", shmPath, "tmpfs", uintptr(syscall.MS_NOEXEC|syscall.MS_NOSUID|syscall.MS_NODEV), label.FormatMountLabel("mode=1777,size=65536k", container.GetMountLabel())); err != nil {
|
|
| 1261 |
+ if err := syscall.Mount("shm", shmPath, "tmpfs", uintptr(syscall.MS_NOEXEC|syscall.MS_NOSUID|syscall.MS_NODEV), label.FormatMountLabel("mode=1777,size=65536k", container.getMountLabel())); err != nil {
|
|
| 1262 | 1262 |
return fmt.Errorf("mounting shm tmpfs: %s", err)
|
| 1263 | 1263 |
} |
| 1264 | 1264 |
|
| ... | ... |
@@ -1283,22 +1283,32 @@ func (container *Container) unmountIpcMounts() error {
|
| 1283 | 1283 |
return nil |
| 1284 | 1284 |
} |
| 1285 | 1285 |
|
| 1286 |
+ var errors []string |
|
| 1286 | 1287 |
shmPath, err := container.shmPath() |
| 1287 | 1288 |
if err != nil {
|
| 1288 |
- return fmt.Errorf("shm path does not exist %v", err)
|
|
| 1289 |
- } |
|
| 1289 |
+ logrus.Error(err) |
|
| 1290 |
+ errors = append(errors, err.Error()) |
|
| 1291 |
+ } else {
|
|
| 1292 |
+ if err := detachMounted(shmPath); err != nil {
|
|
| 1293 |
+ logrus.Errorf("failed to umount %s: %v", shmPath, err)
|
|
| 1294 |
+ errors = append(errors, err.Error()) |
|
| 1295 |
+ } |
|
| 1290 | 1296 |
|
| 1291 |
- if err := syscall.Unmount(shmPath, syscall.MNT_DETACH); err != nil {
|
|
| 1292 |
- return fmt.Errorf("failed to umount %s filesystem %v", shmPath, err)
|
|
| 1293 | 1297 |
} |
| 1294 | 1298 |
|
| 1295 | 1299 |
mqueuePath, err := container.mqueuePath() |
| 1296 | 1300 |
if err != nil {
|
| 1297 |
- return fmt.Errorf("mqueue path does not exist %v", err)
|
|
| 1301 |
+ logrus.Error(err) |
|
| 1302 |
+ errors = append(errors, err.Error()) |
|
| 1303 |
+ } else {
|
|
| 1304 |
+ if err := detachMounted(mqueuePath); err != nil {
|
|
| 1305 |
+ logrus.Errorf("failed to umount %s: %v", mqueuePath, err)
|
|
| 1306 |
+ errors = append(errors, err.Error()) |
|
| 1307 |
+ } |
|
| 1298 | 1308 |
} |
| 1299 | 1309 |
|
| 1300 |
- if err := syscall.Unmount(mqueuePath, syscall.MNT_DETACH); err != nil {
|
|
| 1301 |
- return fmt.Errorf("failed to umount %s filesystem %v", mqueuePath, err)
|
|
| 1310 |
+ if len(errors) > 0 {
|
|
| 1311 |
+ return fmt.Errorf("failed to cleanup ipc mounts:\n%v", strings.Join(errors, "\n"))
|
|
| 1302 | 1312 |
} |
| 1303 | 1313 |
|
| 1304 | 1314 |
return nil |
| ... | ... |
@@ -1322,3 +1332,7 @@ func (container *Container) ipcMounts() []execdriver.Mount {
|
| 1322 | 1322 |
}) |
| 1323 | 1323 |
return mounts |
| 1324 | 1324 |
} |
| 1325 |
+ |
|
| 1326 |
+func detachMounted(path string) error {
|
|
| 1327 |
+ return syscall.Unmount(path, syscall.MNT_DETACH) |
|
| 1328 |
+} |
| ... | ... |
@@ -209,6 +209,9 @@ func (daemon *Daemon) Register(container *Container) error {
|
| 209 | 209 |
} |
| 210 | 210 |
daemon.execDriver.Terminate(cmd) |
| 211 | 211 |
|
| 212 |
+ if err := container.unmountIpcMounts(); err != nil {
|
|
| 213 |
+ logrus.Errorf("%s: Failed to umount ipc filesystems: %v", container.ID, err)
|
|
| 214 |
+ } |
|
| 212 | 215 |
if err := container.Unmount(); err != nil {
|
| 213 | 216 |
logrus.Debugf("unmount error %s", err)
|
| 214 | 217 |
} |
| ... | ... |
@@ -756,13 +759,14 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo |
| 756 | 756 |
d.EventsService = eventsService |
| 757 | 757 |
d.volumes = volStore |
| 758 | 758 |
d.root = config.Root |
| 759 |
- go d.execCommandGC() |
|
| 760 | 759 |
|
| 761 |
- if err := d.restore(); err != nil {
|
|
| 760 |
+ if err := d.cleanupMounts(); err != nil {
|
|
| 762 | 761 |
return nil, err |
| 763 | 762 |
} |
| 764 | 763 |
|
| 765 |
- if err := d.cleanupMounts(); err != nil {
|
|
| 764 |
+ go d.execCommandGC() |
|
| 765 |
+ |
|
| 766 |
+ if err := d.restore(); err != nil {
|
|
| 766 | 767 |
return nil, err |
| 767 | 768 |
} |
| 768 | 769 |
|
| ... | ... |
@@ -2,12 +2,13 @@ package daemon |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 | 4 |
"bufio" |
| 5 |
+ "fmt" |
|
| 6 |
+ "io" |
|
| 5 | 7 |
"os" |
| 6 | 8 |
"path/filepath" |
| 7 | 9 |
"strings" |
| 8 | 10 |
|
| 9 | 11 |
"github.com/Sirupsen/logrus" |
| 10 |
- "github.com/docker/docker/pkg/mount" |
|
| 11 | 12 |
) |
| 12 | 13 |
|
| 13 | 14 |
// cleanupMounts umounts shm/mqueue mounts for old containers |
| ... | ... |
@@ -19,17 +20,24 @@ func (daemon *Daemon) cleanupMounts() error {
|
| 19 | 19 |
} |
| 20 | 20 |
defer f.Close() |
| 21 | 21 |
|
| 22 |
- sc := bufio.NewScanner(f) |
|
| 22 |
+ return daemon.cleanupMountsFromReader(f, detachMounted) |
|
| 23 |
+} |
|
| 24 |
+ |
|
| 25 |
+func (daemon *Daemon) cleanupMountsFromReader(reader io.Reader, unmount func(target string) error) error {
|
|
| 26 |
+ sc := bufio.NewScanner(reader) |
|
| 27 |
+ var errors []string |
|
| 23 | 28 |
for sc.Scan() {
|
| 24 | 29 |
line := sc.Text() |
| 25 |
- fields := strings.Split(line, " ") |
|
| 30 |
+ fields := strings.Fields(line) |
|
| 26 | 31 |
if strings.HasPrefix(fields[4], daemon.repository) {
|
| 32 |
+ logrus.Debugf("Mount base: %v, repository %s", fields[4], daemon.repository)
|
|
| 27 | 33 |
mnt := fields[4] |
| 28 | 34 |
mountBase := filepath.Base(mnt) |
| 29 | 35 |
if mountBase == "mqueue" || mountBase == "shm" {
|
| 30 |
- logrus.Debugf("Unmounting %+v", mnt)
|
|
| 31 |
- if err := mount.Unmount(mnt); err != nil {
|
|
| 32 |
- return err |
|
| 36 |
+ logrus.Debugf("Unmounting %v", mnt)
|
|
| 37 |
+ if err := unmount(mnt); err != nil {
|
|
| 38 |
+ logrus.Error(err) |
|
| 39 |
+ errors = append(errors, err.Error()) |
|
| 33 | 40 |
} |
| 34 | 41 |
} |
| 35 | 42 |
} |
| ... | ... |
@@ -39,6 +47,10 @@ func (daemon *Daemon) cleanupMounts() error {
|
| 39 | 39 |
return err |
| 40 | 40 |
} |
| 41 | 41 |
|
| 42 |
+ if len(errors) > 0 {
|
|
| 43 |
+ return fmt.Errorf("Error cleaningup mounts:\n%v", strings.Join(errors, "\n"))
|
|
| 44 |
+ } |
|
| 45 |
+ |
|
| 42 | 46 |
logrus.Debugf("Cleaning up old shm/mqueue mounts: done.")
|
| 43 | 47 |
return nil |
| 44 | 48 |
} |
| 45 | 49 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,58 @@ |
| 0 |
+// +build linux |
|
| 1 |
+ |
|
| 2 |
+package daemon |
|
| 3 |
+ |
|
| 4 |
+import ( |
|
| 5 |
+ "strings" |
|
| 6 |
+ "testing" |
|
| 7 |
+) |
|
| 8 |
+ |
|
| 9 |
+func TestCleanupMounts(t *testing.T) {
|
|
| 10 |
+ fixture := `230 138 0:60 / / rw,relatime - overlay overlay rw,lowerdir=/var/lib/docker/overlay/0ef9f93d5d365c1385b09d54bbee6afff3d92002c16f22eccb6e1549b2ff97d8/root,upperdir=/var/lib/docker/overlay/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb/upper,workdir=/var/lib/docker/overlay/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb/work |
|
| 11 |
+231 230 0:56 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw |
|
| 12 |
+232 230 0:57 / /dev rw,nosuid - tmpfs tmpfs rw,mode=755 |
|
| 13 |
+233 232 0:58 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 |
|
| 14 |
+234 232 0:59 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k |
|
| 15 |
+235 232 0:55 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw |
|
| 16 |
+236 230 0:61 / /sys rw,nosuid,nodev,noexec,relatime - sysfs sysfs rw |
|
| 17 |
+237 236 0:62 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw |
|
| 18 |
+238 237 0:21 /system.slice/docker.service /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd |
|
| 19 |
+239 237 0:23 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/perf_event rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,perf_event |
|
| 20 |
+240 237 0:24 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpuset,clone_children |
|
| 21 |
+241 237 0:25 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,devices |
|
| 22 |
+242 237 0:26 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/freezer rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,freezer |
|
| 23 |
+243 237 0:27 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpu,cpuacct |
|
| 24 |
+244 237 0:28 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/blkio rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,blkio |
|
| 25 |
+245 237 0:29 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/net_cls,net_prio rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,net_cls,net_prio |
|
| 26 |
+246 237 0:30 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/hugetlb rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,hugetlb |
|
| 27 |
+247 237 0:31 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,memory |
|
| 28 |
+248 230 253:1 /var/lib/docker/volumes/510cc41ac68c48bd4eac932e3e09711673876287abf1b185312cfbfe6261a111/_data /var/lib/docker rw,relatime - ext4 /dev/disk/by-uuid/ba70ea0c-1a8f-4ee4-9687-cb393730e2b5 rw,errors=remount-ro,data=ordered |
|
| 29 |
+250 230 253:1 /var/lib/docker/containers/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb/hostname /etc/hostname rw,relatime - ext4 /dev/disk/by-uuid/ba70ea0c-1a8f-4ee4-9687-cb393730e2b5 rw,errors=remount-ro,data=ordered |
|
| 30 |
+251 230 253:1 /var/lib/docker/containers/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb/hosts /etc/hosts rw,relatime - ext4 /dev/disk/by-uuid/ba70ea0c-1a8f-4ee4-9687-cb393730e2b5 rw,errors=remount-ro,data=ordered |
|
| 31 |
+252 232 0:13 /1 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=000 |
|
| 32 |
+139 236 0:11 / /sys/kernel/security rw,relatime - securityfs none rw |
|
| 33 |
+140 230 0:54 / /tmp rw,relatime - tmpfs none rw |
|
| 34 |
+145 230 0:3 / /run/docker/netns/default rw - nsfs nsfs rw |
|
| 35 |
+130 140 0:45 / /tmp/docker_recursive_mount_test312125472/tmpfs rw,relatime - tmpfs tmpfs rw |
|
| 36 |
+131 230 0:3 / /run/docker/netns/47903e2e6701 rw - nsfs nsfs rw |
|
| 37 |
+133 230 0:55 / /go/src/github.com/docker/docker/bundles/1.9.0-dev/test-integration-cli/d45526097/graph/containers/47903e2e67014246eba27607809d5f5c2437c3bf84c2986393448f84093cc40b/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw` |
|
| 38 |
+ |
|
| 39 |
+ d := &Daemon{
|
|
| 40 |
+ repository: "/go/src/github.com/docker/docker/bundles/1.9.0-dev/test-integration-cli/d45526097/graph/containers/", |
|
| 41 |
+ } |
|
| 42 |
+ |
|
| 43 |
+ expected := "/go/src/github.com/docker/docker/bundles/1.9.0-dev/test-integration-cli/d45526097/graph/containers/47903e2e67014246eba27607809d5f5c2437c3bf84c2986393448f84093cc40b/mqueue" |
|
| 44 |
+ var unmounted bool |
|
| 45 |
+ unmount := func(target string) error {
|
|
| 46 |
+ if target == expected {
|
|
| 47 |
+ unmounted = true |
|
| 48 |
+ } |
|
| 49 |
+ return nil |
|
| 50 |
+ } |
|
| 51 |
+ |
|
| 52 |
+ d.cleanupMountsFromReader(strings.NewReader(fixture), unmount) |
|
| 53 |
+ |
|
| 54 |
+ if !unmounted {
|
|
| 55 |
+ t.Fatalf("Expected to unmount the mqueue")
|
|
| 56 |
+ } |
|
| 57 |
+} |
| ... | ... |
@@ -1474,9 +1474,11 @@ func (s *DockerDaemonSuite) TestCleanupMountsAfterCrash(c *check.C) {
|
| 1474 | 1474 |
id := strings.TrimSpace(out) |
| 1475 | 1475 |
c.Assert(s.d.cmd.Process.Signal(os.Kill), check.IsNil) |
| 1476 | 1476 |
c.Assert(s.d.Start(), check.IsNil) |
| 1477 |
- mountOut, err := exec.Command("mount").CombinedOutput()
|
|
| 1477 |
+ mountOut, err := ioutil.ReadFile("/proc/self/mountinfo")
|
|
| 1478 | 1478 |
c.Assert(err, check.IsNil, check.Commentf("Output: %s", mountOut))
|
| 1479 |
- c.Assert(strings.Contains(string(mountOut), id), check.Equals, false, check.Commentf("Something mounted from older daemon start: %s", mountOut))
|
|
| 1479 |
+ |
|
| 1480 |
+ comment := check.Commentf("%s is still mounted from older daemon start:\nDaemon root repository %s\n%s", id, s.d.folder, mountOut)
|
|
| 1481 |
+ c.Assert(strings.Contains(string(mountOut), id), check.Equals, false, comment) |
|
| 1480 | 1482 |
} |
| 1481 | 1483 |
|
| 1482 | 1484 |
func (s *DockerDaemonSuite) TestRunContainerWithBridgeNone(c *check.C) {
|