Browse code

Don't create source directory while the daemon is being shutdown, fix #30348

If a container mount the socket the daemon is listening on into
container while the daemon is being shutdown, the socket will
not exist on the host, then daemon will assume it's a directory
and create it on the host, this will cause the daemon can't start
next time.

fix issue https://github.com/moby/moby/issues/30348

To reproduce this issue, you can add following code

```
--- a/daemon/oci_linux.go
+++ b/daemon/oci_linux.go
@@ -8,6 +8,7 @@ import (
"sort"
"strconv"
"strings"
+ "time"

"github.com/Sirupsen/logrus"
"github.com/docker/docker/container"
@@ -666,7 +667,8 @@ func (daemon *Daemon) createSpec(c *container.Container) (*libcontainerd.Spec, e
if err := daemon.setupIpcDirs(c); err != nil {
return nil, err
}
-
+ fmt.Printf("===please stop the daemon===\n")
+ time.Sleep(time.Second * 2)
ms, err := daemon.setupMounts(c)
if err != nil {
return nil, err

```

step1 run a container which has `--restart always` and `-v /var/run/docker.sock:/sock`
```
$ docker run -ti --restart always -v /var/run/docker.sock:/sock busybox
/ #

```
step2 exit the the container
```
/ # exit
```
and kill the daemon when you see
```
===please stop the daemon===
```
in the daemon log

The daemon can't restart again and fail with `can't create unix socket /var/run/docker.sock: is a directory`.

Signed-off-by: Lei Jitang <leijitang@huawei.com>

Lei Jitang authored on 2017/05/22 16:44:01
Showing 6 changed files
... ...
@@ -155,6 +155,8 @@ func (cli *DaemonCli) start(opts daemonOptions) (err error) {
155 155
 	api := apiserver.New(serverConfig)
156 156
 	cli.api = api
157 157
 
158
+	var hosts []string
159
+
158 160
 	for i := 0; i < len(cli.Config.Hosts); i++ {
159 161
 		var err error
160 162
 		if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
... ...
@@ -186,6 +188,7 @@ func (cli *DaemonCli) start(opts daemonOptions) (err error) {
186 186
 			}
187 187
 		}
188 188
 		logrus.Debugf("Listener created for HTTP on %s (%s)", proto, addr)
189
+		hosts = append(hosts, protoAddrParts[1])
189 190
 		api.Accept(addr, ls...)
190 191
 	}
191 192
 
... ...
@@ -213,6 +216,8 @@ func (cli *DaemonCli) start(opts daemonOptions) (err error) {
213 213
 		return fmt.Errorf("Error starting daemon: %v", err)
214 214
 	}
215 215
 
216
+	d.StoreHosts(hosts)
217
+
216 218
 	// validate after NewDaemon has restored enabled plugins. Dont change order.
217 219
 	if err := validateAuthzPlugins(cli.Config.AuthorizationPlugins, pluginStore); err != nil {
218 220
 		return fmt.Errorf("Error validating authorization plugin: %v", err)
... ...
@@ -116,6 +116,17 @@ type Daemon struct {
116 116
 
117 117
 	diskUsageRunning int32
118 118
 	pruneRunning     int32
119
+	hosts            map[string]bool // hosts stores the addresses the daemon is listening on
120
+}
121
+
122
+// StoreHosts stores the addresses the daemon is listening on
123
+func (daemon *Daemon) StoreHosts(hosts []string) {
124
+	if daemon.hosts == nil {
125
+		daemon.hosts = make(map[string]bool)
126
+	}
127
+	for _, h := range hosts {
128
+		daemon.hosts[h] = true
129
+	}
119 130
 }
120 131
 
121 132
 // HasExperimental returns whether the experimental features of the daemon are enabled or not
... ...
@@ -46,7 +46,8 @@ func (daemon *Daemon) StateChanged(id string, e libcontainerd.StateInfo) error {
46 46
 		c.StreamConfig.Wait()
47 47
 		c.Reset(false)
48 48
 
49
-		restart, wait, err := c.RestartManager().ShouldRestart(e.ExitCode, c.HasBeenManuallyStopped, time.Since(c.StartedAt))
49
+		// If daemon is being shutdown, don't let the container restart
50
+		restart, wait, err := c.RestartManager().ShouldRestart(e.ExitCode, daemon.IsShuttingDown() || c.HasBeenManuallyStopped, time.Since(c.StartedAt))
50 51
 		if err == nil && restart {
51 52
 			c.RestartCount++
52 53
 			c.SetRestarting(platformConstructExitStatus(e))
... ...
@@ -6,6 +6,7 @@ package daemon
6 6
 
7 7
 import (
8 8
 	"encoding/json"
9
+	"fmt"
9 10
 	"os"
10 11
 	"path/filepath"
11 12
 	"sort"
... ...
@@ -42,8 +43,19 @@ func (daemon *Daemon) setupMounts(c *container.Container) ([]container.Mount, er
42 42
 		if err := daemon.lazyInitializeVolume(c.ID, m); err != nil {
43 43
 			return nil, err
44 44
 		}
45
+		// If the daemon is being shutdown, we should not let a container start if it is trying to
46
+		// mount the socket the daemon is listening on. During daemon shutdown, the socket
47
+		// (/var/run/docker.sock by default) doesn't exist anymore causing the call to m.Setup to
48
+		// create at directory instead. This in turn will prevent the daemon to restart.
49
+		checkfunc := func(m *volume.MountPoint) error {
50
+			if _, exist := daemon.hosts[m.Source]; exist && daemon.IsShuttingDown() {
51
+				return fmt.Errorf("Could not mount %q to container while the daemon is shutting down", m.Source)
52
+			}
53
+			return nil
54
+		}
55
+
45 56
 		rootUID, rootGID := daemon.GetRemappedUIDGID()
46
-		path, err := m.Setup(c.MountLabel, rootUID, rootGID)
57
+		path, err := m.Setup(c.MountLabel, rootUID, rootGID, checkfunc)
47 58
 		if err != nil {
48 59
 			return nil, err
49 60
 		}
... ...
@@ -24,7 +24,7 @@ func (daemon *Daemon) setupMounts(c *container.Container) ([]container.Mount, er
24 24
 		if err := daemon.lazyInitializeVolume(c.ID, mount); err != nil {
25 25
 			return nil, err
26 26
 		}
27
-		s, err := mount.Setup(c.MountLabel, 0, 0)
27
+		s, err := mount.Setup(c.MountLabel, 0, 0, nil)
28 28
 		if err != nil {
29 29
 			return nil, err
30 30
 		}
... ...
@@ -146,7 +146,9 @@ func (m *MountPoint) Cleanup() error {
146 146
 
147 147
 // Setup sets up a mount point by either mounting the volume if it is
148 148
 // configured, or creating the source directory if supplied.
149
-func (m *MountPoint) Setup(mountLabel string, rootUID, rootGID int) (path string, err error) {
149
+// The, optional, checkFun parameter allows doing additional checking
150
+// before creating the source directory on the host.
151
+func (m *MountPoint) Setup(mountLabel string, rootUID, rootGID int, checkFun func(m *MountPoint) error) (path string, err error) {
150 152
 	defer func() {
151 153
 		if err == nil {
152 154
 			if label.RelabelNeeded(m.Mode) {
... ...
@@ -181,6 +183,14 @@ func (m *MountPoint) Setup(mountLabel string, rootUID, rootGID int) (path string
181 181
 
182 182
 	// system.MkdirAll() produces an error if m.Source exists and is a file (not a directory),
183 183
 	if m.Type == mounttypes.TypeBind {
184
+		// Before creating the source directory on the host, invoke checkFun if it's not nil. One of
185
+		// the use case is to forbid creating the daemon socket as a directory if the daemon is in
186
+		// the process of shutting down.
187
+		if checkFun != nil {
188
+			if err := checkFun(m); err != nil {
189
+				return "", err
190
+			}
191
+		}
184 192
 		// idtools.MkdirAllNewAs() produces an error if m.Source exists and is a file (not a directory)
185 193
 		// also, makes sure that if the directory is created, the correct remapped rootUID/rootGID will own it
186 194
 		if err := idtools.MkdirAllNewAs(m.Source, 0755, rootUID, rootGID); err != nil {