If a container mount the socket the daemon is listening on into
container while the daemon is being shutdown, the socket will
not exist on the host, then daemon will assume it's a directory
and create it on the host, this will cause the daemon can't start
next time.
fix issue https://github.com/moby/moby/issues/30348
To reproduce this issue, you can add following code
```
--- a/daemon/oci_linux.go
+++ b/daemon/oci_linux.go
@@ -8,6 +8,7 @@ import (
"sort"
"strconv"
"strings"
+ "time"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/container"
@@ -666,7 +667,8 @@ func (daemon *Daemon) createSpec(c *container.Container) (*libcontainerd.Spec, e
if err := daemon.setupIpcDirs(c); err != nil {
return nil, err
}
-
+ fmt.Printf("===please stop the daemon===\n")
+ time.Sleep(time.Second * 2)
ms, err := daemon.setupMounts(c)
if err != nil {
return nil, err
```
step1 run a container which has `--restart always` and `-v /var/run/docker.sock:/sock`
```
$ docker run -ti --restart always -v /var/run/docker.sock:/sock busybox
/ #
```
step2 exit the the container
```
/ # exit
```
and kill the daemon when you see
```
===please stop the daemon===
```
in the daemon log
The daemon can't restart again and fail with `can't create unix socket /var/run/docker.sock: is a directory`.
Signed-off-by: Lei Jitang <leijitang@huawei.com>
... | ... |
@@ -155,6 +155,8 @@ func (cli *DaemonCli) start(opts daemonOptions) (err error) { |
155 | 155 |
api := apiserver.New(serverConfig) |
156 | 156 |
cli.api = api |
157 | 157 |
|
158 |
+ var hosts []string |
|
159 |
+ |
|
158 | 160 |
for i := 0; i < len(cli.Config.Hosts); i++ { |
159 | 161 |
var err error |
160 | 162 |
if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil { |
... | ... |
@@ -186,6 +188,7 @@ func (cli *DaemonCli) start(opts daemonOptions) (err error) { |
186 | 186 |
} |
187 | 187 |
} |
188 | 188 |
logrus.Debugf("Listener created for HTTP on %s (%s)", proto, addr) |
189 |
+ hosts = append(hosts, protoAddrParts[1]) |
|
189 | 190 |
api.Accept(addr, ls...) |
190 | 191 |
} |
191 | 192 |
|
... | ... |
@@ -213,6 +216,8 @@ func (cli *DaemonCli) start(opts daemonOptions) (err error) { |
213 | 213 |
return fmt.Errorf("Error starting daemon: %v", err) |
214 | 214 |
} |
215 | 215 |
|
216 |
+ d.StoreHosts(hosts) |
|
217 |
+ |
|
216 | 218 |
// validate after NewDaemon has restored enabled plugins. Dont change order. |
217 | 219 |
if err := validateAuthzPlugins(cli.Config.AuthorizationPlugins, pluginStore); err != nil { |
218 | 220 |
return fmt.Errorf("Error validating authorization plugin: %v", err) |
... | ... |
@@ -116,6 +116,17 @@ type Daemon struct { |
116 | 116 |
|
117 | 117 |
diskUsageRunning int32 |
118 | 118 |
pruneRunning int32 |
119 |
+ hosts map[string]bool // hosts stores the addresses the daemon is listening on |
|
120 |
+} |
|
121 |
+ |
|
122 |
+// StoreHosts stores the addresses the daemon is listening on |
|
123 |
+func (daemon *Daemon) StoreHosts(hosts []string) { |
|
124 |
+ if daemon.hosts == nil { |
|
125 |
+ daemon.hosts = make(map[string]bool) |
|
126 |
+ } |
|
127 |
+ for _, h := range hosts { |
|
128 |
+ daemon.hosts[h] = true |
|
129 |
+ } |
|
119 | 130 |
} |
120 | 131 |
|
121 | 132 |
// HasExperimental returns whether the experimental features of the daemon are enabled or not |
... | ... |
@@ -46,7 +46,8 @@ func (daemon *Daemon) StateChanged(id string, e libcontainerd.StateInfo) error { |
46 | 46 |
c.StreamConfig.Wait() |
47 | 47 |
c.Reset(false) |
48 | 48 |
|
49 |
- restart, wait, err := c.RestartManager().ShouldRestart(e.ExitCode, c.HasBeenManuallyStopped, time.Since(c.StartedAt)) |
|
49 |
+ // If daemon is being shutdown, don't let the container restart |
|
50 |
+ restart, wait, err := c.RestartManager().ShouldRestart(e.ExitCode, daemon.IsShuttingDown() || c.HasBeenManuallyStopped, time.Since(c.StartedAt)) |
|
50 | 51 |
if err == nil && restart { |
51 | 52 |
c.RestartCount++ |
52 | 53 |
c.SetRestarting(platformConstructExitStatus(e)) |
... | ... |
@@ -6,6 +6,7 @@ package daemon |
6 | 6 |
|
7 | 7 |
import ( |
8 | 8 |
"encoding/json" |
9 |
+ "fmt" |
|
9 | 10 |
"os" |
10 | 11 |
"path/filepath" |
11 | 12 |
"sort" |
... | ... |
@@ -42,8 +43,19 @@ func (daemon *Daemon) setupMounts(c *container.Container) ([]container.Mount, er |
42 | 42 |
if err := daemon.lazyInitializeVolume(c.ID, m); err != nil { |
43 | 43 |
return nil, err |
44 | 44 |
} |
45 |
+ // If the daemon is being shutdown, we should not let a container start if it is trying to |
|
46 |
+ // mount the socket the daemon is listening on. During daemon shutdown, the socket |
|
47 |
+ // (/var/run/docker.sock by default) doesn't exist anymore causing the call to m.Setup to |
|
48 |
+ // create at directory instead. This in turn will prevent the daemon to restart. |
|
49 |
+ checkfunc := func(m *volume.MountPoint) error { |
|
50 |
+ if _, exist := daemon.hosts[m.Source]; exist && daemon.IsShuttingDown() { |
|
51 |
+ return fmt.Errorf("Could not mount %q to container while the daemon is shutting down", m.Source) |
|
52 |
+ } |
|
53 |
+ return nil |
|
54 |
+ } |
|
55 |
+ |
|
45 | 56 |
rootUID, rootGID := daemon.GetRemappedUIDGID() |
46 |
- path, err := m.Setup(c.MountLabel, rootUID, rootGID) |
|
57 |
+ path, err := m.Setup(c.MountLabel, rootUID, rootGID, checkfunc) |
|
47 | 58 |
if err != nil { |
48 | 59 |
return nil, err |
49 | 60 |
} |
... | ... |
@@ -24,7 +24,7 @@ func (daemon *Daemon) setupMounts(c *container.Container) ([]container.Mount, er |
24 | 24 |
if err := daemon.lazyInitializeVolume(c.ID, mount); err != nil { |
25 | 25 |
return nil, err |
26 | 26 |
} |
27 |
- s, err := mount.Setup(c.MountLabel, 0, 0) |
|
27 |
+ s, err := mount.Setup(c.MountLabel, 0, 0, nil) |
|
28 | 28 |
if err != nil { |
29 | 29 |
return nil, err |
30 | 30 |
} |
... | ... |
@@ -146,7 +146,9 @@ func (m *MountPoint) Cleanup() error { |
146 | 146 |
|
147 | 147 |
// Setup sets up a mount point by either mounting the volume if it is |
148 | 148 |
// configured, or creating the source directory if supplied. |
149 |
-func (m *MountPoint) Setup(mountLabel string, rootUID, rootGID int) (path string, err error) { |
|
149 |
+// The, optional, checkFun parameter allows doing additional checking |
|
150 |
+// before creating the source directory on the host. |
|
151 |
+func (m *MountPoint) Setup(mountLabel string, rootUID, rootGID int, checkFun func(m *MountPoint) error) (path string, err error) { |
|
150 | 152 |
defer func() { |
151 | 153 |
if err == nil { |
152 | 154 |
if label.RelabelNeeded(m.Mode) { |
... | ... |
@@ -181,6 +183,14 @@ func (m *MountPoint) Setup(mountLabel string, rootUID, rootGID int) (path string |
181 | 181 |
|
182 | 182 |
// system.MkdirAll() produces an error if m.Source exists and is a file (not a directory), |
183 | 183 |
if m.Type == mounttypes.TypeBind { |
184 |
+ // Before creating the source directory on the host, invoke checkFun if it's not nil. One of |
|
185 |
+ // the use case is to forbid creating the daemon socket as a directory if the daemon is in |
|
186 |
+ // the process of shutting down. |
|
187 |
+ if checkFun != nil { |
|
188 |
+ if err := checkFun(m); err != nil { |
|
189 |
+ return "", err |
|
190 |
+ } |
|
191 |
+ } |
|
184 | 192 |
// idtools.MkdirAllNewAs() produces an error if m.Source exists and is a file (not a directory) |
185 | 193 |
// also, makes sure that if the directory is created, the correct remapped rootUID/rootGID will own it |
186 | 194 |
if err := idtools.MkdirAllNewAs(m.Source, 0755, rootUID, rootGID); err != nil { |