daemon: fix spurious "ShouldRestart failed" warning on shutdown
When shutting down the daemon, containers are stopped/killed, which calls
`container.ExitOnNext()` to stop the restart manager to prevent containers
with a restart policy from being restarted. However, when the daemon handles
the container exit (Daemon.handleContainerExit), it checks whether the container
should be restarted (`container.RestartManager().ShouldRestart`), which returns
an `ErrRestartCanceled` if the restart manager is stopped.
This patch:
- Prevents the `ErrRestartCanceled` from being logged as a warning; for
other errors, a log is still produced; also splitting the `exitStatus`
field to `exitCode` and `exitedAt` as the string presentation of it
was not human-readable.
- Adds an "info" log to log when the restart-manager is disabled for a
container with a restart-policy.
- Changes the confusing "ignoring event" log for a more informative message
("received task-delete event from containerd"). We don't act on this
event, but do want to log it to verify containerd handled the shutdown.
- Changes the "restarting container" log from "Debug" to "Info"; this
allows verifying that the restart-monitor handled a container-exit
Before this patch:
^CINFO[2026-02-21T20:04:42.937057877Z] Processing signal 'interrupt'
INFO[2026-02-21T20:04:43.074175085Z] ignoring event container=1ed1fcfd71c27b5ff4f04c1b33104c17725a841fba7661acea205879fdd687c8 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
INFO[2026-02-21T20:04:43.073736502Z] shim disconnected id=1ed1fcfd71c27b5ff4f04c1b33104c17725a841fba7661acea205879fdd687c8 namespace=moby
INFO[2026-02-21T20:04:43.074380960Z] cleaning up after shim disconnected id=1ed1fcfd71c27b5ff4f04c1b33104c17725a841fba7661acea205879fdd687c8 namespace=moby
INFO[2026-02-21T20:04:43.074393169Z] cleaning up dead shim id=1ed1fcfd71c27b5ff4f04c1b33104c17725a841fba7661acea205879fdd687c8 namespace=moby
WARN[2026-02-21T20:04:43.084973627Z] ShouldRestart failed, container will not be restarted container=1ed1fcfd71c27b5ff4f04c1b33104c17725a841fba7661acea205879fdd687c8 daemonShuttingDown=true error="restart canceled" execDuration=4.868244252s exitStatus="{0 2026-02-21 20:04:43.061117752 +0000 UTC}" hasBeenManuallyStopped=false restartCount=0
INFO[2026-02-21T20:04:43.231618294Z] stopping event stream following graceful shutdown error="<nil>" module=libcontainerd namespace=moby
INFO[2026-02-21T20:04:43.232019419Z] stopping healthcheck following graceful shutdown module=libcontainerd
INFO[2026-02-21T20:04:43.232244877Z] stopping event stream following graceful shutdown error="context canceled" module=libcontainerd namespace=plugins.moby
INFO[2026-02-21T20:04:44.238598878Z] Daemon shutdown complete
With this patch:
^CINFO[2026-02-21T20:07:39.524114750Z] Processing signal 'interrupt'
INFO[2026-02-21T20:07:39.525043125Z] stopping restart-manager container=1ed1fcfd71c27b5ff4f04c1b33104c17725a841fba7661acea205879fdd687c8
INFO[2026-02-21T20:07:39.592829959Z] received task-delete event from containerd container=1ed1fcfd71c27b5ff4f04c1b33104c17725a841fba7661acea205879fdd687c8 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
INFO[2026-02-21T20:07:39.592859084Z] shim disconnected id=1ed1fcfd71c27b5ff4f04c1b33104c17725a841fba7661acea205879fdd687c8 namespace=moby
INFO[2026-02-21T20:07:39.593019334Z] cleaning up after shim disconnected id=1ed1fcfd71c27b5ff4f04c1b33104c17725a841fba7661acea205879fdd687c8 namespace=moby
INFO[2026-02-21T20:07:39.593038667Z] cleaning up dead shim id=1ed1fcfd71c27b5ff4f04c1b33104c17725a841fba7661acea205879fdd687c8 namespace=moby
INFO[2026-02-21T20:07:39.710357042Z] stopping event stream following graceful shutdown error="<nil>" module=libcontainerd namespace=moby
INFO[2026-02-21T20:07:39.710912875Z] stopping healthcheck following graceful shutdown module=libcontainerd
INFO[2026-02-21T20:07:39.710989959Z] stopping event stream following graceful shutdown error="context canceled" module=libcontainerd namespace=plugins.moby
INFO[2026-02-21T20:07:40.724286834Z] Daemon shutdown complete
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
Sebastiaan van Stijn authored on 2026/02/22 05:37:17Showing 3 changed files
| ... | ... |
@@ -405,7 +405,7 @@ func cleanScopedPath(path string) string {
|
| 405 | 405 |
// path. See symlink.FollowSymlinkInScope for more details. |
| 406 | 406 |
func (container *Container) GetRootResourcePath(path string) (string, error) {
|
| 407 | 407 |
// IMPORTANT - These are paths on the OS where the daemon is running, hence |
| 408 |
- // any filepath operations must be done in an OS agnostic way. |
|
| 408 |
+ // any filepath operations must be done in an OS-agnostic way. |
|
| 409 | 409 |
cleanPath := filepath.Join(string(os.PathSeparator), path) |
| 410 | 410 |
return symlink.FollowSymlinkInScope(filepath.Join(container.Root, cleanPath), container.Root) |
| 411 | 411 |
} |
| ... | ... |
@@ -413,6 +413,11 @@ func (container *Container) GetRootResourcePath(path string) (string, error) {
|
| 413 | 413 |
// ExitOnNext signals to the monitor that it should not restart the container |
| 414 | 414 |
// after we send the kill signal. |
| 415 | 415 |
func (container *Container) ExitOnNext() {
|
| 416 |
+ if !container.HostConfig.RestartPolicy.IsNone() {
|
|
| 417 |
+ log.G(context.TODO()).WithField("container", container.ID).Info("stopping restart-manager")
|
|
| 418 |
+ } |
|
| 419 |
+ // If the container does not have a restartmanager, one is created (through |
|
| 420 |
+ // [container.RestartManager]), which is needed to handle daemon shutdown. |
|
| 416 | 421 |
container.RestartManager().Cancel() |
| 417 | 422 |
} |
| 418 | 423 |
|
| ... | ... |
@@ -687,7 +687,7 @@ func (c *client) processEventStream(ctx context.Context, ns string) {
|
| 687 | 687 |
"topic": ev.Topic, |
| 688 | 688 |
"type": reflect.TypeOf(t), |
| 689 | 689 |
"container": t.ContainerID, |
| 690 |
- }).Info("ignoring event")
|
|
| 690 |
+ }).Info("received task-delete event from containerd")
|
|
| 691 | 691 |
default: |
| 692 | 692 |
c.logger.WithFields(log.Fields{
|
| 693 | 693 |
"topic": ev.Topic, |
| ... | ... |
@@ -97,15 +97,20 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine |
| 97 | 97 |
execDuration := time.Since(c.State.StartedAt) |
| 98 | 98 |
restart, wait, err := c.RestartManager().ShouldRestart(uint32(ctrExitStatus.ExitCode), daemonShutdown || c.HasBeenManuallyStopped, execDuration) |
| 99 | 99 |
if err != nil {
|
| 100 |
- log.G(ctx).WithFields(log.Fields{
|
|
| 101 |
- "error": err, |
|
| 102 |
- "container": c.ID, |
|
| 103 |
- "restartCount": c.RestartCount, |
|
| 104 |
- "exitStatus": ctrExitStatus, |
|
| 105 |
- "daemonShuttingDown": daemonShutdown, |
|
| 106 |
- "hasBeenManuallyStopped": c.HasBeenManuallyStopped, |
|
| 107 |
- "execDuration": execDuration, |
|
| 108 |
- }).Warn("ShouldRestart failed, container will not be restarted")
|
|
| 100 |
+ // Ignore ErrRestartCanceled errors, which mean the restart-manager |
|
| 101 |
+ // was stopped (e.g., during daemon shutdown). |
|
| 102 |
+ if !errors.Is(err, restartmanager.ErrRestartCanceled) {
|
|
| 103 |
+ log.G(ctx).WithFields(log.Fields{
|
|
| 104 |
+ "error": err, |
|
| 105 |
+ "container": c.ID, |
|
| 106 |
+ "restartCount": c.RestartCount, |
|
| 107 |
+ "exitCode": ctrExitStatus.ExitCode, |
|
| 108 |
+ "exitedAt": ctrExitStatus.ExitedAt, |
|
| 109 |
+ "daemonShuttingDown": daemonShutdown, |
|
| 110 |
+ "hasBeenManuallyStopped": c.HasBeenManuallyStopped, |
|
| 111 |
+ "execDuration": execDuration, |
|
| 112 |
+ }).Warn("ShouldRestart failed: container will not be restarted")
|
|
| 113 |
+ } |
|
| 109 | 114 |
restart = false |
| 110 | 115 |
} |
| 111 | 116 |
|
| ... | ... |
@@ -119,10 +124,12 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine |
| 119 | 119 |
c.RestartCount++ |
| 120 | 120 |
log.G(ctx).WithFields(log.Fields{
|
| 121 | 121 |
"container": c.ID, |
| 122 |
+ "restartPolicy": c.HostConfig.RestartPolicy, |
|
| 122 | 123 |
"restartCount": c.RestartCount, |
| 123 |
- "exitStatus": ctrExitStatus, |
|
| 124 |
+ "exitCode": ctrExitStatus.ExitCode, |
|
| 125 |
+ "exitedAt": ctrExitStatus.ExitedAt, |
|
| 124 | 126 |
"manualRestart": c.HasBeenManuallyRestarted, |
| 125 |
- }).Debug("Restarting container")
|
|
| 127 |
+ }).Info("restarting container")
|
|
| 126 | 128 |
c.State.SetRestarting(&ctrExitStatus) |
| 127 | 129 |
} else {
|
| 128 | 130 |
c.State.SetStopped(&ctrExitStatus) |