The `--shutdown-timeout` parameter is added to daemon options and
config file. It will also be updated during daemon reload.
Additional test cases have been added to cover the change.
This fix fixes #22471.
Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
| ... | ... |
@@ -37,6 +37,10 @@ const ( |
| 37 | 37 |
disableNetworkBridge = "none" |
| 38 | 38 |
) |
| 39 | 39 |
|
| 40 |
+const ( |
|
| 41 |
+ defaultShutdownTimeout = 15 |
|
| 42 |
+) |
|
| 43 |
+ |
|
| 40 | 44 |
// flatOptions contains configuration keys |
| 41 | 45 |
// that MUST NOT be parsed as deep structures. |
| 42 | 46 |
// Use this to differentiate these options |
| ... | ... |
@@ -123,6 +127,10 @@ type CommonConfig struct {
|
| 123 | 123 |
// may take place at a time for each push. |
| 124 | 124 |
MaxConcurrentUploads *int `json:"max-concurrent-uploads,omitempty"` |
| 125 | 125 |
|
| 126 |
+ // ShutdownTimeout is the timeout value (in seconds) the daemon will wait for the container |
|
| 127 |
+ // to stop when daemon is being shutdown |
|
| 128 |
+ ShutdownTimeout int `json:"shutdown-timeout,omitempty"` |
|
| 129 |
+ |
|
| 126 | 130 |
Debug bool `json:"debug,omitempty"` |
| 127 | 131 |
Hosts []string `json:"hosts,omitempty"` |
| 128 | 132 |
LogLevel string `json:"log-level,omitempty"` |
| ... | ... |
@@ -176,6 +184,7 @@ func (config *Config) InstallCommonFlags(flags *pflag.FlagSet) {
|
| 176 | 176 |
flags.StringVar(&config.CorsHeaders, "api-cors-header", "", "Set CORS headers in the remote API") |
| 177 | 177 |
flags.IntVar(&maxConcurrentDownloads, "max-concurrent-downloads", defaultMaxConcurrentDownloads, "Set the max concurrent downloads for each pull") |
| 178 | 178 |
flags.IntVar(&maxConcurrentUploads, "max-concurrent-uploads", defaultMaxConcurrentUploads, "Set the max concurrent uploads for each push") |
| 179 |
+ flags.IntVar(&config.ShutdownTimeout, "shutdown-timeout", defaultShutdownTimeout, "Set the default shutdown timeout") |
|
| 179 | 180 |
|
| 180 | 181 |
flags.StringVar(&config.SwarmDefaultAdvertiseAddr, "swarm-default-advertise-addr", "", "Set default address or interface for swarm advertised address") |
| 181 | 182 |
|
| ... | ... |
@@ -732,12 +732,13 @@ func (daemon *Daemon) shutdownContainer(c *container.Container) error {
|
| 732 | 732 |
return nil |
| 733 | 733 |
} |
| 734 | 734 |
|
| 735 |
-// ShutdownTimeout returns the shutdown timeout based on the max stopTimeout of the containers |
|
| 735 |
+// ShutdownTimeout returns the shutdown timeout based on the max stopTimeout of the containers, |
|
| 736 |
+// and is limited by daemon's ShutdownTimeout. |
|
| 736 | 737 |
func (daemon *Daemon) ShutdownTimeout() int {
|
| 737 |
- // By default we use container.DefaultStopTimeout + 5s, which is 15s. |
|
| 738 |
- // TODO (yongtang): Will need to allow shutdown-timeout once #23036 is in place. |
|
| 738 |
+ // By default we use daemon's ShutdownTimeout. |
|
| 739 |
+ shutdownTimeout := daemon.configStore.ShutdownTimeout |
|
| 740 |
+ |
|
| 739 | 741 |
graceTimeout := 5 |
| 740 |
- shutdownTimeout := container.DefaultStopTimeout + graceTimeout |
|
| 741 | 742 |
if daemon.containers != nil {
|
| 742 | 743 |
for _, c := range daemon.containers.List() {
|
| 743 | 744 |
if shutdownTimeout >= 0 {
|
| ... | ... |
@@ -769,7 +770,7 @@ func (daemon *Daemon) Shutdown() error {
|
| 769 | 769 |
} |
| 770 | 770 |
|
| 771 | 771 |
if daemon.containers != nil {
|
| 772 |
- logrus.Debug("starting clean shutdown of all containers...")
|
|
| 772 |
+ logrus.Debugf("start clean shutdown of all containers with a %d seconds timeout...", daemon.configStore.ShutdownTimeout)
|
|
| 773 | 773 |
daemon.containers.ApplyAll(func(c *container.Container) {
|
| 774 | 774 |
if !c.IsRunning() {
|
| 775 | 775 |
return |
| ... | ... |
@@ -995,6 +996,7 @@ func (daemon *Daemon) initDiscovery(config *Config) error {
|
| 995 | 995 |
// - Daemon max concurrent uploads |
| 996 | 996 |
// - Cluster discovery (reconfigure and restart). |
| 997 | 997 |
// - Daemon live restore |
| 998 |
+// - Daemon shutdown timeout (in seconds). |
|
| 998 | 999 |
func (daemon *Daemon) Reload(config *Config) (err error) {
|
| 999 | 1000 |
|
| 1000 | 1001 |
daemon.configStore.reloadLock.Lock() |
| ... | ... |
@@ -1055,6 +1057,11 @@ func (daemon *Daemon) Reload(config *Config) (err error) {
|
| 1055 | 1055 |
daemon.uploadManager.SetConcurrency(*daemon.configStore.MaxConcurrentUploads) |
| 1056 | 1056 |
} |
| 1057 | 1057 |
|
| 1058 |
+ if config.IsValueSet("shutdown-timeout") {
|
|
| 1059 |
+ daemon.configStore.ShutdownTimeout = config.ShutdownTimeout |
|
| 1060 |
+ logrus.Debugf("Reset Shutdown Timeout: %d", daemon.configStore.ShutdownTimeout)
|
|
| 1061 |
+ } |
|
| 1062 |
+ |
|
| 1058 | 1063 |
// We emit daemon reload event here with updatable configurations |
| 1059 | 1064 |
attributes["debug"] = fmt.Sprintf("%t", daemon.configStore.Debug)
|
| 1060 | 1065 |
attributes["live-restore"] = fmt.Sprintf("%t", daemon.configStore.LiveRestoreEnabled)
|
| ... | ... |
@@ -1074,6 +1081,7 @@ func (daemon *Daemon) Reload(config *Config) (err error) {
|
| 1074 | 1074 |
} |
| 1075 | 1075 |
attributes["max-concurrent-downloads"] = fmt.Sprintf("%d", *daemon.configStore.MaxConcurrentDownloads)
|
| 1076 | 1076 |
attributes["max-concurrent-uploads"] = fmt.Sprintf("%d", *daemon.configStore.MaxConcurrentUploads)
|
| 1077 |
+ attributes["shutdown-timeout"] = fmt.Sprintf("%d", daemon.configStore.ShutdownTimeout)
|
|
| 1077 | 1078 |
|
| 1078 | 1079 |
return nil |
| 1079 | 1080 |
} |
| ... | ... |
@@ -64,6 +64,7 @@ Options: |
| 64 | 64 |
--raw-logs Full timestamps without ANSI coloring |
| 65 | 65 |
--registry-mirror value Preferred Docker registry mirror (default []) |
| 66 | 66 |
--selinux-enabled Enable selinux support |
| 67 |
+ --shutdown-timeout=15 Set the shutdown timeout value in seconds |
|
| 67 | 68 |
-s, --storage-driver string Storage driver to use |
| 68 | 69 |
--storage-opt value Storage driver options (default []) |
| 69 | 70 |
--swarm-default-advertise-addr string Set default address or interface for swarm advertised address |
| ... | ... |
@@ -1118,6 +1119,7 @@ This is a full example of the allowed configuration options on Linux: |
| 1118 | 1118 |
"cluster-advertise": "", |
| 1119 | 1119 |
"max-concurrent-downloads": 3, |
| 1120 | 1120 |
"max-concurrent-uploads": 5, |
| 1121 |
+ "shutdown-timeout": 15, |
|
| 1121 | 1122 |
"debug": true, |
| 1122 | 1123 |
"hosts": [], |
| 1123 | 1124 |
"log-level": "", |
| ... | ... |
@@ -1194,6 +1196,7 @@ This is a full example of the allowed configuration options on Windows: |
| 1194 | 1194 |
"graph": "", |
| 1195 | 1195 |
"cluster-store": "", |
| 1196 | 1196 |
"cluster-advertise": "", |
| 1197 |
+ "shutdown-timeout": 15, |
|
| 1197 | 1198 |
"debug": true, |
| 1198 | 1199 |
"hosts": [], |
| 1199 | 1200 |
"log-level": "", |
| ... | ... |
@@ -2920,3 +2920,57 @@ func (s *DockerDaemonSuite) TestDaemonWithUserlandProxyPath(c *check.C) {
|
| 2920 | 2920 |
c.Assert(out, checker.Contains, "driver failed programming external connectivity on endpoint") |
| 2921 | 2921 |
c.Assert(out, checker.Contains, "/does/not/exist: no such file or directory") |
| 2922 | 2922 |
} |
| 2923 |
+ |
|
| 2924 |
+// Test case for #22471 |
|
| 2925 |
+func (s *DockerDaemonSuite) TestDaemonShutdownTimeout(c *check.C) {
|
|
| 2926 |
+ testRequires(c, SameHostDaemon) |
|
| 2927 |
+ |
|
| 2928 |
+ c.Assert(s.d.StartWithBusybox("--shutdown-timeout=3"), check.IsNil)
|
|
| 2929 |
+ |
|
| 2930 |
+ _, err := s.d.Cmd("run", "-d", "busybox", "top")
|
|
| 2931 |
+ c.Assert(err, check.IsNil) |
|
| 2932 |
+ |
|
| 2933 |
+ syscall.Kill(s.d.cmd.Process.Pid, syscall.SIGINT) |
|
| 2934 |
+ |
|
| 2935 |
+ select {
|
|
| 2936 |
+ case <-s.d.wait: |
|
| 2937 |
+ case <-time.After(5 * time.Second): |
|
| 2938 |
+ } |
|
| 2939 |
+ |
|
| 2940 |
+ expectedMessage := `level=debug msg="start clean shutdown of all containers with a 3 seconds timeout..."` |
|
| 2941 |
+ content, _ := ioutil.ReadFile(s.d.logFile.Name()) |
|
| 2942 |
+ c.Assert(string(content), checker.Contains, expectedMessage) |
|
| 2943 |
+} |
|
| 2944 |
+ |
|
| 2945 |
+// Test case for #22471 |
|
| 2946 |
+func (s *DockerDaemonSuite) TestDaemonShutdownTimeoutWithConfigFile(c *check.C) {
|
|
| 2947 |
+ testRequires(c, SameHostDaemon) |
|
| 2948 |
+ |
|
| 2949 |
+ // daemon config file |
|
| 2950 |
+ configFilePath := "test.json" |
|
| 2951 |
+ configFile, err := os.Create(configFilePath) |
|
| 2952 |
+ c.Assert(err, checker.IsNil) |
|
| 2953 |
+ defer os.Remove(configFilePath) |
|
| 2954 |
+ |
|
| 2955 |
+ daemonConfig := `{ "shutdown-timeout" : 8 }`
|
|
| 2956 |
+ fmt.Fprintf(configFile, "%s", daemonConfig) |
|
| 2957 |
+ configFile.Close() |
|
| 2958 |
+ c.Assert(s.d.Start(fmt.Sprintf("--config-file=%s", configFilePath)), check.IsNil)
|
|
| 2959 |
+ |
|
| 2960 |
+ configFile, err = os.Create(configFilePath) |
|
| 2961 |
+ c.Assert(err, checker.IsNil) |
|
| 2962 |
+ daemonConfig = `{ "shutdown-timeout" : 5 }`
|
|
| 2963 |
+ fmt.Fprintf(configFile, "%s", daemonConfig) |
|
| 2964 |
+ configFile.Close() |
|
| 2965 |
+ |
|
| 2966 |
+ syscall.Kill(s.d.cmd.Process.Pid, syscall.SIGHUP) |
|
| 2967 |
+ |
|
| 2968 |
+ select {
|
|
| 2969 |
+ case <-s.d.wait: |
|
| 2970 |
+ case <-time.After(3 * time.Second): |
|
| 2971 |
+ } |
|
| 2972 |
+ |
|
| 2973 |
+ expectedMessage := `level=debug msg="Reset Shutdown Timeout: 5"` |
|
| 2974 |
+ content, _ := ioutil.ReadFile(s.d.logFile.Name()) |
|
| 2975 |
+ c.Assert(string(content), checker.Contains, expectedMessage) |
|
| 2976 |
+} |
| ... | ... |
@@ -418,7 +418,7 @@ func (s *DockerDaemonSuite) TestDaemonEvents(c *check.C) {
|
| 418 | 418 |
|
| 419 | 419 |
configFile, err = os.Create(configFilePath) |
| 420 | 420 |
c.Assert(err, checker.IsNil) |
| 421 |
- daemonConfig = `{"max-concurrent-downloads":1,"labels":["bar=foo"]}`
|
|
| 421 |
+ daemonConfig = `{"max-concurrent-downloads":1,"labels":["bar=foo"], "shutdown-timeout": 10}`
|
|
| 422 | 422 |
fmt.Fprintf(configFile, "%s", daemonConfig) |
| 423 | 423 |
configFile.Close() |
| 424 | 424 |
|
| ... | ... |
@@ -429,7 +429,7 @@ func (s *DockerDaemonSuite) TestDaemonEvents(c *check.C) {
|
| 429 | 429 |
out, err = s.d.Cmd("events", "--since=0", "--until", daemonUnixTime(c))
|
| 430 | 430 |
c.Assert(err, checker.IsNil) |
| 431 | 431 |
|
| 432 |
- c.Assert(out, checker.Contains, fmt.Sprintf("daemon reload %s (cluster-advertise=, cluster-store=, cluster-store-opts={}, debug=true, default-runtime=runc, labels=[\"bar=foo\"], live-restore=false, max-concurrent-downloads=1, max-concurrent-uploads=5, name=%s, runtimes=runc:{docker-runc []})", daemonID, daemonName))
|
|
| 432 |
+ c.Assert(out, checker.Contains, fmt.Sprintf("daemon reload %s (cluster-advertise=, cluster-store=, cluster-store-opts={}, debug=true, default-runtime=runc, labels=[\"bar=foo\"], live-restore=false, max-concurrent-downloads=1, max-concurrent-uploads=5, name=%s, runtimes=runc:{docker-runc []}, shutdown-timeout=10)", daemonID, daemonName))
|
|
| 433 | 433 |
} |
| 434 | 434 |
|
| 435 | 435 |
func (s *DockerDaemonSuite) TestDaemonEventsWithFilters(c *check.C) {
|
| ... | ... |
@@ -56,6 +56,7 @@ dockerd - Enable daemon mode |
| 56 | 56 |
[**--registry-mirror**[=*[]*]] |
| 57 | 57 |
[**-s**|**--storage-driver**[=*STORAGE-DRIVER*]] |
| 58 | 58 |
[**--selinux-enabled**] |
| 59 |
+[**--shutdown-timeout**[=*15*]] |
|
| 59 | 60 |
[**--storage-opt**[=*[]*]] |
| 60 | 61 |
[**--swarm-default-advertise-addr**[=*IP|INTERFACE*]] |
| 61 | 62 |
[**--tls**] |
| ... | ... |
@@ -246,6 +247,9 @@ output otherwise. |
| 246 | 246 |
**--selinux-enabled**=*true*|*false* |
| 247 | 247 |
Enable selinux support. Default is false. |
| 248 | 248 |
|
| 249 |
+**--shutdown-timeout**=*15* |
|
| 250 |
+ Set the shutdown timeout value in seconds. Default is `15`. |
|
| 251 |
+ |
|
| 249 | 252 |
**--storage-opt**=[] |
| 250 | 253 |
Set storage driver options. See STORAGE DRIVER OPTIONS. |
| 251 | 254 |
|