Enable graceful shutdown for services
1] Process using uwsgi:
uwsgi services doesn't support for graceful shutting down [1].
It requires some changes in unit files [2] including adding below
graceful shutdown hook and changing KillSignal:
--hook-master-start "unix_signal:15 gracefully_kill_them_all
All the steps and changes required are specified in etherpad [1].
2] Non uwsgi services needs below changes:
In [service] section:
a. Add KillMode = process
b. Add TimeoutStopSec = infinity
NOTE:
Creating unit file for services other than uwsgi is handled by the
'write_user_unit_file' function [3]. This function is common for all
the services so this patch adds the above mentioned parameters for
services using ServiceLauncher also though they don't require.
Added a new stackrc variable WORKER_TIMEOUT which is required to add
graceful shutdown support to uwsgi services. It will be set as a value
to 'worker-reload-mercy' [4] in uwsgi file of service. The default
value set to this variable is 90.
[1] https://etherpad.openstack.org/p/uwsgi-issues
[2] https://www.freedesktop.org/software/systemd/man/systemd.kill.html
[3] https://github.com/openstack-dev/devstack/blob/2967ca3dfd0d64970dfa5dfa7ac2330ee7aa90ed/functions-common#L1439-L1461
[4] http://uwsgi-docs.readthedocs.io/en/latest/Options.html#worker-reload-mercy
Co-Authored-By: Dinesh Bhor <dinesh.bhor@nttdata.com>
Change-Id: Ia95291325ce4858b47102dd49504250183f339ab
Matthew Treinish authored on 2017/08/05 00:09:26Showing 3 changed files
| ... | ... |
@@ -1451,6 +1451,8 @@ function write_user_unit_file {
|
| 1451 | 1451 |
iniset -sudo $unitfile "Unit" "Description" "Devstack $service" |
| 1452 | 1452 |
iniset -sudo $unitfile "Service" "User" "$user" |
| 1453 | 1453 |
iniset -sudo $unitfile "Service" "ExecStart" "$command" |
| 1454 |
+ iniset -sudo $unitfile "Service" "KillMode" "process" |
|
| 1455 |
+ iniset -sudo $unitfile "Service" "TimeoutStopSec" "infinity" |
|
| 1454 | 1456 |
if [[ -n "$group" ]]; then |
| 1455 | 1457 |
iniset -sudo $unitfile "Service" "Group" "$group" |
| 1456 | 1458 |
fi |
| ... | ... |
@@ -1473,7 +1475,7 @@ function write_uwsgi_user_unit_file {
|
| 1473 | 1473 |
iniset -sudo $unitfile "Service" "User" "$user" |
| 1474 | 1474 |
iniset -sudo $unitfile "Service" "ExecStart" "$command" |
| 1475 | 1475 |
iniset -sudo $unitfile "Service" "Type" "notify" |
| 1476 |
- iniset -sudo $unitfile "Service" "KillSignal" "SIGQUIT" |
|
| 1476 |
+ iniset -sudo $unitfile "Service" "KillMode" "process" |
|
| 1477 | 1477 |
iniset -sudo $unitfile "Service" "Restart" "always" |
| 1478 | 1478 |
iniset -sudo $unitfile "Service" "NotifyAccess" "all" |
| 1479 | 1479 |
iniset -sudo $unitfile "Service" "RestartForceExitStatus" "100" |
| ... | ... |
@@ -260,10 +260,15 @@ function write_uwsgi_config {
|
| 260 | 260 |
# Set die-on-term & exit-on-reload so that uwsgi shuts down |
| 261 | 261 |
iniset "$file" uwsgi die-on-term true |
| 262 | 262 |
iniset "$file" uwsgi exit-on-reload true |
| 263 |
+ # Set worker-reload-mercy so that worker will not exit till the time |
|
| 264 |
+ # configured after graceful shutdown |
|
| 265 |
+ iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT |
|
| 263 | 266 |
iniset "$file" uwsgi enable-threads true |
| 264 | 267 |
iniset "$file" uwsgi plugins python |
| 265 | 268 |
# uwsgi recommends this to prevent thundering herd on accept. |
| 266 | 269 |
iniset "$file" uwsgi thunder-lock true |
| 270 |
+ # Set hook to trigger graceful shutdown on SIGTERM |
|
| 271 |
+ iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all" |
|
| 267 | 272 |
# Override the default size for headers from the 4k default. |
| 268 | 273 |
iniset "$file" uwsgi buffer-size 65535 |
| 269 | 274 |
# Make sure the client doesn't try to re-use the connection. |
| ... | ... |
@@ -316,6 +321,11 @@ function write_local_uwsgi_http_config {
|
| 316 | 316 |
iniset "$file" uwsgi plugins python |
| 317 | 317 |
# uwsgi recommends this to prevent thundering herd on accept. |
| 318 | 318 |
iniset "$file" uwsgi thunder-lock true |
| 319 |
+ # Set hook to trigger graceful shutdown on SIGTERM |
|
| 320 |
+ iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all" |
|
| 321 |
+ # Set worker-reload-mercy so that worker will not exit till the time |
|
| 322 |
+ # configured after graceful shutdown |
|
| 323 |
+ iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT |
|
| 319 | 324 |
# Override the default size for headers from the 4k default. |
| 320 | 325 |
iniset "$file" uwsgi buffer-size 65535 |
| 321 | 326 |
# Make sure the client doesn't try to re-use the connection. |
| ... | ... |
@@ -778,6 +778,9 @@ SERVICE_TIMEOUT=${SERVICE_TIMEOUT:-60}
|
| 778 | 778 |
# Service graceful shutdown timeout |
| 779 | 779 |
SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT=${SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT:-5}
|
| 780 | 780 |
|
| 781 |
+# Service graceful shutdown timeout |
|
| 782 |
+WORKER_TIMEOUT=${WORKER_TIMEOUT:-90}
|
|
| 783 |
+ |
|
| 781 | 784 |
# Support alternative yum -- in future Fedora 'dnf' will become the |
| 782 | 785 |
# only supported installer, but for now 'yum' and 'dnf' are both |
| 783 | 786 |
# available in parallel with compatible CLIs. Allow manual switching |