From 477a962861afe2e859179245d6d39cb41f6c499d Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Fri, 4 Aug 2017 11:09:26 -0400 Subject: [PATCH] Enable graceful shutdown for services 1] Process using uwsgi: uwsgi services doesn't support for graceful shutting down [1]. It requires some changes in unit files [2] including adding below graceful shutdown hook and changing KillSignal: --hook-master-start "unix_signal:15 gracefully_kill_them_all All the steps and changes required are specified in etherpad [1]. 2] Non uwsgi services needs below changes: In [service] section: a. Add KillMode = process b. Add TimeoutStopSec = infinity NOTE: Creating unit file for services other than uwsgi is handled by the 'write_user_unit_file' function [3]. This function is common for all the services so this patch adds the above mentioned parameters for services using ServiceLauncher also though they don't require. Added a new stackrc variable WORKER_TIMEOUT which is required to add graceful shutdown support to uwsgi services. It will be set as a value to 'worker-reload-mercy' [4] in uwsgi file of service. The default value set to this variable is 90. [1] https://etherpad.openstack.org/p/uwsgi-issues [2] https://www.freedesktop.org/software/systemd/man/systemd.kill.html [3] https://github.com/openstack-dev/devstack/blob/2967ca3dfd0d64970dfa5dfa7ac2330ee7aa90ed/functions-common#L1439-L1461 [4] http://uwsgi-docs.readthedocs.io/en/latest/Options.html#worker-reload-mercy Co-Authored-By: Dinesh Bhor Change-Id: Ia95291325ce4858b47102dd49504250183f339ab --- functions-common | 4 +++- lib/apache | 10 ++++++++++ stackrc | 3 +++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/functions-common b/functions-common index a5f770f579..52f53ef147 100644 --- a/functions-common +++ b/functions-common @@ -1451,6 +1451,8 @@ function write_user_unit_file { iniset -sudo $unitfile "Unit" "Description" "Devstack $service" iniset -sudo $unitfile "Service" "User" "$user" iniset -sudo $unitfile "Service" "ExecStart" "$command" + iniset -sudo $unitfile "Service" "KillMode" "process" + iniset -sudo $unitfile "Service" "TimeoutStopSec" "infinity" if [[ -n "$group" ]]; then iniset -sudo $unitfile "Service" "Group" "$group" fi @@ -1473,7 +1475,7 @@ function write_uwsgi_user_unit_file { iniset -sudo $unitfile "Service" "User" "$user" iniset -sudo $unitfile "Service" "ExecStart" "$command" iniset -sudo $unitfile "Service" "Type" "notify" - iniset -sudo $unitfile "Service" "KillSignal" "SIGQUIT" + iniset -sudo $unitfile "Service" "KillMode" "process" iniset -sudo $unitfile "Service" "Restart" "always" iniset -sudo $unitfile "Service" "NotifyAccess" "all" iniset -sudo $unitfile "Service" "RestartForceExitStatus" "100" diff --git a/lib/apache b/lib/apache index dfca25a764..39d5b7b071 100644 --- a/lib/apache +++ b/lib/apache @@ -260,10 +260,15 @@ function write_uwsgi_config { # Set die-on-term & exit-on-reload so that uwsgi shuts down iniset "$file" uwsgi die-on-term true iniset "$file" uwsgi exit-on-reload true + # Set worker-reload-mercy so that worker will not exit till the time + # configured after graceful shutdown + iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT iniset "$file" uwsgi enable-threads true iniset "$file" uwsgi plugins python # uwsgi recommends this to prevent thundering herd on accept. iniset "$file" uwsgi thunder-lock true + # Set hook to trigger graceful shutdown on SIGTERM + iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all" # Override the default size for headers from the 4k default. iniset "$file" uwsgi buffer-size 65535 # Make sure the client doesn't try to re-use the connection. @@ -316,6 +321,11 @@ function write_local_uwsgi_http_config { iniset "$file" uwsgi plugins python # uwsgi recommends this to prevent thundering herd on accept. iniset "$file" uwsgi thunder-lock true + # Set hook to trigger graceful shutdown on SIGTERM + iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all" + # Set worker-reload-mercy so that worker will not exit till the time + # configured after graceful shutdown + iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT # Override the default size for headers from the 4k default. iniset "$file" uwsgi buffer-size 65535 # Make sure the client doesn't try to re-use the connection. diff --git a/stackrc b/stackrc index 3591994f79..e936b337e7 100644 --- a/stackrc +++ b/stackrc @@ -778,6 +778,9 @@ SERVICE_TIMEOUT=${SERVICE_TIMEOUT:-60} # Service graceful shutdown timeout SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT=${SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT:-5} +# Service graceful shutdown timeout +WORKER_TIMEOUT=${WORKER_TIMEOUT:-90} + # Support alternative yum -- in future Fedora 'dnf' will become the # only supported installer, but for now 'yum' and 'dnf' are both # available in parallel with compatible CLIs. Allow manual switching