From 647ff667e6037f1a62de6cf72f0fcb9fca3b8ae0 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Wed, 3 Mar 2021 15:50:03 +0000 Subject: [PATCH] Add variable for changing Apache HTTP timeout In services which use the Apache HTTP server to service HTTP requests, there exists a TimeOut directive [1] which defaults to 60 seconds. APIs which come under heavy load, such as Cinder, can sometimes exceed this which results in a HTTP 504 Gateway timeout, or similar. However, the request can still be serviced without error. For example, if Nova calls the Cinder API to detach a volume, and this operation takes longer than the shortest of the two timeouts, Nova will emit a stack trace with a 504 Gateway timeout. At some time later, the request to detach the volume will succeed. The Nova and Cinder DBs then become out-of-sync with each other, and frequently DB surgery is required. Although strictly this category of bugs should be fixed in OpenStack services, it is not realistic to expect this to happen in the short term. Therefore, this change makes it easier to set the Apache HTTP timeout via a new variable. An example of a related bug is here: https://bugs.launchpad.net/nova/+bug/1888665 Whilst this timeout can currently be set by overriding the WSGI config for individual services, this change makes it much easier. Change-Id: Ie452516655cbd40d63bdad3635fd66693e40ce34 Closes-Bug: #1917648 --- ansible/group_vars/all.yml | 1 + ansible/roles/aodh/templates/wsgi-aodh.conf.j2 | 1 + ansible/roles/cinder/templates/cinder-wsgi.conf.j2 | 1 + .../cloudkitty/templates/wsgi-cloudkitty.conf.j2 | 1 + .../freezer/templates/wsgi-freezer-api.conf.j2 | 1 + .../roles/gnocchi/templates/wsgi-gnocchi.conf.j2 | 1 + .../roles/heat/templates/wsgi-heat-api-cfn.conf.j2 | 1 + ansible/roles/heat/templates/wsgi-heat-api.conf.j2 | 1 + .../roles/ironic/templates/ironic-api-wsgi.conf.j2 | 1 + .../roles/keystone/templates/wsgi-keystone.conf.j2 | 1 + .../roles/masakari/templates/wsgi-masakari.conf.j2 | 1 + .../monasca/templates/monasca-api/wsgi-api.conf.j2 | 1 + ansible/roles/nova/templates/nova-api-wsgi.conf.j2 | 1 + ansible/roles/panko/templates/wsgi-panko.conf.j2 | 1 + .../placement/templates/placement-api-wsgi.conf.j2 | 1 + .../roles/vitrage/templates/wsgi-vitrage.conf.j2 | 1 + ansible/roles/zun/templates/wsgi-zun.conf.j2 | 1 + etc/kolla/globals.yml | 1 + .../notes/add-httpd-timeout-717050ffdc4ea9e8.yaml | 13 +++++++++++++ 19 files changed, 31 insertions(+) create mode 100644 releasenotes/notes/add-httpd-timeout-717050ffdc4ea9e8.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 1454e826e7..f3ac36bbdd 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -794,6 +794,7 @@ haproxy_backend_cacert_dir: "/etc/ssl/certs" # Backend options ################## kolla_httpd_keep_alive: "60" +kolla_httpd_timeout: "60" ###################### # Backend TLS options diff --git a/ansible/roles/aodh/templates/wsgi-aodh.conf.j2 b/ansible/roles/aodh/templates/wsgi-aodh.conf.j2 index 54ff5e11ed..86f5d33c21 100644 --- a/ansible/roles/aodh/templates/wsgi-aodh.conf.j2 +++ b/ansible/roles/aodh/templates/wsgi-aodh.conf.j2 @@ -5,6 +5,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ aodh_api_l ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} ErrorLog "{{ aodh_log_dir }}/apache-error.log" diff --git a/ansible/roles/cinder/templates/cinder-wsgi.conf.j2 b/ansible/roles/cinder/templates/cinder-wsgi.conf.j2 index 16deff8f06..b5a8288730 100644 --- a/ansible/roles/cinder/templates/cinder-wsgi.conf.j2 +++ b/ansible/roles/cinder/templates/cinder-wsgi.conf.j2 @@ -10,6 +10,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ cinder_api ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} {% if cinder_logging_debug | bool %} diff --git a/ansible/roles/cloudkitty/templates/wsgi-cloudkitty.conf.j2 b/ansible/roles/cloudkitty/templates/wsgi-cloudkitty.conf.j2 index 14aac3f048..5da0b72aef 100644 --- a/ansible/roles/cloudkitty/templates/wsgi-cloudkitty.conf.j2 +++ b/ansible/roles/cloudkitty/templates/wsgi-cloudkitty.conf.j2 @@ -5,6 +5,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ cloudkitty ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} ErrorLog "{{ cloudkitty_log_dir }}/apache-error.log" diff --git a/ansible/roles/freezer/templates/wsgi-freezer-api.conf.j2 b/ansible/roles/freezer/templates/wsgi-freezer-api.conf.j2 index 8be3aa8508..7239bcb6e8 100644 --- a/ansible/roles/freezer/templates/wsgi-freezer-api.conf.j2 +++ b/ansible/roles/freezer/templates/wsgi-freezer-api.conf.j2 @@ -5,6 +5,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ freezer_ap ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} ErrorLog "{{ freezer_log_dir }}/apache-error.log" diff --git a/ansible/roles/gnocchi/templates/wsgi-gnocchi.conf.j2 b/ansible/roles/gnocchi/templates/wsgi-gnocchi.conf.j2 index d54942ced9..12eb1748db 100644 --- a/ansible/roles/gnocchi/templates/wsgi-gnocchi.conf.j2 +++ b/ansible/roles/gnocchi/templates/wsgi-gnocchi.conf.j2 @@ -5,6 +5,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ gnocchi_ap ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} ErrorLog "{{ gnocchi_log_dir }}/apache-error.log" diff --git a/ansible/roles/heat/templates/wsgi-heat-api-cfn.conf.j2 b/ansible/roles/heat/templates/wsgi-heat-api-cfn.conf.j2 index af0c5ef31c..ab4e1a1fb1 100644 --- a/ansible/roles/heat/templates/wsgi-heat-api-cfn.conf.j2 +++ b/ansible/roles/heat/templates/wsgi-heat-api-cfn.conf.j2 @@ -12,6 +12,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ heat_api_c ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} diff --git a/ansible/roles/heat/templates/wsgi-heat-api.conf.j2 b/ansible/roles/heat/templates/wsgi-heat-api.conf.j2 index 269a27317f..a164af2dbd 100644 --- a/ansible/roles/heat/templates/wsgi-heat-api.conf.j2 +++ b/ansible/roles/heat/templates/wsgi-heat-api.conf.j2 @@ -12,6 +12,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ heat_api_l ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} diff --git a/ansible/roles/ironic/templates/ironic-api-wsgi.conf.j2 b/ansible/roles/ironic/templates/ironic-api-wsgi.conf.j2 index 2e6bdc72c5..bc5c13c9f4 100644 --- a/ansible/roles/ironic/templates/ironic-api-wsgi.conf.j2 +++ b/ansible/roles/ironic/templates/ironic-api-wsgi.conf.j2 @@ -12,6 +12,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ ironic_api ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} diff --git a/ansible/roles/keystone/templates/wsgi-keystone.conf.j2 b/ansible/roles/keystone/templates/wsgi-keystone.conf.j2 index 447c314e67..124ea2ce69 100644 --- a/ansible/roles/keystone/templates/wsgi-keystone.conf.j2 +++ b/ansible/roles/keystone/templates/wsgi-keystone.conf.j2 @@ -13,6 +13,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ keystone_a ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} ErrorLog "{{ keystone_log_dir }}/apache-error.log" diff --git a/ansible/roles/masakari/templates/wsgi-masakari.conf.j2 b/ansible/roles/masakari/templates/wsgi-masakari.conf.j2 index 214ce91d66..b0148177e4 100644 --- a/ansible/roles/masakari/templates/wsgi-masakari.conf.j2 +++ b/ansible/roles/masakari/templates/wsgi-masakari.conf.j2 @@ -6,6 +6,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ masakari_a ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} ErrorLog "{{ masakari_log_dir }}/apache-error.log" diff --git a/ansible/roles/monasca/templates/monasca-api/wsgi-api.conf.j2 b/ansible/roles/monasca/templates/monasca-api/wsgi-api.conf.j2 index 147df20df9..93e2e3ce36 100644 --- a/ansible/roles/monasca/templates/monasca-api/wsgi-api.conf.j2 +++ b/ansible/roles/monasca/templates/monasca-api/wsgi-api.conf.j2 @@ -4,6 +4,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ monasca_api_port }} TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} ErrorLog "{{ monasca_log_dir }}/apache-api-error.log" diff --git a/ansible/roles/nova/templates/nova-api-wsgi.conf.j2 b/ansible/roles/nova/templates/nova-api-wsgi.conf.j2 index b1592415ee..798547cde0 100644 --- a/ansible/roles/nova/templates/nova-api-wsgi.conf.j2 +++ b/ansible/roles/nova/templates/nova-api-wsgi.conf.j2 @@ -13,6 +13,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ nova_metad ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} diff --git a/ansible/roles/panko/templates/wsgi-panko.conf.j2 b/ansible/roles/panko/templates/wsgi-panko.conf.j2 index d7f94589ee..f94bf02802 100644 --- a/ansible/roles/panko/templates/wsgi-panko.conf.j2 +++ b/ansible/roles/panko/templates/wsgi-panko.conf.j2 @@ -5,6 +5,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ panko_api_ ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} ErrorLog "{{ panko_log_dir }}/apache-error.log" diff --git a/ansible/roles/placement/templates/placement-api-wsgi.conf.j2 b/ansible/roles/placement/templates/placement-api-wsgi.conf.j2 index 282635ecf0..2d99f8d12d 100644 --- a/ansible/roles/placement/templates/placement-api-wsgi.conf.j2 +++ b/ansible/roles/placement/templates/placement-api-wsgi.conf.j2 @@ -12,6 +12,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ placement_ ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} {% if placement_logging_debug | bool %} diff --git a/ansible/roles/vitrage/templates/wsgi-vitrage.conf.j2 b/ansible/roles/vitrage/templates/wsgi-vitrage.conf.j2 index 42bf47f00c..9a843e8aaf 100644 --- a/ansible/roles/vitrage/templates/wsgi-vitrage.conf.j2 +++ b/ansible/roles/vitrage/templates/wsgi-vitrage.conf.j2 @@ -4,6 +4,7 @@ Listen {{ 'api' | kolla_address | put_address_in_context('url') }}:{{ vitrage_ap ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} ErrorLog "{{ vitrage_log_dir }}/apache-error.log" diff --git a/ansible/roles/zun/templates/wsgi-zun.conf.j2 b/ansible/roles/zun/templates/wsgi-zun.conf.j2 index 0c32ead491..9e09779b6e 100644 --- a/ansible/roles/zun/templates/wsgi-zun.conf.j2 +++ b/ansible/roles/zun/templates/wsgi-zun.conf.j2 @@ -4,6 +4,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ zun_api_po ServerSignature Off ServerTokens Prod TraceEnable off +TimeOut {{ kolla_httpd_timeout }} KeepAliveTimeout {{ kolla_httpd_keep_alive }} ErrorLog "{{ zun_log_dir }}/apache-error.log" diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 04b1655f1b..4dea095e60 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -214,6 +214,7 @@ # Backend options ################## #kolla_httpd_keep_alive: "60" +#kolla_httpd_timeout: "60" ##################### # Backend TLS options diff --git a/releasenotes/notes/add-httpd-timeout-717050ffdc4ea9e8.yaml b/releasenotes/notes/add-httpd-timeout-717050ffdc4ea9e8.yaml new file mode 100644 index 0000000000..2244e65544 --- /dev/null +++ b/releasenotes/notes/add-httpd-timeout-717050ffdc4ea9e8.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + Adds configuration parameter ``kolla_httpd_timeout`` to configure the + ``TimeOut`` directive for services that use Apache HTTP server to + handle HTTP requests. The default value is 60 seconds which matches + the original default, but you may wish to increase this. +fixes: + - | + Fixes an issue where it was not easily possible to set the Apache + HTTP timeout directive, where the default of 60s would cause + problems in slow running services. See + `LP#1917648 `__.