From 70279972b6afd75a5755054f51840a376b7cd0b9 Mon Sep 17 00:00:00 2001 From: Aravindh Murugesan Date: Tue, 7 May 2024 17:50:34 +0530 Subject: [PATCH] HAProxy: Switch to L7 Healthchecks Address occasional issues where TCP connections appear healthy, yet the web servers within containers fail to respond, resulting in requests being sent to unhealthy servers. Implemented for services I currently use. - Aodh (OPTIONS to /, expects 2XX or 3XX) - Barbican (OPTIONS to /, expects 2XX or 3XX) - Blazar (OPTIONS to /, expects 401) - Cinder API (OPTIONS to /, expects 2XX or 3XX) - CloudKitty (OPTIONS to /, expects 2XX or 3XX) - Designate (OPTIONS to /, expects 2XX or 3XX) - Glance (OPTIONS to /, expects 2XX or 3XX) - Gnocchi (OPTIONS to /, expects 2XX or 3XX) - Grafana (OPTIONS to /, expects 2XX or 3XX) - Heat (OPTIONS to /, expects 2XX or 3XX) - Horizon (OPTIONS to /, expects 2XX or 3XX) - Ironic (OPTIONS to /, expects 2XX or 3XX) - Keystone (OPTIONS to /, expects 2XX or 3XX) - Magnum (OPTIONS to /, expects 2XX or 3XX) - Manila (OPTIONS to /, expects 2XX or 3XX) - Masakari (OPTIONS to /, expects 2XX or 3XX) - Mistral (OPTIONS to /, expects 2XX or 3XX) - Nova API (OPTIONS to /, expects 2XX or 3XX) - Nova Metadata (OPTIONS to /, expects 2XX or 3XX) - Neutron (OPTIONS to /, expects 2XX or 3XX) - Opensearch (OPTIONS to /, expects 2XX or 3XX) - Opensearch Dashboards (OPTIONS to /, expects 401) - Placement (GET to /, expects 2XX or 3XX) - Prometheus (OPTIONS to /, expects 2XX or 3XX) - Prometheus AlertManager (OPTIONS to /, expects 2XX or 3XX) - Prometheus Openstack Exporter (OPTIONS to /, expects 2XX or 3XX) - Prometheus Server (OPTIONS to /, expects 2XX or 3XX) - Skyline API (OPTIONS to /docs, expects 2XX or 3XX) - Skyline Console (GET to /, expects 2XX or 3XX) - Swift (OPTIONS to /info, expects 2XX or 3XX) - Trove (OPTIONS to /, expects 2XX or 3XX) - Venus (OPTIONS to /, expects 2XX or 3XX) - Watcher (GET to /, expects 2XX or 3XX) - Zun (OPTIONS to /, expects 2XX or 3XX) Change-Id: I839f7f1051182fe797394e5436571d64d5c5b5a4 --- ansible/roles/aodh/defaults/main.yml | 4 ++++ ansible/roles/barbican/defaults/main.yml | 4 ++++ ansible/roles/blazar/defaults/main.yml | 10 ++++++++++ ansible/roles/cinder/defaults/main.yml | 4 ++++ ansible/roles/cloudkitty/defaults/main.yml | 4 ++++ ansible/roles/designate/defaults/main.yml | 4 ++++ ansible/roles/glance/defaults/main.yml | 4 ++++ ansible/roles/gnocchi/defaults/main.yml | 4 ++++ ansible/roles/grafana/defaults/main.yml | 4 ++++ ansible/roles/heat/defaults/main.yml | 8 ++++++++ ansible/roles/horizon/defaults/main.yml | 2 ++ ansible/roles/ironic/defaults/main.yml | 8 ++++++++ ansible/roles/keystone/defaults/main.yml | 2 ++ ansible/roles/magnum/defaults/main.yml | 4 ++++ ansible/roles/manila/defaults/main.yml | 4 ++++ ansible/roles/masakari/defaults/main.yml | 4 ++++ ansible/roles/mistral/defaults/main.yml | 4 ++++ ansible/roles/neutron/defaults/main.yml | 4 ++++ ansible/roles/nova/defaults/main.yml | 8 ++++++++ ansible/roles/opensearch/defaults/main.yml | 8 ++++++++ ansible/roles/placement/defaults/main.yml | 4 ++++ ansible/roles/prometheus/defaults/main.yml | 10 ++++++++++ ansible/roles/skyline/defaults/main.yml | 8 ++++++++ ansible/roles/swift/defaults/main.yml | 4 ++++ ansible/roles/trove/defaults/main.yml | 4 ++++ ansible/roles/venus/defaults/main.yml | 4 ++++ ansible/roles/watcher/defaults/main.yml | 4 ++++ ansible/roles/zun/defaults/main.yml | 4 ++++ .../haproxy_healthchecks_to_l7-b05e8c7b177d1544.yaml | 5 +++++ 29 files changed, 145 insertions(+) create mode 100644 releasenotes/notes/haproxy_healthchecks_to_l7-b05e8c7b177d1544.yaml diff --git a/ansible/roles/aodh/defaults/main.yml b/ansible/roles/aodh/defaults/main.yml index bb18d9cc30..bbe08bc1d7 100644 --- a/ansible/roles/aodh/defaults/main.yml +++ b/ansible/roles/aodh/defaults/main.yml @@ -15,6 +15,8 @@ aodh_services: external: false port: "{{ aodh_api_port }}" listen_port: "{{ aodh_api_listen_port }}" + backend_http_extra: + - "option httpchk" aodh_api_external: enabled: "{{ enable_aodh }}" mode: "http" @@ -22,6 +24,8 @@ aodh_services: external_fqdn: "{{ aodh_external_fqdn }}" port: "{{ aodh_api_public_port }}" listen_port: "{{ aodh_api_listen_port }}" + backend_http_extra: + - "option httpchk" aodh-evaluator: container_name: aodh_evaluator group: aodh-evaluator diff --git a/ansible/roles/barbican/defaults/main.yml b/ansible/roles/barbican/defaults/main.yml index 07e79ece6d..6553375346 100644 --- a/ansible/roles/barbican/defaults/main.yml +++ b/ansible/roles/barbican/defaults/main.yml @@ -16,6 +16,8 @@ barbican_services: port: "{{ barbican_api_port }}" listen_port: "{{ barbican_api_listen_port }}" tls_backend: "{{ barbican_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" barbican_api_external: enabled: "{{ enable_barbican }}" mode: "http" @@ -24,6 +26,8 @@ barbican_services: port: "{{ barbican_api_public_port }}" listen_port: "{{ barbican_api_listen_port }}" tls_backend: "{{ barbican_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" barbican-keystone-listener: container_name: barbican_keystone_listener group: barbican-keystone-listener diff --git a/ansible/roles/blazar/defaults/main.yml b/ansible/roles/blazar/defaults/main.yml index 81e8325b99..be3785543a 100644 --- a/ansible/roles/blazar/defaults/main.yml +++ b/ansible/roles/blazar/defaults/main.yml @@ -15,6 +15,11 @@ blazar_services: external: false port: "{{ blazar_api_port }}" listen_port: "{{ blazar_api_listen_port }}" + backend_http_extra: + ## All of Blazar's API endpoints are private. + ## So we should accept 401 as valid healthy response. + - "option httpchk" + - "http-check expect status 401" blazar_api_external: enabled: "{{ enable_blazar }}" mode: "http" @@ -22,6 +27,11 @@ blazar_services: external_fqdn: "{{ blazar_external_fqdn }}" port: "{{ blazar_api_public_port }}" listen_port: "{{ blazar_api_listen_port }}" + backend_http_extra: + ## All of Blazar's API endpoints are private. + ## So we should accept 401 as valid healthy response. + - "option httpchk" + - "http-check expect status 401" blazar-manager: container_name: blazar_manager group: blazar-manager diff --git a/ansible/roles/cinder/defaults/main.yml b/ansible/roles/cinder/defaults/main.yml index 6e5d30dad0..8faaf3426e 100644 --- a/ansible/roles/cinder/defaults/main.yml +++ b/ansible/roles/cinder/defaults/main.yml @@ -16,6 +16,8 @@ cinder_services: port: "{{ cinder_api_port }}" listen_port: "{{ cinder_api_listen_port }}" tls_backend: "{{ cinder_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" cinder_api_external: enabled: "{{ enable_cinder }}" mode: "http" @@ -24,6 +26,8 @@ cinder_services: port: "{{ cinder_api_public_port }}" listen_port: "{{ cinder_api_listen_port }}" tls_backend: "{{ cinder_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" cinder-scheduler: container_name: cinder_scheduler group: cinder-scheduler diff --git a/ansible/roles/cloudkitty/defaults/main.yml b/ansible/roles/cloudkitty/defaults/main.yml index c462a0450c..b6ba8969d8 100644 --- a/ansible/roles/cloudkitty/defaults/main.yml +++ b/ansible/roles/cloudkitty/defaults/main.yml @@ -15,6 +15,8 @@ cloudkitty_services: external: false port: "{{ cloudkitty_api_port }}" listen_port: "{{ cloudkitty_api_listen_port }}" + backend_http_extra: + - "option httpchk" cloudkitty_api_external: enabled: "{{ enable_cloudkitty }}" mode: "http" @@ -22,6 +24,8 @@ cloudkitty_services: external_fqdn: "{{ cloudkitty_external_fqdn }}" port: "{{ cloudkitty_api_public_port }}" listen_port: "{{ cloudkitty_api_listen_port }}" + backend_http_extra: + - "option httpchk" cloudkitty-processor: container_name: "cloudkitty_processor" group: "cloudkitty-processor" diff --git a/ansible/roles/designate/defaults/main.yml b/ansible/roles/designate/defaults/main.yml index be3d670006..e4dba2ff19 100644 --- a/ansible/roles/designate/defaults/main.yml +++ b/ansible/roles/designate/defaults/main.yml @@ -15,6 +15,8 @@ designate_services: external: false port: "{{ designate_api_port }}" listen_port: "{{ designate_api_listen_port }}" + backend_http_extra: + - "option httpchk" designate_api_external: enabled: "{{ enable_designate }}" mode: "http" @@ -22,6 +24,8 @@ designate_services: external_fqdn: "{{ designate_external_fqdn }}" port: "{{ designate_api_public_port }}" listen_port: "{{ designate_api_listen_port }}" + backend_http_extra: + - "option httpchk" designate-backend-bind9: container_name: designate_backend_bind9 group: designate-backend-bind9 diff --git a/ansible/roles/glance/defaults/main.yml b/ansible/roles/glance/defaults/main.yml index c791b58cf4..494bf6b054 100644 --- a/ansible/roles/glance/defaults/main.yml +++ b/ansible/roles/glance/defaults/main.yml @@ -21,6 +21,7 @@ glance_services: - "timeout client {{ haproxy_glance_api_client_timeout }}" backend_http_extra: - "timeout server {{ haproxy_glance_api_server_timeout }}" + - "option httpchk" custom_member_list: "{{ haproxy_members.split(';') }}" glance_api_external: enabled: "{{ enable_glance | bool and not glance_enable_tls_backend | bool }}" @@ -32,6 +33,7 @@ glance_services: - "timeout client {{ haproxy_glance_api_client_timeout }}" backend_http_extra: - "timeout server {{ haproxy_glance_api_server_timeout }}" + - "option httpchk" custom_member_list: "{{ haproxy_members.split(';') }}" glance-tls-proxy: container_name: glance_tls_proxy @@ -52,6 +54,7 @@ glance_services: - "timeout client {{ haproxy_glance_api_client_timeout }}" backend_http_extra: - "timeout server {{ haproxy_glance_api_server_timeout }}" + - "option httpchk" custom_member_list: "{{ haproxy_tls_members.split(';') }}" tls_backend: "yes" glance_tls_proxy_external: @@ -64,6 +67,7 @@ glance_services: - "timeout client {{ haproxy_glance_api_client_timeout }}" backend_http_extra: - "timeout server {{ haproxy_glance_api_server_timeout }}" + - "option httpchk" custom_member_list: "{{ haproxy_tls_members.split(';') }}" tls_backend: "yes" diff --git a/ansible/roles/gnocchi/defaults/main.yml b/ansible/roles/gnocchi/defaults/main.yml index c27147cf1d..90bb021fbe 100644 --- a/ansible/roles/gnocchi/defaults/main.yml +++ b/ansible/roles/gnocchi/defaults/main.yml @@ -15,6 +15,8 @@ gnocchi_services: external: false port: "{{ gnocchi_api_port }}" listen_port: "{{ gnocchi_api_listen_port }}" + backend_http_extra: + - "option httpchk" gnocchi_api_external: enabled: "{{ enable_gnocchi }}" mode: "http" @@ -22,6 +24,8 @@ gnocchi_services: external_fqdn: "{{ gnocchi_external_fqdn }}" port: "{{ gnocchi_api_public_port }}" listen_port: "{{ gnocchi_api_listen_port }}" + backend_http_extra: + - "option httpchk" gnocchi-metricd: container_name: gnocchi_metricd group: gnocchi-metricd diff --git a/ansible/roles/grafana/defaults/main.yml b/ansible/roles/grafana/defaults/main.yml index cce9562fe2..1b0fee3f1c 100644 --- a/ansible/roles/grafana/defaults/main.yml +++ b/ansible/roles/grafana/defaults/main.yml @@ -14,6 +14,8 @@ grafana_services: external: false port: "{{ grafana_server_port }}" listen_port: "{{ grafana_server_listen_port }}" + backend_http_extra: + - "option httpchk" grafana_server_external: enabled: "{{ enable_grafana_external | bool }}" mode: "http" @@ -21,6 +23,8 @@ grafana_services: external_fqdn: "{{ grafana_external_fqdn }}" port: "{{ grafana_server_public_port }}" listen_port: "{{ grafana_server_listen_port }}" + backend_http_extra: + - "option httpchk" #################### # Database diff --git a/ansible/roles/heat/defaults/main.yml b/ansible/roles/heat/defaults/main.yml index b465f010fd..c659499da7 100644 --- a/ansible/roles/heat/defaults/main.yml +++ b/ansible/roles/heat/defaults/main.yml @@ -16,6 +16,8 @@ heat_services: port: "{{ heat_api_port }}" listen_port: "{{ heat_api_listen_port }}" tls_backend: "{{ heat_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" heat_api_external: enabled: "{{ enable_heat }}" mode: "http" @@ -24,6 +26,8 @@ heat_services: port: "{{ heat_api_public_port }}" listen_port: "{{ heat_api_listen_port }}" tls_backend: "{{ heat_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" heat-api-cfn: container_name: heat_api_cfn group: heat-api-cfn @@ -40,6 +44,8 @@ heat_services: port: "{{ heat_api_cfn_port }}" listen_port: "{{ heat_api_cfn_listen_port }}" tls_backend: "{{ heat_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" heat_api_cfn_external: enabled: "{{ enable_heat }}" mode: "http" @@ -48,6 +54,8 @@ heat_services: port: "{{ heat_api_cfn_public_port }}" listen_port: "{{ heat_api_cfn_listen_port }}" tls_backend: "{{ heat_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" heat-engine: container_name: heat_engine group: heat-engine diff --git a/ansible/roles/horizon/defaults/main.yml b/ansible/roles/horizon/defaults/main.yml index 0caf95ef5c..a17b0f7548 100644 --- a/ansible/roles/horizon/defaults/main.yml +++ b/ansible/roles/horizon/defaults/main.yml @@ -37,6 +37,7 @@ horizon_services: - "use_backend acme_client_back if { path_reg ^/.well-known/acme-challenge/.+ }" backend_http_extra: - "balance {{ 'roundrobin' if enable_memcached | bool or horizon_backend_database | bool else 'source' }}" + - "option httpchk" tls_backend: "{{ horizon_enable_tls_backend }}" horizon_redirect: enabled: "{{ enable_horizon | bool and kolla_enable_tls_internal | bool }}" @@ -57,6 +58,7 @@ horizon_services: - "use_backend acme_client_back if { path_reg ^/.well-known/acme-challenge/.+ }" backend_http_extra: - "balance {{ 'roundrobin' if enable_memcached | bool or horizon_backend_database | bool else 'source' }}" + - "option httpchk" tls_backend: "{{ horizon_enable_tls_backend }}" horizon_external_redirect: enabled: "{{ enable_horizon | bool and kolla_enable_tls_external | bool }}" diff --git a/ansible/roles/ironic/defaults/main.yml b/ansible/roles/ironic/defaults/main.yml index 8f5d7bc9f2..13b95e47e1 100644 --- a/ansible/roles/ironic/defaults/main.yml +++ b/ansible/roles/ironic/defaults/main.yml @@ -16,6 +16,8 @@ ironic_services: port: "{{ ironic_api_port }}" listen_port: "{{ ironic_api_listen_port }}" tls_backend: "{{ ironic_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" ironic_api_external: enabled: "{{ enable_ironic }}" mode: "http" @@ -24,6 +26,8 @@ ironic_services: port: "{{ ironic_api_public_port }}" listen_port: "{{ ironic_api_listen_port }}" tls_backend: "{{ ironic_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" ironic-conductor: container_name: ironic_conductor group: ironic-conductor @@ -49,6 +53,8 @@ ironic_services: external: false port: "{{ ironic_inspector_port }}" listen_port: "{{ ironic_inspector_listen_port }}" + backend_http_extra: + - "option httpchk" ironic_inspector_external: enabled: "{{ enable_ironic }}" mode: "http" @@ -56,6 +62,8 @@ ironic_services: external_fqdn: "{{ ironic_inspector_external_fqdn }}" port: "{{ ironic_inspector_public_port }}" listen_port: "{{ ironic_inspector_listen_port }}" + backend_http_extra: + - "option httpchk" ironic-tftp: container_name: ironic_tftp group: ironic-tftp diff --git a/ansible/roles/keystone/defaults/main.yml b/ansible/roles/keystone/defaults/main.yml index c3266e1773..8282118fa4 100644 --- a/ansible/roles/keystone/defaults/main.yml +++ b/ansible/roles/keystone/defaults/main.yml @@ -18,6 +18,7 @@ keystone_services: listen_port: "{{ keystone_internal_listen_port }}" backend_http_extra: - "balance {{ 'source' if enable_keystone_federation | bool else 'roundrobin' }}" + - "option httpchk" keystone_external: enabled: "{{ enable_keystone }}" mode: "http" @@ -28,6 +29,7 @@ keystone_services: listen_port: "{{ keystone_public_listen_port }}" backend_http_extra: - "balance {{ 'source' if enable_keystone_federation | bool else 'roundrobin' }}" + - "option httpchk" keystone-ssh: container_name: "keystone_ssh" group: "keystone" diff --git a/ansible/roles/magnum/defaults/main.yml b/ansible/roles/magnum/defaults/main.yml index 9f9e1cf3ef..9efa81d333 100644 --- a/ansible/roles/magnum/defaults/main.yml +++ b/ansible/roles/magnum/defaults/main.yml @@ -17,6 +17,8 @@ magnum_services: external: false port: "{{ magnum_api_port }}" listen_port: "{{ magnum_api_listen_port }}" + backend_http_extra: + - "option httpchk" magnum_api_external: enabled: "{{ enable_magnum }}" mode: "http" @@ -24,6 +26,8 @@ magnum_services: external_fqdn: "{{ magnum_external_fqdn }}" port: "{{ magnum_api_public_port }}" listen_port: "{{ magnum_api_listen_port }}" + backend_http_extra: + - "option httpchk" magnum-conductor: container_name: magnum_conductor group: magnum-conductor diff --git a/ansible/roles/manila/defaults/main.yml b/ansible/roles/manila/defaults/main.yml index 3ad48f651a..76c6a530e2 100644 --- a/ansible/roles/manila/defaults/main.yml +++ b/ansible/roles/manila/defaults/main.yml @@ -15,6 +15,8 @@ manila_services: external: false port: "{{ manila_api_port }}" listen_port: "{{ manila_api_listen_port }}" + backend_http_extra: + - "option httpchk" manila_api_external: enabled: "{{ enable_manila }}" mode: "http" @@ -22,6 +24,8 @@ manila_services: external_fqdn: "{{ manila_external_fqdn }}" port: "{{ manila_api_public_port }}" listen_port: "{{ manila_api_listen_port }}" + backend_http_extra: + - "option httpchk" manila-scheduler: container_name: "manila_scheduler" group: "manila-scheduler" diff --git a/ansible/roles/masakari/defaults/main.yml b/ansible/roles/masakari/defaults/main.yml index 8925f6b848..440eed47b1 100644 --- a/ansible/roles/masakari/defaults/main.yml +++ b/ansible/roles/masakari/defaults/main.yml @@ -14,6 +14,8 @@ masakari_services: external: false port: "{{ masakari_api_port }}" listen_port: "{{ masakari_api_listen_port }}" + backend_http_extra: + - "option httpchk" masakari_api_external: enabled: "{{ enable_masakari }}" mode: "http" @@ -21,6 +23,8 @@ masakari_services: external_fqdn: "{{ masakari_external_fqdn }}" port: "{{ masakari_api_public_port }}" listen_port: "{{ masakari_api_listen_port }}" + backend_http_extra: + - "option httpchk" masakari-engine: container_name: masakari_engine group: masakari-engine diff --git a/ansible/roles/mistral/defaults/main.yml b/ansible/roles/mistral/defaults/main.yml index beb9a85fe0..0229bd74b3 100644 --- a/ansible/roles/mistral/defaults/main.yml +++ b/ansible/roles/mistral/defaults/main.yml @@ -15,6 +15,8 @@ mistral_services: external: false port: "{{ mistral_api_port }}" listen_port: "{{ mistral_api_listen_port }}" + backend_http_extra: + - "option httpchk" mistral_api_external: enabled: "{{ enable_mistral }}" mode: "http" @@ -22,6 +24,8 @@ mistral_services: external_fqdn: "{{ mistral_external_fqdn }}" port: "{{ mistral_api_public_port }}" listen_port: "{{ mistral_api_listen_port }}" + backend_http_extra: + - "option httpchk" mistral-engine: container_name: mistral_engine group: mistral-engine diff --git a/ansible/roles/neutron/defaults/main.yml b/ansible/roles/neutron/defaults/main.yml index 2501ce3a42..bb50c84772 100644 --- a/ansible/roles/neutron/defaults/main.yml +++ b/ansible/roles/neutron/defaults/main.yml @@ -16,6 +16,8 @@ neutron_services: external: false port: "{{ neutron_server_port }}" listen_port: "{{ neutron_server_listen_port }}" + backend_http_extra: + - "option httpchk" neutron_server_external: enabled: "{{ enable_neutron | bool and not neutron_enable_tls_backend | bool }}" mode: "http" @@ -23,6 +25,8 @@ neutron_services: external_fqdn: "{{ neutron_external_fqdn }}" port: "{{ neutron_server_public_port }}" listen_port: "{{ neutron_server_listen_port }}" + backend_http_extra: + - "option httpchk" neutron-openvswitch-agent: container_name: "neutron_openvswitch_agent" image: "{{ neutron_openvswitch_agent_image_full }}" diff --git a/ansible/roles/nova/defaults/main.yml b/ansible/roles/nova/defaults/main.yml index cbe554b878..4b508d073b 100644 --- a/ansible/roles/nova/defaults/main.yml +++ b/ansible/roles/nova/defaults/main.yml @@ -17,6 +17,8 @@ nova_services: port: "{{ nova_api_port }}" listen_port: "{{ nova_api_listen_port }}" tls_backend: "{{ nova_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" nova_api_external: enabled: "{{ enable_nova }}" mode: "http" @@ -25,6 +27,8 @@ nova_services: port: "{{ nova_api_public_port }}" listen_port: "{{ nova_api_listen_port }}" tls_backend: "{{ nova_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" nova-metadata: container_name: "nova_metadata" group: "nova-metadata" @@ -41,6 +45,8 @@ nova_services: port: "{{ nova_metadata_port }}" listen_port: "{{ nova_metadata_listen_port }}" tls_backend: "{{ nova_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" nova_metadata_external: enabled: "{{ nova_enable_external_metadata }}" mode: "http" @@ -49,6 +55,8 @@ nova_services: port: "{{ nova_metadata_port }}" listen_port: "{{ nova_metadata_listen_port }}" tls_backend: "{{ nova_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" nova-scheduler: container_name: "nova_scheduler" group: "nova-scheduler" diff --git a/ansible/roles/opensearch/defaults/main.yml b/ansible/roles/opensearch/defaults/main.yml index 5d419e0b49..8a0fa366fc 100644 --- a/ansible/roles/opensearch/defaults/main.yml +++ b/ansible/roles/opensearch/defaults/main.yml @@ -18,6 +18,8 @@ opensearch_services: port: "{{ opensearch_port }}" frontend_http_extra: - "option dontlog-normal" + backend_http_extra: + - "option httpchk" opensearch-dashboards: container_name: opensearch_dashboards group: opensearch-dashboards @@ -36,6 +38,9 @@ opensearch_services: port: "{{ opensearch_dashboards_port }}" auth_user: "{{ opensearch_dashboards_user }}" auth_pass: "{{ opensearch_dashboards_password }}" + backend_http_extra: + - "option httpchk" + - "http-check expect status 401" opensearch_dashboards_external: enabled: "{{ enable_opensearch_dashboards_external | bool }}" mode: "http" @@ -45,6 +50,9 @@ opensearch_services: listen_port: "{{ opensearch_dashboards_listen_port }}" auth_user: "{{ opensearch_dashboards_user }}" auth_pass: "{{ opensearch_dashboards_password }}" + backend_http_extra: + - "option httpchk" + - "http-check expect status 401" #################### diff --git a/ansible/roles/placement/defaults/main.yml b/ansible/roles/placement/defaults/main.yml index d189408e88..788c10dec9 100644 --- a/ansible/roles/placement/defaults/main.yml +++ b/ansible/roles/placement/defaults/main.yml @@ -16,6 +16,8 @@ placement_services: port: "{{ placement_api_port }}" listen_port: "{{ placement_api_listen_port }}" tls_backend: "{{ placement_enable_tls_backend }}" + backend_http_extra: + - "option httpchk GET /" placement_api_external: enabled: "{{ enable_placement }}" mode: "http" @@ -24,6 +26,8 @@ placement_services: port: "{{ placement_api_public_port }}" listen_port: "{{ placement_api_listen_port }}" tls_backend: "{{ placement_enable_tls_backend }}" + backend_http_extra: + - "option httpchk GET /" #################### # Config Validate diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml index 71168fb326..e96988434b 100644 --- a/ansible/roles/prometheus/defaults/main.yml +++ b/ansible/roles/prometheus/defaults/main.yml @@ -14,6 +14,8 @@ prometheus_services: external: false port: "{{ prometheus_port }}" active_passive: "{{ prometheus_active_passive | bool }}" + backend_http_extra: + - "option httpchk" prometheus_server_external: enabled: "{{ enable_prometheus_server_external | bool }}" mode: "http" @@ -22,6 +24,8 @@ prometheus_services: port: "{{ prometheus_public_port }}" listen_port: "{{ prometheus_listen_port }}" active_passive: "{{ prometheus_active_passive | bool }}" + backend_http_extra: + - "option httpchk" prometheus-node-exporter: container_name: prometheus_node_exporter group: prometheus-node-exporter @@ -67,6 +71,8 @@ prometheus_services: auth_user: "{{ prometheus_alertmanager_user }}" auth_pass: "{{ prometheus_alertmanager_password }}" active_passive: "{{ prometheus_alertmanager_active_passive | bool }}" + backend_http_extra: + - "option httpchk" prometheus_alertmanager_external: enabled: "{{ enable_prometheus_alertmanager_external | bool }}" mode: "http" @@ -77,6 +83,8 @@ prometheus_services: auth_user: "{{ prometheus_alertmanager_user }}" auth_pass: "{{ prometheus_alertmanager_password }}" active_passive: "{{ prometheus_alertmanager_active_passive | bool }}" + backend_http_extra: + - "option httpchk" prometheus-openstack-exporter: container_name: "prometheus_openstack_exporter" group: "prometheus-openstack-exporter" @@ -93,6 +101,7 @@ prometheus_services: external: false port: "{{ prometheus_openstack_exporter_port }}" backend_http_extra: + - "option httpchk" - "timeout server {{ prometheus_openstack_exporter_timeout }}" prometheus_openstack_exporter_external: enabled: "{{ enable_prometheus_openstack_exporter_external | bool }}" @@ -100,6 +109,7 @@ prometheus_services: external: true port: "{{ prometheus_openstack_exporter_port }}" backend_http_extra: + - "option httpchk" - "timeout server {{ prometheus_openstack_exporter_timeout }}" prometheus-elasticsearch-exporter: container_name: prometheus_elasticsearch_exporter diff --git a/ansible/roles/skyline/defaults/main.yml b/ansible/roles/skyline/defaults/main.yml index 9cd77327cb..1363748b99 100644 --- a/ansible/roles/skyline/defaults/main.yml +++ b/ansible/roles/skyline/defaults/main.yml @@ -16,6 +16,8 @@ skyline_services: port: "{{ skyline_apiserver_port }}" listen_port: "{{ skyline_apiserver_listen_port }}" tls_backend: "{{ skyline_enable_tls_backend }}" + backend_http_extra: + - "option httpchk GET /docs" skyline_apiserver_external: enabled: "{{ enable_skyline }}" mode: "http" @@ -24,6 +26,8 @@ skyline_services: port: "{{ skyline_apiserver_port }}" listen_port: "{{ skyline_apiserver_listen_port }}" tls_backend: "{{ skyline_enable_tls_backend }}" + backend_http_extra: + - "option httpchk GET /docs" skyline-console: container_name: skyline_console group: skyline-console @@ -40,6 +44,8 @@ skyline_services: port: "{{ skyline_console_port }}" listen_port: "{{ skyline_console_listen_port }}" tls_backend: "{{ skyline_enable_tls_backend }}" + backend_http_extra: + - "option httpchk GET /" skyline_console_external: enabled: "{{ enable_skyline }}" mode: "http" @@ -48,6 +54,8 @@ skyline_services: port: "{{ skyline_console_port }}" listen_port: "{{ skyline_console_listen_port }}" tls_backend: "{{ skyline_enable_tls_backend }}" + backend_http_extra: + - "option httpchk GET /" #################### # Database diff --git a/ansible/roles/swift/defaults/main.yml b/ansible/roles/swift/defaults/main.yml index f1c65f2fa3..9ee665ade7 100644 --- a/ansible/roles/swift/defaults/main.yml +++ b/ansible/roles/swift/defaults/main.yml @@ -9,12 +9,16 @@ swift_services: mode: "http" external: false port: "{{ swift_proxy_server_listen_port }}" + backend_http_extra: + - "option httpchk /info" swift_api_external: enabled: "{{ enable_swift }}" mode: "http" external: true external_fqdn: "{{ swift_external_fqdn }}" port: "{{ swift_proxy_server_listen_port }}" + backend_http_extra: + - "option httpchk /info" #################### # Docker diff --git a/ansible/roles/trove/defaults/main.yml b/ansible/roles/trove/defaults/main.yml index 1977860bd9..a0edcf32ee 100644 --- a/ansible/roles/trove/defaults/main.yml +++ b/ansible/roles/trove/defaults/main.yml @@ -16,6 +16,8 @@ trove_services: port: "{{ trove_api_port }}" listen_port: "{{ trove_api_listen_port }}" tls_backend: "{{ trove_enable_tls_backend }}" + backend_http_extra: + - "option httpchk" trove_api_external: enabled: "{{ enable_trove }}" mode: "http" @@ -24,6 +26,8 @@ trove_services: tls_backend: "{{ trove_enable_tls_backend }}" external_fqdn: "{{ trove_external_fqdn }}" port: "{{ trove_api_public_port }}" + backend_http_extra: + - "option httpchk" trove-conductor: container_name: trove_conductor group: trove-conductor diff --git a/ansible/roles/venus/defaults/main.yml b/ansible/roles/venus/defaults/main.yml index a8a5ad7a54..4afe95f1eb 100644 --- a/ansible/roles/venus/defaults/main.yml +++ b/ansible/roles/venus/defaults/main.yml @@ -14,12 +14,16 @@ venus_services: mode: "http" external: false port: "{{ venus_api_port }}" + backend_http_extra: + - "option httpchk" venus_api_external: enabled: "{{ enable_venus }}" mode: "http" external: true external_fqdn: "{{ venus_external_fqdn }}" port: "{{ venus_api_public_port }}" + backend_http_extra: + - "option httpchk" venus-manager: container_name: venus_manager group: venus-manager diff --git a/ansible/roles/watcher/defaults/main.yml b/ansible/roles/watcher/defaults/main.yml index f3ea3388a6..75f4d1285e 100644 --- a/ansible/roles/watcher/defaults/main.yml +++ b/ansible/roles/watcher/defaults/main.yml @@ -15,6 +15,8 @@ watcher_services: external: false port: "{{ watcher_api_port }}" listen_port: "{{ watcher_api_listen_port }}" + backend_http_extra: + - "option httpchk GET /" watcher_api_external: enabled: "{{ enable_watcher }}" mode: "http" @@ -22,6 +24,8 @@ watcher_services: external_fqdn: "{{ watcher_external_fqdn }}" port: "{{ watcher_api_public_port }}" listen_port: "{{ watcher_api_listen_port }}" + backend_http_extra: + - "option httpchk GET /" watcher-applier: container_name: watcher_applier group: watcher-applier diff --git a/ansible/roles/zun/defaults/main.yml b/ansible/roles/zun/defaults/main.yml index 009368cb1f..bcf7ec80c2 100644 --- a/ansible/roles/zun/defaults/main.yml +++ b/ansible/roles/zun/defaults/main.yml @@ -15,6 +15,8 @@ zun_services: external: false port: "{{ zun_api_port }}" listen_port: "{{ zun_api_listen_port }}" + backend_http_extra: + - "option httpchk" zun_api_external: enabled: "{{ enable_zun }}" mode: "http" @@ -22,6 +24,8 @@ zun_services: external_fqdn: "{{ zun_external_fqdn }}" port: "{{ zun_api_public_port }}" listen_port: "{{ zun_api_listen_port }}" + backend_http_extra: + - "option httpchk" zun-wsproxy: container_name: zun_wsproxy group: zun-wsproxy diff --git a/releasenotes/notes/haproxy_healthchecks_to_l7-b05e8c7b177d1544.yaml b/releasenotes/notes/haproxy_healthchecks_to_l7-b05e8c7b177d1544.yaml new file mode 100644 index 0000000000..e9308c6d47 --- /dev/null +++ b/releasenotes/notes/haproxy_healthchecks_to_l7-b05e8c7b177d1544.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Implement Layer 7 Healthchecks for HA Proxy. + This should fix traffic being sent to unhealthy servers in some scenarios.