From dafac823d1ba92e6e39f91cc502c394c5134034a Mon Sep 17 00:00:00 2001 From: Michal Arbet Date: Mon, 14 Oct 2024 18:56:45 +0200 Subject: [PATCH] Adjust ProxySQL shunning behavior for single-node clusters In single-node clusters, ProxySQL shuns the server on MySQL errors, causing failures during upgrades or container restarts. This change increases the timeout to 10 seconds, allowing the backend time to recover and preventing immediate errors in CI environments. Change-Id: I70becdc3fcb4ca8f7ae31d26097d95bdc6dd67eb --- ansible/group_vars/all.yml | 14 ++++++++++++++ .../templates/proxysql/proxysql.yaml.j2 | 3 +++ ...l-option-shun-on-failures-559781bbee1235e2.yaml | 10 ++++++++++ 3 files changed, 27 insertions(+) create mode 100644 releasenotes/notes/proxysql-option-shun-on-failures-559781bbee1235e2.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index d929a6c8a0..ba0d9d7ceb 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -526,6 +526,20 @@ mariadb_monitor_galera_healthcheck_max_timeout_count: "2" mariadb_monitor_ping_interval: "3000" mariadb_monitor_ping_timeout: "2000" mariadb_monitor_ping_max_failures: "2" +# +# Defaults preserved for multinode setup +# Tweaked for single-node +# +# ProxySQL shuns servers on MySQL errors, which can cause failures +# during upgrades or restarts. In single-node setups, ProxySQL can't reroute +# traffic, leading to "Max connect timeout" errors. To avoid this in CI and +# signle-node environments, delay error responses to clients by 10 seconds, +# giving the backend time to recover without immediate failures. +# +# See ProxySQL docs for more: https://proxysql.com/documentation/global-variables/mysql-variables/#mysql-shun_on_failures +mariadb_shun_on_failures: "{{ '10' if mariadb_shards_info.shards.values() | map(attribute='hosts') | map('length') | select('<=', 1) | list | length > 0 else '5' }}" +mariadb_connect_retries_delay: "{{ '1000' if mariadb_shards_info.shards.values() | map(attribute='hosts') | map('length') | select('<=', 1) | list | length > 0 else '1' }}" +mariadb_connect_retries_on_failure: "{{ '20' if mariadb_shards_info.shards.values() | map(attribute='hosts') | map('length') | select('<=', 1) | list | length > 0 else '10' }}" mariadb_datadir_volume: "mariadb" diff --git a/ansible/roles/loadbalancer/templates/proxysql/proxysql.yaml.j2 b/ansible/roles/loadbalancer/templates/proxysql/proxysql.yaml.j2 index 878dd62a84..ac632c009f 100644 --- a/ansible/roles/loadbalancer/templates/proxysql/proxysql.yaml.j2 +++ b/ansible/roles/loadbalancer/templates/proxysql/proxysql.yaml.j2 @@ -20,6 +20,9 @@ mysql_variables: threads: {{ proxysql_workers }} max_connections: {{ proxysql_max_connections }} interfaces: "{{ kolla_internal_vip_address | put_address_in_context('url') }}:{{ database_port }}" + connect_retries_delay: "{{ mariadb_connect_retries_delay }}" + connect_retries_on_failure: "{{ mariadb_connect_retries_on_failure }}" + shun_on_failures: "{{ mariadb_shun_on_failures }}" monitor_username: "{{ mariadb_monitor_user }}" monitor_password: "{{ mariadb_monitor_password }}" monitor_connect_interval: "{{ mariadb_monitor_connect_interval }}" diff --git a/releasenotes/notes/proxysql-option-shun-on-failures-559781bbee1235e2.yaml b/releasenotes/notes/proxysql-option-shun-on-failures-559781bbee1235e2.yaml new file mode 100644 index 0000000000..c7aabf9fd7 --- /dev/null +++ b/releasenotes/notes/proxysql-option-shun-on-failures-559781bbee1235e2.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Added options to configure ``mariadb_shun_on_failures``, + ``mariadb_connect_retries_delay``, and + ``mariadb_connect_retries_on_failure`` for enhanced + control over ProxySQL's shun behavior. These + adjustments help manage failover responses effectively. + For more details, see `Proxysql Documentation + `__