From 6c037790f2802d626902d377486ec2f8ce2c8cda Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 24 May 2023 12:21:03 +0100 Subject: [PATCH] Refactor MariaDB and RabbitMQ restart procedure Ansible 2.14.3 introduced a change that broke the method used for restarting MariaDB and RabbitMQ serially [1][2]. In I57425680a4cdbf0daeb9b2cc35920f1b933aa4a8 we limited to 2.14.2 to work around this. Ansible upstream claim this behaviour was unintentional, and will not fix it. This change moves to a different approach where we use separate plays with a 'serial' keyword to execute the restart. This change also removes the restriction on the maximum supported version of 2.14.2 on ansible-core - any 2.14 release is now supported. [1] https://github.com/ansible/ansible/commit/65366f663de7d044f42ae6dd53368fd4c1f88b35 [2] https://github.com/ansible/ansible/issues/80848 Depends-On: https://review.opendev.org/c/openstack/kolla/+/884208 Change-Id: I5a12670d07077d24047aaff57ce8d33ccf7156ff --- ansible/mariadb.yml | 72 +++++++++++ ansible/rabbitmq.yml | 115 ++++++++++++++++++ ansible/roles/mariadb/handlers/main.yml | 27 ++-- ansible/roles/mariadb/tasks/deploy.yml | 7 -- ansible/roles/mariadb/tasks/post-deploy.yml | 4 + ansible/roles/mariadb/tasks/post-upgrade.yml | 24 ++++ ansible/roles/mariadb/tasks/upgrade.yml | 24 ---- ansible/roles/prechecks/vars/main.yml | 2 +- ansible/roles/rabbitmq/handlers/main.yml | 26 +--- ansible/roles/rabbitmq/tasks/deploy.yml | 5 - ansible/roles/rabbitmq/tasks/post-deploy.yml | 2 + ansible/roles/rabbitmq/tasks/upgrade.yml | 3 - ansible/site.yml | 50 +------- doc/source/user/quickstart.rst | 4 +- doc/source/user/virtual-environments.rst | 1 - ...adb-restart-refactor-529502bf5d12b0f4.yaml | 10 ++ tests/run.yml | 2 +- 17 files changed, 246 insertions(+), 132 deletions(-) create mode 100644 ansible/mariadb.yml create mode 100644 ansible/rabbitmq.yml create mode 100644 ansible/roles/mariadb/tasks/post-deploy.yml create mode 100644 ansible/roles/mariadb/tasks/post-upgrade.yml create mode 100644 ansible/roles/rabbitmq/tasks/post-deploy.yml create mode 100644 releasenotes/notes/mariadb-restart-refactor-529502bf5d12b0f4.yaml diff --git a/ansible/mariadb.yml b/ansible/mariadb.yml new file mode 100644 index 0000000000..0301adf6a9 --- /dev/null +++ b/ansible/mariadb.yml @@ -0,0 +1,72 @@ +--- +# For MariaDB we need to be careful about restarting services, to avoid losing quorum. +- name: Apply role mariadb + gather_facts: false + hosts: + - mariadb + - '&enable_mariadb_True' + tags: + - mariadb + tasks: + - import_role: + name: mariadb + +- name: Restart mariadb services + gather_facts: false + hosts: + - mariadb_restart + - '&enable_mariadb_True' + # Restart in batches + serial: "33%" + tags: + - mariadb + tasks: + - import_role: + name: mariadb + tasks_from: restart_services.yml + +- name: Start mariadb services + gather_facts: false + hosts: + - mariadb_start + - '&enable_mariadb_True' + # Start in batches + serial: "33%" + tags: + - mariadb + tasks: + - import_role: + name: mariadb + tasks_from: restart_services.yml + +- name: Restart bootstrap mariadb service + gather_facts: false + hosts: + - mariadb_bootstrap_restart + - '&enable_mariadb_True' + tags: + - mariadb + tasks: + - import_role: + name: mariadb + tasks_from: restart_services.yml + +- name: Apply mariadb post-configuration + gather_facts: false + hosts: + - mariadb + - '&enable_mariadb_True' + tags: + - mariadb + tasks: + - name: Include mariadb post-deploy.yml + include_role: + name: mariadb + tasks_from: post-deploy.yml + when: kolla_action in ['deploy', 'reconfigure', 'upgrade'] + + - name: Include mariadb post-upgrade.yml + include_role: + name: mariadb + tasks_from: post-upgrade.yml + when: kolla_action == 'upgrade' diff --git a/ansible/rabbitmq.yml b/ansible/rabbitmq.yml new file mode 100644 index 0000000000..370600c8eb --- /dev/null +++ b/ansible/rabbitmq.yml @@ -0,0 +1,115 @@ +--- +# For RabbitMQ we need to be careful about restarting services, to avoid losing quorum. +- name: Apply role rabbitmq + gather_facts: false + hosts: + - rabbitmq + - '&enable_rabbitmq_True' + tags: + - rabbitmq + tasks: + - import_role: + name: rabbitmq + vars: + role_rabbitmq_cluster_cookie: '{{ rabbitmq_cluster_cookie }}' + role_rabbitmq_cluster_port: '{{ rabbitmq_cluster_port }}' + role_rabbitmq_epmd_port: '{{ rabbitmq_epmd_port }}' + role_rabbitmq_groups: rabbitmq + role_rabbitmq_management_port: '{{ rabbitmq_management_port }}' + role_rabbitmq_monitoring_password: '{{ rabbitmq_monitoring_password }}' + role_rabbitmq_monitoring_user: '{{ rabbitmq_monitoring_user }}' + role_rabbitmq_password: '{{ rabbitmq_password }}' + role_rabbitmq_port: '{{ rabbitmq_port }}' + role_rabbitmq_prometheus_port: '{{ rabbitmq_prometheus_port }}' + role_rabbitmq_user: '{{ rabbitmq_user }}' + +- name: Restart rabbitmq services + gather_facts: false + hosts: + - rabbitmq_restart + - '&enable_rabbitmq_True' + # Restart in batches + serial: "33%" + tags: + - rabbitmq + tasks: + - import_role: + name: rabbitmq + tasks_from: restart_services.yml + vars: + role_rabbitmq_cluster_cookie: '{{ rabbitmq_cluster_cookie }}' + role_rabbitmq_groups: rabbitmq + +- name: Apply rabbitmq post-configuration + gather_facts: false + hosts: + - rabbitmq + - '&enable_rabbitmq_True' + tags: + - rabbitmq + tasks: + - name: Include rabbitmq post-deploy.yml + include_role: + name: rabbitmq + tasks_from: post-deploy.yml + when: kolla_action in ['deploy', 'reconfigure'] + vars: + role_rabbitmq_cluster_cookie: '{{ rabbitmq_cluster_cookie }}' + role_rabbitmq_groups: rabbitmq + +- name: Apply role rabbitmq (outward) + gather_facts: false + hosts: + - outward-rabbitmq + - '&enable_outward_rabbitmq_True' + tags: + - rabbitmq + tasks: + - import_role: + name: rabbitmq + vars: + project_name: outward_rabbitmq + role_rabbitmq_cluster_cookie: '{{ outward_rabbitmq_cluster_cookie }}' + role_rabbitmq_cluster_port: '{{ outward_rabbitmq_cluster_port }}' + role_rabbitmq_epmd_port: '{{ outward_rabbitmq_epmd_port }}' + role_rabbitmq_groups: outward-rabbitmq + role_rabbitmq_management_port: '{{ outward_rabbitmq_management_port }}' + role_rabbitmq_password: '{{ outward_rabbitmq_password }}' + role_rabbitmq_port: '{{ outward_rabbitmq_port }}' + role_rabbitmq_prometheus_port: '{{ outward_rabbitmq_prometheus_port }}' + role_rabbitmq_user: '{{ outward_rabbitmq_user }}' + +- name: Restart rabbitmq (outward) services + gather_facts: false + hosts: + - outward_rabbitmq_restart + - '&enable_outward_rabbitmq_True' + # Restart in batches + serial: "33%" + tags: + - rabbitmq + tasks: + - import_role: + name: rabbitmq + tasks_from: restart_services.yml + vars: + project_name: outward_rabbitmq + role_rabbitmq_cluster_cookie: '{{ outward_rabbitmq_cluster_cookie }}' + role_rabbitmq_groups: outward-rabbitmq + +- name: Apply rabbitmq (outward) post-configuration + gather_facts: false + hosts: + - outward-rabbitmq + - '&enable_outward_rabbitmq_True' + tags: + - rabbitmq + tasks: + - name: Include rabbitmq (outward) post-deploy.yml + include_role: + name: rabbitmq + when: kolla_action in ['deploy', 'reconfigure'] + vars: + project_name: outward_rabbitmq + role_rabbitmq_cluster_cookie: '{{ outward_rabbitmq_cluster_cookie }}' + role_rabbitmq_groups: outward-rabbitmq diff --git a/ansible/roles/mariadb/handlers/main.yml b/ansible/roles/mariadb/handlers/main.yml index 1c76fc48ff..a327668235 100644 --- a/ansible/roles/mariadb/handlers/main.yml +++ b/ansible/roles/mariadb/handlers/main.yml @@ -48,38 +48,29 @@ no_log: true listen: Bootstrap MariaDB cluster +- name: Ensure MariaDB is running normally on bootstrap host + group_by: + key: mariadb_bootstrap_restart + listen: Bootstrap MariaDB cluster + - name: Restart MariaDB on existing cluster members - include_tasks: 'restart_services.yml' + group_by: + key: mariadb_restart when: - groups[mariadb_shard_group + '_port_alive_True'] is defined - inventory_hostname in groups[mariadb_shard_group + '_port_alive_True'] - - groups[mariadb_shard_group + '_port_alive_True'].index(inventory_hostname) % 4 == item - kolla_action != "config" listen: restart mariadb - loop: - - 0 - - 1 - - 2 - - 3 - name: Start MariaDB on new nodes - include_tasks: 'restart_services.yml' + group_by: + key: mariadb_start when: - bootstrap_host is not defined or bootstrap_host != inventory_hostname - groups[mariadb_shard_group + '_port_alive_False'] is defined - inventory_hostname in groups[mariadb_shard_group + '_port_alive_False'] - - groups[mariadb_shard_group + '_port_alive_False'].index(inventory_hostname) % 4 == item - kolla_action != "config" listen: restart mariadb - loop: - - 0 - - 1 - - 2 - - 3 - -- name: Ensure MariaDB is running normally on bootstrap host - include_tasks: 'restart_services.yml' - listen: Bootstrap MariaDB cluster - name: Restart mariadb-clustercheck container vars: diff --git a/ansible/roles/mariadb/tasks/deploy.yml b/ansible/roles/mariadb/tasks/deploy.yml index 348f61b84c..e1f3ad8557 100644 --- a/ansible/roles/mariadb/tasks/deploy.yml +++ b/ansible/roles/mariadb/tasks/deploy.yml @@ -4,10 +4,3 @@ - import_tasks: check-containers.yml - import_tasks: bootstrap.yml - -- name: Flush handlers - meta: flush_handlers - -- import_tasks: register.yml - -- import_tasks: check.yml diff --git a/ansible/roles/mariadb/tasks/post-deploy.yml b/ansible/roles/mariadb/tasks/post-deploy.yml new file mode 100644 index 0000000000..c487e00be4 --- /dev/null +++ b/ansible/roles/mariadb/tasks/post-deploy.yml @@ -0,0 +1,4 @@ +--- +- import_tasks: register.yml + +- import_tasks: check.yml diff --git a/ansible/roles/mariadb/tasks/post-upgrade.yml b/ansible/roles/mariadb/tasks/post-upgrade.yml new file mode 100644 index 0000000000..cb07e7d569 --- /dev/null +++ b/ansible/roles/mariadb/tasks/post-upgrade.yml @@ -0,0 +1,24 @@ +--- +- name: Run upgrade in MariaDB container + vars: + service_name: "mariadb" + service: "{{ mariadb_services[service_name] }}" + become: true + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + detach: False + dimensions: "{{ service.dimensions }}" + environment: + KOLLA_UPGRADE: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + DB_HOST: "{{ api_interface_address }}" + DB_PORT: "{{ mariadb_port }}" + DB_ROOT_PASSWORD: "{{ database_password }}" + image: "{{ service.image }}" + labels: + UPGRADE: + name: "upgrade_mariadb" + restart_policy: no + volumes: "{{ service.volumes }}" + no_log: true diff --git a/ansible/roles/mariadb/tasks/upgrade.yml b/ansible/roles/mariadb/tasks/upgrade.yml index 967d57bdac..5b10a7e111 100644 --- a/ansible/roles/mariadb/tasks/upgrade.yml +++ b/ansible/roles/mariadb/tasks/upgrade.yml @@ -1,26 +1,2 @@ --- - import_tasks: deploy.yml - -- name: Run upgrade in MariaDB container - vars: - service_name: "mariadb" - service: "{{ mariadb_services[service_name] }}" - become: true - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - detach: False - dimensions: "{{ service.dimensions }}" - environment: - KOLLA_UPGRADE: - KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" - DB_HOST: "{{ api_interface_address }}" - DB_PORT: "{{ mariadb_port }}" - DB_ROOT_PASSWORD: "{{ database_password }}" - image: "{{ service.image }}" - labels: - UPGRADE: - name: "upgrade_mariadb" - restart_policy: no - volumes: "{{ service.volumes }}" - no_log: true diff --git a/ansible/roles/prechecks/vars/main.yml b/ansible/roles/prechecks/vars/main.yml index 6e267f3e70..580bb126e1 100644 --- a/ansible/roles/prechecks/vars/main.yml +++ b/ansible/roles/prechecks/vars/main.yml @@ -2,7 +2,7 @@ docker_version_min: '18.09' docker_py_version_min: '3.4.1' ansible_version_min: '2.13' -ansible_version_max: '2.14.2' +ansible_version_max: '2.14' # Top level keys should match ansible_facts.distribution. # These map to lists of supported releases (ansible_facts.distribution_release) or diff --git a/ansible/roles/rabbitmq/handlers/main.yml b/ansible/roles/rabbitmq/handlers/main.yml index cd5e39eb57..4f1b600be9 100644 --- a/ansible/roles/rabbitmq/handlers/main.yml +++ b/ansible/roles/rabbitmq/handlers/main.yml @@ -1,26 +1,6 @@ --- -# NOTE(mgoddard): These tasks perform a 'full stop upgrade', which is necessary when moving between -# major releases. In future kolla-ansible releases we may be able to change this to a rolling -# restart. For info on this process see https://www.rabbitmq.com/upgrade.html - -- name: Restart first rabbitmq container - vars: - service_name: "rabbitmq" - service: "{{ rabbitmq_services[service_name] }}" - include_tasks: 'restart_services.yml' +- name: Restart rabbitmq container + group_by: + key: "{{ project_name }}_restart" when: - kolla_action != "config" - - inventory_hostname == groups[service.group] | first - listen: Restart rabbitmq container - -- name: Restart remaining rabbitmq containers - vars: - service_name: "rabbitmq" - service: "{{ rabbitmq_services[service_name] }}" - include_tasks: 'restart_services.yml' - when: - - kolla_action != "config" - - inventory_hostname == item - - inventory_hostname != groups[service.group] | first - loop: "{{ groups[service.group] }}" - listen: Restart rabbitmq container diff --git a/ansible/roles/rabbitmq/tasks/deploy.yml b/ansible/roles/rabbitmq/tasks/deploy.yml index e790621ad7..7be978c440 100644 --- a/ansible/roles/rabbitmq/tasks/deploy.yml +++ b/ansible/roles/rabbitmq/tasks/deploy.yml @@ -8,8 +8,3 @@ - import_tasks: check-containers.yml - import_tasks: bootstrap.yml - -- name: Flush handlers - meta: flush_handlers - -- import_tasks: feature-flags.yml diff --git a/ansible/roles/rabbitmq/tasks/post-deploy.yml b/ansible/roles/rabbitmq/tasks/post-deploy.yml new file mode 100644 index 0000000000..9c04d85c26 --- /dev/null +++ b/ansible/roles/rabbitmq/tasks/post-deploy.yml @@ -0,0 +1,2 @@ +--- +- import_tasks: feature-flags.yml diff --git a/ansible/roles/rabbitmq/tasks/upgrade.yml b/ansible/roles/rabbitmq/tasks/upgrade.yml index ed672c6aa2..58e5bde46a 100644 --- a/ansible/roles/rabbitmq/tasks/upgrade.yml +++ b/ansible/roles/rabbitmq/tasks/upgrade.yml @@ -8,6 +8,3 @@ - import_tasks: feature-flags.yml - import_tasks: check-containers.yml - -- name: Flush handlers - meta: flush_handlers diff --git a/ansible/site.yml b/ansible/site.yml index 37fee5926b..4ba77cdc71 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -380,14 +380,9 @@ - { role: redis, tags: redis } -- name: Apply role mariadb - gather_facts: false - hosts: - - mariadb - - '&enable_mariadb_True' - roles: - - { role: mariadb, - tags: mariadb } +# MariaDB deployment is more complicated than other services, so is covered in +# its own playbook. +- import_playbook: mariadb.yml - name: Apply role memcached gather_facts: false @@ -440,44 +435,7 @@ - { role: multipathd, tags: multipathd } -- name: Apply role rabbitmq - gather_facts: false - hosts: - - rabbitmq - - '&enable_rabbitmq_True' - roles: - - { role: rabbitmq, - tags: rabbitmq, - role_rabbitmq_cluster_cookie: '{{ rabbitmq_cluster_cookie }}', - role_rabbitmq_cluster_port: '{{ rabbitmq_cluster_port }}', - role_rabbitmq_epmd_port: '{{ rabbitmq_epmd_port }}', - role_rabbitmq_groups: rabbitmq, - role_rabbitmq_management_port: '{{ rabbitmq_management_port }}', - role_rabbitmq_monitoring_password: '{{ rabbitmq_monitoring_password }}', - role_rabbitmq_monitoring_user: '{{ rabbitmq_monitoring_user }}', - role_rabbitmq_password: '{{ rabbitmq_password }}', - role_rabbitmq_port: '{{ rabbitmq_port }}', - role_rabbitmq_prometheus_port: '{{ rabbitmq_prometheus_port }}', - role_rabbitmq_user: '{{ rabbitmq_user }}' } - -- name: Apply role rabbitmq (outward) - gather_facts: false - hosts: - - outward-rabbitmq - - '&enable_outward_rabbitmq_True' - roles: - - { role: rabbitmq, - tags: rabbitmq, - project_name: outward_rabbitmq, - role_rabbitmq_cluster_cookie: '{{ outward_rabbitmq_cluster_cookie }}', - role_rabbitmq_cluster_port: '{{ outward_rabbitmq_cluster_port }}', - role_rabbitmq_epmd_port: '{{ outward_rabbitmq_epmd_port }}', - role_rabbitmq_groups: outward-rabbitmq, - role_rabbitmq_management_port: '{{ outward_rabbitmq_management_port }}', - role_rabbitmq_password: '{{ outward_rabbitmq_password }}', - role_rabbitmq_port: '{{ outward_rabbitmq_port }}', - role_rabbitmq_prometheus_port: '{{ outward_rabbitmq_prometheus_port }}', - role_rabbitmq_user: '{{ outward_rabbitmq_user }}' } +- import_playbook: rabbitmq.yml - name: Apply role etcd gather_facts: false diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst index d71d559c6e..15dfd261ea 100644 --- a/doc/source/user/quickstart.rst +++ b/doc/source/user/quickstart.rst @@ -90,12 +90,10 @@ Install dependencies for the virtual environment pip install -U pip #. Install `Ansible `__. Kolla Ansible requires at least - Ansible ``6`` and supports up to ``7``. Ansible-core must not be greater - than 2.14.2 due to a regression. + Ansible ``6`` and supports up to ``7``. .. code-block:: console - pip install 'ansible-core>=2.13,<=2.14.2' pip install 'ansible>=6,<8' diff --git a/doc/source/user/virtual-environments.rst b/doc/source/user/virtual-environments.rst index 9324429a67..5d52a7e61f 100644 --- a/doc/source/user/virtual-environments.rst +++ b/doc/source/user/virtual-environments.rst @@ -26,7 +26,6 @@ python virtual environment on the Ansible control host. For example: source /path/to/venv/bin/activate pip install -U pip pip install kolla-ansible - pip install 'ansible-core>=2.13,<=2.14.2' pip install 'ansible>=6,<8' deactivate diff --git a/releasenotes/notes/mariadb-restart-refactor-529502bf5d12b0f4.yaml b/releasenotes/notes/mariadb-restart-refactor-529502bf5d12b0f4.yaml new file mode 100644 index 0000000000..28b1faacfc --- /dev/null +++ b/releasenotes/notes/mariadb-restart-refactor-529502bf5d12b0f4.yaml @@ -0,0 +1,10 @@ +--- +upgrade: + - | + Removes the restriction on the maximum supported version of 2.14.2 for + ``ansible-core``. Any 2.14 series release is now supported. +other: + - | + Refactors the MariaDB and RabbitMQ restart procedures to be compatible with + Ansible 2.14.3+. See `Ansible issue 80848 + `__ for details. diff --git a/tests/run.yml b/tests/run.yml index e693db446a..9ab3dbc6ef 100644 --- a/tests/run.yml +++ b/tests/run.yml @@ -245,7 +245,7 @@ - name: install kolla-ansible and dependencies vars: ansible_core_version_min: "==2.13.*" - ansible_core_version_max: "==2.14.2" + ansible_core_version_max: "==2.14.*" # Test latest ansible version on Ubuntu, minimum supported on others. ansible_core_version_constraint: >- {{ ansible_core_version_min if is_upgrade or base_distro != 'ubuntu' else ansible_core_version_max }}