diff --git a/ansible/roles/rabbitmq/tasks/precheck.yml b/ansible/roles/rabbitmq/tasks/precheck.yml index ad663ce729..2f56307728 100644 --- a/ansible/roles/rabbitmq/tasks/precheck.yml +++ b/ansible/roles/rabbitmq/tasks/precheck.yml @@ -223,3 +223,33 @@ - container_facts['rabbitmq'] is defined - om_enable_rabbitmq_high_availability | bool tags: rabbitmq-ha-precheck + +- block: + - name: List RabbitMQ queues + become: true + shell: + # TODO(mattcrees): remove ``| egrep -v '(fanout|reply)'`` once https://review.opendev.org/c/openstack/oslo.messaging/+/888479 is merged. + cmd: "{{ kolla_container_engine }} exec rabbitmq rabbitmqctl list_queues --silent name type | egrep -v '(fanout|reply)' | awk '{ print $NF }'" + register: rabbitmq_queues + changed_when: false + check_mode: false + + - name: Filter RabbitMQ queue types + set_fact: + rabbitmq_queue_types: "{{ rabbitmq_queues.stdout | split | unique }}" + + - name: Check if RabbitMQ quorum queues need to be configured + assert: + that: "{{ (rabbitmq_queue_types | length == 1) and (rabbitmq_queue_types | first == 'quorum') }}" + fail_msg: > + om_enable_rabbitmq_quorum_queues is True but non-quorum queues have been found. + Currently the procedure to migrate to quorum queues is manual. + Please follow the process described here: https://docs.openstack.org/kolla-ansible/latest/reference/message-queues/rabbitmq.html#high-availability. + Note that this process may take several hours on larger systems, and may cause a degredation in performance at large scale. + If you do not wish to enable this feature, set om_enable_rabbitmq_quorum_queues to False. + + run_once: true + when: + - container_facts['rabbitmq'] is defined + - om_enable_rabbitmq_quorum_queues | bool + tags: rabbitmq-ha-precheck diff --git a/releasenotes/notes/rabbitmq-add-precheck-for-quorum-queues-ba10f58538959112.yaml b/releasenotes/notes/rabbitmq-add-precheck-for-quorum-queues-ba10f58538959112.yaml new file mode 100644 index 0000000000..cb2781fb9f --- /dev/null +++ b/releasenotes/notes/rabbitmq-add-precheck-for-quorum-queues-ba10f58538959112.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + A precheck has been added to catch when + ``om_enable_rabbitmq_quorum_queues`` is set to ``True``, but quorum queues + have not been configured on all appropriate queues. A manual migration is + required, see here for details: + https://docs.openstack.org/kolla-ansible/latest/reference/message-queues/rabbitmq.html#high-availability + `LP#2045887 `__ diff --git a/tests/upgrade.sh b/tests/upgrade.sh index 78b945ce49..f897cabb61 100755 --- a/tests/upgrade.sh +++ b/tests/upgrade.sh @@ -12,11 +12,10 @@ function upgrade { source $KOLLA_ANSIBLE_VENV_PATH/bin/activate - kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/upgrade-prechecks + kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks --skip-tags rabbitmq-ha-precheck &> /tmp/logs/ansible/upgrade-prechecks # NOTE(mattcrees): As om_enable_rabbitmq_quorum_queues now defaults to # true in Bobcat, we need to perform a migration to durable queues. - # TODO(mattcrees): Remove these steps in Caracal. SERVICE_TAGS="heat,keystone,neutron,nova" if [[ $SCENARIO == "zun" ]] || [[ $SCENARIO == "cephadm" ]]; then SERVICE_TAGS+=",cinder" @@ -44,6 +43,9 @@ function upgrade { kolla-ansible -i ${RAW_INVENTORY} -vvv pull &> /tmp/logs/ansible/pull-upgrade kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade &> /tmp/logs/ansible/upgrade + # Check that all appropriate RabbitMQ queues are now quorum queues. + kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks --tags rabbitmq-ha-precheck &> /tmp/logs/ansible/rabbitmq-ha-precheck + kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy &> /tmp/logs/ansible/upgrade-post-deploy kolla-ansible -i ${RAW_INVENTORY} -vvv validate-config &> /tmp/logs/ansible/validate-config