enable quorum queues

This implements a global toggle `om_enable_rabbitmq_quorum_queues`
to enable quorum queues for each service in RabbitMQ, similar to
what was done for HA[0].

Quorum Queues are enabled by default.

Quorum queues are more reliable, safer, simpler and faster than
replicated mirrored classic queues[1].

Mirrored classic queues are deprecated and scheduled for removal
in RabbitMQ 4.0[2].

Notice, that we do not need a new policy in the RabbitMQ definitions
template, because their usage is enabled on the client side and can't
be set using a policy[3].

Notice also, that quorum queues are not yet enabled in oslo.messaging
for the usage of reply_ and fanout_ queues (transient queues).
This will change once[4] is merged.

[0]: https://review.opendev.org/c/openstack/kolla-ansible/+/867771
[1]: https://www.rabbitmq.com/quorum-queues.html
[2]: https://blog.rabbitmq.com/posts/2021/08/4.0-deprecation-announcements/
[3]: https://www.rabbitmq.com/quorum-queues.html#declaring
[4]: https://review.opendev.org/c/openstack/oslo.messaging/+/888479

Signed-off-by: Sven Kieske <kieske@osism.tech>
Change-Id: I6c033d460a5c9b93c346e9e47e93b159d3c27830

ansible/group_vars/all.yml

@@ -272,6 +272,8 @@ om_enable_rabbitmq_tls: "{{ rabbitmq_enable_tls | bool }}"
 om_rabbitmq_cacert: "{{ rabbitmq_cacert }}"
 
 om_enable_rabbitmq_high_availability: false
+# Only enable quorum queues if you disable om_enable_rabbitmq_high_availability
+om_enable_rabbitmq_quorum_queues: true
 
 ####################
 # Networking options

ansible/roles/aodh/templates/aodh.conf.j2

@@ -69,3 +69,6 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}

ansible/roles/barbican/templates/barbican.conf.j2

@@ -87,6 +87,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 [oslo_middleware]
 enable_proxy_headers_parsing = True

ansible/roles/blazar/templates/blazar.conf.j2

@@ -62,6 +62,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if blazar_policy_file is defined %}
 [oslo_policy]

ansible/roles/ceilometer/templates/ceilometer.conf.j2

@@ -44,6 +44,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if ceilometer_policy_file is defined %}
 [oslo_policy]

ansible/roles/cinder/templates/cinder.conf.j2

@@ -80,6 +80,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 [oslo_middleware]
 enable_proxy_headers_parsing = True

ansible/roles/cloudkitty/templates/cloudkitty.conf.j2

@@ -53,6 +53,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 [collect]
 collector = {{ cloudkitty_collector_backend }}

ansible/roles/cyborg/templates/cyborg.conf.j2

@@ -66,3 +66,6 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}

ansible/roles/designate/templates/designate.conf.j2

@@ -101,6 +101,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 [oslo_concurrency]
 lock_path = /var/lib/designate/tmp

ansible/roles/glance/templates/glance-api.conf.j2

@@ -131,6 +131,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if glance_policy_file is defined %}
 [oslo_policy]

ansible/roles/heat/templates/heat.conf.j2

@@ -80,6 +80,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if heat_policy_file is defined %}
 [oslo_policy]

ansible/roles/ironic/templates/ironic-inspector.conf.j2

@@ -21,6 +21,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 [ironic]
 {% if ironic_enable_keystone_integration | bool %}

ansible/roles/ironic/templates/ironic.conf.j2

@@ -44,6 +44,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if ironic_policy_file is defined %}
 [oslo_policy]

ansible/roles/keystone/templates/keystone.conf.j2

@@ -68,6 +68,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if enable_osprofiler | bool %}
 [profiler]

ansible/roles/magnum/templates/magnum.conf.j2

@@ -135,6 +135,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if magnum_policy_file is defined %}
 [oslo_policy]

ansible/roles/manila/templates/manila.conf.j2

@@ -65,6 +65,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 [oslo_middleware]
 enable_proxy_headers_parsing = True

ansible/roles/mistral/templates/mistral.conf.j2

@@ -81,6 +81,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if mistral_policy_file is defined %}
 [oslo_policy]

ansible/roles/murano/templates/murano.conf.j2

@@ -70,6 +70,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 [oslo_middleware]
 enable_proxy_headers_parsing = True

ansible/roles/neutron/templates/neutron.conf.j2

@@ -145,6 +145,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if neutron_policy_file is defined %}
 [oslo_policy]

ansible/roles/nova-cell/templates/nova.conf.j2

@@ -193,6 +193,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if service_name in nova_cell_services_require_policy_json and nova_policy_file is defined %}
 [oslo_policy]

ansible/roles/nova/templates/nova.conf.j2

@@ -147,6 +147,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if service_name in nova_services_require_policy_json and nova_policy_file is defined %}
 [oslo_policy]

ansible/roles/prechecks/tasks/service_checks.yml

@@ -72,3 +72,10 @@
       Please consult the release notes.
   when:
     - (enable_zun | default()) | bool
+
+- name: Validate rabbitmq variables
+  run_once: True
+  fail:
+    msg: |
+      Please set only one of om_enable_rabbitmq_high_availability or om_enable_rabbitmq_quorum_queues
+  when: (om_enable_rabbitmq_high_availability | bool) == (om_enable_rabbitmq_quorum_queues | bool)

ansible/roles/sahara/templates/sahara.conf.j2

@@ -49,6 +49,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if sahara_policy_file is defined %}
 [oslo_policy]

ansible/roles/senlin/templates/senlin.conf.j2

@@ -78,6 +78,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if senlin_policy_file is defined %}
 [oslo_policy]

ansible/roles/solum/templates/solum.conf.j2

@@ -76,3 +76,6 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}

ansible/roles/tacker/templates/tacker.conf.j2

@@ -77,6 +77,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if tacker_policy_file is defined %}
 [oslo_policy]

ansible/roles/trove/templates/trove.conf.j2

@@ -83,6 +83,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if enable_osprofiler | bool %}
 [profiler]

ansible/roles/vitrage/templates/vitrage.conf.j2

@@ -78,6 +78,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 [oslo_concurrency]
 lock_path = /var/lib/vitrage/tmp

ansible/roles/watcher/templates/watcher.conf.j2

@@ -68,6 +68,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}
 
 {% if watcher_policy_file is defined %}
 [oslo_policy]

ansible/roles/zun/templates/zun.conf.j2

@@ -135,3 +135,6 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% if om_enable_rabbitmq_high_availability | bool %}
 amqp_durable_queues = true
 {% endif %}
+{% if om_enable_rabbitmq_quorum_queues | bool %}
+rabbit_quorum_queue = true
+{% endif %}

doc/source/reference/message-queues/rabbitmq.rst

@@ -113,15 +113,23 @@ https://www.rabbitmq.com/runtime.html#busy-waiting.
 High Availability
 ~~~~~~~~~~~~~~~~~
 
-RabbitMQ offers two features that, when used together, allow for high
-availability. These are durable queues and classic queue mirroring. Setting the
-flag ``om_enable_rabbitmq_high_availability`` to ``true`` will enable both of
-these features. There are some queue types which are intentionally not mirrored
+RabbitMQ offers two options to configure HA:
+  * Quorum queues (enabled by default and controlled by
+    ``om_enable_rabbitmq_quorum_queues`` variable)
+  * Classic queue mirroring and durable queues (deprecated in RabbitMQ and to
+    be dropped in 4.0, controlled by ``om_enable_rabbitmq_high_availability``)
+
+There are some queue types which are intentionally not mirrored
 using the exclusionary pattern ``^(?!(amq\\.)|(.*_fanout_)|(reply_)).*``.
 
 After enabling this value on a running system, there are some additional steps
 needed to migrate from transient to durable queues.
 
+.. warning::
+
+   Since the default changed from non-HA to Quorum queues in Bobcat release,
+   following procedure is required to be carried out before an upgrade.
+
 1. Stop all OpenStack services which use RabbitMQ, so that they will not
    attempt to recreate any queues yet.
 

releasenotes/notes/make-quorum-queues-usable-552bfb2e8975f83f.yaml

@@ -0,0 +1,18 @@
+---
+features:
+  - |
+    You can now enable the usage of quorum queues in RabbitMQ for all services
+    by setting the variable ``om_enable_rabbitmq_quorum_queues`` to ``true``.
+    Notice that you can't use quorum queues and high availability at the same
+    time. This is caught by a precheck.
+    This feature is enabled by default to improve reliability of the messaging
+    queues.
+upgrade:
+  - |
+    Quorum queues in RabbitMQ (controlled by
+    ``om_enable_rabbitmq_quorum_queues`` variable) is enabled by default from
+    now on.
+    Support for non-HA RabbitMQ queues is dropped. Either quorum queues that
+    are enabled by default, or classic mirrored queues are required now.
+    `Migration procedure from non-HA to HA
+    <https://docs.openstack.org/kolla-ansible/zed/reference/message-queues/rabbitmq.html#high-availability>`__

tests/upgrade.sh

@@ -13,6 +13,34 @@ function upgrade {
     source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
 
     kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/upgrade-prechecks
+
+    # NOTE(mattcrees): As om_enable_rabbitmq_quorum_queues now defaults to
+    # true in Bobcat, we need to perform a migration to durable queues.
+    # TODO(mattcrees): Remove these steps in Caracal.
+    SERVICE_TAGS="heat,keystone,neutron,nova"
+    if [[ $SCENARIO == "zun" ]] || [[ $SCENARIO == "cephadm" ]]; then
+        SERVICE_TAGS+=",cinder"
+    fi
+    if [[ $SCENARIO == "scenario_nfv" ]]; then
+        SERVICE_TAGS+=",barbican"
+    fi
+    if [[ $SCENARIO == "ironic" ]]; then
+        SERVICE_TAGS+=",ironic"
+    fi
+    if [[ $SCENARIO == "masakari" ]]; then
+        SERVICE_TAGS+=",masakari"
+    fi
+    if [[ $SCENARIO == "ovn" ]] || [[ $SCENARIO == "octavia" ]]; then
+        SERVICE_TAGS+=",octavia"
+    fi
+    if [[ $SCENARIO == "magnum" ]]; then
+        SERVICE_TAGS+=",magnum,designate"
+    fi
+    kolla-ansible -i ${RAW_INVENTORY} -vvv stop --tags $SERVICE_TAGS --yes-i-really-really-mean-it &> /tmp/logs/ansible/stop
+    kolla-ansible -i ${RAW_INVENTORY} -vvv genconfig &> /tmp/logs/ansible/genconfig
+    kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure --tags rabbitmq &> /tmp/logs/ansible/reconfigure-rabbitmq
+    kolla-ansible -i ${RAW_INVENTORY} -vvv rabbitmq-reset-state &> /tmp/logs/ansible/rabbitmq-reset-state
+
     kolla-ansible -i ${RAW_INVENTORY} -vvv pull &> /tmp/logs/ansible/pull-upgrade
     kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade &> /tmp/logs/ansible/upgrade