From 4cc4ba59dac8d0835d7eaa1cfc9df5dd481e60c9 Mon Sep 17 00:00:00 2001
From: Victor Chembaev <chembervint@gmail.com>
Date: Wed, 18 Nov 2020 11:55:46 +0300
Subject: [PATCH] RabbitMQ handler refactored to restart services in serial

Change-Id: I1ff4cbdf3f60cb7fd5fe5d3c5d498e05fe2df79a
Closes-Bug: #1904702
---
 ansible/roles/rabbitmq/handlers/main.yml      | 45 ++-----------------
 .../roles/rabbitmq/tasks/restart_services.yml | 21 +++++++++
 .../notes/bug-1904702-7451dd8c4caa309b.yaml   |  5 +++
 3 files changed, 30 insertions(+), 41 deletions(-)
 create mode 100644 ansible/roles/rabbitmq/tasks/restart_services.yml
 create mode 100644 releasenotes/notes/bug-1904702-7451dd8c4caa309b.yaml

diff --git a/ansible/roles/rabbitmq/handlers/main.yml b/ansible/roles/rabbitmq/handlers/main.yml
index d3cb4d717d..f3b78de856 100644
--- a/ansible/roles/rabbitmq/handlers/main.yml
+++ b/ansible/roles/rabbitmq/handlers/main.yml
@@ -1,47 +1,10 @@
 ---
-- name: Restart rabbitmq container (first node)
+- name: Restart rabbitmq container
   vars:
     service_name: "rabbitmq"
     service: "{{ rabbitmq_services[service_name] }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes }}"
-    environment: "{{ service.environment }}"
-    dimensions: "{{ service.dimensions }}"
+  include_tasks: 'restart_services.yml'
   when:
     - kolla_action != "config"
-    - inventory_hostname == groups[service.group]|first
-  notify:
-    - Waiting for rabbitmq to start on first node
-  listen: Restart rabbitmq container
-
-- name: Waiting for rabbitmq to start on first node
-  vars:
-    service_name: "rabbitmq"
-    service: "{{ rabbitmq_services[service_name] }}"
-  become: true
-  command: "docker exec {{ service.container_name }} rabbitmqctl wait {{ rabbitmq_pid_file }}"
-  when:
-    - inventory_hostname == groups[service.group]|first
-
-- name: Restart rabbitmq container (rest of nodes)
-  vars:
-    service_name: "rabbitmq"
-    service: "{{ rabbitmq_services[service_name] }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes }}"
-    environment: "{{ service.environment }}"
-    dimensions: "{{ service.dimensions }}"
-  when:
-    - kolla_action != "config"
-    - inventory_hostname != groups[service.group]|first
-  listen: Restart rabbitmq container
+    - inventory_hostname == item
+  loop: "{{ groups[service.group] }}"
diff --git a/ansible/roles/rabbitmq/tasks/restart_services.yml b/ansible/roles/rabbitmq/tasks/restart_services.yml
new file mode 100644
index 0000000000..d981dfe111
--- /dev/null
+++ b/ansible/roles/rabbitmq/tasks/restart_services.yml
@@ -0,0 +1,21 @@
+---
+- name: Restart rabbitmq container
+  vars:
+    service_name: "rabbitmq"
+    service: "{{ rabbitmq_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    environment: "{{ service.environment }}"
+    dimensions: "{{ service.dimensions }}"
+
+- name: Waiting for rabbitmq to start
+  vars:
+    service_name: "rabbitmq"
+    service: "{{ rabbitmq_services[service_name] }}"
+  become: true
+  command: "docker exec {{ service.container_name }} rabbitmqctl wait {{ rabbitmq_pid_file }}"
diff --git a/releasenotes/notes/bug-1904702-7451dd8c4caa309b.yaml b/releasenotes/notes/bug-1904702-7451dd8c4caa309b.yaml
new file mode 100644
index 0000000000..2118620dad
--- /dev/null
+++ b/releasenotes/notes/bug-1904702-7451dd8c4caa309b.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    RabbitMQ services are now restarted serially to avoid a split brain.
+    `LP#1904702 <https://launchpad.net/bugs/1904702>`__