Fix rabbitmq playbook to allow upgrades

The rabbitmq playbook is designed to run in parallel across the cluster.
This causes an issue when upgrading rabbitmq to a new major or minor
version because RabbitMQ does not support doing an online migration of
datasets between major versions. while a minor release can be upgrade
while online it is recommended to bring down the cluster to do any
upgrade actions. The current configuration takes no account of this.

Reference:
https://www.rabbitmq.com/clustering.html#upgrading for further details.

* A new variable has been added called `rabbitmq_upgrade`. This is set to
  false by default to prevent a new version being installed unintentionally.
  To run the upgrade, which will shutdown the cluster, the variable can be
  set to true on the commandline:

  Example:
    openstack-ansible -e rabbitmq_upgrade=true \
    rabbitmq-install.yml

* A new variable has been added called `rabbitmq_ignore_version_state`
  which can be set "true" to ignore the package and version state tasks.
  This has been provided to allow a deployer to rerun the plays in an
  environment where the playbooks have been upgraded and the default
  version of rabbitmq has changed within the role and the deployer has
  elected to upgraded the installation at that time. This will ensure a
  deployer is able to recluster an environment as needed without
  effecting the package state.

  Example:
    openstack-ansible -e rabbitmq_ignore_version_state=true \
    rabbitmq-install.yml

* A new variable has been added `rabbitmq_primary_cluster_node` which
  allows a deployer to elect / set the primary cluster node in an
  environment. This variable is used to determine the restart order
  of RabbitMQ nodes. IE this will be the last node down and first one
  up in an environment. By default this variable is set to:
  rabbitmq_primary_cluster_node: "{{ groups['rabbitmq_all'][0] }}"

scripts/run-upgrade.sh has been modified to pass 'rabbitmq_upgrade=true'
on the command line so that RabbitMQ can be upgraded as part of the
upgrade between OpenStack versions.

DocImpact
Change-Id: I17d4429b9b94d47c1578dd58a2fb20698d1fe02e
Closes-bug: #1474992

playbooks/rabbitmq-install.yml

@@ -15,10 +15,13 @@
 
 - name: Install rabbitmq server
   hosts: rabbitmq_all
-  max_fail_percentage: 20
+  max_fail_percentage: 0
   user: root
   roles:
-    - { role: "rabbitmq_server", tags: [ "rabbitmq-server" ] }
+    - role: "rabbitmq_server"
+      tags:
+        - "rabbitmq-server"
+        - "upgrade-rabbitmq-server"
     - role: "rsyslog_client"
       rsyslog_client_log_dir: "/var/log/rabbitmq"
       rsyslog_client_config_name: "99-rabbitmq-rsyslog-client.conf"

playbooks/roles/rabbitmq_server/defaults/main.yml

@@ -18,7 +18,16 @@ is_metal: true
 
 rabbitmq_primary_cluster_node: "{{ groups['rabbitmq_all'][0] }}"
 
-rabbitmq_package_url: https://www.rabbitmq.com/releases/rabbitmq-server/v3.5.3/rabbitmq-server_3.5.3-1_all.deb
+# Upgrading the RabbitMQ package requires shutting down the cluster. This variable makes upgrading
+#  the version an explicit action.
+rabbitmq_upgrade: false
+
+# If the user does not want to upgrade but needs to rerun the playbooks for any reason the
+#  upgrade/version state can be ignored by setting `rabbitmq_ignore_version_state=true`
+rabbitmq_ignore_version_state: false
+
+rabbitmq_package_url: "https://www.rabbitmq.com/releases/rabbitmq-server/v3.5.3/rabbitmq-server_3.5.3-1_all.deb"
+rabbitmq_package_version: "{{ rabbitmq_package_url.split('/')[-1].split('_')[1] }}"
 rabbitmq_package_sha256: "0abcae49842c21242da66f1bc02fe4bf73dc3c3c38419861e08bf1fa4ac74cb1"
 rabbitmq_package_path: "/opt/rabbitmq-server.deb"
 

playbooks/roles/rabbitmq_server/files/policy-rc.d

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "All runlevel operations denied by policy" >&2
+exit 101

playbooks/roles/rabbitmq_server/tasks/main.yml

@@ -15,17 +15,14 @@
 
 
 - include: rabbitmq_pre_install.yml
+
+# The install process will be skipped if `rabbitmq_ignore_version_state=true`
 - include: rabbitmq_install.yml
+  when: not rabbitmq_ignore_version_state | bool
+
 - include: rabbitmq_set_cookie.yml
 - include: rabbitmq_post_install.yml
-
-- include: rabbitmq_cluster_create.yml
-  when: >
-    inventory_hostname == groups['rabbitmq_all'][0]
-
-- include: rabbitmq_cluster_join.yml
-  when: >
-    inventory_hostname != groups['rabbitmq_all'][0]
+- include: rabbitmq_cluster.yml
 
 - name: Flush handlers
   meta: flush_handlers

playbooks/roles/rabbitmq_server/tasks/rabbitmq_cluster.yml

@@ -0,0 +1,22 @@
+---
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- include: rabbitmq_cluster_create.yml
+  when: >
+    inventory_hostname == rabbitmq_primary_cluster_node
+
+- include: rabbitmq_cluster_join.yml
+  when: >
+    inventory_hostname != rabbitmq_primary_cluster_node

playbooks/roles/rabbitmq_server/tasks/rabbitmq_install.yml

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- include: rabbitmq_upgrade_check.yml
+
 - name: Update apt sources
   apt:
     update_cache: yes
@@ -54,3 +56,13 @@
   with_items: rabbitmq_pip_packages
   tags:
     - rabbitmq-pip-packages
+
+- name: Ensure there are no policy-rc files
+  file:
+    path: "/usr/sbin/policy-rc.d"
+    state: absent
+  tags:
+    - rabbitmq-upgrade
+    - rabbitmq-policy-file
+
+- include: rabbitmq_started.yml

playbooks/roles/rabbitmq_server/tasks/rabbitmq_pre_install.yml

@@ -13,6 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Remove package rabbit if found for upgrade
+  file:
+    path: "{{ rabbitmq_package_path }}"
+    state: absent
+  tags:
+    - rabbitmq-upgrade
+    - rabbitmq-package-deb
+    - rabbitmq-package-deb-get
+
 - name: Download the RabbitMQ package
   get_url:
     url: "{{ rabbitmq_package_url }}"

playbooks/roles/rabbitmq_server/tasks/rabbitmq_restart.yml

@@ -13,10 +13,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Restart rabbitmq-server
-  service:
-    name: "rabbitmq-server"
-    enabled: "yes"
-    state: "restarted"
-  tags:
-    - rabbitmq-restart
+- include: rabbitmq_stopped.yml
+- include: rabbitmq_started.yml

playbooks/roles/rabbitmq_server/tasks/rabbitmq_set_cookie.yml

@@ -21,14 +21,10 @@
   tags:
     - rabbitmq-config
 
-- name: Stop rabbitmq-server
-  service:
-    name: rabbitmq-server
-    state: stopped
+- include: rabbitmq_stopped.yml
   when: tmp_rabbit_cookie.stdout != rabbitmq_cookie_token
   tags:
     - rabbitmq-config
-    - rabbitmq-stop
 
 - name: Set rabbit cookie
   copy:
@@ -42,5 +38,7 @@
   tags:
     - rabbitmq-config
 
-- include: rabbitmq_restart.yml
+- include: rabbitmq_started.yml
   when: tmp_rabbit_cookie.stdout != rabbitmq_cookie_token
+  tags:
+    - rabbitmq-config

playbooks/roles/rabbitmq_server/tasks/rabbitmq_started.yml

@@ -0,0 +1,38 @@
+---
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "Ensure RabbitMQ node [0] is started"
+  service:
+    name: rabbitmq-server
+    state: started
+  failed_when: false
+  when: >
+    inventory_hostname == rabbitmq_primary_cluster_node
+  tags:
+    - rabbitmq-server-started
+    - rabbitmq-upgrade
+    - rabbitmq-start
+
+- name: "Ensure RabbitMQ nodes [1:] are started"
+  service:
+    name: rabbitmq-server
+    state: started
+  failed_when: false
+  when: >
+    inventory_hostname != rabbitmq_primary_cluster_node
+  tags:
+    - rabbitmq-server-started
+    - rabbitmq-upgrade
+    - rabbitmq-start

playbooks/roles/rabbitmq_server/tasks/rabbitmq_stopped.yml

@@ -0,0 +1,38 @@
+---
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "Ensure RabbitMQ nodes [1:] are stopped"
+  service:
+    name: rabbitmq-server
+    state: stopped
+  failed_when: false
+  when: >
+    inventory_hostname != rabbitmq_primary_cluster_node
+  tags:
+    - rabbitmq-server-stopped
+    - rabbitmq-upgrade
+    - rabbitmq-stop
+
+- name: "Ensure RabbitMQ node [0] is stopped"
+  service:
+    name: rabbitmq-server
+    state: stopped
+  failed_when: false
+  when: >
+    inventory_hostname == rabbitmq_primary_cluster_node
+  tags:
+    - rabbitmq-server-stopped
+    - rabbitmq-upgrade
+    - rabbitmq-stop

playbooks/roles/rabbitmq_server/tasks/rabbitmq_upgrade_check.yml

@@ -0,0 +1,39 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Get version of installed RabbitMQ package
+  shell: |
+    dpkg -l | grep rabbitmq-server
+  failed_when: false
+  register: installed_rabbitmq
+  when:
+    - not rabbitmq_upgrade | bool
+  tags:
+    - rabbitmq-package-deb
+    - rabbitmq-apt-packages
+
+- name: Compare installed version of RabbitMQ with new version variable
+  fail:
+    msg: "To install a new major/minor version of RabbitMQ set '-e rabbitmq_upgrade=true'."
+  when: >
+    not rabbitmq_upgrade | bool and
+    installed_rabbitmq.rc == 0 and
+    installed_rabbitmq.stdout.split()[2] != rabbitmq_package_version
+  tags:
+    - rabbitmq-package-deb
+    - rabbitmq-apt-packages
+
+- include: rabbitmq_upgrade_prep.yml
+  when: rabbitmq_upgrade | bool

playbooks/roles/rabbitmq_server/tasks/rabbitmq_upgrade_prep.yml

@@ -0,0 +1,33 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Prevent RabbitMQ from starting on install
+  copy:
+    src: "policy-rc.d"
+    dest: "/usr/sbin/policy-rc.d"
+    mode: "0755"
+    backup: yes
+  tags:
+    - rabbitmq-policy-file
+    - rabbitmq-upgrade
+
+- include: rabbitmq_stopped.yml
+
+- name: Ensure erlang epmd is stopped
+  shell: |
+    pkill $(pgrep -lf "erlang" | awk '{print $2}')
+  ignore_errors: true
+  tags:
+    - rabbitmq-upgrade

scripts/run-upgrade.sh

@@ -625,7 +625,7 @@ pushd playbooks
   RUN_TASKS+=("openstack-hosts-setup.yml")
 
   # Now run the infrastructure setup
-  RUN_TASKS+=("setup-infrastructure.yml")
+  RUN_TASKS+=("-e 'rabbitmq_upgrade=true' setup-infrastructure.yml")
 
   # Now upgrade the rest of OpenStack
   RUN_TASKS+=("setup-openstack.yml")