From 23e7f6c292dd2f9e9f7423897f3d2c9633c38ee2 Mon Sep 17 00:00:00 2001 From: "Michal (inc0) Jastrzebski" Date: Thu, 3 Mar 2016 16:43:02 +0000 Subject: [PATCH] Playbook for rabbitmq upgrade Main issue with rabbitmq clusterer setup is to shut down gospel node as last one, which is bulk of this change Co-Authored-By: Sam Yaple Change-Id: I88e566a19ed813b0e3eef65ef7139ccfaa0c2700 Implements: blueprint upgrade-rabbitmq Partially-implements: blueprint upgrade-kolla --- ansible/roles/rabbitmq/defaults/main.yml | 1 + ansible/roles/rabbitmq/tasks/upgrade.yml | 21 +++++++ docker/rabbitmq/Dockerfile.j2 | 3 +- docker/rabbitmq/rabbitmq_get_gospel_node.py | 70 +++++++++++++++++++++ 4 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 docker/rabbitmq/rabbitmq_get_gospel_node.py diff --git a/ansible/roles/rabbitmq/defaults/main.yml b/ansible/roles/rabbitmq/defaults/main.yml index b3ae1e9ca0..4fb19feb48 100644 --- a/ansible/roles/rabbitmq/defaults/main.yml +++ b/ansible/roles/rabbitmq/defaults/main.yml @@ -14,3 +14,4 @@ rabbitmq_image_full: "{{ rabbitmq_image }}:{{ rabbitmq_tag }}" #################### rabbitmq_user: "openstack" rabbitmq_cluster_name: "openstack" +rabbitmq_hostname: "{{ ansible_hostname }}" diff --git a/ansible/roles/rabbitmq/tasks/upgrade.yml b/ansible/roles/rabbitmq/tasks/upgrade.yml index ed97d539c0..aaf91cbb9a 100644 --- a/ansible/roles/rabbitmq/tasks/upgrade.yml +++ b/ansible/roles/rabbitmq/tasks/upgrade.yml @@ -1 +1,22 @@ --- +- include: config.yml + +- name: Find gospel node + command: docker exec -t rabbitmq /usr/local/bin/rabbitmq_get_gospel_node + changed_when: "{{ (gospel_node.stdout | from_json).changed }}" + failed_when: "{{ (gospel_node.stdout | from_json).failed }}" + register: gospel_node + run_once: True + +- name: Stopping non-gospel nodes + kolla_docker: + action: "stop_container" + common_options: "{{ docker_common_options }}" + name: "rabbitmq" + when: rabbitmq_hostname != (gospel_node.stdout | from_json).hostname + +- include: start.yml + when: rabbitmq_hostname == (gospel_node.stdout | from_json).hostname + +- include: start.yml + when: rabbitmq_hostname != (gospel_node.stdout | from_json).hostname diff --git a/docker/rabbitmq/Dockerfile.j2 b/docker/rabbitmq/Dockerfile.j2 index b9c967b9a8..1555500113 100644 --- a/docker/rabbitmq/Dockerfile.j2 +++ b/docker/rabbitmq/Dockerfile.j2 @@ -28,7 +28,8 @@ RUN rm -rf /var/lib/rabbitmq/* \ COPY extend_start.sh /usr/local/bin/kolla_extend_start COPY rabbitmq_sudoers /etc/sudoers.d/rabbitmq_sudoers -RUN chmod 755 /usr/local/bin/kolla_extend_start \ +COPY rabbitmq_get_gospel_node.py /usr/local/bin/rabbitmq_get_gospel_node +RUN chmod 755 /usr/local/bin/kolla_extend_start /usr/local/bin/rabbitmq_get_gospel_node \ && chmod 750 /etc/sudoers.d \ && chmod 440 /etc/sudoers.d/rabbitmq_sudoers \ && usermod -a -G kolla rabbitmq diff --git a/docker/rabbitmq/rabbitmq_get_gospel_node.py b/docker/rabbitmq/rabbitmq_get_gospel_node.py new file mode 100644 index 0000000000..3925f00dd6 --- /dev/null +++ b/docker/rabbitmq/rabbitmq_get_gospel_node.py @@ -0,0 +1,70 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re +import subprocess +import traceback + + +def convert_erlang_to_json(term): + term = re.sub('[\s+]', '', term) + term = re.sub('([<>].*?)}', r'"\1"}', term) + term = re.sub('{([a-z].*?),', r'{"\1":', term) + term = re.sub(':([a-z].*?)}', r':"\1"}', term) + return json.loads(term) + + +def main(): + try: + raw_status = subprocess.check_output( + "rabbitmqctl eval 'rabbit_clusterer:status().'", + shell=True, stderr=subprocess.STDOUT + ) + if "Rabbit is running in cluster configuration" not in raw_status: + raise AttributeError + status = convert_erlang_to_json( + '\n'.join(raw_status.split('\n')[1:-3]) + ) + + gospel_node = None + for msg in status: + if 'gospel' in msg: + gospel_node = msg['gospel']['node'].split('@')[1] + if gospel_node is None: + raise AttributeError + except AttributeError: + result = { + 'failed': True, + 'error': raw_status, + 'changed': True + } + except Exception: + result = { + 'failed': True, + 'error': traceback.format_exc(), + 'changed': True + } + else: + result = { + 'failed': False, + 'hostname': gospel_node, + 'changed': False + } + + print(json.dumps(result)) + + +if __name__ == '__main__': + main()