From b163cb02d1486f8844ac52e619de7b62321e42b0 Mon Sep 17 00:00:00 2001 From: Paul Bourke Date: Fri, 20 Jul 2018 16:35:25 +0100 Subject: [PATCH] Update rabbitmq to use new conf & clustering Depends-On: I75e00312b36e1678b90a42cf58d24652323eff27 Change-Id: Ia716fabffca41eff816e59bbf9f4cab79ee8b72f --- ansible/roles/rabbitmq/defaults/main.yml | 1 + ansible/roles/rabbitmq/handlers/main.yml | 37 ++++++++++++++++++- ansible/roles/rabbitmq/tasks/config.yml | 11 ++++-- ansible/roles/rabbitmq/tasks/upgrade.yml | 26 ++++++++++--- .../rabbitmq/templates/definitions.json.j2 | 2 +- .../rabbitmq/templates/rabbitmq-env.conf.j2 | 13 ++----- .../roles/rabbitmq/templates/rabbitmq.conf.j2 | 14 +++++++ .../rabbitmq/templates/rabbitmq.config.j2 | 24 ------------ .../roles/rabbitmq/templates/rabbitmq.json.j2 | 4 +- .../update-rabbitmq-5db770469b9bae18.yaml | 5 +++ tests/templates/globals-default.j2 | 3 -- tools/setup_gate.sh | 3 +- 12 files changed, 89 insertions(+), 54 deletions(-) create mode 100644 ansible/roles/rabbitmq/templates/rabbitmq.conf.j2 delete mode 100644 ansible/roles/rabbitmq/templates/rabbitmq.config.j2 create mode 100644 releasenotes/notes/update-rabbitmq-5db770469b9bae18.yaml diff --git a/ansible/roles/rabbitmq/defaults/main.yml b/ansible/roles/rabbitmq/defaults/main.yml index f709401e95..d9b92040ef 100644 --- a/ansible/roles/rabbitmq/defaults/main.yml +++ b/ansible/roles/rabbitmq/defaults/main.yml @@ -39,3 +39,4 @@ rabbitmq_dimensions: "{{ default_container_dimensions }}" rabbitmq_user: "openstack" rabbitmq_cluster_name: "openstack" rabbitmq_hostname: "{{ ansible_hostname }}" +rabbitmq_pid_file: "/var/lib/rabbitmq/mnesia/rabbitmq.pid" diff --git a/ansible/roles/rabbitmq/handlers/main.yml b/ansible/roles/rabbitmq/handlers/main.yml index 3c31cf700d..d25f4e2ef7 100644 --- a/ansible/roles/rabbitmq/handlers/main.yml +++ b/ansible/roles/rabbitmq/handlers/main.yml @@ -1,5 +1,5 @@ --- -- name: Restart rabbitmq container +- name: Restart rabbitmq container (first node) vars: service_name: "rabbitmq" service: "{{ rabbitmq_services[service_name] }}" @@ -16,7 +16,40 @@ dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" - - inventory_hostname in groups[service.group] + - inventory_hostname == groups[service.group]|first + - service.enabled | bool + - config_json.changed | bool + or rabbitmq_confs.changed | bool + or rabbitmq_container.changed | bool + notify: + - Waiting for rabbitmq to start on first node + +- name: Waiting for rabbitmq to start on first node + vars: + service_name: "rabbitmq" + service: "{{ rabbitmq_services[service_name] }}" + shell: "docker exec {{ service.container_name }} rabbitmqctl wait {{ rabbitmq_pid_file }}" + when: + - inventory_hostname == groups[service.group]|first + +- name: Restart rabbitmq container (rest of nodes) + vars: + service_name: "rabbitmq" + service: "{{ rabbitmq_services[service_name] }}" + config_json: "{{ rabbitmq_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + rabbitmq_container: "{{ check_rabbitmq_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + environment: "{{ service.environment }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" + - inventory_hostname != groups[service.group]|first - service.enabled | bool - config_json.changed | bool or rabbitmq_confs.changed | bool diff --git a/ansible/roles/rabbitmq/tasks/config.yml b/ansible/roles/rabbitmq/tasks/config.yml index 5c96364ddb..df42bbe1b5 100644 --- a/ansible/roles/rabbitmq/tasks/config.yml +++ b/ansible/roles/rabbitmq/tasks/config.yml @@ -24,7 +24,8 @@ - item.value.enabled | bool with_dict: "{{ rabbitmq_services }}" notify: - - Restart rabbitmq container + - Restart rabbitmq container (first node) + - Restart rabbitmq container (rest of nodes) - name: Copying over rabbitmq configs vars: @@ -40,10 +41,11 @@ - service.enabled | bool with_items: - "rabbitmq-env.conf" - - "rabbitmq.config" + - "rabbitmq.conf" - "definitions.json" notify: - - Restart rabbitmq container + - Restart rabbitmq container (first node) + - Restart rabbitmq container (rest of nodes) - name: Check rabbitmq containers become: true @@ -62,4 +64,5 @@ - item.value.enabled | bool with_dict: "{{ rabbitmq_services }}" notify: - - Restart rabbitmq container + - Restart rabbitmq container (first node) + - Restart rabbitmq container (rest of nodes) diff --git a/ansible/roles/rabbitmq/tasks/upgrade.yml b/ansible/roles/rabbitmq/tasks/upgrade.yml index f7f74c97c5..3c176eb532 100644 --- a/ansible/roles/rabbitmq/tasks/upgrade.yml +++ b/ansible/roles/rabbitmq/tasks/upgrade.yml @@ -1,4 +1,7 @@ --- +# NOTE(pbourke): These tasks perform a 'full stop upgrade', which is necessary when moving between +# major releases. In future kolla-ansible releases we may be able to change this to a rolling +# restart. For info on this process see https://www.rabbitmq.com/upgrade.html - name: Checking if rabbitmq container needs upgrading vars: service_name: "rabbitmq" @@ -15,14 +18,25 @@ - include_tasks: config.yml -- name: Flush handlers - meta: flush_handlers +- name: Stopping all rabbitmq instances but the first node + become: true + kolla_docker: + action: "stop_container" + common_options: "{{ docker_common_options }}" + name: "{{ project_name }}" when: - - rabbitmq_hostname == (gospel_node.stdout | from_json).hostname + - inventory_hostname != groups[role_rabbitmq_groups]|first + - rabbitmq_differs['result'] + +- name: Stopping rabbitmq on the first node + become: true + kolla_docker: + action: "stop_container" + common_options: "{{ docker_common_options }}" + name: "{{ project_name }}" + when: + - inventory_hostname == groups[role_rabbitmq_groups]|first - rabbitmq_differs['result'] - name: Flush handlers meta: flush_handlers - when: - - rabbitmq_hostname != (gospel_node.stdout | from_json).hostname - - rabbitmq_differs['result'] diff --git a/ansible/roles/rabbitmq/templates/definitions.json.j2 b/ansible/roles/rabbitmq/templates/definitions.json.j2 index be3c47ee02..e06ae867c3 100644 --- a/ansible/roles/rabbitmq/templates/definitions.json.j2 +++ b/ansible/roles/rabbitmq/templates/definitions.json.j2 @@ -5,7 +5,7 @@ {% endif %} ], "users": [ - {"name": "{{ role_rabbitmq_user }}", "password": "password", "tags": "administrator"}{% if role_rabbitmq_monitoring_user is defined and role_rabbitmq_monitoring_user %}, + {"name": "{{ role_rabbitmq_user }}", "password": "{{ role_rabbitmq_password }}", "tags": "administrator"}{% if role_rabbitmq_monitoring_user is defined and role_rabbitmq_monitoring_user %}, {"name": "{{ role_rabbitmq_monitoring_user }}", "password": "{{ role_rabbitmq_monitoring_password }}", "tags": "monitoring"}{% endif %}{% if project_name == 'outward_rabbitmq' %}, {"name": "{{ murano_agent_rabbitmq_user }}", "password": "{{ murano_agent_rabbitmq_password }}", "tags": "management"} {% endif %} diff --git a/ansible/roles/rabbitmq/templates/rabbitmq-env.conf.j2 b/ansible/roles/rabbitmq/templates/rabbitmq-env.conf.j2 index e2eea5e87c..3d750fdd99 100644 --- a/ansible/roles/rabbitmq/templates/rabbitmq-env.conf.j2 +++ b/ansible/roles/rabbitmq/templates/rabbitmq-env.conf.j2 @@ -1,14 +1,7 @@ RABBITMQ_NODENAME=rabbit@{{ ansible_hostname }} RABBITMQ_LOG_BASE=/var/log/kolla/{{ project_name }} +RABBITMQ_DIST_PORT={{ role_rabbitmq_cluster_port }} +RABBITMQ_PID_FILE={{ rabbitmq_pid_file }} -# TODO(sdake, vhosakot) -# erlang by default binds to wildcard (all interfaces) and can potentially -# interfere with the neutron external or tenant networks. We should in theory -# bind epmd to the host's IPv4 address to address the issue however this also -# has issues and can crash erlang when it is compiled with IPv6 support. -# See bugs: -# https://bugs.launchpad.net/ubuntu/+source/erlang/+bug/1374109 -# https://bugs.launchpad.net/kolla/+bug/1562701 -# https://bugzilla.redhat.com/show_bug.cgi?id=1324922 -#export ERL_EPMD_ADDRESS={{ api_interface_address }} +export ERL_EPMD_ADDRESS={{ api_interface_address }} export ERL_EPMD_PORT={{ role_rabbitmq_epmd_port }} diff --git a/ansible/roles/rabbitmq/templates/rabbitmq.conf.j2 b/ansible/roles/rabbitmq/templates/rabbitmq.conf.j2 new file mode 100644 index 0000000000..e95b7cccab --- /dev/null +++ b/ansible/roles/rabbitmq/templates/rabbitmq.conf.j2 @@ -0,0 +1,14 @@ +listeners.tcp.1 = {{ api_interface_address }}:{{ role_rabbitmq_port }} +{% if rabbitmq_hipe_compile|bool %} +hipe_compile = true +{% endif %} +cluster_partition_handling = autoheal + +management.listener.ip = {{ api_interface_address }} +management.listener.port = {{ role_rabbitmq_management_port }} +management.load_definitions = /etc/rabbitmq/definitions.json + +cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config +{% for host in groups[role_rabbitmq_groups] %} +cluster_formation.classic_config.nodes.{{ loop.index0 }} = rabbit@{{ hostvars[host]['ansible_hostname'] }} +{% endfor %} diff --git a/ansible/roles/rabbitmq/templates/rabbitmq.config.j2 b/ansible/roles/rabbitmq/templates/rabbitmq.config.j2 deleted file mode 100644 index 960f9fb8a7..0000000000 --- a/ansible/roles/rabbitmq/templates/rabbitmq.config.j2 +++ /dev/null @@ -1,24 +0,0 @@ -[ - {kernel, [ - {inet_dist_use_interface, {% raw %}{{% endraw %}{{ api_interface_address | regex_replace('\.', ',') }}}}, - {inet_dist_listen_min, {{ role_rabbitmq_cluster_port }}}, - {inet_dist_listen_max, {{ role_rabbitmq_cluster_port }}} - ]}, - {rabbit, [ -{% if rabbitmq_hipe_compile|bool %} - {hipe_compile, true}, -{% endif %} - {tcp_listeners, [ - {"{{ api_interface_address }}", {{ role_rabbitmq_port }}} - ]}, - {cluster_partition_handling, autoheal} - ]}, - {rabbitmq_management, [ - {listener, [ - {ip, "{{ api_interface_address }}"}, - {port, {{ role_rabbitmq_management_port }}} - ]}, - {load_definitions, "/etc/rabbitmq/definitions.json"} - ]} -]. -% EOF diff --git a/ansible/roles/rabbitmq/templates/rabbitmq.json.j2 b/ansible/roles/rabbitmq/templates/rabbitmq.json.j2 index c2b9269f6c..284a823b0f 100644 --- a/ansible/roles/rabbitmq/templates/rabbitmq.json.j2 +++ b/ansible/roles/rabbitmq/templates/rabbitmq.json.j2 @@ -8,8 +8,8 @@ "perm": "0600" }, { - "source": "{{ container_config_directory }}/rabbitmq.config", - "dest": "/etc/rabbitmq/rabbitmq.config", + "source": "{{ container_config_directory }}/rabbitmq.conf", + "dest": "/etc/rabbitmq/rabbitmq.conf", "owner": "rabbitmq", "perm": "0600" }, diff --git a/releasenotes/notes/update-rabbitmq-5db770469b9bae18.yaml b/releasenotes/notes/update-rabbitmq-5db770469b9bae18.yaml new file mode 100644 index 0000000000..8d919cf6f7 --- /dev/null +++ b/releasenotes/notes/update-rabbitmq-5db770469b9bae18.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Rabbitmq has been updated to 3.7.x. This comes with a new config format + which is now called rabbitmq.conf rather than rabbitmq.config. diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index 2ea3107849..95a7f27c83 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -45,6 +45,3 @@ ceph_pool_pgp_num: 8 {% endif %} keystone_token_provider: "fernet" - -# TODO(pbourke): remove once https://review.openstack.org/#/c/584427/ is merged -rpc_transport_url: "rabbit://openstack:password@{{ api_interface_address }}:5672" diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh index 18903d25c4..bdb0ec6e4f 100755 --- a/tools/setup_gate.sh +++ b/tools/setup_gate.sh @@ -149,8 +149,7 @@ function test_openstack { tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks1 # TODO(jeffrey4l): add pull action when we have a local registry # service in CI - # TODO(pbourke): remove '-e rabbitmq_password=password' once https://review.openstack.org/#/c/584427/ is merged - tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy -e rabbitmq_password=password > /tmp/logs/ansible/deploy + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-deploy