From 0982a823e139560fd97ecc9390f0cd249cc7982e Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Thu, 4 May 2017 09:54:26 -0500 Subject: [PATCH] RabbitMQ: use DNS for cluster discovery, not etcd Implements: blueprint rabbitmq-dns-discovery Some useful things to note: 1. This uses a StatefulSet instead of a Deployment. The reason for this is that when RabbitMQ uses DNS for peer discovery, the first thing it does when trying to join a node is attempt a reverse-dns lookup. This reverse lookup works when using a StatefulSet, but not a Deployment. 2. The RabbitMQ configuration was updated to use the new sysctl-style format. It seems that the new format is required to configure the new autoclustering features. Additionally, I found that this generate much clearer error messages than the straight erlang format. 3. I removed the `is-node-properly-clustered` test in the liveness and readiness probes. This probe isn't directly supported in 3.7.0, and it wasn't clear that a clustering check was appropriate for each node. Change-Id: Ieefbb2205bd77fbac04abcd051fb06fce62e8d97 --- .../bin/_rabbitmq-check-helpers.sh.tpl | 13 ------ .../templates/bin/_rabbitmq-liveness.sh.tpl | 8 ---- .../templates/bin/_rabbitmq-readiness.sh.tpl | 7 ---- rabbitmq/templates/configmap-etc.yaml | 5 +-- rabbitmq/templates/etc/_rabbitmq-env.conf.tpl | 6 +-- rabbitmq/templates/etc/_rabbitmq.conf.tpl | 29 +++++++++++++ rabbitmq/templates/etc/_rabbitmq.config.tpl | 41 ------------------- rabbitmq/templates/service-discovery.yaml | 26 ++++++++++++ .../{deployment.yaml => statefulset.yaml} | 29 +++---------- rabbitmq/values.yaml | 16 +------- 10 files changed, 65 insertions(+), 115 deletions(-) create mode 100644 rabbitmq/templates/etc/_rabbitmq.conf.tpl delete mode 100644 rabbitmq/templates/etc/_rabbitmq.config.tpl create mode 100644 rabbitmq/templates/service-discovery.yaml rename rabbitmq/templates/{deployment.yaml => statefulset.yaml} (75%) diff --git a/rabbitmq/templates/bin/_rabbitmq-check-helpers.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-check-helpers.sh.tpl index bd362f9b3b..6a5e49e31d 100644 --- a/rabbitmq/templates/bin/_rabbitmq-check-helpers.sh.tpl +++ b/rabbitmq/templates/bin/_rabbitmq-check-helpers.sh.tpl @@ -93,16 +93,3 @@ is-node-healthy() { echo "$result" | prepend-log-prefix return 1 } - -is-node-properly-clustered() { - result="$(rabbitmqctl eval 'autocluster:cluster_health_check().' 2>&1)" - if [[ $result =~ ^SUCCESS: ]]; then - return 0 - elif [[ $result =~ ^FAILURE: ]]; then - echo "$result" | prepend-log-prefix - return 1 - fi - log-it "Unexpected health-check output, giving the node the benefit of the doubt" - echo "$result" | prepend-log-prefix - return 0 -} diff --git a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl index 2d3ffc7a7d..48997762de 100644 --- a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl +++ b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl @@ -45,10 +45,6 @@ main() { log-it "Node is unhealthy" return 1 fi - if ! is-node-properly-clustered; then - log-it "Found clustering inconsistency, giving up" - return 1 - fi return 0 ;; stale) # node has started long ago - it shoud be either ready or dead @@ -56,10 +52,6 @@ main() { log-it "Long-running node become unhealthy" return 1 fi - if ! is-node-properly-clustered; then - echo "Long-running node became inconsistent with the rest of the cluster" - return 1 - fi return 0 ;; *) diff --git a/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl index c5134aff84..40ae96756e 100644 --- a/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl +++ b/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl @@ -32,13 +32,6 @@ main() { log-it "Node is unhealthy" return 1 fi - - {{ if gt (.Values.replicas | int) 1 -}} - if ! is-node-properly-clustered; then - log-it "Node is inconsistent with the rest of the cluster" - return 1 - fi - {{- end }} return 0 } diff --git a/rabbitmq/templates/configmap-etc.yaml b/rabbitmq/templates/configmap-etc.yaml index 19abad3c84..8c8143f64e 100644 --- a/rabbitmq/templates/configmap-etc.yaml +++ b/rabbitmq/templates/configmap-etc.yaml @@ -27,6 +27,5 @@ data: {{ tuple "etc/_erlang.cookie.tpl" . | include "helm-toolkit.template" | indent 4 }} rabbitmq-env.conf: | {{ tuple "etc/_rabbitmq-env.conf.tpl" . | include "helm-toolkit.template" | indent 4 }} - rabbitmq.config: | -{{ tuple "etc/_rabbitmq.config.tpl" . | include "helm-toolkit.template" | indent 4 }} - + rabbitmq.conf: | +{{ tuple "etc/_rabbitmq.conf.tpl" . | include "helm-toolkit.template" | indent 4 }} diff --git a/rabbitmq/templates/etc/_rabbitmq-env.conf.tpl b/rabbitmq/templates/etc/_rabbitmq-env.conf.tpl index 43b5af75da..efc30508f1 100644 --- a/rabbitmq/templates/etc/_rabbitmq-env.conf.tpl +++ b/rabbitmq/templates/etc/_rabbitmq-env.conf.tpl @@ -12,10 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +CONFIG_FILE=/etc/rabbitmq/rabbitmq.conf RABBITMQ_LOGS=- RABBITMQ_SASL_LOGS=- -AUTOCLUSTER_TYPE=etcd -AUTOCLUSTER_DELAY={{ .Values.autocluster.delay }} RABBITMQ_USE_LONGNAME=true -AUTOCLUSTER_LOG_LEVEL={{ .Values.autocluster.log_level }} -NODENAME="rabbit@${RABBITMQ_POD_IP}" +NODENAME="rabbit@$(hostname -f)" diff --git a/rabbitmq/templates/etc/_rabbitmq.conf.tpl b/rabbitmq/templates/etc/_rabbitmq.conf.tpl new file mode 100644 index 0000000000..2d0b2997d2 --- /dev/null +++ b/rabbitmq/templates/etc/_rabbitmq.conf.tpl @@ -0,0 +1,29 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +listeners.tcp.other_ip = 0.0.0.0:{{ .Values.network.port.public }} + +default_user = {{ .Values.auth.default_user }} +default_pass = {{ .Values.auth.default_pass }} + +loopback_users.guest = false + +autocluster.peer_discovery_backend = rabbit_peer_discovery_dns +autocluster.dns.hostname = rabbitmq-discovery.{{ .Release.Namespace }}.svc.cluster.local +autocluster.node_type = disc + +cluster_keepalive_interval = 30000 +cluster_partition_handling = ignore + +queue_master_locator = random diff --git a/rabbitmq/templates/etc/_rabbitmq.config.tpl b/rabbitmq/templates/etc/_rabbitmq.config.tpl deleted file mode 100644 index c80b088c77..0000000000 --- a/rabbitmq/templates/etc/_rabbitmq.config.tpl +++ /dev/null @@ -1,41 +0,0 @@ -% Copyright 2017 The Openstack-Helm Authors. -% -% Licensed under the Apache License, Version 2.0 (the "License"); -% you may not use this file except in compliance with the License. -% You may obtain a copy of the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, -% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% See the License for the specific language governing permissions and -% limitations under the License. - -[ - {rabbit, [ - {dummy_param_without_comma, true} - ,{tcp_listeners, [ - {"0.0.0.0", {{ .Values.network.port.public }} } - ]} - ,{default_user, <<"{{ .Values.auth.default_user }}">>} - ,{default_pass, <<"{{ .Values.auth.default_pass }}">>} - ,{loopback_users, []} - ,{cluster_partition_handling, ignore} - ,{queue_master_locator, <<"random">>} - ]} - ,{autocluster, [ - {dummy_param_without_comma, true} - ,{backend, etcd} - ,{autocluster_log_level,{{ .Values.autocluster.log_level }}} - ,{autocluster_failure, stop} - ,{cleanup_interval, 30} - ,{cluster_cleanup, true} - ,{cleanup_warn_only, false} - ,{etcd_node_ttl, 15} - ,{etcd_scheme, http} - ,{etcd_host, {{ .Values.endpoints.etcd.hosts.default }}} - ,{etcd_port, {{ .Values.endpoints.etcd.port }}} - ]} -]. -% EOF diff --git a/rabbitmq/templates/service-discovery.yaml b/rabbitmq/templates/service-discovery.yaml new file mode 100644 index 0000000000..c21387a925 --- /dev/null +++ b/rabbitmq/templates/service-discovery.yaml @@ -0,0 +1,26 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq-discovery + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +spec: + clusterIP: None + selector: + app: rabbitmq + ports: + - port: {{.Values.network.port.public}} diff --git a/rabbitmq/templates/deployment.yaml b/rabbitmq/templates/statefulset.yaml similarity index 75% rename from rabbitmq/templates/deployment.yaml rename to rabbitmq/templates/statefulset.yaml index a5fa4ab31b..34dea8f1ee 100644 --- a/rabbitmq/templates/deployment.yaml +++ b/rabbitmq/templates/statefulset.yaml @@ -12,32 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{- $envAll := . }} -{{- $dependencies := .Values.dependencies }} -kind: Deployment -apiVersion: extensions/v1beta1 +kind: StatefulSet +apiVersion: apps/v1beta1 metadata: name: rabbitmq spec: replicas: {{ .Values.replicas }} - revisionHistoryLimit: {{ .Values.upgrades.revision_history }} - strategy: - type: {{ .Values.upgrades.pod_replacement_strategy }} - {{ if eq .Values.upgrades.pod_replacement_strategy "RollingUpdate" }} - rollingUpdate: - maxUnavailable: {{ .Values.upgrades.rolling_update.max_unavailable }} - maxSurge: {{ .Values.upgrades.rolling_update.max_surge }} - {{ end }} + serviceName: rabbitmq-discovery template: metadata: labels: app: rabbitmq annotations: - configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.hash" }} - configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.hash" }} - pod.beta.kubernetes.io/init-containers: '[ -{{ tuple $envAll $dependencies "[]" | include "helm-toolkit.kubernetes_entrypoint_init_container" | indent 10 }} - ]' # TODO: this needs to be moved to common. scheduler.alpha.kubernetes.io/affinity: > { @@ -82,11 +68,6 @@ spec: command: - bash - /scripts/start.sh - env: - - name: RABBITMQ_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP readinessProbe: timeoutSeconds: {{ .Values.probes_timeout }} exec: @@ -115,5 +96,5 @@ spec: mountPath: /etc/rabbitmq/rabbitmq-env.conf subPath: rabbitmq-env.conf - name: rabbitmq-etc - mountPath: /etc/rabbitmq/rabbitmq.config - subPath: rabbitmq.config + mountPath: /etc/rabbitmq/rabbitmq.conf + subPath: rabbitmq.conf diff --git a/rabbitmq/values.yaml b/rabbitmq/values.yaml index 09b3f93699..3fbaf35c12 100644 --- a/rabbitmq/values.yaml +++ b/rabbitmq/values.yaml @@ -51,28 +51,14 @@ network: management: '15672' images: - rabbitmq: "quay.io/attcomdev/fuel-mcp-rabbitmq:ocata-unstable" - dep_check: "quay.io/stackanetes/kubernetes-entrypoint:v0.1.1" + rabbitmq: "quay.io/attcomdev/rabbitmq:3.7.0-pre-14" pull_policy: "IfNotPresent" -enabled_plugins: - - autocluster - erlang_cookie: openstack-cookie -endpoints: - etcd: - hosts: - default: etcd - port: 2379 - autocluster: log_level: info delay: 15 probes_delay: 180 probes_timeout: 10 - -dependencies: - service: - - etcd