diff --git a/rabbitmq/templates/bin/_rabbitmq-check-helpers.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-check-helpers.sh.tpl index 6a5e49e31d..bd362f9b3b 100644 --- a/rabbitmq/templates/bin/_rabbitmq-check-helpers.sh.tpl +++ b/rabbitmq/templates/bin/_rabbitmq-check-helpers.sh.tpl @@ -93,3 +93,16 @@ is-node-healthy() { echo "$result" | prepend-log-prefix return 1 } + +is-node-properly-clustered() { + result="$(rabbitmqctl eval 'autocluster:cluster_health_check().' 2>&1)" + if [[ $result =~ ^SUCCESS: ]]; then + return 0 + elif [[ $result =~ ^FAILURE: ]]; then + echo "$result" | prepend-log-prefix + return 1 + fi + log-it "Unexpected health-check output, giving the node the benefit of the doubt" + echo "$result" | prepend-log-prefix + return 0 +} diff --git a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl index 48997762de..2d3ffc7a7d 100644 --- a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl +++ b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl @@ -45,6 +45,10 @@ main() { log-it "Node is unhealthy" return 1 fi + if ! is-node-properly-clustered; then + log-it "Found clustering inconsistency, giving up" + return 1 + fi return 0 ;; stale) # node has started long ago - it shoud be either ready or dead @@ -52,6 +56,10 @@ main() { log-it "Long-running node become unhealthy" return 1 fi + if ! is-node-properly-clustered; then + echo "Long-running node became inconsistent with the rest of the cluster" + return 1 + fi return 0 ;; *) diff --git a/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl index 40ae96756e..c5134aff84 100644 --- a/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl +++ b/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl @@ -32,6 +32,13 @@ main() { log-it "Node is unhealthy" return 1 fi + + {{ if gt (.Values.replicas | int) 1 -}} + if ! is-node-properly-clustered; then + log-it "Node is inconsistent with the rest of the cluster" + return 1 + fi + {{- end }} return 0 } diff --git a/rabbitmq/templates/configmap-etc.yaml b/rabbitmq/templates/configmap-etc.yaml index 8c8143f64e..19abad3c84 100644 --- a/rabbitmq/templates/configmap-etc.yaml +++ b/rabbitmq/templates/configmap-etc.yaml @@ -27,5 +27,6 @@ data: {{ tuple "etc/_erlang.cookie.tpl" . | include "helm-toolkit.template" | indent 4 }} rabbitmq-env.conf: | {{ tuple "etc/_rabbitmq-env.conf.tpl" . | include "helm-toolkit.template" | indent 4 }} - rabbitmq.conf: | -{{ tuple "etc/_rabbitmq.conf.tpl" . | include "helm-toolkit.template" | indent 4 }} + rabbitmq.config: | +{{ tuple "etc/_rabbitmq.config.tpl" . | include "helm-toolkit.template" | indent 4 }} + diff --git a/rabbitmq/templates/statefulset.yaml b/rabbitmq/templates/deployment.yaml similarity index 75% rename from rabbitmq/templates/statefulset.yaml rename to rabbitmq/templates/deployment.yaml index 34dea8f1ee..a5fa4ab31b 100644 --- a/rabbitmq/templates/statefulset.yaml +++ b/rabbitmq/templates/deployment.yaml @@ -12,18 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. -kind: StatefulSet -apiVersion: apps/v1beta1 +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies }} +kind: Deployment +apiVersion: extensions/v1beta1 metadata: name: rabbitmq spec: replicas: {{ .Values.replicas }} - serviceName: rabbitmq-discovery + revisionHistoryLimit: {{ .Values.upgrades.revision_history }} + strategy: + type: {{ .Values.upgrades.pod_replacement_strategy }} + {{ if eq .Values.upgrades.pod_replacement_strategy "RollingUpdate" }} + rollingUpdate: + maxUnavailable: {{ .Values.upgrades.rolling_update.max_unavailable }} + maxSurge: {{ .Values.upgrades.rolling_update.max_surge }} + {{ end }} template: metadata: labels: app: rabbitmq annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.hash" }} + pod.beta.kubernetes.io/init-containers: '[ +{{ tuple $envAll $dependencies "[]" | include "helm-toolkit.kubernetes_entrypoint_init_container" | indent 10 }} + ]' # TODO: this needs to be moved to common. scheduler.alpha.kubernetes.io/affinity: > { @@ -68,6 +82,11 @@ spec: command: - bash - /scripts/start.sh + env: + - name: RABBITMQ_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP readinessProbe: timeoutSeconds: {{ .Values.probes_timeout }} exec: @@ -96,5 +115,5 @@ spec: mountPath: /etc/rabbitmq/rabbitmq-env.conf subPath: rabbitmq-env.conf - name: rabbitmq-etc - mountPath: /etc/rabbitmq/rabbitmq.conf - subPath: rabbitmq.conf + mountPath: /etc/rabbitmq/rabbitmq.config + subPath: rabbitmq.config diff --git a/rabbitmq/templates/etc/_rabbitmq-env.conf.tpl b/rabbitmq/templates/etc/_rabbitmq-env.conf.tpl index efc30508f1..43b5af75da 100644 --- a/rabbitmq/templates/etc/_rabbitmq-env.conf.tpl +++ b/rabbitmq/templates/etc/_rabbitmq-env.conf.tpl @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -CONFIG_FILE=/etc/rabbitmq/rabbitmq.conf RABBITMQ_LOGS=- RABBITMQ_SASL_LOGS=- +AUTOCLUSTER_TYPE=etcd +AUTOCLUSTER_DELAY={{ .Values.autocluster.delay }} RABBITMQ_USE_LONGNAME=true -NODENAME="rabbit@$(hostname -f)" +AUTOCLUSTER_LOG_LEVEL={{ .Values.autocluster.log_level }} +NODENAME="rabbit@${RABBITMQ_POD_IP}" diff --git a/rabbitmq/templates/etc/_rabbitmq.conf.tpl b/rabbitmq/templates/etc/_rabbitmq.conf.tpl deleted file mode 100644 index 2d0b2997d2..0000000000 --- a/rabbitmq/templates/etc/_rabbitmq.conf.tpl +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -listeners.tcp.other_ip = 0.0.0.0:{{ .Values.network.port.public }} - -default_user = {{ .Values.auth.default_user }} -default_pass = {{ .Values.auth.default_pass }} - -loopback_users.guest = false - -autocluster.peer_discovery_backend = rabbit_peer_discovery_dns -autocluster.dns.hostname = rabbitmq-discovery.{{ .Release.Namespace }}.svc.cluster.local -autocluster.node_type = disc - -cluster_keepalive_interval = 30000 -cluster_partition_handling = ignore - -queue_master_locator = random diff --git a/rabbitmq/templates/etc/_rabbitmq.config.tpl b/rabbitmq/templates/etc/_rabbitmq.config.tpl new file mode 100644 index 0000000000..c80b088c77 --- /dev/null +++ b/rabbitmq/templates/etc/_rabbitmq.config.tpl @@ -0,0 +1,41 @@ +% Copyright 2017 The Openstack-Helm Authors. +% +% Licensed under the Apache License, Version 2.0 (the "License"); +% you may not use this file except in compliance with the License. +% You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% See the License for the specific language governing permissions and +% limitations under the License. + +[ + {rabbit, [ + {dummy_param_without_comma, true} + ,{tcp_listeners, [ + {"0.0.0.0", {{ .Values.network.port.public }} } + ]} + ,{default_user, <<"{{ .Values.auth.default_user }}">>} + ,{default_pass, <<"{{ .Values.auth.default_pass }}">>} + ,{loopback_users, []} + ,{cluster_partition_handling, ignore} + ,{queue_master_locator, <<"random">>} + ]} + ,{autocluster, [ + {dummy_param_without_comma, true} + ,{backend, etcd} + ,{autocluster_log_level,{{ .Values.autocluster.log_level }}} + ,{autocluster_failure, stop} + ,{cleanup_interval, 30} + ,{cluster_cleanup, true} + ,{cleanup_warn_only, false} + ,{etcd_node_ttl, 15} + ,{etcd_scheme, http} + ,{etcd_host, {{ .Values.endpoints.etcd.hosts.default }}} + ,{etcd_port, {{ .Values.endpoints.etcd.port }}} + ]} +]. +% EOF diff --git a/rabbitmq/templates/service-discovery.yaml b/rabbitmq/templates/service-discovery.yaml deleted file mode 100644 index c21387a925..0000000000 --- a/rabbitmq/templates/service-discovery.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: rabbitmq-discovery - annotations: - service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" -spec: - clusterIP: None - selector: - app: rabbitmq - ports: - - port: {{.Values.network.port.public}} diff --git a/rabbitmq/values.yaml b/rabbitmq/values.yaml index 3fbaf35c12..09b3f93699 100644 --- a/rabbitmq/values.yaml +++ b/rabbitmq/values.yaml @@ -51,14 +51,28 @@ network: management: '15672' images: - rabbitmq: "quay.io/attcomdev/rabbitmq:3.7.0-pre-14" + rabbitmq: "quay.io/attcomdev/fuel-mcp-rabbitmq:ocata-unstable" + dep_check: "quay.io/stackanetes/kubernetes-entrypoint:v0.1.1" pull_policy: "IfNotPresent" +enabled_plugins: + - autocluster + erlang_cookie: openstack-cookie +endpoints: + etcd: + hosts: + default: etcd + port: 2379 + autocluster: log_level: info delay: 15 probes_delay: 180 probes_timeout: 10 + +dependencies: + service: + - etcd