From 0c51d813b34f7d5f8215148dfc3acf0b23dfa545 Mon Sep 17 00:00:00 2001 From: portdirect Date: Fri, 5 Jan 2018 03:10:20 -0500 Subject: [PATCH] RabbitMQ 3.7 with native k8s clustering This PS moves to use RabbitMQ with native k8s clustering. Change-Id: I6a93c8168bc7dce6b73871aa1fd4d637e6192975 --- rabbitmq/templates/_helpers.tpl | 37 ++++++ .../bin/_rabbitmq-check-helpers.sh.tpl | 110 ----------------- .../templates/bin/_rabbitmq-liveness.sh.tpl | 63 +--------- .../templates/bin/_rabbitmq-readiness.sh.tpl | 36 +----- rabbitmq/templates/bin/_rabbitmq-start.sh.tpl | 14 +-- rabbitmq/templates/configmap-bin.yaml | 12 +- rabbitmq/templates/configmap-etc.yaml | 34 ++++-- rabbitmq/templates/etc/_enabled_plugins.tpl | 30 ++--- rabbitmq/templates/etc/_erlang.cookie.tpl | 1 - rabbitmq/templates/etc/_rabbitmq-env.conf.tpl | 24 ---- rabbitmq/templates/etc/_rabbitmq.config.tpl | 45 ------- rabbitmq/templates/service.yaml | 13 +- .../{deployment.yaml => statefulset.yaml} | 90 ++++++++++---- rabbitmq/values.yaml | 114 +++++++++--------- tools/deployment/developer/07-rabbitmq.sh | 8 +- tools/gate/launch-osh/basic.sh | 8 +- 16 files changed, 221 insertions(+), 418 deletions(-) create mode 100644 rabbitmq/templates/_helpers.tpl delete mode 100644 rabbitmq/templates/bin/_rabbitmq-check-helpers.sh.tpl delete mode 100644 rabbitmq/templates/etc/_erlang.cookie.tpl delete mode 100644 rabbitmq/templates/etc/_rabbitmq-env.conf.tpl delete mode 100644 rabbitmq/templates/etc/_rabbitmq.config.tpl rename rabbitmq/templates/{deployment.yaml => statefulset.yaml} (56%) diff --git a/rabbitmq/templates/_helpers.tpl b/rabbitmq/templates/_helpers.tpl new file mode 100644 index 0000000000..c62b49950d --- /dev/null +++ b/rabbitmq/templates/_helpers.tpl @@ -0,0 +1,37 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "rabbitmq.to_rabbit_config" -}} +{{- range $top_key, $top_value := . }} +{{- if kindIs "map" $top_value -}} +{{- range $second_key, $second_value := . }} +{{- if kindIs "map" $second_value -}} +{{- range $third_key, $third_value := . }} +{{- if kindIs "map" $third_value -}} +{{ $top_key }}.{{ $second_key }}.{{ $third_key }} = wow +{{ else -}} +{{ $top_key }}.{{ $second_key }}.{{ $third_key }} = {{ $third_value }} +{{ end -}} +{{- end -}} +{{ else -}} +{{ $top_key }}.{{ $second_key }} = {{ $second_value }} +{{ end -}} +{{- end -}} +{{ else -}} +{{ $top_key }} = {{ $top_value }} +{{ end -}} +{{- end -}} +{{- end -}} diff --git a/rabbitmq/templates/bin/_rabbitmq-check-helpers.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-check-helpers.sh.tpl deleted file mode 100644 index 31b5e29a69..0000000000 --- a/rabbitmq/templates/bin/_rabbitmq-check-helpers.sh.tpl +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bash - -{{/* -Copyright 2017 The Openstack-Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -# This is taken from https://github.com/openstack/fuel-ccp-rabbitmq/blob/master/service/files/rabbitmq-check-helpers.sh.j2 -MARKER_PATH=/tmp/rabbit-startup-marker - -# How many seconds we give a node before successfull liveness checks -# become mandatory. -FRESH_NODE_TIMEOUT={{ .Values.probes_delay }} - -LP="" - -set-log-prefix() { - LP="[${1:?}]" -} - -log-it() { - echo "$LP" "$@" -} - -prepend-log-prefix() { - awk -v lp="$LP" '{print lp " " $0}' -} - -marker-state() { - if [[ ! -f $MARKER_PATH ]]; then - echo "missing" - return 0 - fi - local marker_time - marker_time="$(cat $MARKER_PATH)" - - local end_of_fresh_time=$((FRESH_NODE_TIMEOUT + $marker_time)) - local now - now=$(date +%s) - if [[ $now -le $end_of_fresh_time ]]; then - echo "fresh" - return 0 - fi - echo "stale" - return 0 -} - -ping-node() { - local result - result="$(rabbitmqctl eval 'ok.' 2>&1)" - if [[ "$result" == "ok" ]]; then - return 0 - fi - log-it "ping-node error:" - echo "$result" | prepend-log-prefix - return 1 -} - -is-node-booting() { - local result - result="$(rabbitmqctl eval 'is_pid(erlang:whereis(rabbit_boot)).' 2>&1)" - - case "$result" in - true) - return 0 - ;; - false) - return 1 - ;; - *) - log-it "is-node-booting error:" - echo "$result" | prepend-log-prefix - return 1 - ;; - esac -} - -is-node-healthy() { - local result - result=$(rabbitmqctl node_health_check -t 30 2>&1) - if [[ "$result" =~ "Health check passed" ]]; then - return 0 - fi - echo "$result" | prepend-log-prefix - return 1 -} - -is-node-properly-clustered() { - result="$(rabbitmqctl eval 'autocluster:cluster_health_check().' 2>&1)" - if [[ $result =~ ^SUCCESS: ]]; then - return 0 - elif [[ $result =~ ^FAILURE: ]]; then - echo "$result" | prepend-log-prefix - return 1 - fi - log-it "Unexpected health-check output, giving the node the benefit of the doubt" - echo "$result" | prepend-log-prefix - return 0 -} diff --git a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl index 780fd054b8..4943ef54ef 100644 --- a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl +++ b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl @@ -16,65 +16,4 @@ See the License for the specific language governing permissions and limitations under the License. */}} -# This is taken from https://github.com/openstack/fuel-ccp-rabbitmq/blob/master/service/files/rabbitmq-liveness.sh.j2 -set -eu -set -o pipefail -exec 1>/dev/null 2>&1 - -source $(readlink -f $(dirname $0))/rabbitmq-check-helpers.sh -set-log-prefix "liveness:$$" -log-it "Starting liveness probe at $(date +'%Y-%m-%d %H:%M:%S')" - -main() { - local marker_state - marker_state="$(marker-state)" - case $marker_state in - missing) - log-it "Startup marker missing, probably probe was executed too early" - return 0 - ;; - fresh) # node has recently started - it can still be booting - if ! ping-node; then - log-it "Fresh node, erlang VM hasn't started yet - giving it another chance" - # Erlang VM hasn't started yet - return 0 - fi - if is-node-booting; then - log-it "Node is still booting, giving it some time to finish" - return 0 - fi - if ! is-node-healthy; then - log-it "Node is unhealthy" - return 1 - fi - if ! is-node-properly-clustered; then - log-it "Found clustering inconsistency, giving up" - return 1 - fi - return 0 - ;; - stale) # node has started long ago - it shoud be either ready or dead - if ! is-node-healthy; then - log-it "Long-running node become unhealthy" - return 1 - fi - if ! is-node-properly-clustered; then - echo "Long-running node became inconsistent with the rest of the cluster" - return 1 - fi - return 0 - ;; - *) - log-it "Unexpected marker-state '$marker-state'" - return 1 - ;; - esac -} - -if main; then - rc=0 -else - rc=$? -fi -log-it "Ready to return $rc" -exit $rc +exec rabbitmqctl status diff --git a/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl index d78c957311..4943ef54ef 100644 --- a/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl +++ b/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl @@ -16,38 +16,4 @@ See the License for the specific language governing permissions and limitations under the License. */}} -# This is taken from https://github.com/openstack/fuel-ccp-rabbitmq/blob/master/service/files/rabbitmq-readiness.sh.j2 -set -eu -set -o pipefail -exec 1>/dev/null 2>&1 - -source $(readlink -f $(dirname $0))/rabbitmq-check-helpers.sh -set-log-prefix "readiness:$$" -log-it "Starting readiness probe at $(date +'%Y-%m-%d %H:%M:%S')" - -main() { - if [[ "$(marker-state)" == missing ]]; then - log-it "Startup marker missing, probably probe was executed too early" - return 1 - fi - if ! is-node-healthy; then - log-it "Node is unhealthy" - return 1 - fi - - {{ if gt (.Values.pod.replicas.server | int) 1 -}} - if ! is-node-properly-clustered; then - log-it "Node is inconsistent with the rest of the cluster" - return 1 - fi - {{- end }} - return 0 -} - -if main; then - rc=0 -else - rc=$? -fi -log-it "Ready to return $rc" -exit $rc +exec rabbitmqctl status diff --git a/rabbitmq/templates/bin/_rabbitmq-start.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-start.sh.tpl index 31ba6cfe2b..98394ddfdd 100644 --- a/rabbitmq/templates/bin/_rabbitmq-start.sh.tpl +++ b/rabbitmq/templates/bin/_rabbitmq-start.sh.tpl @@ -16,14 +16,6 @@ See the License for the specific language governing permissions and limitations under the License. */}} -set -eux -set -o pipefail -cp /etc/rabbitmq/erlang.cookie /var/lib/rabbitmq/.erlang.cookie -chmod 600 /var/lib/rabbitmq/.erlang.cookie -# This should be called after rabbitmq-server is started but in current design we don't have -# any other way of doing this. PreStart could not be used here as it's: -# - executed just after container creation (not after entrypoint) -# - Currently, there are (hopefully rare) scenarios where PostStart hooks may not be delivered. -# Startup marker is used by liveness and readiness probes. -date +%s > /tmp/rabbit-startup-marker -exec /usr/lib/rabbitmq/bin/rabbitmq-server +set -ex + +exec /docker-entrypoint.sh rabbitmq-server diff --git a/rabbitmq/templates/configmap-bin.yaml b/rabbitmq/templates/configmap-bin.yaml index 294890e644..4a6ef93bd6 100644 --- a/rabbitmq/templates/configmap-bin.yaml +++ b/rabbitmq/templates/configmap-bin.yaml @@ -14,20 +14,18 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- apiVersion: v1 kind: ConfigMap metadata: name: rabbitmq-bin - labels: - system: openstack - type: configuration - component: messaging data: rabbitmq-liveness.sh: | {{ tuple "bin/_rabbitmq-liveness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} rabbitmq-readiness.sh: | {{ tuple "bin/_rabbitmq-readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - rabbitmq-check-helpers.sh: | -{{ tuple "bin/_rabbitmq-check-helpers.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - start.sh: | + rabbitmq-start.sh: | {{ tuple "bin/_rabbitmq-start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{ end }} diff --git a/rabbitmq/templates/configmap-etc.yaml b/rabbitmq/templates/configmap-etc.yaml index f0004b2f59..9a5491d5bc 100644 --- a/rabbitmq/templates/configmap-etc.yaml +++ b/rabbitmq/templates/configmap-etc.yaml @@ -14,21 +14,33 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} + +{{- if empty .Values.conf.rabbitmq.cluster_formation.k8s.service_name -}} +{{- tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | set .Values.conf.rabbitmq.cluster_formation.k8s "service_name" | quote | trunc 0 -}} +{{- end -}} +{{- if empty .Values.conf.rabbitmq.cluster_formation.k8s.host -}} +{{- print "kubernetes.default.svc." .Values.endpoints.cluster_domain_suffix | set .Values.conf.rabbitmq.cluster_formation.k8s "host" | quote | trunc 0 -}} +{{- end -}} + +{{- print "0.0.0.0:" (tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup") | set .Values.conf.rabbitmq.listeners.tcp "1" | quote | trunc 0 -}} + +{{- if empty .Values.conf.rabbitmq.default_user -}} +{{- set .Values.conf.rabbitmq "default_user" .Values.endpoints.oslo_messaging.auth.user.username | quote | trunc 0 -}} +{{- end -}} +{{- if empty .Values.conf.rabbitmq.default_pass -}} +{{- set .Values.conf.rabbitmq "default_pass" .Values.endpoints.oslo_messaging.auth.user.password | quote | trunc 0 -}} +{{- end -}} + +--- apiVersion: v1 kind: ConfigMap metadata: name: rabbitmq-etc - labels: - system: openstack - type: configuration - component: messaging data: enabled_plugins: | {{ tuple "etc/_enabled_plugins.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - erlang.cookie: | -{{ tuple "etc/_erlang.cookie.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - rabbitmq-env.conf: | -{{ tuple "etc/_rabbitmq-env.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - rabbitmq.config: | -{{ tuple "etc/_rabbitmq.config.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - + rabbitmq.conf: | +{{ include "rabbitmq.to_rabbit_config" .Values.conf.rabbitmq | indent 4 }} +{{ end }} diff --git a/rabbitmq/templates/etc/_enabled_plugins.tpl b/rabbitmq/templates/etc/_enabled_plugins.tpl index a7ba95131f..42f415a660 100644 --- a/rabbitmq/templates/etc/_enabled_plugins.tpl +++ b/rabbitmq/templates/etc/_enabled_plugins.tpl @@ -1,15 +1,17 @@ -% Copyright 2017 The Openstack-Helm Authors. -% -% Licensed under the Apache License, Version 2.0 (the "License"); -% you may not use this file except in compliance with the License. -% You may obtain a copy of the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, -% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% See the License for the specific language governing permissions and -% limitations under the License. +{{/* +Copyright 2017 The Openstack-Helm Authors. -[{{ include "helm-toolkit.utils.joinListWithComma" .Values.enabled_plugins }}]. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[{{ include "helm-toolkit.utils.joinListWithComma" .Values.conf.enabled_plugins }}]. diff --git a/rabbitmq/templates/etc/_erlang.cookie.tpl b/rabbitmq/templates/etc/_erlang.cookie.tpl deleted file mode 100644 index 58116146fd..0000000000 --- a/rabbitmq/templates/etc/_erlang.cookie.tpl +++ /dev/null @@ -1 +0,0 @@ -{{ .Values.erlang_cookie }} diff --git a/rabbitmq/templates/etc/_rabbitmq-env.conf.tpl b/rabbitmq/templates/etc/_rabbitmq-env.conf.tpl deleted file mode 100644 index fc89d5fac9..0000000000 --- a/rabbitmq/templates/etc/_rabbitmq-env.conf.tpl +++ /dev/null @@ -1,24 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -RABBITMQ_LOGS=- -RABBITMQ_SASL_LOGS=- -AUTOCLUSTER_TYPE=etcd -AUTOCLUSTER_DELAY={{ .Values.autocluster.delay }} -RABBITMQ_USE_LONGNAME=true -AUTOCLUSTER_LOG_LEVEL={{ .Values.autocluster.log_level }} -NODENAME="rabbit@${RABBITMQ_POD_IP}" -RABBITMQ_NODE_TYPE={{ .Values.autocluster.node_type }} diff --git a/rabbitmq/templates/etc/_rabbitmq.config.tpl b/rabbitmq/templates/etc/_rabbitmq.config.tpl deleted file mode 100644 index 186c24f844..0000000000 --- a/rabbitmq/templates/etc/_rabbitmq.config.tpl +++ /dev/null @@ -1,45 +0,0 @@ -% Copyright 2017 The Openstack-Helm Authors. -% -% Licensed under the Apache License, Version 2.0 (the "License"); -% you may not use this file except in compliance with the License. -% You may obtain a copy of the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, -% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% See the License for the specific language governing permissions and -% limitations under the License. - -[ - {rabbit, [ - {dummy_param_without_comma, true} - ,{tcp_listeners, [ - {"0.0.0.0", {{ .Values.network.port.public }} } - ]} - ,{default_user, <<"{{ .Values.auth.default_user }}">>} - ,{default_pass, <<"{{ .Values.auth.default_pass }}">>} - ,{loopback_users, []} - ,{cluster_partition_handling, ignore} - ,{queue_master_locator, <<"random">>} - ]} - ,{autocluster, [ - {dummy_param_without_comma, true} - ,{backend, etcd} - ,{autocluster_log_level,{{ .Values.autocluster.log_level }}} - ,{autocluster_failure, stop} - ,{cleanup_interval, 30} - ,{cluster_cleanup, true} - ,{cleanup_warn_only, false} - ,{etcd_node_ttl, 15} - ,{etcd_scheme, http} - ,{etcd_host, {{ .Values.endpoints.etcd.hosts.default }}} - ,{etcd_port, {{ .Values.endpoints.etcd.port }}} - ,{node_type, {{ .Values.autocluster.node_type }} } - ]} - ,{rabbitmq_management, [ - {listener, [{port, {{ .Values.network.port.management }}}]} - ]} -]. -% EOF diff --git a/rabbitmq/templates/service.yaml b/rabbitmq/templates/service.yaml index 1c7368007d..e9ba424ceb 100644 --- a/rabbitmq/templates/service.yaml +++ b/rabbitmq/templates/service.yaml @@ -13,17 +13,20 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */}} + +{{- if .Values.manifests.service }} {{- $envAll := . }} --- apiVersion: v1 kind: Service metadata: - name: rabbitmq + name: {{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: ports: - - port: {{.Values.network.port.public}} - name: public - - port: {{.Values.network.port.management}} - name: management + - port: {{ tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: amqp + - port: {{ tuple "oslo_messaging" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: http selector: {{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{ end }} diff --git a/rabbitmq/templates/deployment.yaml b/rabbitmq/templates/statefulset.yaml similarity index 56% rename from rabbitmq/templates/deployment.yaml rename to rabbitmq/templates/statefulset.yaml index 05ded8a887..b5347411c8 100644 --- a/rabbitmq/templates/deployment.yaml +++ b/rabbitmq/templates/statefulset.yaml @@ -14,19 +14,52 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- if .Values.manifests.statefulset }} {{- $envAll := . }} {{- $dependencies := .Values.dependencies.rabbitmq }} {{- $serviceAccountName := "rabbitmq" }} {{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- -kind: Deployment +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - "" + - extensions + - batch + - apps + verbs: + - get + - list + resources: + - services + - endpoints +--- apiVersion: apps/v1beta1 +kind: StatefulSet metadata: name: rabbitmq spec: + serviceName: {{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} replicas: {{ .Values.pod.replicas.server }} -{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} template: metadata: labels: @@ -47,45 +80,51 @@ spec: image: {{ .Values.images.tags.rabbitmq }} {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} command: - - /scripts/start.sh + - /tmp/rabbitmq-start.sh + ports: + - name: http + protocol: TCP + containerPort: {{ tuple "oslo_messaging" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: amqp + protocol: TCP + containerPort: {{ tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} env: - - name: RABBITMQ_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: RABBITMQ_USE_LONGNAME + value: "true" + - name: RABBITMQ_NODENAME + value: "rabbit@$(MY_POD_IP)" + - name: K8S_SERVICE_NAME + value: {{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }} + - name: RABBITMQ_ERLANG_COOKIE + value: "{{ .Values.endpoints.oslo_messaging.auth.erlang_cookie }}" readinessProbe: - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: {{ .Values.probes_timeout }} + initialDelaySeconds: 10 + timeoutSeconds: 10 exec: command: - - /scripts/rabbitmq-readiness.sh + - /tmp/rabbitmq-readiness.sh livenessProbe: - initialDelaySeconds: {{ .Values.probes_delay }} - timeoutSeconds: {{ .Values.probes_timeout }} + initialDelaySeconds: 30 + timeoutSeconds: 10 exec: command: - - /scripts/rabbitmq-liveness.sh + - /tmp/rabbitmq-liveness.sh volumeMounts: - name: rabbitmq-emptydir mountPath: /var/lib/rabbitmq - name: rabbitmq-bin - mountPath: /scripts + mountPath: /tmp - name: rabbitmq-etc mountPath: /etc/rabbitmq/enabled_plugins subPath: enabled_plugins readOnly: true - name: rabbitmq-etc - mountPath: /etc/rabbitmq/erlang.cookie - subPath: erlang.cookie - readOnly: true - - name: rabbitmq-etc - mountPath: /etc/rabbitmq/rabbitmq-env.conf - subPath: rabbitmq-env.conf - readOnly: true - - name: rabbitmq-etc - mountPath: /etc/rabbitmq/rabbitmq.config - subPath: rabbitmq.config + mountPath: /etc/rabbitmq/rabbitmq.conf + subPath: rabbitmq.conf readOnly: true volumes: - name: rabbitmq-emptydir @@ -98,3 +137,4 @@ spec: configMap: name: rabbitmq-etc defaultMode: 0444 +{{ end }} diff --git a/rabbitmq/values.yaml b/rabbitmq/values.yaml index bac22e44a2..509c4bd55e 100644 --- a/rabbitmq/values.yaml +++ b/rabbitmq/values.yaml @@ -21,22 +21,21 @@ labels: node_selector_key: openstack-control-plane node_selector_value: enabled +images: + tags: + rabbitmq: docker.io/rabbitmq:3.7 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 + pull_policy: "IfNotPresent" + pod: affinity: - anti: - type: - default: preferredDuringSchedulingIgnoredDuringExecution - topologyKey: - default: kubernetes.io/hostname + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname replicas: server: 3 - lifecycle: - upgrades: - revision_history: 3 - pod_replacement_strategy: RollingUpdate - rolling_update: - max_unavailable: 1 - max_surge: 3 resources: enabled: false server: @@ -47,48 +46,55 @@ pod: memory: "128Mi" cpu: "500m" -auth: - default_user: rabbitmq - default_pass: password - admin_user: admin - admin_pass: password - -network: - port: - public: '5672' - management: '15672' - -images: - tags: - rabbitmq: quay.io/attcomdev/fuel-mcp-rabbitmq:ocata-unstable - dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 - pull_policy: "IfNotPresent" - -enabled_plugins: - - autocluster - - rabbitmq_management - -erlang_cookie: openstack-cookie - -endpoints: - cluster_domain_suffix: cluster.local - etcd: - hosts: - default: etcd - host_fqdn_override: - default: null - port: 2379 - -autocluster: - log_level: info - delay: 15 - node_type: disc - -probes_delay: 180 -probes_timeout: 10 +conf: + enabled_plugins: + - rabbitmq_management + - rabbitmq_peer_discovery_k8s + rabbitmq: + listeners: + tcp: + #NOTE(portdirect): This is always defined via the endpoints section. + 1: null + cluster_formation: + peer_discovery_backend: rabbit_peer_discovery_k8s + k8s: + address_type: ip + node_cleanup: + interval: "10" + only_log_warning: "false" + cluster_partition_handling: autoheal + queue_master_locator: min-masters + loopback_users.guest: "false" dependencies: rabbitmq: - services: - - service: etcd - endpoint: internal + jobs: null + +# typically overriden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + cluster_domain_suffix: cluster.local + oslo_messaging: + auth: + erlang_cookie: openstack-cookie + user: + username: rabbitmq + password: password + hosts: + default: rabbitmq + host_fqdn_override: + default: null + path: / + scheme: rabbit + port: + amqp: + default: 5672 + http: + default: 15672 + +manifests: + configmap_bin: true + configmap_etc: true + service: true + statefulset: true diff --git a/tools/deployment/developer/07-rabbitmq.sh b/tools/deployment/developer/07-rabbitmq.sh index cf862c003d..4eb173ce7d 100755 --- a/tools/deployment/developer/07-rabbitmq.sh +++ b/tools/deployment/developer/07-rabbitmq.sh @@ -17,21 +17,15 @@ set -xe #NOTE: Pull images and lint chart -make pull-images etcd make pull-images rabbitmq #NOTE: Deploy command -helm install ./etcd \ - --namespace=openstack \ - --name=etcd-rabbitmq helm install ./rabbitmq \ --namespace=openstack \ - --name=rabbitmq \ - --set pod.replicas.server=1 + --name=rabbitmq #NOTE: Wait for deploy ./tools/deployment/developer/wait-for-pods.sh openstack #NOTE: Validate Deployment info -helm status etcd-rabbitmq helm status rabbitmq diff --git a/tools/gate/launch-osh/basic.sh b/tools/gate/launch-osh/basic.sh index a7577f67cf..9dad6ffd4b 100755 --- a/tools/gate/launch-osh/basic.sh +++ b/tools/gate/launch-osh/basic.sh @@ -119,13 +119,7 @@ if [ "x$OPENSTACK_OBJECT_STORAGE" == "xradosgw" ]; then kube_wait_for_pods openstack ${POD_START_TIMEOUT_OPENSTACK} fi -helm install --namespace=openstack ${WORK_DIR}/etcd --name=etcd-rabbitmq -if [ "x$INTEGRATION" == "xmulti" ]; then - helm install --namespace=openstack ${WORK_DIR}/rabbitmq --name=rabbitmq -else - helm install --namespace=openstack ${WORK_DIR}/rabbitmq --name=rabbitmq \ - --set pod.replicas.server=1 -fi +helm install --namespace=openstack ${WORK_DIR}/rabbitmq --name=rabbitmq if [ "x$INTEGRATION" == "xmulti" ]; then if [ "x$PVC_BACKEND" == "xceph" ]; then