RabbitMQ 3.7 with native k8s clustering

This PS moves to use RabbitMQ with native k8s clustering.

Change-Id: I6a93c8168bc7dce6b73871aa1fd4d637e6192975
This commit is contained in:
portdirect 2018-01-05 03:10:20 -05:00
parent bd3dd6c83a
commit 0c51d813b3
16 changed files with 221 additions and 418 deletions

View File

@ -0,0 +1,37 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "rabbitmq.to_rabbit_config" -}}
{{- range $top_key, $top_value := . }}
{{- if kindIs "map" $top_value -}}
{{- range $second_key, $second_value := . }}
{{- if kindIs "map" $second_value -}}
{{- range $third_key, $third_value := . }}
{{- if kindIs "map" $third_value -}}
{{ $top_key }}.{{ $second_key }}.{{ $third_key }} = wow
{{ else -}}
{{ $top_key }}.{{ $second_key }}.{{ $third_key }} = {{ $third_value }}
{{ end -}}
{{- end -}}
{{ else -}}
{{ $top_key }}.{{ $second_key }} = {{ $second_value }}
{{ end -}}
{{- end -}}
{{ else -}}
{{ $top_key }} = {{ $top_value }}
{{ end -}}
{{- end -}}
{{- end -}}

View File

@ -1,110 +0,0 @@
#!/usr/bin/env bash
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
# This is taken from https://github.com/openstack/fuel-ccp-rabbitmq/blob/master/service/files/rabbitmq-check-helpers.sh.j2
MARKER_PATH=/tmp/rabbit-startup-marker
# How many seconds we give a node before successfull liveness checks
# become mandatory.
FRESH_NODE_TIMEOUT={{ .Values.probes_delay }}
LP=""
set-log-prefix() {
LP="[${1:?}]"
}
log-it() {
echo "$LP" "$@"
}
prepend-log-prefix() {
awk -v lp="$LP" '{print lp " " $0}'
}
marker-state() {
if [[ ! -f $MARKER_PATH ]]; then
echo "missing"
return 0
fi
local marker_time
marker_time="$(cat $MARKER_PATH)"
local end_of_fresh_time=$((FRESH_NODE_TIMEOUT + $marker_time))
local now
now=$(date +%s)
if [[ $now -le $end_of_fresh_time ]]; then
echo "fresh"
return 0
fi
echo "stale"
return 0
}
ping-node() {
local result
result="$(rabbitmqctl eval 'ok.' 2>&1)"
if [[ "$result" == "ok" ]]; then
return 0
fi
log-it "ping-node error:"
echo "$result" | prepend-log-prefix
return 1
}
is-node-booting() {
local result
result="$(rabbitmqctl eval 'is_pid(erlang:whereis(rabbit_boot)).' 2>&1)"
case "$result" in
true)
return 0
;;
false)
return 1
;;
*)
log-it "is-node-booting error:"
echo "$result" | prepend-log-prefix
return 1
;;
esac
}
is-node-healthy() {
local result
result=$(rabbitmqctl node_health_check -t 30 2>&1)
if [[ "$result" =~ "Health check passed" ]]; then
return 0
fi
echo "$result" | prepend-log-prefix
return 1
}
is-node-properly-clustered() {
result="$(rabbitmqctl eval 'autocluster:cluster_health_check().' 2>&1)"
if [[ $result =~ ^SUCCESS: ]]; then
return 0
elif [[ $result =~ ^FAILURE: ]]; then
echo "$result" | prepend-log-prefix
return 1
fi
log-it "Unexpected health-check output, giving the node the benefit of the doubt"
echo "$result" | prepend-log-prefix
return 0
}

View File

@ -16,65 +16,4 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
# This is taken from https://github.com/openstack/fuel-ccp-rabbitmq/blob/master/service/files/rabbitmq-liveness.sh.j2
set -eu
set -o pipefail
exec 1>/dev/null 2>&1
source $(readlink -f $(dirname $0))/rabbitmq-check-helpers.sh
set-log-prefix "liveness:$$"
log-it "Starting liveness probe at $(date +'%Y-%m-%d %H:%M:%S')"
main() {
local marker_state
marker_state="$(marker-state)"
case $marker_state in
missing)
log-it "Startup marker missing, probably probe was executed too early"
return 0
;;
fresh) # node has recently started - it can still be booting
if ! ping-node; then
log-it "Fresh node, erlang VM hasn't started yet - giving it another chance"
# Erlang VM hasn't started yet
return 0
fi
if is-node-booting; then
log-it "Node is still booting, giving it some time to finish"
return 0
fi
if ! is-node-healthy; then
log-it "Node is unhealthy"
return 1
fi
if ! is-node-properly-clustered; then
log-it "Found clustering inconsistency, giving up"
return 1
fi
return 0
;;
stale) # node has started long ago - it shoud be either ready or dead
if ! is-node-healthy; then
log-it "Long-running node become unhealthy"
return 1
fi
if ! is-node-properly-clustered; then
echo "Long-running node became inconsistent with the rest of the cluster"
return 1
fi
return 0
;;
*)
log-it "Unexpected marker-state '$marker-state'"
return 1
;;
esac
}
if main; then
rc=0
else
rc=$?
fi
log-it "Ready to return $rc"
exit $rc
exec rabbitmqctl status

View File

@ -16,38 +16,4 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
# This is taken from https://github.com/openstack/fuel-ccp-rabbitmq/blob/master/service/files/rabbitmq-readiness.sh.j2
set -eu
set -o pipefail
exec 1>/dev/null 2>&1
source $(readlink -f $(dirname $0))/rabbitmq-check-helpers.sh
set-log-prefix "readiness:$$"
log-it "Starting readiness probe at $(date +'%Y-%m-%d %H:%M:%S')"
main() {
if [[ "$(marker-state)" == missing ]]; then
log-it "Startup marker missing, probably probe was executed too early"
return 1
fi
if ! is-node-healthy; then
log-it "Node is unhealthy"
return 1
fi
{{ if gt (.Values.pod.replicas.server | int) 1 -}}
if ! is-node-properly-clustered; then
log-it "Node is inconsistent with the rest of the cluster"
return 1
fi
{{- end }}
return 0
}
if main; then
rc=0
else
rc=$?
fi
log-it "Ready to return $rc"
exit $rc
exec rabbitmqctl status

View File

@ -16,14 +16,6 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -eux
set -o pipefail
cp /etc/rabbitmq/erlang.cookie /var/lib/rabbitmq/.erlang.cookie
chmod 600 /var/lib/rabbitmq/.erlang.cookie
# This should be called after rabbitmq-server is started but in current design we don't have
# any other way of doing this. PreStart could not be used here as it's:
# - executed just after container creation (not after entrypoint)
# - Currently, there are (hopefully rare) scenarios where PostStart hooks may not be delivered.
# Startup marker is used by liveness and readiness probes.
date +%s > /tmp/rabbit-startup-marker
exec /usr/lib/rabbitmq/bin/rabbitmq-server
set -ex
exec /docker-entrypoint.sh rabbitmq-server

View File

@ -14,20 +14,18 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.configmap_bin }}
{{- $envAll := . }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: rabbitmq-bin
labels:
system: openstack
type: configuration
component: messaging
data:
rabbitmq-liveness.sh: |
{{ tuple "bin/_rabbitmq-liveness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
rabbitmq-readiness.sh: |
{{ tuple "bin/_rabbitmq-readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
rabbitmq-check-helpers.sh: |
{{ tuple "bin/_rabbitmq-check-helpers.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
start.sh: |
rabbitmq-start.sh: |
{{ tuple "bin/_rabbitmq-start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{ end }}

View File

@ -14,21 +14,33 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.configmap_etc }}
{{- $envAll := . }}
{{- if empty .Values.conf.rabbitmq.cluster_formation.k8s.service_name -}}
{{- tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | set .Values.conf.rabbitmq.cluster_formation.k8s "service_name" | quote | trunc 0 -}}
{{- end -}}
{{- if empty .Values.conf.rabbitmq.cluster_formation.k8s.host -}}
{{- print "kubernetes.default.svc." .Values.endpoints.cluster_domain_suffix | set .Values.conf.rabbitmq.cluster_formation.k8s "host" | quote | trunc 0 -}}
{{- end -}}
{{- print "0.0.0.0:" (tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup") | set .Values.conf.rabbitmq.listeners.tcp "1" | quote | trunc 0 -}}
{{- if empty .Values.conf.rabbitmq.default_user -}}
{{- set .Values.conf.rabbitmq "default_user" .Values.endpoints.oslo_messaging.auth.user.username | quote | trunc 0 -}}
{{- end -}}
{{- if empty .Values.conf.rabbitmq.default_pass -}}
{{- set .Values.conf.rabbitmq "default_pass" .Values.endpoints.oslo_messaging.auth.user.password | quote | trunc 0 -}}
{{- end -}}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: rabbitmq-etc
labels:
system: openstack
type: configuration
component: messaging
data:
enabled_plugins: |
{{ tuple "etc/_enabled_plugins.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
erlang.cookie: |
{{ tuple "etc/_erlang.cookie.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
rabbitmq-env.conf: |
{{ tuple "etc/_rabbitmq-env.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
rabbitmq.config: |
{{ tuple "etc/_rabbitmq.config.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
rabbitmq.conf: |
{{ include "rabbitmq.to_rabbit_config" .Values.conf.rabbitmq | indent 4 }}
{{ end }}

View File

@ -1,15 +1,17 @@
% Copyright 2017 The Openstack-Helm Authors.
%
% Licensed under the Apache License, Version 2.0 (the "License");
% you may not use this file except in compliance with the License.
% You may obtain a copy of the License at
%
% http://www.apache.org/licenses/LICENSE-2.0
%
% Unless required by applicable law or agreed to in writing, software
% distributed under the License is distributed on an "AS IS" BASIS,
% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
% See the License for the specific language governing permissions and
% limitations under the License.
{{/*
Copyright 2017 The Openstack-Helm Authors.
[{{ include "helm-toolkit.utils.joinListWithComma" .Values.enabled_plugins }}].
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
[{{ include "helm-toolkit.utils.joinListWithComma" .Values.conf.enabled_plugins }}].

View File

@ -1 +0,0 @@
{{ .Values.erlang_cookie }}

View File

@ -1,24 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
RABBITMQ_LOGS=-
RABBITMQ_SASL_LOGS=-
AUTOCLUSTER_TYPE=etcd
AUTOCLUSTER_DELAY={{ .Values.autocluster.delay }}
RABBITMQ_USE_LONGNAME=true
AUTOCLUSTER_LOG_LEVEL={{ .Values.autocluster.log_level }}
NODENAME="rabbit@${RABBITMQ_POD_IP}"
RABBITMQ_NODE_TYPE={{ .Values.autocluster.node_type }}

View File

@ -1,45 +0,0 @@
% Copyright 2017 The Openstack-Helm Authors.
%
% Licensed under the Apache License, Version 2.0 (the "License");
% you may not use this file except in compliance with the License.
% You may obtain a copy of the License at
%
% http://www.apache.org/licenses/LICENSE-2.0
%
% Unless required by applicable law or agreed to in writing, software
% distributed under the License is distributed on an "AS IS" BASIS,
% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
% See the License for the specific language governing permissions and
% limitations under the License.
[
{rabbit, [
{dummy_param_without_comma, true}
,{tcp_listeners, [
{"0.0.0.0", {{ .Values.network.port.public }} }
]}
,{default_user, <<"{{ .Values.auth.default_user }}">>}
,{default_pass, <<"{{ .Values.auth.default_pass }}">>}
,{loopback_users, []}
,{cluster_partition_handling, ignore}
,{queue_master_locator, <<"random">>}
]}
,{autocluster, [
{dummy_param_without_comma, true}
,{backend, etcd}
,{autocluster_log_level,{{ .Values.autocluster.log_level }}}
,{autocluster_failure, stop}
,{cleanup_interval, 30}
,{cluster_cleanup, true}
,{cleanup_warn_only, false}
,{etcd_node_ttl, 15}
,{etcd_scheme, http}
,{etcd_host, {{ .Values.endpoints.etcd.hosts.default }}}
,{etcd_port, {{ .Values.endpoints.etcd.port }}}
,{node_type, {{ .Values.autocluster.node_type }} }
]}
,{rabbitmq_management, [
{listener, [{port, {{ .Values.network.port.management }}}]}
]}
].
% EOF

View File

@ -13,17 +13,20 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.service }}
{{- $envAll := . }}
---
apiVersion: v1
kind: Service
metadata:
name: rabbitmq
name: {{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
spec:
ports:
- port: {{.Values.network.port.public}}
name: public
- port: {{.Values.network.port.management}}
name: management
- port: {{ tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
name: amqp
- port: {{ tuple "oslo_messaging" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
name: http
selector:
{{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
{{ end }}

View File

@ -14,19 +14,52 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.statefulset }}
{{- $envAll := . }}
{{- $dependencies := .Values.dependencies.rabbitmq }}
{{- $serviceAccountName := "rabbitmq" }}
{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
kind: Deployment
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: {{ $serviceAccountName }}
namespace: {{ .Release.Namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ $serviceAccountName }}
subjects:
- kind: ServiceAccount
name: {{ $serviceAccountName }}
namespace: {{ .Release.Namespace }}
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: {{ $serviceAccountName }}
namespace: {{ .Release.Namespace }}
rules:
- apiGroups:
- ""
- extensions
- batch
- apps
verbs:
- get
- list
resources:
- services
- endpoints
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: rabbitmq
spec:
serviceName: {{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
replicas: {{ .Values.pod.replicas.server }}
{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
template:
metadata:
labels:
@ -47,45 +80,51 @@ spec:
image: {{ .Values.images.tags.rabbitmq }}
{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command:
- /scripts/start.sh
- /tmp/rabbitmq-start.sh
ports:
- name: http
protocol: TCP
containerPort: {{ tuple "oslo_messaging" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
- name: amqp
protocol: TCP
containerPort: {{ tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
env:
- name: RABBITMQ_POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: MY_POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: RABBITMQ_USE_LONGNAME
value: "true"
- name: RABBITMQ_NODENAME
value: "rabbit@$(MY_POD_IP)"
- name: K8S_SERVICE_NAME
value: {{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
- name: RABBITMQ_ERLANG_COOKIE
value: "{{ .Values.endpoints.oslo_messaging.auth.erlang_cookie }}"
readinessProbe:
initialDelaySeconds: 30
periodSeconds: 30
timeoutSeconds: {{ .Values.probes_timeout }}
initialDelaySeconds: 10
timeoutSeconds: 10
exec:
command:
- /scripts/rabbitmq-readiness.sh
- /tmp/rabbitmq-readiness.sh
livenessProbe:
initialDelaySeconds: {{ .Values.probes_delay }}
timeoutSeconds: {{ .Values.probes_timeout }}
initialDelaySeconds: 30
timeoutSeconds: 10
exec:
command:
- /scripts/rabbitmq-liveness.sh
- /tmp/rabbitmq-liveness.sh
volumeMounts:
- name: rabbitmq-emptydir
mountPath: /var/lib/rabbitmq
- name: rabbitmq-bin
mountPath: /scripts
mountPath: /tmp
- name: rabbitmq-etc
mountPath: /etc/rabbitmq/enabled_plugins
subPath: enabled_plugins
readOnly: true
- name: rabbitmq-etc
mountPath: /etc/rabbitmq/erlang.cookie
subPath: erlang.cookie
readOnly: true
- name: rabbitmq-etc
mountPath: /etc/rabbitmq/rabbitmq-env.conf
subPath: rabbitmq-env.conf
readOnly: true
- name: rabbitmq-etc
mountPath: /etc/rabbitmq/rabbitmq.config
subPath: rabbitmq.config
mountPath: /etc/rabbitmq/rabbitmq.conf
subPath: rabbitmq.conf
readOnly: true
volumes:
- name: rabbitmq-emptydir
@ -98,3 +137,4 @@ spec:
configMap:
name: rabbitmq-etc
defaultMode: 0444
{{ end }}

View File

@ -21,22 +21,21 @@ labels:
node_selector_key: openstack-control-plane
node_selector_value: enabled
images:
tags:
rabbitmq: docker.io/rabbitmq:3.7
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
pull_policy: "IfNotPresent"
pod:
affinity:
anti:
type:
default: preferredDuringSchedulingIgnoredDuringExecution
topologyKey:
default: kubernetes.io/hostname
anti:
type:
default: preferredDuringSchedulingIgnoredDuringExecution
topologyKey:
default: kubernetes.io/hostname
replicas:
server: 3
lifecycle:
upgrades:
revision_history: 3
pod_replacement_strategy: RollingUpdate
rolling_update:
max_unavailable: 1
max_surge: 3
resources:
enabled: false
server:
@ -47,48 +46,55 @@ pod:
memory: "128Mi"
cpu: "500m"
auth:
default_user: rabbitmq
default_pass: password
admin_user: admin
admin_pass: password
network:
port:
public: '5672'
management: '15672'
images:
tags:
rabbitmq: quay.io/attcomdev/fuel-mcp-rabbitmq:ocata-unstable
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
pull_policy: "IfNotPresent"
enabled_plugins:
- autocluster
- rabbitmq_management
erlang_cookie: openstack-cookie
endpoints:
cluster_domain_suffix: cluster.local
etcd:
hosts:
default: etcd
host_fqdn_override:
default: null
port: 2379
autocluster:
log_level: info
delay: 15
node_type: disc
probes_delay: 180
probes_timeout: 10
conf:
enabled_plugins:
- rabbitmq_management
- rabbitmq_peer_discovery_k8s
rabbitmq:
listeners:
tcp:
#NOTE(portdirect): This is always defined via the endpoints section.
1: null
cluster_formation:
peer_discovery_backend: rabbit_peer_discovery_k8s
k8s:
address_type: ip
node_cleanup:
interval: "10"
only_log_warning: "false"
cluster_partition_handling: autoheal
queue_master_locator: min-masters
loopback_users.guest: "false"
dependencies:
rabbitmq:
services:
- service: etcd
endpoint: internal
jobs: null
# typically overriden by environmental
# values, but should include all endpoints
# required by this chart
endpoints:
cluster_domain_suffix: cluster.local
oslo_messaging:
auth:
erlang_cookie: openstack-cookie
user:
username: rabbitmq
password: password
hosts:
default: rabbitmq
host_fqdn_override:
default: null
path: /
scheme: rabbit
port:
amqp:
default: 5672
http:
default: 15672
manifests:
configmap_bin: true
configmap_etc: true
service: true
statefulset: true

View File

@ -17,21 +17,15 @@
set -xe
#NOTE: Pull images and lint chart
make pull-images etcd
make pull-images rabbitmq
#NOTE: Deploy command
helm install ./etcd \
--namespace=openstack \
--name=etcd-rabbitmq
helm install ./rabbitmq \
--namespace=openstack \
--name=rabbitmq \
--set pod.replicas.server=1
--name=rabbitmq
#NOTE: Wait for deploy
./tools/deployment/developer/wait-for-pods.sh openstack
#NOTE: Validate Deployment info
helm status etcd-rabbitmq
helm status rabbitmq

View File

@ -119,13 +119,7 @@ if [ "x$OPENSTACK_OBJECT_STORAGE" == "xradosgw" ]; then
kube_wait_for_pods openstack ${POD_START_TIMEOUT_OPENSTACK}
fi
helm install --namespace=openstack ${WORK_DIR}/etcd --name=etcd-rabbitmq
if [ "x$INTEGRATION" == "xmulti" ]; then
helm install --namespace=openstack ${WORK_DIR}/rabbitmq --name=rabbitmq
else
helm install --namespace=openstack ${WORK_DIR}/rabbitmq --name=rabbitmq \
--set pod.replicas.server=1
fi
helm install --namespace=openstack ${WORK_DIR}/rabbitmq --name=rabbitmq
if [ "x$INTEGRATION" == "xmulti" ]; then
if [ "x$PVC_BACKEND" == "xceph" ]; then