openstack/openstack-helm-infra
Code Issues Proposed changes

[CEPH] update all ceph daemons startup scripts to support msgr2

Browse Source 
This is to update all ceph daemons startup scripts as per msgr2 protocol and
also to update v2 port for mon_host config.
This also removes setting mon_addr config since we already have mon_host config.

v1 default port: 6789
V2 default port: 3300

Change-Id: I3d95edbd89f5ac8b40a34f41c1099311cee4f875
This commit is contained in:
Chinasubbareddy Mallavarapu 2020-02-27 17:03:27 +00:00
parent 89bc837094
commit 7425e3e5c0
19 changed files with 61 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
ceph-client/templates/bin/mds/_start.sh.tpl
View File

@ -18,11 +18,14 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then
echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
exit 1
else
ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/"ip"/{print $4":"port}' | paste -sd',')
if [[ ${ENDPOINT} == "" ]]; then
ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
-v version=v1 -v msgr_version=v2 \
-v msgr2_port=${MON_PORT_V2} \
'/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
if [[ "${ENDPOINT}" == "" ]]; then
/bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
else
/bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
/bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true
fi
fi

9
ceph-client/templates/bin/mgr/_start.sh.tpl
View File

@ -10,11 +10,14 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then
echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
exit 1
else
ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/"ip"/{print $4":"port}' | paste -sd',')
if [[ ${ENDPOINT} == "" ]]; then
ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
-v version=v1 -v msgr_version=v2 \
-v msgr2_port=${MON_PORT_V2} \
'/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
if [[ "${ENDPOINT}" == "" ]]; then
/bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
else
/bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
/bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true
fi
fi

6
ceph-client/templates/configmap-etc-client.yaml
View File

@ -22,14 +22,10 @@ limitations under the License.
{{- if .Values.deployment.ceph }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $monHost := tuple "ceph_mon" "internal" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.mon_addr -}}
{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.osd.cluster_network -}}
{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}

2
ceph-client/templates/deployment-mds.yaml
View File

@ -94,6 +94,8 @@ spec:
fieldPath: metadata.namespace
- name: MON_PORT
value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: MON_PORT_V2
value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
ports:
- containerPort: 6800
livenessProbe:

2
ceph-client/templates/deployment-mgr.yaml
View File

@ -92,6 +92,8 @@ spec:
fieldPath: metadata.namespace
- name: MON_PORT
value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: MON_PORT_V2
value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
{{- if .Values.ceph_mgr_enabled_modules }}
- name: ENABLED_MODULES
value: |-

10
ceph-mon/templates/bin/mon/_start.sh.tpl
View File

@ -12,7 +12,7 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then
echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
exit 1
else
ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
-v version=v1 -v msgr_version=v2 \
-v msgr2_port=${MON_PORT_V2} \
'/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
@ -54,9 +54,9 @@ function get_mon_config {
while [[ -z "${MONMAP_ADD// }" && "${timeout}" -gt 0 ]]; do
# Get the ceph mon pods (name and IP) from the Kubernetes API. Formatted as a set of monmap params
if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then
MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}")
MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--addv {{`{{.metadata.name}}`}} [v1:{{`{{.status.podIP}}`}}:${MON_PORT},v2:{{`{{.status.podIP}}`}}:${MON_PORT_V2}] {{`{{end}}`}} {{`{{end}}`}}")
else
MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}")
MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--addv {{`{{.spec.nodeName}}`}} [v1:{{`{{.status.podIP}}`}}:${MON_PORT},v2:{{`{{.status.podIP}}`}}:${MON_PORT_V2}] {{`{{end}}`}} {{`{{end}}`}}")
fi
(( timeout-- ))
sleep 1
@ -99,7 +99,7 @@ else
# no mons are up and running yet
timeout 5 ceph --cluster "${CLUSTER}" mon getmap -o ${MONMAP} || true
ceph-mon --setuser ceph --setgroup ceph --cluster "${CLUSTER}" -i ${MON_NAME} --inject-monmap ${MONMAP} --keyring ${MON_KEYRING} --mon-data "${MON_DATA_DIR}"
timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:${MON_PORT}" || true
timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:${MON_PORT_V2}" || true
fi
# start MON
@ -110,4 +110,4 @@ exec /usr/bin/ceph-mon \
-d \
-i ${MON_NAME} \
--mon-data "${MON_DATA_DIR}" \
--public-addr "${MON_IP}:${MON_PORT}"
--public-addr "${MON_IP}:${MON_PORT_V2}"

11
ceph-mon/templates/bin/utils/_checkDNS.sh.tpl
View File

@ -17,17 +17,14 @@ limitations under the License.
*/}}
: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
ENDPOINT=$1
ENDPOINT="{$1}"
function check_mon_dns () {
GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF})
if [[ ${ENDPOINT} == "up" ]]; then
# If DNS is working, we simply clean up the ${CEPH_CONF} file
if [[ ${GREP_CMD} == "" ]]; then
sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" > /dev/null 2>&1
fi
elif [[ ${ENDPOINT} != "" ]]; then
if [[ "${ENDPOINT}" == "up" ]]; then
echo "If DNS is working, we are good here"
elif [[ "${ENDPOINT}" != "" ]]; then
if [[ ${GREP_CMD} != "" ]]; then
# No DNS, write CEPH MONs IPs into ${CEPH_CONF}
sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1

2
ceph-mon/templates/configmap-etc.yaml
View File

@ -22,7 +22,7 @@ limitations under the License.
{{- if .Values.deployment.ceph }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "discovery" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $monHost := tuple "ceph_mon" "discovery" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}

10
ceph-osd/templates/bin/osd/ceph-disk/_common.sh.tpl
View File

@ -44,12 +44,14 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then
echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
exit 1
else
ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/"ip"/{print $4":"port}' | paste -sd',')
if [[ ${ENDPOINT} == "" ]]; then
# No endpoints are available, just copy ceph.conf as-is
ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
-v version=v1 -v msgr_version=v2 \
-v msgr2_port=${MON_PORT_V2} \
'/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
if [[ "${ENDPOINT}" == "" ]]; then
/bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
else
/bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
/bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true
fi
fi

10
ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl
View File

@ -44,12 +44,14 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then
echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
exit 1
else
ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/"ip"/{print $4":"port}' | paste -sd',')
if [[ ${ENDPOINT} == "" ]]; then
# No endpoints are available, just copy ceph.conf as-is
ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
-v version=v1 -v msgr_version=v2 \
-v msgr2_port=${MON_PORT_V2} \
'/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
if [[ "${ENDPOINT}" == "" ]]; then
/bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
else
/bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
/bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true
fi
fi

11
ceph-osd/templates/bin/utils/_checkDNS.sh.tpl
View File

@ -17,17 +17,14 @@ limitations under the License.
*/}}
: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
ENDPOINT=$1
ENDPOINT="{$1}"
function check_mon_dns () {
GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF})
if [[ ${ENDPOINT} == "up" ]]; then
# If DNS is working, we simply clean up the ${CEPH_CONF} file
if [[ ${GREP_CMD} == "" ]]; then
sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" > /dev/null 2>&1
fi
elif [[ ${ENDPOINT} != "" ]]; then
if [[ "${ENDPOINT}" == "up" ]]; then
echo "If DNS is working, we are good here"
elif [[ "${ENDPOINT}" != "" ]]; then
if [[ ${GREP_CMD} != "" ]]; then
# No DNS, write CEPH MONs IPs into ${CEPH_CONF}
sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1

7
ceph-osd/templates/configmap-etc.yaml
View File

@ -20,15 +20,10 @@ limitations under the License.
{{- with $envAll }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $monHost := tuple "ceph_mon" "internal" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.mon_addr -}}
{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.fsid -}}
{{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}}
{{- end -}}

6
ceph-osd/templates/daemonset-osd.yaml
View File

@ -79,6 +79,8 @@ spec:
fieldPath: metadata.namespace
- name: MON_PORT
value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: MON_PORT_V2
value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
volumeMounts:
- name: pod-tmp
mountPath: /tmp
@ -163,6 +165,8 @@ spec:
fieldPath: metadata.namespace
- name: MON_PORT
value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: MON_PORT_V2
value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
command:
- /tmp/osd-init.sh
volumeMounts:
@ -260,6 +264,8 @@ spec:
fieldPath: metadata.namespace
- name: MON_PORT
value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: MON_PORT_V2
value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
command:
- /tmp/osd-start.sh
lifecycle:

2
ceph-osd/values.yaml
View File

@ -332,6 +332,8 @@ endpoints:
port:
mon:
default: 6789
mon_msgr2:
default: 3300
manifests:
configmap_bin: true

6
ceph-provisioners/templates/configmap-etc-client.yaml
View File

@ -22,14 +22,10 @@ limitations under the License.
{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $monHost := tuple "ceph_mon" "internal" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.mon_addr -}}
{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.osd.cluster_network -}}
{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}

3
ceph-provisioners/values.yaml
View File

@ -287,6 +287,9 @@ endpoints:
port:
mon:
default: 6789
mon_msgr2:
default: 3300
manifests:
configmap_bin: true

6
ceph-rgw/templates/configmap-etc-client.yaml
View File

@ -22,14 +22,10 @@ limitations under the License.
{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $monHost := tuple "ceph_mon" "internal" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.mon_addr -}}
{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.osd.cluster_network -}}
{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}

3
ceph-rgw/values.yaml
View File

@ -547,6 +547,9 @@ endpoints:
port:
mon:
default: 6789
mon_msgr2:
default: 3300
kube_dns:
namespace: kube-system
name: kubernetes-dns

2
helm-toolkit/templates/manifests/_ceph-storageclass.tpl
View File

@ -88,7 +88,7 @@ examples:
{{- $envAll := index . "envAll" -}}
{{- $monHost := $envAll.Values.conf.ceph.global.mon_host -}}
{{- if empty $monHost -}}
{{- $monHost = tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}}
{{- $monHost = tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}}
{{- end -}}
{{- $storageclassData := index . "storageclass_data" -}}
---