From 07c62ddfd5f1160384161d4f164a624c154997ed Mon Sep 17 00:00:00 2001 From: Chinasubbareddy Mallavarapu Date: Mon, 24 Feb 2020 16:53:46 -0600 Subject: [PATCH] [ceph-mon] Add mon hosts as per msgr2 protocol This is to update mon_host configuration to support both v1 and v2 of messenger. ex: mon_host = [v1:172.29.0.11:6790/0,v2:172.29.0.11:3300/0] Change-Id: I02785ea42c07d1aecbef2cf0c32dd6a1a236659f Signed-off-by: Pete Birley --- ceph-mon/templates/bin/mon/_start.sh.tpl | 9 ++++++--- ceph-mon/templates/configmap-etc.yaml | 5 ----- ceph-mon/templates/daemonset-mon.yaml | 3 +++ ceph-mon/templates/service-mon-discovery.yaml | 7 ++++++- ceph-mon/templates/service-mon.yaml | 7 ++++++- ceph-mon/values.yaml | 4 +++- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/ceph-mon/templates/bin/mon/_start.sh.tpl b/ceph-mon/templates/bin/mon/_start.sh.tpl index 7d9a0c407..6c6844374 100644 --- a/ceph-mon/templates/bin/mon/_start.sh.tpl +++ b/ceph-mon/templates/bin/mon/_start.sh.tpl @@ -12,11 +12,14 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" exit 1 else - ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/"ip"/{print $4":"port}' | paste -sd',') - if [[ ${ENDPOINT} == "" ]]; then + ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \ + -v version=v1 -v msgr_version=v2 \ + -v msgr2_port=${MON_PORT_V2} \ + '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',') + if [[ "${ENDPOINT}" == "" ]]; then /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true else - /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true fi fi diff --git a/ceph-mon/templates/configmap-etc.yaml b/ceph-mon/templates/configmap-etc.yaml index f0efd3f75..4f482bcac 100644 --- a/ceph-mon/templates/configmap-etc.yaml +++ b/ceph-mon/templates/configmap-etc.yaml @@ -26,11 +26,6 @@ limitations under the License. {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} {{- end -}} -{{- if empty .Values.conf.ceph.global.mon_addr -}} -{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}} -{{- end -}} - {{- if empty .Values.conf.ceph.global.fsid -}} {{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}} {{- end -}} diff --git a/ceph-mon/templates/daemonset-mon.yaml b/ceph-mon/templates/daemonset-mon.yaml index db273f926..41fcff999 100644 --- a/ceph-mon/templates/daemonset-mon.yaml +++ b/ceph-mon/templates/daemonset-mon.yaml @@ -141,6 +141,8 @@ spec: value: {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_kubectl_params" | indent 10 }} - name: MON_PORT value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: MON_PORT_V2 + value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: MON_IP valueFrom: fieldRef: @@ -162,6 +164,7 @@ spec: - /tmp/mon-stop.sh ports: - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - containerPort: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} livenessProbe: exec: command: diff --git a/ceph-mon/templates/service-mon-discovery.yaml b/ceph-mon/templates/service-mon-discovery.yaml index ffe2eacd0..ebff73f6e 100644 --- a/ceph-mon/templates/service-mon-discovery.yaml +++ b/ceph-mon/templates/service-mon-discovery.yaml @@ -31,9 +31,14 @@ metadata: service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" spec: ports: - - port: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: mon + port: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} protocol: TCP targetPort: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: mon-msgr2 + port: {{ tuple "ceph_mon" "discovery" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: {{ tuple "ceph_mon" "discovery" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} selector: {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} clusterIP: None diff --git a/ceph-mon/templates/service-mon.yaml b/ceph-mon/templates/service-mon.yaml index c69aa82c1..ef1165f66 100644 --- a/ceph-mon/templates/service-mon.yaml +++ b/ceph-mon/templates/service-mon.yaml @@ -23,9 +23,14 @@ metadata: name: {{ tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: ports: - - port: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: mon + port: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} protocol: TCP targetPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: mon-msgr2 + port: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} selector: {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} clusterIP: None diff --git a/ceph-mon/values.yaml b/ceph-mon/values.yaml index 8440b308f..f9581a7d5 100644 --- a/ceph-mon/values.yaml +++ b/ceph-mon/values.yaml @@ -46,7 +46,7 @@ labels: pod: mandatory_access_control: - type: apparmor + type: null ceph-mon: ceph-mon: runtime/default security_context: @@ -328,6 +328,8 @@ endpoints: port: mon: default: 6789 + mon_msgr2: + default: 3300 manifests: configmap_bin: true