Switch Ceph to IPs when DNS is down

Add helper scripts that are called by a POD to switch
Ceph from DNS to IPs. This POD will loop every 5 minutes
to catch cases where the DNS might be unavailable.

On a POD's Service start switch ceph.conf to using IPs rather
then DNS.

Change-Id: I402199f55792ca9f5f28e436ff44d4a6ac9b7cf9

ceph-client/templates/bin/mds/_start.sh.tpl

@@ -12,10 +12,18 @@ export LC_ALL=C
 : "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}"
 : "${MDS_KEYRING:=/var/lib/ceph/mds/${CLUSTER}-${MDS_NAME}/keyring}"
 : "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}"
+: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
 
-if [[ ! -e "/etc/ceph/${CLUSTER}.conf" ]]; then
-  echo "ERROR- /etc/ceph/${CLUSTER}.conf must exist; get it from your existing mon"
+if [[ ! -e ${CEPH_CONF}.template ]]; then
+  echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
   exit 1
+else
+  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',')
+  if [[ ${ENDPOINT} == "" ]]; then
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
+  else
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
+  fi
 fi
 
 # Check to see if we are a new MDS

ceph-client/templates/bin/mgr/_start.sh.tpl

@@ -4,10 +4,18 @@ set -ex
 : "${MGR_NAME:=$(uname -n)}"
 : "${MGR_KEYRING:=/var/lib/ceph/mgr/${CLUSTER}-${MGR_NAME}/keyring}"
 : "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}"
+: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
 
-if [[ ! -e /etc/ceph/${CLUSTER}.conf ]]; then
-    echo "ERROR- /etc/ceph/${CLUSTER}.conf must exist; get it from your existing mon"
-    exit 1
+if [[ ! -e ${CEPH_CONF}.template ]]; then
+  echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
+  exit 1
+else
+  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',')
+  if [[ ${ENDPOINT} == "" ]]; then
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
+  else
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
+  fi
 fi
 
 if [ ${CEPH_GET_ADMIN_KEY} -eq 1 ]; then

ceph-client/templates/bin/utils/_checkDNS.sh.tpl

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+{{/*
+Copyright 2018 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
+ENDPOINT=$1
+
+function check_mon_dns () {
+  GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF})
+
+  if [[ ${ENDPOINT} == "up" ]]; then
+    # If DNS is working, we simply restore the ${CEPH_CONF} file
+    if [[ ${GREP_CMD} == "" ]]; then
+      sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" > /dev/null 2>&1
+    fi
+  elif [[ ${ENDPOINT} != "" ]]; then
+    if [[ ${GREP_CMD} != "" ]]; then
+      # No DNS, write CEPH MONs IPs into ${CEPH_CONF}
+      sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1
+    else
+      echo "endpoints are already cached in ${CEPH_CONF}"
+      exit
+    fi
+  fi
+}
+
+check_mon_dns
+
+exit

ceph-client/templates/bin/utils/_checkDNS_start.sh.tpl

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+{{/*
+Copyright 2018 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -xe
+
+function check_mon_dns {
+  DNS_CHECK=$(getent hosts ceph-mon | head -n1)
+  PODS=$(kubectl get pods --namespace=${NAMESPACE} --selector=application=ceph --field-selector=status.phase=Running --output=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E 'ceph-mon|ceph-osd|ceph-mgr|ceph-mds')
+  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',')
+
+  if [[ ${PODS} == "" || ${ENDPOINT} == "" ]]; then
+    echo "Something went wrong, no PODS or ENDPOINTS are available!"
+  elif [[ ${DNS_CHECK} == "" ]]; then
+    for POD in ${PODS}; do
+      kubectl exec -t ${POD} --namespace=${NAMESPACE} -- \
+      sh -c -e "/tmp/utils-checkDNS.sh ${ENDPOINT}"
+    done
+  else
+    for POD in ${PODS}; do
+      kubectl exec -t ${POD} --namespace=${NAMESPACE} -- \
+      sh -c -e "/tmp/utils-checkDNS.sh up"
+    done
+  fi
+}
+
+function watch_mon_dns {
+  while [ true ]; do
+    echo "checking DNS health"
+    check_mon_dns || true
+    echo "sleep 300 sec"
+    sleep 300
+  done
+}
+
+watch_mon_dns
+
+exit

ceph-client/templates/configmap-bin.yaml

@@ -50,4 +50,8 @@ data:
 
   helm-tests.sh: |
 {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  utils-checkDNS.sh: |
+{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  utils-checkDNS_start.sh: |
+{{ tuple "bin/utils/_checkDNS_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}

ceph-client/templates/deployment-checkdns.yaml

@@ -0,0 +1,109 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License: is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.deployment_checkdns .Values.deployment.ceph }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "ceph-checkdns" }}
+{{ tuple $envAll "checkdns" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - endpoints
+      - pods/exec
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: ceph-checkdns
+  annotations:
+    configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
+  labels:
+{{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  selector:
+    matchLabels:
+{{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      affinity:
+{{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+      nodeSelector:
+        {{ .Values.labels.checkdns.node_selector_key }}: {{ .Values.labels.checkdns.node_selector_value }}
+      initContainers:
+{{ tuple $envAll "checkdns" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      hostNetwork: true
+      dnsPolicy: {{ .Values.pod.dns_policy }}
+      containers:
+        - name: ceph-checkdns
+{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.checkdns | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: CLUSTER
+              value: "ceph"
+            - name: K8S_HOST_NETWORK
+              value: "1"
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: MON_PORT
+              value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: KUBECTL_PARAM
+              value: {{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_kubectl_params" | indent 10 }}
+          command:
+            - /tmp/_start.sh
+          volumeMounts:
+            - name: ceph-client-bin
+              mountPath: /tmp/_start.sh
+              subPath: utils-checkDNS_start.sh
+              readOnly: true
+      volumes:
+        - name: ceph-client-bin
+          configMap:
+            name: ceph-client-bin
+            defaultMode: 0555
+{{- end }}

ceph-client/templates/deployment-mds.yaml

@@ -78,6 +78,13 @@ spec:
               value: "ceph"
             - name: CEPHFS_CREATE
               value: "1"
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: MON_PORT
+              value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
           ports:
             - containerPort: 6800
           livenessProbe:
@@ -94,8 +101,12 @@ spec:
               mountPath: /tmp/mds-start.sh
               subPath: mds-start.sh
               readOnly: true
+            - name: ceph-client-bin
+              mountPath: /tmp/utils-checkDNS.sh
+              subPath: utils-checkDNS.sh
+              readOnly: true
             - name: ceph-client-etc
-              mountPath: /etc/ceph/ceph.conf
+              mountPath: /etc/ceph/ceph.conf.template
               subPath: ceph.conf
               readOnly: true
             - name: ceph-client-admin-keyring

ceph-client/templates/deployment-mgr.yaml

@@ -79,6 +79,13 @@ spec:
           env:
             - name: CLUSTER
               value: "ceph"
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: MON_PORT
+              value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             {{- if .Values.ceph_mgr_enabled_modules }}
             - name: ENABLED_MODULES
               value: |-
@@ -126,10 +133,14 @@ spec:
               mountPath: /tmp/mgr-check.sh
               subPath: mgr-check.sh
               readOnly: true
+            - name: ceph-client-bin
+              mountPath: /tmp/utils-checkDNS.sh
+              subPath: utils-checkDNS.sh
+              readOnly: true
             - name: pod-etc-ceph
               mountPath: /etc/ceph
             - name: ceph-client-etc
-              mountPath: /etc/ceph/ceph.conf
+              mountPath: /etc/ceph/ceph.conf.template
               subPath: ceph.conf
               readOnly: true
             - name: ceph-client-admin-keyring

ceph-client/values.yaml

@@ -51,6 +51,9 @@ labels:
   mgr:
     node_selector_key: ceph-mgr
     node_selector_value: enabled
+  checkdns:
+    node_selector_key: ceph-mon
+    node_selector_value: enabled
 
 pod:
   dns_policy: "ClusterFirstWithHostNet"
@@ -79,6 +82,13 @@ pod:
       limits:
         memory: "50Mi"
         cpu: "500m"
+    checkdns:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
     jobs:
       bootstrap:
         limits:
@@ -312,6 +322,10 @@ dependencies:
       services:
         - endpoint: internal
           service: ceph_mon
+    checkdns:
+      services:
+        - endpoint: internal
+          service: ceph_mon
     namespace_client_key_cleaner:
       jobs: null
     namespace_client_key_generator:
@@ -430,6 +444,7 @@ manifests:
   configmap_etc: true
   deployment_mds: true
   deployment_mgr: true
+  deployment_checkdns: true
   job_bootstrap: false
   job_cephfs_client_key: true
   job_image_repo_sync: true

ceph-mon/templates/bin/mon/_start.sh.tpl

@@ -6,6 +6,19 @@ export LC_ALL=C
 : "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}"
 : "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}"
 : "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}"
+: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
+
+if [[ ! -e ${CEPH_CONF}.template ]]; then
+  echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
+  exit 1
+else
+  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',')
+  if [[ ${ENDPOINT} == "" ]]; then
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
+  else
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
+  fi
+fi
 
 if [[ -z "$CEPH_PUBLIC_NETWORK" ]]; then
   echo "ERROR- CEPH_PUBLIC_NETWORK must be defined as the name of the network for the OSDs"

ceph-mon/templates/bin/moncheck/_start.sh.tpl

@@ -1,6 +1,19 @@
 #!/bin/bash
 set -ex
 export LC_ALL=C
+: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
+
+if [[ ! -e ${CEPH_CONF}.template ]]; then
+  echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
+  exit 1
+else
+  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',')
+  if [[ ${ENDPOINT} == "" ]]; then
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
+  else
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
+  fi
+fi
 
 function watch_mon_health {
   while [ true ]; do

ceph-mon/templates/bin/utils/_checkDNS.sh.tpl

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+{{/*
+Copyright 2018 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
+ENDPOINT=$1
+
+function check_mon_dns () {
+  GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF})
+
+  if [[ ${ENDPOINT} == "up" ]]; then
+    # If DNS is working, we simply clean up the ${CEPH_CONF} file
+    if [[ ${GREP_CMD} == "" ]]; then
+      sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" > /dev/null 2>&1
+    fi
+  elif [[ ${ENDPOINT} != "" ]]; then
+    if [[ ${GREP_CMD} != "" ]]; then
+      # No DNS, write CEPH MONs IPs into ${CEPH_CONF}
+      sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1
+    else
+      echo "endpoints are already cached in ${CEPH_CONF}"
+      exit
+    fi
+  fi
+}
+
+check_mon_dns
+
+exit

ceph-mon/templates/configmap-bin.yaml

@@ -56,10 +56,10 @@ data:
 
   utils-checkPGs.py: |
 {{ tuple "bin/utils/_checkPGs.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-
   utils-checkPGs.sh: |
 {{ tuple "bin/utils/_checkPGs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   utils-checkObjectReplication.py: |
 {{ tuple "bin/utils/_checkObjectReplication.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-
+  utils-checkDNS.sh: |
+{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}

ceph-mon/templates/daemonset-mon.yaml

@@ -29,6 +29,7 @@ rules:
       - ""
     resources:
       - pods
+      - endpoints
     verbs:
       - get
       - list
@@ -186,8 +187,12 @@ spec:
               mountPath: /tmp/checkObjectReplication.py
               subPath: utils-checkObjectReplication.py
               readOnly: true
+            - name: ceph-mon-bin
+              mountPath: /tmp/utils-checkDNS.sh
+              subPath: utils-checkDNS.sh
+              readOnly: true
             - name: ceph-mon-etc
-              mountPath: /etc/ceph/ceph.conf
+              mountPath: /etc/ceph/ceph.conf.template
               subPath: ceph.conf
               readOnly: true
             - name: ceph-client-admin-keyring

ceph-mon/templates/deployment-moncheck.yaml

@@ -59,6 +59,8 @@ spec:
                 fieldRef:
                   apiVersion: v1
                   fieldPath: metadata.namespace
+            - name: MON_PORT
+              value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
           command:
             - /tmp/moncheck-start.sh
           ports:
@@ -72,8 +74,12 @@ spec:
               mountPath: /tmp/moncheck-reap-zombies.py
               subPath: moncheck-reap-zombies.py
               readOnly: true
+            - name: ceph-mon-bin
+              mountPath: /tmp/utils-checkDNS.sh
+              subPath: utils-checkDNS.sh
+              readOnly: true
             - name: ceph-mon-etc
-              mountPath: /etc/ceph/ceph.conf
+              mountPath: /etc/ceph/ceph.conf.template
               subPath: ceph.conf
               readOnly: true
             - name: ceph-client-admin-keyring

ceph-osd/templates/bin/osd/_init.sh.tpl

@@ -21,6 +21,20 @@ set -ex
 : "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}"
 : "${OSD_JOURNAL_UUID:=$(uuidgen)}"
 : "${OSD_FORCE_ZAP:=1}"
+: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
+
+if [[ ! -e ${CEPH_CONF}.template ]]; then
+  echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
+  exit 1
+else
+  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',')
+  if [[ ${ENDPOINT} == "" ]]; then
+    # No endpoints are available, just copy ceph.conf as-is
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
+  else
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
+  fi
+fi
 
 if [ "x${STORAGE_TYPE%-*}" == "xdirectory" ]; then
   export OSD_DEVICE="/var/lib/ceph/osd"

ceph-osd/templates/bin/osd/_start.sh.tpl

@@ -17,7 +17,19 @@ limitations under the License.
 */}}
 
 set -ex
+: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
+
+if [[ ! -e ${CEPH_CONF}.template ]]; then
+  echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
+  exit 1
+else
+  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',')
+  if [[ ${ENDPOINT} == "" ]]; then
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
+  else
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
+  fi
+fi
 
 echo "LAUNCHING OSD: in ${STORAGE_TYPE%-*}:${STORAGE_TYPE#*-} mode"
-
 exec "/tmp/osd-${STORAGE_TYPE%-*}.sh"

ceph-osd/templates/bin/utils/_checkDNS.sh.tpl

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+{{/*
+Copyright 2018 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
+ENDPOINT=$1
+
+function check_mon_dns () {
+  GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF})
+
+  if [[ ${ENDPOINT} == "up" ]]; then
+    # If DNS is working, we simply clean up the ${CEPH_CONF} file
+    if [[ ${GREP_CMD} == "" ]]; then
+      sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" > /dev/null 2>&1
+    fi
+  elif [[ ${ENDPOINT} != "" ]]; then
+    if [[ ${GREP_CMD} != "" ]]; then
+      # No DNS, write CEPH MONs IPs into ${CEPH_CONF}
+      sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1
+    else
+      echo "endpoints are already cached in ${CEPH_CONF}"
+      exit
+    fi
+  fi
+}
+
+check_mon_dns
+
+exit

ceph-osd/templates/configmap-bin.yaml

@@ -42,4 +42,6 @@ data:
 {{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   helm-tests.sh: |
 {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  utils-checkDNS.sh: |
+{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}

ceph-osd/templates/daemonset-osd.yaml

@@ -64,6 +64,13 @@ spec:
           #   value: directory
             - name: CLUSTER
               value: "ceph"
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: MON_PORT
+              value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
           volumeMounts:
             - name: ceph-osd-bin
               mountPath: /tmp/init-dirs.sh
@@ -125,6 +132,13 @@ spec:
               value: "ceph"
             - name: CEPH_GET_ADMIN_KEY
               value: "1"
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: MON_PORT
+              value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
           command:
             - /tmp/osd-init.sh
           volumeMounts:
@@ -133,7 +147,7 @@ spec:
               subPath: osd-init.sh
               readOnly: true
             - name: ceph-osd-etc
-              mountPath: /etc/ceph/ceph.conf
+              mountPath: /etc/ceph/ceph.conf.template
               subPath: ceph.conf
               readOnly: true
             - name: ceph-bootstrap-osd-keyring
@@ -185,6 +199,13 @@ spec:
               value: {{ .Values.conf.storage.failure_domain_name | default "false" | quote }}
             - name: CRUSH_FAILURE_DOMAIN_BY_HOSTNAME
               value: {{ .Values.conf.storage.failure_domain_by_hostname | default "false" | quote }}
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: MON_PORT
+              value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
           command:
             - /tmp/osd-start.sh
           lifecycle:
@@ -227,8 +248,12 @@ spec:
               mountPath: /tmp/osd-stop.sh
               subPath: osd-stop.sh
               readOnly: true
+            - name: ceph-osd-bin
+              mountPath: /tmp/utils-checkDNS.sh
+              subPath: utils-checkDNS.sh
+              readOnly: true
             - name: ceph-osd-etc
-              mountPath: /etc/ceph/ceph.conf
+              mountPath: /etc/ceph/ceph.conf.template
               subPath: ceph.conf
               readOnly: true
             - name: ceph-bootstrap-osd-keyring