diff --git a/ceph-client/templates/bin/mds/_start.sh.tpl b/ceph-client/templates/bin/mds/_start.sh.tpl index 50128c76c..357343ff8 100644 --- a/ceph-client/templates/bin/mds/_start.sh.tpl +++ b/ceph-client/templates/bin/mds/_start.sh.tpl @@ -12,10 +12,18 @@ export LC_ALL=C : "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" : "${MDS_KEYRING:=/var/lib/ceph/mds/${CLUSTER}-${MDS_NAME}/keyring}" : "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}" +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" -if [[ ! -e "/etc/ceph/${CLUSTER}.conf" ]]; then - echo "ERROR- /etc/ceph/${CLUSTER}.conf must exist; get it from your existing mon" +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" exit 1 +else + ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',') + if [[ ${ENDPOINT} == "" ]]; then + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true + else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true + fi fi # Check to see if we are a new MDS diff --git a/ceph-client/templates/bin/mgr/_start.sh.tpl b/ceph-client/templates/bin/mgr/_start.sh.tpl index be622ac31..432436d10 100644 --- a/ceph-client/templates/bin/mgr/_start.sh.tpl +++ b/ceph-client/templates/bin/mgr/_start.sh.tpl @@ -4,10 +4,18 @@ set -ex : "${MGR_NAME:=$(uname -n)}" : "${MGR_KEYRING:=/var/lib/ceph/mgr/${CLUSTER}-${MGR_NAME}/keyring}" : "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" -if [[ ! -e /etc/ceph/${CLUSTER}.conf ]]; then - echo "ERROR- /etc/ceph/${CLUSTER}.conf must exist; get it from your existing mon" - exit 1 +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" + exit 1 +else + ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',') + if [[ ${ENDPOINT} == "" ]]; then + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true + else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true + fi fi if [ ${CEPH_GET_ADMIN_KEY} -eq 1 ]; then diff --git a/ceph-client/templates/bin/utils/_checkDNS.sh.tpl b/ceph-client/templates/bin/utils/_checkDNS.sh.tpl new file mode 100644 index 000000000..482dcaf67 --- /dev/null +++ b/ceph-client/templates/bin/utils/_checkDNS.sh.tpl @@ -0,0 +1,43 @@ +#!/bin/bash + +{{/* +Copyright 2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" +ENDPOINT=$1 + +function check_mon_dns () { + GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF}) + + if [[ ${ENDPOINT} == "up" ]]; then + # If DNS is working, we simply restore the ${CEPH_CONF} file + if [[ ${GREP_CMD} == "" ]]; then + sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" > /dev/null 2>&1 + fi + elif [[ ${ENDPOINT} != "" ]]; then + if [[ ${GREP_CMD} != "" ]]; then + # No DNS, write CEPH MONs IPs into ${CEPH_CONF} + sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1 + else + echo "endpoints are already cached in ${CEPH_CONF}" + exit + fi + fi +} + +check_mon_dns + +exit diff --git a/ceph-client/templates/bin/utils/_checkDNS_start.sh.tpl b/ceph-client/templates/bin/utils/_checkDNS_start.sh.tpl new file mode 100644 index 000000000..c91a2b9ff --- /dev/null +++ b/ceph-client/templates/bin/utils/_checkDNS_start.sh.tpl @@ -0,0 +1,52 @@ +#!/bin/bash + +{{/* +Copyright 2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -xe + +function check_mon_dns { + DNS_CHECK=$(getent hosts ceph-mon | head -n1) + PODS=$(kubectl get pods --namespace=${NAMESPACE} --selector=application=ceph --field-selector=status.phase=Running --output=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E 'ceph-mon|ceph-osd|ceph-mgr|ceph-mds') + ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',') + + if [[ ${PODS} == "" || ${ENDPOINT} == "" ]]; then + echo "Something went wrong, no PODS or ENDPOINTS are available!" + elif [[ ${DNS_CHECK} == "" ]]; then + for POD in ${PODS}; do + kubectl exec -t ${POD} --namespace=${NAMESPACE} -- \ + sh -c -e "/tmp/utils-checkDNS.sh ${ENDPOINT}" + done + else + for POD in ${PODS}; do + kubectl exec -t ${POD} --namespace=${NAMESPACE} -- \ + sh -c -e "/tmp/utils-checkDNS.sh up" + done + fi +} + +function watch_mon_dns { + while [ true ]; do + echo "checking DNS health" + check_mon_dns || true + echo "sleep 300 sec" + sleep 300 + done +} + +watch_mon_dns + +exit diff --git a/ceph-client/templates/configmap-bin.yaml b/ceph-client/templates/configmap-bin.yaml index d4aef1a3b..39a9b0fa8 100644 --- a/ceph-client/templates/configmap-bin.yaml +++ b/ceph-client/templates/configmap-bin.yaml @@ -50,4 +50,8 @@ data: helm-tests.sh: | {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-checkDNS.sh: | +{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-checkDNS_start.sh: | +{{ tuple "bin/utils/_checkDNS_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/ceph-client/templates/deployment-checkdns.yaml b/ceph-client/templates/deployment-checkdns.yaml new file mode 100644 index 000000000..c39448174 --- /dev/null +++ b/ceph-client/templates/deployment-checkdns.yaml @@ -0,0 +1,109 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License: is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.deployment_checkdns .Values.deployment.ceph }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-checkdns" }} +{{ tuple $envAll "checkdns" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - pods + - endpoints + - pods/exec + verbs: + - get + - list + - watch + - create +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ceph-checkdns + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + labels: +{{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.checkdns.node_selector_key }}: {{ .Values.labels.checkdns.node_selector_value }} + initContainers: +{{ tuple $envAll "checkdns" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + hostNetwork: true + dnsPolicy: {{ .Values.pod.dns_policy }} + containers: + - name: ceph-checkdns +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.checkdns | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: CLUSTER + value: "ceph" + - name: K8S_HOST_NETWORK + value: "1" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KUBECTL_PARAM + value: {{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_kubectl_params" | indent 10 }} + command: + - /tmp/_start.sh + volumeMounts: + - name: ceph-client-bin + mountPath: /tmp/_start.sh + subPath: utils-checkDNS_start.sh + readOnly: true + volumes: + - name: ceph-client-bin + configMap: + name: ceph-client-bin + defaultMode: 0555 +{{- end }} diff --git a/ceph-client/templates/deployment-mds.yaml b/ceph-client/templates/deployment-mds.yaml index 300e958df..b35bd3ea5 100644 --- a/ceph-client/templates/deployment-mds.yaml +++ b/ceph-client/templates/deployment-mds.yaml @@ -78,6 +78,13 @@ spec: value: "ceph" - name: CEPHFS_CREATE value: "1" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} ports: - containerPort: 6800 livenessProbe: @@ -94,8 +101,12 @@ spec: mountPath: /tmp/mds-start.sh subPath: mds-start.sh readOnly: true + - name: ceph-client-bin + mountPath: /tmp/utils-checkDNS.sh + subPath: utils-checkDNS.sh + readOnly: true - name: ceph-client-etc - mountPath: /etc/ceph/ceph.conf + mountPath: /etc/ceph/ceph.conf.template subPath: ceph.conf readOnly: true - name: ceph-client-admin-keyring diff --git a/ceph-client/templates/deployment-mgr.yaml b/ceph-client/templates/deployment-mgr.yaml index ef853bfd1..034393cba 100644 --- a/ceph-client/templates/deployment-mgr.yaml +++ b/ceph-client/templates/deployment-mgr.yaml @@ -79,6 +79,13 @@ spec: env: - name: CLUSTER value: "ceph" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} {{- if .Values.ceph_mgr_enabled_modules }} - name: ENABLED_MODULES value: |- @@ -126,10 +133,14 @@ spec: mountPath: /tmp/mgr-check.sh subPath: mgr-check.sh readOnly: true + - name: ceph-client-bin + mountPath: /tmp/utils-checkDNS.sh + subPath: utils-checkDNS.sh + readOnly: true - name: pod-etc-ceph mountPath: /etc/ceph - name: ceph-client-etc - mountPath: /etc/ceph/ceph.conf + mountPath: /etc/ceph/ceph.conf.template subPath: ceph.conf readOnly: true - name: ceph-client-admin-keyring diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml index ea54f9c15..3a79d33e5 100644 --- a/ceph-client/values.yaml +++ b/ceph-client/values.yaml @@ -51,6 +51,9 @@ labels: mgr: node_selector_key: ceph-mgr node_selector_value: enabled + checkdns: + node_selector_key: ceph-mon + node_selector_value: enabled pod: dns_policy: "ClusterFirstWithHostNet" @@ -79,6 +82,13 @@ pod: limits: memory: "50Mi" cpu: "500m" + checkdns: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" jobs: bootstrap: limits: @@ -312,6 +322,10 @@ dependencies: services: - endpoint: internal service: ceph_mon + checkdns: + services: + - endpoint: internal + service: ceph_mon namespace_client_key_cleaner: jobs: null namespace_client_key_generator: @@ -430,6 +444,7 @@ manifests: configmap_etc: true deployment_mds: true deployment_mgr: true + deployment_checkdns: true job_bootstrap: false job_cephfs_client_key: true job_image_repo_sync: true diff --git a/ceph-mon/templates/bin/mon/_start.sh.tpl b/ceph-mon/templates/bin/mon/_start.sh.tpl index ad2acc2c4..1340d6bc2 100644 --- a/ceph-mon/templates/bin/mon/_start.sh.tpl +++ b/ceph-mon/templates/bin/mon/_start.sh.tpl @@ -6,6 +6,19 @@ export LC_ALL=C : "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" : "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}" : "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}" +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" + +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" + exit 1 +else + ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',') + if [[ ${ENDPOINT} == "" ]]; then + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true + else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true + fi +fi if [[ -z "$CEPH_PUBLIC_NETWORK" ]]; then echo "ERROR- CEPH_PUBLIC_NETWORK must be defined as the name of the network for the OSDs" diff --git a/ceph-mon/templates/bin/moncheck/_start.sh.tpl b/ceph-mon/templates/bin/moncheck/_start.sh.tpl index 25d4159f7..46510ba2f 100644 --- a/ceph-mon/templates/bin/moncheck/_start.sh.tpl +++ b/ceph-mon/templates/bin/moncheck/_start.sh.tpl @@ -1,6 +1,19 @@ #!/bin/bash set -ex export LC_ALL=C +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" + +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" + exit 1 +else + ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',') + if [[ ${ENDPOINT} == "" ]]; then + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true + else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true + fi +fi function watch_mon_health { while [ true ]; do diff --git a/ceph-mon/templates/bin/utils/_checkDNS.sh.tpl b/ceph-mon/templates/bin/utils/_checkDNS.sh.tpl new file mode 100644 index 000000000..dd99728d8 --- /dev/null +++ b/ceph-mon/templates/bin/utils/_checkDNS.sh.tpl @@ -0,0 +1,43 @@ +#!/bin/bash + +{{/* +Copyright 2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" +ENDPOINT=$1 + +function check_mon_dns () { + GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF}) + + if [[ ${ENDPOINT} == "up" ]]; then + # If DNS is working, we simply clean up the ${CEPH_CONF} file + if [[ ${GREP_CMD} == "" ]]; then + sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" > /dev/null 2>&1 + fi + elif [[ ${ENDPOINT} != "" ]]; then + if [[ ${GREP_CMD} != "" ]]; then + # No DNS, write CEPH MONs IPs into ${CEPH_CONF} + sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1 + else + echo "endpoints are already cached in ${CEPH_CONF}" + exit + fi + fi +} + +check_mon_dns + +exit diff --git a/ceph-mon/templates/configmap-bin.yaml b/ceph-mon/templates/configmap-bin.yaml index 5203f25ae..833089c42 100644 --- a/ceph-mon/templates/configmap-bin.yaml +++ b/ceph-mon/templates/configmap-bin.yaml @@ -56,10 +56,10 @@ data: utils-checkPGs.py: | {{ tuple "bin/utils/_checkPGs.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - utils-checkPGs.sh: | {{ tuple "bin/utils/_checkPGs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} utils-checkObjectReplication.py: | {{ tuple "bin/utils/_checkObjectReplication.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - + utils-checkDNS.sh: | +{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/ceph-mon/templates/daemonset-mon.yaml b/ceph-mon/templates/daemonset-mon.yaml index 73feeb507..60a1a271c 100644 --- a/ceph-mon/templates/daemonset-mon.yaml +++ b/ceph-mon/templates/daemonset-mon.yaml @@ -29,6 +29,7 @@ rules: - "" resources: - pods + - endpoints verbs: - get - list @@ -186,8 +187,12 @@ spec: mountPath: /tmp/checkObjectReplication.py subPath: utils-checkObjectReplication.py readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/utils-checkDNS.sh + subPath: utils-checkDNS.sh + readOnly: true - name: ceph-mon-etc - mountPath: /etc/ceph/ceph.conf + mountPath: /etc/ceph/ceph.conf.template subPath: ceph.conf readOnly: true - name: ceph-client-admin-keyring diff --git a/ceph-mon/templates/deployment-moncheck.yaml b/ceph-mon/templates/deployment-moncheck.yaml index 8ad19aa88..701c9316e 100644 --- a/ceph-mon/templates/deployment-moncheck.yaml +++ b/ceph-mon/templates/deployment-moncheck.yaml @@ -59,6 +59,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} command: - /tmp/moncheck-start.sh ports: @@ -72,8 +74,12 @@ spec: mountPath: /tmp/moncheck-reap-zombies.py subPath: moncheck-reap-zombies.py readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/utils-checkDNS.sh + subPath: utils-checkDNS.sh + readOnly: true - name: ceph-mon-etc - mountPath: /etc/ceph/ceph.conf + mountPath: /etc/ceph/ceph.conf.template subPath: ceph.conf readOnly: true - name: ceph-client-admin-keyring diff --git a/ceph-osd/templates/bin/osd/_init.sh.tpl b/ceph-osd/templates/bin/osd/_init.sh.tpl index 344227071..a5cabfa49 100644 --- a/ceph-osd/templates/bin/osd/_init.sh.tpl +++ b/ceph-osd/templates/bin/osd/_init.sh.tpl @@ -21,6 +21,20 @@ set -ex : "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}" : "${OSD_JOURNAL_UUID:=$(uuidgen)}" : "${OSD_FORCE_ZAP:=1}" +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" + +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" + exit 1 +else + ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',') + if [[ ${ENDPOINT} == "" ]]; then + # No endpoints are available, just copy ceph.conf as-is + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true + else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true + fi +fi if [ "x${STORAGE_TYPE%-*}" == "xdirectory" ]; then export OSD_DEVICE="/var/lib/ceph/osd" diff --git a/ceph-osd/templates/bin/osd/_start.sh.tpl b/ceph-osd/templates/bin/osd/_start.sh.tpl index 5c3b2f748..d71e44574 100644 --- a/ceph-osd/templates/bin/osd/_start.sh.tpl +++ b/ceph-osd/templates/bin/osd/_start.sh.tpl @@ -17,7 +17,19 @@ limitations under the License. */}} set -ex +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" + +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" + exit 1 +else + ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/ip/{print $4":"port}' | paste -sd',') + if [[ ${ENDPOINT} == "" ]]; then + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true + else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true + fi +fi echo "LAUNCHING OSD: in ${STORAGE_TYPE%-*}:${STORAGE_TYPE#*-} mode" - exec "/tmp/osd-${STORAGE_TYPE%-*}.sh" diff --git a/ceph-osd/templates/bin/utils/_checkDNS.sh.tpl b/ceph-osd/templates/bin/utils/_checkDNS.sh.tpl new file mode 100644 index 000000000..dd99728d8 --- /dev/null +++ b/ceph-osd/templates/bin/utils/_checkDNS.sh.tpl @@ -0,0 +1,43 @@ +#!/bin/bash + +{{/* +Copyright 2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" +ENDPOINT=$1 + +function check_mon_dns () { + GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF}) + + if [[ ${ENDPOINT} == "up" ]]; then + # If DNS is working, we simply clean up the ${CEPH_CONF} file + if [[ ${GREP_CMD} == "" ]]; then + sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" > /dev/null 2>&1 + fi + elif [[ ${ENDPOINT} != "" ]]; then + if [[ ${GREP_CMD} != "" ]]; then + # No DNS, write CEPH MONs IPs into ${CEPH_CONF} + sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1 + else + echo "endpoints are already cached in ${CEPH_CONF}" + exit + fi + fi +} + +check_mon_dns + +exit diff --git a/ceph-osd/templates/configmap-bin.yaml b/ceph-osd/templates/configmap-bin.yaml index a9f5ce115..d9b277dcf 100644 --- a/ceph-osd/templates/configmap-bin.yaml +++ b/ceph-osd/templates/configmap-bin.yaml @@ -42,4 +42,6 @@ data: {{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} helm-tests.sh: | {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-checkDNS.sh: | +{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/ceph-osd/templates/daemonset-osd.yaml b/ceph-osd/templates/daemonset-osd.yaml index 2169d45fe..01d20112d 100644 --- a/ceph-osd/templates/daemonset-osd.yaml +++ b/ceph-osd/templates/daemonset-osd.yaml @@ -64,6 +64,13 @@ spec: # value: directory - name: CLUSTER value: "ceph" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} volumeMounts: - name: ceph-osd-bin mountPath: /tmp/init-dirs.sh @@ -125,6 +132,13 @@ spec: value: "ceph" - name: CEPH_GET_ADMIN_KEY value: "1" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} command: - /tmp/osd-init.sh volumeMounts: @@ -133,7 +147,7 @@ spec: subPath: osd-init.sh readOnly: true - name: ceph-osd-etc - mountPath: /etc/ceph/ceph.conf + mountPath: /etc/ceph/ceph.conf.template subPath: ceph.conf readOnly: true - name: ceph-bootstrap-osd-keyring @@ -185,6 +199,13 @@ spec: value: {{ .Values.conf.storage.failure_domain_name | default "false" | quote }} - name: CRUSH_FAILURE_DOMAIN_BY_HOSTNAME value: {{ .Values.conf.storage.failure_domain_by_hostname | default "false" | quote }} + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} command: - /tmp/osd-start.sh lifecycle: @@ -227,8 +248,12 @@ spec: mountPath: /tmp/osd-stop.sh subPath: osd-stop.sh readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/utils-checkDNS.sh + subPath: utils-checkDNS.sh + readOnly: true - name: ceph-osd-etc - mountPath: /etc/ceph/ceph.conf + mountPath: /etc/ceph/ceph.conf.template subPath: ceph.conf readOnly: true - name: ceph-bootstrap-osd-keyring