[ceph-client] Update ceph_mon config to ips from fqdn

As ceph clients expect the ceph_mon config as shown below for Ceph
Nautilus and later releases, this change updates the ceph-client-etc
configmap to reflect the correct mon endpoint specification.

mon_host = [v1:172.29.1.139:6789/0,v2:172.29.1.139:3300/0],
[v1:172.29.1.140:6789/0,v2:172.29.1.140:3300/0],
[v1:172.29.1.145:6789/0,v2:172.29.1.145:3300/0]

Change-Id: Ic3a1cb7e56317a5a5da46f3bf97ee23ece36c99c

ceph-client/Chart.yaml

@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph Client
 name: ceph-client
-version: 0.1.26
+version: 0.1.27
 home: https://github.com/ceph/ceph-client
 ...

ceph-client/templates/bin/_namespace-client-ceph-config-manager.sh.tpl

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+{{- $envAll := . }}
+
+ENDPOINTS=$(kubectl get endpoints ceph-mon-discovery -n ${DEPLOYMENT_NAMESPACE} -o json)
+MON_IPS=$(jq -r '.subsets[0].addresses[].ip?' <<< ${ENDPOINTS})
+V1_PORT=$(jq '.subsets[0].ports[] | select(.name == "mon") | .port' <<< ${ENDPOINTS})
+V2_PORT=$(jq '.subsets[0].ports[] | select(.name == "mon-msgr2") | .port' <<< ${ENDPOINTS})
+ENDPOINT=$(for ip in $MON_IPS; do printf '[v1:%s:%s/0,v2:%s:%s/0]\n' ${ip} ${V1_PORT} ${ip} ${V2_PORT}; done | paste -sd',')
+
+if [[ -z "${V1_PORT}" ]] || [[ -z "${V2_PORT}" ]] || [[ -z "${ENDPOINT}" ]]; then
+  echo "Ceph Mon endpoint is empty"
+  exit 1
+else
+  echo ${ENDPOINT}
+fi
+
+kubectl get cm ${CEPH_CONF_ETC} -n  ${DEPLOYMENT_NAMESPACE}  -o yaml | \
+  sed "s#mon_host.*#mon_host = ${ENDPOINT}#g" | \
+  kubectl apply -f -
+
+kubectl get cm ${CEPH_CONF_ETC} -n  ${DEPLOYMENT_NAMESPACE}  -o yaml

ceph-client/templates/configmap-bin.yaml

@@ -32,6 +32,8 @@ data:
 
   init-dirs.sh: |
 {{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  namespace-client-ceph-config-manager.sh: |
+{{ tuple "bin/_namespace-client-ceph-config-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 
   pool-init.sh: |
 {{ tuple "bin/pool/_init.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}

ceph-client/templates/job-ns-client-ceph-config.yaml

@@ -0,0 +1,137 @@
+---
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.job_ns_client_ceph_config .Values.manifests.configmap_etc }}
+{{- $envAll := . }}
+
+{{- $randStringSuffix := randAlphaNum 5 | lower }}
+
+{{- $serviceAccountName := print $envAll.Release.Name "-ceph-ns-ceph-config-update" }}
+{{ tuple $envAll "namespace_client_ceph_config_update" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - create
+      - update
+      - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
+  namespace: {{ $envAll.Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
+  namespace: {{ $envAll.Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ $serviceAccountName }}
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "ceph" "client-ceph-config-update" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+      annotations:
+{{ dict "envAll" $envAll "podName" $serviceAccountName "containerNames" (list "ceph-client-config-update" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
+    spec:
+{{ dict "envAll" $envAll "application" "client_ceph_config_update" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: {{ $envAll.Values.jobs.client_ceph_config_update.restartPolicy | quote }}
+      nodeSelector:
+        {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }}
+      initContainers:
+{{ tuple $envAll "namespace_client_ceph_config_update" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: ceph-client-config-update
+{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "client_ceph_config_update" "container" "ceph_storage_keys_update" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          env:
+            - name: CEPH_CONF_ETC
+              value: "ceph-client-etc"
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: MON_PORT
+              value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: MON_PORT_V2
+              value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+
+          command:
+            - /tmp/namespace-client-ceph-config-manager.sh
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: pod-etc-ceph
+              mountPath: /etc/ceph
+            - name: ceph-client-bin-clients
+              mountPath: /tmp/namespace-client-ceph-config-manager.sh
+              subPath: namespace-client-ceph-config-manager.sh
+              readOnly: true
+      volumes:
+        - name: pod-tmp
+          emptyDir: {}
+        - name: pod-etc-ceph
+          emptyDir: {}
+        - name: ceph-client-bin-clients
+          configMap:
+            name: ceph-client-bin
+            defaultMode: 0555
+{{- end }}
+...

ceph-client/values.yaml

@@ -253,6 +253,8 @@ jobs:
     startingDeadlineSecs: 60
   rbd_pool:
     restartPolicy: OnFailure
+  client_ceph_config_update:
+    restartPolicy: OnFailure
 
 conf:
   features:
@@ -627,6 +629,7 @@ manifests:
   deployment_mgr: true
   deployment_checkdns: true
   job_bootstrap: false
+  job_ns_client_ceph_config: true
   job_cephfs_client_key: true
   job_image_repo_sync: true
   job_rbd_pool: true

releasenotes/notes/ceph-client.yaml

@@ -27,4 +27,5 @@ ceph-client:
   - 0.1.24 Performance optimizations for the ceph-rbd-pool job
   - 0.1.25 Update htk requirements
   - 0.1.26 Fix ceph-rbd-pool deletion race
+  - 0.1.27 Update ceph_mon config to ips from fqdn
 ...