From fa608d076c86b44086cd18e88389d2df38433e17 Mon Sep 17 00:00:00 2001 From: Chinasubbareddy Mallavarapu Date: Fri, 1 Oct 2021 15:02:18 +0000 Subject: [PATCH] [ceph-client] Update ceph_mon config to ips from fqdn As ceph clients expect the ceph_mon config as shown below for Ceph Nautilus and later releases, this change updates the ceph-client-etc configmap to reflect the correct mon endpoint specification. mon_host = [v1:172.29.1.139:6789/0,v2:172.29.1.139:3300/0], [v1:172.29.1.140:6789/0,v2:172.29.1.140:3300/0], [v1:172.29.1.145:6789/0,v2:172.29.1.145:3300/0] Change-Id: Ic3a1cb7e56317a5a5da46f3bf97ee23ece36c99c --- ceph-client/Chart.yaml | 2 +- ...amespace-client-ceph-config-manager.sh.tpl | 37 +++++ ceph-client/templates/configmap-bin.yaml | 2 + .../templates/job-ns-client-ceph-config.yaml | 137 ++++++++++++++++++ ceph-client/values.yaml | 3 + releasenotes/notes/ceph-client.yaml | 1 + 6 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 ceph-client/templates/bin/_namespace-client-ceph-config-manager.sh.tpl create mode 100644 ceph-client/templates/job-ns-client-ceph-config.yaml diff --git a/ceph-client/Chart.yaml b/ceph-client/Chart.yaml index 0170fb92e..797d36f3e 100644 --- a/ceph-client/Chart.yaml +++ b/ceph-client/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Client name: ceph-client -version: 0.1.26 +version: 0.1.27 home: https://github.com/ceph/ceph-client ... diff --git a/ceph-client/templates/bin/_namespace-client-ceph-config-manager.sh.tpl b/ceph-client/templates/bin/_namespace-client-ceph-config-manager.sh.tpl new file mode 100644 index 000000000..118dacc73 --- /dev/null +++ b/ceph-client/templates/bin/_namespace-client-ceph-config-manager.sh.tpl @@ -0,0 +1,37 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{- $envAll := . }} + +ENDPOINTS=$(kubectl get endpoints ceph-mon-discovery -n ${DEPLOYMENT_NAMESPACE} -o json) +MON_IPS=$(jq -r '.subsets[0].addresses[].ip?' <<< ${ENDPOINTS}) +V1_PORT=$(jq '.subsets[0].ports[] | select(.name == "mon") | .port' <<< ${ENDPOINTS}) +V2_PORT=$(jq '.subsets[0].ports[] | select(.name == "mon-msgr2") | .port' <<< ${ENDPOINTS}) +ENDPOINT=$(for ip in $MON_IPS; do printf '[v1:%s:%s/0,v2:%s:%s/0]\n' ${ip} ${V1_PORT} ${ip} ${V2_PORT}; done | paste -sd',') + +if [[ -z "${V1_PORT}" ]] || [[ -z "${V2_PORT}" ]] || [[ -z "${ENDPOINT}" ]]; then + echo "Ceph Mon endpoint is empty" + exit 1 +else + echo ${ENDPOINT} +fi + +kubectl get cm ${CEPH_CONF_ETC} -n ${DEPLOYMENT_NAMESPACE} -o yaml | \ + sed "s#mon_host.*#mon_host = ${ENDPOINT}#g" | \ + kubectl apply -f - + +kubectl get cm ${CEPH_CONF_ETC} -n ${DEPLOYMENT_NAMESPACE} -o yaml diff --git a/ceph-client/templates/configmap-bin.yaml b/ceph-client/templates/configmap-bin.yaml index cbb44a6e3..6caba70b3 100644 --- a/ceph-client/templates/configmap-bin.yaml +++ b/ceph-client/templates/configmap-bin.yaml @@ -32,6 +32,8 @@ data: init-dirs.sh: | {{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + namespace-client-ceph-config-manager.sh: | +{{ tuple "bin/_namespace-client-ceph-config-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} pool-init.sh: | {{ tuple "bin/pool/_init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/ceph-client/templates/job-ns-client-ceph-config.yaml b/ceph-client/templates/job-ns-client-ceph-config.yaml new file mode 100644 index 000000000..c5948f792 --- /dev/null +++ b/ceph-client/templates/job-ns-client-ceph-config.yaml @@ -0,0 +1,137 @@ +--- +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_ns_client_ceph_config .Values.manifests.configmap_etc }} +{{- $envAll := . }} + +{{- $randStringSuffix := randAlphaNum 5 | lower }} + +{{- $serviceAccountName := print $envAll.Release.Name "-ceph-ns-ceph-config-update" }} +{{ tuple $envAll "namespace_client_ceph_config_update" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }} + namespace: {{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $serviceAccountName }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "client-ceph-config-update" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" $serviceAccountName "containerNames" (list "ceph-client-config-update" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "client_ceph_config_update" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: {{ $envAll.Values.jobs.client_ceph_config_update.restartPolicy | quote }} + nodeSelector: + {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "namespace_client_ceph_config_update" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-client-config-update +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "client_ceph_config_update" "container" "ceph_storage_keys_update" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: CEPH_CONF_ETC + value: "ceph-client-etc" + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: MON_PORT_V2 + value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + + command: + - /tmp/namespace-client-ceph-config-manager.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-client-bin-clients + mountPath: /tmp/namespace-client-ceph-config-manager.sh + subPath: namespace-client-ceph-config-manager.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-client-bin-clients + configMap: + name: ceph-client-bin + defaultMode: 0555 +{{- end }} +... diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml index 92c316329..38bc5e204 100644 --- a/ceph-client/values.yaml +++ b/ceph-client/values.yaml @@ -253,6 +253,8 @@ jobs: startingDeadlineSecs: 60 rbd_pool: restartPolicy: OnFailure + client_ceph_config_update: + restartPolicy: OnFailure conf: features: @@ -627,6 +629,7 @@ manifests: deployment_mgr: true deployment_checkdns: true job_bootstrap: false + job_ns_client_ceph_config: true job_cephfs_client_key: true job_image_repo_sync: true job_rbd_pool: true diff --git a/releasenotes/notes/ceph-client.yaml b/releasenotes/notes/ceph-client.yaml index 1e6e2e6d6..ab99e0080 100644 --- a/releasenotes/notes/ceph-client.yaml +++ b/releasenotes/notes/ceph-client.yaml @@ -27,4 +27,5 @@ ceph-client: - 0.1.24 Performance optimizations for the ceph-rbd-pool job - 0.1.25 Update htk requirements - 0.1.26 Fix ceph-rbd-pool deletion race + - 0.1.27 Update ceph_mon config to ips from fqdn ...