remove pvc-ceph-client-key when deleting ceph

When trying to re-install ceph chart, the
ceph-namespace-client-key-generator job is not working because of the
already created pvc-ceph-client-key secret.
So it needs to remove the secret when deleting ceph chart.

Closes-Bug: #1705177
Change-Id: I7338a516fe3b9f265953b8b1fddd3dbfb97bafd2

ceph/templates/bin/_ceph-namespace-client-key-cleaner.sh.tpl

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -ex
+
+kubectl delete secret ${PVC_CEPH_STORAGECLASS_USER_SECRET_NAME} \
+--namespace ${DEPLOYMENT_NAMESPACE}

ceph/templates/configmap-bin.yaml

@@ -33,6 +33,8 @@ data:
 {{- if .Values.manifests_enabled.client_secrets }}
   ceph-namespace-client-key.sh: |+
 {{ tuple "bin/_ceph-namespace-client-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  ceph-namespace-client-key-cleaner.sh: |+
+{{ tuple "bin/_ceph-namespace-client-key-cleaner.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
   common_functions.sh: |+
 {{ tuple "bin/_common_functions.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

ceph/templates/job-namespace-client-key-cleaner.yaml

@@ -0,0 +1,59 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if .Values.manifests_enabled.client_secrets }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-namespace-client-key-cleaner
+  annotations:
+    "helm.sh/hook": pre-delete
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name:  ceph-namespace-client-keys-cleaner
+          image: {{ .Values.images.ceph_config_helper }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          {{- if .Values.pod.resources.enabled }}
+          resources:
+            requests:
+              memory: {{ .Values.pod.resources.jobs.secret_provisioning.requests.memory | quote }}
+              cpu: {{ .Values.pod.resources.jobs.secret_provisioning.requests.cpu | quote }}
+            limits:
+              memory: {{ .Values.pod.resources.jobs.secret_provisioning.limits.memory | quote }}
+              cpu: {{ .Values.pod.resources.jobs.secret_provisioning.limits.cpu | quote }}
+          {{- end }}
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: PVC_CEPH_STORAGECLASS_USER_SECRET_NAME
+              value: {{ .Values.storageclass.user_secret_name }}
+          command:
+            - /opt/ceph/ceph-namespace-client-key-cleaner.sh
+          volumeMounts:
+            - name: ceph-bin
+              mountPath: /opt/ceph/ceph-namespace-client-key-cleaner.sh
+              subPath: ceph-namespace-client-key-cleaner.sh
+              readOnly: true
+      volumes:
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
+{{- end }}