[Ceph] Add Ceph CSI plugin

The PS adds Ceph CSI plugin (RBD only)

Change-Id: I3ddc69e49d12ff178263f38ac10aff90bb82b902

ceph-mon/Chart.yaml

@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph Mon
 name: ceph-mon
-version: 0.1.4
+version: 0.1.5
 home: https://github.com/ceph/ceph
 ...

ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl

@@ -73,6 +73,8 @@ metadata:
 type: kubernetes.io/rbd
 data:
   key: $( echo ${CEPH_KEYRING} | base64 | tr -d '\n' )
+  userID: $( echo -n "admin" | base64 | tr -d '\n' )
+  userKey: $( echo -n ${CEPH_KEYRING} | base64 | tr -d '\n' )
 EOF
     } | kubectl apply --namespace ${DEPLOYMENT_NAMESPACE} -f -
   fi

ceph-provisioners/Chart.yaml

@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph Provisioner
 name: ceph-provisioners
-version: 0.1.3
+version: 0.1.4
 home: https://github.com/ceph/ceph
 ...

ceph-provisioners/templates/bin/_helm-tests.sh.tpl

@@ -172,6 +172,7 @@ EOF
 
 
 reset_test_env $PVC_NAMESPACE $RBD_TEST_POD_NAME $RBD_TEST_PVC_NAME
+reset_test_env $PVC_NAMESPACE $CSI_RBD_TEST_POD_NAME $CSI_RBD_TEST_PVC_NAME
 reset_test_env $PVC_NAMESPACE $CEPHFS_TEST_POD_NAME $CEPHFS_TEST_PVC_NAME
 
 {{- range $storageclass, $val := .Values.storageclass }}
@@ -184,6 +185,14 @@ then
   reset_test_env $PVC_NAMESPACE $RBD_TEST_POD_NAME $RBD_TEST_PVC_NAME
 fi
 
+if [ {{ $val.provisioner }} == "ceph.rbd.csi.ceph.com" ] && [ {{ $val.provision_storage_class }} == true ];
+then
+  echo "--> Checking CSI RBD storage class."
+  storageclass={{ $val.metadata.name }}
+  storageclass_validation $PVC_NAMESPACE $CSI_RBD_TEST_POD_NAME $CSI_RBD_TEST_PVC_NAME $storageclass
+  reset_test_env $PVC_NAMESPACE $CSI_RBD_TEST_POD_NAME $CSI_RBD_TEST_PVC_NAME
+fi
+
 if [ {{ $val.provisioner }} == "ceph.com/cephfs" ] && [ {{ $val.provision_storage_class }} == true ];
 then
   echo "--> Checking cephfs storage class."

ceph-provisioners/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl

@@ -46,5 +46,5 @@ if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${PVC_CEPH_CEPHFS_S
     ${DEPLOYMENT_NAMESPACE} \
     "kubernetes.io/cephfs" \
     ${PVC_CEPH_CEPHFS_STORAGECLASS_USER_SECRET_NAME} \
-    "$(echo ${CEPH_CEPHFS_KEY} | jq -r '.data | .[]')"
+    "$(echo ${CEPH_CEPHFS_KEY} | jq -r '.data.key')"
 fi

ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl

@@ -41,4 +41,4 @@ EOF
   } | kubectl apply --namespace ${kube_namespace} -f -
 }
 
-ceph_activate_namespace ${DEPLOYMENT_NAMESPACE} "kubernetes.io/rbd" ${PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME} "$(echo ${CEPH_RBD_KEY} | jq -r '.data | .[]')"
+ceph_activate_namespace ${DEPLOYMENT_NAMESPACE} "kubernetes.io/rbd" ${PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME} "$(echo ${CEPH_RBD_KEY} | jq -r '.data.key')"

ceph-provisioners/templates/configmap-etc-csi.yaml

@@ -0,0 +1,48 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "ceph.configmap.etc.csi" }}
+{{- $configMapName := index . 0 }}
+{{- $envAll := index . 1 }}
+{{- with $envAll }}
+
+{{- if and (.Values.deployment.ceph) (.Values.deployment.csi) }}
+
+{{- if empty .Values.conf.ceph.global.mon_host -}}
+{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
+{{- end -}}
+
+---
+apiVersion: v1
+kind: ConfigMap
+data:
+  config.json: |-
+    [
+      {
+        "clusterID": {{ .Release.Namespace | quote }},
+        "monitors": [
+          {{ .Values.conf.ceph.global.mon_host | quote }}
+        ]
+      }
+    ]
+metadata:
+  name: ceph-csi-config
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{- if .Values.manifests.configmap_etc }}
+{{- list  .Values.storageclass.rbd.ceph_configmap_name . | include "ceph.configmap.etc.csi" }}
+{{- end }}

ceph-provisioners/templates/daemonset-csi-rbd-plugin.yaml

@@ -0,0 +1,181 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.rbd_provisioner }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := printf "%s-%s" .Release.Name "ceph-rbd-csi-nodeplugin" }}
+{{ tuple $envAll "rbd_provisioner" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: ceph-rbd-plugin
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+  labels:
+{{ tuple $envAll "rbd" "plugin" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  selector:
+    matchLabels:
+{{ tuple $envAll "rbd" "plugin" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
+{{ tuple $envAll "plugin" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "rbd" "plugin" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+      annotations:
+{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
+{{ dict "envAll" $envAll "podName" "ceph-rbd-plugin" "containerNames" (list "driver-registrar" "csi-rbdplugin" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
+    spec:
+{{ dict "envAll" $envAll "application" "plugin" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      serviceAccountName: {{ $serviceAccountName }}
+      nodeSelector:
+        {{ .Values.labels.csi_rbd_plugin.node_selector_key }}: {{ .Values.labels.csi_rbd_plugin.node_selector_value }}
+      hostNetwork: true
+      hostPID: true
+      dnsPolicy: {{ .Values.pod.dns_policy }}
+      initContainers:
+{{ tuple $envAll "rbd_plugin" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: driver-registrar
+{{ tuple $envAll "csi_registrar" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.rbd_registrar | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "plugin" "container" "ceph_rbd_registrar" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          args:
+            - "--v=0"
+            - "--csi-address=/csi/csi.sock"
+            - "--kubelet-registration-path=/var/lib/kubelet/plugins/$(DEPLOYMENT_NAMESPACE).rbd.csi.ceph.com/csi.sock"
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: csi-rbdplugin
+{{ tuple $envAll "cephcsi" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.rbd_cephcsi | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "plugin" "container" "ceph_csi_rbd_plugin" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          args:
+            - "--nodeid=$(NODE_ID)"
+            - "--type=rbd"
+            - "--nodeserver=true"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--v=0"
+            - "--drivername=$(DEPLOYMENT_NAMESPACE).rbd.csi.ceph.com"
+            - "--pidlimit=-1"
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - mountPath: /dev
+              name: host-dev
+            - mountPath: /sys
+              name: host-sys
+            - mountPath: /run/mount
+              name: host-mount
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
+            - name: plugin-dir
+              mountPath: /var/lib/kubelet/plugins
+              mountPropagation: "Bidirectional"
+            - name: mountpoint-dir
+              mountPath: /var/lib/kubelet/pods
+              mountPropagation: "Bidirectional"
+            - name: keys-tmp-dir
+              mountPath: /tmp/csi/keys
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/ceph.rbd.csi.ceph.com
+            type: DirectoryOrCreate
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins
+            type: Directory
+        - name: mountpoint-dir
+          hostPath:
+            path: /var/lib/kubelet/pods
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: host-dev
+          hostPath:
+            path: /dev
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: host-mount
+          hostPath:
+            path: /run/mount
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: ceph-csi-config
+          configMap:
+            name: ceph-csi-config
+        - name: keys-tmp-dir
+          emptyDir: {
+            medium: "Memory"
+          }
+{{- end }}

ceph-provisioners/templates/deployment-csi-rbd-provisioner.yaml

@@ -0,0 +1,283 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.rbd_provisioner }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := printf "%s-%s" .Release.Name "ceph-rbd-csi-provisioner" }}
+{{ tuple $envAll "rbd_provisioner" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ $serviceAccountName }}-run-rbd-provisioner
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: {{ $envAll.Release.Namespace }}
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ $serviceAccountName }}
+  namespace: {{ $envAll.Release.Namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+roleRef:
+  kind: Role
+  name: {{ $serviceAccountName }}
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: ceph-rbd-csi-provisioner
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+  labels:
+{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  replicas: {{ .Values.pod.replicas.rbd_provisioner }}
+  selector:
+    matchLabels:
+{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
+{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+      annotations:
+{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
+{{ dict "envAll" $envAll "podName" "ceph-rbd-csi-provisioner" "containerNames" (list "ceph-rbd-provisioner" "ceph-rbd-snapshotter" "ceph-rbd-attacher" "csi-resizer" "csi-rbdplugin" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
+    spec:
+{{ dict "envAll" $envAll "application" "provisioner" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      serviceAccountName: {{ $serviceAccountName }}
+      affinity:
+{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+{{ tuple $envAll "rbd_provisioner" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+      nodeSelector:
+        {{ .Values.labels.provisioner.node_selector_key }}: {{ .Values.labels.provisioner.node_selector_value }}
+      initContainers:
+{{ tuple $envAll "rbd_provisioner" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: ceph-rbd-provisioner
+{{ tuple $envAll "csi_provisioner" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.rbd_provisioner | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_provisioner" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: ADDRESS
+              value: unix:///csi/csi-provisioner.sock
+
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=0"
+            - "--timeout=150s"
+            - "--retry-interval-start=500ms"
+            - "--enable-leader-election=true"
+            - "--leader-election-type=leases"
+            - "--leader-election-namespace=$(DEPLOYMENT_NAMESPACE)"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: ceph-rbd-snapshotter
+{{ tuple $envAll "csi_snapshotter" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.rbd_snapshotter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_snapshotter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=0"
+            - "--timeout=150s"
+            - "--leader-election=true"
+            - "--leader-election-namespace=$(DEPLOYMENT_NAMESPACE)"
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: ADDRESS
+              value: unix:///csi/csi-provisioner.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: ceph-rbd-attacher
+{{ tuple $envAll "csi_attacher" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.rbd_attacher | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_attacher" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          args:
+            - "--v=0"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election=true"
+            - "--retry-interval-start=500ms"
+            - "--leader-election-namespace=$(DEPLOYMENT_NAMESPACE)"
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: ADDRESS
+              value: /csi/csi-provisioner.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-resizer
+{{ tuple $envAll "csi_resizer" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.rbd_resizer | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_resizer" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=0"
+            - "--csiTimeout=150s"
+            - "--leader-election"
+            - "--leader-election-namespace=$(DEPLOYMENT_NAMESPACE)"
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: ADDRESS
+              value: unix:///csi/csi-provisioner.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-rbdplugin
+{{ tuple $envAll "cephcsi" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.rbd_cephcsi | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_cephcsi" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          args:
+            - "--nodeid=$(NODE_ID)"
+            - "--type=rbd"
+            - "--controllerserver=true"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--v=0"
+            - "--drivername=$(DEPLOYMENT_NAMESPACE).rbd.csi.ceph.com"
+            - "--pidlimit=-1"
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi-provisioner.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - mountPath: /dev
+              name: host-dev
+            - mountPath: /sys
+              name: host-sys
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
+            - name: keys-tmp-dir
+              mountPath: /tmp/csi/keys
+      volumes:
+        - name: host-dev
+          hostPath:
+            path: /dev
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: socket-dir
+          emptyDir: {
+            medium: "Memory"
+          }
+        - name: ceph-csi-config
+          configMap:
+            name: ceph-csi-config
+        - name: keys-tmp-dir
+          emptyDir: {
+            medium: "Memory"
+          }
+{{- end }}

ceph-provisioners/templates/pod-helm-tests.yaml

@@ -88,6 +88,10 @@ spec:
           value: {{ .Values.pod.test_pod.rbd.name }}
         - name: RBD_TEST_PVC_NAME
           value: {{ .Values.pod.test_pod.rbd.pvc_name }}
+        - name: CSI_RBD_TEST_POD_NAME
+          value: {{ .Values.pod.test_pod.csi_rbd.name }}
+        - name: CSI_RBD_TEST_PVC_NAME
+          value: {{ .Values.pod.test_pod.csi_rbd.pvc_name }}
         - name: CEPHFS_TEST_POD_NAME
           value: {{ .Values.pod.test_pod.cephfs.name }}
         - name: CEPHFS_TEST_PVC_NAME

ceph-provisioners/values.yaml

@@ -20,6 +20,7 @@ deployment:
   ceph: true
   client_secrets: false
   rbd_provisioner: true
+  csi: true
   cephfs_provisioner: true
 
 release_group: null
@@ -31,6 +32,12 @@ images:
     ceph_cephfs_provisioner: 'docker.io/openstackhelm/ceph-cephfs-provisioner:ubuntu_bionic-20200521'
     ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
     ceph_rbd_provisioner: 'docker.io/openstackhelm/ceph-rbd-provisioner:change_770201_ubuntu_bionic-20210113'
+    csi_provisioner: 'quay.io/k8scsi/csi-provisioner:v1.6.0'
+    csi_snapshotter: 'quay.io/k8scsi/csi-snapshotter:v2.1.1'
+    csi_attacher: 'quay.io/k8scsi/csi-attacher:v2.1.1'
+    csi_resizer: 'quay.io/k8scsi/csi-resizer:v0.4.0'
+    csi_registrar: 'quay.io/k8scsi/csi-node-driver-registrar:v1.2.0'
+    cephcsi: 'quay.io/cephcsi/cephcsi:v3.1.0'
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/docker:17.07.0'
   local_registry:
@@ -49,6 +56,9 @@ labels:
   provisioner:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
+  csi_rbd_plugin:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
 
 pod:
   test_pod:
@@ -56,6 +66,9 @@ pod:
     rbd:
       name: rbd-prov-test-pod
       pvc_name: rbd-prov-test-pvc
+    csi_rbd:
+      name: csi-rbd-prov-test-pod
+      pvc_name: csi-rbd-prov-test-pvc
     cephfs:
       name: cephfs-prov-test-pod
       pvc_name: cephfs-prov-test-pvc
@@ -70,6 +83,29 @@ pod:
         ceph_rbd_provisioner:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
+        ceph_rbd_snapshotter:
+          privileged: true
+        ceph_rbd_attacher:
+          privileged: true
+        ceph_rbd_resizer:
+          privileged: true
+        ceph_rbd_cephcsi:
+          privileged: true
+          capabilities:
+            add: ["SYS_ADMIN"]
+    plugin:
+      pod:
+        runAsUser: 0
+      container:
+        ceph_rbd_registrar:
+          privileged: true
+          capabilities:
+            add: ["SYS_ADMIN"]
+        ceph_csi_rbd_plugin:
+          privileged: true
+          capabilities:
+            add: ["SYS_ADMIN"]
+          allowPrivilegeEscalation: true
     bootstrap:
       pod:
         runAsUser: 99
@@ -112,6 +148,12 @@ pod:
     upgrades:
       deployments:
         pod_replacement_strategy: Recreate
+      daemonsets:
+        pod_replacement_strategy: RollingUpdate
+        plugin:
+          enabled: true
+          min_ready_seconds: 0
+          max_unavailable: 1
   affinity:
     anti:
       type:
@@ -136,6 +178,41 @@ pod:
       limits:
         memory: "50Mi"
         cpu: "500m"
+    rbd_attacher:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
+    rbd_registrar:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
+    rbd_resizer:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
+    rbd_snapshotter:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
+    rbd_cephcsi:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
     jobs:
       bootstrap:
         limits:
@@ -263,8 +340,7 @@ storageclass:
     provisioner: ceph.com/rbd
     ceph_configmap_name: ceph-etc
     metadata:
-      default_storage_class: true
+      name: general-rbd
-      name: general
     parameters:
       pool: rbd
       adminId: admin
@@ -274,6 +350,24 @@ storageclass:
       userSecretName: pvc-ceph-client-key
       imageFormat: "2"
       imageFeatures: layering
+  csi_rbd:
+    provision_storage_class: true
+    provisioner: ceph.rbd.csi.ceph.com
+    metadata:
+      default_storage_class: true
+      name: general
+    parameters:
+      clusterID: ceph
+      csi.storage.k8s.io/controller-expand-secret-name: pvc-ceph-conf-combined-storageclass
+      csi.storage.k8s.io/controller-expand-secret-namespace: ceph
+      csi.storage.k8s.io/fstype: ext4
+      csi.storage.k8s.io/node-stage-secret-name: pvc-ceph-conf-combined-storageclass
+      csi.storage.k8s.io/node-stage-secret-namespace: ceph
+      csi.storage.k8s.io/provisioner-secret-name: pvc-ceph-conf-combined-storageclass
+      csi.storage.k8s.io/provisioner-secret-namespace: ceph
+      imageFeatures: layering
+      imageFormat: "2"
+      pool: rbd
   cephfs:
     provision_storage_class: true
     provisioner: ceph.com/cephfs
@@ -317,6 +411,7 @@ manifests:
   configmap_bin_common: true
   configmap_etc: true
   deployment_rbd_provisioner: true
+  deployment_csi_rbd_provisioner: true
   deployment_cephfs_provisioner: true
   job_bootstrap: false
   job_cephfs_client_key: true

ceph-provisioners/values_overrides/apparmor.yaml

@@ -11,6 +11,13 @@ pod:
     ceph-rbd-provisioner:
       ceph-rbd-provisioner: runtime/default
       init: runtime/default
+    ceph-rbd-csi-provisioner:
+      ceph-rbd-provisioner: runtime/default
+      init: runtime/default
+      ceph-rbd-snapshotter: runtime/default
+      ceph-rbd-attacher: runtime/default
+      csi-resizer: runtime/default
+      csi-rbdplugin: runtime/default
     ceph-provisioner-test:
       init: runtime/default
       ceph-provisioner-helm-test: runtime/default
@@ -20,6 +27,10 @@ pod:
     ceph-provisioners-ceph-ns-key-generator:
       ceph-storage-keys-generator: runtime/default
       init: runtime/default
+    ceph-rbd-plugin:
+      driver-registrar: runtime/default
+      csi-rbdplugin: runtime/default
+      init: runtime/default
 
 deployment:
   client_secrets: true

releasenotes/notes/ceph-mon.yaml

@@ -5,4 +5,5 @@ ceph-mon:
   - 0.1.2 Enable shareProcessNamespace in mon daemonset
   - 0.1.3 Run mon container as ceph user
   - 0.1.4 Uplift from Nautilus to Octopus release
+  - 0.1.5 Add Ceph CSI plugin
 ...

releasenotes/notes/ceph-provisioners.yaml

@@ -4,4 +4,5 @@ ceph-provisioners:
   - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0"
   - 0.1.2 Validate each storageclass created
   - 0.1.3 Uplift from Nautilus to Octopus release
+  - 0.1.4 Add Ceph CSI plugin
 ...