d9404f89c2
The current implementation of the Ceph CSI provisioner is tied too closely with the older Ceph RBD provisioner, which doesn't let the deployer deploy Ceph CSI provisioner without the old RBD provisioner. This patchset will decouple them such that they can be deployed independently from one another. A few other changes are needed as well: 1) The deployment/gate scripts are updated so that the old RBD and CSI RBD provisioners are separately enabled/disabled as needed. The original RBD provisioner is now deprecated. 2) Ceph-mon chart is updated because it had some RBD storageclass data in values.yaml that is not needed for ceph-mon deployment. 3) Fixed a couple of bugs in job-cephfs-client-key.yaml where RBD parameters were being used instead of cephfs parameters. Change-Id: Icb5f78dcefa51990baf1b6d92411eb641c2ea9e2
284 lines
11 KiB
YAML
284 lines
11 KiB
YAML
{{/*
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/}}
|
|
|
|
{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.csi_rbd_provisioner }}
|
|
{{- $envAll := . }}
|
|
|
|
{{- $serviceAccountName := printf "%s-%s" .Release.Name "ceph-rbd-csi-provisioner" }}
|
|
{{ tuple $envAll "rbd_provisioner" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
|
|
---
|
|
kind: ClusterRole
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: {{ $serviceAccountName }}
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["nodes"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: [""]
|
|
resources: ["secrets"]
|
|
verbs: ["get", "list"]
|
|
- apiGroups: [""]
|
|
resources: ["events"]
|
|
verbs: ["list", "watch", "create", "update", "patch"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumes"]
|
|
verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumeclaims"]
|
|
verbs: ["get", "list", "watch", "update"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumeclaims/status"]
|
|
verbs: ["update", "patch"]
|
|
- apiGroups: ["storage.k8s.io"]
|
|
resources: ["storageclasses"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
|
resources: ["volumesnapshots"]
|
|
verbs: ["get", "list"]
|
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
|
resources: ["volumesnapshotcontents"]
|
|
verbs: ["create", "get", "list", "watch", "update", "delete"]
|
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
|
resources: ["volumesnapshotclasses"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: ["storage.k8s.io"]
|
|
resources: ["volumeattachments"]
|
|
verbs: ["get", "list", "watch", "update", "patch"]
|
|
- apiGroups: ["storage.k8s.io"]
|
|
resources: ["csinodes"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
|
resources: ["volumesnapshotcontents/status"]
|
|
verbs: ["update"]
|
|
---
|
|
kind: ClusterRoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: {{ $serviceAccountName }}-run-rbd-provisioner
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: {{ $serviceAccountName }}
|
|
namespace: {{ $envAll.Release.Namespace }}
|
|
roleRef:
|
|
kind: ClusterRole
|
|
name: {{ $serviceAccountName }}
|
|
apiGroup: rbac.authorization.k8s.io
|
|
---
|
|
kind: Role
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
namespace: {{ $envAll.Release.Namespace }}
|
|
name: {{ $serviceAccountName }}
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["configmaps"]
|
|
verbs: ["get", "list", "watch", "create", "delete"]
|
|
- apiGroups: ["coordination.k8s.io"]
|
|
resources: ["leases"]
|
|
verbs: ["get", "watch", "list", "delete", "update", "create"]
|
|
---
|
|
kind: RoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: {{ $serviceAccountName }}
|
|
namespace: {{ $envAll.Release.Namespace }}
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: {{ $serviceAccountName }}
|
|
namespace: {{ $envAll.Release.Namespace }}
|
|
roleRef:
|
|
kind: Role
|
|
name: {{ $serviceAccountName }}
|
|
apiGroup: rbac.authorization.k8s.io
|
|
---
|
|
kind: Deployment
|
|
apiVersion: apps/v1
|
|
metadata:
|
|
name: ceph-rbd-csi-provisioner
|
|
annotations:
|
|
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
|
|
labels:
|
|
{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
|
|
spec:
|
|
replicas: {{ .Values.pod.replicas.csi_rbd_provisioner }}
|
|
selector:
|
|
matchLabels:
|
|
{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
|
|
{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
|
|
template:
|
|
metadata:
|
|
labels:
|
|
{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
|
|
annotations:
|
|
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
|
|
{{ dict "envAll" $envAll "podName" "ceph-rbd-csi-provisioner" "containerNames" (list "ceph-rbd-provisioner" "ceph-rbd-snapshotter" "ceph-rbd-attacher" "csi-resizer" "csi-rbdplugin" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
|
|
spec:
|
|
{{ dict "envAll" $envAll "application" "provisioner" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
|
|
serviceAccountName: {{ $serviceAccountName }}
|
|
affinity:
|
|
{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
|
|
{{ tuple $envAll "csi_rbd_provisioner" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
|
|
nodeSelector:
|
|
{{ .Values.labels.provisioner.node_selector_key }}: {{ .Values.labels.provisioner.node_selector_value }}
|
|
initContainers:
|
|
{{ tuple $envAll "rbd_provisioner" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
|
|
containers:
|
|
- name: ceph-rbd-provisioner
|
|
{{ tuple $envAll "csi_provisioner" | include "helm-toolkit.snippets.image" | indent 10 }}
|
|
{{ tuple $envAll $envAll.Values.pod.resources.csi_rbd_provisioner | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
|
|
{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_provisioner" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
|
|
env:
|
|
- name: DEPLOYMENT_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
- name: ADDRESS
|
|
value: unix:///csi/csi-provisioner.sock
|
|
|
|
args:
|
|
- "--csi-address=$(ADDRESS)"
|
|
- "--v=0"
|
|
- "--timeout=150s"
|
|
- "--retry-interval-start=500ms"
|
|
- "--enable-leader-election=true"
|
|
- "--leader-election-type=leases"
|
|
- "--leader-election-namespace=$(DEPLOYMENT_NAMESPACE)"
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /csi
|
|
- name: ceph-rbd-snapshotter
|
|
{{ tuple $envAll "csi_snapshotter" | include "helm-toolkit.snippets.image" | indent 10 }}
|
|
{{ tuple $envAll $envAll.Values.pod.resources.rbd_snapshotter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
|
|
{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_snapshotter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
|
|
args:
|
|
- "--csi-address=$(ADDRESS)"
|
|
- "--v=0"
|
|
- "--timeout=150s"
|
|
- "--leader-election=true"
|
|
- "--leader-election-namespace=$(DEPLOYMENT_NAMESPACE)"
|
|
env:
|
|
- name: DEPLOYMENT_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
- name: ADDRESS
|
|
value: unix:///csi/csi-provisioner.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /csi
|
|
- name: ceph-rbd-attacher
|
|
{{ tuple $envAll "csi_attacher" | include "helm-toolkit.snippets.image" | indent 10 }}
|
|
{{ tuple $envAll $envAll.Values.pod.resources.rbd_attacher | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
|
|
{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_attacher" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
|
|
args:
|
|
- "--v=0"
|
|
- "--csi-address=$(ADDRESS)"
|
|
- "--leader-election=true"
|
|
- "--retry-interval-start=500ms"
|
|
- "--leader-election-namespace=$(DEPLOYMENT_NAMESPACE)"
|
|
env:
|
|
- name: DEPLOYMENT_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
- name: ADDRESS
|
|
value: /csi/csi-provisioner.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /csi
|
|
- name: csi-resizer
|
|
{{ tuple $envAll "csi_resizer" | include "helm-toolkit.snippets.image" | indent 10 }}
|
|
{{ tuple $envAll $envAll.Values.pod.resources.rbd_resizer | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
|
|
{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_resizer" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
|
|
args:
|
|
- "--csi-address=$(ADDRESS)"
|
|
- "--v=0"
|
|
- "--csiTimeout=150s"
|
|
- "--leader-election"
|
|
- "--leader-election-namespace=$(DEPLOYMENT_NAMESPACE)"
|
|
env:
|
|
- name: DEPLOYMENT_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
- name: ADDRESS
|
|
value: unix:///csi/csi-provisioner.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /csi
|
|
- name: csi-rbdplugin
|
|
{{ tuple $envAll "cephcsi" | include "helm-toolkit.snippets.image" | indent 10 }}
|
|
{{ tuple $envAll $envAll.Values.pod.resources.rbd_cephcsi | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
|
|
{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_cephcsi" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
|
|
args:
|
|
- "--nodeid=$(NODE_ID)"
|
|
- "--type=rbd"
|
|
- "--controllerserver=true"
|
|
- "--endpoint=$(CSI_ENDPOINT)"
|
|
- "--v=0"
|
|
- "--drivername=$(DEPLOYMENT_NAMESPACE).rbd.csi.ceph.com"
|
|
- "--pidlimit=-1"
|
|
env:
|
|
- name: DEPLOYMENT_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
- name: POD_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.podIP
|
|
- name: NODE_ID
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
- name: CSI_ENDPOINT
|
|
value: unix:///csi/csi-provisioner.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /csi
|
|
- mountPath: /dev
|
|
name: host-dev
|
|
- mountPath: /sys
|
|
name: host-sys
|
|
- mountPath: /lib/modules
|
|
name: lib-modules
|
|
readOnly: true
|
|
- name: ceph-csi-config
|
|
mountPath: /etc/ceph-csi-config/
|
|
- name: keys-tmp-dir
|
|
mountPath: /tmp/csi/keys
|
|
volumes:
|
|
- name: host-dev
|
|
hostPath:
|
|
path: /dev
|
|
- name: host-sys
|
|
hostPath:
|
|
path: /sys
|
|
- name: lib-modules
|
|
hostPath:
|
|
path: /lib/modules
|
|
- name: socket-dir
|
|
emptyDir: {
|
|
medium: "Memory"
|
|
}
|
|
- name: ceph-csi-config
|
|
configMap:
|
|
name: ceph-csi-config
|
|
- name: keys-tmp-dir
|
|
emptyDir: {
|
|
medium: "Memory"
|
|
}
|
|
{{- end }}
|