Ceph: Split out ceph provsioners from client chart

This PS splits out the ceph provisioners from the client chart
allowing OSH to be used with external (and multiple) cephs more
simply.

Change-Id: I0c5d516c7a9a10bc8435e2423eb1dc05460d2c92
Signed-off-by: Pete Birley <pete@port.direct>

ceph-client/templates/configmap-bin.yaml

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and .Values.manifests.configmap_bin_common .Values.deployment.ceph }}
+{{- if and .Values.manifests.configmap_bin .Values.deployment.ceph }}
 {{- $envAll := . }}
 ---
 apiVersion: v1
@@ -48,14 +48,6 @@ data:
   mgr-check.sh: |
 {{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 
-  provisioner-cephfs-start.sh: |
-{{ tuple "bin/provisioner/cephfs/_start.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
-  provisioner-cephfs-client-key-manager.sh: |
-{{ tuple "bin/provisioner/cephfs/_client-key-manager.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
-
-  provisioner-rbd-start.sh: |
-{{ tuple "bin/provisioner/rbd/_start.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
-
   rgw-start.sh: |
 {{ tuple "bin/rgw/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   rgw-init-keystone.sh: |

ceph-client/templates/configmap-etc-client.yaml

@@ -51,5 +51,5 @@ data:
 {{- end }}
 {{- end }}
 {{- if .Values.manifests.configmap_etc }}
-{{- list "ceph-etc" . | include "ceph.configmap.etc" }}
+{{- list "ceph-client-etc" . | include "ceph.configmap.etc" }}
 {{- end }}

ceph-client/templates/deployment-mds.yaml

@@ -89,7 +89,7 @@ spec:
               mountPath: /tmp/mds-start.sh
               subPath: mds-start.sh
               readOnly: true
-            - name: ceph-etc
+            - name: ceph-client-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -108,9 +108,9 @@ spec:
               mountPath: /run
               readOnly: false
       volumes:
-        - name: ceph-etc
+        - name: ceph-client-etc
           configMap:
-            name: ceph-etc
+            name: ceph-client-etc
             defaultMode: 0444
         - name: ceph-client-bin
           configMap:

ceph-client/templates/deployment-mgr.yaml

@@ -123,7 +123,7 @@ spec:
               readOnly: true
             - name: pod-etc-ceph
               mountPath: /etc/ceph
-            - name: ceph-etc
+            - name: ceph-client-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -148,9 +148,9 @@ spec:
           configMap:
             name: ceph-client-bin
             defaultMode: 0555
-        - name: ceph-etc
+        - name: ceph-client-etc
           configMap:
-            name: ceph-etc
+            name: ceph-client-etc
             defaultMode: 0444
         - name: pod-var-lib-ceph
           emptyDir: {}

ceph-client/templates/deployment-rgw.yaml

@@ -89,7 +89,7 @@ spec:
               mountPath: /tmp/rgw-init-keystone.sh
               subPath: rgw-init-keystone.sh
               readOnly: true
-            - name: ceph-etc
+            - name: ceph-client-etc
               mountPath: /tmp/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -126,7 +126,7 @@ spec:
             - name: pod-etc-ceph
               mountPath: /etc/ceph
 {{- if not .Values.conf.rgw_ks.enabled }}
-            - name: ceph-etc
+            - name: ceph-client-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -152,9 +152,9 @@ spec:
           configMap:
             name: ceph-client-bin
             defaultMode: 0555
-        - name: ceph-etc
+        - name: ceph-client-etc
           configMap:
-            name: ceph-etc
+            name: ceph-client-etc
             defaultMode: 0444
         - name: pod-var-lib-ceph
           emptyDir: {}

ceph-client/templates/job-bootstrap.yaml

@@ -47,7 +47,7 @@ spec:
               mountPath: /tmp/bootstrap.sh
               subPath: bootstrap.sh
               readOnly: true
-            - name: ceph-etc
+            - name: ceph-client-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -60,9 +60,9 @@ spec:
           configMap:
             name: ceph-client-bin
             defaultMode: 0555
-        - name: ceph-etc
+        - name: ceph-client-etc
           configMap:
-            name: ceph-etc
+            name: ceph-client-etc
             defaultMode: 0444
         - name: ceph-client-admin-keyring
           secret:

ceph-client/templates/job-rbd-pool.yaml

@@ -57,7 +57,7 @@ spec:
               mountPath: /tmp/pool-calc.py
               subPath: pool-calc.py
               readOnly: true
-            - name: ceph-etc
+            - name: ceph-client-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -72,9 +72,9 @@ spec:
               mountPath: /run
               readOnly: false
       volumes:
-        - name: ceph-etc
+        - name: ceph-client-etc
           configMap:
-            name: ceph-etc
+            name: ceph-client-etc
             defaultMode: 0444
         - name: ceph-client-bin
           configMap:

ceph-client/values.yaml

@@ -19,9 +19,6 @@
 
 deployment:
   ceph: true
-  client_secrets: false
-  rbd_provisioner: true
-  cephfs_provisioner: true
   rgw_keystone_user_and_endpoints: false
 
 release_group: null
@@ -30,12 +27,10 @@ images:
   pull_policy: IfNotPresent
   tags:
     ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
-    ceph_cephfs_provisioner: 'quay.io/external_storage/cephfs-provisioner:v0.1.1'
     ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3'
     ceph_mds: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     ceph_mgr: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     ceph_rbd_pool: 'docker.io/port/ceph-config-helper:v1.10.3'
-    ceph_rbd_provisioner: 'quay.io/external_storage/rbd-provisioner:v0.1.1'
     ceph_rgw: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
     ks_endpoints: 'docker.io/openstackhelm/heat:newton'
@@ -68,8 +63,6 @@ labels:
 pod:
   dns_policy: "ClusterFirstWithHostNet"
   replicas:
-    cephfs_provisioner: 2
-    rbd_provisioner: 2
     mds: 2
     mgr: 2
     rgw: 2
@@ -95,20 +88,6 @@ pod:
       limits:
         memory: "50Mi"
         cpu: "500m"
-    rbd_provisioner:
-      requests:
-        memory: "5Mi"
-        cpu: "250m"
-      limits:
-        memory: "50Mi"
-        cpu: "500m"
-    cephfs_provisioner:
-      requests:
-        memory: "5Mi"
-        cpu: "250m"
-      limits:
-        memory: "50Mi"
-        cpu: "500m"
     mgr:
       requests:
         memory: "5Mi"
@@ -433,32 +412,6 @@ ceph_mgr_enabled_modules:
 #    num_rep: "3"
 #    min_size: "2"
 
-# if you change provision_storage_class to false
-# it is presumed you manage your own storage
-# class definition externally
-storageclass:
-  rbd:
-    provision_storage_class: true
-    provisioner: ceph.com/rbd
-    name: general
-    monitors: null
-    pool: rbd
-    admin_id: admin
-    admin_secret_name: pvc-ceph-conf-combined-storageclass
-    admin_secret_namespace: ceph
-    user_id: admin
-    user_secret_name: pvc-ceph-client-key
-    image_format: "2"
-    image_features: layering
-  cephfs:
-    provision_storage_class: true
-    provisioner: ceph.com/cephfs
-    name: cephfs
-    admin_id: admin
-    user_secret_name: pvc-ceph-cephfs-client-key
-    admin_secret_name: pvc-ceph-conf-combined-storageclass
-    admin_secret_namespace: ceph
-
 endpoints:
   cluster_domain_suffix: cluster.local
   local_image_registry:
@@ -563,11 +516,8 @@ monitoring:
 manifests:
   configmap_bin: true
   configmap_bin_ks: true
-  configmap_bin_common: true
   configmap_etc: true
   deployment_mds: true
-  deployment_rbd_provisioner: true
-  deployment_cephfs_provisioner: true
   deployment_rgw: true
   deployment_mgr: true
   ingress_rgw: true
@@ -577,8 +527,6 @@ manifests:
   job_ks_endpoints: true
   job_ks_service: true
   job_ks_user: true
-  job_namespace_client_key_cleaner: true
-  job_namespace_client_key: true
   job_rbd_pool: true
   secret_ingress_tls: true
   secret_keystone_rgw: true
@@ -586,5 +534,3 @@ manifests:
   service_ingress_rgw: true
   service_mgr: true
   service_rgw: true
-  storageclass_cephfs: true
-  storageclass_rbd: true

ceph-provisioners/Chart.yaml

@@ -0,0 +1,18 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+description: OpenStack-Helm Ceph Client
+name: ceph-provisioners
+version: 0.1.0

ceph-provisioners/requirements.yaml

@@ -0,0 +1,18 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dependencies:
+  - name: helm-toolkit
+    repository: http://localhost:8879/charts
+    version: 0.1.0

ceph-provisioners/templates/bin/_bootstrap.sh.tpl

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+{{ .Values.bootstrap.script | default "echo 'Not Enabled'" }}

ceph-provisioners/templates/configmap-bin-provisioner.yaml

@@ -20,7 +20,7 @@ limitations under the License.
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: ceph-client-bin-provisioner
+  name: ceph-provisioners-bin-clients
 data:
   provisioner-rbd-namespace-client-key-manager.sh: |
 {{ tuple "bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

ceph-provisioners/templates/configmap-bin.yaml

@@ -0,0 +1,43 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.configmap_bin_common .Values.deployment.ceph }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-provisioners-bin
+data:
+{{- if .Values.images.local_registry.active }}
+  image-repo-sync.sh: |
+{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
+{{- end }}
+
+{{- if .Values.bootstrap.enabled }}
+  bootstrap.sh: |
+{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
+
+  provisioner-cephfs-start.sh: |
+{{ tuple "bin/provisioner/cephfs/_start.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+  provisioner-cephfs-client-key-manager.sh: |
+{{ tuple "bin/provisioner/cephfs/_client-key-manager.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
+  provisioner-rbd-start.sh: |
+{{ tuple "bin/provisioner/rbd/_start.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
+{{- end }}

ceph-provisioners/templates/configmap-etc-client.yaml

@@ -0,0 +1,51 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "ceph.configmap.etc" }}
+{{- $configMapName := index . 0 }}
+{{- $envAll := index . 1 }}
+{{- with $envAll }}
+
+{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
+
+{{- if empty .Values.conf.ceph.global.mon_host -}}
+{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
+{{- end -}}
+
+{{- if empty .Values.conf.ceph.osd.cluster_network -}}
+{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}
+{{- end -}}
+
+{{- if empty .Values.conf.ceph.osd.public_network -}}
+{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
+{{- end -}}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $configMapName }}
+data:
+  ceph.conf: |
+{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
+
+{{- end }}
+{{- end }}
+{{- end }}
+{{- if .Values.manifests.configmap_etc }}
+{{- list "ceph-etc" . | include "ceph.configmap.etc" }}
+{{- end }}

ceph-provisioners/templates/deployment-cephfs-provisioner.yaml

@@ -166,13 +166,13 @@ spec:
           command:
             - /tmp/provisioner-cephfs-start.sh
           volumeMounts:
-            - name: ceph-client-bin
+            - name: ceph-provisioners-bin
               mountPath: /tmp/provisioner-cephfs-start.sh
               subPath: provisioner-cephfs-start.sh
               readOnly: true
       volumes:
-        - name: ceph-client-bin
+        - name: ceph-provisioners-bin
           configMap:
-            name: ceph-client-bin
+            name: ceph-provisioners-bin
             defaultMode: 0555
 {{- end }}

ceph-provisioners/templates/deployment-rbd-provisioner.yaml

@@ -156,13 +156,13 @@ spec:
           command:
             - /tmp/provisioner-rbd-start.sh
           volumeMounts:
-            - name: ceph-client-bin
+            - name: ceph-provisioners-bin
               mountPath: /tmp/provisioner-rbd-start.sh
               subPath: provisioner-rbd-start.sh
               readOnly: true
       volumes:
-        - name: ceph-client-bin
+        - name: ceph-provisioners-bin
           configMap:
-            name: ceph-client-bin
+            name: ceph-provisioners-bin
             defaultMode: 0555
 {{- end }}

ceph-provisioners/templates/job-bootstrap.yaml

@@ -0,0 +1,70 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.job_bootstrap .Values.bootstrap.enabled }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "ceph-client-bootstrap"}}
+{{ tuple $envAll "bootstrap" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-client-bootstrap
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+      initContainers:
+{{ tuple $envAll "bootstrap" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 8 }}
+      containers:
+        - name: ceph-client-bootstrap
+{{ tuple $envAll "ceph_bootstrap" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/bootstrap.sh
+          volumeMounts:
+            - name: ceph-provisioners-bin
+              mountPath: /tmp/bootstrap.sh
+              subPath: bootstrap.sh
+              readOnly: true
+            - name: ceph-etc
+              mountPath: /etc/ceph/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+            - name: ceph-client-admin-keyring
+              mountPath: /etc/ceph/ceph.client.admin.keyring
+              subPath: ceph.client.admin.keyring
+              readOnly: true
+      volumes:
+        - name: ceph-provisioners-bin
+          configMap:
+            name: ceph-provisioners-bin
+            defaultMode: 0555
+        - name: ceph-etc
+          configMap:
+            name: ceph-etc
+            defaultMode: 0444
+        - name: ceph-client-admin-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.admin }}
+{{- end }}

ceph-provisioners/templates/job-cephfs-client-key.yaml

@@ -112,13 +112,13 @@ spec:
           command:
             - /tmp/provisioner-cephfs-client-key-manager.sh
           volumeMounts:
-            - name: ceph-client-bin
+            - name: ceph-provisioners-bin
               mountPath: /tmp/provisioner-cephfs-client-key-manager.sh
               subPath: provisioner-cephfs-client-key-manager.sh
               readOnly: true
       volumes:
-        - name: ceph-client-bin
+        - name: ceph-provisioners-bin
           configMap:
-            name: ceph-client-bin
+            name: ceph-provisioners-bin
             defaultMode: 0555
 {{- end }}

ceph-provisioners/templates/job-namespace-client-key-cleaner.yaml

@@ -81,13 +81,13 @@ spec:
           command:
             - /tmp/provisioner-rbd-namespace-client-key-cleaner.sh
           volumeMounts:
-            - name: ceph-client-bin-provisioner
+            - name: ceph-provisioners-bin-clients
               mountPath: /tmp/provisioner-rbd-namespace-client-key-cleaner.sh
               subPath: provisioner-rbd-namespace-client-key-cleaner.sh
               readOnly: true
       volumes:
-        - name: ceph-client-bin-provisioner
+        - name: ceph-provisioners-bin-clients
           configMap:
-            name: ceph-client-bin-provisioner
+            name: ceph-provisioners-bin-clients
             defaultMode: 0555
 {{- end }}

ceph-provisioners/templates/job-namespace-client-key.yaml

@@ -112,13 +112,13 @@ spec:
           command:
             - /tmp/provisioner-rbd-namespace-client-key-manager.sh
           volumeMounts:
-            - name: ceph-client-bin-provisioner
+            - name: ceph-provisioners-bin-clients
               mountPath: /tmp/provisioner-rbd-namespace-client-key-manager.sh
               subPath: provisioner-rbd-namespace-client-key-manager.sh
               readOnly: true
       volumes:
-        - name: ceph-client-bin-provisioner
+        - name: ceph-provisioners-bin-clients
           configMap:
-            name: ceph-client-bin-provisioner
+            name: ceph-provisioners-bin-clients
             defaultMode: 0555
 {{- end }}

ceph-provisioners/values.yaml

@@ -0,0 +1,234 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default values for ceph-client.
+# This is a YAML-formatted file.
+# Declare name/value pairs to be passed into your templates.
+# name: value
+
+deployment:
+  ceph: true
+  client_secrets: false
+  rbd_provisioner: true
+  cephfs_provisioner: true
+
+release_group: null
+
+images:
+  pull_policy: IfNotPresent
+  tags:
+    ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
+    ceph_cephfs_provisioner: 'quay.io/external_storage/cephfs-provisioner:v0.1.1'
+    ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3'
+    ceph_rbd_provisioner: 'quay.io/external_storage/rbd-provisioner:v0.1.1'
+    dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
+    image_repo_sync: docker.io/docker:17.07.0
+  local_registry:
+    active: false
+    exclude:
+      - dep_check
+      - image_repo_sync
+
+labels:
+  job:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+  provisioner:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+
+pod:
+  dns_policy: "ClusterFirstWithHostNet"
+  replicas:
+    cephfs_provisioner: 2
+    rbd_provisioner: 2
+  affinity:
+    anti:
+      type:
+        default: preferredDuringSchedulingIgnoredDuringExecution
+      topologyKey:
+        default: kubernetes.io/hostname
+  resources:
+    enabled: false
+    rbd_provisioner:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
+    cephfs_provisioner:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
+    jobs:
+      bootstrap:
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+        requests:
+          memory: "128Mi"
+          cpu: "500m"
+      image_repo_sync:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+
+secrets:
+  keyrings:
+    admin: ceph-client-admin-keyring
+
+network:
+  public: 192.168.0.0/16
+  cluster: 192.168.0.0/16
+  port:
+    mon: 6789
+    mgr: 7000
+
+conf:
+  ceph:
+    global:
+      # auth
+      cephx: true
+      cephx_require_signatures: false
+      cephx_cluster_require_signatures: true
+      cephx_service_require_signatures: false
+    osd:
+      osd_mkfs_type: xfs
+      osd_mkfs_options_xfs: -f -i size=2048
+      osd_max_object_name_len: 256
+      ms_bind_port_min: 6800
+      ms_bind_port_max: 7100
+
+dependencies:
+  dynamic:
+    common:
+      local_image_registry:
+        jobs:
+          - ceph-client-image-repo-sync
+        services:
+          - endpoint: node
+            service: local_image_registry
+  static:
+    bootstrap:
+      jobs: null
+      services:
+        - endpoint: internal
+          service: ceph_mon
+    cephfs_client_key_generator:
+      jobs: null
+    cephfs_provisioner:
+      jobs:
+        - ceph-rbd-pool
+      services:
+        - endpoint: internal
+          service: ceph_mon
+    namespace_client_key_cleaner:
+      jobs: null
+    namespace_client_key_generator:
+      jobs: null
+    rbd_provisioner:
+      jobs:
+        - ceph-rbd-pool
+      services:
+        - endpoint: internal
+          service: ceph_mon
+    image_repo_sync:
+      services:
+        - endpoint: internal
+          service: local_image_registry
+
+bootstrap:
+  enabled: false
+  script: |
+    ceph -s
+    function ensure_pool () {
+      ceph osd pool stats $1 || ceph osd pool create $1 $2
+      local test_luminous=$(ceph tell osd.* version | egrep -c "12.2|luminous")
+      if [[ ${test_luminous} -gt 0 ]]; then
+        ceph osd pool application enable $1 $3
+      fi
+    }
+    #ensure_pool volumes 8 cinder
+
+# if you change provision_storage_class to false
+# it is presumed you manage your own storage
+# class definition externally
+storageclass:
+  rbd:
+    provision_storage_class: true
+    provisioner: ceph.com/rbd
+    name: general
+    monitors: null
+    pool: rbd
+    admin_id: admin
+    admin_secret_name: pvc-ceph-conf-combined-storageclass
+    admin_secret_namespace: ceph
+    user_id: admin
+    user_secret_name: pvc-ceph-client-key
+    image_format: "2"
+    image_features: layering
+  cephfs:
+    provision_storage_class: true
+    provisioner: ceph.com/cephfs
+    name: cephfs
+    admin_id: admin
+    user_secret_name: pvc-ceph-cephfs-client-key
+    admin_secret_name: pvc-ceph-conf-combined-storageclass
+    admin_secret_namespace: ceph
+
+endpoints:
+  cluster_domain_suffix: cluster.local
+  local_image_registry:
+    name: docker-registry
+    namespace: docker-registry
+    hosts:
+      default: localhost
+      internal: docker-registry
+      node: localhost
+    host_fqdn_override:
+      default: null
+    port:
+      registry:
+        node: 5000
+  ceph_mon:
+    namespace: null
+    hosts:
+      default: ceph-mon
+      discovery: ceph-mon-discovery
+    host_fqdn_override:
+      default: null
+    port:
+      mon:
+        default: 6789
+
+manifests:
+  configmap_bin: true
+  configmap_bin_common: true
+  configmap_etc: true
+  deployment_rbd_provisioner: true
+  deployment_cephfs_provisioner: true
+  job_bootstrap: false
+  job_cephfs_client_key: true
+  job_image_repo_sync: true
+  job_namespace_client_key_cleaner: true
+  job_namespace_client_key: true
+  storageclass_cephfs: true
+  storageclass_rbd: true

tools/deployment/armada/multinode/armada-ceph.yaml

@@ -273,6 +273,79 @@ data:
   dependencies:
     - helm-toolkit
 ---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ceph-provisioners
+data:
+  chart_name: ceph-provisioners
+  release: ceph-provisioners
+  namespace: ceph
+  wait:
+    timeout: 1800
+    labels:
+      release_group: osh-ceph-provisioners
+  install:
+    no_hooks: False
+  upgrade:
+    no_hooks: False
+    pre:
+      delete:
+        - type: job
+          labels:
+            release_group: osh-ceph-provisioners
+  values:
+    endpoints:
+      identity:
+        namespace: openstack
+        auth:
+          swift:
+            password: ${SWIFT_USER_PASSWORD}
+      object_store:
+        namespace: ceph
+      ceph_mon:
+        namespace: ceph
+    network:
+      public: ${CEPH_NETWORK}
+      cluster: ${CEPH_NETWORK}
+    deployment:
+      storage_secrets: true
+      ceph: true
+      rbd_provisioner: true
+      cephfs_provisioner: true
+      client_secrets: false
+      rgw_keystone_user_and_endpoints: false
+    bootstrap:
+      enabled: true
+    conf:
+      ceph:
+        global:
+          fsid: ${CEPH_FS_ID}
+      rgw_ks:
+        enabled: true
+      pool:
+        crush:
+          tunables: ${CRUSH_TUNABLES}
+        target:
+          # NOTE(portdirect): 5 nodes, with one osd per node
+          osd: 5
+          pg_per_osd: 100
+      storage:
+        osd:
+          - data:
+              type: directory
+              location: /var/lib/openstack-helm/ceph/osd/osd-one
+            journal:
+              type: directory
+              location: /var/lib/openstack-helm/ceph/osd/journal-one
+  source:
+    type: local
+    location: ${OSH_PATH}
+    subpath: ceph-provisioners
+    reference: master
+  dependencies:
+    - helm-toolkit
+---
 schema: armada/ChartGroup/v1
 metadata:
   schema: metadata/Document/v1
@@ -285,6 +358,7 @@ data:
     - ceph-mon
     - ceph-osd
     - ceph-client
+    - ceph-provisioners
 ---
 schema: armada/Manifest/v1
 metadata:

tools/deployment/armada/multinode/armada-lma.yaml

@@ -103,7 +103,7 @@ data:
   source:
     type: local
     location: ${OSH_PATH}
-    subpath: ceph-client
+    subpath: ceph-provisioners
     reference: master
   dependencies:
     - helm-toolkit

tools/deployment/armada/multinode/armada-osh.yaml

@@ -107,7 +107,7 @@ data:
   source:
     type: local
     location: ${OSH_PATH}
-    subpath: ceph-client
+    subpath: ceph-provisioners
     reference: master
   dependencies:
     - helm-toolkit

tools/deployment/developer/ceph/040-ceph.sh

@@ -17,7 +17,7 @@
 set -xe
 
 #NOTE: Lint and package chart
-for CHART in ceph-mon ceph-osd ceph-client; do
+for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do
   make "${CHART}"
 done
 
@@ -163,7 +163,7 @@ pod:
     rgw: 1
 EOF
 
-for CHART in ceph-mon ceph-osd ceph-client; do
+for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do
   helm upgrade --install ${CHART} ./${CHART} \
     --namespace=ceph \
     --values=/tmp/ceph.yaml \

tools/deployment/developer/ceph/045-ceph-ns-activate.sh

@@ -17,7 +17,7 @@
 set -xe
 
 #NOTE: Lint and package chart
-make ceph-client
+make ceph-provisioners
 
 #NOTE: Deploy command
 : ${OSH_EXTRA_HELM_ARGS:=""}
@@ -45,7 +45,7 @@ conf:
   rgw_ks:
     enabled: true
 EOF
-helm upgrade --install ceph-openstack-config ./ceph-client \
+helm upgrade --install ceph-openstack-config ./ceph-provisioners \
   --namespace=openstack \
   --values=/tmp/ceph-openstack-config.yaml \
   ${OSH_EXTRA_HELM_ARGS} \

tools/deployment/multinode/030-ceph.sh

@@ -76,7 +76,7 @@ conf:
           location: /var/lib/openstack-helm/ceph/osd/journal-one
 EOF
 
-for CHART in ceph-mon ceph-osd ceph-client; do
+for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do
   helm upgrade --install ${CHART} ./${CHART} \
     --namespace=ceph \
     --values=/tmp/ceph.yaml \

tools/deployment/multinode/040-ceph-ns-activate.sh

@@ -43,7 +43,7 @@ conf:
   rgw_ks:
     enabled: true
 EOF
-helm upgrade --install ceph-openstack-config ./ceph-client \
+helm upgrade --install ceph-openstack-config ./ceph-provisioners \
   --namespace=openstack \
   --values=/tmp/ceph-openstack-config.yaml \
   ${OSH_EXTRA_HELM_ARGS} \