Merge "Ceph: Split out ceph provsioners from client chart"

ceph-client/templates/configmap-bin.yaml

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and .Values.manifests.configmap_bin_common .Values.deployment.ceph }}
+{{- if and .Values.manifests.configmap_bin .Values.deployment.ceph }}
 {{- $envAll := . }}
 ---
 apiVersion: v1
@@ -48,14 +48,6 @@ data:
   mgr-check.sh: |
 {{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 
-  provisioner-cephfs-start.sh: |
-{{ tuple "bin/provisioner/cephfs/_start.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
-  provisioner-cephfs-client-key-manager.sh: |
-{{ tuple "bin/provisioner/cephfs/_client-key-manager.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
-
-  provisioner-rbd-start.sh: |
-{{ tuple "bin/provisioner/rbd/_start.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
-
   rgw-start.sh: |
 {{ tuple "bin/rgw/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   rgw-init-keystone.sh: |

ceph-client/templates/configmap-etc-client.yaml

@@ -51,5 +51,5 @@ data:
 {{- end }}
 {{- end }}
 {{- if .Values.manifests.configmap_etc }}
-{{- list "ceph-etc" . | include "ceph.configmap.etc" }}
+{{- list "ceph-client-etc" . | include "ceph.configmap.etc" }}
 {{- end }}

ceph-client/templates/deployment-mds.yaml

@@ -89,7 +89,7 @@ spec:
               mountPath: /tmp/mds-start.sh
               subPath: mds-start.sh
               readOnly: true
-            - name: ceph-etc
+            - name: ceph-client-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -108,9 +108,9 @@ spec:
               mountPath: /run
               readOnly: false
       volumes:
-        - name: ceph-etc
+        - name: ceph-client-etc
           configMap:
-            name: ceph-etc
+            name: ceph-client-etc
             defaultMode: 0444
         - name: ceph-client-bin
           configMap:

ceph-client/templates/deployment-mgr.yaml

@@ -123,7 +123,7 @@ spec:
               readOnly: true
             - name: pod-etc-ceph
               mountPath: /etc/ceph
-            - name: ceph-etc
+            - name: ceph-client-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -148,9 +148,9 @@ spec:
           configMap:
             name: ceph-client-bin
             defaultMode: 0555
-        - name: ceph-etc
+        - name: ceph-client-etc
           configMap:
-            name: ceph-etc
+            name: ceph-client-etc
             defaultMode: 0444
         - name: pod-var-lib-ceph
           emptyDir: {}

ceph-client/templates/deployment-rgw.yaml

@@ -89,7 +89,7 @@ spec:
               mountPath: /tmp/rgw-init-keystone.sh
               subPath: rgw-init-keystone.sh
               readOnly: true
-            - name: ceph-etc
+            - name: ceph-client-etc
               mountPath: /tmp/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -126,7 +126,7 @@ spec:
             - name: pod-etc-ceph
               mountPath: /etc/ceph
 {{- if not .Values.conf.rgw_ks.enabled }}
-            - name: ceph-etc
+            - name: ceph-client-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -152,9 +152,9 @@ spec:
           configMap:
             name: ceph-client-bin
             defaultMode: 0555
-        - name: ceph-etc
+        - name: ceph-client-etc
           configMap:
-            name: ceph-etc
+            name: ceph-client-etc
             defaultMode: 0444
         - name: pod-var-lib-ceph
           emptyDir: {}

ceph-client/templates/job-bootstrap.yaml

@@ -47,7 +47,7 @@ spec:
               mountPath: /tmp/bootstrap.sh
               subPath: bootstrap.sh
               readOnly: true
-            - name: ceph-etc
+            - name: ceph-client-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -60,9 +60,9 @@ spec:
           configMap:
             name: ceph-client-bin
             defaultMode: 0555
-        - name: ceph-etc
+        - name: ceph-client-etc
           configMap:
-            name: ceph-etc
+            name: ceph-client-etc
             defaultMode: 0444
         - name: ceph-client-admin-keyring
           secret:

ceph-client/templates/job-rbd-pool.yaml

@@ -57,7 +57,7 @@ spec:
               mountPath: /tmp/pool-calc.py
               subPath: pool-calc.py
               readOnly: true
-            - name: ceph-etc
+            - name: ceph-client-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -72,9 +72,9 @@ spec:
               mountPath: /run
               readOnly: false
       volumes:
-        - name: ceph-etc
+        - name: ceph-client-etc
           configMap:
-            name: ceph-etc
+            name: ceph-client-etc
             defaultMode: 0444
         - name: ceph-client-bin
           configMap:

ceph-client/values.yaml

@@ -19,9 +19,6 @@
 
 deployment:
   ceph: true
-  client_secrets: false
-  rbd_provisioner: true
-  cephfs_provisioner: true
   rgw_keystone_user_and_endpoints: false
 
 release_group: null
@@ -30,12 +27,10 @@ images:
   pull_policy: IfNotPresent
   tags:
     ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
-    ceph_cephfs_provisioner: 'quay.io/external_storage/cephfs-provisioner:v0.1.1'
     ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3'
     ceph_mds: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     ceph_mgr: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     ceph_rbd_pool: 'docker.io/port/ceph-config-helper:v1.10.3'
-    ceph_rbd_provisioner: 'quay.io/external_storage/rbd-provisioner:v0.1.1'
     ceph_rgw: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
     ks_endpoints: 'docker.io/openstackhelm/heat:newton'
@@ -68,8 +63,6 @@ labels:
 pod:
   dns_policy: "ClusterFirstWithHostNet"
   replicas:
-    cephfs_provisioner: 2
-    rbd_provisioner: 2
     mds: 2
     mgr: 2
     rgw: 2
@@ -95,20 +88,6 @@ pod:
       limits:
         memory: "50Mi"
         cpu: "500m"
-    rbd_provisioner:
-      requests:
-        memory: "5Mi"
-        cpu: "250m"
-      limits:
-        memory: "50Mi"
-        cpu: "500m"
-    cephfs_provisioner:
-      requests:
-        memory: "5Mi"
-        cpu: "250m"
-      limits:
-        memory: "50Mi"
-        cpu: "500m"
     mgr:
       requests:
         memory: "5Mi"
@@ -435,32 +414,6 @@ ceph_mgr_enabled_modules:
 #    num_rep: "3"
 #    min_size: "2"
 
-# if you change provision_storage_class to false
-# it is presumed you manage your own storage
-# class definition externally
-storageclass:
-  rbd:
-    provision_storage_class: true
-    provisioner: ceph.com/rbd
-    name: general
-    monitors: null
-    pool: rbd
-    admin_id: admin
-    admin_secret_name: pvc-ceph-conf-combined-storageclass
-    admin_secret_namespace: ceph
-    user_id: admin
-    user_secret_name: pvc-ceph-client-key
-    image_format: "2"
-    image_features: layering
-  cephfs:
-    provision_storage_class: true
-    provisioner: ceph.com/cephfs
-    name: cephfs
-    admin_id: admin
-    user_secret_name: pvc-ceph-cephfs-client-key
-    admin_secret_name: pvc-ceph-conf-combined-storageclass
-    admin_secret_namespace: ceph
-
 endpoints:
   cluster_domain_suffix: cluster.local
   local_image_registry:
@@ -565,11 +518,8 @@ monitoring:
 manifests:
   configmap_bin: true
   configmap_bin_ks: true
-  configmap_bin_common: true
   configmap_etc: true
   deployment_mds: true
-  deployment_rbd_provisioner: true
-  deployment_cephfs_provisioner: true
   deployment_rgw: true
   deployment_mgr: true
   ingress_rgw: true
@@ -579,8 +529,6 @@ manifests:
   job_ks_endpoints: true
   job_ks_service: true
   job_ks_user: true
-  job_namespace_client_key_cleaner: true
-  job_namespace_client_key: true
   job_rbd_pool: true
   secret_ingress_tls: true
   secret_keystone_rgw: true
@@ -588,5 +536,3 @@ manifests:
   service_ingress_rgw: true
   service_mgr: true
   service_rgw: true
-  storageclass_cephfs: true
-  storageclass_rbd: true

ceph-provisioners/Chart.yaml

@@ -0,0 +1,18 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+description: OpenStack-Helm Ceph Client
+name: ceph-provisioners
+version: 0.1.0

ceph-provisioners/requirements.yaml

@@ -0,0 +1,18 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dependencies:
+  - name: helm-toolkit
+    repository: http://localhost:8879/charts
+    version: 0.1.0

ceph-provisioners/templates/bin/_bootstrap.sh.tpl

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+{{ .Values.bootstrap.script | default "echo 'Not Enabled'" }}

ceph-provisioners/templates/configmap-bin-provisioner.yaml

@@ -20,7 +20,7 @@ limitations under the License.
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: ceph-client-bin-provisioner
+  name: ceph-provisioners-bin-clients
 data:
   provisioner-rbd-namespace-client-key-manager.sh: |
 {{ tuple "bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

ceph-provisioners/templates/configmap-bin.yaml

@@ -0,0 +1,43 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.configmap_bin_common .Values.deployment.ceph }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-provisioners-bin
+data:
+{{- if .Values.images.local_registry.active }}
+  image-repo-sync.sh: |
+{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
+{{- end }}
+
+{{- if .Values.bootstrap.enabled }}
+  bootstrap.sh: |
+{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
+
+  provisioner-cephfs-start.sh: |
+{{ tuple "bin/provisioner/cephfs/_start.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+  provisioner-cephfs-client-key-manager.sh: |
+{{ tuple "bin/provisioner/cephfs/_client-key-manager.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
+  provisioner-rbd-start.sh: |
+{{ tuple "bin/provisioner/rbd/_start.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
+{{- end }}

ceph-provisioners/templates/configmap-etc-client.yaml

@@ -0,0 +1,51 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "ceph.configmap.etc" }}
+{{- $configMapName := index . 0 }}
+{{- $envAll := index . 1 }}
+{{- with $envAll }}
+
+{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
+
+{{- if empty .Values.conf.ceph.global.mon_host -}}
+{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
+{{- end -}}
+
+{{- if empty .Values.conf.ceph.osd.cluster_network -}}
+{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}
+{{- end -}}
+
+{{- if empty .Values.conf.ceph.osd.public_network -}}
+{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
+{{- end -}}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $configMapName }}
+data:
+  ceph.conf: |
+{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
+
+{{- end }}
+{{- end }}
+{{- end }}
+{{- if .Values.manifests.configmap_etc }}
+{{- list "ceph-etc" . | include "ceph.configmap.etc" }}
+{{- end }}

ceph-provisioners/templates/deployment-cephfs-provisioner.yaml

@@ -166,13 +166,13 @@ spec:
           command:
             - /tmp/provisioner-cephfs-start.sh
           volumeMounts:
-            - name: ceph-client-bin
+            - name: ceph-provisioners-bin
               mountPath: /tmp/provisioner-cephfs-start.sh
               subPath: provisioner-cephfs-start.sh
               readOnly: true
       volumes:
-        - name: ceph-client-bin
+        - name: ceph-provisioners-bin
           configMap:
-            name: ceph-client-bin
+            name: ceph-provisioners-bin
             defaultMode: 0555
 {{- end }}

ceph-provisioners/templates/deployment-rbd-provisioner.yaml

@@ -156,13 +156,13 @@ spec:
           command:
             - /tmp/provisioner-rbd-start.sh
           volumeMounts:
-            - name: ceph-client-bin
+            - name: ceph-provisioners-bin
               mountPath: /tmp/provisioner-rbd-start.sh
               subPath: provisioner-rbd-start.sh
               readOnly: true
       volumes:
-        - name: ceph-client-bin
+        - name: ceph-provisioners-bin
           configMap:
-            name: ceph-client-bin
+            name: ceph-provisioners-bin
             defaultMode: 0555
 {{- end }}

ceph-provisioners/templates/job-bootstrap.yaml

@@ -0,0 +1,70 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.job_bootstrap .Values.bootstrap.enabled }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "ceph-client-bootstrap"}}
+{{ tuple $envAll "bootstrap" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-client-bootstrap
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+      initContainers:
+{{ tuple $envAll "bootstrap" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 8 }}
+      containers:
+        - name: ceph-client-bootstrap
+{{ tuple $envAll "ceph_bootstrap" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/bootstrap.sh
+          volumeMounts:
+            - name: ceph-provisioners-bin
+              mountPath: /tmp/bootstrap.sh
+              subPath: bootstrap.sh
+              readOnly: true
+            - name: ceph-etc
+              mountPath: /etc/ceph/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+            - name: ceph-client-admin-keyring
+              mountPath: /etc/ceph/ceph.client.admin.keyring
+              subPath: ceph.client.admin.keyring
+              readOnly: true
+      volumes:
+        - name: ceph-provisioners-bin
+          configMap:
+            name: ceph-provisioners-bin
+            defaultMode: 0555
+        - name: ceph-etc
+          configMap:
+            name: ceph-etc
+            defaultMode: 0444
+        - name: ceph-client-admin-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.admin }}
+{{- end }}

ceph-provisioners/templates/job-cephfs-client-key.yaml

@@ -112,13 +112,13 @@ spec:
           command:
             - /tmp/provisioner-cephfs-client-key-manager.sh
           volumeMounts:
-            - name: ceph-client-bin
+            - name: ceph-provisioners-bin
               mountPath: /tmp/provisioner-cephfs-client-key-manager.sh
               subPath: provisioner-cephfs-client-key-manager.sh
               readOnly: true
       volumes:
-        - name: ceph-client-bin
+        - name: ceph-provisioners-bin
           configMap:
-            name: ceph-client-bin
+            name: ceph-provisioners-bin
             defaultMode: 0555
 {{- end }}

ceph-provisioners/templates/job-namespace-client-key-cleaner.yaml

@@ -81,13 +81,13 @@ spec:
           command:
             - /tmp/provisioner-rbd-namespace-client-key-cleaner.sh
           volumeMounts:
-            - name: ceph-client-bin-provisioner
+            - name: ceph-provisioners-bin-clients
               mountPath: /tmp/provisioner-rbd-namespace-client-key-cleaner.sh
               subPath: provisioner-rbd-namespace-client-key-cleaner.sh
               readOnly: true
       volumes:
-        - name: ceph-client-bin-provisioner
+        - name: ceph-provisioners-bin-clients
           configMap:
-            name: ceph-client-bin-provisioner
+            name: ceph-provisioners-bin-clients
             defaultMode: 0555
 {{- end }}

ceph-provisioners/templates/job-namespace-client-key.yaml

@@ -112,13 +112,13 @@ spec:
           command:
             - /tmp/provisioner-rbd-namespace-client-key-manager.sh
           volumeMounts:
-            - name: ceph-client-bin-provisioner
+            - name: ceph-provisioners-bin-clients
               mountPath: /tmp/provisioner-rbd-namespace-client-key-manager.sh
               subPath: provisioner-rbd-namespace-client-key-manager.sh
               readOnly: true
       volumes:
-        - name: ceph-client-bin-provisioner
+        - name: ceph-provisioners-bin-clients
           configMap:
-            name: ceph-client-bin-provisioner
+            name: ceph-provisioners-bin-clients
             defaultMode: 0555
 {{- end }}

ceph-provisioners/values.yaml

@@ -0,0 +1,234 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default values for ceph-client.
+# This is a YAML-formatted file.
+# Declare name/value pairs to be passed into your templates.
+# name: value
+
+deployment:
+  ceph: true
+  client_secrets: false
+  rbd_provisioner: true
+  cephfs_provisioner: true
+
+release_group: null
+
+images:
+  pull_policy: IfNotPresent
+  tags:
+    ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
+    ceph_cephfs_provisioner: 'quay.io/external_storage/cephfs-provisioner:v0.1.1'
+    ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3'
+    ceph_rbd_provisioner: 'quay.io/external_storage/rbd-provisioner:v0.1.1'
+    dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
+    image_repo_sync: docker.io/docker:17.07.0
+  local_registry:
+    active: false
+    exclude:
+      - dep_check
+      - image_repo_sync
+
+labels:
+  job:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+  provisioner:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+
+pod:
+  dns_policy: "ClusterFirstWithHostNet"
+  replicas:
+    cephfs_provisioner: 2
+    rbd_provisioner: 2
+  affinity:
+    anti:
+      type:
+        default: preferredDuringSchedulingIgnoredDuringExecution
+      topologyKey:
+        default: kubernetes.io/hostname
+  resources:
+    enabled: false
+    rbd_provisioner:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
+    cephfs_provisioner:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
+    jobs:
+      bootstrap:
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+        requests:
+          memory: "128Mi"
+          cpu: "500m"
+      image_repo_sync:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+
+secrets:
+  keyrings:
+    admin: ceph-client-admin-keyring
+
+network:
+  public: 192.168.0.0/16
+  cluster: 192.168.0.0/16
+  port:
+    mon: 6789
+    mgr: 7000
+
+conf:
+  ceph:
+    global:
+      # auth
+      cephx: true
+      cephx_require_signatures: false
+      cephx_cluster_require_signatures: true
+      cephx_service_require_signatures: false
+    osd:
+      osd_mkfs_type: xfs
+      osd_mkfs_options_xfs: -f -i size=2048
+      osd_max_object_name_len: 256
+      ms_bind_port_min: 6800
+      ms_bind_port_max: 7100
+
+dependencies:
+  dynamic:
+    common:
+      local_image_registry:
+        jobs:
+          - ceph-client-image-repo-sync
+        services:
+          - endpoint: node
+            service: local_image_registry
+  static:
+    bootstrap:
+      jobs: null
+      services:
+        - endpoint: internal
+          service: ceph_mon
+    cephfs_client_key_generator:
+      jobs: null
+    cephfs_provisioner:
+      jobs:
+        - ceph-rbd-pool
+      services:
+        - endpoint: internal
+          service: ceph_mon
+    namespace_client_key_cleaner:
+      jobs: null
+    namespace_client_key_generator:
+      jobs: null
+    rbd_provisioner:
+      jobs:
+        - ceph-rbd-pool
+      services:
+        - endpoint: internal
+          service: ceph_mon
+    image_repo_sync:
+      services:
+        - endpoint: internal
+          service: local_image_registry
+
+bootstrap:
+  enabled: false
+  script: |
+    ceph -s
+    function ensure_pool () {
+      ceph osd pool stats $1 || ceph osd pool create $1 $2
+      local test_luminous=$(ceph tell osd.* version | egrep -c "12.2|luminous")
+      if [[ ${test_luminous} -gt 0 ]]; then
+        ceph osd pool application enable $1 $3
+      fi
+    }
+    #ensure_pool volumes 8 cinder
+
+# if you change provision_storage_class to false
+# it is presumed you manage your own storage
+# class definition externally
+storageclass:
+  rbd:
+    provision_storage_class: true
+    provisioner: ceph.com/rbd
+    name: general
+    monitors: null
+    pool: rbd
+    admin_id: admin
+    admin_secret_name: pvc-ceph-conf-combined-storageclass
+    admin_secret_namespace: ceph
+    user_id: admin
+    user_secret_name: pvc-ceph-client-key
+    image_format: "2"
+    image_features: layering
+  cephfs:
+    provision_storage_class: true
+    provisioner: ceph.com/cephfs
+    name: cephfs
+    admin_id: admin
+    user_secret_name: pvc-ceph-cephfs-client-key
+    admin_secret_name: pvc-ceph-conf-combined-storageclass
+    admin_secret_namespace: ceph
+
+endpoints:
+  cluster_domain_suffix: cluster.local
+  local_image_registry:
+    name: docker-registry
+    namespace: docker-registry
+    hosts:
+      default: localhost
+      internal: docker-registry
+      node: localhost
+    host_fqdn_override:
+      default: null
+    port:
+      registry:
+        node: 5000
+  ceph_mon:
+    namespace: null
+    hosts:
+      default: ceph-mon
+      discovery: ceph-mon-discovery
+    host_fqdn_override:
+      default: null
+    port:
+      mon:
+        default: 6789
+
+manifests:
+  configmap_bin: true
+  configmap_bin_common: true
+  configmap_etc: true
+  deployment_rbd_provisioner: true
+  deployment_cephfs_provisioner: true
+  job_bootstrap: false
+  job_cephfs_client_key: true
+  job_image_repo_sync: true
+  job_namespace_client_key_cleaner: true
+  job_namespace_client_key: true
+  storageclass_cephfs: true
+  storageclass_rbd: true

tools/deployment/armada/multinode/armada-ceph.yaml

@@ -273,6 +273,79 @@ data:
   dependencies:
     - helm-toolkit
 ---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ceph-provisioners
+data:
+  chart_name: ceph-provisioners
+  release: ceph-provisioners
+  namespace: ceph
+  wait:
+    timeout: 1800
+    labels:
+      release_group: osh-ceph-provisioners
+  install:
+    no_hooks: False
+  upgrade:
+    no_hooks: False
+    pre:
+      delete:
+        - type: job
+          labels:
+            release_group: osh-ceph-provisioners
+  values:
+    endpoints:
+      identity:
+        namespace: openstack
+        auth:
+          swift:
+            password: ${SWIFT_USER_PASSWORD}
+      object_store:
+        namespace: ceph
+      ceph_mon:
+        namespace: ceph
+    network:
+      public: ${CEPH_NETWORK}
+      cluster: ${CEPH_NETWORK}
+    deployment:
+      storage_secrets: true
+      ceph: true
+      rbd_provisioner: true
+      cephfs_provisioner: true
+      client_secrets: false
+      rgw_keystone_user_and_endpoints: false
+    bootstrap:
+      enabled: true
+    conf:
+      ceph:
+        global:
+          fsid: ${CEPH_FS_ID}
+      rgw_ks:
+        enabled: true
+      pool:
+        crush:
+          tunables: ${CRUSH_TUNABLES}
+        target:
+          # NOTE(portdirect): 5 nodes, with one osd per node
+          osd: 5
+          pg_per_osd: 100
+      storage:
+        osd:
+          - data:
+              type: directory
+              location: /var/lib/openstack-helm/ceph/osd/osd-one
+            journal:
+              type: directory
+              location: /var/lib/openstack-helm/ceph/osd/journal-one
+  source:
+    type: local
+    location: ${OSH_PATH}
+    subpath: ceph-provisioners
+    reference: master
+  dependencies:
+    - helm-toolkit
+---
 schema: armada/ChartGroup/v1
 metadata:
   schema: metadata/Document/v1
@@ -285,6 +358,7 @@ data:
     - ceph-mon
     - ceph-osd
     - ceph-client
+    - ceph-provisioners
 ---
 schema: armada/Manifest/v1
 metadata:

tools/deployment/armada/multinode/armada-lma.yaml

@@ -103,7 +103,7 @@ data:
   source:
     type: local
     location: ${OSH_PATH}
-    subpath: ceph-client
+    subpath: ceph-provisioners
     reference: master
   dependencies:
     - helm-toolkit

tools/deployment/armada/multinode/armada-osh.yaml

@@ -107,7 +107,7 @@ data:
   source:
     type: local
     location: ${OSH_PATH}
-    subpath: ceph-client
+    subpath: ceph-provisioners
     reference: master
   dependencies:
     - helm-toolkit

tools/deployment/developer/ceph/040-ceph.sh

@@ -17,7 +17,7 @@
 set -xe
 
 #NOTE: Lint and package chart
-for CHART in ceph-mon ceph-osd ceph-client; do
+for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do
   make "${CHART}"
 done
 
@@ -163,7 +163,7 @@ pod:
     rgw: 1
 EOF
 
-for CHART in ceph-mon ceph-osd ceph-client; do
+for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do
   helm upgrade --install ${CHART} ./${CHART} \
     --namespace=ceph \
     --values=/tmp/ceph.yaml \

tools/deployment/developer/ceph/045-ceph-ns-activate.sh

@@ -17,7 +17,7 @@
 set -xe
 
 #NOTE: Lint and package chart
-make ceph-client
+make ceph-provisioners
 
 #NOTE: Deploy command
 : ${OSH_EXTRA_HELM_ARGS:=""}
@@ -45,7 +45,7 @@ conf:
   rgw_ks:
     enabled: true
 EOF
-helm upgrade --install ceph-openstack-config ./ceph-client \
+helm upgrade --install ceph-openstack-config ./ceph-provisioners \
   --namespace=openstack \
   --values=/tmp/ceph-openstack-config.yaml \
   ${OSH_EXTRA_HELM_ARGS} \

tools/deployment/multinode/030-ceph.sh

@@ -76,7 +76,7 @@ conf:
           location: /var/lib/openstack-helm/ceph/osd/journal-one
 EOF
 
-for CHART in ceph-mon ceph-osd ceph-client; do
+for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do
   helm upgrade --install ${CHART} ./${CHART} \
     --namespace=ceph \
     --values=/tmp/ceph.yaml \

tools/deployment/multinode/040-ceph-ns-activate.sh

@@ -43,7 +43,7 @@ conf:
   rgw_ks:
     enabled: true
 EOF
-helm upgrade --install ceph-openstack-config ./ceph-client \
+helm upgrade --install ceph-openstack-config ./ceph-provisioners \
   --namespace=openstack \
   --values=/tmp/ceph-openstack-config.yaml \
   ${OSH_EXTRA_HELM_ARGS} \