diff --git a/ceph-client/Chart.yaml b/ceph-client/Chart.yaml new file mode 100644 index 0000000000..225179ea7b --- /dev/null +++ b/ceph-client/Chart.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Ceph Client +name: ceph-client +version: 0.1.0 diff --git a/ceph/requirements.yaml b/ceph-client/requirements.yaml similarity index 100% rename from ceph/requirements.yaml rename to ceph-client/requirements.yaml diff --git a/ceph/templates/bin/_bootstrap.sh.tpl b/ceph-client/templates/bin/_bootstrap.sh.tpl similarity index 100% rename from ceph/templates/bin/_bootstrap.sh.tpl rename to ceph-client/templates/bin/_bootstrap.sh.tpl diff --git a/ceph-client/templates/bin/_init-dirs.sh.tpl b/ceph-client/templates/bin/_init-dirs.sh.tpl new file mode 100644 index 0000000000..3db82a767d --- /dev/null +++ b/ceph-client/templates/bin/_init-dirs.sh.tpl @@ -0,0 +1,51 @@ +#!/bin/bash + +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export LC_ALL=C +: "${HOSTNAME:=$(uname -n)}" +: "${MGR_NAME:=${HOSTNAME}}" +: "${RGW_NAME:=${HOSTNAME}}" +: "${MDS_NAME:=mds-${HOSTNAME}}" +: "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}" +: "${RGW_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-rgw/${CLUSTER}.keyring}" +: "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}" + +for keyring in ${OSD_BOOTSTRAP_KEYRING} ${MDS_BOOTSTRAP_KEYRING} ${RGW_BOOTSTRAP_KEYRING}; do + mkdir -p "$(dirname "$keyring")" +done + +# Let's create the ceph directories +for DIRECTORY in mds radosgw tmp mgr; do + mkdir -p "/var/lib/ceph/${DIRECTORY}" +done + +# Create socket directory +mkdir -p /run/ceph + +# Creating rados directories +mkdir -p "/var/lib/ceph/radosgw/${RGW_NAME}" + +# Create the MDS directory +mkdir -p "/var/lib/ceph/mds/${CLUSTER}-${MDS_NAME}" + +# Create the MGR directory +mkdir -p "/var/lib/ceph/mgr/${CLUSTER}-${MGR_NAME}" + +# Adjust the owner of all those directories +chown -R ceph. /run/ceph/ /var/lib/ceph/* diff --git a/ceph/templates/bin/mds/_start.sh.tpl b/ceph-client/templates/bin/mds/_start.sh.tpl similarity index 100% rename from ceph/templates/bin/mds/_start.sh.tpl rename to ceph-client/templates/bin/mds/_start.sh.tpl diff --git a/ceph/templates/bin/mgr/_check.sh.tpl b/ceph-client/templates/bin/mgr/_check.sh.tpl similarity index 100% rename from ceph/templates/bin/mgr/_check.sh.tpl rename to ceph-client/templates/bin/mgr/_check.sh.tpl diff --git a/ceph/templates/bin/mgr/_start.sh.tpl b/ceph-client/templates/bin/mgr/_start.sh.tpl similarity index 100% rename from ceph/templates/bin/mgr/_start.sh.tpl rename to ceph-client/templates/bin/mgr/_start.sh.tpl diff --git a/ceph/templates/bin/pool/_calc.py.tpl b/ceph-client/templates/bin/pool/_calc.py.tpl similarity index 100% rename from ceph/templates/bin/pool/_calc.py.tpl rename to ceph-client/templates/bin/pool/_calc.py.tpl diff --git a/ceph/templates/bin/pool/_init.sh.tpl b/ceph-client/templates/bin/pool/_init.sh.tpl similarity index 100% rename from ceph/templates/bin/pool/_init.sh.tpl rename to ceph-client/templates/bin/pool/_init.sh.tpl diff --git a/ceph/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl b/ceph-client/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl similarity index 100% rename from ceph/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl rename to ceph-client/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl diff --git a/ceph/templates/bin/provisioner/cephfs/_start.sh.tpl b/ceph-client/templates/bin/provisioner/cephfs/_start.sh.tpl similarity index 100% rename from ceph/templates/bin/provisioner/cephfs/_start.sh.tpl rename to ceph-client/templates/bin/provisioner/cephfs/_start.sh.tpl diff --git a/ceph/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl b/ceph-client/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl similarity index 100% rename from ceph/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl rename to ceph-client/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl diff --git a/ceph/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl b/ceph-client/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl similarity index 100% rename from ceph/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl rename to ceph-client/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl diff --git a/ceph/templates/bin/provisioner/rbd/_start.sh.tpl b/ceph-client/templates/bin/provisioner/rbd/_start.sh.tpl similarity index 100% rename from ceph/templates/bin/provisioner/rbd/_start.sh.tpl rename to ceph-client/templates/bin/provisioner/rbd/_start.sh.tpl diff --git a/ceph/templates/bin/rgw/_init_keystone.sh.tpl b/ceph-client/templates/bin/rgw/_init_keystone.sh.tpl similarity index 100% rename from ceph/templates/bin/rgw/_init_keystone.sh.tpl rename to ceph-client/templates/bin/rgw/_init_keystone.sh.tpl diff --git a/ceph/templates/bin/rgw/_start.sh.tpl b/ceph-client/templates/bin/rgw/_start.sh.tpl similarity index 100% rename from ceph/templates/bin/rgw/_start.sh.tpl rename to ceph-client/templates/bin/rgw/_start.sh.tpl diff --git a/ceph/templates/configmap-bin-ks.yaml b/ceph-client/templates/configmap-bin-ks.yaml similarity index 97% rename from ceph/templates/configmap-bin-ks.yaml rename to ceph-client/templates/configmap-bin-ks.yaml index 2a502a3d63..64258219a2 100644 --- a/ceph/templates/configmap-bin-ks.yaml +++ b/ceph-client/templates/configmap-bin-ks.yaml @@ -20,7 +20,7 @@ limitations under the License. apiVersion: v1 kind: ConfigMap metadata: - name: ceph-bin-ks + name: ceph-client-bin-ks data: ks-service.sh: | {{- include "helm-toolkit.scripts.keystone_service" . | indent 4 }} diff --git a/ceph/templates/configmap-bin-clients.yaml b/ceph-client/templates/configmap-bin-provisioner.yaml similarity index 89% rename from ceph/templates/configmap-bin-clients.yaml rename to ceph-client/templates/configmap-bin-provisioner.yaml index d4647db3b0..8a52855eaa 100644 --- a/ceph/templates/configmap-bin-clients.yaml +++ b/ceph-client/templates/configmap-bin-provisioner.yaml @@ -14,13 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and .Values.manifests.configmap_bin_clients .Values.deployment.client_secrets }} +{{- if and .Values.manifests.configmap_bin .Values.deployment.client_secrets }} {{- $envAll := . }} --- apiVersion: v1 kind: ConfigMap metadata: - name: ceph-bin-clients + name: ceph-client-bin-provisioner data: provisioner-rbd-namespace-client-key-manager.sh: | {{ tuple "bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/ceph/templates/configmap-bin.yaml b/ceph-client/templates/configmap-bin.yaml similarity index 54% rename from ceph/templates/configmap-bin.yaml rename to ceph-client/templates/configmap-bin.yaml index 0fc7eea748..e34a33ad2f 100644 --- a/ceph/templates/configmap-bin.yaml +++ b/ceph-client/templates/configmap-bin.yaml @@ -14,13 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and .Values.manifests.configmap_bin .Values.deployment.ceph }} +{{- if and .Values.manifests.configmap_bin_common .Values.deployment.ceph }} {{- $envAll := . }} --- apiVersion: v1 kind: ConfigMap metadata: - name: ceph-bin + name: ceph-client-bin data: {{- if .Values.bootstrap.enabled }} bootstrap.sh: | @@ -35,13 +35,6 @@ data: pool-calc.py: | {{ tuple "bin/pool/_calc.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - keys-bootstrap-keyring-generator.py: | -{{ tuple "bin/keys/_bootstrap-keyring-generator.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - keys-bootstrap-keyring-manager.sh: | -{{ tuple "bin/keys/_bootstrap-keyring-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - keys-storage-keyring-manager.sh: | -{{ tuple "bin/keys/_storage-keyring-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - mds-start.sh: | {{ tuple "bin/mds/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} @@ -50,31 +43,6 @@ data: mgr-check.sh: | {{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - mon-start.sh: | -{{ tuple "bin/mon/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - mon-stop.sh: | -{{ tuple "bin/mon/_stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - mon-check.sh: | -{{ tuple "bin/mon/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - - moncheck-start.sh: | -{{ tuple "bin/moncheck/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - moncheck-reap-zombies.py: | -{{ tuple "bin/moncheck/_reap-zombies.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - - osd-start.sh: | -{{ tuple "bin/osd/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - osd-directory.sh: | -{{ tuple "bin/osd/_directory.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - osd-block.sh: | -{{ tuple "bin/osd/_block.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - osd-init.sh: | -{{ tuple "bin/osd/_init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - osd-check.sh: | -{{ tuple "bin/osd/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - osd-stop.sh: | -{{ tuple "bin/osd/_stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - provisioner-cephfs-start.sh: | {{ tuple "bin/provisioner/cephfs/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} provisioner-cephfs-client-key-manager.sh: | @@ -89,3 +57,4 @@ data: {{ tuple "bin/rgw/_init_keystone.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} + diff --git a/ceph/templates/configmap-etc.yaml b/ceph-client/templates/configmap-etc-client.yaml similarity index 100% rename from ceph/templates/configmap-etc.yaml rename to ceph-client/templates/configmap-etc-client.yaml diff --git a/ceph/templates/deployment-cephfs-provisioner.yaml b/ceph-client/templates/deployment-cephfs-provisioner.yaml similarity index 97% rename from ceph/templates/deployment-cephfs-provisioner.yaml rename to ceph-client/templates/deployment-cephfs-provisioner.yaml index 557196cccc..3930b67309 100644 --- a/ceph/templates/deployment-cephfs-provisioner.yaml +++ b/ceph-client/templates/deployment-cephfs-provisioner.yaml @@ -163,13 +163,13 @@ spec: command: - /tmp/provisioner-cephfs-start.sh volumeMounts: - - name: ceph-bin + - name: ceph-client-bin mountPath: /tmp/provisioner-cephfs-start.sh subPath: provisioner-cephfs-start.sh readOnly: true volumes: - - name: ceph-bin + - name: ceph-client-bin configMap: - name: ceph-bin + name: ceph-client-bin defaultMode: 0555 {{- end }} diff --git a/ceph/templates/deployment-mds.yaml b/ceph-client/templates/deployment-mds.yaml similarity index 96% rename from ceph/templates/deployment-mds.yaml rename to ceph-client/templates/deployment-mds.yaml index 5553f1262a..469fc8cfc9 100644 --- a/ceph/templates/deployment-mds.yaml +++ b/ceph-client/templates/deployment-mds.yaml @@ -49,7 +49,7 @@ spec: - name: CLUSTER value: "ceph" volumeMounts: - - name: ceph-bin + - name: ceph-client-bin mountPath: /tmp/init-dirs.sh subPath: init-dirs.sh readOnly: true @@ -83,7 +83,7 @@ spec: port: 6800 timeoutSeconds: 5 volumeMounts: - - name: ceph-bin + - name: ceph-client-bin mountPath: /tmp/mds-start.sh subPath: mds-start.sh readOnly: true @@ -110,9 +110,9 @@ spec: configMap: name: ceph-etc defaultMode: 0444 - - name: ceph-bin + - name: ceph-client-bin configMap: - name: ceph-bin + name: ceph-client-bin defaultMode: 0555 - name: pod-var-lib-ceph emptyDir: {} diff --git a/ceph/templates/deployment-mgr.yaml b/ceph-client/templates/deployment-mgr.yaml similarity index 97% rename from ceph/templates/deployment-mgr.yaml rename to ceph-client/templates/deployment-mgr.yaml index b54c22fba5..513fe9ad7c 100644 --- a/ceph/templates/deployment-mgr.yaml +++ b/ceph-client/templates/deployment-mgr.yaml @@ -52,7 +52,7 @@ spec: - name: CLUSTER value: "ceph" volumeMounts: - - name: ceph-bin + - name: ceph-client-bin mountPath: /tmp/init-dirs.sh subPath: init-dirs.sh readOnly: true @@ -111,11 +111,11 @@ spec: initialDelaySeconds: 30 timeoutSeconds: 5 volumeMounts: - - name: ceph-bin + - name: ceph-client-bin mountPath: /mgr-start.sh subPath: mgr-start.sh readOnly: true - - name: ceph-bin + - name: ceph-client-bin mountPath: /tmp/mgr-check.sh subPath: mgr-check.sh readOnly: true @@ -142,9 +142,9 @@ spec: volumes: - name: pod-etc-ceph emptyDir: {} - - name: ceph-bin + - name: ceph-client-bin configMap: - name: ceph-bin + name: ceph-client-bin defaultMode: 0555 - name: ceph-etc configMap: diff --git a/ceph/templates/deployment-rbd-provisioner.yaml b/ceph-client/templates/deployment-rbd-provisioner.yaml similarity index 97% rename from ceph/templates/deployment-rbd-provisioner.yaml rename to ceph-client/templates/deployment-rbd-provisioner.yaml index 253e4988c6..45ac0fdae6 100644 --- a/ceph/templates/deployment-rbd-provisioner.yaml +++ b/ceph-client/templates/deployment-rbd-provisioner.yaml @@ -153,13 +153,13 @@ spec: command: - /tmp/provisioner-rbd-start.sh volumeMounts: - - name: ceph-bin + - name: ceph-client-bin mountPath: /tmp/provisioner-rbd-start.sh subPath: provisioner-rbd-start.sh readOnly: true volumes: - - name: ceph-bin + - name: ceph-client-bin configMap: - name: ceph-bin + name: ceph-client-bin defaultMode: 0555 {{- end }} diff --git a/ceph/templates/deployment-rgw.yaml b/ceph-client/templates/deployment-rgw.yaml similarity index 97% rename from ceph/templates/deployment-rgw.yaml rename to ceph-client/templates/deployment-rgw.yaml index db0f9926f3..91abf8ff96 100644 --- a/ceph/templates/deployment-rgw.yaml +++ b/ceph-client/templates/deployment-rgw.yaml @@ -48,7 +48,7 @@ spec: - name: CLUSTER value: "ceph" volumeMounts: - - name: ceph-bin + - name: ceph-client-bin mountPath: /tmp/init-dirs.sh subPath: init-dirs.sh readOnly: true @@ -83,7 +83,7 @@ spec: volumeMounts: - name: pod-etc-ceph mountPath: /etc/ceph - - name: ceph-bin + - name: ceph-client-bin mountPath: /tmp/rgw-init-keystone.sh subPath: rgw-init-keystone.sh readOnly: true @@ -118,7 +118,7 @@ spec: port: {{ .Values.network.port.rgw }} timeoutSeconds: 5 volumeMounts: - - name: ceph-bin + - name: ceph-client-bin mountPath: /tmp/rgw-start.sh subPath: rgw-start.sh readOnly: true @@ -147,9 +147,9 @@ spec: volumes: - name: pod-etc-ceph emptyDir: {} - - name: ceph-bin + - name: ceph-client-bin configMap: - name: ceph-bin + name: ceph-client-bin defaultMode: 0555 - name: ceph-etc configMap: diff --git a/ceph-client/templates/job-bootstrap.yaml b/ceph-client/templates/job-bootstrap.yaml new file mode 100644 index 0000000000..eeafaefd9f --- /dev/null +++ b/ceph-client/templates/job-bootstrap.yaml @@ -0,0 +1,72 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_bootstrap .Values.bootstrap.enabled }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.static.bootstrap }} + +{{- $serviceAccountName := "ceph-client-bootstrap"}} +{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-client-bootstrap +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-client-bootstrap + image: {{ .Values.images.tags.ceph_bootstrap }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/bootstrap.sh + volumeMounts: + - name: ceph-client-bin + mountPath: /tmp/bootstrap.sh + subPath: bootstrap.sh + readOnly: true + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + volumes: + - name: ceph-client-bin + configMap: + name: ceph-client-bin + defaultMode: 0555 + - name: ceph-etc + configMap: + name: ceph-etc + defaultMode: 0444 + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} +{{- end }} diff --git a/ceph/templates/job-cephfs-client-key.yaml b/ceph-client/templates/job-cephfs-client-key.yaml similarity index 97% rename from ceph/templates/job-cephfs-client-key.yaml rename to ceph-client/templates/job-cephfs-client-key.yaml index 6578b92159..fc17463269 100644 --- a/ceph/templates/job-cephfs-client-key.yaml +++ b/ceph-client/templates/job-cephfs-client-key.yaml @@ -114,13 +114,13 @@ spec: command: - /tmp/provisioner-cephfs-client-key-manager.sh volumeMounts: - - name: ceph-bin + - name: ceph-client-bin mountPath: /tmp/provisioner-cephfs-client-key-manager.sh subPath: provisioner-cephfs-client-key-manager.sh readOnly: true volumes: - - name: ceph-bin + - name: ceph-client-bin configMap: - name: ceph-bin + name: ceph-client-bin defaultMode: 0555 {{- end }} diff --git a/ceph/templates/job-ks-endpoints.yaml b/ceph-client/templates/job-ks-endpoints.yaml similarity index 84% rename from ceph/templates/job-ks-endpoints.yaml rename to ceph-client/templates/job-ks-endpoints.yaml index 8d3a952e2d..c560f2e825 100644 --- a/ceph/templates/job-ks-endpoints.yaml +++ b/ceph-client/templates/job-ks-endpoints.yaml @@ -15,6 +15,6 @@ limitations under the License. */}} {{- if and .Values.manifests.job_ks_endpoints .Values.deployment.rgw_keystone_user_and_endpoints }} -{{- $ksServiceJob := dict "envAll" . "configMapBin" "ceph-bin-ks" "serviceName" "ceph" "serviceTypes" ( tuple "object-store" ) -}} +{{- $ksServiceJob := dict "envAll" . "configMapBin" "ceph-client-bin-ks" "serviceName" "ceph" "serviceTypes" ( tuple "object-store" ) -}} {{ $ksServiceJob | include "helm-toolkit.manifests.job_ks_endpoints" }} {{- end }} diff --git a/ceph/templates/job-ks-service.yaml b/ceph-client/templates/job-ks-service.yaml similarity index 84% rename from ceph/templates/job-ks-service.yaml rename to ceph-client/templates/job-ks-service.yaml index e8f6590f6b..80a440de62 100644 --- a/ceph/templates/job-ks-service.yaml +++ b/ceph-client/templates/job-ks-service.yaml @@ -15,6 +15,6 @@ limitations under the License. */}} {{- if and .Values.manifests.job_ks_service .Values.deployment.rgw_keystone_user_and_endpoints }} -{{- $ksServiceJob := dict "envAll" . "configMapBin" "ceph-bin-ks" "serviceName" "ceph" "serviceTypes" ( tuple "object-store" ) -}} +{{- $ksServiceJob := dict "envAll" . "configMapBin" "ceph-client-bin-ks" "serviceName" "ceph" "serviceTypes" ( tuple "object-store" ) -}} {{ $ksServiceJob | include "helm-toolkit.manifests.job_ks_service" }} {{- end }} diff --git a/ceph/templates/job-ks-user.yaml b/ceph-client/templates/job-ks-user.yaml similarity index 86% rename from ceph/templates/job-ks-user.yaml rename to ceph-client/templates/job-ks-user.yaml index 80c5b1e579..b1f1edfe7d 100644 --- a/ceph/templates/job-ks-user.yaml +++ b/ceph-client/templates/job-ks-user.yaml @@ -15,6 +15,6 @@ limitations under the License. */}} {{- if and .Values.manifests.job_ks_user .Values.deployment.rgw_keystone_user_and_endpoints }} -{{- $ksUserJob := dict "envAll" . "configMapBin" "ceph-bin-ks" "serviceName" "ceph" "serviceUser" "swift" -}} +{{- $ksUserJob := dict "envAll" . "configMapBin" "ceph-client-bin-ks" "serviceName" "ceph" "serviceUser" "swift" -}} {{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }} {{- end }} diff --git a/ceph/templates/job-namespace-client-key-cleaner.yaml b/ceph-client/templates/job-namespace-client-key-cleaner.yaml similarity index 96% rename from ceph/templates/job-namespace-client-key-cleaner.yaml rename to ceph-client/templates/job-namespace-client-key-cleaner.yaml index 3514e88403..7e6b37cccf 100644 --- a/ceph/templates/job-namespace-client-key-cleaner.yaml +++ b/ceph-client/templates/job-namespace-client-key-cleaner.yaml @@ -87,13 +87,13 @@ spec: command: - /tmp/provisioner-rbd-namespace-client-key-cleaner.sh volumeMounts: - - name: ceph-bin + - name: ceph-client-bin-provisioner mountPath: /tmp/provisioner-rbd-namespace-client-key-cleaner.sh subPath: provisioner-rbd-namespace-client-key-cleaner.sh readOnly: true volumes: - - name: ceph-bin + - name: ceph-client-bin-provisioner configMap: - name: ceph-bin-clients + name: ceph-client-bin-provisioner defaultMode: 0555 {{- end }} diff --git a/ceph/templates/job-namespace-client-key.yaml b/ceph-client/templates/job-namespace-client-key.yaml similarity index 96% rename from ceph/templates/job-namespace-client-key.yaml rename to ceph-client/templates/job-namespace-client-key.yaml index 124a0af717..bcb00fb16e 100644 --- a/ceph/templates/job-namespace-client-key.yaml +++ b/ceph-client/templates/job-namespace-client-key.yaml @@ -114,13 +114,13 @@ spec: command: - /tmp/provisioner-rbd-namespace-client-key-manager.sh volumeMounts: - - name: ceph-bin + - name: ceph-client-bin-provisioner mountPath: /tmp/provisioner-rbd-namespace-client-key-manager.sh subPath: provisioner-rbd-namespace-client-key-manager.sh readOnly: true volumes: - - name: ceph-bin + - name: ceph-client-bin-provisioner configMap: - name: ceph-bin-clients + name: ceph-client-bin-provisioner defaultMode: 0555 {{- end }} diff --git a/ceph/templates/job-rbd-pool.yaml b/ceph-client/templates/job-rbd-pool.yaml similarity index 95% rename from ceph/templates/job-rbd-pool.yaml rename to ceph-client/templates/job-rbd-pool.yaml index c8205220ae..1f4b210b39 100644 --- a/ceph/templates/job-rbd-pool.yaml +++ b/ceph-client/templates/job-rbd-pool.yaml @@ -51,11 +51,11 @@ spec: command: - /tmp/pool-init.sh volumeMounts: - - name: ceph-bin + - name: ceph-client-bin mountPath: /tmp/pool-init.sh subPath: pool-init.sh readOnly: true - - name: ceph-bin + - name: ceph-client-bin mountPath: /tmp/pool-calc.py subPath: pool-calc.py readOnly: true @@ -78,9 +78,9 @@ spec: configMap: name: ceph-etc defaultMode: 0444 - - name: ceph-bin + - name: ceph-client-bin configMap: - name: ceph-bin + name: ceph-client-bin defaultMode: 0555 - name: pod-var-lib-ceph emptyDir: {} diff --git a/ceph/templates/secret-keystone-rgw.yaml b/ceph-client/templates/secret-keystone-rgw.yaml similarity index 100% rename from ceph/templates/secret-keystone-rgw.yaml rename to ceph-client/templates/secret-keystone-rgw.yaml diff --git a/ceph/templates/secret-keystone.yaml b/ceph-client/templates/secret-keystone.yaml similarity index 100% rename from ceph/templates/secret-keystone.yaml rename to ceph-client/templates/secret-keystone.yaml diff --git a/ceph/templates/service-mgr.yaml b/ceph-client/templates/service-mgr.yaml similarity index 100% rename from ceph/templates/service-mgr.yaml rename to ceph-client/templates/service-mgr.yaml diff --git a/ceph/templates/service-rgw.yaml b/ceph-client/templates/service-rgw.yaml similarity index 100% rename from ceph/templates/service-rgw.yaml rename to ceph-client/templates/service-rgw.yaml diff --git a/ceph/templates/storageclass-cephfs.yaml b/ceph-client/templates/storageclass-cephfs.yaml similarity index 100% rename from ceph/templates/storageclass-cephfs.yaml rename to ceph-client/templates/storageclass-cephfs.yaml diff --git a/ceph/templates/storageclass-rbd.yaml b/ceph-client/templates/storageclass-rbd.yaml similarity index 100% rename from ceph/templates/storageclass-rbd.yaml rename to ceph-client/templates/storageclass-rbd.yaml diff --git a/ceph/values.yaml b/ceph-client/values.yaml similarity index 76% rename from ceph/values.yaml rename to ceph-client/values.yaml index 12d391450a..7707590341 100644 --- a/ceph/values.yaml +++ b/ceph-client/values.yaml @@ -14,8 +14,7 @@ deployment: ceph: true - storage_secrets: true - client_secrets: true + client_secrets: false rbd_provisioner: true cephfs_provisioner: true rgw_keystone_user_and_endpoints: false @@ -28,9 +27,6 @@ images: ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.1' ceph_mds: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' ceph_mgr: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' - ceph_mon: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' - ceph_mon_check: 'docker.io/port/ceph-config-helper:v1.10.1' - ceph_osd: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' ceph_rbd_pool: 'docker.io/port/ceph-config-helper:v1.10.1' ceph_rbd_provisioner: 'quay.io/external_storage/rbd-provisioner:v0.1.1' ceph_rgw: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' @@ -46,15 +42,9 @@ labels: provisioner: node_selector_key: openstack-control-plane node_selector_value: enabled - mon: - node_selector_key: ceph-mon - node_selector_value: enabled mds: node_selector_key: ceph-mds node_selector_value: enabled - osd: - node_selector_key: ceph-osd - node_selector_value: enabled rgw: node_selector_key: ceph-rgw node_selector_value: enabled @@ -66,7 +56,6 @@ pod: dns_policy: "ClusterFirstWithHostNet" replicas: rgw: 1 - mon_check: 1 rbd_provisioner: 2 cephfs_provisioner: 2 mgr: 1 @@ -78,13 +67,6 @@ pod: default: kubernetes.io/hostname resources: enabled: false - osd: - requests: - memory: "512Mi" - cpu: "500m" - limits: - memory: "1024Mi" - cpu: "1000m" mds: requests: memory: "10Mi" @@ -92,20 +74,6 @@ pod: limits: memory: "50Mi" cpu: "500m" - mon: - requests: - memory: "50Mi" - cpu: "250m" - limits: - memory: "100Mi" - cpu: "500m" - mon_check: - requests: - memory: "5Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" rgw: requests: memory: "5Mi" @@ -142,13 +110,6 @@ pod: requests: memory: "128Mi" cpu: "500m" - secret_provisioning: - limits: - memory: "1024Mi" - cpu: "2000m" - requests: - memory: "128Mi" - cpu: "500m" ks_endpoints: requests: memory: "128Mi" @@ -193,37 +154,6 @@ network: mgr: 7000 conf: - templates: - keyring: - admin: | - [client.admin] - key = {{ key }} - auid = 0 - caps mds = "allow" - caps mon = "allow *" - caps osd = "allow *" - caps mgr = "allow *" - mon: | - [mon.] - key = {{ key }} - caps mon = "allow *" - bootstrap: - mds: | - [client.bootstrap-mds] - key = {{ key }} - caps mon = "allow profile bootstrap-mds" - mgr: | - [client.bootstrap-mgr] - key = {{ key }} - caps mgr = "allow profile bootstrap-mgr" - osd: | - [client.bootstrap-osd] - key = {{ key }} - caps mon = "allow profile bootstrap-osd" - rgw: | - [client.bootstrap-rgw] - key = {{ key }} - caps mon = "allow profile bootstrap-rgw" features: mds: true rgw: true @@ -345,45 +275,6 @@ conf: osd_max_object_name_len: 256 ms_bind_port_min: 6800 ms_bind_port_max: 7100 - storage: - mon: - directory: /var/lib/openstack-helm/ceph/mon - # NOTE(portdirect): for homogeneous clusters the `osd` key can be used to - # define OSD pods that will be deployed across the cluster. - osd: - - data: - type: directory - location: /var/lib/openstack-helm/ceph/osd/osd-one - journal: - type: directory - location: /var/lib/openstack-helm/ceph/osd/journal-one - # - data: - # type: block-logical - # location: /dev/sde - # journal: - # type: block-logical - # location: /dev/sdf - # - data: - # type: block-logical - # location: /dev/sdg - # journal: - # type: directory - # location: /var/lib/openstack-helm/ceph/osd/journal-sdg -# NOTE(portdirect): for heterogeneous clusters the overrides section can be used to define -# OSD pods that will be deployed upon specifc nodes. -# overrides: -# ceph_osd: -# hosts: -# - name: host1.fqdn -# conf: -# storage: -# osd: -# - data: -# type: directory -# location: /var/lib/openstack-helm/ceph/osd/data-three -# journal: -# type: directory -# location: /var/lib/openstack-helm/ceph/osd/journal-three dependencies: static: @@ -400,8 +291,6 @@ dependencies: services: - endpoint: internal service: ceph_mon - job_keyring_generator: - jobs: null ks_endpoints: jobs: - ceph-ks-service @@ -431,28 +320,10 @@ dependencies: services: - endpoint: internal service: ceph_mon - mon: - jobs: - - ceph-storage-keys-generator - - ceph-mon-keyring-generator - moncheck: - jobs: - - ceph-storage-keys-generator - - ceph-mon-keyring-generator - services: - - endpoint: discovery - service: ceph_mon namespace_client_key_cleaner: jobs: null namespace_client_key_generator: jobs: null - osd: - jobs: - - ceph-storage-keys-generator - - ceph-osd-keyring-generator - services: - - endpoint: internal - service: ceph_mon rbd_pool: services: - endpoint: internal @@ -471,8 +342,6 @@ dependencies: services: - endpoint: internal service: ceph_mon - storage_keys_generator: - jobs: null bootstrap: enabled: false @@ -618,34 +487,26 @@ monitoring: port: 9283 manifests: - configmap_bin_clients: true - configmap_bin_ks: true configmap_bin: true + configmap_bin_ks: true + configmap_bin_common: true configmap_etc: true - configmap_templates: true - daemonset_mon: true - daemonset_osd: true deployment_mds: true - deployment_moncheck: true deployment_rbd_provisioner: true deployment_cephfs_provisioner: true deployment_rgw: true deployment_mgr: true - job_bootstrap: true + job_bootstrap: false job_cephfs_client_key: true - job_keyring: true job_ks_endpoints: true job_ks_service: true job_ks_user: true job_namespace_client_key_cleaner: true job_namespace_client_key: true job_rbd_pool: true - job_storage_admin_keys: true secret_keystone_rgw: true secret_keystone: true service_mgr: true - service_mon: true - service_mon_discovery: true service_rgw: true storageclass_cephfs: true storageclass_rbd: true diff --git a/ceph/Chart.yaml b/ceph-mon/Chart.yaml similarity index 92% rename from ceph/Chart.yaml rename to ceph-mon/Chart.yaml index 78f56d1c2b..ba425831b1 100644 --- a/ceph/Chart.yaml +++ b/ceph-mon/Chart.yaml @@ -13,6 +13,6 @@ # limitations under the License. apiVersion: v1 -description: OpenStack-Helm Ceph -name: ceph +description: OpenStack-Helm Ceph Mon +name: ceph-mon version: 0.1.0 diff --git a/ceph-mon/requirements.yaml b/ceph-mon/requirements.yaml new file mode 100644 index 0000000000..53782e69b2 --- /dev/null +++ b/ceph-mon/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/ceph-mon/templates/bin/_bootstrap.sh.tpl b/ceph-mon/templates/bin/_bootstrap.sh.tpl new file mode 100644 index 0000000000..533c0a5a3f --- /dev/null +++ b/ceph-mon/templates/bin/_bootstrap.sh.tpl @@ -0,0 +1,20 @@ +#!/bin/bash + +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{ .Values.bootstrap.script | default "echo 'Not Enabled'" }} diff --git a/ceph/templates/bin/_init-dirs.sh.tpl b/ceph-mon/templates/bin/_init-dirs.sh.tpl similarity index 100% rename from ceph/templates/bin/_init-dirs.sh.tpl rename to ceph-mon/templates/bin/_init-dirs.sh.tpl diff --git a/ceph/templates/bin/keys/_bootstrap-keyring-generator.py.tpl b/ceph-mon/templates/bin/keys/_bootstrap-keyring-generator.py.tpl similarity index 100% rename from ceph/templates/bin/keys/_bootstrap-keyring-generator.py.tpl rename to ceph-mon/templates/bin/keys/_bootstrap-keyring-generator.py.tpl diff --git a/ceph/templates/bin/keys/_bootstrap-keyring-manager.sh.tpl b/ceph-mon/templates/bin/keys/_bootstrap-keyring-manager.sh.tpl similarity index 100% rename from ceph/templates/bin/keys/_bootstrap-keyring-manager.sh.tpl rename to ceph-mon/templates/bin/keys/_bootstrap-keyring-manager.sh.tpl diff --git a/ceph/templates/bin/keys/_storage-keyring-manager.sh.tpl b/ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl similarity index 100% rename from ceph/templates/bin/keys/_storage-keyring-manager.sh.tpl rename to ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl diff --git a/ceph/templates/bin/mon/_check.sh.tpl b/ceph-mon/templates/bin/mon/_check.sh.tpl similarity index 100% rename from ceph/templates/bin/mon/_check.sh.tpl rename to ceph-mon/templates/bin/mon/_check.sh.tpl diff --git a/ceph/templates/bin/mon/_start.sh.tpl b/ceph-mon/templates/bin/mon/_start.sh.tpl similarity index 100% rename from ceph/templates/bin/mon/_start.sh.tpl rename to ceph-mon/templates/bin/mon/_start.sh.tpl diff --git a/ceph/templates/bin/mon/_stop.sh.tpl b/ceph-mon/templates/bin/mon/_stop.sh.tpl similarity index 100% rename from ceph/templates/bin/mon/_stop.sh.tpl rename to ceph-mon/templates/bin/mon/_stop.sh.tpl diff --git a/ceph/templates/bin/moncheck/_reap-zombies.py.tpl b/ceph-mon/templates/bin/moncheck/_reap-zombies.py.tpl similarity index 100% rename from ceph/templates/bin/moncheck/_reap-zombies.py.tpl rename to ceph-mon/templates/bin/moncheck/_reap-zombies.py.tpl diff --git a/ceph/templates/bin/moncheck/_start.sh.tpl b/ceph-mon/templates/bin/moncheck/_start.sh.tpl similarity index 100% rename from ceph/templates/bin/moncheck/_start.sh.tpl rename to ceph-mon/templates/bin/moncheck/_start.sh.tpl diff --git a/ceph-mon/templates/configmap-bin.yaml b/ceph-mon/templates/configmap-bin.yaml new file mode 100644 index 0000000000..7a17232726 --- /dev/null +++ b/ceph-mon/templates/configmap-bin.yaml @@ -0,0 +1,52 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.configmap_bin .Values.deployment.ceph }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-mon-bin +data: +{{- if .Values.bootstrap.enabled }} + bootstrap.sh: | +{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} + + init-dirs.sh: | +{{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + keys-bootstrap-keyring-generator.py: | +{{ tuple "bin/keys/_bootstrap-keyring-generator.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + keys-bootstrap-keyring-manager.sh: | +{{ tuple "bin/keys/_bootstrap-keyring-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + keys-storage-keyring-manager.sh: | +{{ tuple "bin/keys/_storage-keyring-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + mon-start.sh: | +{{ tuple "bin/mon/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + mon-stop.sh: | +{{ tuple "bin/mon/_stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + mon-check.sh: | +{{ tuple "bin/mon/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + moncheck-start.sh: | +{{ tuple "bin/moncheck/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + moncheck-reap-zombies.py: | +{{ tuple "bin/moncheck/_reap-zombies.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + +{{- end }} diff --git a/ceph-mon/templates/configmap-etc.yaml b/ceph-mon/templates/configmap-etc.yaml new file mode 100644 index 0000000000..f6030ac6a3 --- /dev/null +++ b/ceph-mon/templates/configmap-etc.yaml @@ -0,0 +1,55 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.configmap.etc" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} + +{{- if .Values.deployment.ceph }} + +{{- if empty .Values.conf.ceph.global.mon_host -}} +{{- $monHost := tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $monHost | set .Values.conf.ceph.global "mon_host" | quote | trunc 0 -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.global.fsid -}} +{{- uuidv4 | set .Values.conf.ceph.global "fsid" | quote | trunc 0 -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.cluster_network -}} +{{- .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" | quote | trunc 0 -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.public_network -}} +{{- .Values.network.public | set .Values.conf.ceph.osd "public_network" | quote | trunc 0 -}} +{{- end -}} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapName }} +data: + ceph.conf: | +{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }} + +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.manifests.configmap_etc }} +{{- list "ceph-mon-etc" . | include "ceph.configmap.etc" }} +{{- end }} diff --git a/ceph/templates/configmap-templates.yaml b/ceph-mon/templates/configmap-templates.yaml similarity index 100% rename from ceph/templates/configmap-templates.yaml rename to ceph-mon/templates/configmap-templates.yaml diff --git a/ceph/templates/daemonset-mon.yaml b/ceph-mon/templates/daemonset-mon.yaml similarity index 96% rename from ceph/templates/daemonset-mon.yaml rename to ceph-mon/templates/daemonset-mon.yaml index a0354cd4e2..7ae43f0a7c 100644 --- a/ceph/templates/daemonset-mon.yaml +++ b/ceph-mon/templates/daemonset-mon.yaml @@ -73,7 +73,7 @@ spec: - name: CLUSTER value: "ceph" volumeMounts: - - name: ceph-bin + - name: ceph-mon-bin mountPath: /tmp/init-dirs.sh subPath: init-dirs.sh readOnly: true @@ -140,19 +140,19 @@ spec: initialDelaySeconds: 60 periodSeconds: 60 volumeMounts: - - name: ceph-bin + - name: ceph-mon-bin mountPath: /tmp/mon-start.sh subPath: mon-start.sh readOnly: true - - name: ceph-bin + - name: ceph-mon-bin mountPath: /tmp/mon-stop.sh subPath: mon-stop.sh readOnly: true - - name: ceph-bin + - name: ceph-mon-bin mountPath: /tmp/mon-check.sh subPath: mon-check.sh readOnly: true - - name: ceph-etc + - name: ceph-mon-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -183,13 +183,13 @@ spec: mountPath: /run readOnly: false volumes: - - name: ceph-bin + - name: ceph-mon-bin configMap: - name: ceph-bin + name: ceph-mon-bin defaultMode: 0555 - - name: ceph-etc + - name: ceph-mon-etc configMap: - name: ceph-etc + name: ceph-mon-etc defaultMode: 0444 - name: pod-var-lib-ceph hostPath: diff --git a/ceph/templates/deployment-moncheck.yaml b/ceph-mon/templates/deployment-moncheck.yaml similarity index 94% rename from ceph/templates/deployment-moncheck.yaml rename to ceph-mon/templates/deployment-moncheck.yaml index 8c739608ff..576a0cfb48 100644 --- a/ceph/templates/deployment-moncheck.yaml +++ b/ceph-mon/templates/deployment-moncheck.yaml @@ -59,15 +59,15 @@ spec: ports: - containerPort: 6789 volumeMounts: - - name: ceph-bin + - name: ceph-mon-bin mountPath: /tmp/moncheck-start.sh subPath: moncheck-start.sh readOnly: true - - name: ceph-bin + - name: ceph-mon-bin mountPath: /tmp/moncheck-reap-zombies.py subPath: moncheck-reap-zombies.py readOnly: true - - name: ceph-etc + - name: ceph-mon-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -86,13 +86,13 @@ spec: mountPath: /run readOnly: false volumes: - - name: ceph-etc + - name: ceph-mon-etc configMap: - name: ceph-etc + name: ceph-mon-etc defaultMode: 0444 - - name: ceph-bin + - name: ceph-mon-bin configMap: - name: ceph-bin + name: ceph-mon-bin defaultMode: 0555 - name: pod-var-lib-ceph emptyDir: {} diff --git a/ceph/templates/job-bootstrap.yaml b/ceph-mon/templates/job-bootstrap.yaml similarity index 93% rename from ceph/templates/job-bootstrap.yaml rename to ceph-mon/templates/job-bootstrap.yaml index 63c46d1f04..1d8b02df2a 100644 --- a/ceph/templates/job-bootstrap.yaml +++ b/ceph-mon/templates/job-bootstrap.yaml @@ -45,11 +45,11 @@ spec: command: - /tmp/bootstrap.sh volumeMounts: - - name: ceph-bin + - name: ceph-mon-bin mountPath: /tmp/bootstrap.sh subPath: bootstrap.sh readOnly: true - - name: ceph-etc + - name: ceph-mon-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -58,13 +58,13 @@ spec: subPath: ceph.client.admin.keyring readOnly: true volumes: - - name: ceph-bin + - name: ceph-mon-bin configMap: - name: ceph-bin + name: ceph-mon-bin defaultMode: 0555 - - name: ceph-etc + - name: ceph-mon-etc configMap: - name: ceph-etc + name: ceph-mon-etc defaultMode: 0444 - name: ceph-client-admin-keyring secret: diff --git a/ceph/templates/job-keyring.yaml b/ceph-mon/templates/job-keyring.yaml similarity index 97% rename from ceph/templates/job-keyring.yaml rename to ceph-mon/templates/job-keyring.yaml index 54493e9a79..caf5aa6c15 100644 --- a/ceph/templates/job-keyring.yaml +++ b/ceph-mon/templates/job-keyring.yaml @@ -98,11 +98,11 @@ spec: command: - /tmp/keys-bootstrap-keyring-manager.sh volumeMounts: - - name: ceph-bin + - name: ceph-mon-bin mountPath: /tmp/keys-bootstrap-keyring-manager.sh subPath: keys-bootstrap-keyring-manager.sh readOnly: true - - name: ceph-bin + - name: ceph-mon-bin mountPath: /tmp/keys-bootstrap-keyring-generator.py subPath: keys-bootstrap-keyring-generator.py readOnly: true @@ -110,9 +110,9 @@ spec: mountPath: /tmp/templates readOnly: true volumes: - - name: ceph-bin + - name: ceph-mon-bin configMap: - name: ceph-bin + name: ceph-mon-bin defaultMode: 0555 - name: ceph-templates configMap: diff --git a/ceph/templates/job-storage-admin-keys.yaml b/ceph-mon/templates/job-storage-admin-keys.yaml similarity index 96% rename from ceph/templates/job-storage-admin-keys.yaml rename to ceph-mon/templates/job-storage-admin-keys.yaml index afec9f74be..1ebd3d4af7 100644 --- a/ceph/templates/job-storage-admin-keys.yaml +++ b/ceph-mon/templates/job-storage-admin-keys.yaml @@ -89,11 +89,11 @@ spec: command: - /tmp/keys-storage-keyring-manager.sh volumeMounts: - - name: ceph-bin + - name: ceph-mon-bin mountPath: /tmp/keys-storage-keyring-manager.sh subPath: keys-storage-keyring-manager.sh readOnly: true - - name: ceph-bin + - name: ceph-mon-bin mountPath: /tmp/keys-bootstrap-keyring-generator.py subPath: keys-bootstrap-keyring-generator.py readOnly: true @@ -101,9 +101,9 @@ spec: mountPath: /tmp/templates readOnly: true volumes: - - name: ceph-bin + - name: ceph-mon-bin configMap: - name: ceph-bin + name: ceph-mon-bin defaultMode: 0555 - name: ceph-templates configMap: diff --git a/ceph/templates/service-mon-discovery.yaml b/ceph-mon/templates/service-mon-discovery.yaml similarity index 100% rename from ceph/templates/service-mon-discovery.yaml rename to ceph-mon/templates/service-mon-discovery.yaml diff --git a/ceph/templates/service-mon.yaml b/ceph-mon/templates/service-mon.yaml similarity index 100% rename from ceph/templates/service-mon.yaml rename to ceph-mon/templates/service-mon.yaml diff --git a/ceph-mon/values.yaml b/ceph-mon/values.yaml new file mode 100644 index 0000000000..e295f6f371 --- /dev/null +++ b/ceph-mon/values.yaml @@ -0,0 +1,228 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +deployment: + ceph: true + storage_secrets: true + +images: + pull_policy: IfNotPresent + tags: + ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' + ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.1' + ceph_mon: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' + ceph_mon_check: 'docker.io/port/ceph-config-helper:v1.10.1' + dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.0' + +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + mon: + node_selector_key: ceph-mon + node_selector_value: enabled + +pod: + dns_policy: "ClusterFirstWithHostNet" + replicas: + mon_check: 1 + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + resources: + enabled: false + mon: + requests: + memory: "50Mi" + cpu: "250m" + limits: + memory: "100Mi" + cpu: "500m" + mon_check: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + jobs: + bootstrap: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + secret_provisioning: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + +secrets: + keyrings: + mon: ceph-mon-keyring + mds: ceph-bootstrap-mds-keyring + osd: ceph-bootstrap-osd-keyring + rgw: ceph-bootstrap-rgw-keyring + mgr: ceph-bootstrap-mgr-keyring + admin: ceph-client-admin-keyring + +network: + public: 192.168.0.0/16 + cluster: 192.168.0.0/16 + port: + mon: 6789 + +conf: + templates: + keyring: + admin: | + [client.admin] + key = {{ key }} + auid = 0 + caps mds = "allow" + caps mon = "allow *" + caps osd = "allow *" + caps mgr = "allow *" + mon: | + [mon.] + key = {{ key }} + caps mon = "allow *" + bootstrap: + mds: | + [client.bootstrap-mds] + key = {{ key }} + caps mon = "allow profile bootstrap-mds" + mgr: | + [client.bootstrap-mgr] + key = {{ key }} + caps mgr = "allow profile bootstrap-mgr" + osd: | + [client.bootstrap-osd] + key = {{ key }} + caps mon = "allow profile bootstrap-osd" + rgw: | + [client.bootstrap-rgw] + key = {{ key }} + caps mon = "allow profile bootstrap-rgw" + ceph: + global: + # auth + cephx: true + cephx_require_signatures: false + cephx_cluster_require_signatures: true + cephx_service_require_signatures: false + osd: + osd_mkfs_type: xfs + osd_mkfs_options_xfs: -f -i size=2048 + osd_max_object_name_len: 256 + ms_bind_port_min: 6800 + ms_bind_port_max: 7100 + storage: + mon: + directory: /var/lib/openstack-helm/ceph/mon + +dependencies: + static: + bootstrap: + jobs: null + services: + - endpoint: internal + service: ceph_mon + job_keyring_generator: + jobs: null + mon: + jobs: + - ceph-storage-keys-generator + - ceph-mon-keyring-generator + moncheck: + jobs: + - ceph-storage-keys-generator + - ceph-mon-keyring-generator + services: + - endpoint: discovery + service: ceph_mon + storage_keys_generator: + jobs: null + +bootstrap: + enabled: false + script: | + ceph -s + function ensure_pool () { + ceph osd pool stats $1 || ceph osd pool create $1 $2 + local test_luminous=$(ceph tell osd.* version | egrep -c "12.2|luminous") + if [[ ${test_luminous} -gt 0 ]]; then + ceph osd pool application enable $1 $3 + fi + } + #ensure_pool volumes 8 cinder + +# if you change provision_storage_class to false +# it is presumed you manage your own storage +# class definition externally +storageclass: + rbd: + provision_storage_class: true + provisioner: ceph.com/rbd + name: general + monitors: null + pool: rbd + admin_id: admin + admin_secret_name: pvc-ceph-conf-combined-storageclass + admin_secret_namespace: ceph + user_id: admin + user_secret_name: pvc-ceph-client-key + image_format: "2" + image_features: layering + cephfs: + provision_storage_class: true + provisioner: ceph.com/cephfs + name: cephfs + admin_id: admin + user_secret_name: pvc-ceph-cephfs-client-key + admin_secret_name: pvc-ceph-conf-combined-storageclass + admin_secret_namespace: ceph + +endpoints: + cluster_domain_suffix: cluster.local + ceph_mon: + namespace: null + hosts: + default: ceph-mon + discovery: ceph-mon-discovery + host_fqdn_override: + default: null + port: + mon: + default: 6789 + +manifests: + configmap_bin: true + configmap_etc: true + configmap_templates: true + daemonset_mon: true + deployment_moncheck: true + deployment_rgw: true + job_bootstrap: true + job_keyring: true + service_mon: true + service_mon_discovery: true + job_storage_admin_keys: true diff --git a/ceph-osd/Chart.yaml b/ceph-osd/Chart.yaml new file mode 100644 index 0000000000..b2e5376c57 --- /dev/null +++ b/ceph-osd/Chart.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Ceph OSD +name: ceph-osd +version: 0.1.0 diff --git a/ceph-osd/requirements.yaml b/ceph-osd/requirements.yaml new file mode 100644 index 0000000000..53782e69b2 --- /dev/null +++ b/ceph-osd/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/ceph-osd/templates/bin/_init-dirs.sh.tpl b/ceph-osd/templates/bin/_init-dirs.sh.tpl new file mode 100644 index 0000000000..b15731c4ae --- /dev/null +++ b/ceph-osd/templates/bin/_init-dirs.sh.tpl @@ -0,0 +1,34 @@ +#!/bin/bash + +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export LC_ALL=C +: "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}" + +mkdir -p "$(dirname "${OSD_BOOTSTRAP_KEYRING}")" + +# Let's create the ceph directories +for DIRECTORY in osd tmp; do + mkdir -p "/var/lib/ceph/${DIRECTORY}" +done + +# Create socket directory +mkdir -p /run/ceph + +# Adjust the owner of all those directories +chown -R ceph. /run/ceph/ /var/lib/ceph/* diff --git a/ceph/templates/bin/osd/_block.sh.tpl b/ceph-osd/templates/bin/osd/_block.sh.tpl similarity index 100% rename from ceph/templates/bin/osd/_block.sh.tpl rename to ceph-osd/templates/bin/osd/_block.sh.tpl diff --git a/ceph/templates/bin/osd/_check.sh.tpl b/ceph-osd/templates/bin/osd/_check.sh.tpl similarity index 100% rename from ceph/templates/bin/osd/_check.sh.tpl rename to ceph-osd/templates/bin/osd/_check.sh.tpl diff --git a/ceph/templates/bin/osd/_directory.sh.tpl b/ceph-osd/templates/bin/osd/_directory.sh.tpl similarity index 100% rename from ceph/templates/bin/osd/_directory.sh.tpl rename to ceph-osd/templates/bin/osd/_directory.sh.tpl diff --git a/ceph/templates/bin/osd/_init.sh.tpl b/ceph-osd/templates/bin/osd/_init.sh.tpl similarity index 100% rename from ceph/templates/bin/osd/_init.sh.tpl rename to ceph-osd/templates/bin/osd/_init.sh.tpl diff --git a/ceph/templates/bin/osd/_start.sh.tpl b/ceph-osd/templates/bin/osd/_start.sh.tpl similarity index 100% rename from ceph/templates/bin/osd/_start.sh.tpl rename to ceph-osd/templates/bin/osd/_start.sh.tpl diff --git a/ceph/templates/bin/osd/_stop.sh.tpl b/ceph-osd/templates/bin/osd/_stop.sh.tpl similarity index 100% rename from ceph/templates/bin/osd/_stop.sh.tpl rename to ceph-osd/templates/bin/osd/_stop.sh.tpl diff --git a/ceph-osd/templates/configmap-bin.yaml b/ceph-osd/templates/configmap-bin.yaml new file mode 100644 index 0000000000..f4bc33a5b8 --- /dev/null +++ b/ceph-osd/templates/configmap-bin.yaml @@ -0,0 +1,39 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-osd-bin +data: + osd-start.sh: | +{{ tuple "bin/osd/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-directory.sh: | +{{ tuple "bin/osd/_directory.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-block.sh: | +{{ tuple "bin/osd/_block.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-init.sh: | +{{ tuple "bin/osd/_init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-check.sh: | +{{ tuple "bin/osd/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-stop.sh: | +{{ tuple "bin/osd/_stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + init-dirs.sh: | +{{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/ceph-osd/templates/configmap-etc.yaml b/ceph-osd/templates/configmap-etc.yaml new file mode 100644 index 0000000000..dfccaf470d --- /dev/null +++ b/ceph-osd/templates/configmap-etc.yaml @@ -0,0 +1,52 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.osd.configmap.etc" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} + +{{- if empty .Values.conf.ceph.global.mon_host -}} +{{- $monHost := tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $monHost | set .Values.conf.ceph.global "mon_host" | quote | trunc 0 -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.global.fsid -}} +{{- uuidv4 | set .Values.conf.ceph.global "fsid" | quote | trunc 0 -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.cluster_network -}} +{{- .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" | quote | trunc 0 -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.public_network -}} +{{- .Values.network.public | set .Values.conf.ceph.osd "public_network" | quote | trunc 0 -}} +{{- end -}} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapName }} +data: + ceph.conf: | +{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }} + +{{- end }} +{{- end }} +{{- if .Values.manifests.configmap_etc }} +{{- list "ceph-osd-etc" . | include "ceph.osd.configmap.etc" }} +{{- end }} diff --git a/ceph/templates/daemonset-osd.yaml b/ceph-osd/templates/daemonset-osd.yaml similarity index 94% rename from ceph/templates/daemonset-osd.yaml rename to ceph-osd/templates/daemonset-osd.yaml index 33e174adc5..558313c7d7 100644 --- a/ceph/templates/daemonset-osd.yaml +++ b/ceph-osd/templates/daemonset-osd.yaml @@ -60,7 +60,7 @@ spec: - name: CLUSTER value: "ceph" volumeMounts: - - name: ceph-bin + - name: ceph-osd-bin mountPath: /tmp/init-dirs.sh subPath: init-dirs.sh readOnly: true @@ -94,11 +94,11 @@ spec: command: - /tmp/osd-init.sh volumeMounts: - - name: ceph-bin + - name: ceph-osd-bin mountPath: /tmp/osd-init.sh subPath: osd-init.sh readOnly: true - - name: ceph-etc + - name: ceph-osd-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -168,27 +168,27 @@ spec: initialDelaySeconds: 60 periodSeconds: 60 volumeMounts: - - name: ceph-bin + - name: ceph-osd-bin mountPath: /tmp/osd-start.sh subPath: osd-start.sh readOnly: true - - name: ceph-bin + - name: ceph-osd-bin mountPath: /tmp/osd-directory.sh subPath: osd-directory.sh readOnly: true - - name: ceph-bin + - name: ceph-osd-bin mountPath: /tmp/osd-block.sh subPath: osd-block.sh readOnly: true - - name: ceph-bin + - name: ceph-osd-bin mountPath: /tmp/osd-check.sh subPath: osd-check.sh readOnly: true - - name: ceph-bin + - name: ceph-osd-bin mountPath: /tmp/osd-stop.sh subPath: osd-stop.sh readOnly: true - - name: ceph-etc + - name: ceph-osd-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -226,11 +226,11 @@ spec: - name: pod-run emptyDir: medium: "Memory" - - name: ceph-bin + - name: ceph-osd-bin configMap: - name: ceph-bin + name: ceph-osd-bin defaultMode: 0555 - - name: ceph-etc + - name: ceph-osd-etc configMap: name: {{ $configMapName }} defaultMode: 0444 @@ -249,13 +249,13 @@ spec: {{- end }} {{- end }} -{{- if and .Values.manifests.daemonset_osd .Values.deployment.ceph }} +{{- if .Values.manifests.daemonset_osd }} {{- $daemonset := "osd" }} -{{- $configMapName := "ceph-etc" }} +{{- $configMapName := "ceph-osd-etc" }} {{- $serviceAccountName := "ceph-osd"}} {{- $dependencies := .Values.dependencies.static.osd }} {{ tuple . $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} {{- $daemonset_yaml := list $daemonset $configMapName $serviceAccountName $dependencies . | include "ceph.osd.daemonset" | toString | fromYaml }} -{{- $configmap_yaml := "ceph.configmap.etc" }} +{{- $configmap_yaml := "ceph.osd.configmap.etc" }} {{- list $daemonset $daemonset_yaml $configmap_yaml $configMapName . | include "ceph.utils.osd_daemonset_overrides" }} {{- end }} diff --git a/ceph/templates/utils/_osd_daemonset_overrides.tpl b/ceph-osd/templates/utils/_osd_daemonset_overrides.tpl similarity index 100% rename from ceph/templates/utils/_osd_daemonset_overrides.tpl rename to ceph-osd/templates/utils/_osd_daemonset_overrides.tpl diff --git a/ceph-osd/values.yaml b/ceph-osd/values.yaml new file mode 100644 index 0000000000..4905d31530 --- /dev/null +++ b/ceph-osd/values.yaml @@ -0,0 +1,150 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +images: + pull_policy: IfNotPresent + tags: + ceph_osd: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' + ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' + dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.0' + +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + osd: + node_selector_key: ceph-osd + node_selector_value: enabled + +pod: + dns_policy: "ClusterFirstWithHostNet" + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + resources: + enabled: false + osd: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1024Mi" + cpu: "1000m" +secrets: + keyrings: + osd: ceph-bootstrap-osd-keyring + +network: + public: 192.168.0.0/16 + cluster: 192.168.0.0/16 + port: + mon: 6789 + rgw: 8088 + mgr: 7000 + +conf: + ceph: + global: + # auth + cephx: true + cephx_require_signatures: false + cephx_cluster_require_signatures: true + cephx_service_require_signatures: false + osd: + osd_mkfs_type: xfs + osd_mkfs_options_xfs: -f -i size=2048 + osd_max_object_name_len: 256 + ms_bind_port_min: 6800 + ms_bind_port_max: 7100 + storage: + # NOTE(portdirect): for homogeneous clusters the `osd` key can be used to + # define OSD pods that will be deployed across the cluster. + osd: + - data: + type: directory + location: /var/lib/openstack-helm/ceph/osd/osd-one + journal: + type: directory + location: /var/lib/openstack-helm/ceph/osd/journal-one + # - data: + # type: block-logical + # location: /dev/sde + # journal: + # type: block-logical + # location: /dev/sdf + # - data: + # type: block-logical + # location: /dev/sdg + # journal: + # type: directory + # location: /var/lib/openstack-helm/ceph/osd/journal-sdg +# NOTE(portdirect): for heterogeneous clusters the overrides section can be used to define +# OSD pods that will be deployed upon specifc nodes. +# overrides: +# ceph_osd: +# hosts: +# - name: host1.fqdn +# conf: +# storage: +# osd: +# - data: +# type: directory +# location: /var/lib/openstack-helm/ceph/osd/data-three +# journal: +# type: directory +# location: /var/lib/openstack-helm/ceph/osd/journal-three + +dependencies: + static: + osd: + jobs: + - ceph-storage-keys-generator + - ceph-osd-keyring-generator + services: + - endpoint: internal + service: ceph_mon + +bootstrap: + enabled: false + script: | + ceph -s + function ensure_pool () { + ceph osd pool stats $1 || ceph osd pool create $1 $2 + local test_luminous=$(ceph tell osd.* version | egrep -c "12.2|luminous") + if [[ ${test_luminous} -gt 0 ]]; then + ceph osd pool application enable $1 $3 + fi + } + #ensure_pool volumes 8 cinder + +endpoints: + cluster_domain_suffix: cluster.local + ceph_mon: + namespace: null + hosts: + default: ceph-mon + discovery: ceph-mon-discovery + host_fqdn_override: + default: null + port: + mon: + default: 6789 + +manifests: + configmap_bin: true + configmap_etc: true + daemonset_osd: true diff --git a/ceph/.gitignore b/ceph/.gitignore deleted file mode 100644 index ce71aabd5c..0000000000 --- a/ceph/.gitignore +++ /dev/null @@ -1 +0,0 @@ -secrets/* diff --git a/ceph/.helmignore b/ceph/.helmignore deleted file mode 100644 index 6fcf34ee21..0000000000 --- a/ceph/.helmignore +++ /dev/null @@ -1,26 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj - -secrets/ -patches/ -*.py -Makefile \ No newline at end of file diff --git a/tools/deployment/armada/multinode/armada-ceph.yaml b/tools/deployment/armada/multinode/armada-ceph.yaml index ed8cc27565..50abcff192 100644 --- a/tools/deployment/armada/multinode/armada-ceph.yaml +++ b/tools/deployment/armada/multinode/armada-ceph.yaml @@ -19,10 +19,10 @@ data: schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 - name: ceph + name: ceph-mon data: - chart_name: ceph - release: ceph + chart_name: ceph-mon + release: ceph-mon namespace: ceph timeout: 1800 install: @@ -33,39 +33,35 @@ data: delete: - type: job labels: - application: ceph + application: ceph-mon component: bootstrap - type: job labels: - application: ceph + application: ceph-mon component: cephfs-client-key-generator - type: job labels: - application: ceph + application: ceph-mon component: mds-keyring-generator - type: job labels: - application: ceph + application: ceph-mon component: osd-keyring-generator - type: job labels: - application: ceph + application: ceph-mon component: rgw-keyring-generator - type: job labels: - application: ceph + application: ceph-mon component: mon-keyring-generator - type: job labels: - application: ceph + application: ceph-mon component: mgr-keyring-generator - type: job labels: - application: ceph - component: rbd-pool - - type: job - labels: - application: ceph + application: ceph-mon component: storage-keys-generator values: endpoints: @@ -114,7 +110,151 @@ data: source: type: local location: ${OSH_PATH} - subpath: ceph + subpath: ceph-mon + reference: master + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: ceph-osd +data: + chart_name: ceph-osd + release: ceph-osd + namespace: ceph + timeout: 1800 + install: + no_hooks: false + upgrade: + no_hooks: false + values: + endpoints: + identity: + namespace: openstack + object_store: + namespace: ceph + ceph_mon: + namespace: ceph + network: + public: ${CEPH_NETWORK} + cluster: ${CEPH_NETWORK} + deployment: + storage_secrets: true + ceph: true + rbd_provisioner: true + cephfs_provisioner: true + client_secrets: false + rgw_keystone_user_and_endpoints: false + bootstrap: + enabled: true + conf: + ceph: + global: + #NOTE (portdirect): any uuidv4 value is acceptable here, though must + # be consistent if you wish to redploy whist maintaining data. + fsid: C1150D08-707D-4FEB-85E7-692BD9206DC3 + rgw_ks: + enabled: true + pool: + crush: + #NOTE(portdirect): set to hammer if using kernel < 4.5 + tunables: null + target: + # NOTE(portdirect): 5 nodes, with one osd per node + osd: 5 + pg_per_osd: 100 + storage: + osd: + - data: + type: directory + location: /var/lib/openstack-helm/ceph/osd/osd-one + journal: + type: directory + location: /var/lib/openstack-helm/ceph/osd/journal-one + source: + type: local + location: ${OSH_PATH} + subpath: ceph-osd + reference: master + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: ceph-client +data: + chart_name: ceph-client + release: ceph-client + namespace: ceph + timeout: 1800 + install: + no_hooks: false + upgrade: + no_hooks: false + pre: + delete: + - type: job + labels: + application: ceph-client + component: bootstrap + - type: job + labels: + application: ceph-client + component: cephfs-client-key-generator + - type: job + labels: + application: ceph-client + component: rbd-pool + values: + endpoints: + identity: + namespace: openstack + object_store: + namespace: ceph + ceph_mon: + namespace: ceph + network: + public: ${CEPH_NETWORK} + cluster: ${CEPH_NETWORK} + deployment: + storage_secrets: true + ceph: true + rbd_provisioner: true + cephfs_provisioner: true + client_secrets: false + rgw_keystone_user_and_endpoints: false + bootstrap: + enabled: true + conf: + ceph: + global: + #NOTE (portdirect): any uuidv4 value is acceptable here, though must + # be consistent if you wish to redploy whist maintaining data. + fsid: C1150D08-707D-4FEB-85E7-692BD9206DC3 + rgw_ks: + enabled: true + pool: + crush: + #NOTE(portdirect): set to hammer if using kernel < 4.5 + tunables: null + target: + # NOTE(portdirect): 5 nodes, with one osd per node + osd: 5 + pg_per_osd: 100 + storage: + osd: + - data: + type: directory + location: /var/lib/openstack-helm/ceph/osd/osd-one + journal: + type: directory + location: /var/lib/openstack-helm/ceph/osd/journal-one + source: + type: local + location: ${OSH_PATH} + subpath: ceph-client reference: master dependencies: - helm-toolkit @@ -127,7 +267,9 @@ data: description: "Ceph Storage" sequenced: True chart_group: - - ceph + - ceph-mon + - ceph-osd + - ceph-client --- schema: armada/Manifest/v1 metadata: diff --git a/tools/deployment/armada/multinode/armada-lma.yaml b/tools/deployment/armada/multinode/armada-lma.yaml index 1609693e8c..ca5c945e8e 100644 --- a/tools/deployment/armada/multinode/armada-lma.yaml +++ b/tools/deployment/armada/multinode/armada-lma.yaml @@ -105,7 +105,7 @@ data: source: type: local location: ${OSH_PATH} - subpath: ceph + subpath: ceph-client reference: master dependencies: - helm-toolkit diff --git a/tools/deployment/armada/multinode/armada-osh.yaml b/tools/deployment/armada/multinode/armada-osh.yaml index 0abb7573f7..5660ae575a 100644 --- a/tools/deployment/armada/multinode/armada-osh.yaml +++ b/tools/deployment/armada/multinode/armada-osh.yaml @@ -120,7 +120,7 @@ data: source: type: local location: ${OSH_PATH} - subpath: ceph + subpath: ceph-client reference: master dependencies: - helm-toolkit @@ -383,7 +383,7 @@ data: source: type: local location: ${OSH_PATH} - subpath: ceph + subpath: ceph-client reference: master dependencies: - helm-toolkit diff --git a/tools/deployment/baremetal/030-ceph.sh b/tools/deployment/baremetal/030-ceph.sh index d3053bdd36..bef54dc9c1 100755 --- a/tools/deployment/baremetal/030-ceph.sh +++ b/tools/deployment/baremetal/030-ceph.sh @@ -17,7 +17,9 @@ set -xe #NOTE: Pull images and lint chart -make pull-images ceph +for CHART in ceph-mon ceph-osd ceph-client; do + make pull-images "${CHART}" +done #NOTE: Deploy command uuidgen > /tmp/ceph-fs-uuid.txt @@ -165,18 +167,21 @@ conf: type: directory location: /var/lib/openstack-helm/ceph/osd/journal-one EOF -helm install ./ceph \ - --namespace=ceph \ - --name=ceph \ - --values=/tmp/ceph.yaml -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh ceph +for CHART in ceph-mon ceph-osd ceph-client; do + helm install ./${CHART} \ + --namespace=ceph \ + --name=${CHART} \ + --values=/tmp/ceph.yaml -#NOTE: Validate deploy -MON_POD=$(kubectl get pods \ - --namespace=ceph \ - --selector="application=ceph" \ - --selector="component=mon" \ - --no-headers | awk '{ print $1; exit }') -kubectl exec -n ceph ${MON_POD} -- ceph -s + #NOTE: Wait for deploy + ./tools/deployment/common/wait-for-pods.sh ceph + + #NOTE: Validate deploy + MON_POD=$(kubectl get pods \ + --namespace=ceph \ + --selector="application=ceph" \ + --selector="component=mon" \ + --no-headers | awk '{ print $1; exit }') + kubectl exec -n ceph ${MON_POD} -- ceph -s +done diff --git a/tools/deployment/baremetal/035-ceph-ns-activate.sh b/tools/deployment/baremetal/035-ceph-ns-activate.sh index 0270432f84..fc6dbe203a 100755 --- a/tools/deployment/baremetal/035-ceph-ns-activate.sh +++ b/tools/deployment/baremetal/035-ceph-ns-activate.sh @@ -17,7 +17,7 @@ set -xe #NOTE: Pull images and lint chart -make pull-images ceph +make pull-images ceph-client #NOTE: Deploy command tee /tmp/ceph-openstack-config.yaml <