From 8ef5d946740bacb2af361761500e2204c365d5c7 Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Wed, 14 Jun 2017 13:57:21 -0500 Subject: [PATCH] Refactor Ceph secret generation This PS refactors the ceph chart and secret generation process. The updated chart replaces the existing "bootstrap" chart. Additionally, Ceph manifests and deployment guides were modified accordingly. Change-Id: I6f5bb88fc0f40cfee8865d9dab83859d765e7537 Co-Authored-By: Larry Rensing --- Makefile | 4 +- bootstrap/.gitignore | 3 - bootstrap/.helmignore | 27 --- bootstrap/Chart.yaml | 18 -- bootstrap/requirements.yaml | 18 -- bootstrap/values.yaml | 18 -- ceph/templates/bin/_ceph-key.py.tpl | 14 ++ ceph/templates/bin/_ceph-key.sh.tpl | 37 +++ .../bin/_ceph-namespace-client-key.sh.tpl | 22 ++ ceph/templates/bin/_ceph-storage-key.sh.tpl | 62 +++++ ceph/templates/configmap-bin.yaml | 31 +++ .../templates/configmap-etc.yaml | 25 +- ceph/templates/configmap-templates.yaml | 31 +++ ceph/templates/daemonset-osd.yaml | 124 ++++++---- ceph/templates/deployment-mds.yaml | 85 ++++--- ceph/templates/deployment-moncheck.yaml | 82 +++++-- ceph/templates/deployment-rgw.yaml | 84 ++++--- ceph/templates/etc/_ceph.conf.tpl | 77 +++++++ ceph/templates/job-keyring.yaml | 85 +++++++ ceph/templates/job-namespace-client-key.yaml | 61 +++++ ceph/templates/job-storage-admin-keys.yaml | 77 +++++++ ceph/templates/pdb-mon.yaml | 4 +- ceph/templates/secrets.yaml | 65 ------ ceph/templates/service-rgw.yaml | 4 +- ceph/templates/service.yaml | 4 +- ceph/templates/statefulset-mon.yaml | 101 +++++--- ceph/templates/storage.yaml | 10 +- ceph/templates/templates/_admin.keyring.tpl | 6 + .../templates/_bootstrap.keyring.mds.tpl | 3 + .../templates/_bootstrap.keyring.osd.tpl | 3 + .../templates/_bootstrap.keyring.rgw.tpl | 3 + ceph/templates/templates/_mon.keyring.tpl | 3 + ceph/values.yaml | 170 +++++++++++--- cinder/templates/_funcs.tpl | 9 +- .../_ceph-keyring.sh.tpl} | 11 +- cinder/templates/configmap-bin.yaml | 2 + cinder/templates/configmap-etc.yaml | 13 -- cinder/templates/deployment-backup.yaml | 41 +++- cinder/templates/deployment-volume.yaml | 39 +++- cinder/templates/etc/_ceph.conf.tpl | 30 --- cinder/values.yaml | 2 + doc/source/install/multinode.rst | 217 ++++++------------ .../_ceph-keyring.sh.tpl} | 11 +- glance/templates/configmap-bin.yaml | 2 + glance/templates/configmap-etc.yaml | 24 -- glance/templates/deployment-api.yaml | 60 ++++- glance/templates/etc/_ceph.conf.tpl | 30 --- helm-toolkit/secrets/.gitkeep | 0 .../utils/secret-generator/README.rst | 78 ------- .../secret-generator/generate_secrets.sh | 96 -------- .../templates/ceph/admin.keyring.tmpl | 20 -- .../templates/ceph/bootstrap.keyring.tmpl | 17 -- .../templates/ceph/ceph.conf.tmpl | 85 ------- .../templates/ceph/mon.keyring.tmpl | 17 -- helm-toolkit/utils/test/ceph-rbd-test.yaml | 40 ---- .../_ceph-keyring.sh.tpl} | 11 +- nova/templates/configmap-bin.yaml | 4 +- nova/templates/configmap-etc.yaml | 2 - nova/templates/daemonset-compute.yaml | 39 +++- nova/templates/daemonset-libvirt.yaml | 37 ++- nova/templates/etc/_ceph.conf.tpl | 32 --- nova/values.yaml | 1 - tests/pvc-test.yaml | 35 ++- tools/gate/README.rst | 1 + tools/gate/basic_launch.sh | 50 +++- tools/gate/dump_logs.sh | 10 + tools/gate/funcs/common.sh | 14 ++ tools/gate/funcs/kube.sh | 6 + tools/gate/funcs/network.sh | 1 + tools/gate/kubeadm_aio.sh | 6 + tools/gate/provision_gate_worker_node.sh | 7 + tools/gate/setup_gate.sh | 16 +- tools/gate/setup_gate_worker_nodes.sh | 1 + tools/images/ceph-config-helper/Dockerfile | 20 ++ tools/images/ceph-config-helper/README.rst | 39 ++++ tools/kubeadm-aio/Dockerfile | 3 +- tools/kubeadm-aio/README.rst | 2 +- tools/kubeadm-aio/assets/usr/bin/kubelet | 3 +- tools/kubeadm-aio/kubeadm-aio-launcher.sh | 1 + .../overrides/mvp/cinder.yaml | 30 +-- tools/overrides/mvp/nova.yaml | 3 + 81 files changed, 1518 insertions(+), 1061 deletions(-) delete mode 100644 bootstrap/.gitignore delete mode 100644 bootstrap/.helmignore delete mode 100644 bootstrap/Chart.yaml delete mode 100644 bootstrap/requirements.yaml delete mode 100644 bootstrap/values.yaml create mode 100644 ceph/templates/bin/_ceph-key.py.tpl create mode 100644 ceph/templates/bin/_ceph-key.sh.tpl create mode 100644 ceph/templates/bin/_ceph-namespace-client-key.sh.tpl create mode 100644 ceph/templates/bin/_ceph-storage-key.sh.tpl create mode 100644 ceph/templates/configmap-bin.yaml rename bootstrap/templates/secrets.yaml => ceph/templates/configmap-etc.yaml (53%) create mode 100644 ceph/templates/configmap-templates.yaml create mode 100644 ceph/templates/etc/_ceph.conf.tpl create mode 100644 ceph/templates/job-keyring.yaml create mode 100644 ceph/templates/job-namespace-client-key.yaml create mode 100644 ceph/templates/job-storage-admin-keys.yaml delete mode 100644 ceph/templates/secrets.yaml create mode 100644 ceph/templates/templates/_admin.keyring.tpl create mode 100644 ceph/templates/templates/_bootstrap.keyring.mds.tpl create mode 100644 ceph/templates/templates/_bootstrap.keyring.osd.tpl create mode 100644 ceph/templates/templates/_bootstrap.keyring.rgw.tpl create mode 100644 ceph/templates/templates/_mon.keyring.tpl rename cinder/templates/{etc/_ceph-cinder.keyring.tpl => bin/_ceph-keyring.sh.tpl} (81%) delete mode 100644 cinder/templates/etc/_ceph.conf.tpl rename glance/templates/{etc/_ceph.client.glance.keyring.tpl => bin/_ceph-keyring.sh.tpl} (79%) delete mode 100644 glance/templates/etc/_ceph.conf.tpl delete mode 100644 helm-toolkit/secrets/.gitkeep delete mode 100644 helm-toolkit/utils/secret-generator/README.rst delete mode 100755 helm-toolkit/utils/secret-generator/generate_secrets.sh delete mode 100644 helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl delete mode 100644 helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl delete mode 100644 helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl delete mode 100644 helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl delete mode 100644 helm-toolkit/utils/test/ceph-rbd-test.yaml rename nova/templates/{etc/_ceph.client.cinder.keyring.yaml.tpl => bin/_ceph-keyring.sh.tpl} (85%) delete mode 100644 nova/templates/etc/_ceph.conf.tpl create mode 100644 tools/images/ceph-config-helper/Dockerfile create mode 100644 tools/images/ceph-config-helper/README.rst rename helm-toolkit/utils/secret-generator/ceph-key.py => tools/overrides/mvp/cinder.yaml (58%) diff --git a/Makefile b/Makefile index 391c9aac68..91d8233a10 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ HELM = helm TASK = build -CHARTS = helm-toolkit bootstrap ceph mariadb etcd rabbitmq memcached -CHARTS += keystone glance cinder horizon neutron nova heat +CHARTS = helm-toolkit ceph mariadb etcd rabbitmq +CHARTS += memcached keystone glance cinder horizon neutron nova heat CHARTS += barbican mistral senlin magnum ingress all: $(CHARTS) diff --git a/bootstrap/.gitignore b/bootstrap/.gitignore deleted file mode 100644 index e1bd7e85af..0000000000 --- a/bootstrap/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -secrets/* -!secrets/.gitkeep -templates/_secrets.tpl diff --git a/bootstrap/.helmignore b/bootstrap/.helmignore deleted file mode 100644 index e8ef5ffab2..0000000000 --- a/bootstrap/.helmignore +++ /dev/null @@ -1,27 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj - -bin/ -etc/ -patches/ -*.py -Makefile diff --git a/bootstrap/Chart.yaml b/bootstrap/Chart.yaml deleted file mode 100644 index 2a9e7d09b3..0000000000 --- a/bootstrap/Chart.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -description: OpenStack-Helm namespace bootstrap -name: bootstrap -version: 0.1.0 diff --git a/bootstrap/requirements.yaml b/bootstrap/requirements.yaml deleted file mode 100644 index 53782e69b2..0000000000 --- a/bootstrap/requirements.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -dependencies: - - name: helm-toolkit - repository: http://localhost:8879/charts - version: 0.1.0 diff --git a/bootstrap/values.yaml b/bootstrap/values.yaml deleted file mode 100644 index 923e85d7da..0000000000 --- a/bootstrap/values.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for bootstrap. -# This is a YAML-formatted file. -# Declare name/value pairs to be passed into your templates. -# name: value diff --git a/ceph/templates/bin/_ceph-key.py.tpl b/ceph/templates/bin/_ceph-key.py.tpl new file mode 100644 index 0000000000..a0a279c7b2 --- /dev/null +++ b/ceph/templates/bin/_ceph-key.py.tpl @@ -0,0 +1,14 @@ +#!/bin/python +import os +import struct +import time +import base64 +key = os.urandom(16) +header = struct.pack( + ' +create_kube_key $(ceph_gen_key) ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${KUBE_SECRET_NAME} diff --git a/ceph/templates/bin/_ceph-namespace-client-key.sh.tpl b/ceph/templates/bin/_ceph-namespace-client-key.sh.tpl new file mode 100644 index 0000000000..6817e53331 --- /dev/null +++ b/ceph/templates/bin/_ceph-namespace-client-key.sh.tpl @@ -0,0 +1,22 @@ +#!/bin/bash +set -ex + +ceph_activate_namespace() { + kube_namespace=$1 + { + cat < +create_kube_key ${CEPH_CLIENT_KEY} ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${CEPH_KEYRING_ADMIN_NAME} + +function create_kube_storage_key () { + CEPH_KEYRING=$1 + KUBE_SECRET_NAME=$2 + + if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then + { + cat < +create_kube_storage_key ${CEPH_CLIENT_KEY} ${CEPH_STORAGECLASS_ADMIN_SECRET_NAME} diff --git a/ceph/templates/configmap-bin.yaml b/ceph/templates/configmap-bin.yaml new file mode 100644 index 0000000000..d8de23d0b8 --- /dev/null +++ b/ceph/templates/configmap-bin.yaml @@ -0,0 +1,31 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-bin +data: +{{- if .Values.manifests_enabled.storage_secrets }} + ceph-key.py: |+ +{{ tuple "bin/_ceph-key.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-key.sh: |+ +{{ tuple "bin/_ceph-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-storage-key.sh: |+ +{{ tuple "bin/_ceph-storage-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} +{{- if .Values.manifests_enabled.client_secrets }} + ceph-namespace-client-key.sh: |+ +{{ tuple "bin/_ceph-namespace-client-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/bootstrap/templates/secrets.yaml b/ceph/templates/configmap-etc.yaml similarity index 53% rename from bootstrap/templates/secrets.yaml rename to ceph/templates/configmap-etc.yaml index 2bd43ac310..3a1203562a 100644 --- a/bootstrap/templates/secrets.yaml +++ b/ceph/templates/configmap-etc.yaml @@ -12,21 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if empty .Values.conf.ceph.config.global.mon_host -}} +{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_endpoint_lookup" }} +{{- $monHostDomain := default .Release.Namespace .Values.ceph.namespace }} +{{- $monHostURI := cat $monHost "." $monHostDomain | nospace -}} +{{- $monHostURI | set .Values.conf.ceph.config.global "mon_host" | quote | trunc 0 -}} +{{- end -}} + --- apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: - name: "pvc-ceph-conf-combined-storageclass" -type: kubernetes.io/rbd + name: ceph-etc data: - key: | -{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: "pvc-ceph-client-key" -type: kubernetes.io/rbd -data: - key: | -{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }} + ceph.conf: |+ +{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/ceph/templates/configmap-templates.yaml b/ceph/templates/configmap-templates.yaml new file mode 100644 index 0000000000..ce18d56fcb --- /dev/null +++ b/ceph/templates/configmap-templates.yaml @@ -0,0 +1,31 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +{{- if .Values.manifests_enabled.storage_secrets }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-templates +data: + admin.keyring: |+ +{{ tuple "templates/_admin.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + bootstrap.keyring.mds: |+ +{{ tuple "templates/_bootstrap.keyring.mds.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + bootstrap.keyring.osd: |+ +{{ tuple "templates/_bootstrap.keyring.osd.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + bootstrap.keyring.rgw: |+ +{{ tuple "templates/_bootstrap.keyring.rgw.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + mon.keyring: |+ +{{ tuple "templates/_mon.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/ceph/templates/daemonset-osd.yaml b/ceph/templates/daemonset-osd.yaml index 0092830c17..f0c5ffa54a 100644 --- a/ceph/templates/daemonset-osd.yaml +++ b/ceph/templates/daemonset-osd.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if .Values.manifests_enabled.deployment }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.osd }} --- kind: DaemonSet apiVersion: extensions/v1beta1 @@ -29,47 +32,21 @@ spec: spec: nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} - volumes: - - name: devices - hostPath: - path: /dev - - name: ceph - hostPath: - path: {{ .Values.storage.var_directory }} - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - - name: osd-directory - hostPath: - path: {{ .Values.storage.osd_directory }} + initContainers: +{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: osd-pod image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} - volumeMounts: - - name: devices - mountPath: /dev - - name: ceph - mountPath: /var/lib/ceph - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - - name: osd-directory - mountPath: /var/lib/ceph/osd + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.osd.requests.memory | quote }} + cpu: {{ .Values.resources.osd.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.osd.limits.memory | quote }} + cpu: {{ .Values.resources.osd.limits.cpu | quote }} + {{- end }} securityContext: privileged: true env: @@ -81,6 +58,10 @@ spec: value: ceph - name: CEPH_GET_ADMIN_KEY value: "1" + command: + - /entrypoint.sh + ports: + - containerPort: 6800 livenessProbe: tcpSocket: port: 6800 @@ -90,10 +71,65 @@ spec: tcpSocket: port: 6800 timeoutSeconds: 5 - resources: - requests: - memory: {{ .Values.resources.osd.requests.memory | quote }} - cpu: {{ .Values.resources.osd.requests.cpu | quote }} - limits: - memory: {{ .Values.resources.osd.limits.memory | quote }} - cpu: {{ .Values.resources.osd.limits.cpu | quote }} + volumeMounts: + - name: devices + mountPath: /dev + readOnly: false + - name: ceph + mountPath: /var/lib/ceph + readOnly: false + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: false + - name: ceph-mon-keyring + mountPath: /etc/ceph/ceph.mon.keyring + subPath: ceph.mon.keyring + readOnly: false + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: false + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + subPath: ceph.keyring + readOnly: false + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring + subPath: ceph.keyring + readOnly: false + - name: osd-directory + mountPath: /var/lib/ceph/osd + volumes: + - name: devices + hostPath: + path: /dev + - name: ceph + hostPath: + path: {{ .Values.ceph.storage.var_directory }} + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-mon-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mon }} + - name: ceph-bootstrap-osd-keyring + secret: + secretName: {{ .Values.secrets.keyrings.osd }} + - name: ceph-bootstrap-mds-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mds }} + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: {{ .Values.secrets.keyrings.rgw }} + - name: osd-directory + hostPath: + path: {{ .Values.ceph.storage.osd_directory }} +{{- end }} diff --git a/ceph/templates/deployment-mds.yaml b/ceph/templates/deployment-mds.yaml index 925f449dde..964677ec40 100644 --- a/ceph/templates/deployment-mds.yaml +++ b/ceph/templates/deployment-mds.yaml @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. ---- +{{- if .Values.manifests_enabled.deployment }} +{{- if .Values.ceph.enabled.mds }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.mds }} kind: Deployment apiVersion: apps/v1beta1 metadata: @@ -31,24 +34,22 @@ spec: spec: nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring containers: - name: ceph-mds image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.mds.requests.memory | quote }} + cpu: {{ .Values.resources.mds.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mds.limits.memory | quote }} + cpu: {{ .Values.resources.mds.limits.cpu | quote }} + {{- end }} ports: - containerPort: 6800 env: @@ -60,15 +61,33 @@ spec: value: k8s - name: CLUSTER value: ceph + command: + - /entrypoint.sh volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-mon-keyring + mountPath: /etc/ceph/ceph.mon.keyring + subPath: ceph.mon.keyring + readOnly: true - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds + mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw + mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring + subPath: ceph.keyring + readOnly: false livenessProbe: tcpSocket: port: 6800 @@ -78,10 +97,24 @@ spec: tcpSocket: port: 6800 timeoutSeconds: 5 - resources: - requests: - memory: {{ .Values.resources.mds.requests.memory | quote }} - cpu: {{ .Values.resources.mds.requests.cpu | quote }} - limits: - memory: {{ .Values.resources.mds.limits.memory | quote }} - cpu: {{ .Values.resources.mds.limits.cpu | quote }} + volumes: + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-mon-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mon }} + - name: ceph-bootstrap-osd-keyring + secret: + secretName: {{ .Values.secrets.keyrings.osd }} + - name: ceph-bootstrap-mds-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mds }} + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: {{ .Values.secrets.keyrings.rgw }} +{{- end }} +{{- end }} diff --git a/ceph/templates/deployment-moncheck.yaml b/ceph/templates/deployment-moncheck.yaml index a62c039fdc..54228cf3ad 100644 --- a/ceph/templates/deployment-moncheck.yaml +++ b/ceph/templates/deployment-moncheck.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if .Values.manifests_enabled.deployment }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.moncheck }} --- kind: Deployment apiVersion: apps/v1beta1 @@ -31,24 +34,22 @@ spec: spec: nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring containers: - name: ceph-mon image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.mon_check.requests.memory | quote }} + cpu: {{ .Values.resources.mon_check.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mon_check.limits.memory | quote }} + cpu: {{ .Values.resources.mon_check.limits.cpu | quote }} + {{- end }} ports: - containerPort: 6789 env: @@ -60,19 +61,50 @@ spec: value: "1" - name: CLUSTER value: ceph + command: + - /entrypoint.sh volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-mon-keyring + mountPath: /etc/ceph/ceph.mon.keyring + subPath: ceph.mon.keyring + readOnly: true - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds + mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - resources: - requests: - memory: {{ .Values.resources.mon_check.requests.memory | quote }} - cpu: {{ .Values.resources.mon_check.requests.cpu | quote }} - limits: - memory: {{ .Values.resources.mon_check.limits.memory | quote }} - cpu: {{ .Values.resources.mon_check.limits.cpu | quote }} + mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring + subPath: ceph.keyring + readOnly: false + volumes: + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-mon-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mon }} + - name: ceph-bootstrap-osd-keyring + secret: + secretName: {{ .Values.secrets.keyrings.osd }} + - name: ceph-bootstrap-mds-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mds }} + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: {{ .Values.secrets.keyrings.rgw }} +{{- end }} diff --git a/ceph/templates/deployment-rgw.yaml b/ceph/templates/deployment-rgw.yaml index 6521046c85..c754b1ae40 100644 --- a/ceph/templates/deployment-rgw.yaml +++ b/ceph/templates/deployment-rgw.yaml @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{- if .Values.rgw.enabled }} +{{- if .Values.manifests_enabled.deployment }} +{{- if .Values.ceph.enabled.rgw }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.rgw }} --- kind: Deployment apiVersion: apps/v1beta1 @@ -32,24 +35,22 @@ spec: spec: nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring containers: - name: ceph-rgw image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.rgw.requests.memory | quote }} + cpu: {{ .Values.resources.rgw.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.rgw.limits.memory | quote }} + cpu: {{ .Values.resources.rgw.limits.cpu | quote }} + {{- end }} ports: - containerPort: {{ .Values.network.port.rgw_target }} env: @@ -61,15 +62,33 @@ spec: value: k8s - name: CLUSTER value: ceph + command: + - /entrypoint.sh volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-mon-keyring + mountPath: /etc/ceph/ceph.mon.keyring + subPath: ceph.mon.keyring + readOnly: true - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds + mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw + mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring + subPath: ceph.keyring + readOnly: false livenessProbe: httpGet: path: / @@ -81,11 +100,24 @@ spec: path: / port: {{ .Values.network.port.rgw_target }} timeoutSeconds: 5 - resources: - requests: - memory: {{ .Values.resources.rgw.requests.memory | quote }} - cpu: {{ .Values.resources.rgw.requests.cpu | quote }} - limits: - memory: {{ .Values.resources.rgw.limits.memory | quote }} - cpu: {{ .Values.resources.rgw.limits.cpu | quote }} + volumes: + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-mon-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mon }} + - name: ceph-bootstrap-osd-keyring + secret: + secretName: {{ .Values.secrets.keyrings.osd }} + - name: ceph-bootstrap-mds-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mds }} + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: {{ .Values.secrets.keyrings.rgw }} +{{- end }} {{- end }} diff --git a/ceph/templates/etc/_ceph.conf.tpl b/ceph/templates/etc/_ceph.conf.tpl new file mode 100644 index 0000000000..7a9dd76f28 --- /dev/null +++ b/ceph/templates/etc/_ceph.conf.tpl @@ -0,0 +1,77 @@ +[global] +fsid = {{ uuidv4 | default .Values.conf.ceph.config.global.uuid | quote }} +cephx = {{ .Values.conf.ceph.config.global.cephx | default "true" | quote }} +cephx_require_signatures = {{ .Values.conf.ceph.config.global.cephx_require_signatures | default "false" | quote }} +cephx_cluster_require_signatures = {{ .Values.conf.ceph.config.global.cephx_cluster_require_signatures | default "true" | quote }} +cephx_service_require_signatures = {{ .Values.conf.ceph.config.global.cephx_service_require_signatures | default "false" | quote }} + +# auth +max_open_files = {{ .Values.conf.ceph.config.global.max_open_files | default "131072" | quote }} + +osd_pool_default_pg_num = {{ .Values.conf.ceph.config.global.osd_pool_default_pg_num | default "128" | quote }} +osd_pool_default_pgp_num = {{ .Values.conf.ceph.config.global.osd_pool_default_pgp_num | default "128" | quote }} +osd_pool_default_size = {{ .Values.conf.ceph.config.global.osd_pool_default_size | default "3" | quote }} +osd_pool_default_min_size = {{ .Values.conf.ceph.config.global.osd_pool_default_min_size | default "1" | quote }} + +mon_osd_full_ratio = {{ .Values.conf.ceph.config.global.mon_osd_full_ratio | default ".95" | quote }} +mon_osd_nearfull_ratio = {{ .Values.conf.ceph.config.global.mon_osd_nearfull_ratio | default ".85" | quote }} +mon_host = {{ .Values.conf.ceph.config.global.mon_host | quote }} + +rgw_thread_pool_size = {{ .Values.conf.ceph.config.global.rgw_thread_pool_size | default "1024" | quote }} +rgw_num_rados_handles = {{ .Values.conf.ceph.config.global.rgw_num_rados_handles | default "100" | quote }} + +[mon] +mon_osd_down_out_interval = {{ .Values.conf.ceph.config.mon.mon_osd_down_out_interval | default "600" | quote }} +mon_osd_min_down_reporters = {{ .Values.conf.ceph.config.mon.mon_osd_min_down_reporters | default "4" | quote }} +mon_clock_drift_allowed = {{ .Values.conf.ceph.config.mon.mon_clock_drift_allowed | default "0.15" | quote }} +mon_clock_drift_warn_backoff = {{ .Values.conf.ceph.config.mon.mon_clock_drift_warn_backoff | default "30" | quote }} +mon_osd_report_timeout = {{ .Values.conf.ceph.config.mon.mon_osd_report_timeout | default "300" | quote }} + +[osd] +# network +cluster_network = {{ .Values.network.cluster | default "192.168.0.0/16" | quote }} +public_network = {{ .Values.network.public | default "192.168.0.0/16" | quote }} +osd_mon_heartbeat_interval = {{ .Values.conf.ceph.config.osd.osd_mon_heartbeat_interval | default "30" | quote }} + +# ports +ms_bind_port_min = {{ .Values.conf.ceph.config.osd.ms_bind_port_min | default "6800" | quote }} +ms_bind_port_max = {{ .Values.conf.ceph.config.osd.ms_bind_port_max | default "7100" | quote }} + +# journal +journal_size = {{ .Values.conf.ceph.config.osd.journal_size | default "100" | quote }} + +# filesystem +osd_mkfs_type = {{ .Values.conf.ceph.config.osd.osd_mkfs_type | default "xfs" | quote }} +osd_mkfs_options_xfs = {{ .Values.conf.ceph.config.osd.osd_mkfs_options_xfs | default "-f -i size=2048" | quote }} +osd_max_object_name_len = {{ .Values.conf.ceph.config.osd.osd_max_object_name_len | default "256" | quote }} + +# crush +osd_pool_default_crush_rule = {{ .Values.conf.ceph.config.osd.osd_pool_default_crush_rule | default "0" | quote }} +osd_crush_update_on_start = {{ .Values.conf.ceph.config.osd.osd_crush_update_on_start | default "true" | quote }} + +# backend +osd_objectstore = {{ .Values.conf.ceph.config.osd.osd_objectstore | default "filestore" | quote }} + +# performance tuning +filestore_merge_threshold = {{ .Values.conf.ceph.config.osd.filestore_merge_threshold | default "40" | quote }} +filestore_split_multiple = {{ .Values.conf.ceph.config.osd.filestore_split_multiple | default "8" | quote }} +osd_op_threads = {{ .Values.conf.ceph.config.osd.osd_op_threads | default "8" | quote }} +filestore_op_threads = {{ .Values.conf.ceph.config.osd.filestore_op_threads | default "8" | quote }} +filestore_max_sync_interval = {{ .Values.conf.ceph.config.osd.filestore_max_sync_interval | default "5" | quote }} +osd_max_scrubs = {{ .Values.conf.ceph.config.osd.osd_max_scrubs | default "1" | quote }} + +# recovery tuning +osd_recovery_max_active = {{ .Values.conf.ceph.config.osd.osd_recovery_max_active | default "5" | quote }} +osd_max_backfills = {{ .Values.conf.ceph.config.osd.osd_max_backfills | default "2" | quote }} +osd_recovery_op_priority = {{ .Values.conf.ceph.config.osd.osd_recovery_op_priority | default "2" | quote }} +osd_client_op_priority = {{ .Values.conf.ceph.config.osd.osd_client_op_priority | default "63" | quote }} +osd_recovery_max_chunk = {{ .Values.conf.ceph.config.osd.osd_client_op_priority | default "osd_recovery_max_chunk" | quote }} +osd_recovery_threads = {{ .Values.conf.ceph.config.osd.osd_recovery_threads | default "1" | quote }} + +[client] +rbd_cache_enabled = {{ .Values.conf.ceph.config.client.rbd_cache_enabled | default "true" | quote }} +rbd_cache_writethrough_until_flush = {{ .Values.conf.ceph.config.client.rbd_cache_writethrough_until_flush | default "true" | quote }} +rbd_default_features = {{ .Values.conf.ceph.config.client.rbd_default_features | default "1" | quote }} + +[mds] +mds_cache_size = {{ .Values.conf.ceph.config.client.mds_mds_cache_size | default "100000" | quote }} diff --git a/ceph/templates/job-keyring.yaml b/ceph/templates/job-keyring.yaml new file mode 100644 index 0000000000..677167dfaf --- /dev/null +++ b/ceph/templates/job-keyring.yaml @@ -0,0 +1,85 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.manifests_enabled.storage_secrets }} +{{- $envAll := . }} +{{- range $key1, $cephBootstrapKey := tuple "mds" "osd" "rgw" "mon" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-{{ $cephBootstrapKey }}-keyring-generator +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: ceph-secret-generator + image: {{ $envAll.Values.images.ceph_config_helper }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} + {{- if $envAll.Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }} + cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }} + cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }} + {{- end }} + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CEPH_GEN_DIR + value: /opt/ceph + - name: CEPH_TEMPLATES_DIR + value: /opt/ceph/templates + {{- if eq $cephBootstrapKey "mon"}} + - name: CEPH_KEYRING_NAME + value: ceph.mon.keyring + - name: CEPH_KEYRING_TEMPLATE + value: mon.keyring + {{- else }} + - name: CEPH_KEYRING_NAME + value: ceph.keyring + - name: CEPH_KEYRING_TEMPLATE + value: bootstrap.keyring.{{ $cephBootstrapKey }} + {{- end }} + - name: KUBE_SECRET_NAME + value: {{ index $envAll.Values.secrets.keyrings $cephBootstrapKey }} + command: + - /opt/ceph/ceph-key.sh + volumeMounts: + - name: ceph-bin + mountPath: /opt/ceph/ceph-key.sh + subPath: ceph-key.sh + readOnly: true + - name: ceph-bin + mountPath: /opt/ceph/ceph-key.py + subPath: ceph-key.py + readOnly: true + - name: ceph-templates + mountPath: /opt/ceph/templates + readOnly: true + volumes: + - name: ceph-bin + configMap: + name: ceph-bin + defaultMode: 0555 + - name: ceph-templates + configMap: + name: ceph-templates +{{ end }} +{{ end }} diff --git a/ceph/templates/job-namespace-client-key.yaml b/ceph/templates/job-namespace-client-key.yaml new file mode 100644 index 0000000000..aa9a2f09ed --- /dev/null +++ b/ceph/templates/job-namespace-client-key.yaml @@ -0,0 +1,61 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.manifests_enabled.client_secrets }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-namespace-client-key-generator +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: ceph-storage-keys-generator + image: {{ .Values.images.ceph_config_helper }} + imagePullPolicy: {{ .Values.images.pull_policy }} + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }} + cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }} + cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }} + {{- end }} + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PVC_CEPH_STORAGECLASS_USER_SECRET_NAME + value: {{ .Values.storageclass.user_secret_name }} + - name: PVC_CEPH_STORAGECLASS_ADMIN_SECRET_NAME + value: {{ .Values.storageclass.admin_secret_name }} + - name: PVC_CEPH_STORAGECLASS_DEPLOYED_NAMESPACE + value: {{ .Values.storageclass.admin_secret_namespace }} + command: + - /opt/ceph/ceph-namespace-client-key.sh + volumeMounts: + - name: ceph-bin + mountPath: /opt/ceph/ceph-namespace-client-key.sh + subPath: ceph-namespace-client-key.sh + readOnly: true + volumes: + - name: ceph-bin + configMap: + name: ceph-bin + defaultMode: 0555 +{{- end }} diff --git a/ceph/templates/job-storage-admin-keys.yaml b/ceph/templates/job-storage-admin-keys.yaml new file mode 100644 index 0000000000..571057b8d7 --- /dev/null +++ b/ceph/templates/job-storage-admin-keys.yaml @@ -0,0 +1,77 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.manifests_enabled.storage_secrets }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-storage-keys-generator +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: ceph-storage-keys-generator + image: {{ .Values.images.ceph_config_helper }} + imagePullPolicy: {{ .Values.images.pull_policy }} + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }} + cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }} + cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }} + {{- end }} + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CEPH_GEN_DIR + value: /opt/ceph + - name: CEPH_TEMPLATES_DIR + value: /opt/ceph/templates + - name: CEPH_KEYRING_NAME + value: ceph.client.admin.keyring + - name: CEPH_KEYRING_TEMPLATE + value: admin.keyring + - name: CEPH_KEYRING_ADMIN_NAME + value: {{ .Values.secrets.keyrings.admin }} + - name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME + value: {{ .Values.storageclass.admin_secret_name }} + command: + - /opt/ceph/ceph-storage-key.sh + volumeMounts: + - name: ceph-bin + mountPath: /opt/ceph/ceph-storage-key.sh + subPath: ceph-storage-key.sh + readOnly: true + - name: ceph-bin + mountPath: /opt/ceph/ceph-key.py + subPath: ceph-key.py + readOnly: true + - name: ceph-templates + mountPath: /opt/ceph/templates + readOnly: true + volumes: + - name: ceph-bin + configMap: + name: ceph-bin + defaultMode: 0555 + - name: ceph-templates + configMap: + name: ceph-templates +{{- end }} diff --git a/ceph/templates/pdb-mon.yaml b/ceph/templates/pdb-mon.yaml index 9852700e95..e357496855 100644 --- a/ceph/templates/pdb-mon.yaml +++ b/ceph/templates/pdb-mon.yaml @@ -1,3 +1,4 @@ +{{- if .Values.manifests_enabled.deployment }} apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: @@ -7,4 +8,5 @@ spec: selector: matchLabels: app: ceph - daemon: mon \ No newline at end of file + daemon: mon +{{- end }} diff --git a/ceph/templates/secrets.yaml b/ceph/templates/secrets.yaml deleted file mode 100644 index 8c11ec2f7d..0000000000 --- a/ceph/templates/secrets.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.secrets.use_common_secrets -}} ---- -apiVersion: v1 -kind: Secret -metadata: - name: "ceph-conf-combined" -type: Opaque -data: - ceph.conf: | -{{ include "secrets/ceph.conf" . | b64enc | indent 4 }} - ceph.client.admin.keyring: | -{{ include "secrets/ceph.client.admin.keyring" . | b64enc | indent 4 }} - ceph.mon.keyring: | -{{ include "secrets/ceph.mon.keyring" . | b64enc | indent 4 }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: "ceph-bootstrap-rgw-keyring" -type: Opaque -data: - ceph.keyring: | -{{ include "secrets/ceph.rgw.keyring" . | b64enc | indent 4 }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: "ceph-bootstrap-mds-keyring" -type: Opaque -data: - ceph.keyring: | -{{ include "secrets/ceph.mds.keyring" . | b64enc | indent 4 }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: "ceph-bootstrap-osd-keyring" -type: Opaque -data: - ceph.keyring: | -{{ include "secrets/ceph.osd.keyring" . | b64enc | indent 4 }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: "ceph-client-key" -type: Opaque -data: - ceph-client-key: | -{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }} -{{- end -}} diff --git a/ceph/templates/service-rgw.yaml b/ceph/templates/service-rgw.yaml index 4ae4055c96..c93e82de3d 100644 --- a/ceph/templates/service-rgw.yaml +++ b/ceph/templates/service-rgw.yaml @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{- if .Values.rgw.enabled }} +{{- if .Values.manifests_enabled.deployment }} +{{- if .Values.ceph.enabled.rgw }} --- apiVersion: v1 kind: Service @@ -30,3 +31,4 @@ spec: app: ceph daemon: rgw {{- end }} +{{- end }} diff --git a/ceph/templates/service.yaml b/ceph/templates/service.yaml index ef6d98e0a9..e0202e557e 100644 --- a/ceph/templates/service.yaml +++ b/ceph/templates/service.yaml @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if .Values.manifests_enabled.deployment }} --- kind: Service apiVersion: v1 metadata: - name: ceph-mon + name: {{ .Values.endpoints.ceph_mon.hosts.default }} labels: app: ceph daemon: mon @@ -37,3 +38,4 @@ spec: app: ceph daemon: mon clusterIP: None +{{- end }} diff --git a/ceph/templates/statefulset-mon.yaml b/ceph/templates/statefulset-mon.yaml index 47a8ebc279..0f1ca1e169 100644 --- a/ceph/templates/statefulset-mon.yaml +++ b/ceph/templates/statefulset-mon.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if .Values.manifests_enabled.deployment }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.mon }} --- apiVersion: apps/v1beta1 kind: StatefulSet @@ -21,7 +24,7 @@ metadata: daemon: mon name: ceph-mon spec: - serviceName: {{ .Values.service.mon.name | quote }} + serviceName: {{ tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_endpoint_lookup" }} replicas: {{ .Values.replicas.mon }} template: metadata: @@ -46,33 +49,22 @@ spec: weight: 10 nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - - name: ceph-monfs - hostPath: - path: {{ .Values.storage.mon_directory }} containers: - name: ceph-mon image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} - lifecycle: - preStop: - exec: - # remove the mon on Pod stop. - command: - - "/remove-mon.sh" + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.mon.requests.memory | quote }} + cpu: {{ .Values.resources.mon.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mon.limits.memory | quote }} + cpu: {{ .Values.resources.mon.limits.cpu | quote }} + {{- end }} ports: - containerPort: 6789 env: @@ -90,17 +82,39 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + command: + - /entrypoint.sh + lifecycle: + preStop: + exec: + # remove the mon on Pod stop. + command: + - "/remove-mon.sh" volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-mon-keyring + mountPath: /etc/ceph/ceph.mon.keyring + subPath: ceph.mon.keyring + readOnly: false - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds + mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - - name: ceph-monfs - mountPath: /var/lib/ceph/mon + mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring + subPath: ceph.keyring + readOnly: false livenessProbe: tcpSocket: port: 6789 @@ -110,10 +124,23 @@ spec: tcpSocket: port: 6789 timeoutSeconds: 5 - resources: - requests: - memory: {{ .Values.resources.mon.requests.memory | quote }} - cpu: {{ .Values.resources.mon.requests.cpu | quote }} - limits: - memory: {{ .Values.resources.mon.limits.memory | quote }} - cpu: {{ .Values.resources.mon.limits.cpu | quote }} + volumes: + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-mon-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mon }} + - name: ceph-bootstrap-osd-keyring + secret: + secretName: {{ .Values.secrets.keyrings.osd }} + - name: ceph-bootstrap-mds-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mds }} + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: {{ .Values.secrets.keyrings.rgw }} +{{- end }} diff --git a/ceph/templates/storage.yaml b/ceph/templates/storage.yaml index b104269e56..e75fd35df6 100644 --- a/ceph/templates/storage.yaml +++ b/ceph/templates/storage.yaml @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# note that these secrets are handled by the common chart, not the ceph -# chart, as we likely want them "everywhere" +{{- if .Values.manifests_enabled.deployment }} --- apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -21,12 +20,11 @@ metadata: name: {{ .Values.storageclass.name }} provisioner: kubernetes.io/rbd parameters: - monitors: {{ .Values.storageclass.monitors | default "ceph-mon.ceph:6789" }} + monitors: {{ tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.hostname_endpoint_uri_lookup" }} adminId: {{ .Values.storageclass.admin_id }} adminSecretName: {{ .Values.storageclass.admin_secret_name }} - # forcing namespace due to issue with default pipeline of "{{ .Release.Namespace }}" }} - # during helm lint - adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace | default "ceph" }} + adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace }} pool: {{ .Values.storageclass.pool }} userId: {{ .Values.storageclass.user_id }} userSecretName: {{ .Values.storageclass.user_secret_name }} +{{- end }} diff --git a/ceph/templates/templates/_admin.keyring.tpl b/ceph/templates/templates/_admin.keyring.tpl new file mode 100644 index 0000000000..7c5de607b6 --- /dev/null +++ b/ceph/templates/templates/_admin.keyring.tpl @@ -0,0 +1,6 @@ +[client.admin] + key = {{"{{"}} key {{"}}"}} + auid = 0 + caps mds = "allow" + caps mon = "allow *" + caps osd = "allow *" diff --git a/ceph/templates/templates/_bootstrap.keyring.mds.tpl b/ceph/templates/templates/_bootstrap.keyring.mds.tpl new file mode 100644 index 0000000000..c52fd6397a --- /dev/null +++ b/ceph/templates/templates/_bootstrap.keyring.mds.tpl @@ -0,0 +1,3 @@ +[client.bootstrap-mds] + key = {{"{{"}} key {{"}}"}} + caps mon = "allow profile bootstrap-mds" diff --git a/ceph/templates/templates/_bootstrap.keyring.osd.tpl b/ceph/templates/templates/_bootstrap.keyring.osd.tpl new file mode 100644 index 0000000000..c5fe618d99 --- /dev/null +++ b/ceph/templates/templates/_bootstrap.keyring.osd.tpl @@ -0,0 +1,3 @@ +[client.bootstrap-osd] + key = {{"{{"}} key {{"}}"}} + caps mon = "allow profile bootstrap-osd" diff --git a/ceph/templates/templates/_bootstrap.keyring.rgw.tpl b/ceph/templates/templates/_bootstrap.keyring.rgw.tpl new file mode 100644 index 0000000000..1f2a58d6ab --- /dev/null +++ b/ceph/templates/templates/_bootstrap.keyring.rgw.tpl @@ -0,0 +1,3 @@ +[client.bootstrap-rgw] + key = {{"{{"}} key {{"}}"}} + caps mon = "allow profile bootstrap-rgw" diff --git a/ceph/templates/templates/_mon.keyring.tpl b/ceph/templates/templates/_mon.keyring.tpl new file mode 100644 index 0000000000..f9681f2d90 --- /dev/null +++ b/ceph/templates/templates/_mon.keyring.tpl @@ -0,0 +1,3 @@ +[mon.] + key = {{"{{"}} key {{"}}"}} + caps mon = "allow *" diff --git a/ceph/values.yaml b/ceph/values.yaml index ade9681e81..ebfd7160d4 100644 --- a/ceph/values.yaml +++ b/ceph/values.yaml @@ -12,6 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +manifests_enabled: + storage_secrets: true + client_secrets: true + deployment: true + replicas: mon: 3 rgw: 3 @@ -22,8 +27,10 @@ service: name: ceph-mon images: - daemon: docker.io/library/ceph/daemon:tag-build-master-jewel-ubuntu-16.04 - pull_policy: IfNotPresent + dep_check: docker.io/kolla/ubuntu-source-kubernetes-entrypoint:4.0.0 + daemon: quay.io/attcomdev/ceph-daemon:tag-build-master-jewel-ubuntu-16.04 + ceph_config_helper: docker.io/port/ceph-config-helper:v1.6.5 + pull_policy: Always labels: node_selector_key: ceph-storage @@ -33,23 +40,125 @@ pod_disruption_budget: mon: min_available: 0 +secrets: + keyrings: + mon: ceph-mon-keyring + mds: ceph-bootstrap-mds-keyring + osd: ceph-bootstrap-osd-keyring + rgw: ceph-bootstrap-rgw-keyring + admin: ceph-client-admin-keyring + network: - public: "10.25.0.0/16" + public: "192.168.0.0/16" + cluster: "192.168.0.0/16" port: mon: 6789 rgw_ingress: 80 rgw_target: 8088 -storage: - osd_directory: /var/lib/openstack-helm/ceph/osd - var_directory: /var/lib/openstack-helm/ceph/ceph - mon_directory: /var/lib/openstack-helm/ceph/mon +conf: + ceph: + override: + append: + config: + global: + # auth + cephx: true + cephx_require_signatures: false + cephx_cluster_require_signatures: true + cephx_service_require_signatures: false + + max_open_files: 131072 + osd_pool_default_pg_num: 128 + osd_pool_default_pgp_num: 128 + osd_pool_default_size: 3 + osd_pool_default_min_size: 1 + mon_osd_full_ratio: .95 + mon_osd_nearfull_ratio: .85 + mon_host: null + mon: + mon_osd_down_out_interval: 600 + mon_osd_min_down_reporters: 4 + mon_clock_drift_allowed: .15 + mon_clock_drift_warn_backoff: 30 + mon_osd_report_timeout: 300 + osd: + journal_size: 100 + osd_mkfs_type: xfs + osd_mkfs_options_xfs: -f -i size=2048 + osd_mon_heartbeat_interval: 30 + osd_max_object_name_len: 256 + #crush + osd_pool_default_crush_rule: 0 + osd_crush_update_on_start: true + #backend + osd_objectstore: filestore + #performance tuning + filestore_merge_threshold: 40 + filestore_split_multiple: 8 + osd_op_threads: 8 + filestore_op_threads: 8 + filestore_max_sync_interval: 5 + osd_max_scrubs: 1 + #recovery tuning + osd_recovery_max_active: 5 + osd_max_backfills: 2 + osd_recovery_op_priority: 2 + osd_client_op_priority: 63 + osd_recovery_max_chunk: 1048576 + osd_recovery_threads: 1 + #ports + ms_bind_port_min: 6800 + ms_bind_port_max: 7100 + client: + rbd_cache_enabled: true + rbd_cache_writethrough_until_flush: true + rbd_default_features: "1" + mds: + mds_cache_size: 100000 + + +dependencies: + mon: + jobs: + service: + osd: + jobs: + services: + - service: ceph_mon + endpoint: internal + moncheck: + jobs: + services: + - service: ceph_mon + endpoint: internal + rgw: + jobs: + services: + - service: ceph_mon + endpoint: internal + mds: + jobs: + services: + - service: ceph_mon + endpoint: internal + + +ceph: + enabled: + mds: true + rgw: false + storage: + osd_directory: /var/lib/openstack-helm/ceph/osd + var_directory: /var/lib/openstack-helm/ceph/ceph + mon_directory: /var/lib/openstack-helm/ceph/mon # rgw is optionally disabled rgw: enabled: false resources: + enabled: false osd: requests: memory: "512Mi" @@ -85,36 +194,14 @@ resources: limits: memory: "50Mi" cpu: "500m" - -# Setting this to false will assume you will -# setup and orchestrate your own secrets and -# configmaps outside of this helm chart -# -# The list below is in the format of -# -# configMapName: -# elementKeyName -# -# ceph.conf: -# ceph.conf -# ceph.client.admin.keyring -# ceph.client.admin.keyring -# ceph.mon.keyring: -# ceph.mon.keyring -# ceph-bootstrap-rgw-keyring: -# ceph.keyring -# ceph.rgw.keyring -# ceph-bootstrap-mds-keyring: -# ceph.keyring -# ceph.mds.keyring -# ceph-bootstrap-osd-keyring: -# ceph.keyring -# ceph.osd.keyring -# ceph-client-key: -# ceph-client-key -secrets: - use_common_secrets: true - + jobs: + secret_provisioning: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" # if you change provision_storage_class to false # it is presumed you manage your own storage # class definition externally @@ -125,6 +212,13 @@ storageclass: pool: rbd admin_id: admin admin_secret_name: pvc-ceph-conf-combined-storageclass - admin_secret_namespace: null + admin_secret_namespace: ceph user_id: admin user_secret_name: pvc-ceph-client-key + +endpoints: + ceph_mon: + hosts: + default: ceph-mon + port: + mon: 6789 diff --git a/cinder/templates/_funcs.tpl b/cinder/templates/_funcs.tpl index 3d62b8f9b0..8f42e5cc27 100644 --- a/cinder/templates/_funcs.tpl +++ b/cinder/templates/_funcs.tpl @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{- define "cinder.is_ceph_configured" -}} +{{- define "cinder.is_ceph_volume_configured" -}} {{- range $section, $values := .Values.conf.backends -}} {{- if kindIs "map" $values -}} {{- if eq $values.volume_driver "cinder.volume.drivers.rbd.RBDDriver" -}} @@ -21,3 +21,10 @@ true {{- end -}} {{- end -}} {{- end -}} + +{{- define "cinder.is_ceph_backup_configured" -}} +{{- $values := .Values.conf.cinder.default.cinder -}} +{{- if eq $values.backup_driver "cinder.backup.drivers.ceph" -}} +true +{{- end -}} +{{- end -}} diff --git a/cinder/templates/etc/_ceph-cinder.keyring.tpl b/cinder/templates/bin/_ceph-keyring.sh.tpl similarity index 81% rename from cinder/templates/etc/_ceph-cinder.keyring.tpl rename to cinder/templates/bin/_ceph-keyring.sh.tpl index 6a33704297..1d248258d9 100644 --- a/cinder/templates/etc/_ceph-cinder.keyring.tpl +++ b/cinder/templates/bin/_ceph-keyring.sh.tpl @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright 2017 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,9 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -ex +export HOME=/tmp + +cat < /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring [client.{{ .Values.conf.backends.rbd1.rbd_user }}] {{- if .Values.conf.ceph.cinder_keyring }} key = {{ .Values.conf.ceph.cinder_keyring }} {{- else }} - key = {{- include "secrets/ceph-client-key" . -}} + key = $(cat /tmp/client-keyring) {{- end }} +EOF + +exit 0 \ No newline at end of file diff --git a/cinder/templates/configmap-bin.yaml b/cinder/templates/configmap-bin.yaml index 961363e16d..338239e269 100644 --- a/cinder/templates/configmap-bin.yaml +++ b/cinder/templates/configmap-bin.yaml @@ -33,6 +33,8 @@ data: {{ tuple "bin/_cinder-scheduler.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} cinder-volume.sh: | {{ tuple "bin/_cinder-volume.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-keyring.sh: |+ +{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- if .Values.bootstrap.enabled }} bootstrap.sh: |+ {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/cinder/templates/configmap-etc.yaml b/cinder/templates/configmap-etc.yaml index 6e050e26cb..f817c45bae 100644 --- a/cinder/templates/configmap-etc.yaml +++ b/cinder/templates/configmap-etc.yaml @@ -97,17 +97,4 @@ data: {{ .Values.conf.policy.override | indent 4 }} {{- else -}} {{ tuple "etc/_policy.json.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} -{{- end }} - ceph.conf: |+ -{{- if or (include "cinder.is_ceph_configured" .) (eq .Values.conf.cinder.default.cinder.backup_driver "cinder.backup.drivers.ceph") }} -{{ if .Values.conf.ceph.override -}} -{{ .Values.conf.ceph.override | indent 4 }} -{{- else -}} -{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} -{{- end }} -{{- if .Values.conf.ceph.append -}} -{{ .Values.conf.ceph.append | indent 4 }} -{{- end }} - ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring: |+ -{{ tuple "etc/_ceph-cinder.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/cinder/templates/deployment-backup.yaml b/cinder/templates/deployment-backup.yaml index 020ab575a2..55086c2352 100644 --- a/cinder/templates/deployment-backup.yaml +++ b/cinder/templates/deployment-backup.yaml @@ -42,6 +42,23 @@ spec: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} initContainers: {{ tuple $envAll $dependencies $mounts_cinder_backup_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + {{- if include "cinder.is_ceph_backup_configured" . }} + - name: ceph-keyring-placement + image: {{ .Values.images.backup }} + imagePullPolicy: {{ .Values.images.pull_policy }} + command: + - /tmp/ceph-keyring.sh + volumeMounts: + - name: etcceph + mountPath: /etc/ceph + - name: cinder-bin + mountPath: /tmp/ceph-keyring.sh + subPath: ceph-keyring.sh + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} containers: - name: cinder-backup image: {{ .Values.images.backup }} @@ -67,13 +84,15 @@ spec: subPath: cinder.conf readOnly: true {{- if eq .Values.conf.cinder.default.cinder.backup_driver "cinder.backup.drivers.ceph" }} - - name: cinder-etc + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true - - name: cinder-etc - mountPath: /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring - subPath: ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key readOnly: true {{- end -}} {{ if $mounts_cinder_backup.volumeMounts }}{{ toYaml $mounts_cinder_backup.volumeMounts | indent 12 }}{{ end }} @@ -83,4 +102,18 @@ spec: - name: cinder-etc configMap: name: cinder-etc + - name: cinder-bin + configMap: + name: cinder-bin + defaultMode: 0555 + {{- if include "cinder.is_ceph_backup_configured" . }} + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key + {{ end }} {{ if $mounts_cinder_backup.volumes }}{{ toYaml $mounts_cinder_backup.volumes | indent 8 }}{{ end }} diff --git a/cinder/templates/deployment-volume.yaml b/cinder/templates/deployment-volume.yaml index 754452ffbe..afe59779c6 100644 --- a/cinder/templates/deployment-volume.yaml +++ b/cinder/templates/deployment-volume.yaml @@ -42,6 +42,23 @@ spec: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} initContainers: {{ tuple $envAll $dependencies $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + {{- if include "cinder.is_ceph_volume_configured" . }} + - name: ceph-keyring-placement + image: {{ .Values.images.volume }} + imagePullPolicy: {{ .Values.images.pull_policy }} + command: + - /tmp/ceph-keyring.sh + volumeMounts: + - name: etcceph + mountPath: /etc/ceph + - name: cinder-bin + mountPath: /tmp/ceph-keyring.sh + subPath: ceph-keyring.sh + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} containers: - name: cinder-volume image: {{ .Values.images.volume }} @@ -72,14 +89,16 @@ spec: mountPath: /etc/cinder/conf/backends.conf subPath: backends.conf readOnly: true - {{- if include "cinder.is_ceph_configured" . }} - - name: cinder-etc + {{- if include "cinder.is_ceph_volume_configured" . }} + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true - - name: cinder-etc - mountPath: /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring - subPath: ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key readOnly: true {{- end }} {{ if $mounts_cinder_volume.volumeMounts }}{{ toYaml $mounts_cinder_volume.volumeMounts | indent 12 }}{{ end }} @@ -93,4 +112,14 @@ spec: - name: cinder-etc configMap: name: cinder-etc + {{- if include "cinder.is_ceph_volume_configured" . }} + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key + {{ end }} {{ if $mounts_cinder_volume.volumes }}{{ toYaml $mounts_cinder_volume.volumes | indent 8 }}{{ end }} diff --git a/cinder/templates/etc/_ceph.conf.tpl b/cinder/templates/etc/_ceph.conf.tpl deleted file mode 100644 index e42694fc27..0000000000 --- a/cinder/templates/etc/_ceph.conf.tpl +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[global] -rgw_thread_pool_size = 1024 -rgw_num_rados_handles = 100 -{{- if .Values.conf.ceph.monitors }} -[mon] -{{ range .Values.conf.ceph.monitors }} - [mon.{{ . }}] - host = {{ . }} - mon_addr = {{ . }} -{{ end }} -{{- else }} -mon_host = ceph-mon.ceph -{{- end }} -[client] - rbd_cache_enabled = true - rbd_cache_writethrough_until_flush = true diff --git a/cinder/values.yaml b/cinder/values.yaml index fd9ce2d6af..433cea4be7 100644 --- a/cinder/values.yaml +++ b/cinder/values.yaml @@ -23,6 +23,8 @@ replicas: scheduler: 1 backup: 1 +storage: ceph + labels: node_selector_key: openstack-control-plane node_selector_value: enabled diff --git a/doc/source/install/multinode.rst b/doc/source/install/multinode.rst index a8f3fb4a65..28ee45511c 100644 --- a/doc/source/install/multinode.rst +++ b/doc/source/install/multinode.rst @@ -141,32 +141,13 @@ completed. Installing Ceph Host Requirements ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -At some future point, we want to ensure that our solution is -cloud-native, allowing installation on any host system without a package -manager and only a container runtime (i.e. CoreOS). Until this happens, -we will need to ensure that ``ceph-common`` is installed on each of our -hosts. Using our Ubuntu example: +You need to ensure that ``ceph-common`` or equivalent is +installed on each of our hosts. Using our Ubuntu example: :: sudo apt-get install ceph-common -y -We will always attempt to keep host-specific requirements to a minimum, -and we are working with the Ceph team (Sébastien Han) to quickly address -this Ceph requirement. - -Ceph Secrets Generation -~~~~~~~~~~~~~~~~~~~~~~~ - -Another thing of interest is that our deployment assumes that you can -generate secrets at the time of the container deployment. We require the -`sigil `__ -binary on your deployment host in order to perform this action. - -:: - - curl -L https://github.com/gliderlabs/sigil/releases/download/v0.4.0/sigil_0.4.0_Linux_x86_64.tgz | sudo tar -zxC /usr/local/bin - Kubernetes Controller Manager ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -200,20 +181,18 @@ Kubernetes v1.6.5. export kube_version=v1.6.5 sudo sed -i "s|gcr.io/google_containers/kube-controller-manager-amd64:$kube_version|quay.io/attcomdev/kube-controller-manager:$kube_version|g" /etc/kubernetes/manifests/kube-controller-manager.yaml -Now you will want to ``restart`` your Kubernetes master server to +Now you will want to ``restart`` the Kubernetes master server to continue. Kube Controller Manager DNS Resolution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Until the following `Kubernetes Pull -Request `__ is -merged, you will need to allow the Kubernetes Controller to use the -internal container ``skydns`` endpoint as a DNS server, and add the -Kubernetes search suffix into the controller's resolv.conf. As of now, -the Kubernetes controller only mirrors the host's ``resolv.conf``. This -is not sufficient if you want the controller to know how to correctly -resolve container service endpoints (in the case of DaemonSets). +You will need to allow the Kubernetes Controller to use the +Kubernetes service DNS server, and add the Kubernetes search suffix +to the controller's resolv.conf. As of now, the Kubernetes controller +only mirrors the host's ``resolv.conf``. This is not sufficient if you +want the controller to know how to correctly resolve container service +endpoints. First, find out what the IP Address of your ``kube-dns`` deployment is: @@ -224,82 +203,16 @@ First, find out what the IP Address of your ``kube-dns`` deployment is: kube-dns 10.96.0.10 53/UDP,53/TCP 1d admin@kubenode01:~$ -As you can see by this example, ``10.96.0.10`` is the -``CLUSTER-IP``\ IP. Now, have a look at the current -``kube-controller-manager-kubenode01`` ``/etc/resolv.conf``: +Then update the controller manager configuration to match: :: - admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -n kube-system -- cat /etc/resolv.conf - # Dynamic resolv.conf(5) file for glibc resolver(3) generated by resolvconf(8) - # DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN - nameserver 192.168.1.70 - nameserver 8.8.8.8 - search jinkit.com - admin@kubenode01:~$ - -What we need is for ``kube-controller-manager-kubenode01`` -``/etc/resolv.conf`` to look like this: - -:: - - admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -n kube-system -- cat /etc/resolv.conf + admin@kubenode01:~$ CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | head -1 | awk -F '/' '{ print $NF }') + admin@kubenode01:~$ kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf < /etc/resolv.conf - nameserver 10.96.0.10 - nameserver 192.168.1.70 - nameserver 8.8.8.8 - search svc.cluster.local jinkit.com - EOF - root@kubenode01:/# - -Now you can test your changes by deploying a service to your cluster, -and resolving this from the controller. As an example, lets deploy -something useful, like `Kubernetes -dashboard `__: - -:: - - kubectl create -f https://rawgit.com/kubernetes/dashboard/master/src/deploy/kubernetes-dashboard.yaml - -Note the ``IP`` field: - -:: - - admin@kubenode01:~$ kubectl describe svc kubernetes-dashboard -n kube-system - Name: kubernetes-dashboard - Namespace: kube-system - Labels: app=kubernetes-dashboard - Selector: app=kubernetes-dashboard - Type: NodePort - IP: 10.110.207.144 - Port: 80/TCP - NodePort: 32739/TCP - Endpoints: 10.25.178.65:9090 - Session Affinity: None - No events. - admin@kubenode01:~$ - -Now you should be able to resolve the host -``kubernetes-dashboard.kube-system.svc.cluster.local``: - -:: - - admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -it -n kube-system -- ping kubernetes-dashboard.kube-system.svc.cluster.local - PING kubernetes-dashboard.kube-system.svc.cluster.local (10.110.207.144) 56(84) bytes of data. - -.. note:: - This host example above has ``iputils-ping`` installed. + search cluster.local svc.cluster.local + EOF" Kubernetes Node DNS Resolution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -358,42 +271,15 @@ Download the latest copy of Openstack-Helm: Ceph Preparation and Installation --------------------------------- -Ceph must be aware of the OSX cluster and public networks. These CIDR +Ceph must be aware of the OSD cluster and public networks. These CIDR ranges are the exact same ranges you used earlier in your Calico -deployment yaml (our example was 10.25.0.0/16 due to our 192.168.0.0/16 -overlap). Explore this variable to your deployment environment by +deployment yaml. Export this variable to your deployment environment by issuing the following commands: :: - export osd_cluster_network=10.25.0.0/16 - export osd_public_network=10.25.0.0/16 - -Ceph Storage Volumes --------------------- - -Ceph must also have volumes to mount on each host labeled for -``ceph-storage``. On each host that you labeled, create the following -directory (can be overriden): - -:: - - mkdir -p /var/lib/openstack-helm/ceph - -*Repeat this step for each node labeled: ``ceph-storage``* - -Ceph Secrets Generation ------------------------ - -Although you can bring your own secrets, we have conveniently created a -secret generation tool for you (for greenfield deployments). You can -create secrets for your project by issuing the following: - -:: - - cd helm-toolkit/utils/secret-generator - ./generate_secrets.sh all `./generate_secrets.sh fsid` - cd ../../.. + export osd_cluster_network=192.168.0.0/16 + export osd_public_network=192.168.0.0/16 Nova Compute Instance Storage ----------------------------- @@ -468,30 +354,55 @@ the following command to install Ceph: :: - helm install --set network.public=$osd_public_network --name=ceph local/ceph --namespace=ceph + helm install --namespace=ceph local/ceph --name=ceph \ + --set manifests_enabled.client_secrets=false \ + --set network.public=$osd_public_network \ + --set network.cluster=$osd_cluster_network -Bootstrap Installation ----------------------- +Activating Control-Plane Namespace for Ceph +------------------------------------------- -At this time (and before verification of Ceph) you'll need to install -the ``bootstrap`` chart. The ``bootstrap`` chart will install secrets -for both the ``ceph`` and ``openstack`` namespaces for the general -StorageClass: +In order for Ceph to fulfill PersistentVolumeClaims within Kubernetes namespaces +outside of Ceph's namespace, a client keyring needs to be present within that +namespace. For the rest of the OpenStack and supporting core services, this guide +will be deploying the control plane to a seperate namespace ``openstack``. To +deploy the aforementioned client keyring to the ``openstack`` namespace: :: - helm install --name=bootstrap-ceph local/bootstrap --namespace=ceph - helm install --name=bootstrap-openstack local/bootstrap --namespace=openstack + helm install --namespace=openstack local/ceph --name=ceph-openstack-config \ + --set manifests_enabled.storage_secrets=false \ + --set manifests_enabled.deployment=false \ + --set ceph.namespace=ceph \ + --set network.public=$osd_public_network \ + --set network.cluster=$osd_cluster_network + +This will load the client keyring as well as the same ``ceph.conf`` into +the specified namespace. Deploying ceph.conf into this namespace allows +OpenStack services to consume this ConfigMap for their Ceph-specific +configurations. You may want to validate that Ceph is deployed successfully. For more information on this, please see the section entitled `Ceph Troubleshooting <../../operator/troubleshooting/persistent-storage.html>`__. +Ceph pool creation +------------------ + +You should now be ready to create the pools for OpenStack services to consume, +using the following commands: + +:: + + kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create volumes 8 + kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create images 8 + kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create vms 8 + MariaDB Installation and Verification ------------------------------------- -We are using Galera to cluster MariaDB and establish a quorum. To -install the MariaDB, issue the following command: +We are using Galera to cluster MariaDB. To install MariaDB, issue the following +command: :: @@ -514,43 +425,49 @@ Now you can easily install the other services simply by going in order: :: - helm install --name=keystone local/keystone --set replicas=2 --namespace=openstack + helm install --namespace=openstack --name=keystone local/keystone \ + --set replicas=2 **Install Horizon:** :: - helm install --name=horizon local/horizon --set network.enable_node_port=true --namespace=openstack + helm install --namespace=openstack --name=horizon local/horizon \ + --set network.enable_node_port=true **Install Glance:** :: - helm install --name=glance local/glance --set replicas.api=2,replicas.registry=2 --namespace=openstack + helm install --namespace=openstack --name=glance local/glance \ + --set replicas.api=2,replicas.registry=2 **Install Heat:** :: - helm install --name=heat local/heat --namespace=openstack + helm install --namespace=openstack --name=heat local/heat **Install Neutron:** :: - helm install --name=neutron local/neutron --set replicas.server=2 --namespace=openstack + helm install --namespace=openstack --name=neutron local/neutron \ + --set replicas.server=2 **Install Nova:** :: - helm install --name=nova local/nova --set control_replicas=2 --namespace=openstack + helm install --namespace=openstack --name=nova local/nova \ + --set control_replicas=2 **Install Cinder:** :: - helm install --name=cinder local/cinder --set replicas.api=2 --namespace=openstack + helm install --namespace=openstack --name=cinder local/cinder \ + --set replicas.api=2 Final Checks ------------ diff --git a/glance/templates/etc/_ceph.client.glance.keyring.tpl b/glance/templates/bin/_ceph-keyring.sh.tpl similarity index 79% rename from glance/templates/etc/_ceph.client.glance.keyring.tpl rename to glance/templates/bin/_ceph-keyring.sh.tpl index 0e0e639a2e..411b2bb639 100644 --- a/glance/templates/etc/_ceph.client.glance.keyring.tpl +++ b/glance/templates/bin/_ceph-keyring.sh.tpl @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright 2017 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,9 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -ex +export HOME=/tmp + +cat < /etc/ceph/ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring [client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}] {{- if .Values.conf.ceph.keyring }} key = {{ .Values.conf.ceph.keyring }} {{- else }} - key = {{- include "secrets/ceph-client-key" . -}} + key = $(cat /tmp/client-keyring) {{- end }} +EOF + +exit 0 diff --git a/glance/templates/configmap-bin.yaml b/glance/templates/configmap-bin.yaml index b5d90d5f2a..b104293f73 100644 --- a/glance/templates/configmap-bin.yaml +++ b/glance/templates/configmap-bin.yaml @@ -35,3 +35,5 @@ data: {{ tuple "bin/_glance-registry.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} bootstrap.sh: |+ {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-keyring.sh: |+ +{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/glance/templates/configmap-etc.yaml b/glance/templates/configmap-etc.yaml index 2999cd472b..13655b773a 100644 --- a/glance/templates/configmap-etc.yaml +++ b/glance/templates/configmap-etc.yaml @@ -78,30 +78,6 @@ data: {{- end }} {{- if .Values.conf.rally_tests.append -}} {{ .Values.conf.rally_tests.append | indent 4 }} -{{- end }} - ceph.conf: |+ -{{ if .Values.conf.ceph.override -}} -{{ .Values.conf.ceph.override | indent 4 }} -{{- else -}} -{{- if .Values.conf.ceph.prefix -}} -{{ .Values.conf.ceph.prefix | indent 4 }} -{{- end }} -{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} -{{- end }} -{{- if .Values.conf.ceph.append -}} -{{ .Values.conf.ceph.append | indent 4 }} -{{- end }} - ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring: |+ -{{ if .Values.conf.ceph_client.override -}} -{{ .Values.conf.ceph_client.override | indent 4 }} -{{- else -}} -{{- if .Values.conf.ceph_client.prefix -}} -{{ .Values.conf.ceph_client.prefix | indent 4 }} -{{- end }} -{{ tuple "etc/_ceph.client.glance.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} -{{- end }} -{{- if .Values.conf.ceph_client.append -}} -{{ .Values.conf.ceph_client.append | indent 4 }} {{- end }} glance-api.conf: |+ {{ if .Values.conf.glance.override -}} diff --git a/glance/templates/deployment-api.yaml b/glance/templates/deployment-api.yaml index 5a6c015b6d..83d5fdca62 100644 --- a/glance/templates/deployment-api.yaml +++ b/glance/templates/deployment-api.yaml @@ -43,6 +43,47 @@ spec: terminationGracePeriodSeconds: {{ .Values.termination_grace_period.api.timeout | default "600" }} initContainers: {{ tuple $envAll $dependencies $mounts_glance_api_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + {{- if eq .Values.storage "pvc" }} + - name: glance-perms + image: {{ .Values.images.api }} + imagePullPolicy: {{ .Values.images.pull_policy }} + securityContext: + runAsUser: 0 + {{- if .Values.resources.enabled }} + resources: + limits: + cpu: {{ .Values.resources.api.limits.cpu | quote }} + memory: {{ .Values.resources.api.limits.memory | quote }} + requests: + cpu: {{ .Values.resources.api.requests.cpu | quote }} + memory: {{ .Values.resources.api.requests.memory | quote }} + {{- end }} + command: + - chown + - -R + - "glance:" + - {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }} + volumeMounts: + - name: glance-images + mountPath: {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }} + {{- end }} + {{ if eq .Values.storage "ceph" }} + - name: ceph-keyring-placement + image: {{ .Values.images.api }} + imagePullPolicy: {{ .Values.images.pull_policy }} + command: + - /tmp/ceph-keyring.sh + volumeMounts: + - name: etcceph + mountPath: /etc/ceph + - name: glance-bin + mountPath: /tmp/ceph-keyring.sh + subPath: ceph-keyring.sh + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} containers: - name: glance-api image: {{ .Values.images.api }} @@ -93,13 +134,15 @@ spec: - name: glance-images mountPath: {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }} {{- else }} - - name: glance-etc + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true - - name: glance-etc - mountPath: /etc/ceph/ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring - subPath: ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key readOnly: true {{- end }} {{ if $mounts_glance_api.volumeMounts }}{{ toYaml $mounts_glance_api.volumeMounts | indent 12 }}{{ end }} @@ -117,5 +160,14 @@ spec: - name: glance-images persistentVolumeClaim: claimName: glance-images +{{ else }} + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key {{- end }} {{ if $mounts_glance_api.volumes }}{{ toYaml $mounts_glance_api.volumes | indent 8 }}{{ end }} diff --git a/glance/templates/etc/_ceph.conf.tpl b/glance/templates/etc/_ceph.conf.tpl deleted file mode 100644 index e42694fc27..0000000000 --- a/glance/templates/etc/_ceph.conf.tpl +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[global] -rgw_thread_pool_size = 1024 -rgw_num_rados_handles = 100 -{{- if .Values.conf.ceph.monitors }} -[mon] -{{ range .Values.conf.ceph.monitors }} - [mon.{{ . }}] - host = {{ . }} - mon_addr = {{ . }} -{{ end }} -{{- else }} -mon_host = ceph-mon.ceph -{{- end }} -[client] - rbd_cache_enabled = true - rbd_cache_writethrough_until_flush = true diff --git a/helm-toolkit/secrets/.gitkeep b/helm-toolkit/secrets/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/helm-toolkit/utils/secret-generator/README.rst b/helm-toolkit/utils/secret-generator/README.rst deleted file mode 100644 index ace977e6bb..0000000000 --- a/helm-toolkit/utils/secret-generator/README.rst +++ /dev/null @@ -1,78 +0,0 @@ -Ceph Kubernetes Secret Generation -================================= - -This script will generate ceph keyrings and configs as Kubernetes -secrets. - -Sigil is required for template handling and must be installed in system -``PATH``. Instructions can be found`here -`__ - -The following functions are provided: - -Generate raw FSID (can be used for other functions) ---------------------------------------------------- - -.. code:: bash - - ./generate_secrets.sh fsid - -Generate raw ceph.conf (For verification) ------------------------------------------ - -.. code:: bash - - ./generate_secrets.sh ceph-conf-raw "overridekey=value" - -Take a look at ``ceph/ceph.conf.tmpl`` for the default values - -Generate encoded ceph.conf secret ---------------------------------- - -.. code:: bash - - ./generate_secrets.sh ceph-conf "overridekey=value" - -Generate encoded admin keyring secret -------------------------------------- - -.. code:: bash - - ./generate_secrets.sh admin-keyring - -Generate encoded mon keyring secret ------------------------------------ - -.. code:: bash - - ./generate_secrets.sh mon-keyring - -Generate a combined secret --------------------------- - -Contains ceph.conf, admin keyring and mon keyring. Useful for generating -the ``/etc/ceph`` directory - -.. code:: bash - - ./generate_secrets.sh combined-conf - -Generate encoded boostrap keyring secret ----------------------------------------- - -.. code:: bash - - ./generate_secrets.sh bootstrap-keyring - -Kubernetes workflow -=================== - -.. code:: bash - - ./generator/generate_secrets.sh all `./generate_secrets.sh fsid` - - kubectl create secret generic ceph-conf-combined --from-file=ceph.conf --from-file=ceph.client.admin.keyring --from-file=ceph.mon.keyring --namespace=ceph - kubectl create secret generic ceph-bootstrap-rgw-keyring --from-file=ceph.keyring=ceph.rgw.keyring --namespace=ceph - kubectl create secret generic ceph-bootstrap-mds-keyring --from-file=ceph.keyring=ceph.mds.keyring --namespace=ceph - kubectl create secret generic ceph-bootstrap-osd-keyring --from-file=ceph.keyring=ceph.osd.keyring --namespace=ceph - kubectl create secret generic ceph-client-key --from-file=ceph-client-key --namespace=ceph diff --git a/helm-toolkit/utils/secret-generator/generate_secrets.sh b/helm-toolkit/utils/secret-generator/generate_secrets.sh deleted file mode 100755 index a74bf1d040..0000000000 --- a/helm-toolkit/utils/secret-generator/generate_secrets.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash - -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -gen-fsid() { - echo "$(uuidgen)" -} - -gen-ceph-conf-raw() { - fsid=${1:?} - shift - conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@) - echo "${conf}" -} - -gen-ceph-conf() { - fsid=${1:?} - shift - conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@) - echo "${conf}" -} - -gen-admin-keyring() { - key=$(python ceph-key.py) - keyring=$(sigil -f templates/ceph/admin.keyring.tmpl "key=${key}") - echo "${keyring}" -} - -gen-mon-keyring() { - key=$(python ceph-key.py) - keyring=$(sigil -f templates/ceph/mon.keyring.tmpl "key=${key}") - echo "${keyring}" -} - -gen-combined-conf() { - fsid=${1:?} - shift - conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@) - echo "${conf}" > ../../secrets/ceph.conf - - key=$(python ceph-key.py) - keyring=$(sigil -f templates/ceph/admin.keyring.tmpl "key=${key}") - echo "${key}" > ../../secrets/ceph-client-key - echo "${keyring}" > ../../secrets/ceph.client.admin.keyring - - key=$(python ceph-key.py) - keyring=$(sigil -f templates/ceph/mon.keyring.tmpl "key=${key}") - echo "${keyring}" > ../../secrets/ceph.mon.keyring -} - -gen-bootstrap-keyring() { - service="${1:-osd}" - key=$(python ceph-key.py) - bootstrap=$(sigil -f templates/ceph/bootstrap.keyring.tmpl "key=${key}" "service=${service}") - echo "${bootstrap}" -} - -gen-all-bootstrap-keyrings() { - gen-bootstrap-keyring osd > ../../secrets/ceph.osd.keyring - gen-bootstrap-keyring mds > ../../secrets/ceph.mds.keyring - gen-bootstrap-keyring rgw > ../../secrets/ceph.rgw.keyring -} - -gen-all() { - gen-combined-conf $@ - gen-all-bootstrap-keyrings -} - - -main() { - set -eo pipefail - case "$1" in - fsid) shift; gen-fsid $@;; - ceph-conf-raw) shift; gen-ceph-conf-raw $@;; - ceph-conf) shift; gen-ceph-conf $@;; - admin-keyring) shift; gen-admin-keyring $@;; - mon-keyring) shift; gen-mon-keyring $@;; - bootstrap-keyring) shift; gen-bootstrap-keyring $@;; - combined-conf) shift; gen-combined-conf $@;; - all) shift; gen-all $@;; - esac -} - -main "$@" diff --git a/helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl b/helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl deleted file mode 100644 index b0f04dba26..0000000000 --- a/helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[client.admin] - key = {{ $key }} - auid = 0 - caps mds = "allow" - caps mon = "allow *" - caps osd = "allow *" diff --git a/helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl b/helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl deleted file mode 100644 index 639a1010be..0000000000 --- a/helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[client.bootstrap-{{ $service }}] - key = {{ $key }} - caps mon = "allow profile bootstrap-{{ $service }}" diff --git a/helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl b/helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl deleted file mode 100644 index b7c925bf9f..0000000000 --- a/helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[global] -fsid = ${fsid:?} -cephx = ${auth_cephx:-"true"} -cephx_require_signatures = ${auth_cephx_require_signatures:-"false"} -cephx_cluster_require_signatures = ${auth_cephx_cluster_require_signatures:-"true"} -cephx_service_require_signatures = ${auth_cephx_service_require_signatures:-"false"} - -# auth -max_open_files = ${global_max_open_files:-"131072"} -osd_pool_default_pg_num = ${global_osd_pool_default_pg_num:-"128"} -osd_pool_default_pgp_num = ${global_osd_pool_default_pgp_num:-"128"} -osd_pool_default_size = ${global_osd_pool_default_size:-"3"} -osd_pool_default_min_size = ${global_osd_pool_default_min_size:-"1"} - -mon_osd_full_ratio = ${global_mon_osd_full_ratio:-".95"} -mon_osd_nearfull_ratio = ${global_mon_osd_nearfull_ratio:-".85"} - -mon_host = ${global_mon_host:-'ceph-mon'} - -[mon] -mon_osd_down_out_interval = ${mon_mon_osd_down_out_interval:-"600"} -mon_osd_min_down_reporters = ${mon_mon_osd_min_down_reporters:-"4"} -mon_clock_drift_allowed = ${mon_mon_clock_drift_allowed:-".15"} -mon_clock_drift_warn_backoff = ${mon_mon_clock_drift_warn_backoff:-"30"} -mon_osd_report_timeout = ${mon_mon_osd_report_timeout:-"300"} - - -[osd] -journal_size = ${osd_journal_size:-"100"} -cluster_network = ${osd_cluster_network:-'192.168.0.0/16'} -public_network = ${osd_public_network:-'192.168.0.0/16'} -osd_mkfs_type = ${osd_osd_mkfs_type:-"xfs"} -osd_mkfs_options_xfs = ${osd_osd_mkfs_options_xfs:-"-f -i size=2048"} -osd_mon_heartbeat_interval = ${osd_osd_mon_heartbeat_interval:-"30"} -osd_max_object_name_len = ${osd_max_object_name_len:-"256"} - -#crush -osd_pool_default_crush_rule = ${osd_pool_default_crush_rule:-"0"} -osd_crush_update_on_start = ${osd_osd_crush_update_on_start:-"true"} - -#backend -osd_objectstore = ${osd_osd_objectstore:-"filestore"} - -#performance tuning -filestore_merge_threshold = ${osd_filestore_merge_threshold:-"40"} -filestore_split_multiple = ${osd_filestore_split_multiple:-"8"} -osd_op_threads = ${osd_osd_op_threads:-"8"} -filestore_op_threads = ${osd_filestore_op_threads:-"8"} -filestore_max_sync_interval = ${osd_filestore_max_sync_interval:-"5"} -osd_max_scrubs = ${osd_osd_max_scrubs:-"1"} - - -#recovery tuning -osd_recovery_max_active = ${osd_osd_recovery_max_active:-"5"} -osd_max_backfills = ${osd_osd_max_backfills:-"2"} -osd_recovery_op_priority = ${osd_osd_recovery_op_priority:-"2"} -osd_client_op_priority = ${osd_osd_client_op_priority:-"63"} -osd_recovery_max_chunk = ${osd_osd_recovery_max_chunk:-"1048576"} -osd_recovery_threads = ${osd_osd_recovery_threads:-"1"} - -#ports -ms_bind_port_min = ${osd_ms_bind_port_min:-"6800"} -ms_bind_port_max = ${osd_ms_bind_port_max:-"7100"} - -[client] -rbd_cache_enabled = ${client_rbd_cache_enabled:-"true"} -rbd_cache_writethrough_until_flush = ${client_rbd_cache_writethrough_until_flush:-"true"} -rbd_default_features = ${client_rbd_default_features:-"1"} - -[mds] -mds_cache_size = ${mds_mds_cache_size:-"100000"} diff --git a/helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl b/helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl deleted file mode 100644 index 9a88a2421c..0000000000 --- a/helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[mon.] - key = {{ $key }} - caps mon = "allow *" diff --git a/helm-toolkit/utils/test/ceph-rbd-test.yaml b/helm-toolkit/utils/test/ceph-rbd-test.yaml deleted file mode 100644 index 32b91d2a28..0000000000 --- a/helm-toolkit/utils/test/ceph-rbd-test.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Pod -metadata: - name: ceph-rbd-test -spec: - containers: - - name: cephrbd-rw - image: busybox - command: - - sh - - -c - - while true; do sleep 1; done - volumeMounts: - - mountPath: "/mnt/cephrbd" - name: cephrbd - volumes: - - name: cephrbd - rbd: - monitors: -#This only works if you have skyDNS resolveable from the kubernetes node. Otherwise you must manually put in one or more mon pod ips. - - ceph-mon.ceph:6789 - user: admin - image: ceph-rbd-test - pool: rbd - secretRef: - name: ceph-client-key diff --git a/nova/templates/etc/_ceph.client.cinder.keyring.yaml.tpl b/nova/templates/bin/_ceph-keyring.sh.tpl similarity index 85% rename from nova/templates/etc/_ceph.client.cinder.keyring.yaml.tpl rename to nova/templates/bin/_ceph-keyring.sh.tpl index 1a0200d098..0407e50271 100644 --- a/nova/templates/etc/_ceph.client.cinder.keyring.yaml.tpl +++ b/nova/templates/bin/_ceph-keyring.sh.tpl @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright 2017 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,9 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -ex +export HOME=/tmp + +cat < /etc/ceph/ceph.client.keyring [client.{{ .Values.ceph.cinder_user }}] {{- if .Values.ceph.cinder_keyring }} key = {{ .Values.ceph.cinder_keyring }} {{- else }} - key = {{- include "secrets/ceph-client-key" . -}} + key = $(cat /tmp/client-keyring) {{- end }} +EOF + +exit 0 diff --git a/nova/templates/configmap-bin.yaml b/nova/templates/configmap-bin.yaml index 76188e5c67..aee5c9ea23 100644 --- a/nova/templates/configmap-bin.yaml +++ b/nova/templates/configmap-bin.yaml @@ -33,7 +33,7 @@ data: {{ tuple "bin/_libvirt.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- if .Values.ceph.enabled }} ceph-secret-define.sh: | -{{ tuple "bin/_ceph-secret-define.sh.tpl" . | include "helm-toolkit.template" | indent 4 }} +{{ tuple "bin/_ceph-secret-define.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} bootstrap.sh: | {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} @@ -51,3 +51,5 @@ data: {{ tuple "bin/_nova-scheduler.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} fake-iptables.sh: | {{ tuple "bin/_fake-iptables.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-keyring.sh: |+ +{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/nova/templates/configmap-etc.yaml b/nova/templates/configmap-etc.yaml index bf7e5b5801..706fb2ef41 100644 --- a/nova/templates/configmap-etc.yaml +++ b/nova/templates/configmap-etc.yaml @@ -92,8 +92,6 @@ data: {{- else -}} {{ tuple "etc/_policy.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} - ceph.client.cinder.keyring.yaml: |+ -{{ tuple "etc/_ceph.client.cinder.keyring.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} resolv.conf: |+ {{ tuple "etc/_resolv.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} libvirtd.conf: |+ diff --git a/nova/templates/daemonset-compute.yaml b/nova/templates/daemonset-compute.yaml index a6f82fea62..7f1bd6e162 100644 --- a/nova/templates/daemonset-compute.yaml +++ b/nova/templates/daemonset-compute.yaml @@ -39,6 +39,23 @@ spec: dnsPolicy: ClusterFirst initContainers: {{ tuple $envAll $dependencies $mounts_nova_compute_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + {{- if .Values.ceph.enabled }} + - name: ceph-keyring-placement + image: {{ .Values.images.compute }} + imagePullPolicy: {{ .Values.images.pull_policy }} + command: + - /tmp/ceph-keyring.sh + volumeMounts: + - name: etcceph + mountPath: /etc/ceph + - name: nova-bin + mountPath: /tmp/ceph-keyring.sh + subPath: ceph-keyring.sh + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} containers: - name: nova-compute image: {{ .Values.images.compute }} @@ -77,14 +94,18 @@ spec: mountPath: /etc/resolv.conf subPath: resolv.conf readOnly: true - - name: nova-etc + {{- if .Values.ceph.enabled }} + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true - - name: nova-etc - mountPath: /etc/ceph/ceph.client.keyring - subPath: ceph.client.keyring + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key readOnly: true + {{ end }} - mountPath: /lib/modules name: libmodules readOnly: true @@ -105,6 +126,16 @@ spec: - name: nova-etc configMap: name: nova-etc + {{- if .Values.ceph.enabled }} + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key + {{ end }} - name: libmodules hostPath: path: /lib/modules diff --git a/nova/templates/daemonset-libvirt.yaml b/nova/templates/daemonset-libvirt.yaml index 32e4b7aba3..dce550c71e 100644 --- a/nova/templates/daemonset-libvirt.yaml +++ b/nova/templates/daemonset-libvirt.yaml @@ -38,6 +38,23 @@ spec: dnsPolicy: ClusterFirst initContainers: {{ tuple $envAll $dependencies $mounts_nova_libvirt_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + {{- if .Values.ceph.enabled }} + - name: ceph-keyring-placement + image: {{ .Values.images.libvirt }} + imagePullPolicy: {{ .Values.images.pull_policy }} + command: + - /tmp/ceph-keyring.sh + volumeMounts: + - name: etcceph + mountPath: /etc/ceph + - name: nova-bin + mountPath: /tmp/ceph-keyring.sh + subPath: ceph-keyring.sh + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} containers: - name: nova-libvirt image: {{ .Values.images.libvirt }} @@ -101,13 +118,15 @@ spec: - name: cgroup mountPath: /sys/fs/cgroup {{- if .Values.ceph.enabled }} - - name: nova-etc + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true - - name: nova-etc - mountPath: /etc/ceph/ceph.client.keyring - subPath: ceph.client.keyring + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key readOnly: true - name: nova-bin mountPath: /tmp/ceph-secret-define.sh @@ -123,6 +142,16 @@ spec: - name: nova-etc configMap: name: nova-etc + {{- if .Values.ceph.enabled }} + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key + {{ end }} - name: libmodules hostPath: path: /lib/modules diff --git a/nova/templates/etc/_ceph.conf.tpl b/nova/templates/etc/_ceph.conf.tpl deleted file mode 100644 index f8fbaf64a3..0000000000 --- a/nova/templates/etc/_ceph.conf.tpl +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[global] -rgw_thread_pool_size = 1024 -rgw_num_rados_handles = 100 -{{- if .Values.ceph.enabled }} -[mon] -{{- if .Values.ceph.monitors }} -{{ range .Values.ceph.monitors }} - [mon.{{ . }}] - host = {{ . }} - mon_addr = {{ . }} -{{ end }} -{{- else }} -mon_host = ceph-mon.ceph -{{- end }} -{{- end }} -[client] - rbd_cache_enabled = true - rbd_cache_writethrough_until_flush = true diff --git a/nova/values.yaml b/nova/values.yaml index df1e99ed98..b0239e30a1 100644 --- a/nova/values.yaml +++ b/nova/values.yaml @@ -130,7 +130,6 @@ keystone: admin_region_name: "RegionOne" nova_user_role: "admin" - ceph: enabled: false monitors: [] diff --git a/tests/pvc-test.yaml b/tests/pvc-test.yaml index 475d995ece..a448ecb63e 100644 --- a/tests/pvc-test.yaml +++ b/tests/pvc-test.yaml @@ -12,15 +12,42 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + version: v0.1.0 + test: ceph + name: ceph-test-job +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: test + image: docker.io/alpine:latest + imagePullPolicy: Always + command: + - /bin/sh + - -ec + - | + echo "Ceph PVC Mount Test Passed" + volumeMounts: + - name: ceph-mount + mountPath: /mnt/ceph + volumes: + - name: ceph-mount + persistentVolumeClaim: + claimName: ceph-test --- kind: PersistentVolumeClaim apiVersion: v1 metadata: - name: pvc-test + name: ceph-test spec: - accessModes: - - ReadWriteOnce + storageClassName: general + accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 1Gi - storageClassName: general diff --git a/tools/gate/README.rst b/tools/gate/README.rst index a73422beb7..bdaed3b23e 100644 --- a/tools/gate/README.rst +++ b/tools/gate/README.rst @@ -17,6 +17,7 @@ integration test is below: export INTEGRATION=aio export INTEGRATION_TYPE=basic + export PVC_BACKEND=ceph ./tools/gate/setup_gate.sh Supported Platforms diff --git a/tools/gate/basic_launch.sh b/tools/gate/basic_launch.sh index 810c86ce1e..3a721bd19b 100755 --- a/tools/gate/basic_launch.sh +++ b/tools/gate/basic_launch.sh @@ -29,14 +29,53 @@ if [ "x$HOST_OS" == "xfedora" ]; then sudo modprobe ip6_tables fi +if [ "x$PVC_BACKEND" == "xceph" ]; then + kubectl label nodes ceph-storage=enabled --all + CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | head -1 | awk -F '/' '{ print $NF }') + kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf < \ + ${LOGS_DIR}/k8s/pvc/$NAMESPACE-$NAME.txt +done + mkdir -p ${LOGS_DIR}/k8s/rbac for OBJECT_TYPE in clusterroles \ roles \ @@ -76,5 +85,6 @@ sudo iptables-save > ${LOGS_DIR}/nodes/$(hostname)/iptables.txt sudo ip a > ${LOGS_DIR}/nodes/$(hostname)/ip.txt sudo route -n > ${LOGS_DIR}/nodes/$(hostname)/routes.txt arp -a > ${LOGS_DIR}/nodes/$(hostname)/arp.txt +cat /etc/resolv.conf > ${LOGS_DIR}/nodes/$(hostname)/resolv.conf exit $1 diff --git a/tools/gate/funcs/common.sh b/tools/gate/funcs/common.sh index 90176491e8..72d88d49f3 100644 --- a/tools/gate/funcs/common.sh +++ b/tools/gate/funcs/common.sh @@ -29,3 +29,17 @@ function base_install { iptables fi } + +function ceph_support_install { + if [ "x$HOST_OS" == "xubuntu" ]; then + sudo apt-get update -y + sudo apt-get install -y --no-install-recommends -qq \ + ceph-common + elif [ "x$HOST_OS" == "xcentos" ]; then + sudo yum install -y \ + ceph + elif [ "x$HOST_OS" == "xfedora" ]; then + sudo dnf install -y \ + ceph + fi +} diff --git a/tools/gate/funcs/kube.sh b/tools/gate/funcs/kube.sh index 75a445421d..900b49b15e 100755 --- a/tools/gate/funcs/kube.sh +++ b/tools/gate/funcs/kube.sh @@ -130,3 +130,9 @@ function kubeadm_aio_launch { kube_wait_for_pods kube-system 240 kube_wait_for_pods default 240 } + +function ceph_kube_controller_manager_replace { + sudo docker pull ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE} + IMAGE_ID=$(sudo docker images ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE} -q) + sudo docker tag ${IMAGE_ID} ${BASE_KUBE_CONTROLLER_MANAGER_IMAGE} +} diff --git a/tools/gate/funcs/network.sh b/tools/gate/funcs/network.sh index 85f724914c..1b1d0d9dfb 100755 --- a/tools/gate/funcs/network.sh +++ b/tools/gate/funcs/network.sh @@ -42,3 +42,4 @@ function net_hosts_pre_kube { function net_hosts_post_kube { sudo cp -f /etc/hosts-pre-kube /etc/hosts } + diff --git a/tools/gate/kubeadm_aio.sh b/tools/gate/kubeadm_aio.sh index 7dd8d4acb6..b60d5900dd 100755 --- a/tools/gate/kubeadm_aio.sh +++ b/tools/gate/kubeadm_aio.sh @@ -18,5 +18,11 @@ source ${WORK_DIR}/tools/gate/funcs/kube.sh kubeadm_aio_reqs_install sudo docker pull ${KUBEADM_IMAGE} || kubeadm_aio_build + +if [ "x$PVC_BACKEND" == "xceph" ]; then + ceph_kube_controller_manager_replace + sudo modprobe rbd +fi + kubeadm_aio_launch net_resolv_kube diff --git a/tools/gate/provision_gate_worker_node.sh b/tools/gate/provision_gate_worker_node.sh index 1f236163f7..e1cd5e4ee4 100755 --- a/tools/gate/provision_gate_worker_node.sh +++ b/tools/gate/provision_gate_worker_node.sh @@ -37,8 +37,15 @@ sudo rm -rfv \ /var/lib/etcd \ /var/etcd \ /var/lib/kubelet/* \ + /var/lib/nova \ + /var/lib/openstack-helm \ /run/openvswitch || true +# Load ceph kernel module if required +if [ "x$PVC_BACKEND" == "xceph" ]; then + sudo modprobe rbd +fi + # Launch Container sudo docker run \ -dt \ diff --git a/tools/gate/setup_gate.sh b/tools/gate/setup_gate.sh index e31dd17a9e..f9b72bf0d8 100755 --- a/tools/gate/setup_gate.sh +++ b/tools/gate/setup_gate.sh @@ -13,10 +13,12 @@ # limitations under the License. set -ex -export HELM_VERSION=${2:-v2.3.1} +export HELM_VERSION=${2:-v2.4.1} export KUBE_VERSION=${3:-v1.6.5} export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf -export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION} +export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION}-ceph +export BASE_KUBE_CONTROLLER_MANAGER_IMAGE=gcr.io/google_containers/kube-controller-manager-amd64:${KUBE_VERSION} +export CEPH_KUBE_CONTROLLER_MANAGER_IMAGE=quay.io/attcomdev/kube-controller-manager:${KUBE_VERSION} export WORK_DIR=$(pwd) source /etc/os-release @@ -24,6 +26,7 @@ export HOST_OS=${ID} source ${WORK_DIR}/tools/gate/funcs/common.sh source ${WORK_DIR}/tools/gate/funcs/network.sh source ${WORK_DIR}/tools/gate/funcs/helm.sh +export PVC_BACKEND=ceph # Setup the logging location: by default use the working dir as the root. export LOGS_DIR=${LOGS_DIR:-"${WORK_DIR}/logs"} @@ -34,14 +37,17 @@ function dump_logs () { } trap 'dump_logs "$?"' ERR -# Install base requirements -base_install - # Moving the ws-linter here to avoid it blocking all the jobs just for ws if [ "x$INTEGRATION_TYPE" == "xlinter" ]; then bash ${WORK_DIR}/tools/gate/whitespace.sh fi +# Install base requirements +base_install +if [ "x$PVC_BACKEND" == "xceph" ]; then + ceph_support_install +fi + # We setup the network for pre kube here, to enable cluster restarts on # development machines net_resolv_pre_kube diff --git a/tools/gate/setup_gate_worker_nodes.sh b/tools/gate/setup_gate_worker_nodes.sh index da6c6683f0..91db2904b7 100755 --- a/tools/gate/setup_gate_worker_nodes.sh +++ b/tools/gate/setup_gate_worker_nodes.sh @@ -31,6 +31,7 @@ cat /etc/nodepool/sub_nodes_private | while read SUB_NODE; do export KUBEADM_TOKEN=${KUBEADM_TOKEN}; \ export PRIMARY_NODE_IP=${PRIMARY_NODE_IP}; \ export KUBEADM_IMAGE=${KUBEADM_IMAGE}; \ + export PVC_BACKEND=${PVC_BACKEND}; \ bash ${WORK_DIR}/tools/gate/provision_gate_worker_node.sh" EOS done diff --git a/tools/images/ceph-config-helper/Dockerfile b/tools/images/ceph-config-helper/Dockerfile new file mode 100644 index 0000000000..f310618847 --- /dev/null +++ b/tools/images/ceph-config-helper/Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:16.04 +MAINTAINER pete.birley@att.com + +ARG KUBE_VERSION=v1.6.5 + +RUN set -x \ + && TMP_DIR=$(mktemp --directory) \ + && cd ${TMP_DIR} \ + && apt-get update \ + && apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + python \ + jq \ +# Install kubectl: + && curl -sSL https://dl.k8s.io/${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \ + && mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \ + && chmod +x /usr/bin/kubectl \ + && rm -rf ${TMP_DIR} diff --git a/tools/images/ceph-config-helper/README.rst b/tools/images/ceph-config-helper/README.rst new file mode 100644 index 0000000000..7b7196b368 --- /dev/null +++ b/tools/images/ceph-config-helper/README.rst @@ -0,0 +1,39 @@ +Ceph Config Helper Container +===================== + +This container builds a small image with kubectl and some other utilites for +use in the ceph-config chart. + +Instructions +------------ + +OS Specific Host setup: +~~~~~~~~~~~~~~~~~~~~~~~ + +Ubuntu: +^^^^^^^aa + +From a freshly provisioned Ubuntu 16.04 LTS host run: + +.. code:: bash + + sudo apt-get update -y + sudo apt-get install -y \ + docker.io \ + git + +Build the Ceph-Helper Image environment (optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A known good image is published to dockerhub on a fairly regular basis, but if +you wish to build your own image, from the root directory of the OpenStack-Helm +repo run: + +.. code:: bash + + export KUBE_VERSION=v1.6.5 + sudo docker build \ + --build-arg KUBE_VERSION=${KUBE_VERSION} \ + -t docker.io/port/ceph-config-helper:${KUBE_VERSION} \ + tools/images/ceph-config-helper + sudo docker push docker.io/port/ceph-config-helper:${KUBE_VERSION} diff --git a/tools/kubeadm-aio/Dockerfile b/tools/kubeadm-aio/Dockerfile index 90e9d32ea2..ef14dfc670 100644 --- a/tools/kubeadm-aio/Dockerfile +++ b/tools/kubeadm-aio/Dockerfile @@ -61,8 +61,9 @@ RUN set -x \ git \ vim \ jq \ -# Install nfs utils for development PVC provisioner +# Install utils for PVC provisioners nfs-common \ + ceph-common \ # Tweak Systemd units and targets for running in a container && find /lib/systemd/system/sysinit.target.wants/ ! -name 'systemd-tmpfiles-setup.service' -type l -exec rm -fv {} + \ && rm -fv \ diff --git a/tools/kubeadm-aio/README.rst b/tools/kubeadm-aio/README.rst index dd9ac2ce18..f2ba595249 100644 --- a/tools/kubeadm-aio/README.rst +++ b/tools/kubeadm-aio/README.rst @@ -57,7 +57,7 @@ repo run: .. code:: bash - export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.4 + export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.5 sudo docker build --pull -t ${KUBEADM_IMAGE} tools/kubeadm-aio Deploy the AIO environment diff --git a/tools/kubeadm-aio/assets/usr/bin/kubelet b/tools/kubeadm-aio/assets/usr/bin/kubelet index 0972ffc339..a851686625 100755 --- a/tools/kubeadm-aio/assets/usr/bin/kubelet +++ b/tools/kubeadm-aio/assets/usr/bin/kubelet @@ -36,7 +36,8 @@ else --restart=always \ --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \ --volume=/:/rootfs:ro \ - --volume=/dev/net:/dev/net:rw \ + --volume=/dev:/dev:rshared \ + --volume=/lib/modules:/lib/modules:ro \ --volume=/var/run/netns:/var/run/netns:rw \ --volume=/sys:/sys:ro \ --volume=/etc/machine-id:/etc/machine-id:ro \ diff --git a/tools/kubeadm-aio/kubeadm-aio-launcher.sh b/tools/kubeadm-aio/kubeadm-aio-launcher.sh index d9bab230e9..1a9224befa 100755 --- a/tools/kubeadm-aio/kubeadm-aio-launcher.sh +++ b/tools/kubeadm-aio/kubeadm-aio-launcher.sh @@ -33,6 +33,7 @@ sudo rm -rfv \ /run/openvswitch \ /var/lib/nova \ ${HOME}/.kubeadm-aio/admin.conf \ + /var/lib/openstack-helm \ /var/lib/nfs-provisioner || true # Launch Container diff --git a/helm-toolkit/utils/secret-generator/ceph-key.py b/tools/overrides/mvp/cinder.yaml similarity index 58% rename from helm-toolkit/utils/secret-generator/ceph-key.py rename to tools/overrides/mvp/cinder.yaml index 761f41a86f..bc44c55300 100644 --- a/helm-toolkit/utils/secret-generator/ceph-key.py +++ b/tools/overrides/mvp/cinder.yaml @@ -1,5 +1,3 @@ -#!/bin/python - # Copyright 2017 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,17 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import struct -import time -import base64 +# MVP values for glance. +# This file contains overrides to launch a MVP deployment of glance for the +# OpenStack-Helm Single node gates, and local development use. It should be +# kept to the bare minimum required for this purpose. + +storage: "gate" + +conf: + cinder: + default: + cinder: + backup_driver: "cinder.backup.drivers.nfs" + backends: + rbd1: + volume_driver: "cinder.volume.drivers.nfs.NfsDriver" -key = os.urandom(16) -header = struct.pack( - '