From d4d99cf9fcdf9302975e1434dacc525af0f35775 Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Mon, 9 Jul 2018 17:44:24 -0500 Subject: [PATCH] Split out RGW from ceph client charts Co-Authored-By: Chinasubbareddy Mallavarapu Change-Id: I8b091493bb79c7eab525fbaf7801028398772c5d Signed-off-by: Pete Birley --- ceph-client/templates/bin/_init-dirs.sh.tpl | 9 +- ceph-client/templates/configmap-bin.yaml | 5 - .../templates/configmap-etc-client.yaml | 4 - ceph-client/values.yaml | 160 ------- ceph-rgw/Chart.yaml | 18 + ceph-rgw/requirements.yaml | 18 + .../templates/bin/_ceph-admin-keyring.sh.tpl | 27 ++ .../bin/_ceph-rgw-storage-init.sh.tpl | 59 +++ ceph-rgw/templates/bin/_init-dirs.sh.tpl | 41 ++ .../templates/bin/rgw/_init_keystone.sh.tpl | 0 .../templates/bin/rgw/_start.sh.tpl | 0 .../templates/configmap-bin-ks.yaml | 2 +- ceph-rgw/templates/configmap-bin.yaml | 42 ++ .../configmap-ceph-rgw-templates.yaml | 27 ++ ceph-rgw/templates/configmap-etc-client.yaml | 55 +++ .../templates/deployment-rgw.yaml | 25 +- .../templates/ingress-rgw.yaml | 0 .../templates/job-ks-endpoints.yaml | 2 +- .../templates/job-ks-service.yaml | 2 +- .../templates/job-ks-user.yaml | 2 +- ceph-rgw/templates/job-rgw-storage-init.yaml | 133 ++++++ .../templates/secret-ingress-tls.yaml | 0 .../templates/secret-keystone-rgw.yaml | 0 .../templates/secret-keystone.yaml | 0 .../templates/service-ingress-rgw.yaml | 0 .../templates/service-rgw.yaml | 0 ceph-rgw/values.yaml | 432 ++++++++++++++++++ .../install/developer/deploy-with-ceph.rst | 4 +- doc/source/install/multinode.rst | 4 +- .../armada/multinode/armada-osh.yaml | 8 +- .../developer/ceph/110-ceph-radosgateway.sh | 9 +- .../multinode/090-ceph-radosgateway.sh | 6 +- 32 files changed, 884 insertions(+), 210 deletions(-) create mode 100644 ceph-rgw/Chart.yaml create mode 100644 ceph-rgw/requirements.yaml create mode 100644 ceph-rgw/templates/bin/_ceph-admin-keyring.sh.tpl create mode 100644 ceph-rgw/templates/bin/_ceph-rgw-storage-init.sh.tpl create mode 100644 ceph-rgw/templates/bin/_init-dirs.sh.tpl rename {ceph-client => ceph-rgw}/templates/bin/rgw/_init_keystone.sh.tpl (100%) rename {ceph-client => ceph-rgw}/templates/bin/rgw/_start.sh.tpl (100%) rename {ceph-client => ceph-rgw}/templates/configmap-bin-ks.yaml (97%) create mode 100644 ceph-rgw/templates/configmap-bin.yaml create mode 100644 ceph-rgw/templates/configmap-ceph-rgw-templates.yaml create mode 100644 ceph-rgw/templates/configmap-etc-client.yaml rename {ceph-client => ceph-rgw}/templates/deployment-rgw.yaml (90%) rename {ceph-client => ceph-rgw}/templates/ingress-rgw.yaml (100%) rename {ceph-client => ceph-rgw}/templates/job-ks-endpoints.yaml (84%) rename {ceph-client => ceph-rgw}/templates/job-ks-service.yaml (84%) rename {ceph-client => ceph-rgw}/templates/job-ks-user.yaml (86%) create mode 100644 ceph-rgw/templates/job-rgw-storage-init.yaml rename {ceph-client => ceph-rgw}/templates/secret-ingress-tls.yaml (100%) rename {ceph-client => ceph-rgw}/templates/secret-keystone-rgw.yaml (100%) rename {ceph-client => ceph-rgw}/templates/secret-keystone.yaml (100%) rename {ceph-client => ceph-rgw}/templates/service-ingress-rgw.yaml (100%) rename {ceph-client => ceph-rgw}/templates/service-rgw.yaml (100%) create mode 100644 ceph-rgw/values.yaml diff --git a/ceph-client/templates/bin/_init-dirs.sh.tpl b/ceph-client/templates/bin/_init-dirs.sh.tpl index 3db82a767d..dd186d4c0a 100644 --- a/ceph-client/templates/bin/_init-dirs.sh.tpl +++ b/ceph-client/templates/bin/_init-dirs.sh.tpl @@ -20,27 +20,22 @@ set -ex export LC_ALL=C : "${HOSTNAME:=$(uname -n)}" : "${MGR_NAME:=${HOSTNAME}}" -: "${RGW_NAME:=${HOSTNAME}}" : "${MDS_NAME:=mds-${HOSTNAME}}" : "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}" -: "${RGW_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-rgw/${CLUSTER}.keyring}" : "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}" -for keyring in ${OSD_BOOTSTRAP_KEYRING} ${MDS_BOOTSTRAP_KEYRING} ${RGW_BOOTSTRAP_KEYRING}; do +for keyring in ${OSD_BOOTSTRAP_KEYRING} ${MDS_BOOTSTRAP_KEYRING}; do mkdir -p "$(dirname "$keyring")" done # Let's create the ceph directories -for DIRECTORY in mds radosgw tmp mgr; do +for DIRECTORY in mds tmp mgr; do mkdir -p "/var/lib/ceph/${DIRECTORY}" done # Create socket directory mkdir -p /run/ceph -# Creating rados directories -mkdir -p "/var/lib/ceph/radosgw/${RGW_NAME}" - # Create the MDS directory mkdir -p "/var/lib/ceph/mds/${CLUSTER}-${MDS_NAME}" diff --git a/ceph-client/templates/configmap-bin.yaml b/ceph-client/templates/configmap-bin.yaml index 0edfe1f1b2..d4f31c0a85 100644 --- a/ceph-client/templates/configmap-bin.yaml +++ b/ceph-client/templates/configmap-bin.yaml @@ -48,9 +48,4 @@ data: mgr-check.sh: | {{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - rgw-start.sh: | -{{ tuple "bin/rgw/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - rgw-init-keystone.sh: | -{{ tuple "bin/rgw/_init_keystone.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - {{- end }} diff --git a/ceph-client/templates/configmap-etc-client.yaml b/ceph-client/templates/configmap-etc-client.yaml index fb2a5adc8e..7464532a31 100644 --- a/ceph-client/templates/configmap-etc-client.yaml +++ b/ceph-client/templates/configmap-etc-client.yaml @@ -39,10 +39,6 @@ limitations under the License. {{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}} {{- end -}} -{{- if empty .Values.conf.rgw_ks.config.rgw_swift_url -}} -{{- $_ := tuple "object_store" "public" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.object_store.path.default | set .Values.conf.rgw_ks.config "rgw_swift_url" -}} -{{- end -}} - --- apiVersion: v1 kind: ConfigMap diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml index 2f5039b8cc..218cb48796 100644 --- a/ceph-client/values.yaml +++ b/ceph-client/values.yaml @@ -19,7 +19,6 @@ deployment: ceph: true - rgw_keystone_user_and_endpoints: false release_group: null @@ -31,11 +30,7 @@ images: ceph_mds: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' ceph_mgr: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' ceph_rbd_pool: 'docker.io/port/ceph-config-helper:v1.10.3' - ceph_rgw: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1' - ks_endpoints: 'docker.io/openstackhelm/heat:newton' - ks_service: 'docker.io/openstackhelm/heat:newton' - ks_user: 'docker.io/openstackhelm/heat:newton' image_repo_sync: docker.io/docker:17.07.0 local_registry: active: false @@ -53,9 +48,6 @@ labels: mds: node_selector_key: ceph-mds node_selector_value: enabled - rgw: - node_selector_key: ceph-rgw - node_selector_value: enabled mgr: node_selector_key: ceph-mgr node_selector_value: enabled @@ -65,7 +57,6 @@ pod: replicas: mds: 2 mgr: 2 - rgw: 2 affinity: anti: type: @@ -81,13 +72,6 @@ pod: limits: memory: "50Mi" cpu: "500m" - rgw: - requests: - memory: "5Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" mgr: requests: memory: "5Mi" @@ -103,27 +87,6 @@ pod: requests: memory: "128Mi" cpu: "500m" - ks_endpoints: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "1024Mi" - cpu: "2000m" - ks_service: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "1024Mi" - cpu: "2000m" - ks_user: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "1024Mi" - cpu: "2000m" image_repo_sync: requests: memory: "128Mi" @@ -140,37 +103,14 @@ secrets: rgw: ceph-bootstrap-rgw-keyring mgr: ceph-bootstrap-mgr-keyring admin: ceph-client-admin-keyring - identity: - admin: ceph-keystone-admin - swift: ceph-keystone-user - user_rgw: ceph-keystone-user-rgw - tls: - object_store: - api: - public: ceph-tls-public network: - api: - ingress: - public: true - classes: - namespace: "nginx" - cluster: "nginx-cluster" - annotations: - nginx.ingress.kubernetes.io/rewrite-target: / - nginx.ingress.kubernetes.io/proxy-body-size: "0" - nginx.org/proxy-max-temp-file-size: "0" - external_policy_local: false - node_port: - enabled: false - port: 30004 public: 192.168.0.0/16 cluster: 192.168.0.0/16 conf: features: mds: true - rgw: true mgr: true pool: #NOTE(portdirect): this drives a simple approximation of @@ -269,18 +209,6 @@ conf: application: rgw replication: 3 percent_total_data: 34.8 - rgw_ks: - enabled: false - config: - rgw_keystone_api_version: 3 - rgw_keystone_accepted_roles: "admin, member" - rgw_keystone_implicit_tenants: true - rgw_keystone_make_new_tenants: true - rgw_s3_auth_use_keystone: true - rgw_swift_account_in_url: true - rgw_swift_url: null - #NOTE (portdirect): See http://tracker.ceph.com/issues/21226 - rgw_keystone_token_cache_size: 0 ceph: global: # auth @@ -318,20 +246,6 @@ dependencies: services: - endpoint: internal service: ceph_mon - ks_endpoints: - jobs: - - ceph-ks-service - services: - - endpoint: internal - service: identity - ks_service: - services: - - endpoint: internal - service: identity - ks_user: - services: - - endpoint: internal - service: identity mds: jobs: - ceph-storage-keys-generator @@ -361,14 +275,6 @@ dependencies: services: - endpoint: internal service: ceph_mon - rgw: - jobs: - - ceph-storage-keys-generator - - ceph-rgw-keyring-generator - - ceph-rbd-pool - services: - - endpoint: internal - service: ceph_mon image_repo_sync: services: - endpoint: internal @@ -425,61 +331,6 @@ endpoints: port: registry: node: 5000 - identity: - name: keystone - namespace: null - auth: - admin: - region_name: RegionOne - username: admin - password: password - project_name: admin - user_domain_name: default - project_domain_name: default - swift: - role: admin - region_name: RegionOne - username: swift - password: password - project_name: service - user_domain_name: service - project_domain_name: service - hosts: - default: keystone-api - public: keystone - host_fqdn_override: - default: null - path: - default: /v3 - scheme: - default: http - port: - api: - default: 80 - internal: 5000 - object_store: - name: swift - namespace: null - hosts: - default: ceph-rgw - public: radosgw - host_fqdn_override: - default: null - # NOTE(portdirect): this chart supports TLS for fqdn over-ridden public - # endpoints using the following format: - # public: - # host: null - # tls: - # crt: null - # key: null - path: - default: /swift/v1/KEY_$(tenant_id)s - scheme: - default: http - port: - api: - default: 8088 - public: 80 ceph_mon: namespace: null hosts: @@ -513,22 +364,11 @@ monitoring: manifests: configmap_bin: true - configmap_bin_ks: true configmap_etc: true deployment_mds: true - deployment_rgw: true deployment_mgr: true - ingress_rgw: true job_bootstrap: false job_cephfs_client_key: true job_image_repo_sync: true - job_ks_endpoints: true - job_ks_service: true - job_ks_user: true job_rbd_pool: true - secret_ingress_tls: true - secret_keystone_rgw: true - secret_keystone: true - service_ingress_rgw: true service_mgr: true - service_rgw: true diff --git a/ceph-rgw/Chart.yaml b/ceph-rgw/Chart.yaml new file mode 100644 index 0000000000..7e9c7d809c --- /dev/null +++ b/ceph-rgw/Chart.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Ceph RadosGW +name: ceph-rgw +version: 0.1.0 diff --git a/ceph-rgw/requirements.yaml b/ceph-rgw/requirements.yaml new file mode 100644 index 0000000000..53782e69b2 --- /dev/null +++ b/ceph-rgw/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/ceph-rgw/templates/bin/_ceph-admin-keyring.sh.tpl b/ceph-rgw/templates/bin/_ceph-admin-keyring.sh.tpl new file mode 100644 index 0000000000..8384abf4e9 --- /dev/null +++ b/ceph-rgw/templates/bin/_ceph-admin-keyring.sh.tpl @@ -0,0 +1,27 @@ +#!/bin/bash + +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export HOME=/tmp + +cat < /etc/ceph/ceph.client.admin.keyring +[client.admin] + key = $(cat /tmp/client-keyring) +EOF + +exit 0 diff --git a/ceph-rgw/templates/bin/_ceph-rgw-storage-init.sh.tpl b/ceph-rgw/templates/bin/_ceph-rgw-storage-init.sh.tpl new file mode 100644 index 0000000000..81236a115c --- /dev/null +++ b/ceph-rgw/templates/bin/_ceph-rgw-storage-init.sh.tpl @@ -0,0 +1,59 @@ +#!/bin/bash + +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -x +if [ "x$STORAGE_BACKEND" == "xceph-rgw" ]; then + SECRET=$(mktemp --suffix .yaml) + KEYRING=$(mktemp --suffix .keyring) + function cleanup { + rm -f ${SECRET} ${KEYRING} + } + trap cleanup EXIT +fi + +function kube_ceph_keyring_gen () { + CEPH_KEY=$1 + CEPH_KEY_TEMPLATE=$2 + sed "s|{{"{{"}} key {{"}}"}}|${CEPH_KEY}|" /tmp/ceph-templates/${CEPH_KEY_TEMPLATE} | base64 -w0 | tr -d '\n' +} + +set -ex +if [ "x$STORAGE_BACKEND" == "xceph-rgw" ]; then + ceph -s + if USERINFO=$(ceph auth get client.bootstrap-rgw); then + KEYSTR=$(echo $USERINFO | sed 's/.*\( key = .*\) caps mon.*/\1/') + echo $KEYSTR > ${KEYRING} + else + #NOTE(Portdirect): Determine proper privs to assign keyring + ceph auth get-or-create client.bootstrap-rgw \ + mon "allow profile bootstrap-rgw" \ + -o ${KEYRING} + fi + FINAL_KEYRING=$(sed -n 's/^[[:blank:]]*key[[:blank:]]\+=[[:blank:]]\(.*\)/\1/p' ${KEYRING}) + cat > ${SECRET} <