From 05cad716e515b203fbe7b6890af02d6c31735319 Mon Sep 17 00:00:00 2001 From: "Ritchie, Frank (fr801x)" Date: Mon, 8 Mar 2021 11:10:33 -0500 Subject: [PATCH] Add support for rgw placement targets This PS adds support for rgw placement targets: https://docs.ceph.com/en/latest/radosgw/placement/#placement-targets Change-Id: I6fc643994dcf2c15a04f07b8703968a76c009c18 --- ceph-rgw/Chart.yaml | 2 +- .../bin/_create-rgw-placement-targets.sh.tpl | 48 +++++++ ceph-rgw/templates/configmap-bin.yaml | 2 + .../templates/job-rgw-placement-targets.yaml | 131 ++++++++++++++++++ ceph-rgw/values.yaml | 26 ++++ releasenotes/notes/ceph-rgw.yaml | 1 + 6 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 ceph-rgw/templates/bin/_create-rgw-placement-targets.sh.tpl create mode 100644 ceph-rgw/templates/job-rgw-placement-targets.yaml diff --git a/ceph-rgw/Chart.yaml b/ceph-rgw/Chart.yaml index 916248c84..8e6b9ac74 100644 --- a/ceph-rgw/Chart.yaml +++ b/ceph-rgw/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph RadosGW name: ceph-rgw -version: 0.1.3 +version: 0.1.4 home: https://github.com/ceph/ceph ... diff --git a/ceph-rgw/templates/bin/_create-rgw-placement-targets.sh.tpl b/ceph-rgw/templates/bin/_create-rgw-placement-targets.sh.tpl new file mode 100644 index 000000000..7f3b6d78d --- /dev/null +++ b/ceph-rgw/templates/bin/_create-rgw-placement-targets.sh.tpl @@ -0,0 +1,48 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e + +function create_rgw_placement_target () { + echo "Creating rgw placement target $2" + radosgw-admin zonegroup placement add \ + --rgw-zonegroup "$1" \ + --placement-id "$2" +} + +function add_rgw_zone_placement () { + echo "Adding rgw zone placement for placement target $2 data pool $3" + radosgw-admin zone placement add \ + --rgw-zone $1 \ + --placement-id "$2" \ + --data-pool "$3" \ + --index-pool "$4" \ + --data-extra-pool "$5" +} + +{{- range $i, $placement_target := .Values.conf.rgw_placement_targets }} +RGW_PLACEMENT_TARGET={{ $placement_target.name | quote }} +RGW_PLACEMENT_TARGET_DATA_POOL={{ $placement_target.data_pool | quote }} +RGW_PLACEMENT_TARGET_INDEX_POOL={{ $placement_target.index_pool | default "default.rgw.buckets.index" | quote }} +RGW_PLACEMENT_TARGET_DATA_EXTRA_POOL={{ $placement_target.data_extra_pool | default "default.rgw.buckets.non-ec" | quote }} +RGW_ZONEGROUP={{ $placement_target.zonegroup | default "default" | quote }} +RGW_ZONE={{ $placement_target.zone | default "default" | quote }} +RGW_PLACEMENT_TARGET_EXISTS=$(radosgw-admin zonegroup placement get --placement-id "$RGW_PLACEMENT_TARGET" 2>/dev/null || true) +if [[ -z "$RGW_PLACEMENT_TARGET_EXISTS" ]]; then + create_rgw_placement_target "$RGW_ZONEGROUP" "$RGW_PLACEMENT_TARGET" + add_rgw_zone_placement "$RGW_ZONE" "$RGW_PLACEMENT_TARGET" "$RGW_PLACEMENT_TARGET_DATA_POOL" "$RGW_PLACEMENT_TARGET_INDEX_POOL" "$RGW_PLACEMENT_TARGET_DATA_EXTRA_POOL" +fi +{{- end }} diff --git a/ceph-rgw/templates/configmap-bin.yaml b/ceph-rgw/templates/configmap-bin.yaml index e8aaa8bc3..4a0212780 100644 --- a/ceph-rgw/templates/configmap-bin.yaml +++ b/ceph-rgw/templates/configmap-bin.yaml @@ -45,6 +45,8 @@ data: {{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} rgw-s3-admin.sh: | {{- include "helm-toolkit.scripts.create_s3_user" . | indent 4 }} + create-rgw-placement-targets.sh: | +{{ tuple "bin/_create-rgw-placement-targets.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} helm-tests.sh: | {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/ceph-rgw/templates/job-rgw-placement-targets.yaml b/ceph-rgw/templates/job-rgw-placement-targets.yaml new file mode 100644 index 000000000..9a5155a69 --- /dev/null +++ b/ceph-rgw/templates/job-rgw-placement-targets.yaml @@ -0,0 +1,131 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_rgw_placement_targets .Values.conf.features.rgw }} +{{- $envAll := . }} + +{{- $serviceAccountName := "rgw-placement-targets" }} +{{ tuple $envAll "rgw_placement_targets" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-rgw-placement-targets + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "rgw-placement-targets" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-rgw-placement-targets" "containerNames" (list "ceph-keyring-placement" "init" "create-rgw-placement-targets") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "rgw_placement_targets" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "rgw_placement_targets" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: ceph-keyring-placement +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw_placement_targets" "container" "keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/ceph-admin-keyring.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/ceph-admin-keyring.sh + subPath: ceph-admin-keyring.sh + readOnly: true + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + containers: + - name: create-rgw-placement-targets + image: {{ .Values.images.tags.rgw_placement_targets }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.rgw_placement_targets | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw_placement_targets" "container" "create_rgw_placement_targets" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/create-rgw-placement-targets.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/create-rgw-placement-targets.sh + subPath: create-rgw-placement-targets.sh + readOnly: true + - name: ceph-rgw-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-rgw-bin + configMap: + name: ceph-rgw-bin + defaultMode: 0555 + - name: ceph-rgw-etc + configMap: + name: ceph-rgw-etc + defaultMode: 0444 + - name: ceph-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin | quote }} +{{- end }} diff --git a/ceph-rgw/values.yaml b/ceph-rgw/values.yaml index 19da50477..b9e29265e 100644 --- a/ceph-rgw/values.yaml +++ b/ceph-rgw/values.yaml @@ -30,6 +30,7 @@ images: dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0' image_repo_sync: 'docker.io/docker:17.07.0' rgw_s3_admin: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113' + rgw_placement_targets: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113' ks_endpoints: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial' ks_service: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial' ks_user: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial' @@ -92,6 +93,16 @@ pod: create_s3_admin: allowPrivilegeEscalation: false readOnlyRootFilesystem: true + rgw_placement_targets: + pod: + runAsUser: 64045 + container: + keyring_placement: + runAsUser: 0 + readOnlyRootFilesystem: true + create_rgw_placement_targets: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true rgw_test: pod: runAsUser: 64045 @@ -190,6 +201,13 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + rgw_placement_targets: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" rgw_restart: limits: memory: "1024Mi" @@ -379,6 +397,9 @@ conf: application: rgw replication: 3 percent_total_data: 34.8 + rgw_placement_targets: + - name: default-placement + data_pool: default.rgw.buckets.data rgw: config: # NOTE (portdirect): See http://tracker.ceph.com/issues/21226 @@ -478,6 +499,10 @@ dependencies: services: - endpoint: internal service: ceph_object_store + rgw_placement_targets: + services: + - endpoint: internal + service: ceph_object_store tests: services: - endpoint: internal @@ -639,6 +664,7 @@ manifests: job_ks_service: true job_ks_user: true job_s3_admin: true + job_rgw_placement_targets: false secret_s3_rgw: true secret_keystone_rgw: true secret_ingress_tls: true diff --git a/releasenotes/notes/ceph-rgw.yaml b/releasenotes/notes/ceph-rgw.yaml index 237d1f7f0..23c3c3e69 100644 --- a/releasenotes/notes/ceph-rgw.yaml +++ b/releasenotes/notes/ceph-rgw.yaml @@ -4,4 +4,5 @@ ceph-rgw: - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" - 0.1.2 Uplift from Nautilus to Octopus release - 0.1.3 update rbac api version + - 0.1.4 Rgw placement target support ...