Add support for rgw placement targets

This PS adds support for rgw placement targets:

https://docs.ceph.com/en/latest/radosgw/placement/#placement-targets

Change-Id: I6fc643994dcf2c15a04f07b8703968a76c009c18

ceph-rgw/Chart.yaml

@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph RadosGW
 name: ceph-rgw
-version: 0.1.3
+version: 0.1.4
 home: https://github.com/ceph/ceph
 ...

ceph-rgw/templates/bin/_create-rgw-placement-targets.sh.tpl

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -e
+
+function create_rgw_placement_target () {
+  echo "Creating rgw placement target $2"
+  radosgw-admin zonegroup placement add \
+    --rgw-zonegroup "$1" \
+    --placement-id "$2"
+}
+
+function add_rgw_zone_placement () {
+  echo "Adding rgw zone placement for placement target $2 data pool $3"
+  radosgw-admin zone placement add \
+    --rgw-zone $1 \
+    --placement-id "$2" \
+    --data-pool "$3" \
+    --index-pool "$4" \
+    --data-extra-pool "$5"
+}
+
+{{- range $i, $placement_target := .Values.conf.rgw_placement_targets }}
+RGW_PLACEMENT_TARGET={{ $placement_target.name | quote }}
+RGW_PLACEMENT_TARGET_DATA_POOL={{ $placement_target.data_pool | quote }}
+RGW_PLACEMENT_TARGET_INDEX_POOL={{ $placement_target.index_pool | default "default.rgw.buckets.index" | quote }}
+RGW_PLACEMENT_TARGET_DATA_EXTRA_POOL={{ $placement_target.data_extra_pool | default "default.rgw.buckets.non-ec" | quote }}
+RGW_ZONEGROUP={{ $placement_target.zonegroup | default "default" | quote }}
+RGW_ZONE={{ $placement_target.zone | default "default" | quote }}
+RGW_PLACEMENT_TARGET_EXISTS=$(radosgw-admin zonegroup placement get --placement-id "$RGW_PLACEMENT_TARGET" 2>/dev/null || true)
+if [[ -z "$RGW_PLACEMENT_TARGET_EXISTS" ]]; then
+  create_rgw_placement_target "$RGW_ZONEGROUP" "$RGW_PLACEMENT_TARGET"
+  add_rgw_zone_placement "$RGW_ZONE" "$RGW_PLACEMENT_TARGET" "$RGW_PLACEMENT_TARGET_DATA_POOL" "$RGW_PLACEMENT_TARGET_INDEX_POOL" "$RGW_PLACEMENT_TARGET_DATA_EXTRA_POOL"
+fi
+{{- end }}

ceph-rgw/templates/configmap-bin.yaml

@@ -45,6 +45,8 @@ data:
 {{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   rgw-s3-admin.sh: |
 {{- include "helm-toolkit.scripts.create_s3_user" . | indent 4 }}
+  create-rgw-placement-targets.sh: |
+{{ tuple "bin/_create-rgw-placement-targets.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   helm-tests.sh: |
 {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}

ceph-rgw/templates/job-rgw-placement-targets.yaml

@@ -0,0 +1,131 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.job_rgw_placement_targets .Values.conf.features.rgw }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "rgw-placement-targets" }}
+{{ tuple $envAll "rgw_placement_targets" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+      - update
+      - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-rgw-placement-targets
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "ceph" "rgw-placement-targets" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+      annotations:
+{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
+{{ dict "envAll" $envAll "podName" "ceph-rgw-placement-targets" "containerNames" (list "ceph-keyring-placement" "init" "create-rgw-placement-targets") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
+    spec:
+{{ dict "envAll" $envAll "application" "rgw_placement_targets" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+      initContainers:
+{{ tuple $envAll "rgw_placement_targets" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+        - name: ceph-keyring-placement
+{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ dict "envAll" $envAll "application" "rgw_placement_targets" "container" "keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - /tmp/ceph-admin-keyring.sh
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: pod-etc-ceph
+              mountPath: /etc/ceph
+            - name: ceph-rgw-bin
+              mountPath: /tmp/ceph-admin-keyring.sh
+              subPath: ceph-admin-keyring.sh
+              readOnly: true
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+      containers:
+        - name: create-rgw-placement-targets
+          image: {{ .Values.images.tags.rgw_placement_targets }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.rgw_placement_targets | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "rgw_placement_targets" "container" "create_rgw_placement_targets" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - /tmp/create-rgw-placement-targets.sh
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: pod-etc-ceph
+              mountPath: /etc/ceph
+            - name: ceph-rgw-bin
+              mountPath: /tmp/create-rgw-placement-targets.sh
+              subPath: create-rgw-placement-targets.sh
+              readOnly: true
+            - name: ceph-rgw-etc
+              mountPath: /etc/ceph/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+      volumes:
+        - name: pod-tmp
+          emptyDir: {}
+        - name: pod-etc-ceph
+          emptyDir: {}
+        - name: ceph-rgw-bin
+          configMap:
+            name: ceph-rgw-bin
+            defaultMode: 0555
+        - name: ceph-rgw-etc
+          configMap:
+            name: ceph-rgw-etc
+            defaultMode: 0444
+        - name: ceph-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.admin | quote }}
+{{- end }}

ceph-rgw/values.yaml

@@ -30,6 +30,7 @@ images:
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/docker:17.07.0'
     rgw_s3_admin: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
+    rgw_placement_targets: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
     ks_endpoints: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial'
     ks_service: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial'
     ks_user: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial'
@@ -92,6 +93,16 @@ pod:
         create_s3_admin:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
+    rgw_placement_targets:
+      pod:
+        runAsUser: 64045
+      container:
+        keyring_placement:
+          runAsUser: 0
+          readOnlyRootFilesystem: true
+        create_rgw_placement_targets:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
     rgw_test:
       pod:
         runAsUser: 64045
@@ -190,6 +201,13 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      rgw_placement_targets:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
       rgw_restart:
         limits:
           memory: "1024Mi"
@@ -379,6 +397,9 @@ conf:
         application: rgw
         replication: 3
         percent_total_data: 34.8
+  rgw_placement_targets:
+    - name: default-placement
+      data_pool: default.rgw.buckets.data
   rgw:
     config:
       # NOTE (portdirect): See http://tracker.ceph.com/issues/21226
@@ -478,6 +499,10 @@ dependencies:
       services:
         - endpoint: internal
           service: ceph_object_store
+    rgw_placement_targets:
+      services:
+        - endpoint: internal
+          service: ceph_object_store
     tests:
       services:
         - endpoint: internal
@@ -639,6 +664,7 @@ manifests:
   job_ks_service: true
   job_ks_user: true
   job_s3_admin: true
+  job_rgw_placement_targets: false
   secret_s3_rgw: true
   secret_keystone_rgw: true
   secret_ingress_tls: true

releasenotes/notes/ceph-rgw.yaml

@@ -4,4 +4,5 @@ ceph-rgw:
   - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0"
   - 0.1.2 Uplift from Nautilus to Octopus release
   - 0.1.3 update rbac api version
+  - 0.1.4 Rgw placement target support
 ...