Elasticsearch S3 Update

This change updates how the Elasticsearch chart handles
S3 configuration and snapshot repository registration.

This allows for
  - Multiple snapshot destinations to be configued
  - Repositories to use a specific placement target
  - Management of multiple account credentials

Change-Id: I12de918adc5964a4ded46f6f6cd3fa94c7235112

elasticsearch/Chart.yaml

@@ -12,10 +12,10 @@
 
 ---
 apiVersion: v1
-appVersion: v7.1.0
+appVersion: v7.6.2
 description: OpenStack-Helm ElasticSearch
 name: elasticsearch
-version: 0.1.8
+version: 0.2.0
 home: https://www.elastic.co/
 sources:
   - https://github.com/elastic/elasticsearch

elasticsearch/templates/bin/_create_s3_users.sh.tpl

@@ -0,0 +1,75 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+   http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+#!/bin/bash
+
+set -e
+
+function create_s3_user () {
+  echo "Creating s3 user and key pair"
+  radosgw-admin user create \
+    --uid=${S3_USERNAME} \
+    --display-name=${S3_USERNAME} \
+    --key-type=s3 \
+    --access-key ${S3_ACCESS_KEY} \
+    --secret-key ${S3_SECRET_KEY}
+}
+
+function update_s3_user () {
+  # Retrieve old access keys, if they exist
+  old_access_keys=$(radosgw-admin user info --uid=${S3_USERNAME} \
+    | jq -r '.keys[].access_key' || true)
+  if [[ ! -z ${old_access_keys} ]]; then
+    for access_key in $old_access_keys; do
+      # If current access key is the same as the key supplied, do nothing.
+      if [ "$access_key" == "${S3_ACCESS_KEY}" ]; then
+        echo "Current user and key pair exists."
+        continue
+      else
+        # If keys differ, remove previous key
+        radosgw-admin key rm --uid=${S3_USERNAME} --key-type=s3 --access-key=$access_key
+      fi
+    done
+  fi
+  # Perform one more additional check to account for scenarios where multiple
+  # key pairs existed previously, but one existing key was the supplied key
+  current_access_key=$(radosgw-admin user info --uid=${S3_USERNAME} \
+    | jq -r '.keys[].access_key' || true)
+  # If the supplied key does not exist, modify the user
+  if [[ -z ${current_access_key} ]]; then
+    # Modify user with new access and secret keys
+    echo "Updating existing user's key pair"
+    radosgw-admin user modify \
+      --uid=${S3_USERNAME}\
+      --access-key ${S3_ACCESS_KEY} \
+      --secret-key ${S3_SECRET_KEY}
+  fi
+}
+
+{{- range $client, $config := .Values.storage.s3.clients -}}
+{{- if $config.create_user | default false }}
+
+S3_USERNAME=${{ printf "%s_S3_USERNAME" ($client | replace "-" "_" | upper)  }}
+S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ($client | replace "-" "_" | upper)  }}
+S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ($client | replace "-" "_" | upper)  }}
+
+user_exists=$(radosgw-admin user info --uid=${S3_USERNAME} || true)
+if [[ -z ${user_exists} ]]; then
+  echo "Creating $S3_USERNAME"
+  create_s3_user > /dev/null 2>&1
+else
+  echo "Updating $S3_USERNAME"
+  update_s3_user > /dev/null 2>&1
+fi
+
+{{- end }}
+{{- end }}

elasticsearch/templates/bin/_create_template.sh.tpl

@@ -1,4 +1,17 @@
 #!/bin/bash
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
 
 set -ex
 

elasticsearch/templates/bin/_elasticsearch.sh.tpl

@@ -19,9 +19,17 @@ set -e
 COMMAND="${@:-start}"
 
 function initiate_keystore () {
+  set -ex
   bin/elasticsearch-keystore create
-  echo ${S3_ACCESS_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.default.access_key
+
-  echo ${S3_SECRET_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.default.secret_key
+  {{- if .Values.conf.elasticsearch.snapshots.enabled }}
+  {{- range $client, $settings := .Values.storage.s3.clients -}}
+  {{- $access_key := printf "%s_S3_ACCESS_KEY" ( $client | replace "-" "_" | upper) }}
+  {{- $secret_key := printf "%s_S3_SECRET_KEY" ( $client | replace "-" "_" | upper) }}
+  echo ${{$access_key}} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.{{ $client }}.access_key
+  echo ${{$secret_key}} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.{{ $client }}.secret_key
+  {{- end }}
+  {{- end }}
 }
 
 function start () {

elasticsearch/templates/bin/_helm-tests.sh.tpl

@@ -36,19 +36,39 @@ function create_test_index () {
   fi
 }
 
-function check_snapshot_repositories () {
+{{ if .Values.conf.elasticsearch.snapshots.enabled }}
-  {{ range $repository := .Values.conf.elasticsearch.snapshots.repositories }}
+function check_snapshot_repositories_registered () {
-  repository={{$repository.name}}
+  total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-  repository_search_result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  "${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq length)
-  "${ELASTICSEARCH_ENDPOINT}/_cat/repositories" | awk '{print $1}' | grep "\<$repository\>")
+  if [ "$total_hits" -gt 0 ]; then
-  if [ "$repository_search_result" == "$repository" ]; then
+    echo "PASS: $total_hits Snapshot repositories have been registered!"
-     echo "PASS: The snapshot repository $repository exists!"
   else
-     echo "FAIL: The snapshot repository $respository does not exist! Exiting now";
+    echo "FAIL: No snapshot repositories found! Exiting";
     exit 1;
   fi
-  {{ end }}
 }
+{{ end }}
+
+{{ if .Values.conf.elasticsearch.snapshots.enabled }}
+function check_snapshot_repositories_verified () {
+  repositories=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+                  "${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq -r "keys | @sh" )
+
+  repositories=$(echo $repositories | sed "s/'//g") # Strip single quotes from jq output
+
+  for repository in $repositories; do
+    error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+            -XPOST "${ELASTICSEARCH_ENDPOINT}/_snapshot/${repository}/_verify" | jq -r '.error')
+
+    if [ $error == "null" ]; then
+      echo "PASS: $repository is verified."
+    else
+      echo "FAIL: Error for $repository: $(echo $error | jq -r)"
+      exit 1;
+    fi
+  done
+}
+{{ end }}
 
 {{ if .Values.manifests.job_elasticsearch_templates }}
 # Tests whether elasticsearch has successfully generated the elasticsearch index mapping
@@ -74,7 +94,8 @@ function remove_test_index () {
 remove_test_index || true
 create_test_index
 {{ if .Values.conf.elasticsearch.snapshots.enabled }}
-check_snapshot_repositories
+check_snapshot_repositories_registered
+check_snapshot_repositories_verified
 {{ end }}
 check_templates
 remove_test_index

elasticsearch/templates/bin/_register-repository.sh.tpl

@@ -1,62 +0,0 @@
-#!/bin/bash
-{{/*
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{ $envAll := . }}
-
-set -ex
-
-function contains() {
-    [[ $1 =~ (^|[[:space:]])$2($|[[:space:]]) ]] && return 0 || return 1
-}
-
-function register_snapshot_repository() {
-  result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-    "${ELASTICSEARCH_HOST}/_snapshot/$1" \
-    -H 'Content-Type: application/json' -d'
-    {
-      "type": "s3",
-      "settings": {
-        "endpoint": "'"$RGW_HOST"'",
-        "protocol": "http",
-        "bucket": "'"$S3_BUCKET"'"
-      }
-    }' | python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
-  if [ "$result" == "True" ];
-  then
-     echo "Snapshot repository $1 created!";
-  else
-     echo "Snapshot repository $1 not created!";
-     exit 1;
-  fi
-}
-
-function verify_snapshot_repository() {
-  curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-    -XPOST "${ELASTICSEARCH_HOST}/_snapshot/$1/_verify"
-}
-
-# Get names of all current snapshot repositories
-snapshot_repos=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-  "${ELASTICSEARCH_HOST}"/_cat/repositories?format=json | jq -r '.[].id')
-
-# Create snapshot repositories if they don't exist
-{{ range $repository := $envAll.Values.conf.elasticsearch.snapshots.repositories }}
-if contains "$snapshot_repos" {{$repository.name}}; then
-  echo "Snapshot repository {{$repository.name}} exists!"
-else
-  register_snapshot_repository {{$repository.name}}
-  verify_snapshot_repository {{$repository.name}}
-fi
-{{ end }}

elasticsearch/templates/bin/_verify-repositories.sh.tpl

@@ -22,6 +22,17 @@ function verify_snapshot_repository() {
     -XPOST "${ELASTICSEARCH_HOST}/_snapshot/$1/_verify"
 }
 
-{{ range $repository := $envAll.Values.conf.elasticsearch.snapshots.repositories }}
+repositories=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-  verify_snapshot_repository {{$repository.name}}
+                "${ELASTICSEARCH_HOST}/_snapshot" | jq -r 'keys | @sh')
-{{ end }}
+
+repositories=$(echo $repositories | sed "s/'//g") # Strip single quotes from jq output
+
+for repository in $repositories; do
+  error=$(verify_snapshot_repository $repository | jq -r '.error' )
+  if [ $error == "null" ]; then
+    echo "$repository is verified."
+  else
+    echo "Error for $repository: $(echo $error | jq -r)"
+    exit 1;
+  fi
+done

elasticsearch/templates/configmap-bin-elasticsearch.yaml

@@ -31,9 +31,7 @@ data:
   create-s3-bucket.sh: |
 {{- include "helm-toolkit.scripts.create_s3_bucket" . | indent 4 }}
   create-s3-user.sh: |
-{{- include "helm-toolkit.scripts.create_s3_user" . | indent 4 }}
+{{ tuple "bin/_create_s3_users.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  register-repository.sh: |
-{{ tuple "bin/_register-repository.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   create_template.sh: |
 {{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   verify-repositories.sh: |

elasticsearch/templates/configmap-etc-elasticsearch.yaml

@@ -15,19 +15,22 @@ limitations under the License.
 {{- if .Values.manifests.configmap_etc_elasticsearch }}
 {{- $envAll := . }}
 
-{{- if empty .Values.endpoints.ceph_object_store.path.default -}}
+{{- if .Values.conf.elasticsearch.snapshots.enabled }}
-{{- set .Values.endpoints.ceph_object_store.path "default" .Values.conf.elasticsearch.snapshots.bucket -}}
+{{- range $client, $config := $envAll.Values.storage.s3.clients }}
+{{- $settings := $config.settings }}
+{{- $endpoint := $settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup") }}
+{{- $_ := set $settings "endpoint" $endpoint }}
+{{- $protocol := $settings.protocol | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") }}
+{{- $_ := set $settings "protocol" $protocol }}
+{{- $_:= set $envAll.Values.conf.elasticsearch.config.s3.client $client $settings }}
 {{- end -}}
-
-{{- if empty .Values.conf.elasticsearch.config.s3.client.default.endpoint -}}
-{{- $radosgw_host := tuple "ceph_object_store" "internal" "api" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}}
-{{- set .Values.conf.elasticsearch.config.s3.client.default "endpoint" $radosgw_host -}}
 {{- end -}}
 
 {{- if empty .Values.conf.elasticsearch.config.discovery.seed_hosts -}}
 {{- $discovery_svc := tuple "elasticsearch" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" -}}
-{{- set .Values.conf.elasticsearch.config.discovery "seed_hosts" $discovery_svc -}}
+{{- $_:= set .Values.conf.elasticsearch.config.discovery "seed_hosts" $discovery_svc -}}
 {{- end -}}
+
 ---
 apiVersion: v1
 kind: Secret

elasticsearch/templates/deployment-client.yaml

@@ -162,16 +162,9 @@ spec:
               value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
             - name: ES_JAVA_OPTS
               value: "{{ .Values.conf.elasticsearch.env.java_opts.client }}"
-            - name: S3_ACCESS_KEY
+{{- if .Values.conf.elasticsearch.snapshots.enabled }}
-              valueFrom:
+{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
-                secretKeyRef:
+{{- end }}
-                  name: {{ $s3UserSecret }}
-                  key: S3_ACCESS_KEY
-            - name: S3_SECRET_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: {{ $s3UserSecret }}
-                  key: S3_SECRET_KEY
 {{- if .Values.pod.env.client }}
 {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.client | indent 12 }}
 {{- end }}

elasticsearch/templates/deployment-gateway.yaml

@@ -115,16 +115,9 @@ spec:
               value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
             - name: ES_JAVA_OPTS
               value: "{{ .Values.conf.elasticsearch.env.java_opts.client }}"
-            - name: S3_ACCESS_KEY
+{{- if .Values.conf.elasticsearch.snapshots.enabled }}
-              valueFrom:
+{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
-                secretKeyRef:
+{{- end }}
-                  name: {{ $s3UserSecret }}
-                  key: S3_ACCESS_KEY
-            - name: S3_SECRET_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: {{ $s3UserSecret }}
-                  key: S3_SECRET_KEY
 {{- if .Values.pod.env.gateway }}
 {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.gateway | indent 12 }}
 {{- end }}

elasticsearch/templates/job-register-snapshot-repository.yaml

@@ -1,93 +0,0 @@
-{{/*
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if and (.Values.manifests.job_snapshot_repository) (.Values.conf.elasticsearch.snapshots.enabled) }}
-{{- $envAll := . }}
-
-{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
-{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }}
-
-{{- $serviceAccountName := "elasticsearch-register-snapshot-repository" }}
-{{ tuple $envAll "snapshot_repository" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: elasticsearch-register-snapshot-repository
-  annotations:
-    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
-spec:
-  template:
-    metadata:
-      labels:
-{{ tuple $envAll "elasticsearch" "snapshot-repository" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
-      annotations:
-{{ dict "envAll" $envAll "podName" "elasticsearch-register-snapshot-repository" "containerNames" (list "register-snapshot-repository" "init" ) | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
-    spec:
-{{ dict "envAll" $envAll "application" "snapshot_repository" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
-      serviceAccountName: {{ $serviceAccountName }}
-      restartPolicy: OnFailure
-      nodeSelector:
-        {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }}
-      initContainers:
-{{ tuple $envAll "snapshot_repository" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 8 }}
-      containers:
-        - name: register-snapshot-repository
-{{ tuple $envAll "snapshot_repository" | include "helm-toolkit.snippets.image" | indent 10 }}
-{{ tuple $envAll $envAll.Values.pod.resources.jobs.snapshot_repository | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-{{ dict "envAll" $envAll "application" "snapshot_repository" "container" "register_snapshot_repository" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
-          env:
-            - name: ELASTICSEARCH_USERNAME
-              valueFrom:
-                secretKeyRef:
-                  name: {{ $esUserSecret }}
-                  key: ELASTICSEARCH_USERNAME
-            - name: ELASTICSEARCH_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ $esUserSecret }}
-                  key: ELASTICSEARCH_PASSWORD
-            - name: ELASTICSEARCH_HOST
-              value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
-            - name: S3_BUCKET
-              value: {{ .Values.conf.elasticsearch.snapshots.bucket | quote }}
-            - name: S3_ACCESS_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: {{ $s3UserSecret }}
-                  key: S3_ACCESS_KEY
-            - name: S3_SECRET_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: {{ $s3UserSecret }}
-                  key: S3_SECRET_KEY
-            - name: RGW_HOST
-              value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
-          command:
-            - /tmp/register-repository.sh
-          volumeMounts:
-            - name: pod-tmp
-              mountPath: /tmp
-            - name: elasticsearch-bin
-              mountPath: /tmp/register-repository.sh
-              subPath: register-repository.sh
-              readOnly: true
-      volumes:
-        - name: pod-tmp
-          emptyDir: {}
-        - name: elasticsearch-bin
-          configMap:
-            name: elasticsearch-bin
-            defaultMode: 0555
-{{- end }}

elasticsearch/templates/job-s3-bucket.yaml

@@ -16,4 +16,4 @@ limitations under the License.
 {{- $esBucket := .Values.conf.elasticsearch.snapshots.bucket }}
 {{- $s3BucketJob := dict "envAll" . "serviceName" "elasticsearch" "s3Bucket" $esBucket -}}
 {{ $s3BucketJob | include "helm-toolkit.manifests.job_s3_bucket" }}
-{{- end }}
+{{- end -}}

elasticsearch/templates/secret-s3-user.yaml

@@ -13,14 +13,5 @@ limitations under the License.
 */}}
 
 {{- if .Values.manifests.secret_s3 }}
-{{- $envAll := . }}
+{{ include "helm-toolkit.snippets.rgw_s3_secret_creds" . }}
-{{- $secretName := index $envAll.Values.secrets.rgw.elasticsearch }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ $secretName }}
-type: Opaque
-data:
-{{- tuple "elasticsearch" $envAll | include "helm-toolkit.snippets.rgw_s3_secret_creds" | indent 2 -}}
 {{- end }}

elasticsearch/templates/statefulset-data.yaml

@@ -127,18 +127,11 @@ spec:
               value: "false"
             - name: ES_JAVA_OPTS
               value: "{{ .Values.conf.elasticsearch.env.java_opts.data }}"
-            - name: S3_ACCESS_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: {{ $s3UserSecret }}
-                  key: S3_ACCESS_KEY
-            - name: S3_SECRET_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: {{ $s3UserSecret }}
-                  key: S3_SECRET_KEY
             - name: DISCOVERY_SERVICE
               value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+{{- if .Values.conf.elasticsearch.snapshots.enabled }}
+{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
+{{- end }}
 {{- if .Values.pod.env.data }}
 {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.data | indent 12 }}
 {{- end }}

elasticsearch/templates/statefulset-master.yaml

@@ -122,16 +122,9 @@ spec:
               value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
             - name: ES_JAVA_OPTS
               value: "{{ .Values.conf.elasticsearch.env.java_opts.master }}"
-            - name: S3_ACCESS_KEY
+{{- if .Values.conf.elasticsearch.snapshots.enabled }}
-              valueFrom:
+{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
-                secretKeyRef:
+{{- end }}
-                  name: {{ $s3UserSecret }}
-                  key: S3_ACCESS_KEY
-            - name: S3_SECRET_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: {{ $s3UserSecret }}
-                  key: S3_SECRET_KEY
 {{- if .Values.pod.env.master }}
 {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.master | indent 12 }}
 {{- end }}

elasticsearch/values.yaml

@@ -103,6 +103,8 @@ dependencies:
       services:
         - endpoint: internal
           service: elasticsearch
+      jobs:
+        - elasticsearch-s3-bucket
     image_repo_sync:
       services:
         - endpoint: internal
@@ -120,7 +122,7 @@ dependencies:
     verify_repositories:
       services: null
       jobs:
-        - elasticsearch-register-snapshot-repository
+        - create-elasticsearch-templates
     s3_user:
       services:
         - endpoint: internal
@@ -131,7 +133,7 @@ dependencies:
     tests:
       services: null
       jobs:
-        - elasticsearch-register-snapshot-repository
+        - create-elasticsearch-templates
 
 pod:
   env:
@@ -705,12 +707,7 @@ conf:
       network:
         host: 0.0.0.0
       s3:
-        client:
+        client: {}
-          default:
-            # NOTE(srwilkers): This gets configured dynamically via endpoint
-            # lookups
-            endpoint: null
-            protocol: http
       node:
         ingest: ${NODE_INGEST}
         master: ${NODE_MASTER}
@@ -722,13 +719,6 @@ conf:
         logs: /logs
     snapshots:
       enabled: false
-      # NOTE(srwilkers): The path for the radosgw s3 endpoint gets populated
-      # dynamically with this value to ensure the bucket name and s3 compatible
-      # radosgw endpoint/path match
-      bucket: elasticsearch_bucket
-      repositories:
-        logstash:
-          name: logstash_snapshots
     env:
       java_opts:
         client: "-Xms256m -Xmx256m"
@@ -783,40 +773,7 @@ conf:
               min_age: 14d
               actions:
                 delete: {}
-    - endpoint: _slm/policy/non-security-snapshots
+
-      body:
-        schedule: "0 30 1 * * ?"
-        name: "<non-security-logs-snapshot-{now/d}>"
-        repository: logstash_snapshots
-        config:
-          indices: ["^(.*calico-|.*ceph-|.*jenkins-|.*journal-|.*kernel_syslog-|.*kubernetes-|.*libvirt-|.*logstash-|.*openvswitch-|.*utility_access-).*$"]
-          ignore_unavailable: true
-          include_global_state: false
-          wait_for_completion: true
-          max_wait: 64800
-          wait_interval: 30
-          ignore_empty_list: true
-          continue_if_exception: true
-          disable_action: false
-        retention:
-          expire_after: 29d
-    - endpoint: _slm/policy/security-snapshots
-      body:
-        schedule: "0 30 1 * * ?"
-        name: "<security-logs-snapshot-{now/d}>"
-        repository: logstash_snapshots
-        config:
-          indices: ["^(.*airship-|.*audit_tsee-|.*auth-|.*flows-|.*lma-|.*openstack-).*$"]
-          ignore_unavailable: true
-          include_global_state: false
-          wait_for_completion: true
-          max_wait: 18000
-          wait_interval: 30
-          ignore_empty_list: true
-          continue_if_exception: true
-          disable_action: false
-        retention:
-          expire_after: 179d
 endpoints:
   cluster_domain_suffix: cluster.local
   local_image_registry:
@@ -900,15 +857,6 @@ endpoints:
   ceph_object_store:
     name: radosgw
     namespace: null
-    auth:
-      elasticsearch:
-        username: elasticsearch
-        access_key: "elastic_access_key"
-        secret_key: "elastic_secret_key"
-      admin:
-        username: s3_admin
-        access_key: "admin_access_key"
-        secret_key: "admin_secret_key"
     hosts:
       default: ceph-rgw
       public: radosgw
@@ -963,7 +911,43 @@ storage:
     requests:
       storage: 1Gi
     storage_class: general
-
+  s3:
+    clients: {}
+    # These values configure the s3 clients section of elasticsearch.yml
+    # See: https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3-client.html
+    #   default:
+    #     auth:
+    #       # Values under auth are written to the Secret $client-s3-user-secret
+    #       # and the access & secret keys are added to the elasticsearch keystore
+    #       username: elasticsearch
+    #       access_key: "elastic_access_key"
+    #       secret_key: "elastic_secret_key"
+    #     settings:
+    #       # Configure Client Settings here (https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3-client.html)
+    #       # endpoint: Defaults to the ceph-rgw endpoint
+    #       # protocol: Defaults to http
+    #       path_style_access: true # Required for ceph-rgw S3 API
+    #     create_user: true # Attempt to create the user at the ceph_object_store endpoint, authenticating using the secret named at .Values.secrets.rgw.admin
+    #   backup:
+    #     auth:
+    #       username: elasticsearch
+    #       access_key: "backup_access_key"
+    #       secret_key: "backup_secret_key"
+    #     settings:
+    #       endpoint: s3.example.com # Specify your own s3 endpoint (defaults to the ceph_object_store endpoint)
+    #       path_style_access: false
+    #     create_user: false
+    buckets: {}
+    # List of buckets to create (if required).
+    # (The client field references one of the clients defined above)
+    #   - name: elasticsearch-bucket
+    #     client: default
+    #     options: # list of extra options for s3cmd
+    #       - --region="default:osh-infra"
+    #   - name: backup-bucket
+    #     client: backup
+    #     options: # list of extra options for s3cmd
+    #       - --region="default:backup"
 
 manifests:
   configmap_bin_curator: false

elasticsearch/values_overrides/apparmor.yaml

@@ -6,9 +6,6 @@ pod:
     master: null
   mandatory_access_control:
     type: apparmor
-    elasticsearch-register-snapshot-repository:
-      register-snapshot-repository: runtime/default
-      init: runtime/default
     elasticsearch-master:
       elasticsearch-master: runtime/default
       init: runtime/default

helm-toolkit/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Helm-Toolkit
 name: helm-toolkit
-version: 0.2.9
+version: 0.2.10
 home: https://docs.openstack.org/openstack-helm
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
 sources:

helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl

@@ -77,15 +77,7 @@ spec:
 {{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }}
 {{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }}
 {{- end }}
-{{- with $env := dict "s3UserSecret" $s3UserSecret }}
+{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" $envAll | indent 12 }}
-{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" $env | indent 12 }}
-{{- end }}
-            - name: S3_BUCKET
-              value: {{ $s3Bucket }}
-            - name: RGW_HOST
-              value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
-            - name: RGW_PROTO
-              value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp

helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl

@@ -96,9 +96,7 @@ spec:
 {{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }}
 {{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }}
 {{- end }}
-{{- with $env := dict "s3UserSecret" $s3UserSecret }}
+{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" $envAll | indent 12 }}
-{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" $env | indent 12 }}
-{{- end }}
             - name: RGW_HOST
               value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
           volumeMounts:

helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl

@@ -16,29 +16,51 @@ limitations under the License.
 #!/bin/bash
 
 set -e
-CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST"
 
+function check_rgw_s3_bucket () {
+  echo "Checking if bucket exists"
+  s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET
+}
+
+function create_rgw_s3_bucket () {
+  echo "Creating bucket"
+  s3cmd $CONNECTION_ARGS $S3_BUCKET_OPTS $USER_AUTH_ARGS mb s3://$S3_BUCKET
+}
+
+function modify_bucket_acl () {
+  echo "Updating bucket ACL"
+  s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME
+}
+
+ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY"
+
+{{- $envAll := . }}
+{{- range $bucket := .Values.storage.s3.buckets }}
+
+S3_BUCKET={{ $bucket.name }}
+S3_BUCKET_OPTS={{ $bucket.options | default nil | include "helm-toolkit.utils.joinListWithSpace" }}
+
+S3_USERNAME=${{ printf "%s_S3_USERNAME" ( $bucket.client | replace "-" "_" | upper) }}
+S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ( $bucket.client | replace "-" "_" | upper) }}
+S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ( $bucket.client | replace "-" "_" | upper) }}
+
+{{- with $client := index $envAll.Values.storage.s3.clients $bucket.client }}
+
+RGW_HOST={{ $client.settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup")  }}
+RGW_PROTO={{ $client.settings.protocool | tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup"   }}
+
+{{- end }}
+
+CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST"
 if [ "$RGW_PROTO" = "http" ]; then
   CONNECTION_ARGS+=" --no-ssl"
 else
   CONNECTION_ARGS+=" --no-check-certificate"
 fi
-
-ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY"
 USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY"
 
-function check_rgw_s3_bucket () {
+echo "Creating Bucket $S3_BUCKET at $RGW_HOST"
-  s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET
-}
-
-function create_rgw_s3_bucket () {
-  s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS mb s3://$S3_BUCKET
-}
-
-function modify_bucket_acl () {
-  s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME
-}
-
 check_rgw_s3_bucket || ( create_rgw_s3_bucket && modify_bucket_acl )
 
 {{- end }}
+{{- end }}

helm-toolkit/templates/snippets/_rgw_s3_secret_creds.tpl

@@ -13,10 +13,17 @@ limitations under the License.
 */}}
 
 {{- define "helm-toolkit.snippets.rgw_s3_secret_creds" }}
-{{- $userClass := index . 0 -}}
+{{- range $client, $config := .Values.storage.s3.clients -}}
-{{- $context := index . 1 -}}
+---
-{{- $userContext := index $context.Values.endpoints.ceph_object_store.auth $userClass }}
+apiVersion: v1
-S3_USERNAME: {{ $userContext.username | b64enc }}
+kind: Secret
-S3_ACCESS_KEY: {{ $userContext.access_key | b64enc }}
+metadata:
-S3_SECRET_KEY: {{ $userContext.secret_key | b64enc }}
+  name: {{ printf "%s-s3-user-secret" ( $client | replace "_" "-" | lower ) }}
+type: Opaque
+data:
+{{- range $key, $value := $config.auth }}
+  {{ $key | upper }}: {{ $value | toString | b64enc}}
+{{- end }}
+
+{{ end }}
 {{- end }}

helm-toolkit/templates/snippets/_rgw_s3_user_env_vars.tpl

@@ -13,20 +13,22 @@ limitations under the License.
 */}}
 
 {{- define "helm-toolkit.snippets.rgw_s3_user_env_vars" }}
-{{- $s3UserSecret := .s3UserSecret }}
+{{- range $client, $user := .Values.storage.s3.clients }}
-- name: S3_USERNAME
+{{- $s3secret := printf "%s-s3-user-secret" ( $client | replace "_" "-" | lower ) }}
+- name: {{ printf "%s_S3_USERNAME" ($client | replace "-" "_" | upper) }}
   valueFrom:
     secretKeyRef:
-      name: {{ $s3UserSecret }}
+      name: {{ $s3secret }}
-      key: S3_USERNAME
+      key: USERNAME
-- name: S3_ACCESS_KEY
+- name: {{ printf "%s_S3_ACCESS_KEY" ($client | replace "-" "_" | upper) }}
   valueFrom:
     secretKeyRef:
-      name: {{ $s3UserSecret }}
+      name: {{ $s3secret }}
-      key: S3_ACCESS_KEY
+      key: ACCESS_KEY
-- name: S3_SECRET_KEY
+- name: {{ printf "%s_S3_SECRET_KEY" ($client | replace "-" "_" | upper) }}
   valueFrom:
     secretKeyRef:
-      name: {{ $s3UserSecret }}
+      name: {{ $s3secret }}
-      key: S3_SECRET_KEY
+      key: SECRET_KEY
+{{- end }}
 {{- end }}

releasenotes/notes/elasticsearch.yaml

@@ -9,4 +9,5 @@ elasticsearch:
   - 0.1.6 Fix elasticsearch-master rendering error
   - 0.1.7 Pin Java options to specific versions
   - 0.1.8 Disable Curator in Gate & Chart Defaults
+  - 0.2.0 Add more S3 configuration options
 ...

releasenotes/notes/helm-toolkit.yaml

@@ -16,4 +16,5 @@ helm-toolkit:
   - 0.2.7 Replace brace expansion with more standardized Posix approach
   - 0.2.8 Override the expiry of Ingress TLS certificate
   - 0.2.9 Jobs; put labels only in the template spec
+  - 0.2.10 Add more S3 configuration options
 ...

tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh

@@ -41,11 +41,24 @@ conf:
     enabled: false
   rgw_s3:
     enabled: true
+    config:
+      rgw_relaxed_s3_bucket_names: false
+  rgw_placement_targets:
+    - name: osh-infra
+      index_pool: default.rgw.osh-infra.index
+      data_pool: default.rgw.osh-infra.data
+      data-extra-pool: default.rgw.osh-infra.non-ec
+    - name: backup
+      index_pool: default.rgw.backup.index
+      data_pool: default.rgw.backup.data
+      data-extra-pool: default.rgw.backup.non-ec
 pod:
   replicas:
     rgw: 1
 manifests:
   job_bootstrap: true
+  job_rgw_placement_targets: true
+
 EOF
 helm upgrade --install radosgw-osh-infra ./ceph-rgw \
   --namespace=osh-infra \
@@ -61,3 +74,6 @@ helm status radosgw-osh-infra
 kubectl delete pods -l application=ceph,release_group=radosgw-osh-infra,component=rgw-test --namespace=osh-infra --ignore-not-found
 #NOTE: Test Deployment
 helm test radosgw-osh-infra --timeout 900
+
+#NOTE: RGW needs to be restarted for placement-targets to become accessible
+kubectl delete pods -l application=ceph,component=rgw -n osh-infra

tools/deployment/osh-infra-logging/050-elasticsearch.sh

@@ -34,6 +34,70 @@ conf:
   elasticsearch:
     snapshots:
       enabled: true
+  api_objects:
+    - endpoint: _snapshot/ceph-rgw
+      body:
+        type: s3
+        settings:
+          client: default
+          bucket: elasticsearch-bucket
+    - endpoint: _snapshot/backup
+      body:
+        type: s3
+        settings:
+          client: backup
+          bucket: backup-bucket
+    - endpoint: _slm/policy/rgw-snapshots
+      body:
+        schedule: "0 */3 * * * ?"
+        name: "<snapshot-{now/d}>"
+        repository: ceph-rgw
+        config:
+          indices: ["*"]
+        retention:
+          expire_after: 30d
+    - endpoint: _slm/policy/backup-snapshots
+      body:
+        schedule: "0 */3 * * * ?"
+        name: "<snapshot-{now/d}>"
+        repository: backup
+        config:
+          indices: ["*"]
+        retention:
+          expire_after: 180d
+storage:
+  s3:
+    clients:
+      # These values configure the s3 clients section of elasticsearch.yml, with access_key and secret_key being saved to the keystore
+      default:
+        auth:
+          username: elasticsearch
+          access_key: "elastic_access_key"
+          secret_key: "elastic_secret_key"
+        settings:
+          # endpoint: Defaults to the ceph-rgw endpoint
+          # protocol: Defaults to http
+          path_style_access: true # Required for ceph-rgw S3 API
+        create_user: true # Attempt to create the user at the ceph_object_store endpoint, authenticating using the secret named at .Values.secrets.rgw.admin
+      backup: # Change this as you'd like
+        auth:
+          username: backup
+          access_key: "backup_access_key"
+          secret_key: "backup_secret_key"
+        settings:
+          endpoint: radosgw.osh-infra.svc.cluster.local # Using the ingress here to test the endpoint override
+          path_style_access: true
+        create_user: true
+    buckets: # List of buckets to create (if required).
+      - name: elasticsearch-bucket
+        client: default
+        options: # list of extra options for s3cmd
+          - --region="default:osh-infra"
+      - name: backup-bucket
+        client: backup
+        options: # list of extra options for s3cmd
+          - --region="default:backup"
+
 EOF
 
 : ${OSH_INFRA_EXTRA_HELM_ARGS_ELASTICSEARCH:="$(./tools/deployment/common/get-values-overrides.sh elasticsearch)"}

zuul.d/jobs.yaml

@@ -151,7 +151,7 @@
           - ./tools/deployment/osh-infra-logging/020-ceph.sh
         - - ./tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
           - ./tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh
-          - ./tools/deployment/osh-infra-logging/040-ldap.sh
+        - - ./tools/deployment/osh-infra-logging/040-ldap.sh
           - ./tools/deployment/osh-infra-logging/050-elasticsearch.sh
         - - ./tools/deployment/osh-infra-logging/060-fluentd.sh
           - ./tools/deployment/osh-infra-logging/070-kibana.sh