Merge "Elasticsearch: Make templates job more robust"

elasticsearch/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v7.6.2
 description: OpenStack-Helm ElasticSearch
 name: elasticsearch
-version: 0.2.0
+version: 0.2.1
 home: https://www.elastic.co/
 sources:
   - https://github.com/elastic/elasticsearch

elasticsearch/templates/bin/_create_s3_buckets.sh.tpl

@@ -0,0 +1,63 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+#!/bin/bash
+
+set -e
+
+function check_rgw_s3_bucket () {
+  echo "Checking if bucket exists"
+  s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET
+}
+
+function create_rgw_s3_bucket () {
+  echo "Creating bucket"
+  s3cmd $CONNECTION_ARGS $S3_BUCKET_OPTS $USER_AUTH_ARGS mb s3://$S3_BUCKET
+}
+
+function modify_bucket_acl () {
+  echo "Updating bucket ACL"
+  s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME
+}
+
+ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY"
+
+{{- $envAll := . }}
+{{- range $bucket := .Values.storage.s3.buckets }}
+
+S3_BUCKET={{ $bucket.name }}
+S3_BUCKET_OPTS={{ $bucket.options | default nil | include "helm-toolkit.utils.joinListWithSpace" }}
+
+S3_USERNAME=${{ printf "%s_S3_USERNAME" ( $bucket.client | replace "-" "_" | upper) }}
+S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ( $bucket.client | replace "-" "_" | upper) }}
+S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ( $bucket.client | replace "-" "_" | upper) }}
+
+{{- with $client := index $envAll.Values.storage.s3.clients $bucket.client }}
+
+RGW_HOST={{ $client.settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup")  }}
+RGW_PROTO={{ $client.settings.protocol | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup")   }}
+
+{{- end }}
+
+CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST"
+if [ "$RGW_PROTO" = "http" ]; then
+  CONNECTION_ARGS+=" --no-ssl"
+fi
+
+USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY"
+
+echo "Creating Bucket $S3_BUCKET at $RGW_HOST"
+check_rgw_s3_bucket || ( create_rgw_s3_bucket && modify_bucket_acl )
+
+{{- end }}

elasticsearch/templates/bin/_create_template.sh.tpl

@@ -13,11 +13,31 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-set -ex
+set -e
 
-{{ range $object := .Values.conf.api_objects }}
+NUM_ERRORS=0
-curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+
+{{ range $name, $object := .Values.conf.api_objects }}
+{{ if not (empty $object) }}
+
+echo "creating {{$name}}"
+error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
    -X{{ $object.method | default "PUT" | upper }} \
    "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/{{ $object.endpoint }}" \
-   -H 'Content-Type: application/json' -d '{{ $object.body | toJson }}'
+   -H 'Content-Type: application/json' -d '{{ $object.body | toJson }}' | jq -r '.error')
+
+if [ $error == "null" ]; then
+   echo "Object {{$name}} was created."
+else
+   echo "Error when creating object {{$name}}: $(echo $error | jq -r)"
+   NUM_ERRORS=$(($NUM_ERRORS+1))
+fi
+
 {{ end }}
+{{ end }}
+
+if [ $NUM_ERRORS -gt 0 ]; then
+   exit 1
+else
+   echo "leaving normally"
+fi

elasticsearch/templates/bin/_helm-tests.sh.tpl

@@ -36,17 +36,30 @@ function create_test_index () {
   fi
 }
 
-{{ if .Values.conf.elasticsearch.snapshots.enabled }}
+{{ if not (empty .Values.conf.api_objects) }}
-function check_snapshot_repositories_registered () {
+
-  total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+function test_api_object_creation () {
-  "${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq length)
+  NUM_ERRORS=0
-  if [ "$total_hits" -gt 0 ]; then
+  {{ range $object, $config := .Values.conf.api_objects }}
-    echo "PASS: $total_hits Snapshot repositories have been registered!"
+  error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+            -XGET "${ELASTICSEARCH_ENDPOINT}/{{ $config.endpoint }}" | jq -r '.error')
+
+  if [ $error == "null" ]; then
+      echo "PASS: {{ $object }} is verified."
     else
-    echo "FAIL: No snapshot repositories found! Exiting";
+      echo "FAIL: Error for {{ $object }}: $(echo $error | jq -r)"
-    exit 1;
+      NUM_ERRORS=$(($NUM_ERRORS+1))
+    fi
+  {{ end }}
+
+  if [ $NUM_ERRORS -gt 0 ]; then
+    echo "FAIL: Some API Objects were not created!"
+    exit 1
+  else
+    echo "PASS: API Objects are verified!"
   fi
 }
+
 {{ end }}
 
 {{ if .Values.conf.elasticsearch.snapshots.enabled }}
@@ -70,21 +83,6 @@ function check_snapshot_repositories_verified () {
 }
 {{ end }}
 
-{{ if .Values.manifests.job_elasticsearch_templates }}
-# Tests whether elasticsearch has successfully generated the elasticsearch index mapping
-# templates defined by values.yaml
-function check_templates () {
-  total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-              -XGET "${ELASTICSEARCH_ENDPOINT}/_template" | jq length)
-  if [ "$total_hits" -gt 0 ]; then
-    echo "PASS: Successful hits on templates!"
-  else
-    echo "FAIL: No hits on query for templates! Exiting";
-    exit 1;
-  fi
-}
-{{ end }}
-
 function remove_test_index () {
   echo "Deleting index created for service testing"
   curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
@@ -93,9 +91,8 @@ function remove_test_index () {
 
 remove_test_index || true
 create_test_index
+remove_test_index
+test_api_object_creation
 {{ if .Values.conf.elasticsearch.snapshots.enabled }}
-check_snapshot_repositories_registered
 check_snapshot_repositories_verified
 {{ end }}
-check_templates
-remove_test_index

elasticsearch/templates/configmap-bin-elasticsearch.yaml

@@ -29,7 +29,7 @@ data:
   ceph-admin-keyring.sh: |
 {{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   create-s3-bucket.sh: |
-{{- include "helm-toolkit.scripts.create_s3_bucket" . | indent 4 }}
+{{ tuple "bin/_create_s3_buckets.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   create-s3-user.sh: |
 {{ tuple "bin/_create_s3_users.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   create_template.sh: |

elasticsearch/values.yaml

@@ -736,43 +736,23 @@ conf:
       ca: null
       client_private_key: null
       client_cert: null
-  api_objects:
+
-    - endpoint: _template/fluent
+  api_objects: {}
-      body:
+    # Fill this map with API objects to create once Elasticsearch is deployed
-        index_patterns: "logstash-*"
+    # name: # This name can be completely arbitrary
-        settings:
+    #   method: # Defaults to PUT
-          index:
+    #   endpoint: # Path for the request
-            number_of_shards: 1
+    #   body: # Body of the request in yaml (Converted to Json in Template)
-        mappings:
+    # Example: ILM Policy
-          properties:
+    # ilm_policy:
-            kubernetes:
+    #   endpoint: _ilm/policy/delete_all_indexes
-              properties:
+    #   body:
-                container_name:
+    #     policy:
-                  type: keyword
+    #       phases:
-                  index: false
+    #         delete:
-                docker_id:
+    #           min_age: 14d
-                  type: keyword
+    #           actions:
-                  index: false
+    #             delete: {}
-                host:
-                  type: keyword
-                  index: false
-                namespace_name:
-                  type: keyword
-                  index: false
-                pod_id:
-                  type: keyword
-                  index: false
-                pod_name:
-                  type: keyword
-                  index: false
-    - endpoint: _ilm/policy/delete_all_indexes
-      body:
-        policy:
-          phases:
-            delete:
-              min_age: 14d
-              actions:
-                delete: {}
 
 endpoints:
   cluster_domain_suffix: cluster.local

helm-toolkit/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Helm-Toolkit
 name: helm-toolkit
-version: 0.2.10
+version: 0.2.11
 home: https://docs.openstack.org/openstack-helm
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
 sources:

helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl

@@ -11,56 +11,25 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */}}
-
 {{- define "helm-toolkit.scripts.create_s3_bucket" }}
 #!/bin/bash
-
 set -e
-
-function check_rgw_s3_bucket () {
-  echo "Checking if bucket exists"
-  s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET
-}
-
-function create_rgw_s3_bucket () {
-  echo "Creating bucket"
-  s3cmd $CONNECTION_ARGS $S3_BUCKET_OPTS $USER_AUTH_ARGS mb s3://$S3_BUCKET
-}
-
-function modify_bucket_acl () {
-  echo "Updating bucket ACL"
-  s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME
-}
-
-ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY"
-
-{{- $envAll := . }}
-{{- range $bucket := .Values.storage.s3.buckets }}
-
-S3_BUCKET={{ $bucket.name }}
-S3_BUCKET_OPTS={{ $bucket.options | default nil | include "helm-toolkit.utils.joinListWithSpace" }}
-
-S3_USERNAME=${{ printf "%s_S3_USERNAME" ( $bucket.client | replace "-" "_" | upper) }}
-S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ( $bucket.client | replace "-" "_" | upper) }}
-S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ( $bucket.client | replace "-" "_" | upper) }}
-
-{{- with $client := index $envAll.Values.storage.s3.clients $bucket.client }}
-
-RGW_HOST={{ $client.settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup")  }}
-RGW_PROTO={{ $client.settings.protocool | tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup"   }}
-
-{{- end }}
-
 CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST"
 if [ "$RGW_PROTO" = "http" ]; then
   CONNECTION_ARGS+=" --no-ssl"
 else
   CONNECTION_ARGS+=" --no-check-certificate"
 fi
+ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY"
 USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY"
-
+function check_rgw_s3_bucket () {
-echo "Creating Bucket $S3_BUCKET at $RGW_HOST"
+  s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET
+}
+function create_rgw_s3_bucket () {
+  s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS mb s3://$S3_BUCKET
+}
+function modify_bucket_acl () {
+  s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME
+}
 check_rgw_s3_bucket || ( create_rgw_s3_bucket && modify_bucket_acl )
-
-{{- end }}
 {{- end }}

helm-toolkit/templates/scripts/_create-s3-user.sh.tpl

@@ -11,12 +11,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */}}
-
 {{- define "helm-toolkit.scripts.create_s3_user" }}
 #!/bin/bash
-
 set -e
-
 function create_s3_user () {
   echo "Creating s3 user and key pair"
   radosgw-admin user create \
@@ -26,7 +23,6 @@ function create_s3_user () {
     --access-key ${S3_ACCESS_KEY} \
     --secret-key ${S3_SECRET_KEY}
 }
-
 function update_s3_user () {
   # Retrieve old access keys, if they exist
   old_access_keys=$(radosgw-admin user info --uid=${S3_USERNAME} \
@@ -60,12 +56,10 @@ function update_s3_user () {
       --secret-key ${S3_SECRET_KEY}
   fi
 }
-
 user_exists=$(radosgw-admin user info --uid=${S3_USERNAME} || true)
 if [[ -z ${user_exists} ]]; then
   create_s3_user
 else
   update_s3_user
 fi
-
 {{- end }}

releasenotes/notes/elasticsearch.yaml

@@ -10,4 +10,5 @@ elasticsearch:
   - 0.1.7 Pin Java options to specific versions
   - 0.1.8 Disable Curator in Gate & Chart Defaults
   - 0.2.0 Add more S3 configuration options
+  - 0.2.1 Make templates job more robust & allow overrides
 ...

releasenotes/notes/helm-toolkit.yaml

@@ -17,4 +17,5 @@ helm-toolkit:
   - 0.2.8 Override the expiry of Ingress TLS certificate
   - 0.2.9 Jobs; put labels only in the template spec
   - 0.2.10 Add more S3 configuration options
+  - 0.2.11 Revert S3 User & Bucket job scripts to v0.2.9
 ...

tools/deployment/osh-infra-logging/050-elasticsearch.sh

@@ -35,36 +35,34 @@ conf:
     snapshots:
       enabled: true
   api_objects:
-    - endpoint: _snapshot/ceph-rgw
+    snapshot_repo:
+      endpoint: _snapshot/ceph-rgw
       body:
         type: s3
         settings:
           client: default
           bucket: elasticsearch-bucket
-    - endpoint: _snapshot/backup
+    slm_policy:
-      body:
+      endpoint: _slm/policy/snapshots
-        type: s3
-        settings:
-          client: backup
-          bucket: backup-bucket
-    - endpoint: _slm/policy/rgw-snapshots
       body:
         schedule: "0 */3 * * * ?"
         name: "<snapshot-{now/d}>"
         repository: ceph-rgw
         config:
-          indices: ["*"]
+          indices:
+            - "<*-{now/d}>"
         retention:
           expire_after: 30d
-    - endpoint: _slm/policy/backup-snapshots
+    ilm_policy:
+      endpoint: _ilm/policy/cleanup
       body:
-        schedule: "0 */3 * * * ?"
+        policy:
-        name: "<snapshot-{now/d}>"
+          phases:
-        repository: backup
+            delete:
-        config:
+              min_age: 5d
-          indices: ["*"]
+              actions:
-        retention:
+                delete: {}
-          expire_after: 180d
+    test_empty: {}
 storage:
   s3:
     clients: