From d3c6069be3013e5b508a081fc0053b90b9a5ab3a Mon Sep 17 00:00:00 2001 From: Steven Fitzpatrick Date: Tue, 6 Apr 2021 17:58:21 +0000 Subject: [PATCH] Elasticsearch: Make templates job more robust This change primarily changes the type of the api_objects yaml structure to a map, which allows for additional objects to be added by values overrides (Arrays/Lists are not mutable like this) Also, in the previous change, some scripts in HTK were modified, while other were copied over to the Elasticsearch chart. To simplify the chart's structure, this change also moves the create_s3_bucket script to Elasticsearch, and reverts the changes in HTK. Those HTK scripts are no longer referenced by osh charts, and could be candidates for removal if that chart needed to be pruned Change-Id: I7d8d7ef28223948437450dcb64bd03f2975ad54d --- elasticsearch/Chart.yaml | 2 +- .../templates/bin/_create_s3_buckets.sh.tpl | 63 +++++++++++++++++++ .../templates/bin/_create_template.sh.tpl | 28 +++++++-- .../templates/bin/_helm-tests.sh.tpl | 49 +++++++-------- .../configmap-bin-elasticsearch.yaml | 2 +- elasticsearch/values.yaml | 54 +++++----------- helm-toolkit/Chart.yaml | 2 +- .../scripts/_create-s3-bucket.sh.tpl | 53 ++++------------ .../templates/scripts/_create-s3-user.sh.tpl | 8 +-- releasenotes/notes/elasticsearch.yaml | 1 + releasenotes/notes/helm-toolkit.yaml | 1 + .../osh-infra-logging/050-elasticsearch.sh | 32 +++++----- 12 files changed, 159 insertions(+), 136 deletions(-) create mode 100644 elasticsearch/templates/bin/_create_s3_buckets.sh.tpl diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml index 7c1d6c7c8..600dd5c8f 100644 --- a/elasticsearch/Chart.yaml +++ b/elasticsearch/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v7.6.2 description: OpenStack-Helm ElasticSearch name: elasticsearch -version: 0.2.0 +version: 0.2.1 home: https://www.elastic.co/ sources: - https://github.com/elastic/elasticsearch diff --git a/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl b/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl new file mode 100644 index 000000000..e1563a69d --- /dev/null +++ b/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl @@ -0,0 +1,63 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +#!/bin/bash + +set -e + +function check_rgw_s3_bucket () { + echo "Checking if bucket exists" + s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET +} + +function create_rgw_s3_bucket () { + echo "Creating bucket" + s3cmd $CONNECTION_ARGS $S3_BUCKET_OPTS $USER_AUTH_ARGS mb s3://$S3_BUCKET +} + +function modify_bucket_acl () { + echo "Updating bucket ACL" + s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME +} + +ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY" + +{{- $envAll := . }} +{{- range $bucket := .Values.storage.s3.buckets }} + +S3_BUCKET={{ $bucket.name }} +S3_BUCKET_OPTS={{ $bucket.options | default nil | include "helm-toolkit.utils.joinListWithSpace" }} + +S3_USERNAME=${{ printf "%s_S3_USERNAME" ( $bucket.client | replace "-" "_" | upper) }} +S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ( $bucket.client | replace "-" "_" | upper) }} +S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ( $bucket.client | replace "-" "_" | upper) }} + +{{- with $client := index $envAll.Values.storage.s3.clients $bucket.client }} + +RGW_HOST={{ $client.settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup") }} +RGW_PROTO={{ $client.settings.protocol | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") }} + +{{- end }} + +CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST" +if [ "$RGW_PROTO" = "http" ]; then + CONNECTION_ARGS+=" --no-ssl" +fi + +USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY" + +echo "Creating Bucket $S3_BUCKET at $RGW_HOST" +check_rgw_s3_bucket || ( create_rgw_s3_bucket && modify_bucket_acl ) + +{{- end }} diff --git a/elasticsearch/templates/bin/_create_template.sh.tpl b/elasticsearch/templates/bin/_create_template.sh.tpl index d90dd05ba..c61bb868a 100644 --- a/elasticsearch/templates/bin/_create_template.sh.tpl +++ b/elasticsearch/templates/bin/_create_template.sh.tpl @@ -13,11 +13,31 @@ See the License for the specific language governing permissions and limitations under the License. */}} -set -ex +set -e -{{ range $object := .Values.conf.api_objects }} -curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ +NUM_ERRORS=0 + +{{ range $name, $object := .Values.conf.api_objects }} +{{ if not (empty $object) }} + +echo "creating {{$name}}" +error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ -X{{ $object.method | default "PUT" | upper }} \ "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/{{ $object.endpoint }}" \ - -H 'Content-Type: application/json' -d '{{ $object.body | toJson }}' + -H 'Content-Type: application/json' -d '{{ $object.body | toJson }}' | jq -r '.error') + +if [ $error == "null" ]; then + echo "Object {{$name}} was created." +else + echo "Error when creating object {{$name}}: $(echo $error | jq -r)" + NUM_ERRORS=$(($NUM_ERRORS+1)) +fi + {{ end }} +{{ end }} + +if [ $NUM_ERRORS -gt 0 ]; then + exit 1 +else + echo "leaving normally" +fi diff --git a/elasticsearch/templates/bin/_helm-tests.sh.tpl b/elasticsearch/templates/bin/_helm-tests.sh.tpl index e6a7d2d08..79381733a 100644 --- a/elasticsearch/templates/bin/_helm-tests.sh.tpl +++ b/elasticsearch/templates/bin/_helm-tests.sh.tpl @@ -36,17 +36,30 @@ function create_test_index () { fi } -{{ if .Values.conf.elasticsearch.snapshots.enabled }} -function check_snapshot_repositories_registered () { - total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ - "${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq length) - if [ "$total_hits" -gt 0 ]; then - echo "PASS: $total_hits Snapshot repositories have been registered!" +{{ if not (empty .Values.conf.api_objects) }} + +function test_api_object_creation () { + NUM_ERRORS=0 + {{ range $object, $config := .Values.conf.api_objects }} + error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XGET "${ELASTICSEARCH_ENDPOINT}/{{ $config.endpoint }}" | jq -r '.error') + + if [ $error == "null" ]; then + echo "PASS: {{ $object }} is verified." + else + echo "FAIL: Error for {{ $object }}: $(echo $error | jq -r)" + NUM_ERRORS=$(($NUM_ERRORS+1)) + fi + {{ end }} + + if [ $NUM_ERRORS -gt 0 ]; then + echo "FAIL: Some API Objects were not created!" + exit 1 else - echo "FAIL: No snapshot repositories found! Exiting"; - exit 1; + echo "PASS: API Objects are verified!" fi } + {{ end }} {{ if .Values.conf.elasticsearch.snapshots.enabled }} @@ -70,21 +83,6 @@ function check_snapshot_repositories_verified () { } {{ end }} -{{ if .Values.manifests.job_elasticsearch_templates }} -# Tests whether elasticsearch has successfully generated the elasticsearch index mapping -# templates defined by values.yaml -function check_templates () { - total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ - -XGET "${ELASTICSEARCH_ENDPOINT}/_template" | jq length) - if [ "$total_hits" -gt 0 ]; then - echo "PASS: Successful hits on templates!" - else - echo "FAIL: No hits on query for templates! Exiting"; - exit 1; - fi -} -{{ end }} - function remove_test_index () { echo "Deleting index created for service testing" curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ @@ -93,9 +91,8 @@ function remove_test_index () { remove_test_index || true create_test_index +remove_test_index +test_api_object_creation {{ if .Values.conf.elasticsearch.snapshots.enabled }} -check_snapshot_repositories_registered check_snapshot_repositories_verified {{ end }} -check_templates -remove_test_index diff --git a/elasticsearch/templates/configmap-bin-elasticsearch.yaml b/elasticsearch/templates/configmap-bin-elasticsearch.yaml index afaa06534..645f16d7d 100644 --- a/elasticsearch/templates/configmap-bin-elasticsearch.yaml +++ b/elasticsearch/templates/configmap-bin-elasticsearch.yaml @@ -29,7 +29,7 @@ data: ceph-admin-keyring.sh: | {{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} create-s3-bucket.sh: | -{{- include "helm-toolkit.scripts.create_s3_bucket" . | indent 4 }} +{{ tuple "bin/_create_s3_buckets.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} create-s3-user.sh: | {{ tuple "bin/_create_s3_users.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} create_template.sh: | diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index e88253a04..250b75239 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -736,43 +736,23 @@ conf: ca: null client_private_key: null client_cert: null - api_objects: - - endpoint: _template/fluent - body: - index_patterns: "logstash-*" - settings: - index: - number_of_shards: 1 - mappings: - properties: - kubernetes: - properties: - container_name: - type: keyword - index: false - docker_id: - type: keyword - index: false - host: - type: keyword - index: false - namespace_name: - type: keyword - index: false - pod_id: - type: keyword - index: false - pod_name: - type: keyword - index: false - - endpoint: _ilm/policy/delete_all_indexes - body: - policy: - phases: - delete: - min_age: 14d - actions: - delete: {} + + api_objects: {} + # Fill this map with API objects to create once Elasticsearch is deployed + # name: # This name can be completely arbitrary + # method: # Defaults to PUT + # endpoint: # Path for the request + # body: # Body of the request in yaml (Converted to Json in Template) + # Example: ILM Policy + # ilm_policy: + # endpoint: _ilm/policy/delete_all_indexes + # body: + # policy: + # phases: + # delete: + # min_age: 14d + # actions: + # delete: {} endpoints: cluster_domain_suffix: cluster.local diff --git a/helm-toolkit/Chart.yaml b/helm-toolkit/Chart.yaml index 20994d77e..5b4c7b6ed 100644 --- a/helm-toolkit/Chart.yaml +++ b/helm-toolkit/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Helm-Toolkit name: helm-toolkit -version: 0.2.10 +version: 0.2.11 home: https://docs.openstack.org/openstack-helm icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png sources: diff --git a/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl b/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl index 22b1f57b5..bf1465b23 100644 --- a/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl +++ b/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl @@ -11,56 +11,25 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */}} - {{- define "helm-toolkit.scripts.create_s3_bucket" }} #!/bin/bash - set -e - -function check_rgw_s3_bucket () { - echo "Checking if bucket exists" - s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET -} - -function create_rgw_s3_bucket () { - echo "Creating bucket" - s3cmd $CONNECTION_ARGS $S3_BUCKET_OPTS $USER_AUTH_ARGS mb s3://$S3_BUCKET -} - -function modify_bucket_acl () { - echo "Updating bucket ACL" - s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME -} - -ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY" - -{{- $envAll := . }} -{{- range $bucket := .Values.storage.s3.buckets }} - -S3_BUCKET={{ $bucket.name }} -S3_BUCKET_OPTS={{ $bucket.options | default nil | include "helm-toolkit.utils.joinListWithSpace" }} - -S3_USERNAME=${{ printf "%s_S3_USERNAME" ( $bucket.client | replace "-" "_" | upper) }} -S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ( $bucket.client | replace "-" "_" | upper) }} -S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ( $bucket.client | replace "-" "_" | upper) }} - -{{- with $client := index $envAll.Values.storage.s3.clients $bucket.client }} - -RGW_HOST={{ $client.settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup") }} -RGW_PROTO={{ $client.settings.protocool | tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} - -{{- end }} - CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST" if [ "$RGW_PROTO" = "http" ]; then CONNECTION_ARGS+=" --no-ssl" else CONNECTION_ARGS+=" --no-check-certificate" fi +ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY" USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY" - -echo "Creating Bucket $S3_BUCKET at $RGW_HOST" +function check_rgw_s3_bucket () { + s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET +} +function create_rgw_s3_bucket () { + s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS mb s3://$S3_BUCKET +} +function modify_bucket_acl () { + s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME +} check_rgw_s3_bucket || ( create_rgw_s3_bucket && modify_bucket_acl ) - -{{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl b/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl index c2d9ded15..08796d29c 100644 --- a/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl +++ b/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl @@ -11,12 +11,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */}} - {{- define "helm-toolkit.scripts.create_s3_user" }} #!/bin/bash - set -e - function create_s3_user () { echo "Creating s3 user and key pair" radosgw-admin user create \ @@ -26,7 +23,6 @@ function create_s3_user () { --access-key ${S3_ACCESS_KEY} \ --secret-key ${S3_SECRET_KEY} } - function update_s3_user () { # Retrieve old access keys, if they exist old_access_keys=$(radosgw-admin user info --uid=${S3_USERNAME} \ @@ -60,12 +56,10 @@ function update_s3_user () { --secret-key ${S3_SECRET_KEY} fi } - user_exists=$(radosgw-admin user info --uid=${S3_USERNAME} || true) if [[ -z ${user_exists} ]]; then create_s3_user else update_s3_user fi - -{{- end }} +{{- end }} \ No newline at end of file diff --git a/releasenotes/notes/elasticsearch.yaml b/releasenotes/notes/elasticsearch.yaml index 0cc8c6649..811773096 100644 --- a/releasenotes/notes/elasticsearch.yaml +++ b/releasenotes/notes/elasticsearch.yaml @@ -10,4 +10,5 @@ elasticsearch: - 0.1.7 Pin Java options to specific versions - 0.1.8 Disable Curator in Gate & Chart Defaults - 0.2.0 Add more S3 configuration options + - 0.2.1 Make templates job more robust & allow overrides ... diff --git a/releasenotes/notes/helm-toolkit.yaml b/releasenotes/notes/helm-toolkit.yaml index a511bf40f..1103cf8b5 100644 --- a/releasenotes/notes/helm-toolkit.yaml +++ b/releasenotes/notes/helm-toolkit.yaml @@ -17,4 +17,5 @@ helm-toolkit: - 0.2.8 Override the expiry of Ingress TLS certificate - 0.2.9 Jobs; put labels only in the template spec - 0.2.10 Add more S3 configuration options + - 0.2.11 Revert S3 User & Bucket job scripts to v0.2.9 ... diff --git a/tools/deployment/osh-infra-logging/050-elasticsearch.sh b/tools/deployment/osh-infra-logging/050-elasticsearch.sh index 8fa950b32..5e62ef05b 100755 --- a/tools/deployment/osh-infra-logging/050-elasticsearch.sh +++ b/tools/deployment/osh-infra-logging/050-elasticsearch.sh @@ -35,36 +35,34 @@ conf: snapshots: enabled: true api_objects: - - endpoint: _snapshot/ceph-rgw + snapshot_repo: + endpoint: _snapshot/ceph-rgw body: type: s3 settings: client: default bucket: elasticsearch-bucket - - endpoint: _snapshot/backup - body: - type: s3 - settings: - client: backup - bucket: backup-bucket - - endpoint: _slm/policy/rgw-snapshots + slm_policy: + endpoint: _slm/policy/snapshots body: schedule: "0 */3 * * * ?" name: "" repository: ceph-rgw config: - indices: ["*"] + indices: + - "<*-{now/d}>" retention: expire_after: 30d - - endpoint: _slm/policy/backup-snapshots + ilm_policy: + endpoint: _ilm/policy/cleanup body: - schedule: "0 */3 * * * ?" - name: "" - repository: backup - config: - indices: ["*"] - retention: - expire_after: 180d + policy: + phases: + delete: + min_age: 5d + actions: + delete: {} + test_empty: {} storage: s3: clients: