From fcb4681cb16066b075c99b6e757d099fad8d3daa Mon Sep 17 00:00:00 2001 From: Graham Steffaniak Date: Tue, 8 Dec 2020 14:31:34 +0000 Subject: [PATCH] Add elasticsearch snapshot policy template for SLM ADD: new snapshot policy template job which creates templates for ES SLM manager to snapshot indicies instead of curator. Change-Id: I629d30691d6d3f77646bde7d4838056b117ce091 --- elasticsearch/Chart.yaml | 2 +- .../templates/bin/_create_template.sh.tpl | 14 ++++++++ .../templates/configmap-etc-templates.yaml | 4 +++ .../templates/job-elasticsearch-template.yaml | 6 ++++ elasticsearch/values.yaml | 33 +++++++++++++++++++ 5 files changed, 58 insertions(+), 1 deletion(-) diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml index 52ad5dbbc..808cd0b46 100644 --- a/elasticsearch/Chart.yaml +++ b/elasticsearch/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v7.1.0 description: OpenStack-Helm ElasticSearch name: elasticsearch -version: 0.1.2 +version: 0.1.3 home: https://www.elastic.co/ sources: - https://github.com/elastic/elasticsearch diff --git a/elasticsearch/templates/bin/_create_template.sh.tpl b/elasticsearch/templates/bin/_create_template.sh.tpl index 22ee33f82..6eb873617 100644 --- a/elasticsearch/templates/bin/_create_template.sh.tpl +++ b/elasticsearch/templates/bin/_create_template.sh.tpl @@ -15,3 +15,17 @@ else fi {{ end }} + +{{ range $policy_name, $fields := .Values.conf.snapshot_policies }} + +result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ +-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_slm/policy/{{$policy_name}}" \ +-H 'Content-Type: application/json' -d @/tmp/{{$policy_name}}.json \ +| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])") +if [ "$result" == "True" ]; then + echo "Policy {{$policy_name}} created!" +else + echo "Policy {{$policy_name}} not created!" +fi + +{{ end }} \ No newline at end of file diff --git a/elasticsearch/templates/configmap-etc-templates.yaml b/elasticsearch/templates/configmap-etc-templates.yaml index 0a80d164f..7f4d0f36f 100644 --- a/elasticsearch/templates/configmap-etc-templates.yaml +++ b/elasticsearch/templates/configmap-etc-templates.yaml @@ -25,4 +25,8 @@ data: {{ range $template, $fields := .Values.conf.templates }} {{ $template }}.json: {{ toJson $fields | b64enc }} {{ end }} +{{ range $policy_name, $fields := .Values.conf.snapshot_policies }} + {{ $policy_name }}.json: {{ toJson $fields | b64enc }} +{{ end }} + {{- end }} diff --git a/elasticsearch/templates/job-elasticsearch-template.yaml b/elasticsearch/templates/job-elasticsearch-template.yaml index 18c9df073..38ee62d9c 100644 --- a/elasticsearch/templates/job-elasticsearch-template.yaml +++ b/elasticsearch/templates/job-elasticsearch-template.yaml @@ -78,6 +78,12 @@ spec: subPath: {{$template}}.json readOnly: true {{ end }} + {{ range $policy_name, $fields := .Values.conf.snapshot_policies }} + - name: elasticsearch-templates-etc + mountPath: /tmp/{{$policy_name}}.json + subPath: {{$policy_name}}.json + readOnly: true + {{ end }} {{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }} volumes: - name: pod-tmp diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 2c9d95dbb..b1f79f5f6 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -774,6 +774,39 @@ conf: pod_name: type: keyword index: false + snapshot_policies: + non-security-snapshots: + schedule: "0 30 1 * * ?" + name: "" + repository: logstash_snapshots + config: + indices: ["^(.*calico-|.*ceph-|.*jenkins-|.*journal-|.*kernel_syslog-|.*kubernetes-|.*libvirt-|.*logstash-|.*openvswitch-|.*utility_access-).*$"] + ignore_unavailable: true + include_global_state: false + wait_for_completion: true + max_wait: 64800 + wait_interval: 30 + ignore_empty_list: true + continue_if_exception: true + disable_action: false + retention: + expire_after: 29d + security-snapshots: + schedule: "0 30 1 * * ?" + name: "" + repository: logstash_snapshots + config: + indices: ["^(.*airship-|.*audit_tsee-|.*auth-|.*flows-|.*lma-|.*openstack-).*$"] + ignore_unavailable: true + include_global_state: false + wait_for_completion: true + max_wait: 18000 + wait_interval: 30 + ignore_empty_list: true + continue_if_exception: true + disable_action: false + retention: + expire_after: 179d endpoints: cluster_domain_suffix: cluster.local