Elasticsearch: Add curator snapshot action and PVC for fs repo

This provides an example action in the curator config for taking
snapshots of the elasticsearch indexes. As the snapshot action
requires a repository registered with Elasticsearch, this also
adds a PVC for a filesystem repository backed with NFS and a job
for registering the repository with Elasticsearch.

Change-Id: I26b788c58f52844e997bde5002459bddc1bb685e

elasticsearch/templates/bin/_register-repository.sh.tpl

@@ -1,3 +1,4 @@
+#!/bin/bash
 {{/*
 Copyright 2017 The Openstack-Helm Authors.
 
@@ -14,9 +15,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests.serviceaccount }}
+set -ex
-apiVersion: v1
+
-kind: ServiceAccount
+exec curl -X PUT "${ELASTICSEARCH_ENDPOINT}/_snapshot/${REPO_NAME}" -H 'Content-Type: application/json' -d'
-metadata:
+{
-  name: elasticsearch
+  "type": "'"$REPO_TYPE"'",
-{{- end }}
+  "settings": {
+    "location": "'"$REPO_LOCATION"'",
+    "compress": true
+  }
+}'

elasticsearch/templates/configmap-bin.yaml

@@ -26,6 +26,8 @@ data:
 {{ tuple "bin/_elasticsearch.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   helm-tests.sh: |
 {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  register-repository.sh: |
+{{ tuple "bin/_register-repository.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   curator.sh: |
 {{ tuple "bin/_curator.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   image-repo-sync.sh: |+

elasticsearch/templates/cron-job-curator.yaml

@@ -19,13 +19,13 @@ limitations under the License.
 {{- $envAll := . }}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.curator -}}
 
-{{- $serviceAccountName := "curator"}}
+{{- $serviceAccountName := "elastic-curator"}}
 {{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: batch/v2alpha1
 kind: CronJob
 metadata:
-  name: curator
+  name: elastic-curator
 spec:
   schedule: {{ .Values.conf.curator.schedule | quote }}
   jobTemplate:

elasticsearch/templates/deployment-client.yaml

@@ -22,6 +22,8 @@ limitations under the License.
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.elasticsearch_client -}}
 {{- end -}}
 
+{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
+
 {{- $serviceAccountName := "elasticsearch-client"}}
 {{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
@@ -126,6 +128,11 @@ spec:
               readOnly: true
             - name: storage
               mountPath: {{ .Values.conf.elasticsearch.path.data }}
+            {{ if .Values.storage.filesystem_repository.enabled }}
+            - name: snapshots
+              mountPath: {{ .Values.conf.elasticsearch.path.repo }}
+            {{ end }}
+{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: elastic-logs
           emptyDir: {}
@@ -141,4 +148,10 @@ spec:
             defaultMode: 0444
         - name: storage
           emptyDir: {}
+        {{ if .Values.storage.filesystem_repository.enabled }}
+        - name: snapshots
+          persistentVolumeClaim:
+            claimName: {{ .Values.storage.filesystem_repository.pvc.name }}
+        {{ end }}
+{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
 {{- end }}

elasticsearch/templates/deployment-master.yaml

@@ -22,6 +22,8 @@ limitations under the License.
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.elasticsearch_master -}}
 {{- end -}}
 
+{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
+
 {{- $serviceAccountName := "elasticsearch-master"}}
 {{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
@@ -120,6 +122,11 @@ spec:
               readOnly: true
             - name: storage
               mountPath: {{ .Values.conf.elasticsearch.path.data }}
+            {{ if .Values.storage.filesystem_repository.enabled }}
+            - name: snapshots
+              mountPath: {{ .Values.conf.elasticsearch.path.repo }}
+            {{ end }}
+{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: elastic-logs
           emptyDir: {}
@@ -135,4 +142,10 @@ spec:
             defaultMode: 0444
         - name: storage
           emptyDir: {}
+        {{ if .Values.storage.filesystem_repository.enabled }}
+        - name: snapshots
+          persistentVolumeClaim:
+            claimName: {{ .Values.storage.filesystem_repository.pvc.name }}
+        {{ end }}
+{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
 {{- end }}

elasticsearch/templates/etc/_elasticsearch.yml.tpl

@@ -28,6 +28,7 @@ network.host: {{ .Values.conf.elasticsearch.network.host }}
 path:
   data: {{ .Values.conf.elasticsearch.path.data }}
   logs: {{ .Values.conf.elasticsearch.path.logs }}
+  repo: {{ .Values.conf.elasticsearch.path.repo }}
 
 bootstrap:
   memory_lock: {{ .Values.conf.elasticsearch.bootstrap.memory_lock }}

elasticsearch/templates/job-image-repo-sync.yaml

@@ -50,16 +50,16 @@ spec:
           command:
             - /tmp/image-repo-sync.sh
           volumeMounts:
-            - name: elasticsearch-bin
+            - name: elastic-bin
               mountPath: /tmp/image-repo-sync.sh
               subPath: image-repo-sync.sh
               readOnly: true
             - name: docker-socket
               mountPath: /var/run/docker.sock
       volumes:
-        - name: elasticsearch-bin
+        - name: elastic-bin
           configMap:
-            name: elasticsearch-bin
+            name: elastic-bin
             defaultMode: 0555
         - name: docker-socket
           hostPath:

elasticsearch/templates/job-register-snapshot-repository.yaml

@@ -0,0 +1,65 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_snapshot_repository }}
+{{- $envAll := . }}
+{{- $_ := set .Values "pod_dependency" .Values.dependencies.snapshot_repository -}}
+
+{{- $serviceAccountName := "elasticsearch-register-snapshot-repository" }}
+{{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: elasticsearch-register-snapshot-repository
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "elasticsearch" "snapshot-repository" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+      initContainers:
+{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 8 }}
+      containers:
+        - name: register-snapshot-repository
+{{ tuple $envAll "snapshot_repository" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.snapshot_repository | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: ELASTICSEARCH_ENDPOINT
+              value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+            - name: REPO_NAME
+              value: {{ .Values.conf.elasticsearch.repository.name | quote }}
+            - name: REPO_TYPE
+              value: {{ .Values.conf.elasticsearch.repository.type | quote }}
+            - name: REPO_LOCATION
+              value: {{ .Values.conf.elasticsearch.path.repo | quote }}
+          command:
+            - /tmp/register-repository.sh
+          volumeMounts:
+            - name: elastic-bin
+              mountPath: /tmp/register-repository.sh
+              subPath: register-repository.sh
+              readOnly: true
+      volumes:
+        - name: elastic-bin
+          configMap:
+            name: elastic-bin
+            defaultMode: 0555
+{{- end }}

elasticsearch/templates/pvc-snapshots.yaml

@@ -0,0 +1,33 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.pvc_snapshots }}
+{{- if .Values.storage.filesystem_repository.enabled }}
+{{- $envAll := . }}
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ .Values.storage.filesystem_repository.pvc.name }}
+spec:
+  accessModes:
+    - {{ .Values.storage.filesystem_repository.pvc.access_mode }}
+  resources:
+    requests:
+      storage: {{ .Values.storage.filesystem_repository.requests.storage }}
+  storageClassName: {{ .Values.storage.filesystem_repository.storage_class }}
+{{- end }}
+{{- end }}

elasticsearch/templates/statefulset-data.yaml

@@ -22,6 +22,8 @@ limitations under the License.
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.elasticsearch_data -}}
 {{- end -}}
 
+{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
+
 {{- $serviceAccountName := "elasticsearch-data"}}
 {{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
@@ -115,8 +117,13 @@ spec:
               mountPath: /usr/share/elasticsearch/config/log4j2.properties
               subPath: log4j2.properties
               readOnly: true
+            {{ if .Values.storage.filesystem_repository.enabled }}
+            - name: snapshots
+              mountPath: {{ .Values.conf.elasticsearch.path.repo }}
+            {{ end }}
             - name: storage
               mountPath: {{ .Values.conf.elasticsearch.path.data }}
+{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: elastic-logs
           emptyDir: {}
@@ -130,7 +137,13 @@ spec:
           configMap:
             name: elastic-etc
             defaultMode: 0444
-{{- if not .Values.storage.enabled }}
+        {{ if .Values.storage.filesystem_repository.enabled }}
+        - name: snapshots
+          persistentVolumeClaim:
+            claimName: {{ .Values.storage.filesystem_repository.pvc.name }}
+        {{ end }}
+{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
+{{- if not .Values.storage.elasticsearch.enabled }}
         - name: storage
           emptyDir: {}
 {{- else }}
@@ -138,10 +151,10 @@ spec:
     - metadata:
         name: storage
       spec:
-        accessModes: {{ .Values.storage.pvc.access_mode }}
+        accessModes: {{ .Values.storage.elasticsearch.pvc.access_mode }}
         resources:
           requests:
-            storage: {{ .Values.storage.requests.storage  }}
+            storage: {{ .Values.storage.elasticsearch.requests.storage  }}
-        storageClassName: {{ .Values.storage.storage_class }}
+        storageClassName: {{ .Values.storage.elasticsearch.storage_class }}
 {{- end }}
 {{- end }}

elasticsearch/values.yaml

@@ -18,11 +18,12 @@
 
 images:
   tags:
-    memory_init: docker.io/kolla/ubuntu-source-kolla-toolbox:4.0.0
+    memory_init: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
     curator:  docker.io/bobrik/curator:5.2.0
     elasticsearch: docker.io/elasticsearch:5.4.2
-    helm_tests: docker.io/kolla/ubuntu-source-kolla-toolbox:3.0.3
+    helm_tests: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
+    snapshot_repository: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: "IfNotPresent"
   local_registry:
@@ -48,6 +49,10 @@ dependencies:
     services:
       - service: local_image_registry
         endpoint: internal
+  snapshot_repository:
+    services:
+      - service: elasticsearch
+        endpoint: internal
 
 conditional_dependencies:
   local_image_registry:
@@ -83,6 +88,9 @@ pod:
         timeout: 600
       client:
         timeout: 600
+  mounts:
+    elasticsearch:
+      elasticsearch:
   resources:
     enabled: false
     client:
@@ -121,6 +129,13 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      snapshot_repository:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
       tests:
         requests:
           memory: "128Mi"
@@ -149,7 +164,33 @@ conf:
           options:
             timeout_override:
             continue_if_exception: False
-            disable_action: False
+            ignore_empty_list: True
+            disable_action: True
+          filters:
+          - filtertype: age
+            source: name
+            direction: older
+            timestring: '%Y.%m.%d'
+            unit: days
+            unit_count: 30
+            field:
+            stats_result:
+            epoch:
+            exclude: False
+        2:
+          action: snapshot
+          description: "Snapshot indices and send to configured repository"
+          options:
+            repository: default_repo
+            # Leaving this blank results in the default name format
+            name:
+            wait_for_completion: True
+            max_wait: 3600
+            wait_interval: 10
+            timeout_override:
+            ignore_empty_list: True
+            continue_if_exception: False
+            disable_action: True
           filters:
           - filtertype: age
             source: name
@@ -195,6 +236,10 @@ conf:
     path:
       data: /usr/share/elasticsearch/data
       logs: /usr/share/elasticsearch/logs
+      repo: /usr/share/elasticsearch/repo
+    repository:
+      name: default_repo
+      type: fs
     zen:
       min_masters: 2
     env:
@@ -244,6 +289,7 @@ network:
       port: 30931
 
 storage:
+  elasticsearch:
     enabled: true
     pvc:
       name: pvc-elastic
@@ -251,6 +297,15 @@ storage:
     requests:
       storage: 5Gi
     storage_class: general
+  filesystem_repository:
+    enabled: true
+    pvc:
+      name: pvc-snapshots
+      access_mode: ReadWriteMany
+    requests:
+      storage: 5Gi
+    storage_class: general
+
 
 manifests:
   clusterrole: true
@@ -262,8 +317,9 @@ manifests:
   deployment_client: true
   deployment_master: true
   job_image_repo_sync: true
+  job_snapshot_repository: true
   helm_tests: true
-  serviceaccount: true
+  pvc_snapshots: true
   service_data: true
   service_discovery: true
   service_logging: true

tools/gate/chart-deploys/default.yaml

@@ -141,7 +141,10 @@ charts:
       output: false
     values:
       storage:
-        enabled: false
+        elasticsearch:
+          storage_class: openstack-helm-bootstrap
+        filesystem_repository:
+          storage_class: openstack-helm-bootstrap
 
   fluent_logging:
     chart_name: fluent-logging