Elasticsearch: Add curator snapshot action and PVC for fs repo

This provides an example action in the curator config for taking snapshots of the elasticsearch indexes. As the snapshot action requires a repository registered with Elasticsearch, this also adds a PVC for a filesystem repository backed with NFS and a job for registering the repository with Elasticsearch. Change-Id: I26b788c58f52844e997bde5002459bddc1bb685e
2017-12-12 16:54:23 -06:00 · 2017-12-12 16:54:23 -06:00 · 45ba95a2de
commit 45ba95a2de
parent 2be5abec3d
12 changed files with 231 additions and 27 deletions
--- a/elasticsearch/templates/bin/_register-repository.sh.tpl
+++ b/elasticsearch/templates/bin/_register-repository.sh.tpl
@ -1,3 +1,4 @@
+#!/bin/bash
 {{/*
 Copyright 2017 The Openstack-Helm Authors.

@ -14,9 +15,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}

-{{- if .Values.manifests.serviceaccount }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: elasticsearch
-{{- end }}
+set -ex
+
+exec curl -X PUT "${ELASTICSEARCH_ENDPOINT}/_snapshot/${REPO_NAME}" -H 'Content-Type: application/json' -d'
+{
+  "type": "'"$REPO_TYPE"'",
+  "settings": {
+    "location": "'"$REPO_LOCATION"'",
+    "compress": true
+  }
+}'
--- a/elasticsearch/templates/configmap-bin.yaml
+++ b/elasticsearch/templates/configmap-bin.yaml
@ -26,6 +26,8 @@ data:
 {{ tuple "bin/_elasticsearch.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
  helm-tests.sh: |
 {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  register-repository.sh: |
+{{ tuple "bin/_register-repository.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
  curator.sh: |
 {{ tuple "bin/_curator.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
  image-repo-sync.sh: |+
--- a/elasticsearch/templates/cron-job-curator.yaml
+++ b/elasticsearch/templates/cron-job-curator.yaml
@ -19,13 +19,13 @@ limitations under the License.
 {{- $envAll := . }}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.curator -}}

-{{- $serviceAccountName := "curator"}}
+{{- $serviceAccountName := "elastic-curator"}}
 {{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: batch/v2alpha1
 kind: CronJob
 metadata:
-  name: curator
+  name: elastic-curator
 spec:
  schedule: {{ .Values.conf.curator.schedule | quote }}
  jobTemplate:
--- a/elasticsearch/templates/deployment-client.yaml
+++ b/elasticsearch/templates/deployment-client.yaml
@ -22,6 +22,8 @@ limitations under the License.
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.elasticsearch_client -}}
 {{- end -}}

+{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
+
 {{- $serviceAccountName := "elasticsearch-client"}}
 {{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
@ -126,6 +128,11 @@ spec:
              readOnly: true
            - name: storage
              mountPath: {{ .Values.conf.elasticsearch.path.data }}
+            {{ if .Values.storage.filesystem_repository.enabled }}
+            - name: snapshots
+              mountPath: {{ .Values.conf.elasticsearch.path.repo }}
+            {{ end }}
+{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
      volumes:
        - name: elastic-logs
          emptyDir: {}
@ -141,4 +148,10 @@ spec:
            defaultMode: 0444
        - name: storage
          emptyDir: {}
+        {{ if .Values.storage.filesystem_repository.enabled }}
+        - name: snapshots
+          persistentVolumeClaim:
+            claimName: {{ .Values.storage.filesystem_repository.pvc.name }}
+        {{ end }}
+{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
 {{- end }}
--- a/elasticsearch/templates/deployment-master.yaml
+++ b/elasticsearch/templates/deployment-master.yaml
@ -22,6 +22,8 @@ limitations under the License.
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.elasticsearch_master -}}
 {{- end -}}

+{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
+
 {{- $serviceAccountName := "elasticsearch-master"}}
 {{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
@ -120,6 +122,11 @@ spec:
              readOnly: true
            - name: storage
              mountPath: {{ .Values.conf.elasticsearch.path.data }}
+            {{ if .Values.storage.filesystem_repository.enabled }}
+            - name: snapshots
+              mountPath: {{ .Values.conf.elasticsearch.path.repo }}
+            {{ end }}
+{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
      volumes:
        - name: elastic-logs
          emptyDir: {}
@ -135,4 +142,10 @@ spec:
            defaultMode: 0444
        - name: storage
          emptyDir: {}
+        {{ if .Values.storage.filesystem_repository.enabled }}
+        - name: snapshots
+          persistentVolumeClaim:
+            claimName: {{ .Values.storage.filesystem_repository.pvc.name }}
+        {{ end }}
+{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
 {{- end }}
--- a/elasticsearch/templates/etc/_elasticsearch.yml.tpl
+++ b/elasticsearch/templates/etc/_elasticsearch.yml.tpl
@ -28,6 +28,7 @@ network.host: {{ .Values.conf.elasticsearch.network.host }}
 path:
  data: {{ .Values.conf.elasticsearch.path.data }}
  logs: {{ .Values.conf.elasticsearch.path.logs }}
+  repo: {{ .Values.conf.elasticsearch.path.repo }}

 bootstrap:
  memory_lock: {{ .Values.conf.elasticsearch.bootstrap.memory_lock }}
--- a/elasticsearch/templates/job-image-repo-sync.yaml
+++ b/elasticsearch/templates/job-image-repo-sync.yaml
@ -50,16 +50,16 @@ spec:
          command:
            - /tmp/image-repo-sync.sh
          volumeMounts:
-            - name: elasticsearch-bin
+            - name: elastic-bin
              mountPath: /tmp/image-repo-sync.sh
              subPath: image-repo-sync.sh
              readOnly: true
            - name: docker-socket
              mountPath: /var/run/docker.sock
      volumes:
-        - name: elasticsearch-bin
+        - name: elastic-bin
          configMap:
-            name: elasticsearch-bin
+            name: elastic-bin
            defaultMode: 0555
        - name: docker-socket
          hostPath:
--- a/elasticsearch/templates/job-register-snapshot-repository.yaml
+++ b/elasticsearch/templates/job-register-snapshot-repository.yaml
@ -0,0 +1,65 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_snapshot_repository }}
+{{- $envAll := . }}
+{{- $_ := set .Values "pod_dependency" .Values.dependencies.snapshot_repository -}}
+
+{{- $serviceAccountName := "elasticsearch-register-snapshot-repository" }}
+{{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: elasticsearch-register-snapshot-repository
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "elasticsearch" "snapshot-repository" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+      initContainers:
+{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 8 }}
+      containers:
+        - name: register-snapshot-repository
+{{ tuple $envAll "snapshot_repository" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.snapshot_repository | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: ELASTICSEARCH_ENDPOINT
+              value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+            - name: REPO_NAME
+              value: {{ .Values.conf.elasticsearch.repository.name | quote }}
+            - name: REPO_TYPE
+              value: {{ .Values.conf.elasticsearch.repository.type | quote }}
+            - name: REPO_LOCATION
+              value: {{ .Values.conf.elasticsearch.path.repo | quote }}
+          command:
+            - /tmp/register-repository.sh
+          volumeMounts:
+            - name: elastic-bin
+              mountPath: /tmp/register-repository.sh
+              subPath: register-repository.sh
+              readOnly: true
+      volumes:
+        - name: elastic-bin
+          configMap:
+            name: elastic-bin
+            defaultMode: 0555
+{{- end }}
--- a/elasticsearch/templates/pvc-snapshots.yaml
+++ b/elasticsearch/templates/pvc-snapshots.yaml
@ -0,0 +1,33 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.pvc_snapshots }}
+{{- if .Values.storage.filesystem_repository.enabled }}
+{{- $envAll := . }}
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ .Values.storage.filesystem_repository.pvc.name }}
+spec:
+  accessModes:
+    - {{ .Values.storage.filesystem_repository.pvc.access_mode }}
+  resources:
+    requests:
+      storage: {{ .Values.storage.filesystem_repository.requests.storage }}
+  storageClassName: {{ .Values.storage.filesystem_repository.storage_class }}
+{{- end }}
+{{- end }}
--- a/elasticsearch/templates/statefulset-data.yaml
+++ b/elasticsearch/templates/statefulset-data.yaml
@ -22,6 +22,8 @@ limitations under the License.
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.elasticsearch_data -}}
 {{- end -}}

+{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
+
 {{- $serviceAccountName := "elasticsearch-data"}}
 {{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
@ -115,8 +117,13 @@ spec:
              mountPath: /usr/share/elasticsearch/config/log4j2.properties
              subPath: log4j2.properties
              readOnly: true
+            {{ if .Values.storage.filesystem_repository.enabled }}
+            - name: snapshots
+              mountPath: {{ .Values.conf.elasticsearch.path.repo }}
+            {{ end }}
            - name: storage
              mountPath: {{ .Values.conf.elasticsearch.path.data }}
+{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
      volumes:
        - name: elastic-logs
          emptyDir: {}
@ -130,7 +137,13 @@ spec:
          configMap:
            name: elastic-etc
            defaultMode: 0444
-{{- if not .Values.storage.enabled }}
+        {{ if .Values.storage.filesystem_repository.enabled }}
+        - name: snapshots
+          persistentVolumeClaim:
+            claimName: {{ .Values.storage.filesystem_repository.pvc.name }}
+        {{ end }}
+{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
+{{- if not .Values.storage.elasticsearch.enabled }}
        - name: storage
          emptyDir: {}
 {{- else }}
@ -138,10 +151,10 @@ spec:
    - metadata:
        name: storage
      spec:
-        accessModes: {{ .Values.storage.pvc.access_mode }}
+        accessModes: {{ .Values.storage.elasticsearch.pvc.access_mode }}
        resources:
          requests:
-            storage: {{ .Values.storage.requests.storage  }}
-        storageClassName: {{ .Values.storage.storage_class }}
+            storage: {{ .Values.storage.elasticsearch.requests.storage  }}
+        storageClassName: {{ .Values.storage.elasticsearch.storage_class }}
 {{- end }}
 {{- end }}
--- a/elasticsearch/values.yaml
+++ b/elasticsearch/values.yaml
@ -18,11 +18,12 @@

 images:
  tags:
-    memory_init: docker.io/kolla/ubuntu-source-kolla-toolbox:4.0.0
+    memory_init: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
    curator:  docker.io/bobrik/curator:5.2.0
    elasticsearch: docker.io/elasticsearch:5.4.2
-    helm_tests: docker.io/kolla/ubuntu-source-kolla-toolbox:3.0.3
+    helm_tests: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
    dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
+    snapshot_repository: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
    image_repo_sync: docker.io/docker:17.07.0
  pull_policy: "IfNotPresent"
  local_registry:
@ -48,6 +49,10 @@ dependencies:
    services:
      - service: local_image_registry
        endpoint: internal
+  snapshot_repository:
+    services:
+      - service: elasticsearch
+        endpoint: internal

 conditional_dependencies:
  local_image_registry:
@ -83,6 +88,9 @@ pod:
        timeout: 600
      client:
        timeout: 600
+  mounts:
+    elasticsearch:
+      elasticsearch:
  resources:
    enabled: false
    client:
@ -121,6 +129,13 @@ pod:
        limits:
          memory: "1024Mi"
          cpu: "2000m"
+      snapshot_repository:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
      tests:
        requests:
          memory: "128Mi"
@ -149,7 +164,33 @@ conf:
          options:
            timeout_override:
            continue_if_exception: False
-            disable_action: False
+            ignore_empty_list: True
+            disable_action: True
+          filters:
+          - filtertype: age
+            source: name
+            direction: older
+            timestring: '%Y.%m.%d'
+            unit: days
+            unit_count: 30
+            field:
+            stats_result:
+            epoch:
+            exclude: False
+        2:
+          action: snapshot
+          description: "Snapshot indices and send to configured repository"
+          options:
+            repository: default_repo
+            # Leaving this blank results in the default name format
+            name:
+            wait_for_completion: True
+            max_wait: 3600
+            wait_interval: 10
+            timeout_override:
+            ignore_empty_list: True
+            continue_if_exception: False
+            disable_action: True
          filters:
          - filtertype: age
            source: name
@ -195,6 +236,10 @@ conf:
    path:
      data: /usr/share/elasticsearch/data
      logs: /usr/share/elasticsearch/logs
+      repo: /usr/share/elasticsearch/repo
+    repository:
+      name: default_repo
+      type: fs
    zen:
      min_masters: 2
    env:
@ -244,13 +289,23 @@ network:
      port: 30931

 storage:
-  enabled: true
-  pvc:
-    name: pvc-elastic
-    access_mode: [ "ReadWriteOnce" ]
-  requests:
-    storage: 5Gi
-  storage_class: general
+  elasticsearch:
+    enabled: true
+    pvc:
+      name: pvc-elastic
+      access_mode: [ "ReadWriteOnce" ]
+    requests:
+      storage: 5Gi
+    storage_class: general
+  filesystem_repository:
+    enabled: true
+    pvc:
+      name: pvc-snapshots
+      access_mode: ReadWriteMany
+    requests:
+      storage: 5Gi
+    storage_class: general
+

 manifests:
  clusterrole: true
@ -262,8 +317,9 @@ manifests:
  deployment_client: true
  deployment_master: true
  job_image_repo_sync: true
+  job_snapshot_repository: true
  helm_tests: true
-  serviceaccount: true
+  pvc_snapshots: true
  service_data: true
  service_discovery: true
  service_logging: true
--- a/tools/gate/chart-deploys/default.yaml
+++ b/tools/gate/chart-deploys/default.yaml
@ -141,7 +141,10 @@ charts:
      output: false
    values:
      storage:
-        enabled: false
+        elasticsearch:
+          storage_class: openstack-helm-bootstrap
+        filesystem_repository:
+          storage_class: openstack-helm-bootstrap

  fluent_logging:
    chart_name: fluent-logging