Elasticsearch: Add cron job to verify snapshot repositories

This adds a cron job to manually verify all snapshot repositories
are registered to any active master and data nodes. This is to
address scenarios where master and data nodes do not have the
desired snapshot repositories registered following node outages
or reboots

Change-Id: Ie6f42e95c3ca4dc2ec70f2852a2bde11e59ec097
Signed-off-by: Steve Wilkerson <sw5822@att.com>

elasticsearch/templates/bin/_verify-repositories.sh.tpl

@@ -0,0 +1,29 @@
+#!/bin/bash
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{ $envAll := . }}
+
+set -ex
+
+function verify_snapshot_repository() {
+  curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+    -XPOST "${ELASTICSEARCH_HOST}/_snapshot/$1/_verify"
+}
+
+{{ range $repository := $envAll.Values.conf.elasticsearch.snapshots.repositories }}
+  verify_snapshot_repository {{$repository.name}}
+{{ end }}

elasticsearch/templates/configmap-bin-elasticsearch.yaml

@@ -40,6 +40,8 @@ data:
 {{ tuple "bin/_es-cluster-wait.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   create_template.sh: |
 {{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  verify-repositories.sh: |
+{{ tuple "bin/_verify-repositories.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   image-repo-sync.sh: |
 {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
 {{- end }}

elasticsearch/templates/cron-job-verify-repositories.yaml

@@ -0,0 +1,85 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and (.Values.manifests.cron_verify_repositories) (.Values.conf.elasticsearch.snapshots.enabled) }}
+{{- $envAll := . }}
+
+{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
+
+{{- $serviceAccountName := "verify-repositories" }}
+{{ tuple $envAll "verify_repositories" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: elasticsearch-verify-repositories
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+  schedule: {{ .Values.jobs.verify_repositories.cron | quote }}
+  successfulJobsHistoryLimit: {{ .Values.jobs.verify_repositories.history.success }}
+  failedJobsHistoryLimit: {{ .Values.jobs.verify_repositories.history.failed }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    metadata:
+      labels:
+{{ tuple $envAll "elasticsearch" "verify-repositories" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      template:
+        metadata:
+          labels:
+{{ tuple $envAll "elasticsearch" "verify-repositories" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
+        spec:
+          nodeSelector:
+            {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }}
+          serviceAccountName: {{ $serviceAccountName }}
+          restartPolicy: OnFailure
+          initContainers:
+{{ tuple $envAll "verify_repositories" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 12 }}
+          containers:
+            - name: elasticsearch-verify-repositories
+{{ tuple $envAll "snapshot_repository" | include "helm-toolkit.snippets.image" | indent 14 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.snapshot_repository | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
+              command:
+                - /tmp/verify-repositories.sh
+              env:
+                - name: ELASTICSEARCH_USERNAME
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ $esUserSecret }}
+                      key: ELASTICSEARCH_USERNAME
+                - name: ELASTICSEARCH_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ $esUserSecret }}
+                      key: ELASTICSEARCH_PASSWORD
+                - name: ELASTICSEARCH_HOST
+                  value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+              volumeMounts:
+                - name: pod-tmp
+                  mountPath: /tmp
+                - name: elasticsearch-bin
+                  mountPath: /tmp/verify-repositories.sh
+                  subPath: verify-repositories.sh
+                  readOnly: true
+          volumes:
+            - name: pod-tmp
+              emptyDir: {}
+            - name: elasticsearch-bin
+              configMap:
+                name: elasticsearch-bin
+                defaultMode: 0555
+{{- end }}

elasticsearch/values.yaml

@@ -106,6 +106,10 @@ dependencies:
       jobs:
         - elasticsearch-s3-bucket
         - elasticsearch-cluster-wait
+    verify_repositories:
+      services: null
+      jobs:
+        - elasticsearch-register-snapshot-repository
     s3_user:
       services:
         - endpoint: internal
@@ -373,6 +377,11 @@ jobs:
   snapshot_repository:
     backoffLimit: 6
     activeDeadlineSeconds: 600
+  verify_repositories:
+    cron: "*/30 * * * *"
+    history:
+      success: 3
+      failed: 1
 
 conf:
   httpd: |
@@ -836,6 +845,7 @@ manifests:
   configmap_etc_elasticsearch: true
   configmap_etc_templates: true
   cron_curator: true
+  cron_verify_repositories: true
   deployment_client: true
   deployment_master: true
   ingress: true

tools/deployment/multinode/120-elasticsearch.sh

@@ -21,6 +21,9 @@ make elasticsearch
 
 #NOTE: Deploy command
 tee /tmp/elasticsearch.yaml << EOF
+jobs:
+  verify_repositories:
+    cron: "*/3 * * * *"
 pod:
   replicas:
     data: 1

tools/deployment/osh-infra-logging/050-elasticsearch.sh

@@ -21,6 +21,9 @@ make elasticsearch
 
 #NOTE: Deploy command
 tee /tmp/elasticsearch.yaml << EOF
+jobs:
+  verify_repositories:
+    cron: "*/3 * * * *"
 monitoring:
   prometheus:
     enabled: true