From ff291b5abbcfd4ae758a10e6e5a44210c59fc2ea Mon Sep 17 00:00:00 2001 From: Steven Fitzpatrick Date: Fri, 15 May 2020 15:13:09 -0500 Subject: [PATCH] Kibana - Add hook to delete .kibana indices This hook is enabled for post-delete and pre-upgrade triggers. The indices deleted by this hook are Kibana's meta indices - .kibana - .kibana_1 - .kibana_2 etc This is done to get around https://github.com/elastic/kibana/issues/58388 which sometimes prevents Kibana deployments from upgrading successfully. Change-Id: I99ccc7de20c6dadb5154e4bb714dfd302a694a78 --- .../bin/_flush_kibana_metadata.sh.tpl | 19 ++++ kibana/templates/configmap-bin.yaml | 2 + .../templates/job-flush-kibana-metadata.yaml | 100 ++++++++++++++++++ kibana/values.yaml | 25 +++++ kibana/values_overrides/apparmor.yaml | 3 + 5 files changed, 149 insertions(+) create mode 100644 kibana/templates/bin/_flush_kibana_metadata.sh.tpl create mode 100644 kibana/templates/job-flush-kibana-metadata.yaml diff --git a/kibana/templates/bin/_flush_kibana_metadata.sh.tpl b/kibana/templates/bin/_flush_kibana_metadata.sh.tpl new file mode 100644 index 000000000..76c82a165 --- /dev/null +++ b/kibana/templates/bin/_flush_kibana_metadata.sh.tpl @@ -0,0 +1,19 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +set -ex +echo "Deleting index created for metadata" + +curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XDELETE "${ELASTICSEARCH_ENDPOINT}/.kibana*" diff --git a/kibana/templates/configmap-bin.yaml b/kibana/templates/configmap-bin.yaml index 57b676b3d..d7c3c11af 100644 --- a/kibana/templates/configmap-bin.yaml +++ b/kibana/templates/configmap-bin.yaml @@ -26,6 +26,8 @@ data: {{ tuple "bin/_kibana.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} create_kibana_index_patterns.sh: | {{ tuple "bin/_create_kibana_index_patterns.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + flush_kibana_metadata.sh: | +{{ tuple "bin/_flush_kibana_metadata.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} image-repo-sync.sh: | {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} {{- end }} diff --git a/kibana/templates/job-flush-kibana-metadata.yaml b/kibana/templates/job-flush-kibana-metadata.yaml new file mode 100644 index 000000000..e96a2c7cb --- /dev/null +++ b/kibana/templates/job-flush-kibana-metadata.yaml @@ -0,0 +1,100 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +This hook is enabled for post-delete and pre-upgrade triggers. +The indices deleted by this hook are Kibana's meta indices + - .kibana + - .kibana_1 + - .kibana_2 + etc + +This is done to get around https://github.com/elastic/kibana/issues/58388 +which sometimes prevents Kibana deployments from upgrading successfully. +*/}} + +{{- if .Values.manifests.job_flush_kibana_metadata }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $serviceAccountName := "flush-kibana-metadata" }} +{{ tuple $envAll "flush_kibana_metadata" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: flush-kibana-metadata +spec: + backoffLimit: {{ .Values.jobs.flush_kibana_metadata.backoffLimit }} + template: + metadata: + labels: +{{ tuple $envAll "kibana" "flush_kibana_metadata" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + "helm.sh/hook": post-delete, pre-upgrade + "helm.sh/hook-delete-policy": hook-succeeded +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "flush-kibana-metadata" "containerNames" (list "flush-kibana-metadata" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "flush_kibana_metadata" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + activeDeadlineSeconds: {{ .Values.jobs.flush_kibana_metadata.activeDeadlineSeconds }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "flush_kibana_metadata" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: flush-kibana-metadata +{{ tuple $envAll "flush_kibana_metadata" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.flush_kibana_metadata | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "flush_kibana_metadata" "container" "flush_kibana_metadata" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: KIBANA_ENDPOINT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + - name: ELASTICSEARCH_ENDPOINT + value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + command: + - /tmp/flush_kibana_metadata.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: kibana-bin + mountPath: /tmp/flush_kibana_metadata.sh + subPath: flush_kibana_metadata.sh + readOnly: false + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-run + emptyDir: + medium: "Memory" + - name: kibana-bin + configMap: + name: kibana-bin + defaultMode: 0755 +{{- end }} diff --git a/kibana/values.yaml b/kibana/values.yaml index 27de90d4e..3ef5785fd 100644 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -25,6 +25,7 @@ images: dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 register_kibana_indexes: docker.io/openstackhelm/heat:newton-ubuntu_xenial + flush_kibana_metadata: docker.io/openstackhelm/heat:newton-ubuntu_xenial pull_policy: IfNotPresent local_registry: active: false @@ -53,6 +54,13 @@ pod: register_kibana_indexes: allowPrivilegeEscalation: false readOnlyRootFilesystem: true + flush_kibana_metadata: + pod: + runAsUser: 1000 + container: + flush_kibana_metadata: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true affinity: anti: type: @@ -102,6 +110,13 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + flush_kibana_metadata: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" probes: kibana: kibana: @@ -152,6 +167,15 @@ dependencies: services: - endpoint: internal service: kibana + flush_kibana_metadata: + services: + - endpoint: internal + service: kibana + +jobs: + flush_kibana_metadata: + backoffLimit: 6 + activeDeadlineSeconds: 600 conf: httpd: | @@ -393,3 +417,4 @@ manifests: service: true service_ingress: true job_register_kibana_indexes: true + job_flush_kibana_metadata: true diff --git a/kibana/values_overrides/apparmor.yaml b/kibana/values_overrides/apparmor.yaml index 7481673ad..3ecc51094 100644 --- a/kibana/values_overrides/apparmor.yaml +++ b/kibana/values_overrides/apparmor.yaml @@ -8,3 +8,6 @@ pod: register-kibana-indexes: register-kibana-indexes: runtime/default init: runtime/default + flush-kibana-metadata: + flush-kibana-metadata: runtime/default + init: runtime/default