diff --git a/centos_tarball-dl.lst b/centos_tarball-dl.lst index 8806d0f..732664d 100644 --- a/centos_tarball-dl.lst +++ b/centos_tarball-dl.lst @@ -1,2 +1,2 @@ helm-charts-92b6289ae93816717a8453cfe62bad51cbdb8ad0.tar.gz#helm-charts#https://github.com/helm/charts/archive/92b6289ae93816717a8453cfe62bad51cbdb8ad0.tar.gz#http## -helm-charts-elastic-2bd7616ceddbdf2eee88965e2028ee37d304c79c.tar.gz#helm-charts-elastic#https://github.com/elastic/helm-charts/archive/2bd7616ceddbdf2eee88965e2028ee37d304c79c.tar.gz#http## +helm-charts-elastic-945017287598479ba8653d9baf3ff26f7fe31e50.tar.gz#helm-charts-elastic#https://github.com/elastic/helm-charts/archive/945017287598479ba8653d9baf3ff26f7fe31e50.tar.gz#http## diff --git a/monitor-helm-elastic/Readme.rst b/monitor-helm-elastic/Readme.rst index fdd8396..402ed99 100644 --- a/monitor-helm-elastic/Readme.rst +++ b/monitor-helm-elastic/Readme.rst @@ -1,5 +1,9 @@ This repo is for https://github.com/elastic/helm-charts/tree/master/elasticsearch +https://github.com/elastic/helm-charts/tree/master/filebeat +https://github.com/elastic/helm-charts/tree/master/kibana +https://github.com/elastic/helm-charts/tree/master/logstash +https://github.com/elastic/helm-charts/tree/master/metricbeat Changes to this repo are needed for StarlingX and those changes are not yet merged. diff --git a/monitor-helm-elastic/centos/build_srpm.data b/monitor-helm-elastic/centos/build_srpm.data index 77d0643..ec1821b 100644 --- a/monitor-helm-elastic/centos/build_srpm.data +++ b/monitor-helm-elastic/centos/build_srpm.data @@ -1,5 +1,5 @@ TAR_NAME=helm-charts-elastic -SHA=2bd7616ceddbdf2eee88965e2028ee37d304c79c +SHA=945017287598479ba8653d9baf3ff26f7fe31e50 VERSION=1.0.0 TAR="$TAR_NAME-$SHA.tar.gz" diff --git a/monitor-helm-elastic/centos/monitor-helm-elastic.spec b/monitor-helm-elastic/centos/monitor-helm-elastic.spec index 1be1423..ada3aa5 100644 --- a/monitor-helm-elastic/centos/monitor-helm-elastic.spec +++ b/monitor-helm-elastic/centos/monitor-helm-elastic.spec @@ -1,4 +1,4 @@ -%global sha 2bd7616ceddbdf2eee88965e2028ee37d304c79c +%global sha 945017287598479ba8653d9baf3ff26f7fe31e50 %global helm_folder /usr/lib/helm %global helmchart_version 0.1.0 %global _default_patch_flags --no-backup-if-mismatch --prefix=/tmp/junk @@ -19,11 +19,10 @@ Source2: index.yaml BuildArch: noarch Patch01: 0001-add-makefile.patch -Patch02: 0002-Add-compatibility-for-k8s-1.16.patch -Patch03: 0003-use-oss-image.patch -Patch04: 0004-Update-to-Elastic-7.4.0-Release.patch -Patch05: 0005-set-initial-masters-to-master-0.patch -Patch06: 0006-readiness-probe-enhancements.patch +Patch02: 0002-use-oss-image.patch +Patch03: 0003-set-initial-masters-to-master-0.patch +Patch04: 0004-Update-Elastic-Apps-to-7.6.0-Releases.patch +Patch05: 0005-readiness-probe-enhancements.patch BuildRequires: helm @@ -37,7 +36,6 @@ Monitor Helm elasticsearch charts %patch03 -p1 %patch04 -p1 %patch05 -p1 -%patch06 -p1 %build # initialize helm and build the toolkit @@ -66,7 +64,15 @@ helm repo add local http://localhost:8879/charts # Create the tgz files rm elasticsearch/Makefile +rm kibana/Makefile +rm filebeat/Makefile +rm metricbeat/Makefile +rm logstash/Makefile make elasticsearch +make kibana +make filebeat +make metricbeat +make logstash # terminate helm server (the last backgrounded task) kill %1 diff --git a/monitor-helm-elastic/files/0001-add-makefile.patch b/monitor-helm-elastic/files/0001-add-makefile.patch index b8ffa0e..b61fc2a 100644 --- a/monitor-helm-elastic/files/0001-add-makefile.patch +++ b/monitor-helm-elastic/files/0001-add-makefile.patch @@ -4,16 +4,15 @@ Date: Tue, 1 Oct 2019 15:25:32 -0400 Subject: [PATCH 1/1] add makefile --- - Makefile | 43 +++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 43 insertions(+) - create mode 100644 Makefile + Makefile | 44 +++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 43 insertions(+), 1 deletions(-) diff --git a/Makefile b/Makefile -new file mode 100644 -index 0000000..5cf4447 ---- /dev/null +index 00e154d..5cf4447 +--- a/Makefile +++ b/Makefile -@@ -0,0 +1,43 @@ +@@ -1,1 +1,43 @@ +-include helpers/common.mk +# +# Copyright 2017 The Openstack-Helm Authors. +# diff --git a/monitor-helm-elastic/files/0002-Add-compatibility-for-k8s-1.16.patch b/monitor-helm-elastic/files/0002-Add-compatibility-for-k8s-1.16.patch deleted file mode 100644 index e6aa4ab..0000000 --- a/monitor-helm-elastic/files/0002-Add-compatibility-for-k8s-1.16.patch +++ /dev/null @@ -1,68 +0,0 @@ -From f79169b8e6a57881952ca4d2c338eeda179471e9 Mon Sep 17 00:00:00 2001 -From: Kevin Smith -Date: Mon, 7 Oct 2019 11:18:05 -0400 -Subject: [PATCH 1/1] Add compatibility for k8s 1.16 - ---- - elasticsearch/templates/_helpers.tpl | 22 ++++++++++++++++++++++ - elasticsearch/templates/ingress.yaml | 2 +- - elasticsearch/templates/statefulset.yaml | 2 +- - 3 files changed, 24 insertions(+), 2 deletions(-) - -diff --git a/elasticsearch/templates/_helpers.tpl b/elasticsearch/templates/_helpers.tpl -index d651d56..ccfd8c3 100755 ---- a/elasticsearch/templates/_helpers.tpl -+++ b/elasticsearch/templates/_helpers.tpl -@@ -47,3 +47,25 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this - {{- end -}} - {{- end -}} - {{- end -}} -+ -+{{/* -+Return the appropriate apiVersion for statefulset. -+*/}} -+{{- define "elasticsearch.statefulset.apiVersion" -}} -+{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.GitVersion -}} -+{{- print "apps/v1beta2" -}} -+{{- else -}} -+{{- print "apps/v1" -}} -+{{- end -}} -+{{- end -}} -+ -+{{/* -+Return the appropriate apiVersion for ingress. -+*/}} -+{{- define "elasticsearch.ingress.apiVersion" -}} -+{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} -+{{- print "extensions/v1beta1" -}} -+{{- else -}} -+{{- print "networking.k8s.io/v1beta1" -}} -+{{- end -}} -+{{- end -}} -diff --git a/elasticsearch/templates/ingress.yaml b/elasticsearch/templates/ingress.yaml -index 39167ca..1715b97 100644 ---- a/elasticsearch/templates/ingress.yaml -+++ b/elasticsearch/templates/ingress.yaml -@@ -2,7 +2,7 @@ - {{- $fullName := include "uname" . -}} - {{- $servicePort := .Values.httpPort -}} - {{- $ingressPath := .Values.ingress.path -}} --apiVersion: extensions/v1beta1 -+apiVersion: {{ template "elasticsearch.ingress.apiVersion" . }} - kind: Ingress - metadata: - name: {{ $fullName }} -diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml -index 004578f..36c7af6 100644 ---- a/elasticsearch/templates/statefulset.yaml -+++ b/elasticsearch/templates/statefulset.yaml -@@ -1,5 +1,5 @@ - --- --apiVersion: apps/v1beta1 -+apiVersion: {{ template "elasticsearch.statefulset.apiVersion" . }} - kind: StatefulSet - metadata: - name: {{ template "uname" . }} --- -1.8.3.1 - diff --git a/monitor-helm-elastic/files/0003-use-oss-image.patch b/monitor-helm-elastic/files/0002-use-oss-image.patch similarity index 97% rename from monitor-helm-elastic/files/0003-use-oss-image.patch rename to monitor-helm-elastic/files/0002-use-oss-image.patch index 8c3c0d6..ff198a0 100644 --- a/monitor-helm-elastic/files/0003-use-oss-image.patch +++ b/monitor-helm-elastic/files/0002-use-oss-image.patch @@ -17,7 +17,7 @@ index ccab462..f4fca1f 100755 -image: "docker.elastic.co/elasticsearch/elasticsearch" +image: "docker.elastic.co/elasticsearch/elasticsearch-oss" - imageTag: "7.3.2" + imageTag: "7.6.0" imagePullPolicy: "IfNotPresent" -- diff --git a/monitor-helm-elastic/files/0005-set-initial-masters-to-master-0.patch b/monitor-helm-elastic/files/0003-set-initial-masters-to-master-0.patch similarity index 89% rename from monitor-helm-elastic/files/0005-set-initial-masters-to-master-0.patch rename to monitor-helm-elastic/files/0003-set-initial-masters-to-master-0.patch index 9640e4a..e8baacd 100644 --- a/monitor-helm-elastic/files/0005-set-initial-masters-to-master-0.patch +++ b/monitor-helm-elastic/files/0003-set-initial-masters-to-master-0.patch @@ -12,11 +12,11 @@ diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/ index 36c7af6..e17d39e 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml -@@ -233,7 +233,7 @@ spec: +@@ -241,7 +241,7 @@ spec: {{- if eq .Values.roles.master "true" }} - {{- if ge (int (include "esMajorVersion" .)) 7 }} + {{- if ge (int (include "elasticsearch.esMajorVersion" .)) 7 }} - name: cluster.initial_master_nodes -- value: "{{ template "endpoints" . }}" +- value: "{{ template "elasticsearch.endpoints" . }}" + value: "{{ .Values.initialMasterNodes }}" {{- else }} - name: discovery.zen.minimum_master_nodes diff --git a/monitor-helm-elastic/files/0004-Update-Elastic-Apps-to-7.6.0-Releases.patch b/monitor-helm-elastic/files/0004-Update-Elastic-Apps-to-7.6.0-Releases.patch new file mode 100644 index 0000000..e205fee --- /dev/null +++ b/monitor-helm-elastic/files/0004-Update-Elastic-Apps-to-7.6.0-Releases.patch @@ -0,0 +1,944 @@ +From cf8a4f75684e78920f1c464d06558daa8fc61fbc Mon Sep 17 00:00:00 2001 +From: Simon Cousineau +Date: Mon, 6 Apr 2020 18:10:31 -0400 +Subject: [PATCH 1/1] Update Elastic Apps to 7.6.0 Releases + +--- + elasticsearch/templates/statefulset.yaml | 5 ++ + elasticsearch/values.yaml | 2 + + filebeat/templates/_helpers.tpl | 9 +++ + filebeat/templates/configmap.yaml | 4 + + filebeat/templates/daemonset.yaml | 92 ++++++++++++++++++++++- + filebeat/values.yaml | 16 +++- + kibana/templates/_helpers.tpl | 24 +++++- + kibana/templates/deployment.yaml | 8 +- + kibana/templates/service.yaml | 26 ++++++- + kibana/templates/serviceaccount.yaml | 11 +++ + logstash/templates/_helpers.tpl | 5 ++ + logstash/templates/configmap-config.yaml | 4 + + logstash/templates/statefulset.yaml | 49 +++++++++++- + logstash/values.yaml | 19 +++-- + metricbeat/requirements.yaml | 8 +- + metricbeat/templates/_helpers.tpl | 9 +++ + metricbeat/templates/configmap.yaml | 4 + + metricbeat/templates/daemonset.yaml | 92 ++++++++++++++++++++++- + metricbeat/templates/deployment.yaml | 96 +++++++++++++++++++++++- + metricbeat/templates/serviceaccount.yaml | 2 + + metricbeat/values.yaml | 20 +++-- + 21 files changed, 468 insertions(+), 37 deletions(-) + create mode 100644 kibana/templates/serviceaccount.yaml + +diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml +index 5217270..61b59af 100644 +--- a/elasticsearch/templates/statefulset.yaml ++++ b/elasticsearch/templates/statefulset.yaml +@@ -251,8 +251,13 @@ spec: + - name: discovery.zen.ping.unicast.hosts + value: "{{ template "elasticsearch.masterService" . }}-headless" + {{- else }} ++ {{- if empty .Values.seedHosts }} + - name: discovery.seed_hosts + value: "{{ template "elasticsearch.masterService" . }}-headless" ++ {{- else }} ++ - name: discovery.seed_hosts ++ value: "{{ .Values.seedHosts }}" ++ {{- end }} + {{- end }} + - name: cluster.name + value: "{{ .Values.clusterName }}" +diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml +index e2a5d96..a446316 100755 +--- a/elasticsearch/values.yaml ++++ b/elasticsearch/values.yaml +@@ -259,3 +259,5 @@ sysctlInitContainer: + enabled: true + + keystore: [] ++ ++seedHosts: "" +diff --git a/filebeat/templates/_helpers.tpl b/filebeat/templates/_helpers.tpl +index a52a937..df294e9 100755 +--- a/filebeat/templates/_helpers.tpl ++++ b/filebeat/templates/_helpers.tpl +@@ -9,15 +9,20 @@ Expand the name of the chart. + {{/* + Create a default fully qualified app name. + We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). ++If release name contains chart name it will be used as a full name. + */}} + {{- define "filebeat.fullname" -}} + {{- if .Values.fullnameOverride -}} + {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} + {{- else -}} + {{- $name := default .Chart.Name .Values.nameOverride -}} ++{{- if contains $name .Release.Name -}} ++{{- .Release.Name | trunc 63 | trimSuffix "-" -}} ++{{- else -}} + {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} + {{- end -}} + {{- end -}} ++{{- end -}} + + {{/* + Use the fullname if the serviceAccount value is not set +@@ -27,6 +32,10 @@ Use the fullname if the serviceAccount value is not set + {{- .Values.serviceAccount -}} + {{- else }} + {{- $name := default .Chart.Name .Values.nameOverride -}} ++{{- if contains $name .Release.Name -}} ++{{- .Release.Name | trunc 63 | trimSuffix "-" -}} ++{{- else -}} + {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} + {{- end -}} + {{- end -}} ++{{- end -}} +diff --git a/filebeat/templates/configmap.yaml b/filebeat/templates/configmap.yaml +index 32df8d8..581179f 100644 +--- a/filebeat/templates/configmap.yaml ++++ b/filebeat/templates/configmap.yaml +@@ -12,6 +12,10 @@ metadata: + data: + {{- range $path, $config := .Values.filebeatConfig }} + {{ $path }}: | ++{{- if eq $path "filebeat.yml" }} ++{{ toYaml $config | indent 4 -}} ++{{- else }} + {{ $config | indent 4 -}} + {{- end -}} + {{- end -}} ++{{- end -}} +diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml +index d2f5a23..6e8be91 100644 +--- a/filebeat/templates/daemonset.yaml ++++ b/filebeat/templates/daemonset.yaml +@@ -26,7 +26,7 @@ spec: + {{- end }} + {{/* This forces a restart if the configmap has changed */}} + {{- if .Values.filebeatConfig }} +- configChecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} ++ configchecksum: {{ toYaml .Values.filebeatConfig | sha256sum | trunc 63 }} + {{- end }} + name: "{{ template "filebeat.fullname" . }}" + labels: +@@ -87,10 +87,64 @@ spec: + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: +-{{ toYaml .Values.extraInitContainers | indent 8 }} ++{{- range $name, $config := .Values.extraInitContainers }} ++ - name: {{ $name }} ++{{- if $config.image }} ++ image: "{{ $config.image.repository }}:{{ $config.image.tag }}" ++ imagePullPolicy: {{ $config.image.pullPolicy }} ++{{- else }} ++ image: "{{ $.Values.image }}:{{ $.Values.imageTag }}" ++ imagePullPolicy: {{ $.Values.imagePullPolicy }} ++{{- end }} ++{{ toYaml $config.container | indent 10 }} ++ env: ++ - name: POD_NAMESPACE ++ valueFrom: ++ fieldRef: ++ fieldPath: metadata.namespace ++ - name: NODE_NAME ++ valueFrom: ++ fieldRef: ++ fieldPath: spec.nodeName ++ ## make output.elasticsearch parms visible ++ {{- range $path, $config := $.Values.filebeatConfig}} ++ {{- if eq $path "filebeat.yml" -}} ++ {{- range $key, $value := $config}} ++ {{- $configname := $key | upper | replace "." "_" }} ++ {{- if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ ## make system name and index parms visible ++ {{- if $.Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" $.Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ - name: INDEX_PATTERN ++ value: "{{ $.Values.indexNamePrefix }}-{{$systemNameFormatted}}-*" ++ - name: INDEX_NAME ++ value: "{{ $.Values.indexNamePrefix }}-{{$systemNameFormatted}}" ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "unset" ++ - name: INDEX_PATTERN ++ value: "{{ $.Values.indexNamePrefix }}-*" ++ - name: INDEX_NAME ++ value: "{{ $.Values.indexNamePrefix }}" ++ {{- end }} ++{{- if $.Values.extraEnvs }} ++{{ toYaml $.Values.extraEnvs | indent 12 }} ++{{- end }} ++{{- end }} + {{- end }} + containers: +- - name: "filebeat" ++ - name: {{ template "filebeat.name" . }} + image: "{{ .Values.image }}:{{ .Values.imageTag }}" + imagePullPolicy: "{{ .Values.imagePullPolicy }}" + args: +@@ -112,6 +166,38 @@ spec: + valueFrom: + fieldRef: + fieldPath: spec.nodeName ++ ## make output.elasticsearch parms visible ++ {{- range $path, $config := .Values.filebeatConfig}} ++ {{- if eq $path "filebeat.yml" -}} ++ {{- range $key, $value := $config}} ++ {{- $configname := $key | upper | replace "." "_" }} ++ {{- if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ ## make system name and index parms visible ++ {{- if .Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" .Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}" ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "unset" ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}" ++ {{- end }} + {{- if .Values.extraEnvs }} + {{ toYaml .Values.extraEnvs | indent 8 }} + {{- end }} +diff --git a/filebeat/values.yaml b/filebeat/values.yaml +index d3c575c..f83a19f 100755 +--- a/filebeat/values.yaml ++++ b/filebeat/values.yaml +@@ -2,7 +2,7 @@ + # Allows you to add any config files in /usr/share/filebeat + # such as filebeat.yml + filebeatConfig: +- filebeat.yml: | ++ filebeat.yml: + filebeat.inputs: + - type: docker + containers.ids: +@@ -36,9 +36,13 @@ extraContainers: "" + # command: ['echo', 'hey'] + + extraInitContainers: [] +-# - name: dummy-init +-# image: busybox +-# command: ['echo', 'hey'] ++ # "dummy-init": ++ # image: ++ # repository: busybox ++ # tag: latest ++ # pullPolicy: IfNotPresent ++ # container: ++ # command: ['echo', 'hey'] + + envFrom: [] + # - configMapRef: +@@ -132,3 +136,7 @@ updateStrategy: RollingUpdate + # Only edit these if you know what you're doing + nameOverride: "" + fullnameOverride: "" ++ ++# Will be used in index name ++systemNameForIndex: "" ++indexNamePrefix: filebeat-%{[agent.version]} +diff --git a/kibana/templates/_helpers.tpl b/kibana/templates/_helpers.tpl +index 2fe259e..0558ced 100755 +--- a/kibana/templates/_helpers.tpl ++++ b/kibana/templates/_helpers.tpl +@@ -9,13 +9,18 @@ Expand the name of the chart. + {{/* + Create a default fully qualified app name. + We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). ++If release name contains chart name it will be used as a full name. + */}} + {{- define "kibana.fullname" -}} + {{- if .Values.fullnameOverride -}} + {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} + {{- else -}} +-{{- $name := default .Release.Name .Values.nameOverride -}} +-{{- printf "%s-%s" $name .Chart.Name | trunc 63 | trimSuffix "-" -}} ++{{- $name := default .Chart.Name .Values.nameOverride -}} ++{{- if contains $name .Release.Name -}} ++{{- printf .Release.Name | trunc 63 | trimSuffix "-" -}} ++{{- else -}} ++{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} ++{{- end -}} + {{- end -}} + {{- end -}} + +@@ -29,3 +34,18 @@ Return the appropriate apiVersion for ingress. + {{- print "networking.k8s.io/v1beta1" -}} + {{- end -}} + {{- end -}} ++ ++{{/* ++Create the name of the service account to use ++*/}} ++{{- define "serviceAccountName" -}} ++{{- if .Values.serviceAccountCreate -}} ++{{ default (include "kibana.fullname" .) .Values.serviceAccountName }} ++{{- else -}} ++{{- if .Values.serviceAccountName -}} ++{{- .Values.serviceAccountName }} ++{{- else -}} ++{{ default "default" .Values.serviceAccountName }} ++{{- end -}} ++{{- end -}} ++{{- end -}} +diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml +index ad563b3..15343eb 100644 +--- a/kibana/templates/deployment.yaml ++++ b/kibana/templates/deployment.yaml +@@ -14,12 +14,12 @@ spec: + {{ toYaml .Values.updateStrategy | indent 4 }} + selector: + matchLabels: +- app: kibana ++ app: {{ template "kibana.name" . }} + release: {{ .Release.Name | quote }} + template: + metadata: + labels: +- app: kibana ++ app: {{ template "kibana.name" . }} + release: {{ .Release.Name | quote }} + {{- range $key, $value := .Values.labels }} + {{ $key }}: {{ $value | quote }} +@@ -30,7 +30,7 @@ spec: + {{- end }} + {{/* This forces a restart if the configmap has changed */}} + {{- if .Values.kibanaConfig }} +- configchecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} ++ configchecksum: {{ toYaml .Values.kibanaConfig | sha256sum | trunc 63 }} + {{- end }} + spec: + {{- if .Values.priorityClassName }} +@@ -73,7 +73,7 @@ spec: + {{ tpl .Values.extraInitContainers . | indent 6 }} + {{- end }} + containers: +- - name: kibana ++ - name: {{ template "kibana.name" . }} + securityContext: + {{ toYaml .Values.securityContext | indent 10 }} + image: "{{ .Values.image }}:{{ .Values.imageTag }}" +diff --git a/kibana/templates/service.yaml b/kibana/templates/service.yaml +index 5734580..c2f8d9e 100644 +--- a/kibana/templates/service.yaml ++++ b/kibana/templates/service.yaml +@@ -20,14 +20,36 @@ spec: + loadBalancerSourceRanges: + {{ toYaml . | indent 4 }} + {{- end }} ++ {{- if and (eq .Values.service.type "ClusterIP") .Values.service.clusterIP }} ++ clusterIP: {{ .Values.service.clusterIP }} ++ {{- end }} + ports: + - port: {{ .Values.service.port }} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + protocol: TCP +- name: http +- targetPort: {{ .Values.httpPort }} ++{{ if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} ++ nodePort: {{ .Values.service.nodePort }} ++{{ end }} ++{{- if .Values.service.portName }} ++ name: {{ .Values.service.portName }} ++{{- end }} ++{{- if not .Values.authProxyEnabled }} ++ targetPort: {{ .Values.service.internalPort }} ++{{- else }} ++ targetPort: {{ .Values.service.authProxyPort }} ++{{- end }} ++{{- if .Values.service.externalIPs }} ++ externalIPs: ++{{ toYaml .Values.service.externalIPs | indent 4 }} ++{{- end }} + selector: + app: {{ .Chart.Name }} + release: {{ .Release.Name | quote }} ++{{- range $key, $value := .Values.service.selector }} ++ {{ $key }}: {{ $value | quote }} ++{{- end }} ++{{- if .Values.service.loadBalancerIP }} ++ loadBalancerIP: {{ .Values.service.loadBalancerIP }} ++{{- end }} +diff --git a/kibana/templates/serviceaccount.yaml b/kibana/templates/serviceaccount.yaml +new file mode 100644 +index 0000000..c40b687 +--- /dev/null ++++ b/kibana/templates/serviceaccount.yaml +@@ -0,0 +1,11 @@ ++{{- if .Values.managedServiceAccount -}} ++apiVersion: v1 ++kind: ServiceAccount ++metadata: ++ name: {{ template "serviceAccountName" . }} ++ labels: ++ app: {{ template "kibana.name" . }} ++ chart: {{ .Chart.Name }}-{{ .Chart.Version }} ++ heritage: {{ .Release.Service }} ++ release: {{ .Release.Name }} ++{{- end -}} +diff --git a/logstash/templates/_helpers.tpl b/logstash/templates/_helpers.tpl +index 162a3ee..a0a0321 100755 +--- a/logstash/templates/_helpers.tpl ++++ b/logstash/templates/_helpers.tpl +@@ -9,15 +9,20 @@ Expand the name of the chart. + {{/* + Create a default fully qualified app name. + We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). ++If release name contains chart name it will be used as a full name. + */}} + {{- define "logstash.fullname" -}} + {{- if .Values.fullnameOverride -}} + {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} + {{- else -}} + {{- $name := default .Chart.Name .Values.nameOverride -}} ++{{- if contains $name .Release.Name -}} ++{{- .Release.Name | trunc 63 | trimSuffix "-" -}} ++{{- else -}} + {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} + {{- end -}} + {{- end -}} ++{{- end -}} + + {{/* + Return the appropriate apiVersion for statefulset. +diff --git a/logstash/templates/configmap-config.yaml b/logstash/templates/configmap-config.yaml +index e69d524..e9e7a5b 100644 +--- a/logstash/templates/configmap-config.yaml ++++ b/logstash/templates/configmap-config.yaml +@@ -12,6 +12,10 @@ metadata: + data: + {{- range $path, $config := .Values.logstashConfig }} + {{ $path }}: | ++{{- if eq $path "logstash.yml" }} ++{{ toYaml $config | indent 4 -}} ++{{- else }} + {{ $config | indent 4 -}} + {{- end -}} + {{- end -}} ++{{- end -}} +diff --git a/logstash/templates/statefulset.yaml b/logstash/templates/statefulset.yaml +index f953ec8..09ddfd0 100644 +--- a/logstash/templates/statefulset.yaml ++++ b/logstash/templates/statefulset.yaml +@@ -51,11 +51,11 @@ spec: + {{- end }} + {{/* This forces a restart if the configmap has changed */}} + {{- if .Values.logstashConfig }} +- configchecksum: {{ include (print .Template.BasePath "/configmap-config.yaml") . | sha256sum | trunc 63 }} ++ configchecksum: {{ toYaml .Values.logstashConfig | sha256sum | trunc 63 }} + {{- end }} + {{/* This forces a restart if the configmap has changed */}} + {{- if .Values.logstashPipeline }} +- pipelinechecksum: {{ include (print .Template.BasePath "/configmap-pipeline.yaml") . | sha256sum | trunc 63 }} ++ pipelinechecksum: {{ toYaml .Values.logstashPipeline | sha256sum | trunc 63 }} + {{- end }} + spec: + {{- if .Values.schedulerName }} +@@ -135,7 +135,34 @@ spec: + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: +-{{ tpl .Values.extraInitContainers . | indent 6 }} ++{{- range $name, $config := .Values.extraInitContainers }} ++ - name: {{ $name }} ++{{- if $config.image }} ++ image: "{{ $config.image.repository }}:{{ $config.image.tag }}" ++ imagePullPolicy: {{ $config.image.pullPolicy }} ++{{- else }} ++ image: "{{ $.Values.image }}:{{ $.Values.imageTag }}" ++ imagePullPolicy: {{ $.Values.imagePullPolicy }} ++{{- end }} ++{{ toYaml $config.container | indent 8 }} ++ env: ++ ## Elasticsearch output ++ - name: ELASTICSEARCH_HOST ++ value: {{ $.Values.elasticsearchHosts | quote }} ++ ## make system name visible for in setting index. ++ {{- if $.Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" $.Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "" ++ {{- end }} ++{{- if $.Values.extraEnvs }} ++{{ toYaml $.Values.extraEnvs | indent 10 }} ++{{- end }} ++{{- end }} + {{- end }} + containers: + - name: "{{ template "logstash.name" . }}" +@@ -150,9 +177,25 @@ spec: + ports: + - name: http + containerPort: {{ .Values.httpPort }} ++{{- if .Values.ports }} ++{{ toYaml .Values.ports | indent 8 }} ++{{- end }} + resources: + {{ toYaml .Values.resources | indent 10 }} + env: ++ ## Elasticsearch output ++ - name: ELASTICSEARCH_HOST ++ value: {{ .Values.elasticsearchHosts | quote }} ++ ## make system name visible for in setting index. ++ {{- if .Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" .Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "" ++ {{- end }} + - name: LS_JAVA_OPTS + value: "{{ .Values.logstashJavaOpts }}" + {{- if .Values.extraEnvs }} +diff --git a/logstash/values.yaml b/logstash/values.yaml +index 498136a..990287b 100755 +--- a/logstash/values.yaml ++++ b/logstash/values.yaml +@@ -4,7 +4,7 @@ replicas: 1 + # Allows you to add any config files in /usr/share/logstash/config/ + # such as logstash.yml and log4j2.properties + logstashConfig: {} +-# logstash.yml: | ++# logstash.yml: + # key: + # nestedkey: value + # log4j2.properties: | +@@ -97,10 +97,14 @@ extraContainers: "" + # image: busybox + # command: ['do', 'something'] + +-extraInitContainers: "" +- # - name: do-something +- # image: busybox +- # command: ['do', 'something'] ++extraInitContainers: [] ++ # "do-something": ++ # image: ++ # repository: busybox ++ # tag: latest ++ # pullPolicy: IfNotPresent ++ # container: ++ # command: ['do', 'something'] + + # This is the PriorityClass settings as defined in + # https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass +@@ -185,6 +189,11 @@ lifecycle: {} + # exec: + # command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"] + ++# Will be used in index names ++systemNameForIndex: "" ++ ++ports: [] ++ + service: {} + # annotations: {} + # type: ClusterIP +diff --git a/metricbeat/requirements.yaml b/metricbeat/requirements.yaml +index 0f8c03d..7ee66b5 100644 +--- a/metricbeat/requirements.yaml ++++ b/metricbeat/requirements.yaml +@@ -1,4 +1,4 @@ +-dependencies: +- - name: 'kube-state-metrics' +- version: '2.4.1' +- repository: '@stable' ++dependencies: [] ++ # - name: 'kube-state-metrics' ++ # version: '2.4.1' ++ # repository: '@stable' +diff --git a/metricbeat/templates/_helpers.tpl b/metricbeat/templates/_helpers.tpl +index 35f776c..38e2cc7 100755 +--- a/metricbeat/templates/_helpers.tpl ++++ b/metricbeat/templates/_helpers.tpl +@@ -9,15 +9,20 @@ Expand the name of the chart. + {{/* + Create a default fully qualified app name. + We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). ++If release name contains chart name it will be used as a full name. + */}} + {{- define "metricbeat.fullname" -}} + {{- if .Values.fullnameOverride -}} + {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} + {{- else -}} + {{- $name := default .Chart.Name .Values.nameOverride -}} ++{{- if contains $name .Release.Name -}} ++{{- .Release.Name | trunc 63 | trimSuffix "-" -}} ++{{- else -}} + {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} + {{- end -}} + {{- end -}} ++{{- end -}} + + {{/* + Use the fullname if the serviceAccount value is not set +@@ -27,6 +32,10 @@ Use the fullname if the serviceAccount value is not set + {{- .Values.serviceAccount -}} + {{- else }} + {{- $name := default .Chart.Name .Values.nameOverride -}} ++{{- if contains $name .Release.Name -}} ++{{- .Release.Name | trunc 63 | trimSuffix "-" -}} ++{{- else -}} + {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} + {{- end -}} + {{- end -}} ++{{- end -}} +diff --git a/metricbeat/templates/configmap.yaml b/metricbeat/templates/configmap.yaml +index 09e381c..56af441 100644 +--- a/metricbeat/templates/configmap.yaml ++++ b/metricbeat/templates/configmap.yaml +@@ -12,6 +12,10 @@ metadata: + data: + {{- range $path, $config := .Values.metricbeatConfig }} + {{ $path }}: | ++{{- if or (eq $path "metricbeat.yml") (eq $path "kube-state-metrics-metricbeat.yml") }} ++{{ toYaml $config | indent 4 -}} ++{{- else }} + {{ $config | indent 4 -}} + {{- end -}} + {{- end -}} ++{{- end -}} +diff --git a/metricbeat/templates/daemonset.yaml b/metricbeat/templates/daemonset.yaml +index 392b7cf..db02a29 100644 +--- a/metricbeat/templates/daemonset.yaml ++++ b/metricbeat/templates/daemonset.yaml +@@ -26,7 +26,7 @@ spec: + {{- end }} + {{/* This forces a restart if the configmap has changed */}} + {{- if .Values.metricbeatConfig }} +- configChecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} ++ configchecksum: {{ toYaml .Values.metricbeatConfig | sha256sum | trunc 63 }} + {{- end }} + name: "{{ template "metricbeat.fullname" . }}" + labels: +@@ -52,6 +52,8 @@ spec: + {{- end }} + serviceAccountName: {{ template "metricbeat.serviceAccount" . }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }} ++ hostNetwork: true ++ dnsPolicy: ClusterFirstWithHostNet + volumes: + {{- range .Values.secretMounts }} + - name: {{ .name }} +@@ -89,7 +91,61 @@ spec: + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: +-{{ tpl .Values.extraInitContainers . | indent 6 }} ++{{- range $name, $config := .Values.extraInitContainers }} ++ - name: {{ $name }} ++{{- if $config.image }} ++ image: "{{ $config.image.repository }}:{{ $config.image.tag }}" ++ imagePullPolicy: {{ $config.image.pullPolicy }} ++{{- else }} ++ image: "{{ $.Values.image }}:{{ $.Values.imageTag }}" ++ imagePullPolicy: {{ $.Values.imagePullPolicy }} ++{{- end }} ++{{ toYaml $config.container | indent 8 }} ++ env: ++ - name: POD_NAMESPACE ++ valueFrom: ++ fieldRef: ++ fieldPath: metadata.namespace ++ - name: NODE_NAME ++ valueFrom: ++ fieldRef: ++ fieldPath: spec.nodeName ++ ## make output.elasticsearch parms visible ++ {{- range $path, $config := $.Values.metricbeatConfig}} ++ {{- if eq $path "metricbeat.yml" -}} ++ {{- range $key, $value := $config}} ++ {{- $configname := $key | upper | replace "." "_" }} ++ {{- if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ ## make system name and index parms visible ++ {{- if $.Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" $.Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ - name: INDEX_PATTERN ++ value: "{{ $.Values.indexNamePrefix }}-{{$systemNameFormatted}}-*" ++ - name: INDEX_NAME ++ value: "{{ $.Values.indexNamePrefix }}-{{$systemNameFormatted}}" ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "unset" ++ - name: INDEX_PATTERN ++ value: "{{ $.Values.indexNamePrefix }}-*" ++ - name: INDEX_NAME ++ value: "{{ $.Values.indexNamePrefix }}" ++ {{- end }} ++{{- if $.Values.extraEnvs }} ++{{ toYaml $.Values.extraEnvs | indent 10 }} ++{{- end }} ++{{- end }} + {{- end }} + containers: + - name: "metricbeat" +@@ -115,6 +171,38 @@ spec: + valueFrom: + fieldRef: + fieldPath: spec.nodeName ++ ## make output.elasticsearch parms visible ++ {{- range $path, $config := .Values.metricbeatConfig}} ++ {{- if eq $path "metricbeat.yml" -}} ++ {{- range $key, $value := $config}} ++ {{- $configname := $key | upper | replace "." "_" }} ++ {{- if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ ## make system name and index parms visible ++ {{- if .Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" .Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}" ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "unset" ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}" ++ {{- end }} + {{- if .Values.extraEnvs }} + {{ toYaml .Values.extraEnvs | indent 8 }} + {{- end }} +diff --git a/metricbeat/templates/deployment.yaml b/metricbeat/templates/deployment.yaml +index 225160b..2585651 100644 +--- a/metricbeat/templates/deployment.yaml ++++ b/metricbeat/templates/deployment.yaml +@@ -24,7 +24,7 @@ spec: + {{- end }} + {{/* This forces a restart if the configmap has changed */}} + {{- if .Values.metricbeatConfig }} +- configChecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} ++ configchecksum: {{ toYaml .Values.metricbeatConfig | sha256sum | trunc 63 }} + {{- end }} + labels: + app: '{{ template "metricbeat.fullname" . }}-metrics' +@@ -62,7 +62,63 @@ spec: + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: +-{{ tpl .Values.extraInitContainers . | indent 6 }} ++{{- range $name, $config := .Values.extraInitContainers }} ++ - name: {{ $name }} ++{{- if $config.image }} ++ image: "{{ $config.image.repository }}:{{ $config.image.tag }}" ++ imagePullPolicy: {{ $config.image.pullPolicy }} ++{{- else }} ++ image: "{{ $.Values.image }}:{{ $.Values.imageTag }}" ++ imagePullPolicy: {{ $.Values.imagePullPolicy }} ++{{- end }} ++{{ toYaml $config.container | indent 8 }} ++ env: ++ - name: POD_NAMESPACE ++ valueFrom: ++ fieldRef: ++ fieldPath: metadata.namespace ++ - name: KUBE_STATE_METRICS_HOSTS ++ value: "$({{ $.Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_HOST):$({{ $.Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_PORT_HTTP)" ++ - name: NODE_NAME ++ valueFrom: ++ fieldRef: ++ fieldPath: spec.nodeName ++ ## make output.elasticsearch parms visible ++ {{- range $path, $config := $.Values.metricbeatConfig}} ++ {{- if eq $path "metricbeat.yml" -}} ++ {{- range $key, $value := $config}} ++ {{- $configname := $key | upper | replace "." "_" }} ++ {{- if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ ## make system name and index parms visible ++ {{- if $.Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" $.Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ - name: INDEX_PATTERN ++ value: "{{ $.Values.indexNamePrefix }}-{{$systemNameFormatted}}-*" ++ - name: INDEX_NAME ++ value: "{{ $.Values.indexNamePrefix }}-{{$systemNameFormatted}}" ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "unset" ++ - name: INDEX_PATTERN ++ value: "{{ $.Values.indexNamePrefix }}-*" ++ - name: INDEX_NAME ++ value: "{{ $.Values.indexNamePrefix }}" ++ {{- end }} ++{{- if $.Values.extraEnvs }} ++{{ toYaml $.Values.extraEnvs | indent 10 }} ++{{- end }} ++{{- end }} + {{- end }} + containers: + - name: "metricbeat" +@@ -87,6 +143,42 @@ spec: + fieldPath: metadata.namespace + - name: KUBE_STATE_METRICS_HOSTS + value: "$({{ .Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_HOST):$({{ .Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_PORT_HTTP)" ++ - name: NODE_NAME ++ valueFrom: ++ fieldRef: ++ fieldPath: spec.nodeName ++ ## make output.elasticsearch parms visible ++ {{- range $path, $config := .Values.metricbeatConfig}} ++ {{- if eq $path "metricbeat.yml" -}} ++ {{- range $key, $value := $config}} ++ {{- $configname := $key | upper | replace "." "_" }} ++ {{- if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ ## make system name and index parms visible ++ {{- if .Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" .Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}" ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "unset" ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}" ++ {{- end }} + {{- if .Values.extraEnvs }} + {{ toYaml .Values.extraEnvs | indent 8 }} + {{- end }} +diff --git a/metricbeat/templates/serviceaccount.yaml b/metricbeat/templates/serviceaccount.yaml +index 2330646..49d6782 100644 +--- a/metricbeat/templates/serviceaccount.yaml ++++ b/metricbeat/templates/serviceaccount.yaml +@@ -8,4 +8,6 @@ metadata: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} ++imagePullSecrets: ++ - name: default-registry-key + {{- end -}} +diff --git a/metricbeat/values.yaml b/metricbeat/values.yaml +index 3ea4af3..78ad798 100755 +--- a/metricbeat/values.yaml ++++ b/metricbeat/values.yaml +@@ -2,7 +2,7 @@ + # Allows you to add any config files in /usr/share/metricbeat + # such as metricbeat.yml + metricbeatConfig: +- metricbeat.yml: | ++ metricbeat.yml: + metricbeat.modules: + - module: kubernetes + metricsets: +@@ -45,7 +45,7 @@ metricbeatConfig: + output.elasticsearch: + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' + +- kube-state-metrics-metricbeat.yml: | ++ kube-state-metrics-metricbeat.yml: + metricbeat.modules: + - module: kubernetes + enabled: true +@@ -64,15 +64,23 @@ metricbeatConfig: + + replicas: 1 + ++# Will be used in index name ++systemNameForIndex: "" ++indexNamePrefix: metricbeat-%{[agent.version]} ++ + extraContainers: "" + # - name: dummy-init + # image: busybox + # command: ['echo', 'hey'] + +-extraInitContainers: "" +-# - name: dummy-init +-# image: busybox +-# command: ['echo', 'hey'] ++extraInitContainers: [] ++ # "dummy-init": ++ # image: ++ # repository: busybox ++ # tag: latest ++ # pullPolicy: IfNotPresent ++ # container: ++ # command: ['echo', 'hey'] + + # Extra environment variables to append to the DaemonSet pod spec. + # This will be appended to the current 'env:' key. You can use any of the kubernetes env +-- +2.20.1 + diff --git a/monitor-helm-elastic/files/0004-Update-to-Elastic-7.4.0-Release.patch b/monitor-helm-elastic/files/0004-Update-to-Elastic-7.4.0-Release.patch deleted file mode 100644 index 87559ea..0000000 --- a/monitor-helm-elastic/files/0004-Update-to-Elastic-7.4.0-Release.patch +++ /dev/null @@ -1,649 +0,0 @@ -From 560555d13b524eb5209ab42144a3d40989cc2ac7 Mon Sep 17 00:00:00 2001 -From: John Kung -Date: Tue, 15 Oct 2019 12:56:39 -0400 -Subject: [PATCH 1/1] Update to Elastic 7.4.0 Release - ---- - CHANGELOG.md | 45 ++++++++++++++++++++-- - elasticsearch/Chart.yaml | 4 +- - elasticsearch/README.md | 8 ++-- - elasticsearch/examples/default/test/goss.yaml | 2 +- - elasticsearch/examples/openshift/test/goss.yaml | 2 +- - elasticsearch/examples/oss/test/goss.yaml | 2 +- - elasticsearch/examples/upgrade/test/goss.yaml | 2 +- - elasticsearch/values.yaml | 2 +- - filebeat/Chart.yaml | 4 +- - filebeat/README.md | 8 ++-- - filebeat/examples/default/test/goss.yaml | 4 +- - filebeat/examples/oss/test/goss.yaml | 2 +- - filebeat/examples/security/test/goss.yaml | 2 +- - filebeat/values.yaml | 2 +- - helpers/bumper.py | 4 +- - helpers/examples.mk | 2 +- - kibana/Chart.yaml | 4 +- - kibana/README.md | 8 ++-- - kibana/examples/default/test/goss.yaml | 2 +- - kibana/values.yaml | 2 +- - metricbeat/Chart.yaml | 4 +- - metricbeat/README.md | 8 ++-- - metricbeat/examples/default/test/goss-metrics.yaml | 6 +-- - metricbeat/examples/default/test/goss.yaml | 6 +-- - metricbeat/examples/oss/test/goss-metrics.yaml | 6 +-- - metricbeat/examples/oss/test/goss.yaml | 6 +-- - .../examples/security/test/goss-metrics.yaml | 6 +-- - metricbeat/examples/security/test/goss.yaml | 6 +-- - metricbeat/values.yaml | 2 +- - 29 files changed, 99 insertions(+), 62 deletions(-) - -diff --git a/CHANGELOG.md b/CHANGELOG.md -index 250d11a..bb2fc24 100644 ---- a/CHANGELOG.md -+++ b/CHANGELOG.md -@@ -1,9 +1,46 @@ - # Changelog - -+## 7.4.0 - 2019/10/01 -+ -+* 7.4.0 as the default stack version -+* Helm-tester Docker image migrated to Python 3 in [#297](https://github.com/elastic/helm-charts/pull/297) [@jmlrt](https://github.com/jmlrt) -+* Helm-tester Python dependencies freeze in [#309](https://github.com/elastic/helm-charts/pull/309) [@jmlrt](https://github.com/jmlrt) -+ -+### Elasticsearch -+ -+| PR | Author | Title | -+| ------------------------------------------------------- | -------------------------------------------------- | ----------------------------------------------------------------------------------- | -+|[#296](https://github.com/elastic/helm-charts/pull/296) | [@jmlrt](https://github.com/jmlrt) | Fix "; \" when there is no additional command in the Makefiles | -+|[#298](https://github.com/elastic/helm-charts/pull/298) | [@floretan](https://github.com/floretan) | Make it possible to override the endpoint template. | -+|[#263](https://github.com/elastic/helm-charts/pull/263) | [@Crazybus](https://github.com/Crazybus) | Add working examples for running Elasticsearch and Kibana on OpenShift | -+|[#301](https://github.com/elastic/helm-charts/pull/301) | [@ravishivt](https://github.com/ravishivt) | Fix bug in keystore initContainer | -+ -+### Kibana -+ -+| PR | Author | Title | -+| ------------------------------------------------------- | -------------------------------------------------- | ----------------------------------------------------------------------------------- | -+|[#295](https://github.com/elastic/helm-charts/pull/295) | [@karlbohlmark](https://github.com/karlbohlmark) | Allow configuring lifecycle events | -+|[#263](https://github.com/elastic/helm-charts/pull/263) | [@Crazybus](https://github.com/Crazybus) | Add working examples for running Elasticsearch and Kibana on OpenShift | -+|[#303](https://github.com/elastic/helm-charts/pull/303) | [@code-chris](https://github.com/code-chris) | Add compatibility for k8s 1.16 and change min k8s version due to ingress apiVersion | -+ -+ -+### Filebeat -+ -+| PR | Author | Title | -+| ------------------------------------------------------- | -------------------------------------------------- | ----------------------------------------------------------------------------------- | -+|[#304](https://github.com/elastic/helm-charts/pull/304) | [@code-chris](https://github.com/code-chris) | Change min k8s version due to daemonset apiVersion | -+ -+### Metricbeat -+ -+| PR | Author | Title | -+| ------------------------------------------------------- | -------------------------------------------------- | ----------------------------------------------------------------------------------- | -+|[#305](https://github.com/elastic/helm-charts/pull/305) | [@code-chris](https://github.com/code-chris) | Change min k8s version due to used apiVersions | -+ -+ - ## 7.3.2 - 2019/09/19 - - * 7.3.2 as the default stack version --* Testing of GKE for 1.11 dropped and 1.14 added [#287](https://github.com/elastic/helm-charts/pull/287)) -+* Testing of GKE for 1.11 dropped and 1.14 added [#287](https://github.com/elastic/helm-charts/pull/287) - * Make helper scripts python3 compatible [#255](https://github.com/elastic/helm-charts/pull/255) [@cclauss](https://github.com/cclauss) - - ### Elasticsearch -@@ -176,14 +213,14 @@ - - ### Elasticsearch - --* [#100](https://github.com/elastic/helm-charts/pull/100) - @kuisathaverat - Remove deprecated zen ping unicast hosts setting -+* [#100](https://github.com/elastic/helm-charts/pull/100) - @kuisathaverat - Remove deprecated zen ping unicast hosts setting - * [#114](https://github.com/elastic/helm-charts/pull/114) - @Crazybus - Make persistent volumes optional - * [#115](https://github.com/elastic/helm-charts/pull/115) - @Crazybus - Added an integration test for upgrading from the previous release and testing rolling upgrades - - - ### Kibana - --* [#107](https://github.com/elastic/helm-charts/pull/107) - @Crazybus - Make the health check path configurable to support webroots and other customizations. -+* [#107](https://github.com/elastic/helm-charts/pull/107) - @Crazybus - Make the health check path configurable to support webroots and other customizations. - - --- - ## 7.0.0-alpha1 - 2019/04/17 -@@ -203,7 +240,7 @@ esMajorVersion: 6 - imageTag: 6.7.1 - ``` - --If you are upgrading an existing cluster that did not override the default `storageClassName` you will now need to specify the `storageClassName`. This only affects existing clusters and was changed in https://github.com/elastic/helm-charts/pull/94. The advantage of this is that now the helm chart will just use the default storageClassName rather than needing to override it for any providers where it is not called `standard`. -+If you are upgrading an existing cluster that did not override the default `storageClassName` you will now need to specify the `storageClassName`. This only affects existing clusters and was changed in https://github.com/elastic/helm-charts/pull/94. The advantage of this is that now the helm chart will just use the default storageClassName rather than needing to override it for any providers where it is not called `standard`. - - ``` - volumeClaimTemplate: -diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml -index cfb40d8..e9d5d24 100755 ---- a/elasticsearch/Chart.yaml -+++ b/elasticsearch/Chart.yaml -@@ -4,8 +4,8 @@ maintainers: - - email: helm-charts@elastic.co - name: Elastic - name: elasticsearch --version: 7.3.2 --appVersion: 7.3.2 -+version: 7.4.0 -+appVersion: 7.4.0 - sources: - - https://github.com/elastic/elasticsearch - icon: https://helm.elastic.co/icons/elasticsearch.png -diff --git a/elasticsearch/README.md b/elasticsearch/README.md -index ff90f09..ab9fe7f 100644 ---- a/elasticsearch/README.md -+++ b/elasticsearch/README.md -@@ -43,14 +43,14 @@ This chart is tested with the latest supported versions. The currently tested ve - - | 6.x | 7.x | - | ----- | ----- | --| 6.8.3 | 7.3.2 | -+| 6.8.3 | 7.4.0 | - - Examples of installing older major versions can be found in the [examples](./examples) directory. - --While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.3.2` of Elasticsearch it would look like this: -+While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.4.0` of Elasticsearch it would look like this: - - ``` --helm install --name elasticsearch elastic/elasticsearch --set imageTag=7.3.2 -+helm install --name elasticsearch elastic/elasticsearch --set imageTag=7.4.0 - ``` - - ## Configuration -@@ -71,7 +71,7 @@ helm install --name elasticsearch elastic/elasticsearch --set imageTag=7.3.2 - | `extraInitContainers` | Templatable string of additional init containers to be passed to the `tpl` function | `""` | - | `secretMounts` | Allows you easily mount a secret as a file inside the statefulset. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | - | `image` | The Elasticsearch docker image | `docker.elastic.co/elasticsearch/elasticsearch` | --| `imageTag` | The Elasticsearch docker image tag | `7.3.2` | -+| `imageTag` | The Elasticsearch docker image tag | `7.4.0` | - | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | - | `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Elasticsearch pods | `{}` | - | `labels` | Configurable [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) applied to all Elasticsearch pods | `{}` | -diff --git a/elasticsearch/examples/default/test/goss.yaml b/elasticsearch/examples/default/test/goss.yaml -index 182981d..49b1994 100644 ---- a/elasticsearch/examples/default/test/goss.yaml -+++ b/elasticsearch/examples/default/test/goss.yaml -@@ -15,7 +15,7 @@ http: - status: 200 - timeout: 2000 - body: -- - '"number" : "7.3.2"' -+ - '"number" : "7.4.0"' - - '"cluster_name" : "elasticsearch"' - - '"name" : "elasticsearch-master-0"' - - 'You Know, for Search' -diff --git a/elasticsearch/examples/openshift/test/goss.yaml b/elasticsearch/examples/openshift/test/goss.yaml -index 275754b..3a7a1b0 100644 ---- a/elasticsearch/examples/openshift/test/goss.yaml -+++ b/elasticsearch/examples/openshift/test/goss.yaml -@@ -11,7 +11,7 @@ http: - status: 200 - timeout: 2000 - body: -- - '"number" : "7.3.0"' -+ - '"number" : "7.4.0"' - - '"cluster_name" : "elasticsearch"' - - '"name" : "elasticsearch-master-0"' - - 'You Know, for Search' -diff --git a/elasticsearch/examples/oss/test/goss.yaml b/elasticsearch/examples/oss/test/goss.yaml -index 21e268b..1eec5a8 100644 ---- a/elasticsearch/examples/oss/test/goss.yaml -+++ b/elasticsearch/examples/oss/test/goss.yaml -@@ -11,7 +11,7 @@ http: - status: 200 - timeout: 2000 - body: -- - '"number" : "7.3.2"' -+ - '"number" : "7.4.0"' - - '"cluster_name" : "oss"' - - '"name" : "oss-master-0"' - - 'You Know, for Search' -diff --git a/elasticsearch/examples/upgrade/test/goss.yaml b/elasticsearch/examples/upgrade/test/goss.yaml -index 5e20f7f..f7dbf10 100644 ---- a/elasticsearch/examples/upgrade/test/goss.yaml -+++ b/elasticsearch/examples/upgrade/test/goss.yaml -@@ -11,7 +11,7 @@ http: - status: 200 - timeout: 2000 - body: -- - '"number" : "7.3.2"' -+ - '"number" : "7.4.0"' - - '"cluster_name" : "upgrade"' - - '"name" : "upgrade-master-0"' - - 'You Know, for Search' -diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml -index f4fca1f..6175540 100755 ---- a/elasticsearch/values.yaml -+++ b/elasticsearch/values.yaml -@@ -43,7 +43,7 @@ secretMounts: [] - # path: /usr/share/elasticsearch/config/certs - - image: "docker.elastic.co/elasticsearch/elasticsearch-oss" --imageTag: "7.3.2" -+imageTag: "7.4.0" - imagePullPolicy: "IfNotPresent" - - podAnnotations: {} -diff --git a/filebeat/Chart.yaml b/filebeat/Chart.yaml -index 59d7ebe..42cffb3 100755 ---- a/filebeat/Chart.yaml -+++ b/filebeat/Chart.yaml -@@ -4,8 +4,8 @@ maintainers: - - email: helm-charts@elastic.co - name: Elastic - name: filebeat --version: 7.3.2 --appVersion: 7.3.2 -+version: 7.4.0 -+appVersion: 7.4.0 - sources: - - https://github.com/elastic/beats - icon: https://helm.elastic.co/icons/filebeat.png -diff --git a/filebeat/README.md b/filebeat/README.md -index 901515d..5b56ca1 100644 ---- a/filebeat/README.md -+++ b/filebeat/README.md -@@ -30,14 +30,14 @@ This chart is tested with the latest supported versions. The currently tested ve - - | 6.x | 7.x | - | ----- | ----- | --| 6.8.3 | 7.3.2 | -+| 6.8.3 | 7.4.0 | - - Examples of installing older major versions can be found in the [examples](./examples) directory. - --While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.3.2` of Filebeat it would look like this: -+While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.4.0` of Filebeat it would look like this: - - ``` --helm install --name filebeat elastic/filebeat --set imageTag=7.3.2 -+helm install --name filebeat elastic/filebeat --set imageTag=7.4.0 - ``` - - -@@ -50,7 +50,7 @@ helm install --name filebeat elastic/filebeat --set imageTag=7.3.2 - | `extraVolumes` | Templatable string of additional volumes to be passed to the `tpl` function | `""` | - | `hostPathRoot` | Fully-qualified [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) that will be used to persist Filebeat registry data | `/var/lib` | - | `image` | The Filebeat docker image | `docker.elastic.co/beats/filebeat` | --| `imageTag` | The Filebeat docker image tag | `7.3.2` | -+| `imageTag` | The Filebeat docker image tag | `7.4.0` | - | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | - | `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | - | `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles. | `true` | -diff --git a/filebeat/examples/default/test/goss.yaml b/filebeat/examples/default/test/goss.yaml -index ea0ca18..6713348 100644 ---- a/filebeat/examples/default/test/goss.yaml -+++ b/filebeat/examples/default/test/goss.yaml -@@ -29,7 +29,7 @@ http: - status: 200 - timeout: 2000 - body: -- - 'filebeat-7.3.2' -+ - 'filebeat-7.4.0' - - file: - /usr/share/filebeat/filebeat.yml: -@@ -44,4 +44,4 @@ command: - exit-status: 0 - stdout: - - 'elasticsearch: http://elasticsearch-master:9200' -- - 'version: 7.3.2' -+ - 'version: 7.4.0' -diff --git a/filebeat/examples/oss/test/goss.yaml b/filebeat/examples/oss/test/goss.yaml -index cf9d0a9..f38cbfe 100644 ---- a/filebeat/examples/oss/test/goss.yaml -+++ b/filebeat/examples/oss/test/goss.yaml -@@ -19,4 +19,4 @@ http: - status: 200 - timeout: 2000 - body: -- - 'filebeat-7.3.2' -+ - 'filebeat-7.4.0' -diff --git a/filebeat/examples/security/test/goss.yaml b/filebeat/examples/security/test/goss.yaml -index f5efc42..a7ef6c3 100644 ---- a/filebeat/examples/security/test/goss.yaml -+++ b/filebeat/examples/security/test/goss.yaml -@@ -3,7 +3,7 @@ http: - status: 200 - timeout: 2000 - body: -- - 'filebeat-7.3.2' -+ - 'filebeat-7.4.0' - allow-insecure: true - username: '{{ .Env.ELASTICSEARCH_USERNAME }}' - password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' -diff --git a/filebeat/values.yaml b/filebeat/values.yaml -index 284185a..1e6af75 100755 ---- a/filebeat/values.yaml -+++ b/filebeat/values.yaml -@@ -34,7 +34,7 @@ extraVolumes: "" - hostPathRoot: /var/lib - - image: "docker.elastic.co/beats/filebeat" --imageTag: "7.3.2" -+imageTag: "7.4.0" - imagePullPolicy: "IfNotPresent" - imagePullSecrets: [] - -diff --git a/helpers/bumper.py b/helpers/bumper.py -index c1cf2a8..5d4f0d6 100755 ---- a/helpers/bumper.py -+++ b/helpers/bumper.py -@@ -7,11 +7,11 @@ import fileinput - - os.chdir(os.path.join(os.path.dirname(__file__), '..')) - --chart_version = '7.3.2' -+chart_version = '7.4.0' - - versions = { - 6: '6.8.3', -- 7: '7.3.2', -+ 7: '7.4.0', - } - - file_patterns = [ -diff --git a/helpers/examples.mk b/helpers/examples.mk -index 9a429f8..56f749c 100644 ---- a/helpers/examples.mk -+++ b/helpers/examples.mk -@@ -1,7 +1,7 @@ - GOSS_VERSION := v0.3.6 - GOSS_FILE ?= goss.yaml - GOSS_SELECTOR ?= release=$(RELEASE) --STACK_VERSION := 7.3.2 -+STACK_VERSION := 7.4.0 - - goss: - GOSS_CONTAINER=$$(kubectl get --no-headers=true pods -l $(GOSS_SELECTOR) -o custom-columns=:metadata.name | sed -n 1p ) && \ -diff --git a/kibana/Chart.yaml b/kibana/Chart.yaml -index 2754b57..d858b0a 100755 ---- a/kibana/Chart.yaml -+++ b/kibana/Chart.yaml -@@ -4,8 +4,8 @@ maintainers: - - email: helm-charts@elastic.co - name: Elastic - name: kibana --version: 7.3.2 --appVersion: 7.3.2 -+version: 7.4.0 -+appVersion: 7.4.0 - sources: - - https://github.com/elastic/kibana - icon: https://helm.elastic.co/icons/kibana.png -diff --git a/kibana/README.md b/kibana/README.md -index ef30957..6f2cf2e 100644 ---- a/kibana/README.md -+++ b/kibana/README.md -@@ -26,14 +26,14 @@ This chart is tested with the latest supported versions. The currently tested ve - - | 6.x | 7.x | - | ----- | ----- | --| 6.8.3 | 7.3.2 | -+| 6.8.3 | 7.4.0 | - - Examples of installing older major versions can be found in the [examples](./examples) directory. - --While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.3.2` of Kibana it would look like this: -+While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.4.0` of Kibana it would look like this: - - ``` --helm install --name kibana elastic/kibana --set imageTag=7.3.2 -+helm install --name kibana elastic/kibana --set imageTag=7.4.0 - ``` - - ## Configuration -@@ -46,7 +46,7 @@ helm install --name kibana elastic/kibana --set imageTag=7.3.2 - | `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | - | `secretMounts` | Allows you easily mount a secret as a file inside the deployment. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | - | `image` | The Kibana docker image | `docker.elastic.co/kibana/kibana` | --| `imageTag` | The Kibana docker image tag | `7.3.2` | -+| `imageTag` | The Kibana docker image tag | `7.4.0` | - | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | - | `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Kibana pods | `{}` | - | `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 500Mi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | -diff --git a/kibana/examples/default/test/goss.yaml b/kibana/examples/default/test/goss.yaml -index 760d38a..2bafc00 100644 ---- a/kibana/examples/default/test/goss.yaml -+++ b/kibana/examples/default/test/goss.yaml -@@ -3,7 +3,7 @@ http: - status: 200 - timeout: 2000 - body: -- - '"number":"7.3.2"' -+ - '"number":"7.4.0"' - - http://localhost:5601/app/kibana: - status: 200 -diff --git a/kibana/values.yaml b/kibana/values.yaml -index 1af52b8..4e3668c 100755 ---- a/kibana/values.yaml -+++ b/kibana/values.yaml -@@ -22,7 +22,7 @@ secretMounts: [] - # subPath: kibana.keystore # optional - - image: "docker.elastic.co/kibana/kibana" --imageTag: "7.3.2" -+imageTag: "7.4.0" - imagePullPolicy: "IfNotPresent" - - # additionals labels -diff --git a/metricbeat/Chart.yaml b/metricbeat/Chart.yaml -index 569d47c..d6f0f4b 100755 ---- a/metricbeat/Chart.yaml -+++ b/metricbeat/Chart.yaml -@@ -4,8 +4,8 @@ maintainers: - - email: helm-charts@elastic.co - name: Elastic - name: metricbeat --version: 7.3.2 --appVersion: 7.3.2 -+version: 7.4.0 -+appVersion: 7.4.0 - sources: - - https://github.com/elastic/beats - icon: https://helm.elastic.co/icons/metricbeat.png -diff --git a/metricbeat/README.md b/metricbeat/README.md -index 1e53f47..ecca6f4 100644 ---- a/metricbeat/README.md -+++ b/metricbeat/README.md -@@ -26,14 +26,14 @@ This chart is tested with the latest supported versions. The currently tested ve - - | 6.x | 7.x | - | ----- | ----- | --| 6.8.3 | 7.3.2 | -+| 6.8.3 | 7.4.0 | - - Examples of installing older major versions can be found in the [examples](./examples) directory. - --While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.3.2` of metricbeat it would look like this: -+While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.4.0` of metricbeat it would look like this: - - ``` --helm install --name metricbeat elastic/metricbeat --set imageTag=7.3.2 -+helm install --name metricbeat elastic/metricbeat --set imageTag=7.4.0 - ``` - - -@@ -46,7 +46,7 @@ helm install --name metricbeat elastic/metricbeat --set imageTag=7.3.2 - | `extraVolumes` | Templatable string of additional volumes to be passed to the `tpl` function | `""` | - | `hostPathRoot` | Fully-qualified [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) that will be used to persist Metricbeat registry data | `/var/lib` | - | `image` | The Metricbeat docker image | `docker.elastic.co/beats/metricbeat` | --| `imageTag` | The Metricbeat docker image tag | `7.3.2` | -+| `imageTag` | The Metricbeat docker image tag | `7.4.0` | - | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | - | `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | - | `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles. | `true` | -diff --git a/metricbeat/examples/default/test/goss-metrics.yaml b/metricbeat/examples/default/test/goss-metrics.yaml -index 3e0304d..b4e5163 100644 ---- a/metricbeat/examples/default/test/goss-metrics.yaml -+++ b/metricbeat/examples/default/test/goss-metrics.yaml -@@ -21,13 +21,13 @@ http: - status: 200 - timeout: 2000 - body: -- - 'metricbeat-7.3.2' -+ - 'metricbeat-7.4.0' - - 'http://elasticsearch-master:9200/_search?q=metricset.name:state_container%20AND%20kubernetes.container.name:metricbeat': - status: 200 - timeout: 2000 - body: -- - 'metricbeat-7.3.2' -+ - 'metricbeat-7.4.0' - - file: - /usr/share/metricbeat/metricbeat.yml: -@@ -42,4 +42,4 @@ command: - exit-status: 0 - stdout: - - 'elasticsearch: http://elasticsearch-master:9200' -- - 'version: 7.3.2' -+ - 'version: 7.4.0' -diff --git a/metricbeat/examples/default/test/goss.yaml b/metricbeat/examples/default/test/goss.yaml -index edb4e13..c1614ab 100644 ---- a/metricbeat/examples/default/test/goss.yaml -+++ b/metricbeat/examples/default/test/goss.yaml -@@ -29,12 +29,12 @@ http: - status: 200 - timeout: 2000 - body: -- - 'metricbeat-7.3.2' -+ - 'metricbeat-7.4.0' - 'http://elasticsearch-master:9200/_search?q=metricset.name:container%20AND%20kubernetes.container.name:metricbeat': - status: 200 - timeout: 2000 - body: -- - 'metricbeat-7.3.2' -+ - 'metricbeat-7.4.0' - - file: - /usr/share/metricbeat/metricbeat.yml: -@@ -49,4 +49,4 @@ command: - exit-status: 0 - stdout: - - 'elasticsearch: http://elasticsearch-master:9200' -- - 'version: 7.3.2' -+ - 'version: 7.4.0' -diff --git a/metricbeat/examples/oss/test/goss-metrics.yaml b/metricbeat/examples/oss/test/goss-metrics.yaml -index 947a71d..ea98e76 100644 ---- a/metricbeat/examples/oss/test/goss-metrics.yaml -+++ b/metricbeat/examples/oss/test/goss-metrics.yaml -@@ -21,12 +21,12 @@ http: - status: 200 - timeout: 2000 - body: -- - 'metricbeat-7.3.2' -+ - 'metricbeat-7.4.0' - http://oss-master:9200/_search?q=metricset.name:state_deployment: - status: 200 - timeout: 2000 - body: -- - 'metricbeat-7.3.2' -+ - 'metricbeat-7.4.0' - - file: - /usr/share/metricbeat/metricbeat.yml: -@@ -40,4 +40,4 @@ command: - exit-status: 0 - stdout: - - 'elasticsearch: http://oss-master:9200' -- - 'version: 7.3.2' -+ - 'version: 7.4.0' -diff --git a/metricbeat/examples/oss/test/goss.yaml b/metricbeat/examples/oss/test/goss.yaml -index d39d1b2..3b5c523 100644 ---- a/metricbeat/examples/oss/test/goss.yaml -+++ b/metricbeat/examples/oss/test/goss.yaml -@@ -29,12 +29,12 @@ http: - status: 200 - timeout: 2000 - body: -- - 'metricbeat-7.3.2' -+ - 'metricbeat-7.4.0' - http://oss-master:9200/_search?q=metricset.name:container: - status: 200 - timeout: 2000 - body: -- - 'metricbeat-7.3.2' -+ - 'metricbeat-7.4.0' - - file: - /usr/share/metricbeat/metricbeat.yml: -@@ -48,4 +48,4 @@ command: - exit-status: 0 - stdout: - - 'elasticsearch: http://oss-master:9200' -- - 'version: 7.3.2' -+ - 'version: 7.4.0' -diff --git a/metricbeat/examples/security/test/goss-metrics.yaml b/metricbeat/examples/security/test/goss-metrics.yaml -index 80d57a8..ecb788f 100644 ---- a/metricbeat/examples/security/test/goss-metrics.yaml -+++ b/metricbeat/examples/security/test/goss-metrics.yaml -@@ -21,7 +21,7 @@ http: - status: 200 - timeout: 2000 - body: -- - 'metricbeat-7.3.2' -+ - 'metricbeat-7.4.0' - allow-insecure: true - username: '{{ .Env.ELASTICSEARCH_USERNAME }}' - password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' -@@ -29,7 +29,7 @@ http: - status: 200 - timeout: 2000 - body: -- - 'metricbeat-7.3.2' -+ - 'metricbeat-7.4.0' - allow-insecure: true - username: '{{ .Env.ELASTICSEARCH_USERNAME }}' - password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' -@@ -46,4 +46,4 @@ command: - exit-status: 0 - stdout: - - 'elasticsearch: https://security-master:9200' -- - 'version: 7.3.2' -+ - 'version: 7.4.0' -diff --git a/metricbeat/examples/security/test/goss.yaml b/metricbeat/examples/security/test/goss.yaml -index d25b9de..0e2f04c 100644 ---- a/metricbeat/examples/security/test/goss.yaml -+++ b/metricbeat/examples/security/test/goss.yaml -@@ -29,7 +29,7 @@ http: - status: 200 - timeout: 2000 - body: -- - 'metricbeat-7.3.2' -+ - 'metricbeat-7.4.0' - allow-insecure: true - username: '{{ .Env.ELASTICSEARCH_USERNAME }}' - password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' -@@ -37,7 +37,7 @@ http: - status: 200 - timeout: 2000 - body: -- - 'metricbeat-7.3.2' -+ - 'metricbeat-7.4.0' - allow-insecure: true - username: '{{ .Env.ELASTICSEARCH_USERNAME }}' - password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' -@@ -54,4 +54,4 @@ command: - exit-status: 0 - stdout: - - 'elasticsearch: https://security-master:9200' -- - 'version: 7.3.2' -+ - 'version: 7.4.0' -diff --git a/metricbeat/values.yaml b/metricbeat/values.yaml -index 5c85249..3f9f0b1 100755 ---- a/metricbeat/values.yaml -+++ b/metricbeat/values.yaml -@@ -86,7 +86,7 @@ extraVolumes: "" - hostPathRoot: /var/lib - - image: "docker.elastic.co/beats/metricbeat" --imageTag: "7.3.2" -+imageTag: "7.4.0" - imagePullPolicy: "IfNotPresent" - imagePullSecrets: [] - --- -1.8.3.1 - diff --git a/monitor-helm-elastic/files/0006-readiness-probe-enhancements.patch b/monitor-helm-elastic/files/0005-readiness-probe-enhancements.patch similarity index 94% rename from monitor-helm-elastic/files/0006-readiness-probe-enhancements.patch rename to monitor-helm-elastic/files/0005-readiness-probe-enhancements.patch index 5a1f362..448b0af 100644 --- a/monitor-helm-elastic/files/0006-readiness-probe-enhancements.patch +++ b/monitor-helm-elastic/files/0005-readiness-probe-enhancements.patch @@ -12,7 +12,7 @@ diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/ index e17d39e..483e1f4 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml -@@ -194,7 +194,7 @@ spec: +@@ -202,7 +202,7 @@ spec: # If the node is starting up wait for the cluster to be ready (request params: '{{ .Values.clusterHealthCheckParams }}' ) # Once it has started only check that the node itself is responding START_FILE=/tmp/.es_start_file @@ -21,11 +21,11 @@ index e17d39e..483e1f4 100644 http () { local path="${1}" if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then -@@ -209,13 +209,45 @@ spec: - echo 'Elasticsearch is already running, lets check the node is healthy' - http "/" +@@ -217,13 +217,45 @@ spec: + echo 'Elasticsearch is already running, lets check the node is healthy and there are master nodes available' + http "/_cluster/health?timeout=0s" else -- echo 'Waiting for elasticsearch cluster to become cluster to be ready (request params: "{{ .Values.clusterHealthCheckParams }}" )' +- echo 'Waiting for elasticsearch cluster to become ready (request params: "{{ .Values.clusterHealthCheckParams }}" )' - if http "/_cluster/health?{{ .Values.clusterHealthCheckParams }}" ; then - touch ${START_FILE} - exit 0 @@ -77,7 +77,7 @@ diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 0d983eb..ebbae6c 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml -@@ -197,6 +197,8 @@ readinessProbe: +@@ -204,6 +204,8 @@ readinessProbe: # https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#request-params wait_for_status clusterHealthCheckParams: "wait_for_status=green&timeout=1s" diff --git a/monitor-helm/Readme.rst b/monitor-helm/Readme.rst index bc59ac9..e5b54f6 100644 --- a/monitor-helm/Readme.rst +++ b/monitor-helm/Readme.rst @@ -1,7 +1,4 @@ This repo is for -https://github.com/helm/charts/tree/master/stable/filebeat -https://github.com/helm/charts/tree/master/stable/metricbeat -https://github.com/helm/charts/tree/master/stable/kibana https://github.com/helm/charts/tree/master/stable/kube-state-metrics Changes to this repo are needed for StarlingX and those changes are diff --git a/monitor-helm/centos/monitor-helm.spec b/monitor-helm/centos/monitor-helm.spec index 8ea72ce..f1518de 100644 --- a/monitor-helm/centos/monitor-helm.spec +++ b/monitor-helm/centos/monitor-helm.spec @@ -86,12 +86,8 @@ helm repo add local http://localhost:8879/charts # Create the tgz files cd stable -make filebeat -make metricbeat make kube-state-metrics -make kibana make nginx-ingress -make logstash make elasticsearch-curator # terminate helm server (the last backgrounded task) diff --git a/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml b/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml index aa6cc43..def3b36 100644 --- a/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml +++ b/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml @@ -74,7 +74,7 @@ data: namespace: monitor release: elasticsearch-data source: - location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-7.4.0.tgz + location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-7.6.0.tgz reference: master subpath: elasticsearch type: tar @@ -104,7 +104,8 @@ data: elastic-data: enabled imagePullSecrets: [{"name": "default-registry-key"}] esMajorVersion: 7 - masterService: 'mon-elasticsearch-data-headless, mon-elasticsearch-master' + masterService: 'mon-elasticsearch-master' + seedHosts: "mon-elasticsearch-master-headless, mon-elasticsearch-data-headless" podManagementPolicy: OrderedReady clusterHealthCheckParams: 'wait_for_no_relocating_shards&wait_for_no_initializing_shards&timeout=1s' maxUnavailable: 1 @@ -136,7 +137,7 @@ data: namespace: monitor release: elasticsearch-client source: - location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-7.4.0.tgz + location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-7.6.0.tgz reference: master subpath: elasticsearch type: tar @@ -186,7 +187,8 @@ data: elastic-client: enabled imagePullSecrets: [{"name": "default-registry-key"}] esMajorVersion: 7 - masterService: 'mon-elasticsearch-data-headless, mon-elasticsearch-master' + masterService: 'mon-elasticsearch-master' + seedHosts: "mon-elasticsearch-master-headless, mon-elasticsearch-data-headless" podManagementPolicy: OrderedReady clusterHealthCheckParams: 'local=true' maxUnavailable: 1 @@ -211,7 +213,7 @@ data: namespace: monitor release: elasticsearch-master source: - location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-7.4.0.tgz + location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-7.6.0.tgz reference: master subpath: elasticsearch type: tar @@ -241,7 +243,8 @@ data: elastic-master: enabled imagePullSecrets: [{"name": "default-registry-key"}] esMajorVersion: 7 - masterService: 'mon-elasticsearch-data-headless, mon-elasticsearch-master' + masterService: 'mon-elasticsearch-master' + seedHosts: "mon-elasticsearch-master-headless, mon-elasticsearch-data-headless" podManagementPolicy: OrderedReady clusterHealthCheckParams: 'local=true' maxUnavailable: 1 @@ -362,7 +365,7 @@ data: - 'source /tmp/cur_limits/set_factor.sh; echo LIMIT_FACTOR=$LIMIT_FACTOR; CD_GB=$(printenv COLLECTD_INDEX_LIMIT_GB);MB_GB=$(printenv METRICBEAT_INDEX_LIMIT_GB);FB_GB=$(printenv FILEBEAT_INDEX_LIMIT_GB);echo export COLLECTD_INDEX_LIMIT_GB=$(($CD_GB / $LIMIT_FACTOR)) > /tmp/cur_limits/new_limits.sh; echo export METRICBEAT_INDEX_LIMIT_GB=$(($MB_GB / $LIMIT_FACTOR)) >> /tmp/cur_limits/new_limits.sh;echo export FILEBEAT_INDEX_LIMIT_GB=$(($FB_GB / $LIMIT_FACTOR)) >> /tmp/cur_limits/new_limits.sh; source /tmp/cur_limits/new_limits.sh; cat /tmp/cur_limits/new_limits.sh; /curator/curator --config /etc/es-curator/config.yml /etc/es-curator/action_file.yml' extraInitContainers: limitset: - image: docker.elastic.co/beats/filebeat-oss:7.4.0 + image: docker.elastic.co/beats/filebeat-oss:7.6.0 command: - 'sh' - '-c' @@ -416,41 +419,153 @@ data: release: mon-filebeat component: test values: - image: - repository: docker.elastic.co/beats/filebeat-oss - tag: 7.4.0 - pullSecrets: ['default-registry-key'] - monitoring: - enabled: true - config: - output.file: - enabled: false - output.logstash: - enabled: true - hosts: ["mon-logstash:5044"] - output.elasticsearch: - enabled: false - hosts: ["http://mon-elasticsearch-client:9200"] - ilm.pattern: "000001" - index: ${INDEX_NAME}-%{+yyyy.MM.dd} - setup.template: - name: ${INDEX_NAME} - pattern: ${INDEX_PATTERN} - setup.kibana: - # Note port 5601 is default - host: "mon-kibana" - setup.dashboards: - enabled: true - filebeat.autodiscover: - providers: - - type: kubernetes - host: ${NODE_NAME} - hints.enabled: true - # Mount the setup_script to main container for visibility - # even though not used there. + filebeatConfig: + filebeat.yml: + filebeat.inputs: + - type: log + enabled: true + paths: + - /var/log/*.log + - /var/log/messages + - /var/log/syslog + - type: docker + containers.ids: + - '*' + processors: + - add_kubernetes_metadata: + in_cluster: true + output.file: + enabled: false + output.logstash: + enabled: true + hosts: ["mon-logstash:5044"] + output.elasticsearch: + enabled: false + hosts: "http://mon-elasticsearch-client:9200" + ilm.pattern: "000001" + index: ${INDEX_NAME}-%{+yyyy.MM.dd} + setup.template: + name: ${INDEX_NAME} + pattern: ${INDEX_PATTERN} + setup.kibana: + # Note port 5601 is default + host: "mon-kibana" + setup.dashboard: + enabled: true + filebeat.autodiscover: + providers: + - type: kubernetes + host: ${NODE_NAME} + hints.enabled: true + setup-script.sh: | + #!/bin/bash + # + # This is best effort to load the template into elasticsearch + # if beats are going to elasticsearch via logstash. + # + BEAT='filebeat' + BEAT_VER=$($BEAT version | awk '{print $3}') + + ADDR=$OUTPUT_ELASTICSEARCH_HOSTS + ESPATH=$OUTPUT_ELASTICSEARCH_PATH + + echo $ADDR$ESPATH + + # Check if this is a map of entries. If so, we'll only export the template + # to the first address + if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then + # Remove the square brackets + ADDR=${ADDR:1:-1} + # Take the first entry in case there are more than one + ADDR=${ADDR%%,*} + fi + echo $ADDR$ESPATH + + # Check if user has formatted with http:// on front, if not we need to add + HTTP='http://' + if [[ ${ADDR} == http* ]]; then + HTTP='' + fi + echo $HTTP + + # Check for ip address containing special characters where -g must be used in curl command + # IPV6 Addresses should come in with square braces around the address. + CURL_G='' + if [[ ${ADDR} == *[* ]]; then + CURL_G=' -g ' + fi + echo $CURL_G + + # check if logstash output is enabled, via crude parsing of the .yml file. + sed -e '/output.logstash/,/enabled:/!d' files/${BEAT}.yml | grep -i true + if [[ $? -eq 0 ]]; then + echo "logstash configured, exporting template to elasticsearch" + $BEAT export template > /tmp/beat.template.json + # remove the lifecycle section of the yaml, as elasticsearch will choke + # on it as oss version does not support ilm. + sed -i '/lifecycle/,+3d' /tmp/beat.template.json + + # "unset" is the special system name when none has been set through + # overrides.. We key on that to know what template name and index pattern to set + if [[ $SYSTEM_NAME_FOR_INDEX != unset ]]; then + # replace the standard index pattern with one including our system name, + # which will match our created indices + sed -i "s/$BEAT-$BEAT_VER/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX/g" /tmp/beat.template.json + # give the template a name with the system name appended. + + # check if the template exists first before creating + curl -v $CURL_G -XGET -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX | grep "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX" + if [[ $? -ne 0 ]]; then + echo "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX template not found, creating..." + curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX -d@/tmp/beat.template.json + else + echo "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX template found, not creating..." + fi + else + # apply a higher order to this template in case there are templates with system names + # which should be applied first + curl -v $CURL_G -XGET -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER | grep "$BEAT-$BEAT_VER" + if [[ $? -ne 0 ]]; then + echo "$BEAT-$BEAT_VER template not found, creating..." + sed -i "s/\"order\": 1/\"order\": 2/g" /tmp/beat.template.json + curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json + else + echo "$BEAT-$BEAT_VER template found, not creating..." + fi + fi + else + echo "logstash not configured, not exporting template, should be done for us." + fi + # Above is non-fatal if there is a problem. Always pass. + exit 0 + extraInitContainers: + "setup-script": + container: + command: + - /bin/bash + - -c + - /usr/share/filebeat/files/setup-script.sh + volumeMounts: + - name: setup-script + mountPath: /usr/share/filebeat/files + - name: setup-script + mountPath: /usr/share/filebeat/filebeat.yml + subPath: filebeat.yml + extraVolumes: + - name: setup-script + configMap: + defaultMode: 0755 + name: mon-filebeat-config + - name: varlog + hostPath: + path: /var/log extraVolumeMounts: - - name: files - mountPath: /usr/share/filebeat/files + - name: varlog + mountPath: /var/log + readOnly: true + image: docker.elastic.co/beats/filebeat-oss + imageTag: 7.6.0 + imagePullSecrets: [{"name": "default-registry-key"}] tolerations: - key: "services" operator: "Equal" @@ -458,7 +573,7 @@ data: effect: "NoExecute" source: type: tar - location: http://172.17.0.1:8080/helm_charts/starlingx/filebeat-1.5.1.tgz + location: http://172.17.0.1:8080/helm_charts/starlingx/filebeat-7.6.0.tgz subpath: filebeat reference: master dependencies: [] @@ -491,13 +606,8 @@ data: release: mon-metricbeat component: test values: - image: - repository: docker.elastic.co/beats/metricbeat-oss - tag: 7.4.0 - serviceAccount: - imagePullSecrets: [{"name": "default-registry-key"}] - daemonset: - config: + metricbeatConfig: + metricbeat.yml: output.file: enabled: false output.logstash: @@ -506,7 +616,7 @@ data: output.elasticsearch: # this must be opposite of above output.logstash.enabled enabled: false - hosts: ["http://mon-elasticsearch-client:9200"] + hosts: "http://mon-elasticsearch-client:9200" ilm.pattern: "000001" index: ${INDEX_NAME}-%{+yyyy.MM.dd} setup.template: @@ -520,38 +630,29 @@ data: host: "mon-kibana" setup.dashboards: enabled: true - metricbeat.autodiscover: - providers: - - type: kubernetes - host: ${NODE_NAME} - hints.enabled: true - tolerations: - - key: "services" - operator: "Equal" - value: "disabled" - effect: "NoExecute" - deployment: - nodeSelector: - elastic-controller: "enabled" - config: + metricbeat.autodiscover: + providers: + - type: kubernetes + host: ${NODE_NAME} + hints.enabled: true + kube-state-metrics-metricbeat.yml: output.file: enabled: false output.logstash: enabled: true hosts: ["mon-logstash:5044"] output.elasticsearch: + # this must be opposite of above output.logstash.enabled enabled: false - hosts: ["http://mon-elasticsearch-client:9200"] + hosts: "http://mon-elasticsearch-client:9200" ilm.pattern: "000001" index: ${INDEX_NAME}-%{+yyyy.MM.dd} setup.template: name: ${INDEX_NAME} pattern: ${INDEX_PATTERN} setup.kibana: - # for onbox kibana, note port 5601 is default + # for on box kibana, note port 5601 is default host: "mon-kibana" - # for offbox kibana, will fail if we can't connect - # host: "128.224.140.223" setup.dashboards: enabled: true metricbeat.autodiscover: @@ -559,6 +660,100 @@ data: - type: kubernetes host: ${NODE_NAME} hints.enabled: true + setup-script.sh: | + #!/bin/bash + # + # This is best effort to load the template into elasticsearch + # if beats are going to elasticsearch via logstash. + # + BEAT='metricbeat' + BEAT_VER=$($BEAT version | awk '{print $3}') + + ADDR=$OUTPUT_ELASTICSEARCH_HOSTS + ESPATH=$OUTPUT_ELASTICSEARCH_PATH + + echo $ADDR$ESPATH + + # Check if this is a map of entries. If so, we'll only export the template + # to the first address + if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then + # Remove the square brackets + ADDR=${ADDR:1:-1} + # Take the first entry in case there are more than one + ADDR=${ADDR%%,*} + fi + echo $ADDR$ESPATH + + # Check if user has formatted with http:// on front, if not we need to add + HTTP='http://' + if [[ ${ADDR} == http* ]]; then + HTTP='' + fi + echo $HTTP + + # Check for ip address containing special characters where -g must be used in curl command + # IPV6 Addresses should come in with square braces around the address. + CURL_G='' + if [[ ${ADDR} == *[* ]]; then + CURL_G=' -g ' + fi + echo $CURL_G + + # check if logstash output is enabled, via crude parsing of the .yml file. + sed -e '/output.logstash/,/enabled:/!d' files/${BEAT}.yml | grep -i true + if [[ $? -eq 0 ]]; then + echo "logstash configured, exporting template to elasticsearch" + $BEAT export template > /tmp/beat.template.json + # remove the lifecycle section of the yaml, as elasticsearch will choke + # on it as oss version does not support ilm. + sed -i '/lifecycle/,+3d' /tmp/beat.template.json + + # "unset" is the special system name when none has been set through + # overrides.. We key on that to know what template name and index pattern to set + if [[ $SYSTEM_NAME_FOR_INDEX != unset ]]; then + # replace the standard index pattern with one including our system name, + # which will match our created indices + sed -i "s/$BEAT-$BEAT_VER/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX/g" /tmp/beat.template.json + # give the template a name with the system name appended. + + # check if the template exists first before creating + curl -v $CURL_G -XGET -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX | grep "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX" + if [[ $? -ne 0 ]]; then + echo "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX template not found, creating..." + curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX -d@/tmp/beat.template.json + else + echo "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX template found, not creating..." + fi + else + # apply a higher order to this template in case there are templates with system names + # which should be applied first + curl -v $CURL_G -XGET -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER | grep "$BEAT-$BEAT_VER" + if [[ $? -ne 0 ]]; then + echo "$BEAT-$BEAT_VER template not found, creating..." + sed -i "s/\"order\": 1/\"order\": 2/g" /tmp/beat.template.json + curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json + else + echo "$BEAT-$BEAT_VER template found, not creating..." + fi + fi + else + echo "logstash not configured, not exporting template, should be done for us." + fi + # Above is non-fatal if there is a problem. Always pass. + exit 0 + extraInitContainers: + "setup-script": + container: + command: + - /bin/bash + - -c + - /usr/share/metricbeat/files/setup-script.sh + volumeMounts: + - name: setup-script + mountPath: /usr/share/metricbeat/files + - name: setup-script + mountPath: /usr/share/metricbeat/metricbeat.yml + subPath: metricbeat.yml extraVolumeMounts: - name: root mountPath: /hostfs @@ -568,9 +763,51 @@ data: - name: root hostPath: path: / + - name: setup-script + configMap: + defaultMode: 0755 + name: mon-metricbeat-config + extraEnvs: + - name: KUBE_STATE_METRICS_HOSTS + value: http://mon-kube-state-metrics:8080 + image: docker.elastic.co/beats/metricbeat-oss + imageTag: 7.6.0 + imagePullPolicy: "IfNotPresent" + imagePullSecrets: [{"name": "default-registry-key"}] + clusterRoleRules: + - apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + verbs: ["get", "list", "watch"] + - apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - statefulsets + - deployments + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: + - nodes/stats + - nodes/metrics + verbs: ["get"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] + tolerations: + - key: "services" + operator: "Equal" + value: "disabled" + effect: "NoExecute" + nodeSelector: + elastic-controller: "enabled" source: type: tar - location: http://172.17.0.1:8080/helm_charts/starlingx/metricbeat-1.6.0.tgz + location: http://172.17.0.1:8080/helm_charts/starlingx/metricbeat-7.6.0.tgz subpath: metricbeat reference: master dependencies: [] @@ -652,10 +889,9 @@ data: release: mon-kibana component: test values: - image: - repository: docker.elastic.co/kibana/kibana-oss - tag: 7.4.0 - pullSecrets: [{'name': 'default-registry-key'}] + image: docker.elastic.co/kibana/kibana-oss + imageTag: 7.6.0 + imagePullSecrets: [{'name': 'default-registry-key'}] ingress: enabled: true annotations: @@ -665,15 +901,13 @@ data: nginx.ingress.kubernetes.io/force-ssl-redirect: "false" hosts: - "" - files: - kibana.yml: - elasticsearch.hosts: http://mon-elasticsearch-client:9200 - server.host: "::" + elasticsearchHosts: "http://mon-elasticsearch-client:9200" + serverHost: "::" nodeSelector: elastic-controller: "enabled" source: type: tar - location: http://172.17.0.1:8080/helm_charts/starlingx/kibana-2.2.0.tgz + location: http://172.17.0.1:8080/helm_charts/starlingx/kibana-7.6.0.tgz subpath: kibana reference: master dependencies: [] @@ -710,54 +944,8 @@ data: release: mon-logstash component: test values: - image: - repository: docker.elastic.co/logstash/logstash-oss - tag: 7.4.0 - pullSecrets: [{"name": "default-registry-key"}] - exporter: - logstash: - image: - repository: null - tag: null - nodeSelector: - elastic-controller: "enabled" - elasticsearch: - host: mon-elasticsearch-client - config: - elasticsearch.path: "" - queue.type: "memory" - persistence: - enabled: false - livenessProbe: - initialDelaySeconds: 90 - periodSeconds: 60 - timeoutSeconds: 60 - readinessProbe: - initialDelaySeconds: 90 - periodSeconds: 60 - timeoutSeconds: 60 - outputs: - main: |- - output { - # Note uncomment below and can see raw input in logs - #stdout { codec => rubydebug } - if [type] == "collectd" { - elasticsearch { - hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}${ELASTICSEARCH_PATH}"] - manage_template => false - index => "collectd${SYSTEM_NAME_FOR_INDEX}-%{+YYYY.MM.dd}" - } - } - if [type] == "beats" { - elasticsearch { - hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}${ELASTICSEARCH_PATH}"] - manage_template => false - index => "%{[@metadata][beat]}-%{[@metadata][version]}${SYSTEM_NAME_FOR_INDEX}-%{+YYYY.MM.dd}" - } - } - } - inputs: - main: |- + logstashPipeline: + logstash.conf: | input { udp { #Note default port 25826 in use by influxdb. @@ -773,8 +961,6 @@ data: type => 'beats' } } - filters: - main: |- filter { mutate { gsub => [ @@ -783,6 +969,40 @@ data: ] } } + output { + # Note uncomment below and can see raw input in logs + #stdout { codec => rubydebug } + if [type] == "collectd" { + elasticsearch { + hosts => ["${ELASTICSEARCH_HOST}"] + manage_template => false + index => "collectd${SYSTEM_NAME_FOR_INDEX}-%{+YYYY.MM.dd}" + } + } + if [type] == "beats" { + elasticsearch { + hosts => ["${ELASTICSEARCH_HOST}"] + manage_template => false + index => "%{[@metadata][beat]}-%{[@metadata][version]}${SYSTEM_NAME_FOR_INDEX}-%{+YYYY.MM.dd}" + } + } + } + image: docker.elastic.co/logstash/logstash-oss + imageTag: 7.6.0 + imagePullSecrets: [{"name": "default-registry-key"}] + elasticsearchHosts: "http://mon-elasticsearch-client:9200" + nodeSelector: + elastic-controller: "enabled" + persistence: + enabled: false + livenessProbe: + initialDelaySeconds: 90 + periodSeconds: 60 + timeoutSeconds: 60 + readinessProbe: + initialDelaySeconds: 90 + periodSeconds: 60 + timeoutSeconds: 60 ports: # influxdb port conflict, can't use 25826 # - containerPort: 25826 @@ -796,33 +1016,21 @@ data: protocol: TCP service: ports: - # collectd-udp: - # port: 25826 - # targetPort: collectd-udp - # protocol: UDP - collectd-udp: + - name: collectd-udp port: 31005 targetPort: collectd-udp protocol: UDP - monitor: + - name: monitor port: 9600 targetPort: monitor protocol: TCP - - ingress: - enabled: true - annotations: - kubernetes.io/ingress.class: "nginx" - nginx.ingress.kubernetes.io/rewrite-target: /$2 - nginx.ingress.kubernetes.io/ssl-redirect: "false" - nginx.ingress.kubernetes.io/force-ssl-redirect: "false" - path: /mon-logstash(/|$)(.*) - servicePort: 9600 - hosts: - - "" + - name: beats + port: 5044 + targetPort: beats + protocol: TCP source: type: tar - location: http://172.17.0.1:8080/helm_charts/starlingx/logstash-1.7.0.tgz + location: http://172.17.0.1:8080/helm_charts/starlingx/logstash-7.6.0.tgz subpath: logstash reference: master dependencies: [] @@ -917,9 +1125,9 @@ data: release_prefix: mon chart_groups: - nginx-ingress - - kibana - elasticsearch - elasticsearch-curator + - kibana - logstash - filebeat - metricbeat