From efa0ac8a6ecfe0482f669cf5574687eaf34d41df Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Wed, 22 Jan 2020 09:56:22 -0500 Subject: [PATCH 1/1] script flexibility --- stable/filebeat/templates/configmap.yaml | 73 ++------------------------ stable/filebeat/templates/daemonset.yaml | 14 +++-- stable/filebeat/values.yaml | 84 ++++++++++++++++++++++++++++++ stable/logstash/templates/statefulset.yaml | 33 ++++++++++++ stable/logstash/values.yaml | 10 ++++ stable/metricbeat/templates/configmap.yaml | 73 ++------------------------ stable/metricbeat/templates/daemonset.yaml | 19 +++---- stable/metricbeat/values.yaml | 83 +++++++++++++++++++++++++++++ 8 files changed, 240 insertions(+), 149 deletions(-) diff --git a/stable/filebeat/templates/configmap.yaml b/stable/filebeat/templates/configmap.yaml index c08a4bc..340f2b9 100644 --- a/stable/filebeat/templates/configmap.yaml +++ b/stable/filebeat/templates/configmap.yaml @@ -1,77 +1,14 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "filebeat.fullname" . }} + name: {{ template "filebeat.fullname" . }}-files labels: app: {{ template "filebeat.fullname" . }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" data: - setup-script.sh: |- - #!/bin/bash - # - # This is best effort to load the template into elasticsearch - # if beats are going to elasticsearch via logstash. - # - BEAT='filebeat' - BEAT_VER=$($BEAT version | awk '{print $3}') - - ADDR=$OUTPUT_ELASTICSEARCH_HOSTS - ESPATH=$OUTPUT_ELASTICSEARCH_PATH - - echo $ADDR$ESPATH - - # Check if this is a map of entries. If so, we'll only export the template - # to the first address - if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then - # Remove the square brackets - ADDR=${ADDR:1:-1} - # Take the first entry in case there are more than one - ADDR=${ADDR%%,*} - fi - echo $ADDR$ESPATH - - # Check if user has formatted with http:// on front, if not we need to add - HTTP='http://' - if [[ ${ADDR} == http* ]]; then - HTTP='' - fi - echo $HTTP - - # Check for ip address containing special characters where -g must be used in curl command - # IPV6 Addresses should come in with square braces around the address. - CURL_G='' - if [[ ${ADDR} == *[* ]]; then - CURL_G=' -g ' - fi - echo $CURL_G - - # check if logstash output is enabled, via crude parsing of the .yml file. - sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true - if [[ $? -eq 0 ]]; then - echo "logstash configured, exporting template to elasticsearch" - $BEAT export template > /tmp/beat.template.json - # remove the lifecycle section of the yaml, as elasticsearch will choke - # on it as oss version does not support ilm. - sed -i '/lifecycle/,+3d' /tmp/beat.template.json - - # "unset" is the special system name when none has been set through - # overrides.. We key on that to know what template name and index pattern to set - if [[ $SYSTEM_NAME_FOR_INDEX != unset ]]; then - # replace the standard index pattern with one including our system name, - # which will match our created indices - sed -i "s/$BEAT-$BEAT_VER/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX/g" /tmp/beat.template.json - # give the template a name with the system name appended. - curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX -d@/tmp/beat.template.json - else - # apply a higher order to this template in case there are templates with system names - # which should be applied first - sed -i "s/\"order\": 1/\"order\": 2/g" /tmp/beat.template.json - curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json - fi - else - echo "logstash not configured, not exporting template, should be done for us." - fi - # Above is non-fatal if there is a problem. Always pass. - exit 0 +{{- range $key, $value := .Values.files }} + {{ $key }}: |- +{{ $value | indent 4 }} +{{- end }} diff --git a/stable/filebeat/templates/daemonset.yaml b/stable/filebeat/templates/daemonset.yaml index 2429892..ea76007 100644 --- a/stable/filebeat/templates/daemonset.yaml +++ b/stable/filebeat/templates/daemonset.yaml @@ -44,11 +44,10 @@ spec: command: - /bin/bash - -c - - /usr/share/filebeat/setup-script.sh + - /usr/share/filebeat/files/setup-script.sh volumeMounts: - - mountPath: /usr/share/filebeat/setup-script.sh - name: setupscript - subPath: setup-script.sh + - name: files + mountPath: /usr/share/filebeat/files - name: filebeat-config mountPath: /usr/share/filebeat/filebeat.yml readOnly: true @@ -90,6 +89,9 @@ spec: - name: INDEX_NAME value: "{{ .Values.indexNamePrefix }}" {{- end }} +{{- if .Values.extraVars }} +{{ toYaml .Values.extraVars | indent 8 }} +{{- end }} {{- if .Values.indexTemplateLoad }} - name: "load-es-template" image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -221,6 +223,10 @@ spec: hostPath: path: /var/lib/filebeat type: DirectoryOrCreate + - name: files + configMap: + name: {{ template "filebeat.fullname" . }}-files + defaultMode: 0755 {{- if .Values.extraVolumes }} {{ toYaml .Values.extraVolumes | indent 6 }} {{- end }} diff --git a/stable/filebeat/values.yaml b/stable/filebeat/values.yaml index 3897c82..2490242 100644 --- a/stable/filebeat/values.yaml +++ b/stable/filebeat/values.yaml @@ -90,6 +90,90 @@ extraInitContainers: [] # - echo # - hello +files: + setup-script.sh: |- + #!/bin/bash + # + # This is best effort to load the template into elasticsearch + # if beats are going to elasticsearch via logstash. + # + BEAT='filebeat' + BEAT_VER=$($BEAT version | awk '{print $3}') + + ADDR=$OUTPUT_ELASTICSEARCH_HOSTS + ESPATH=$OUTPUT_ELASTICSEARCH_PATH + + echo $ADDR$ESPATH + + # Check if this is a map of entries. If so, we'll only export the template + # to the first address + if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then + # Remove the square brackets + ADDR=${ADDR:1:-1} + # Take the first entry in case there are more than one + ADDR=${ADDR%%,*} + fi + echo $ADDR$ESPATH + + # Check if user has formatted with http:// on front, if not we need to add + HTTP='http://' + if [[ ${ADDR} == http* ]]; then + HTTP='' + fi + echo $HTTP + + # Check for ip address containing special characters where -g must be used in curl command + # IPV6 Addresses should come in with square braces around the address. + CURL_G='' + if [[ ${ADDR} == *[* ]]; then + CURL_G=' -g ' + fi + echo $CURL_G + + # check if logstash output is enabled, via crude parsing of the .yml file. + sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true + if [[ $? -eq 0 ]]; then + echo "logstash configured, exporting template to elasticsearch" + $BEAT export template > /tmp/beat.template.json + # remove the lifecycle section of the yaml, as elasticsearch will choke + # on it as oss version does not support ilm. + sed -i '/lifecycle/,+3d' /tmp/beat.template.json + + # "unset" is the special system name when none has been set through + # overrides.. We key on that to know what template name and index pattern to set + if [[ $SYSTEM_NAME_FOR_INDEX != unset ]]; then + # replace the standard index pattern with one including our system name, + # which will match our created indices + sed -i "s/$BEAT-$BEAT_VER/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX/g" /tmp/beat.template.json + # give the template a name with the system name appended. + + # check if the template exists first before creating + curl -v $CURL_G -XGET -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX | grep "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX" + if [[ $? -ne 0 ]]; then + echo "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX template not found, creating..." + curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX -d@/tmp/beat.template.json + else + echo "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX template found, not creating..." + fi + else + # apply a higher order to this template in case there are templates with system names + # which should be applied first + curl -v $CURL_G -XGET -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER | grep "$BEAT-$BEAT_VER" + if [[ $? -ne 0 ]]; then + echo "$BEAT-$BEAT_VER template not found, creating..." + sed -i "s/\"order\": 1/\"order\": 2/g" /tmp/beat.template.json + curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json + else + echo "$BEAT-$BEAT_VER template found, not creating..." + fi + fi + else + echo "logstash not configured, not exporting template, should be done for us." + fi + # Above is non-fatal if there is a problem. Always pass. + exit 0 + + resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little diff --git a/stable/logstash/templates/statefulset.yaml b/stable/logstash/templates/statefulset.yaml index 4749bb1..d34dcec 100644 --- a/stable/logstash/templates/statefulset.yaml +++ b/stable/logstash/templates/statefulset.yaml @@ -45,6 +45,36 @@ spec: {{- end }} hostNetwork: true dnsPolicy: ClusterFirstWithHostNet + initContainers: +{{- if .Values.extraInitContainers }} +{{ toYaml .Values.extraInitContainers | indent 8 }} + env: + ## Elasticsearch output + - name: ELASTICSEARCH_HOST + value: {{ .Values.elasticsearch.host | quote }} + - name: ELASTICSEARCH_PORT + value: {{ .Values.elasticsearch.port | quote }} + ## make system name visible for in setting index. + {{- if .Values.systemNameForIndex}} + {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" .Values.systemNameForIndex "" }} + {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} + - name: SYSTEM_NAME_FOR_INDEX + value: {{ printf "-%s" $systemNameFormatted | quote }} + {{- else }} + - name: SYSTEM_NAME_FOR_INDEX + value: "" + {{- end }} + ## Additional env vars + {{- range $key, $value := .Values.config }} + - name: {{ $key | upper | replace "." "_" }} + value: {{ $value | quote }} + {{- end }} + {{- if .Values.extraEnv }} +{{ .Values.extraEnv | toYaml | indent 12 }} + {{- end }} + volumeMounts: +{{ toYaml .Values.volumeMounts | indent 12 }} +{{- end }} containers: ## logstash @@ -86,6 +116,9 @@ spec: - name: {{ $key | upper | replace "." "_" }} value: {{ $value | quote }} {{- end }} + {{- if .Values.extraEnv }} +{{ .Values.extraEnv | toYaml | indent 12 }} + {{- end }} resources: {{ toYaml .Values.resources | indent 12 }} volumeMounts: diff --git a/stable/logstash/values.yaml b/stable/logstash/values.yaml index b8b8de2..2fdebde 100644 --- a/stable/logstash/values.yaml +++ b/stable/logstash/values.yaml @@ -167,6 +167,16 @@ volumes: [] # hostPath: # path: /tmp +extraEnv: [] + +extraInitContainers: [] + # - name: echo + # image: busybox + # imagePullPolicy: Always + # args: + # - echo + # - hello + exporter: logstash: env: {} diff --git a/stable/metricbeat/templates/configmap.yaml b/stable/metricbeat/templates/configmap.yaml index 9ae276d..8b281b4 100644 --- a/stable/metricbeat/templates/configmap.yaml +++ b/stable/metricbeat/templates/configmap.yaml @@ -1,77 +1,14 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "metricbeat.fullname" . }} + name: {{ template "metricbeat.fullname" . }}-files labels: app: {{ template "metricbeat.fullname" . }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" data: - setup-script.sh: |- - #!/bin/bash - # - # This is best effort to load the template into elasticsearch - # if beats are going to elasticsearch via logstash. - # - BEAT='metricbeat' - BEAT_VER=$($BEAT version | awk '{print $3}') - - ADDR=$OUTPUT_ELASTICSEARCH_HOSTS - ESPATH=$OUTPUT_ELASTICSEARCH_PATH - - echo $ADDR$ESPATH - - # Check if this is a map of entries. If so, we'll only export the template - # to the first address - if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then - # Remove the square brackets - ADDR=${ADDR:1:-1} - # Take the first entry in case there are more than one - ADDR=${ADDR%%,*} - fi - echo $ADDR$ESPATH - - # Check if user has formatted with http:// on front, if not we need to add - HTTP='http://' - if [[ ${ADDR} == http* ]]; then - HTTP='' - fi - echo $HTTP - - # Check for ip address containing special characters where -g must be used in curl command - # IPV6 Addresses should come in with square braces around the address. - CURL_G='' - if [[ ${ADDR} == *[* ]]; then - CURL_G=' -g ' - fi - echo $CURL_G - - # check if logstash output is enabled, via crude parsing of the .yml file. - sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true - if [[ $? -eq 0 ]]; then - echo "logstash configured, exporting template to elasticsearch" - $BEAT export template > /tmp/beat.template.json - # remove the lifecycle section of the yaml, as elasticsearch will choke - # on it as oss version does not support ilm. - sed -i '/lifecycle/,+3d' /tmp/beat.template.json - - # "unset" is the special system name when none has been set through - # overrides.. We key on that to know what template name and index pattern to set - if [[ $SYSTEM_NAME_FOR_INDEX != unset ]]; then - # replace the standard index pattern with one including our system name, - # which will match our created indices - sed -i "s/$BEAT-$BEAT_VER/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX/g" /tmp/beat.template.json - # give the template a name with the system name appended. - curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX -d@/tmp/beat.template.json - else - # apply a higher order to this template in case there are templates with system names - # which should be applied first - sed -i "s/\"order\": 1/\"order\": 2/g" /tmp/beat.template.json - curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json - fi - else - echo "logstash not configured, not exporting template, should be done for us." - fi - # Above is non-fatal if there is a problem. Always pass. - exit 0 +{{- range $key, $value := .Values.files }} + {{ $key }}: |- +{{ $value | indent 4 }} +{{- end }} diff --git a/stable/metricbeat/templates/daemonset.yaml b/stable/metricbeat/templates/daemonset.yaml index 247304e..581c120 100644 --- a/stable/metricbeat/templates/daemonset.yaml +++ b/stable/metricbeat/templates/daemonset.yaml @@ -39,11 +39,10 @@ spec: command: - /bin/bash - -c - - /usr/share/metricbeat/setup-script.sh + - /usr/share/metricbeat/files/setup-script.sh volumeMounts: - - mountPath: /usr/share/metricbeat/setup-script.sh - name: setupscript - subPath: setup-script.sh + - name: files + mountPath: /usr/share/metricbeat/files - name: config mountPath: /usr/share/metricbeat/metricbeat.yml readOnly: true @@ -85,6 +84,9 @@ spec: - name: INDEX_NAME value: "{{ .Values.indexNamePrefix }}" {{- end }} +{{- if .Values.extraEnv }} +{{ toYaml .Values.extraEnv | indent 8 }} +{{- end }} containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -168,9 +170,8 @@ spec: - name: dockersock mountPath: /var/run/docker.sock #Mount the script so we can see it from running containter - - mountPath: /usr/share/metricbeat/setup-script.sh - name: setupscript - subPath: setup-script.sh + - name: files + mountPath: /usr/share/metricbeat/files {{- if .Values.extraVolumeMounts }} {{ toYaml .Values.extraVolumeMounts | indent 8 }} {{- end }} @@ -194,9 +195,9 @@ spec: - name: dockersock hostPath: path: /var/run/docker.sock - - name: setupscript + - name: files configMap: - name: {{ template "metricbeat.fullname" . }} + name: {{ template "metricbeat.fullname" . }}-files defaultMode: 0755 {{- if .Values.extraVolumes }} {{ toYaml .Values.extraVolumes | indent 6 }} diff --git a/stable/metricbeat/values.yaml b/stable/metricbeat/values.yaml index b59c428..8f80bb6 100644 --- a/stable/metricbeat/values.yaml +++ b/stable/metricbeat/values.yaml @@ -144,6 +144,89 @@ extraVolumeMounts: [] # mountPath: /host/var/log # readOnly: true +files: + setup-script.sh: |- + #!/bin/bash + # + # This is best effort to load the template into elasticsearch + # if beats are going to elasticsearch via logstash. + # + BEAT='metricbeat' + BEAT_VER=$($BEAT version | awk '{print $3}') + + ADDR=$OUTPUT_ELASTICSEARCH_HOSTS + ESPATH=$OUTPUT_ELASTICSEARCH_PATH + + echo $ADDR$ESPATH + + # Check if this is a map of entries. If so, we'll only export the template + # to the first address + if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then + # Remove the square brackets + ADDR=${ADDR:1:-1} + # Take the first entry in case there are more than one + ADDR=${ADDR%%,*} + fi + echo $ADDR$ESPATH + + # Check if user has formatted with http:// on front, if not we need to add + HTTP='http://' + if [[ ${ADDR} == http* ]]; then + HTTP='' + fi + echo $HTTP + + # Check for ip address containing special characters where -g must be used in curl command + # IPV6 Addresses should come in with square braces around the address. + CURL_G='' + if [[ ${ADDR} == *[* ]]; then + CURL_G=' -g ' + fi + echo $CURL_G + + # check if logstash output is enabled, via crude parsing of the .yml file. + sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true + if [[ $? -eq 0 ]]; then + echo "logstash configured, exporting template to elasticsearch" + $BEAT export template > /tmp/beat.template.json + # remove the lifecycle section of the yaml, as elasticsearch will choke + # on it as oss version does not support ilm. + sed -i '/lifecycle/,+3d' /tmp/beat.template.json + + # "unset" is the special system name when none has been set through + # overrides.. We key on that to know what template name and index pattern to set + if [[ $SYSTEM_NAME_FOR_INDEX != unset ]]; then + # replace the standard index pattern with one including our system name, + # which will match our created indices + sed -i "s/$BEAT-$BEAT_VER/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX/g" /tmp/beat.template.json + # give the template a name with the system name appended. + + # check if the template exists first before creating + curl -v $CURL_G -XGET -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX | grep "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX" + if [[ $? -ne 0 ]]; then + echo "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX template not found, creating..." + curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX -d@/tmp/beat.template.json + else + echo "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX template found, not creating..." + fi + else + # apply a higher order to this template in case there are templates with system names + # which should be applied first + curl -v $CURL_G -XGET -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER | grep "$BEAT-$BEAT_VER" + if [[ $? -ne 0 ]]; then + echo "$BEAT-$BEAT_VER template not found, creating..." + sed -i "s/\"order\": 1/\"order\": 2/g" /tmp/beat.template.json + curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json + else + echo "$BEAT-$BEAT_VER template found, not creating..." + fi + fi + else + echo "logstash not configured, not exporting template, should be done for us." + fi + # Above is non-fatal if there is a problem. Always pass. + exit 0 + resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little -- 1.8.3.1