From 4b8f46abeee0e7afd0f127e0fad4ab8fe082a604 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Fri, 1 Jun 2018 14:26:59 -0500 Subject: [PATCH] Fluent-logging: Support creation of arbitrary number of templates This updates the fluent-logging chart to support the creation of an arbitrary number of templates for elasticsearch. This allows for the definition of multiple index mappings driven via the chart's values. This provides flexibility in determining specific structures for indexes that may differ between log types. This also moves to define these mappings via json instead of XML. As gotpl can convert yaml directly to json, and elasticsearch can ingest json directly for index creation, we no longer need an XML helper function to generate the required configuration. This helps reduce the number of helper functions we need to maintain Change-Id: I3c85fb9a1e700eb1592d96f83e632172d0eb2681 --- fluent-logging/templates/_helpers.tpl | 35 -------- .../templates/bin/_create_template.sh.tpl | 13 +-- .../templates/bin/_helm-tests.sh.tpl | 20 +++-- fluent-logging/templates/configmap-etc.yaml | 6 +- .../templates/job-elasticsearch-template.yaml | 6 +- fluent-logging/values.yaml | 89 ++++++++++--------- 6 files changed, 72 insertions(+), 97 deletions(-) diff --git a/fluent-logging/templates/_helpers.tpl b/fluent-logging/templates/_helpers.tpl index 672209051..6cbf26ca9 100644 --- a/fluent-logging/templates/_helpers.tpl +++ b/fluent-logging/templates/_helpers.tpl @@ -108,38 +108,3 @@ section): {{- end -}} {{- end -}} {{- end -}} - -# This function generates elasticsearch template files with entries in the -# fluent-logging values.yaml. It results in a configuration section with the -# following format (for as many key/value pairs defined in values for a section): -# { -# key: value -# key: { -# key: { ... } -# } -# } -# The configuration schema can be found here: -# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html - -{{- define "fluent_logging.to_elasticsearch_template" -}} -{ -{{- include "fluent_logging.recursive_tuple" . | indent 2 }} -} -{{- end }} - -{{- define "fluent_logging.recursive_tuple" -}} -{{- range $key, $value := . -}} -, -{{- if or (kindIs "map" $value) }} -{{ $key | quote -}}:{ -{{- include "fluent_logging.recursive_tuple" $value | indent 2 }} -} -{{- else }} -{{- if eq $key "index_patterns"}} -{{ $key | quote -}}: [{{ $value | quote }}] -{{- else }} -{{ $key | quote -}}:{{ $value | quote }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} diff --git a/fluent-logging/templates/bin/_create_template.sh.tpl b/fluent-logging/templates/bin/_create_template.sh.tpl index 6e9fd39fc..f071a2639 100644 --- a/fluent-logging/templates/bin/_create_template.sh.tpl +++ b/fluent-logging/templates/bin/_create_template.sh.tpl @@ -2,13 +2,16 @@ set -ex -sed 's/ ,//' /tmp/template.xml.raw > /tmp/template.xml +{{ range $template, $fields := .Values.conf.templates }} + result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ --XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/template_fluent_logging" \ --H 'Content-Type: application/json' -d @/tmp/template.xml \ +-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/{{$template}}" \ +-H 'Content-Type: application/json' -d @/tmp/{{$template}}.json \ | python -c "import sys, json; print json.load(sys.stdin)['acknowledged']") if [ "$result" == "True" ]; then - echo "template created!" + echo "{{$template}} template created!" else - echo "template not created!" + echo "{{$template}} template not created!" fi + +{{ end }} diff --git a/fluent-logging/templates/bin/_helm-tests.sh.tpl b/fluent-logging/templates/bin/_helm-tests.sh.tpl index af34ce524..e9875b5a1 100644 --- a/fluent-logging/templates/bin/_helm-tests.sh.tpl +++ b/fluent-logging/templates/bin/_helm-tests.sh.tpl @@ -46,22 +46,24 @@ function check_kubernetes_tag () { fi } -# Tests whether fluent-logging has successfully generate template_fluent_logging template -# defined by value.yaml -function check_template () { - total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ - -XGET "${ELASTICSEARCH_ENDPOINT}/_template/template_fluent_logging" -H 'Content-Type: application/json' \ +# Tests whether fluent-logging has successfully generated the elasticsearch index mapping +# templates defined by values.yaml +function check_templates () { + {{ range $template, $fields := .Values.conf.templates }} + {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \ | python -c "import sys, json; print len(json.load(sys.stdin))") - if [ "$total_hits" -gt 0 ]; then - echo "PASS: Successful hits on template, provided by fluent-logging!" + if [ "${{$template}}_total_hits" -gt 0 ]; then + echo "PASS: Successful hits on {{$template}} template, provided by fluent-logging!" else - echo "FAIL: No hits on query for template_fluent_logging template! Exiting"; + echo "FAIL: No hits on query for {{$template}} template! Exiting"; exit 1; fi + {{ end }} } # Sleep for at least the buffer flush time to allow for indices to be populated sleep 30 -check_template +check_templates check_logstash_index check_kubernetes_tag diff --git a/fluent-logging/templates/configmap-etc.yaml b/fluent-logging/templates/configmap-etc.yaml index a81a8371a..430de5f31 100644 --- a/fluent-logging/templates/configmap-etc.yaml +++ b/fluent-logging/templates/configmap-etc.yaml @@ -28,6 +28,8 @@ data: {{ include "fluent_logging.to_fluentbit_conf" .Values.conf.parsers | indent 4 }} td-agent.conf: | {{ include "fluent_logging.to_fluentd_conf" .Values.conf.td_agent | indent 4 }} - template.xml.raw: | -{{ include "fluent_logging.to_elasticsearch_template" .Values.conf.template | indent 4 }} +{{ range $template, $fields := .Values.conf.templates }} + {{ $template }}.json: | +{{ toJson $fields | indent 4 }} +{{ end }} {{- end }} diff --git a/fluent-logging/templates/job-elasticsearch-template.yaml b/fluent-logging/templates/job-elasticsearch-template.yaml index 0f9d58b98..e766163be 100644 --- a/fluent-logging/templates/job-elasticsearch-template.yaml +++ b/fluent-logging/templates/job-elasticsearch-template.yaml @@ -65,10 +65,12 @@ spec: mountPath: /tmp/create_template.sh subPath: create_template.sh readOnly: true + {{ range $template, $fields := .Values.conf.templates }} - name: fluent-logging-etc - mountPath: /tmp/template.xml.raw - subPath: template.xml.raw + mountPath: /tmp/{{$template}}.json + subPath: {{$template}}.json readOnly: true + {{ end }} {{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 12 }}{{ end }} volumes: - name: fluent-logging-bin diff --git a/fluent-logging/values.yaml b/fluent-logging/values.yaml index d6d81fc1e..5e0f2aed8 100644 --- a/fluent-logging/values.yaml +++ b/fluent-logging/values.yaml @@ -166,50 +166,51 @@ conf: log: format: "logger:stdout?json=true" level: "info" - template: - template: "logstash-*" - index_patterns: "logstash-*" - settings: - number_of_shards: 1 - mappings: - fluent: - properties: - kubernetes: - properties: - container_name: - type: keyword - index: false - docker_id: - type: keyword - index: false - host: - type: keyword - index: false - labels: - properties: - app: - type: keyword - index: false - application: - type: keyword - index: false - component: - type: keyword - index: false - release_group: - type: keyword - index: false - namespace_name: - type: keyword - index: false - pod_id: - type: keyword - index: false - pod_name: - type: keyword - index: false - log: - type: text + templates: + fluent: + template: "logstash-*" + index_patterns: "logstash-*" + settings: + number_of_shards: 1 + mappings: + fluent: + properties: + kubernetes: + properties: + container_name: + type: keyword + index: false + docker_id: + type: keyword + index: false + host: + type: keyword + index: false + labels: + properties: + app: + type: keyword + index: false + application: + type: keyword + index: false + component: + type: keyword + index: false + release_group: + type: keyword + index: false + namespace_name: + type: keyword + index: false + pod_id: + type: keyword + index: false + pod_name: + type: keyword + index: false + log: + type: text endpoints: cluster_domain_suffix: cluster.local