Fluent-logging: Support creation of arbitrary number of templates

This updates the fluent-logging chart to support the creation of
an arbitrary number of templates for elasticsearch. This allows
for the definition of multiple index mappings driven via the
chart's values. This provides flexibility in determining specific
structures for indexes that may differ between log types.

This also moves to define these mappings via json instead of XML.
As gotpl can convert yaml directly to json, and elasticsearch can
ingest json directly for index creation, we no longer need an XML
helper function to generate the required configuration. This helps
reduce the number of helper functions we need to maintain

Change-Id: I3c85fb9a1e700eb1592d96f83e632172d0eb2681

fluent-logging/templates/_helpers.tpl

@@ -108,38 +108,3 @@ section):
 {{- end -}}
 {{- end -}}
 {{- end -}}
-
-# This function generates elasticsearch template files with entries in the
-# fluent-logging values.yaml.  It results in a configuration section with the
-# following format (for as many key/value pairs defined in values for a section):
-# {
-#     key: value
-#     key: {
-#         key: { ... }
-#     }
-# }
-# The configuration schema can be found here:
-# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html
-
-{{- define "fluent_logging.to_elasticsearch_template" -}}
-{
-{{- include "fluent_logging.recursive_tuple" . | indent 2 }}
-}
-{{- end }}
-
-{{- define "fluent_logging.recursive_tuple" -}}
-{{- range $key, $value := . -}}
-,
-{{- if or (kindIs "map" $value) }}
-{{ $key | quote -}}:{
-{{- include "fluent_logging.recursive_tuple" $value | indent 2 }}
-}
-{{- else }}
-{{- if eq $key "index_patterns"}}
-{{ $key | quote -}}: [{{ $value | quote }}]
-{{- else }}
-{{ $key | quote -}}:{{ $value | quote }}
-{{- end }}
-{{- end }}
-{{- end }}
-{{- end }}

fluent-logging/templates/bin/_create_template.sh.tpl

@@ -2,13 +2,16 @@
 
 set -ex
 
-sed 's/ ,//' /tmp/template.xml.raw > /tmp/template.xml
+{{ range $template, $fields := .Values.conf.templates }}
+
 result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
--XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/template_fluent_logging" \
--H 'Content-Type: application/json' -d @/tmp/template.xml \
+-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/{{$template}}" \
+-H 'Content-Type: application/json' -d @/tmp/{{$template}}.json \
 | python -c "import sys, json; print json.load(sys.stdin)['acknowledged']")
 if [ "$result" == "True" ]; then
-   echo "template created!"
+   echo "{{$template}} template created!"
 else
-   echo "template not created!"
+   echo "{{$template}} template not created!"
 fi
+
+{{ end }}

fluent-logging/templates/bin/_helm-tests.sh.tpl

@@ -46,22 +46,24 @@ function check_kubernetes_tag () {
   fi
 }
 
-# Tests whether fluent-logging has successfully generate template_fluent_logging template
-# defined by value.yaml
-function check_template () {
-  total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-              -XGET "${ELASTICSEARCH_ENDPOINT}/_template/template_fluent_logging" -H 'Content-Type: application/json' \
+# Tests whether fluent-logging has successfully generated the elasticsearch index mapping
+# templates defined by values.yaml
+function check_templates () {
+  {{ range $template, $fields := .Values.conf.templates }}
+  {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+              -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
               | python -c "import sys, json; print len(json.load(sys.stdin))")
-  if [ "$total_hits" -gt 0 ]; then
-     echo "PASS: Successful hits on template, provided by fluent-logging!"
+  if [ "${{$template}}_total_hits" -gt 0 ]; then
+     echo "PASS: Successful hits on {{$template}} template, provided by fluent-logging!"
   else
-     echo "FAIL: No hits on query for template_fluent_logging template! Exiting";
+     echo "FAIL: No hits on query for {{$template}} template! Exiting";
      exit 1;
   fi
+  {{ end }}
 }
 
 # Sleep for at least the buffer flush time to allow for indices to be populated
 sleep 30
-check_template
+check_templates
 check_logstash_index
 check_kubernetes_tag

fluent-logging/templates/configmap-etc.yaml

@@ -28,6 +28,8 @@ data:
 {{ include "fluent_logging.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
   td-agent.conf: |
 {{ include "fluent_logging.to_fluentd_conf" .Values.conf.td_agent | indent 4 }}
-  template.xml.raw: |
-{{ include "fluent_logging.to_elasticsearch_template" .Values.conf.template | indent 4 }}
+{{ range $template, $fields := .Values.conf.templates }}
+  {{ $template }}.json: |
+{{ toJson $fields | indent 4 }}
+{{ end }}
 {{- end }}

fluent-logging/templates/job-elasticsearch-template.yaml

@@ -65,10 +65,12 @@ spec:
               mountPath: /tmp/create_template.sh
               subPath: create_template.sh
               readOnly: true
+            {{ range $template, $fields := .Values.conf.templates }}
             - name: fluent-logging-etc
-              mountPath: /tmp/template.xml.raw
-              subPath: template.xml.raw
+              mountPath: /tmp/{{$template}}.json
+              subPath: {{$template}}.json
               readOnly: true
+            {{ end }}
 {{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: fluent-logging-bin

fluent-logging/values.yaml

@@ -166,7 +166,8 @@ conf:
     log:
       format: "logger:stdout?json=true"
       level: "info"
-  template:
+  templates:
+    fluent:
       template: "logstash-*"
       index_patterns: "logstash-*"
       settings: