openstack/openstack-helm-infra
Code Issues Proposed changes

Fluent-logging: Support creation of arbitrary number of templates

Browse Source 
This updates the fluent-logging chart to support the creation of
an arbitrary number of templates for elasticsearch. This allows
for the definition of multiple index mappings driven via the
chart's values. This provides flexibility in determining specific
structures for indexes that may differ between log types.

This also moves to define these mappings via json instead of XML.
As gotpl can convert yaml directly to json, and elasticsearch can
ingest json directly for index creation, we no longer need an XML
helper function to generate the required configuration. This helps
reduce the number of helper functions we need to maintain

Change-Id: I3c85fb9a1e700eb1592d96f83e632172d0eb2681
This commit is contained in:
Steve Wilkerson 2018-06-01 14:26:59 -05:00
parent f9c186fc55
commit 4b8f46abee
6 changed files with 72 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
fluent-logging/templates/_helpers.tpl
View File

@ -108,38 +108,3 @@ section):
{{- end -}} {{- end -}}
{{- end -}} {{- end -}}
{{- end -}} {{- end -}}
# This function generates elasticsearch template files with entries in the
# fluent-logging values.yaml. It results in a configuration section with the
# following format (for as many key/value pairs defined in values for a section):
# {
# key: value
# key: {
# key: { ... }
# }
# }
# The configuration schema can be found here:
# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html
{{- define "fluent_logging.to_elasticsearch_template" -}}
{
{{- include "fluent_logging.recursive_tuple" . | indent 2 }}
}
{{- end }}
{{- define "fluent_logging.recursive_tuple" -}}
{{- range $key, $value := . -}}
,
{{- if or (kindIs "map" $value) }}
{{ $key | quote -}}:{
{{- include "fluent_logging.recursive_tuple" $value | indent 2 }}
}
{{- else }}
{{- if eq $key "index_patterns"}}
{{ $key | quote -}}: [{{ $value | quote }}]
{{- else }}
{{ $key | quote -}}:{{ $value | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

13
fluent-logging/templates/bin/_create_template.sh.tpl
View File

@ -2,13 +2,16 @@
set -ex set -ex
sed 's/ ,//' /tmp/template.xml.raw > /tmp/template.xml {{ range $template, $fields := .Values.conf.templates }}
result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/template_fluent_logging" \ -XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/{{$template}}" \
-H 'Content-Type: application/json' -d @/tmp/template.xml \ -H 'Content-Type: application/json' -d @/tmp/{{$template}}.json \
| python -c "import sys, json; print json.load(sys.stdin)['acknowledged']") | python -c "import sys, json; print json.load(sys.stdin)['acknowledged']")
if [ "$result" == "True" ]; then if [ "$result" == "True" ]; then
echo "template created!" echo "{{$template}} template created!"
else else
echo "template not created!" echo "{{$template}} template not created!"
fi fi
{{ end }}

20
fluent-logging/templates/bin/_helm-tests.sh.tpl
View File

@ -46,22 +46,24 @@ function check_kubernetes_tag () {
fi fi
} }
# Tests whether fluent-logging has successfully generate template_fluent_logging template # Tests whether fluent-logging has successfully generated the elasticsearch index mapping
# defined by value.yaml # templates defined by values.yaml
function check_template () { function check_templates () {
total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ {{ range $template, $fields := .Values.conf.templates }}
-XGET "${ELASTICSEARCH_ENDPOINT}/_template/template_fluent_logging" -H 'Content-Type: application/json' \ {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
| python -c "import sys, json; print len(json.load(sys.stdin))") | python -c "import sys, json; print len(json.load(sys.stdin))")
if [ "$total_hits" -gt 0 ]; then if [ "${{$template}}_total_hits" -gt 0 ]; then
echo "PASS: Successful hits on template, provided by fluent-logging!" echo "PASS: Successful hits on {{$template}} template, provided by fluent-logging!"
else else
echo "FAIL: No hits on query for template_fluent_logging template! Exiting"; echo "FAIL: No hits on query for {{$template}} template! Exiting";
exit 1; exit 1;
fi fi
{{ end }}
} }
# Sleep for at least the buffer flush time to allow for indices to be populated # Sleep for at least the buffer flush time to allow for indices to be populated
sleep 30 sleep 30
check_template check_templates
check_logstash_index check_logstash_index
check_kubernetes_tag check_kubernetes_tag

6
fluent-logging/templates/configmap-etc.yaml
View File

@ -28,6 +28,8 @@ data:
{{ include "fluent_logging.to_fluentbit_conf" .Values.conf.parsers | indent 4 }} {{ include "fluent_logging.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
td-agent.conf: | td-agent.conf: |
{{ include "fluent_logging.to_fluentd_conf" .Values.conf.td_agent | indent 4 }} {{ include "fluent_logging.to_fluentd_conf" .Values.conf.td_agent | indent 4 }}
template.xml.raw: | {{ range $template, $fields := .Values.conf.templates }}
{{ include "fluent_logging.to_elasticsearch_template" .Values.conf.template | indent 4 }} {{ $template }}.json: |
{{ toJson $fields | indent 4 }}
{{ end }}
{{- end }} {{- end }}

6
fluent-logging/templates/job-elasticsearch-template.yaml
View File

@ -65,10 +65,12 @@ spec:
mountPath: /tmp/create_template.sh mountPath: /tmp/create_template.sh
subPath: create_template.sh subPath: create_template.sh
readOnly: true readOnly: true
{{ range $template, $fields := .Values.conf.templates }}
- name: fluent-logging-etc - name: fluent-logging-etc
mountPath: /tmp/template.xml.raw mountPath: /tmp/{{$template}}.json
subPath: template.xml.raw subPath: {{$template}}.json
readOnly: true readOnly: true
{{ end }}
{{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 12 }}{{ end }} {{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 12 }}{{ end }}
volumes: volumes:
- name: fluent-logging-bin - name: fluent-logging-bin

89
fluent-logging/values.yaml
View File

@ -166,50 +166,51 @@ conf:
log: log:
format: "logger:stdout?json=true" format: "logger:stdout?json=true"
level: "info" level: "info"
template: templates:
template: "logstash-*" fluent:
index_patterns: "logstash-*" template: "logstash-*"
settings: index_patterns: "logstash-*"
number_of_shards: 1 settings:
mappings: number_of_shards: 1
fluent: mappings:
properties: fluent:
kubernetes: properties:
properties: kubernetes:
container_name: properties:
type: keyword container_name:
index: false type: keyword
docker_id: index: false
type: keyword docker_id:
index: false type: keyword
host: index: false
type: keyword host:
index: false type: keyword
labels: index: false
properties: labels:
app: properties:
type: keyword app:
index: false type: keyword
application: index: false
type: keyword application:
index: false type: keyword
component: index: false
type: keyword component:
index: false type: keyword
release_group: index: false
type: keyword release_group:
index: false type: keyword
namespace_name: index: false
type: keyword namespace_name:
index: false type: keyword
pod_id: index: false
type: keyword pod_id:
index: false type: keyword
pod_name: index: false
type: keyword pod_name:
index: false type: keyword
log: index: false
type: text log:
type: text
endpoints: endpoints:
cluster_domain_suffix: cluster.local cluster_domain_suffix: cluster.local