Fluent-logging: Support creation of arbitrary number of templates
This updates the fluent-logging chart to support the creation of an arbitrary number of templates for elasticsearch. This allows for the definition of multiple index mappings driven via the chart's values. This provides flexibility in determining specific structures for indexes that may differ between log types. This also moves to define these mappings via json instead of XML. As gotpl can convert yaml directly to json, and elasticsearch can ingest json directly for index creation, we no longer need an XML helper function to generate the required configuration. This helps reduce the number of helper functions we need to maintain Change-Id: I3c85fb9a1e700eb1592d96f83e632172d0eb2681
This commit is contained in:
Steve Wilkerson
parent
f9c186fc55
commit
4b8f46abee
@ -108,38 +108,3 @@ section):
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
# This function generates elasticsearch template files with entries in the
|
||||
# fluent-logging values.yaml. It results in a configuration section with the
|
||||
# following format (for as many key/value pairs defined in values for a section):
|
||||
# {
|
||||
# key: value
|
||||
# key: {
|
||||
# key: { ... }
|
||||
# }
|
||||
# }
|
||||
# The configuration schema can be found here:
|
||||
# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html
|
||||
|
||||
{{- define "fluent_logging.to_elasticsearch_template" -}}
|
||||
{
|
||||
{{- include "fluent_logging.recursive_tuple" . | indent 2 }}
|
||||
}
|
||||
{{- end }}
|
||||
|
||||
{{- define "fluent_logging.recursive_tuple" -}}
|
||||
{{- range $key, $value := . -}}
|
||||
,
|
||||
{{- if or (kindIs "map" $value) }}
|
||||
{{ $key | quote -}}:{
|
||||
{{- include "fluent_logging.recursive_tuple" $value | indent 2 }}
|
||||
}
|
||||
{{- else }}
|
||||
{{- if eq $key "index_patterns"}}
|
||||
{{ $key | quote -}}: [{{ $value | quote }}]
|
||||
{{- else }}
|
||||
{{ $key | quote -}}:{{ $value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@ -2,13 +2,16 @@
|
||||
|
||||
set -ex
|
||||
|
||||
sed 's/ ,//' /tmp/template.xml.raw > /tmp/template.xml
|
||||
{{ range $template, $fields := .Values.conf.templates }}
|
||||
|
||||
result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/template_fluent_logging" \
|
||||
-H 'Content-Type: application/json' -d @/tmp/template.xml \
|
||||
-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/{{$template}}" \
|
||||
-H 'Content-Type: application/json' -d @/tmp/{{$template}}.json \
|
||||
| python -c "import sys, json; print json.load(sys.stdin)['acknowledged']")
|
||||
if [ "$result" == "True" ]; then
|
||||
echo "template created!"
|
||||
echo "{{$template}} template created!"
|
||||
else
|
||||
echo "template not created!"
|
||||
echo "{{$template}} template not created!"
|
||||
fi
|
||||
|
||||
{{ end }}
|
||||
|
||||
@ -46,22 +46,24 @@ function check_kubernetes_tag () {
|
||||
fi
|
||||
}
|
||||
|
||||
# Tests whether fluent-logging has successfully generate template_fluent_logging template
|
||||
# defined by value.yaml
|
||||
function check_template () {
|
||||
total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-XGET "${ELASTICSEARCH_ENDPOINT}/_template/template_fluent_logging" -H 'Content-Type: application/json' \
|
||||
# Tests whether fluent-logging has successfully generated the elasticsearch index mapping
|
||||
# templates defined by values.yaml
|
||||
function check_templates () {
|
||||
{{ range $template, $fields := .Values.conf.templates }}
|
||||
{{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
|
||||
| python -c "import sys, json; print len(json.load(sys.stdin))")
|
||||
if [ "$total_hits" -gt 0 ]; then
|
||||
echo "PASS: Successful hits on template, provided by fluent-logging!"
|
||||
if [ "${{$template}}_total_hits" -gt 0 ]; then
|
||||
echo "PASS: Successful hits on {{$template}} template, provided by fluent-logging!"
|
||||
else
|
||||
echo "FAIL: No hits on query for template_fluent_logging template! Exiting";
|
||||
echo "FAIL: No hits on query for {{$template}} template! Exiting";
|
||||
exit 1;
|
||||
fi
|
||||
{{ end }}
|
||||
}
|
||||
|
||||
# Sleep for at least the buffer flush time to allow for indices to be populated
|
||||
sleep 30
|
||||
check_template
|
||||
check_templates
|
||||
check_logstash_index
|
||||
check_kubernetes_tag
|
||||
|
||||
@ -28,6 +28,8 @@ data:
|
||||
{{ include "fluent_logging.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
|
||||
td-agent.conf: |
|
||||
{{ include "fluent_logging.to_fluentd_conf" .Values.conf.td_agent | indent 4 }}
|
||||
template.xml.raw: |
|
||||
{{ include "fluent_logging.to_elasticsearch_template" .Values.conf.template | indent 4 }}
|
||||
{{ range $template, $fields := .Values.conf.templates }}
|
||||
{{ $template }}.json: |
|
||||
{{ toJson $fields | indent 4 }}
|
||||
{{ end }}
|
||||
{{- end }}
|
||||
|
||||
@ -65,10 +65,12 @@ spec:
|
||||
mountPath: /tmp/create_template.sh
|
||||
subPath: create_template.sh
|
||||
readOnly: true
|
||||
{{ range $template, $fields := .Values.conf.templates }}
|
||||
- name: fluent-logging-etc
|
||||
mountPath: /tmp/template.xml.raw
|
||||
subPath: template.xml.raw
|
||||
mountPath: /tmp/{{$template}}.json
|
||||
subPath: {{$template}}.json
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
{{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 12 }}{{ end }}
|
||||
volumes:
|
||||
- name: fluent-logging-bin
|
||||
|
||||
@ -166,50 +166,51 @@ conf:
|
||||
log:
|
||||
format: "logger:stdout?json=true"
|
||||
level: "info"
|
||||
template:
|
||||
template: "logstash-*"
|
||||
index_patterns: "logstash-*"
|
||||
settings:
|
||||
number_of_shards: 1
|
||||
mappings:
|
||||
fluent:
|
||||
properties:
|
||||
kubernetes:
|
||||
properties:
|
||||
container_name:
|
||||
type: keyword
|
||||
index: false
|
||||
docker_id:
|
||||
type: keyword
|
||||
index: false
|
||||
host:
|
||||
type: keyword
|
||||
index: false
|
||||
labels:
|
||||
properties:
|
||||
app:
|
||||
type: keyword
|
||||
index: false
|
||||
application:
|
||||
type: keyword
|
||||
index: false
|
||||
component:
|
||||
type: keyword
|
||||
index: false
|
||||
release_group:
|
||||
type: keyword
|
||||
index: false
|
||||
namespace_name:
|
||||
type: keyword
|
||||
index: false
|
||||
pod_id:
|
||||
type: keyword
|
||||
index: false
|
||||
pod_name:
|
||||
type: keyword
|
||||
index: false
|
||||
log:
|
||||
type: text
|
||||
templates:
|
||||
fluent:
|
||||
template: "logstash-*"
|
||||
index_patterns: "logstash-*"
|
||||
settings:
|
||||
number_of_shards: 1
|
||||
mappings:
|
||||
fluent:
|
||||
properties:
|
||||
kubernetes:
|
||||
properties:
|
||||
container_name:
|
||||
type: keyword
|
||||
index: false
|
||||
docker_id:
|
||||
type: keyword
|
||||
index: false
|
||||
host:
|
||||
type: keyword
|
||||
index: false
|
||||
labels:
|
||||
properties:
|
||||
app:
|
||||
type: keyword
|
||||
index: false
|
||||
application:
|
||||
type: keyword
|
||||
index: false
|
||||
component:
|
||||
type: keyword
|
||||
index: false
|
||||
release_group:
|
||||
type: keyword
|
||||
index: false
|
||||
namespace_name:
|
||||
type: keyword
|
||||
index: false
|
||||
pod_id:
|
||||
type: keyword
|
||||
index: false
|
||||
pod_name:
|
||||
type: keyword
|
||||
index: false
|
||||
log:
|
||||
type: text
|
||||
|
||||
endpoints:
|
||||
cluster_domain_suffix: cluster.local
|
||||
|
||||
Loading…
Reference in New Issue
Block a user