From e9a196f787c6a823e75ea42c5292b25b7a452820 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Fri, 12 Oct 2018 15:48:17 +0000 Subject: [PATCH] Monasca specific config for Fluentd This commit adds some filters which format logs so that they can be correctly sent to the Monasca Log API by the Monasca Fluentd plugin. In the future the Fluentd plugin could be extended and this config could be removed. Partially-Implements: blueprint monasca-roles Change-Id: I87b6dfb3052d03f87349d30b66078c39d625195d --- .../conf/filter/00-record_transformer.conf.j2 | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 index 37ea4c19e6..a63fd5e73f 100644 --- a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 +++ b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 @@ -35,3 +35,39 @@ programname haproxy + +{% if enable_monasca | bool %} +# Kolla Fluentd input config configures Fluentd to save all log messages +# under the 'Payload' field, however the fluentd-monasca plugin assumes +# that log messages are saved under the 'message' field. Here we map the +# 'Payload' field to the 'message' field so that log can be forwarded to +# Monasca. + + @type record_transformer + enable_ruby true + + message ${record["Payload"]} + + remove_keys Payload + + +# Kolla configures Fluentd to extract timestamps from OpenStack service +# logs, however these timestamps are not saved in the event and are not +# forwarded to Monasca. Here we save the timestamp which has been +# *parsed* by Fluentd to a field which is part of the event and *is* +# therefore forwarded to Monasca. If no timestamp is parsed, then this +# should stamp the event with the current time. Note that since Kolla +# configures Fluentd to keep the time key, the original, *unparsed* +# timestamp, if present, will also be forwarded to Monasca as part of the +# event. However, because the logs which are collected by Fluentd use a +# variety of time formats the format of this timestamp is irregular and +# is therefore dropped in the Monasca log pipeline in favour of the +# timestamp added here. In the future we could investigate getting the +# Fluentd Monasca plugin to add this timestamp. + + @type record_transformer + + timestamp ${time} + + +{% endif %}