Fluent-logging: Remove utils for generating configuration files
This removes the utilities for generating the fluentd, fluentbit, and parser configuration files from yaml and moves to instead consume the configuration files as strings from the values.yaml. This allows for easier creation and maintenance of configuration files for fluentd and fluentbit, as the utilities became unwieldy with complex configuration files. This necessitated the removal of the core test executed by the charts helm tests, but this would be required as we move to split the charts regardless Change-Id: Ied4a76bbdf58b54a6d702db04a7120b64f54dcac Signed-off-by: Steve Wilkerson <sw5822@att.com>
This commit is contained in:
Steve Wilkerson
parent
85c03c422e
commit
b44d99b36a
@ -18,73 +18,6 @@ limitations under the License.
|
||||
|
||||
set -ex
|
||||
|
||||
# Test whether indexes have been created for each Elasticsearch output defined
|
||||
function check_output_indexes_exist () {
|
||||
{{/*
|
||||
First, determine the sum of Fluentbit and Fluentd's flush intervals. This
|
||||
ensures we wait long enough for recorded events to be indexed
|
||||
*/}}
|
||||
{{ $fluentBitConf := first .Values.conf.fluentbit }}
|
||||
{{ $fluentBitServiceConf := index $fluentBitConf "service" }}
|
||||
{{ $fluentBitFlush := index $fluentBitServiceConf "Flush" }}
|
||||
fluentBitFlush={{$fluentBitFlush}}
|
||||
|
||||
{{/*
|
||||
The generic Elasticsearch output should always be last, and intervals for all
|
||||
Elasticsearch outputs should match. This means we can safely use the last item
|
||||
in fluentd's configuration to get the Fluentd flush output interval
|
||||
*/}}
|
||||
{{- $fluentdConf := last .Values.conf.fluentd -}}
|
||||
{{- $fluentdElasticsearchConf := index $fluentdConf "elasticsearch" -}}
|
||||
{{- $fluentdFlush := index $fluentdElasticsearchConf "flush_interval" -}}
|
||||
fluentdFlush={{$fluentdFlush}}
|
||||
|
||||
totalFlush=$(($fluentBitFlush + $fluentdFlush))
|
||||
sleep $totalFlush
|
||||
|
||||
{{/*
|
||||
Iterate over Fluentd's config and for each Elasticsearch output, determine
|
||||
the logstash index prefix and check Elasticsearch for that index
|
||||
*/}}
|
||||
{{ range $key, $config := .Values.conf.td_agent -}}
|
||||
|
||||
{{/* Get list of keys to determine config header to index on */}}
|
||||
{{- $keyList := keys $config -}}
|
||||
{{- $configSection := first $keyList -}}
|
||||
|
||||
{{/* Index config section dictionary */}}
|
||||
{{- $configEntry := index $config $configSection -}}
|
||||
|
||||
{{- if hasKey $configEntry "type" -}}
|
||||
{{- $type := index $configEntry "type" -}}
|
||||
{{- if eq $type "elasticsearch" -}}
|
||||
{{- if hasKey $configEntry "logstash_prefix" -}}
|
||||
{{- $logstashPrefix := index $configEntry "logstash_prefix" }}
|
||||
{{$logstashPrefix}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-XGET "${ELASTICSEARCH_ENDPOINT}/{{$logstashPrefix}}-*/_search?pretty" -H 'Content-Type: application/json' \
|
||||
| python -c "import sys, json; print json.load(sys.stdin)['hits']['total']")
|
||||
if [ "${{$logstashPrefix}}_total_hits" -gt 0 ]; then
|
||||
echo "PASS: Successful hits on {{$logstashPrefix}}-* index!"
|
||||
else
|
||||
echo "FAIL: No hits on query for {{$logstashPrefix}}-* index! Exiting";
|
||||
exit 1;
|
||||
fi
|
||||
{{ else }}
|
||||
logstash_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-XGET "${ELASTICSEARCH_ENDPOINT}/logstash-*/_search?pretty" -H 'Content-Type: application/json' \
|
||||
| python -c "import sys, json; print json.load(sys.stdin)['hits']['total']")
|
||||
if [ "$logstash_total_hits" -gt 0 ]; then
|
||||
echo "PASS: Successful hits on logstash-* index!"
|
||||
else
|
||||
echo "FAIL: No hits on query for logstash-* index! Exiting";
|
||||
exit 1;
|
||||
fi
|
||||
{{ end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
}
|
||||
|
||||
{{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }}
|
||||
# Tests whether fluent-logging has successfully generated the elasticsearch index mapping
|
||||
# templates defined by values.yaml
|
||||
@ -106,4 +39,3 @@ function check_templates () {
|
||||
{{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }}
|
||||
check_templates
|
||||
{{ end }}
|
||||
check_output_indexes_exist
|
||||
|
||||
@ -23,9 +23,9 @@ metadata:
|
||||
name: fluent-logging-etc
|
||||
type: Opaque
|
||||
data:
|
||||
fluent-bit.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.fluentbit | b64enc }}
|
||||
parsers.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.parsers | b64enc }}
|
||||
fluent.conf: {{ include "fluent_logging.utils.to_fluentd_conf" .Values.conf.fluentd | b64enc }}
|
||||
fluent-bit.conf: {{ .Values.conf.fluentbit.template | b64enc }}
|
||||
parsers.conf: {{ .Values.conf.parsers.template | b64enc }}
|
||||
fluent.conf: {{ .Values.conf.fluentd.template | b64enc }}
|
||||
{{ range $template, $fields := .Values.conf.templates }}
|
||||
{{ $template }}.json: {{ toJson $fields | b64enc }}
|
||||
{{ end }}
|
||||
|
||||
@ -1,44 +0,0 @@
|
||||
{{/*
|
||||
Copyright 2017 The Openstack-Helm Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/}}
|
||||
|
||||
# This function generates fluentbit configuration files with entries in the
|
||||
# fluent-logging values.yaml. It results in a configuration section with the
|
||||
# following format (for as many key/value pairs defined in values for a section):
|
||||
# [HEADER]
|
||||
# key value
|
||||
# key value
|
||||
# key value
|
||||
# The configuration schema can be found here:
|
||||
# http://fluentbit.io/documentation/0.12/configuration/schema.html
|
||||
|
||||
{{- define "fluent_logging.utils.to_fluentbit_conf" -}}
|
||||
{{- range $values := . -}}
|
||||
{{- range $section := . -}}
|
||||
{{- $header := pick . "header" -}}
|
||||
{{- $config := omit . "header" }}
|
||||
[{{$header.header | upper }}]
|
||||
{{range $key, $value := $config -}}
|
||||
{{ if eq $key "Rename" }}
|
||||
{{- range $original, $new := $value -}}
|
||||
{{ printf "Rename %s %s" $original $new | indent 4 }}
|
||||
{{end}}
|
||||
{{- else -}}
|
||||
{{ $key | indent 4 }} {{ $value }}
|
||||
{{end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@ -1,90 +0,0 @@
|
||||
{{/*
|
||||
Copyright 2017 The Openstack-Helm Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/}}
|
||||
|
||||
# This function generates fluentd configuration files with entries in the
|
||||
# fluent-logging values.yaml. It results in a configuration section with either
|
||||
# of the following formats (for as many key/value pairs defined in values for a
|
||||
section):
|
||||
# <HEADER>
|
||||
# key value
|
||||
# key value
|
||||
# key value
|
||||
# </HEADER>
|
||||
# or
|
||||
# <HEADER>
|
||||
# key value
|
||||
# <INNER_HEADER>
|
||||
# key value
|
||||
# </INNER_HEADER>
|
||||
# </HEADER>
|
||||
# The configuration schema can be found here:
|
||||
# https://docs.fluentd.org/v0.12/articles/config-file
|
||||
|
||||
{{- define "fluent_logging.utils.to_fluentd_conf" -}}
|
||||
{{- range $values := . -}}
|
||||
{{- range $section := . -}}
|
||||
{{- $header := pick . "header" -}}
|
||||
{{- $config := omit . "header" "expression" -}}
|
||||
{{- if hasKey . "expression" -}}
|
||||
{{ $regex := pick . "expression" }}
|
||||
{{ printf "<%s %s>" $header.header $regex.expression }}
|
||||
{{- else }}
|
||||
{{ printf "<%s>" $header.header }}
|
||||
{{- end }}
|
||||
{{- range $key, $value := $config -}}
|
||||
{{- if kindIs "slice" $value }}
|
||||
{{- range $value := . -}}
|
||||
{{- range $innerSection := . -}}
|
||||
{{- $innerHeader := pick . "header" -}}
|
||||
{{- $innerConfig := omit . "header" "expression" -}}
|
||||
{{- if hasKey . "expression" -}}
|
||||
{{ $innerRegex := pick . "expression" }}
|
||||
{{ printf "<%s %s>" $innerHeader.header $innerRegex.expression | indent 2 }}
|
||||
{{- else }}
|
||||
{{ printf "<%s>" $innerHeader.header | indent 2 }}
|
||||
{{- end }}
|
||||
{{- range $innerKey, $innerValue := $innerConfig -}}
|
||||
{{- if eq $innerKey "type" -}}
|
||||
{{ $type := list "@" "type" | join "" }}
|
||||
{{ $type | indent 4 }} {{ $innerValue }}
|
||||
{{- else if contains "ENV" ($innerValue | quote) }}
|
||||
{{ $innerKey | indent 4 }} {{ $innerValue | quote }}
|
||||
{{- else if eq $innerKey "flush_interval" }}
|
||||
{{ $innerKey | indent 4 }} {{ printf "%ss" $innerValue }}
|
||||
{{- else }}
|
||||
{{ $innerKey | indent 4 }} {{ $innerValue }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{ printf "</%s>" $innerHeader.header | indent 2 }}
|
||||
{{- end -}}
|
||||
{{ end -}}
|
||||
{{- else }}
|
||||
{{- if eq $key "type" -}}
|
||||
{{ $type := list "@" "type" | join "" }}
|
||||
{{ $type | indent 2 }} {{ $value }}
|
||||
{{- else if contains "ENV" ($value | quote) }}
|
||||
{{ $key | indent 2 }} {{ $value | quote }}
|
||||
{{- else if eq $key "flush_interval" }}
|
||||
{{ $key | indent 2 }} {{ printf "%ss" $value }}
|
||||
{{- else }}
|
||||
{{ $key | indent 2 }} {{ $value }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{ printf "</%s>" $header.header }}
|
||||
{{- end }}
|
||||
{{ end -}}
|
||||
{{- end -}}
|
||||
@ -103,347 +103,340 @@ dependencies:
|
||||
service: fluentd
|
||||
|
||||
conf:
|
||||
fluentbit:
|
||||
- service:
|
||||
header: service
|
||||
Flush: 30
|
||||
Daemon: Off
|
||||
Log_Level: info
|
||||
Parsers_File: parsers.conf
|
||||
- kernel_messages:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: kernel
|
||||
Path: /var/log/kern.log
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- containers_tail:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: kube.*
|
||||
Path: /var/log/containers/*.log
|
||||
Parser: docker
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- libvirt:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: libvirt
|
||||
Path: /var/log/libvirt/libvirtd.log
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- qemu:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: qemu
|
||||
Path: /var/log/libvirt/qemu/*.log
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- kubelet:
|
||||
header: input
|
||||
Name: systemd
|
||||
Tag: journal.*
|
||||
Path: ${JOURNAL_PATH}
|
||||
Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- docker_daemon:
|
||||
header: input
|
||||
Name: systemd
|
||||
Tag: journal.*
|
||||
Path: ${JOURNAL_PATH}
|
||||
Systemd_Filter: _SYSTEMD_UNIT=docker.service
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- throttle_filter:
|
||||
header: filter
|
||||
Name: throttle
|
||||
Match: "**"
|
||||
Rate: 1000
|
||||
Window: 300
|
||||
Interval: 1s
|
||||
- libvirt_record_modifier:
|
||||
header: filter
|
||||
Name: record_modifier
|
||||
Match: libvirt
|
||||
Record: hostname ${HOSTNAME}
|
||||
- qemu_record_modifier:
|
||||
header: filter
|
||||
Name: record_modifier
|
||||
Match: qemu
|
||||
Record: hostname ${HOSTNAME}
|
||||
- kernel_record_modifier:
|
||||
header: filter
|
||||
Name: record_modifier
|
||||
Match: kernel
|
||||
Record: hostname ${HOSTNAME}
|
||||
- systemd_modify_fields:
|
||||
header: filter
|
||||
Name: modify
|
||||
Match: journal.**
|
||||
Rename:
|
||||
_BOOT_ID: BOOT_ID
|
||||
_CAP_EFFECTIVE: CAP_EFFECTIVE
|
||||
_CMDLINE: CMDLINE
|
||||
_COMM: COMM
|
||||
_EXE: EXE
|
||||
_GID: GID
|
||||
_HOSTNAME: HOSTNAME
|
||||
_MACHINE_ID: MACHINE_ID
|
||||
_PID: PID
|
||||
_SYSTEMD_CGROUP: SYSTEMD_CGROUP
|
||||
_SYSTEMD_SLICE: SYSTEMD_SLICE
|
||||
_SYSTEMD_UNIT: SYSTEMD_UNIT
|
||||
_UID: UID
|
||||
_TRANSPORT: TRANSPORT
|
||||
- drop_fluentd_logs:
|
||||
header: output
|
||||
Name: "null"
|
||||
Match: "**.fluentd**"
|
||||
- kube_filter:
|
||||
header: filter
|
||||
Name: kubernetes
|
||||
Match: kube.*
|
||||
Merge_JSON_Log: On
|
||||
- fluentd_output:
|
||||
header: output
|
||||
Name: forward
|
||||
Match: "*"
|
||||
Host: ${FLUENTD_HOST}
|
||||
Port: ${FLUENTD_PORT}
|
||||
parsers:
|
||||
- docker:
|
||||
header: parser
|
||||
Name: docker
|
||||
Format: json
|
||||
Time_Key: time
|
||||
Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
|
||||
Time_Keep: On
|
||||
Decode_Field_As: escaped_utf8 log
|
||||
fluentd:
|
||||
- metrics_agent:
|
||||
header: source
|
||||
type: monitor_agent
|
||||
bind: 0.0.0.0
|
||||
port: 24220
|
||||
- fluentbit_forward:
|
||||
header: source
|
||||
type: forward
|
||||
port: "#{ENV['FLUENTD_PORT']}"
|
||||
bind: 0.0.0.0
|
||||
- filter_fluentd_logs:
|
||||
header: match
|
||||
expression: "fluent.**"
|
||||
type: "null"
|
||||
# NOTE(srwilkers): Look for specific keywords in the log key to determine
|
||||
# log level of event
|
||||
- tag_kubernetes_log_level:
|
||||
header: match
|
||||
type: rewrite_tag_filter
|
||||
expression: "kube.var.log.containers.**.log"
|
||||
rule:
|
||||
-
|
||||
- header: rule
|
||||
key: log
|
||||
pattern: /info/i
|
||||
tag: info.${tag}
|
||||
- header: rule
|
||||
key: log
|
||||
pattern: /warn/i
|
||||
tag: warn.${tag}
|
||||
- header: rule
|
||||
key: log
|
||||
pattern: /error/i
|
||||
tag: error.${tag}
|
||||
- header: rule
|
||||
key: log
|
||||
pattern: /critical/i
|
||||
tag: critical.${tag}
|
||||
- header: rule
|
||||
key: log
|
||||
pattern: (.+)
|
||||
tag: info.${tag}
|
||||
# NOTE(srwilkers): Create new key for log level, and use the tag prefix
|
||||
# added previously
|
||||
- add_kubernetes_log_level_and_application_key:
|
||||
header: filter
|
||||
type: record_transformer
|
||||
enable_ruby: true
|
||||
expression: "**.kube.var.log.containers.**.log"
|
||||
record:
|
||||
-
|
||||
- header: record
|
||||
level: ${tag_parts[0]}
|
||||
application: ${record["kubernetes"]["labels"]["application"]}
|
||||
- add_openstack_application_key:
|
||||
header: filter
|
||||
type: record_transformer
|
||||
expression: "openstack.**"
|
||||
record:
|
||||
-
|
||||
- header: record
|
||||
application: ${tag_parts[1]}
|
||||
#NOTE(srwilkers): This prefixes the tag for oslo.log entries from the
|
||||
# fluent handler/formatter with the log level, allowing for lookups on
|
||||
# openstack logs with a particular log level (ie: error.openstack.keystone)
|
||||
- tag_openstack_log_level:
|
||||
header: match
|
||||
type: rewrite_tag_filter
|
||||
expression: "openstack.**"
|
||||
rule:
|
||||
-
|
||||
- header: rule
|
||||
key: level
|
||||
pattern: INFO
|
||||
tag: info.${tag}
|
||||
- header: rule
|
||||
key: level
|
||||
pattern: WARN
|
||||
tag: warn.${tag}
|
||||
- header: rule
|
||||
key: level
|
||||
pattern: ERROR
|
||||
tag: error.${tag}
|
||||
- header: rule
|
||||
key: level
|
||||
pattern: CRITICAL
|
||||
tag: critical.${tag}
|
||||
#NOTE(tp6510): This prefixes the tag for auth entries
|
||||
# it allows for lookups on openstack logs with
|
||||
# a particular auth log (ie: auth.openstack.keystone)
|
||||
- tag_auth_log:
|
||||
header: match
|
||||
type: rewrite_tag_filter
|
||||
expression: "*.openstack.**"
|
||||
rule:
|
||||
-
|
||||
- header: rule
|
||||
key: application
|
||||
pattern: keystone
|
||||
tag: auth.${tag}
|
||||
- header: rule
|
||||
key: application
|
||||
pattern: horizon
|
||||
tag: auth.${tag}
|
||||
- header: rule
|
||||
key: application
|
||||
pattern: mariadb
|
||||
tag: auth.${tag}
|
||||
- header: rule
|
||||
key: application
|
||||
pattern: memcached
|
||||
tag: auth.${tag}
|
||||
- header: rule
|
||||
key: application
|
||||
pattern: rabbitmq
|
||||
tag: auth.${tag}
|
||||
- libvirt_elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "libvirt"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
logstash_prefix: libvirt
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
- qemu_elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "qemu"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
logstash_prefix: qemu
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
- journal_elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "journal.**"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
logstash_prefix: journal
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
- kernel_elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "kernel"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
logstash_prefix: kernel
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
- elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "**"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
type_name: fluent
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
flush_interval: "15"
|
||||
template: |
|
||||
<source>
|
||||
bind 0.0.0.0
|
||||
port 24220
|
||||
@type monitor_agent
|
||||
</source>
|
||||
|
||||
<source>
|
||||
bind 0.0.0.0
|
||||
port "#{ENV['FLUENTD_PORT']}"
|
||||
@type forward
|
||||
</source>
|
||||
|
||||
<match fluent.**>
|
||||
@type null
|
||||
</match>
|
||||
|
||||
<match kube.var.log.containers.**.log>
|
||||
<rule>
|
||||
key log
|
||||
pattern /info/i
|
||||
tag info.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key log
|
||||
pattern /warn/i
|
||||
tag warn.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key log
|
||||
pattern /error/i
|
||||
tag error.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key log
|
||||
pattern /critical/i
|
||||
tag critical.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key log
|
||||
pattern (.+)
|
||||
tag info.${tag}
|
||||
</rule>
|
||||
@type rewrite_tag_filter
|
||||
</match>
|
||||
|
||||
<filter **.kube.var.log.containers.**.log>
|
||||
enable_ruby true
|
||||
<record>
|
||||
application ${record["kubernetes"]["labels"]["application"]}
|
||||
level ${tag_parts[0]}
|
||||
</record>
|
||||
@type record_transformer
|
||||
</filter>
|
||||
|
||||
<filter openstack.**>
|
||||
<record>
|
||||
application ${tag_parts[1]}
|
||||
</record>
|
||||
@type record_transformer
|
||||
</filter>
|
||||
|
||||
<match openstack.**>
|
||||
<rule>
|
||||
key level
|
||||
pattern INFO
|
||||
tag info.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key level
|
||||
pattern WARN
|
||||
tag warn.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key level
|
||||
pattern ERROR
|
||||
tag error.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key level
|
||||
pattern CRITICAL
|
||||
tag critical.${tag}
|
||||
</rule>
|
||||
@type rewrite_tag_filter
|
||||
</match>
|
||||
|
||||
<match *.openstack.**>
|
||||
<rule>
|
||||
key application
|
||||
pattern keystone
|
||||
tag auth.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key application
|
||||
pattern horizon
|
||||
tag auth.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key application
|
||||
pattern mariadb
|
||||
tag auth.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key application
|
||||
pattern memcached
|
||||
tag auth.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key application
|
||||
pattern rabbitmq
|
||||
tag auth.${tag}
|
||||
</rule>
|
||||
@type rewrite_tag_filter
|
||||
</match>
|
||||
|
||||
<match libvirt>
|
||||
<buffer>
|
||||
chunk_limit_size 8MB
|
||||
flush_interval 15s
|
||||
flush_thread_count 8
|
||||
queue_limit_length 256
|
||||
retry_forever false
|
||||
retry_max_interval 30
|
||||
</buffer>
|
||||
host "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
include_tag_key true
|
||||
logstash_format true
|
||||
logstash_prefix libvirt
|
||||
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
port "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
@type elasticsearch
|
||||
user "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
</match>
|
||||
|
||||
<match qemu>
|
||||
<buffer>
|
||||
chunk_limit_size 8MB
|
||||
flush_interval 15s
|
||||
flush_thread_count 8
|
||||
queue_limit_length 256
|
||||
retry_forever false
|
||||
retry_max_interval 30
|
||||
</buffer>
|
||||
host "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
include_tag_key true
|
||||
logstash_format true
|
||||
logstash_prefix qemu
|
||||
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
port "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
@type elasticsearch
|
||||
user "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
</match>
|
||||
|
||||
<match journal.**>
|
||||
<buffer>
|
||||
chunk_limit_size 8MB
|
||||
flush_interval 15s
|
||||
flush_thread_count 8
|
||||
queue_limit_length 256
|
||||
retry_forever false
|
||||
retry_max_interval 30
|
||||
</buffer>
|
||||
host "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
include_tag_key true
|
||||
logstash_format true
|
||||
logstash_prefix journal
|
||||
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
port "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
@type elasticsearch
|
||||
user "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
</match>
|
||||
|
||||
<match kernel>
|
||||
<buffer>
|
||||
chunk_limit_size 8MB
|
||||
flush_interval 15s
|
||||
flush_thread_count 8
|
||||
queue_limit_length 256
|
||||
retry_forever false
|
||||
retry_max_interval 30
|
||||
</buffer>
|
||||
host "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
include_tag_key true
|
||||
logstash_format true
|
||||
logstash_prefix kernel
|
||||
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
port "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
@type elasticsearch
|
||||
user "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
</match>
|
||||
|
||||
<match **>
|
||||
<buffer>
|
||||
chunk_limit_size 8MB
|
||||
flush_interval 15s
|
||||
flush_thread_count 8
|
||||
queue_limit_length 256
|
||||
retry_forever false
|
||||
retry_max_interval 30
|
||||
</buffer>
|
||||
flush_interval 15s
|
||||
host "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
include_tag_key true
|
||||
logstash_format true
|
||||
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
port "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
@type elasticsearch
|
||||
type_name fluent
|
||||
user "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
</match>
|
||||
fluentbit:
|
||||
template: |
|
||||
[SERVICE]
|
||||
Daemon false
|
||||
Flush 30
|
||||
Log_Level info
|
||||
Parsers_File parsers.conf
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name tail
|
||||
Path /var/log/kern.log
|
||||
Tag kernel
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name tail
|
||||
Parser docker
|
||||
Path /var/log/containers/*.log
|
||||
Tag kube.*
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name tail
|
||||
Path /var/log/libvirt/libvirtd.log
|
||||
Tag libvirt
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name tail
|
||||
Path /var/log/libvirt/qemu/*.log
|
||||
Tag qemu
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name systemd
|
||||
Path ${JOURNAL_PATH}
|
||||
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
|
||||
Tag journal.*
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name systemd
|
||||
Path ${JOURNAL_PATH}
|
||||
Systemd_Filter _SYSTEMD_UNIT=docker.service
|
||||
Tag journal.*
|
||||
|
||||
[FILTER]
|
||||
Interval 1s
|
||||
Match **
|
||||
Name throttle
|
||||
Rate 1000
|
||||
Window 300
|
||||
|
||||
[FILTER]
|
||||
Match libvirt
|
||||
Name record_modifier
|
||||
Record hostname ${HOSTNAME}
|
||||
|
||||
[FILTER]
|
||||
Match qemu
|
||||
Name record_modifier
|
||||
Record hostname ${HOSTNAME}
|
||||
|
||||
[FILTER]
|
||||
Match kernel
|
||||
Name record_modifier
|
||||
Record hostname ${HOSTNAME}
|
||||
|
||||
[FILTER]
|
||||
Match journal.**
|
||||
Name modify
|
||||
Rename _BOOT_ID BOOT_ID
|
||||
Rename _CAP_EFFECTIVE CAP_EFFECTIVE
|
||||
Rename _CMDLINE CMDLINE
|
||||
Rename _COMM COMM
|
||||
Rename _EXE EXE
|
||||
Rename _GID GID
|
||||
Rename _HOSTNAME HOSTNAME
|
||||
Rename _MACHINE_ID MACHINE_ID
|
||||
Rename _PID PID
|
||||
Rename _SYSTEMD_CGROUP SYSTEMD_CGROUP
|
||||
Rename _SYSTEMD_SLICE SYSTEMD_SLICE
|
||||
Rename _SYSTEMD_UNIT SYSTEMD_UNIT
|
||||
Rename _TRANSPORT TRANSPORT
|
||||
Rename _UID UID
|
||||
|
||||
[OUTPUT]
|
||||
Match **.fluentd**
|
||||
Name null
|
||||
|
||||
[FILTER]
|
||||
Match kube.*
|
||||
Merge_JSON_Log true
|
||||
Name kubernetes
|
||||
|
||||
[OUTPUT]
|
||||
Host ${FLUENTD_HOST}
|
||||
Match *
|
||||
Name forward
|
||||
Port ${FLUENTD_PORT}
|
||||
parsers:
|
||||
template: |
|
||||
[PARSER]
|
||||
Decode_Field_As escaped_utf8 log
|
||||
Format json
|
||||
Name docker
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%L
|
||||
Time_Keep true
|
||||
Time_Key time
|
||||
|
||||
fluentd_exporter:
|
||||
log:
|
||||
format: "logger:stdout?json=true"
|
||||
|
||||
@ -369,461 +369,6 @@ data:
|
||||
component: test
|
||||
values:
|
||||
release_uuid: ${RELEASE_UUID}
|
||||
conf:
|
||||
fluentbit:
|
||||
- service:
|
||||
header: service
|
||||
Flush: 30
|
||||
Daemon: Off
|
||||
Log_Level: info
|
||||
Parsers_File: parsers.conf
|
||||
- ceph_cluster_logs:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: ceph.cluster.*
|
||||
Path: /var/log/ceph/ceph.log
|
||||
Parsers: syslog
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- ceph_audit_logs:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: ceph.audit.*
|
||||
Path: /var/log/ceph/ceph.audit.log
|
||||
Parsers: syslog
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- ceph_mon_logs:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: ceph.mon.*
|
||||
Path: /var/log/ceph/ceph-mon**.log
|
||||
Parsers: syslog
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- ceph_osd_logs:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: ceph.osd.*
|
||||
Path: /var/log/ceph/ceph-osd**.log
|
||||
Parsers: syslog
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- kernel_messages:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: kernel
|
||||
Path: /var/log/kern.log
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- kubelet:
|
||||
header: input
|
||||
Name: systemd
|
||||
Tag: journal.*
|
||||
Path: ${JOURNAL_PATH}
|
||||
Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- libvirt:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: libvirt
|
||||
Path: /var/log/libvirt/libvirtd.log
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- qemu:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: qemu
|
||||
Path: /var/log/libvirt/qemu/*.log
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- docker_daemon:
|
||||
header: input
|
||||
Name: systemd
|
||||
Tag: journal.*
|
||||
Path: ${JOURNAL_PATH}
|
||||
Systemd_Filter: _SYSTEMD_UNIT=docker.service
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- throttle_filter:
|
||||
header: filter
|
||||
Name: throttle
|
||||
Match: "**"
|
||||
Rate: 1000
|
||||
Window: 300
|
||||
Interval: 1s
|
||||
- libvirt_record_modifier:
|
||||
header: filter
|
||||
Name: record_modifier
|
||||
Match: libvirt
|
||||
Record: hostname ${HOSTNAME}
|
||||
- qemu_record_modifier:
|
||||
header: filter
|
||||
Name: record_modifier
|
||||
Match: qemu
|
||||
Record: hostname ${HOSTNAME}
|
||||
- kernel_record_modifier:
|
||||
header: filter
|
||||
Name: record_modifier
|
||||
Match: kernel
|
||||
Record: hostname ${HOSTNAME}
|
||||
- systemd_modify_fields:
|
||||
header: filter
|
||||
Name: modify
|
||||
Match: journal.**
|
||||
Rename:
|
||||
_BOOT_ID: BOOT_ID
|
||||
_CAP_EFFECTIVE: CAP_EFFECTIVE
|
||||
_CMDLINE: CMDLINE
|
||||
_COMM: COMM
|
||||
_EXE: EXE
|
||||
_GID: GID
|
||||
_HOSTNAME: HOSTNAME
|
||||
_MACHINE_ID: MACHINE_ID
|
||||
_PID: PID
|
||||
_SYSTEMD_CGROUP: SYSTEMD_CGROUP
|
||||
_SYSTEMD_SLICE: SYSTEMD_SLICE
|
||||
_SYSTEMD_UNIT: SYSTEMD_UNIT
|
||||
_UID: UID
|
||||
_TRANSPORT: TRANSPORT
|
||||
- containers_tail:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: kube.*
|
||||
Path: /var/log/containers/*.log
|
||||
Parser: docker
|
||||
DB: /var/log/flb_kube.db
|
||||
Mem_Buf_Limit: 5MB
|
||||
DB.Sync: Normal
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- drop_fluentd_logs:
|
||||
header: output
|
||||
Name: "null"
|
||||
Match: "**.fluentd**"
|
||||
- kube_filter:
|
||||
header: filter
|
||||
Name: kubernetes
|
||||
Match: kube.*
|
||||
Merge_JSON_Log: On
|
||||
- fluentd_output:
|
||||
header: output
|
||||
Name: forward
|
||||
Match: "*"
|
||||
Host: ${FLUENTD_HOST}
|
||||
Port: ${FLUENTD_PORT}
|
||||
parsers:
|
||||
- docker:
|
||||
header: parser
|
||||
Name: docker
|
||||
Format: json
|
||||
Time_Key: time
|
||||
Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
|
||||
Time_Keep: On
|
||||
- syslog:
|
||||
header: parser
|
||||
Name: syslog
|
||||
Format: regex
|
||||
Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
|
||||
Time_Key: time
|
||||
Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
|
||||
Time_Keep: On
|
||||
Types: "pid:integer"
|
||||
fluentd:
|
||||
- metrics_agent:
|
||||
header: source
|
||||
type: monitor_agent
|
||||
bind: 0.0.0.0
|
||||
port: 24220
|
||||
- fluentbit_forward:
|
||||
header: source
|
||||
type: forward
|
||||
port: "#{ENV['FLUENTD_PORT']}"
|
||||
bind: 0.0.0.0
|
||||
- drop_fluent_logs:
|
||||
header: match
|
||||
type: "null"
|
||||
expression: "fluent.*"
|
||||
- add_container_name:
|
||||
header: filter
|
||||
type: record_transformer
|
||||
expression: "kube.**"
|
||||
enable_ruby: true
|
||||
record:
|
||||
-
|
||||
- header: record
|
||||
container_name: ${record["kubernetes"]["container_name"]}
|
||||
- remove_openstack_pod_logged_events:
|
||||
header: filter
|
||||
type: grep
|
||||
expression: "kube.**"
|
||||
exclude:
|
||||
-
|
||||
- header: exclude
|
||||
key: container_name
|
||||
pattern: ^(cinder-api|cinder-scheduler|cinder-volume|cinder-backup|glance-api|glance-registry|heat-api|heat-cfn|heat-engine|keystone-api|neutron-dhcp-agent|neutron-l3-agent|neutron-server|nova-osapi|nova-api|nova-compute|nova-conductor|nova-consoleauth|nova-novncproxy|nova-scheduler)$
|
||||
# NOTE(srwilkers): Look for specific keywords in the log key to determine
|
||||
# log level of event
|
||||
- tag_kubernetes_log_level:
|
||||
header: match
|
||||
type: rewrite_tag_filter
|
||||
expression: "kube.var.log.containers.**.log"
|
||||
rule:
|
||||
-
|
||||
- header: rule
|
||||
key: log
|
||||
pattern: /info/i
|
||||
tag: info.${tag}
|
||||
- header: rule
|
||||
key: log
|
||||
pattern: /warn/i
|
||||
tag: warn.${tag}
|
||||
- header: rule
|
||||
key: log
|
||||
pattern: /error/i
|
||||
tag: error.${tag}
|
||||
- header: rule
|
||||
key: log
|
||||
pattern: /critical/i
|
||||
tag: critical.${tag}
|
||||
- header: rule
|
||||
key: log
|
||||
pattern: (.+)
|
||||
tag: info.${tag}
|
||||
# NOTE(srwilkers): Create new key for log level, and use the tag prefix
|
||||
# added previously
|
||||
- add_kubernetes_log_level_and_application_key:
|
||||
header: filter
|
||||
type: record_transformer
|
||||
enable_ruby: true
|
||||
expression: "**.kube.var.log.containers.**.log"
|
||||
record:
|
||||
-
|
||||
- header: record
|
||||
level: ${tag_parts[0]}
|
||||
application: ${record["kubernetes"]["labels"]["application"]}
|
||||
- add_openstack_application_key:
|
||||
header: filter
|
||||
type: record_transformer
|
||||
expression: "openstack.**"
|
||||
record:
|
||||
-
|
||||
- header: record
|
||||
application: ${tag_parts[1]}
|
||||
#NOTE(srwilkers): This prefixes the tag for oslo.log entries from the
|
||||
# fluent handler/formatter with the log level, allowing for lookups on
|
||||
# openstack logs with a particular log level (ie: error.openstack.keystone)
|
||||
- tag_openstack_log_level:
|
||||
header: match
|
||||
type: rewrite_tag_filter
|
||||
expression: "openstack.**"
|
||||
rule:
|
||||
-
|
||||
- header: rule
|
||||
key: level
|
||||
pattern: INFO
|
||||
tag: info.${tag}
|
||||
- header: rule
|
||||
key: level
|
||||
pattern: WARN
|
||||
tag: warn.${tag}
|
||||
- header: rule
|
||||
key: level
|
||||
pattern: ERROR
|
||||
tag: error.${tag}
|
||||
- header: rule
|
||||
key: level
|
||||
pattern: CRITICAL
|
||||
tag: critical.${tag}
|
||||
- libvirt_elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "libvirt"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
logstash_prefix: libvirt
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
- qemu_elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "qemu"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
logstash_prefix: qemu
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
- journal_elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "journal.**"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
logstash_prefix: journal
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
- kernel_elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "kernel"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
logstash_prefix: kernel
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
- elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "**"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
flush_interval: "15"
|
||||
fluentd_exporter:
|
||||
log:
|
||||
format: "logger:stdout?json=true"
|
||||
level: "info"
|
||||
templates:
|
||||
syslog:
|
||||
template: "syslog-*"
|
||||
index_patterns: "syslog-*"
|
||||
settings:
|
||||
number_of_shards: 1
|
||||
mappings:
|
||||
syslog:
|
||||
properties:
|
||||
cluster:
|
||||
type: keyword
|
||||
app:
|
||||
type: keyword
|
||||
pid:
|
||||
type: integer
|
||||
host:
|
||||
type: keyword
|
||||
log:
|
||||
type: text
|
||||
oslo_openstack_fluentd:
|
||||
template: "openstack-*"
|
||||
index_patterns: "openstack-*"
|
||||
settings:
|
||||
number_of_shards: 1
|
||||
mappings:
|
||||
oslo_openstack_fluentd:
|
||||
properties:
|
||||
extra:
|
||||
properties:
|
||||
project:
|
||||
type: text
|
||||
norms: false
|
||||
version:
|
||||
type: text
|
||||
norms: false
|
||||
filename:
|
||||
type: text
|
||||
norms: false
|
||||
funcname:
|
||||
type: text
|
||||
norms: false
|
||||
message:
|
||||
type: text
|
||||
norms: false
|
||||
process_name:
|
||||
type: keyword
|
||||
index: false
|
||||
docker_fluentd:
|
||||
template: "logstash-*"
|
||||
index_patterns: "logstash-*"
|
||||
settings:
|
||||
number_of_shards: 1
|
||||
mappings:
|
||||
docker_fluentd:
|
||||
properties:
|
||||
kubernetes:
|
||||
properties:
|
||||
container_name:
|
||||
type: keyword
|
||||
index: false
|
||||
docker_id:
|
||||
type: keyword
|
||||
index: false
|
||||
host:
|
||||
type: keyword
|
||||
index: false
|
||||
namespace_name:
|
||||
type: keyword
|
||||
index: false
|
||||
pod_id:
|
||||
type: keyword
|
||||
index: false
|
||||
pod_name:
|
||||
type: keyword
|
||||
index: false
|
||||
monitoring:
|
||||
prometheus:
|
||||
enabled: true
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user