From b44d99b36a1b896ea6aac926bb2066f24c534272 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Wed, 22 May 2019 13:12:32 -0500 Subject: [PATCH] Fluent-logging: Remove utils for generating configuration files This removes the utilities for generating the fluentd, fluentbit, and parser configuration files from yaml and moves to instead consume the configuration files as strings from the values.yaml. This allows for easier creation and maintenance of configuration files for fluentd and fluentbit, as the utilities became unwieldy with complex configuration files. This necessitated the removal of the core test executed by the charts helm tests, but this would be required as we move to split the charts regardless Change-Id: Ied4a76bbdf58b54a6d702db04a7120b64f54dcac Signed-off-by: Steve Wilkerson --- .../templates/bin/_helm-tests.sh.tpl | 68 -- fluent-logging/templates/configmap-etc.yaml | 6 +- .../templates/utils/_to_fluentbit_conf.tpl | 44 -- .../templates/utils/_to_fluentd_conf.tpl | 90 --- fluent-logging/values.yaml | 673 +++++++++--------- .../armada/manifests/armada-lma.yaml | 455 ------------ 6 files changed, 336 insertions(+), 1000 deletions(-) delete mode 100644 fluent-logging/templates/utils/_to_fluentbit_conf.tpl delete mode 100644 fluent-logging/templates/utils/_to_fluentd_conf.tpl diff --git a/fluent-logging/templates/bin/_helm-tests.sh.tpl b/fluent-logging/templates/bin/_helm-tests.sh.tpl index 12bbf3786..24d94f11a 100644 --- a/fluent-logging/templates/bin/_helm-tests.sh.tpl +++ b/fluent-logging/templates/bin/_helm-tests.sh.tpl @@ -18,73 +18,6 @@ limitations under the License. set -ex -# Test whether indexes have been created for each Elasticsearch output defined -function check_output_indexes_exist () { - {{/* - First, determine the sum of Fluentbit and Fluentd's flush intervals. This - ensures we wait long enough for recorded events to be indexed - */}} - {{ $fluentBitConf := first .Values.conf.fluentbit }} - {{ $fluentBitServiceConf := index $fluentBitConf "service" }} - {{ $fluentBitFlush := index $fluentBitServiceConf "Flush" }} - fluentBitFlush={{$fluentBitFlush}} - - {{/* - The generic Elasticsearch output should always be last, and intervals for all - Elasticsearch outputs should match. This means we can safely use the last item - in fluentd's configuration to get the Fluentd flush output interval - */}} - {{- $fluentdConf := last .Values.conf.fluentd -}} - {{- $fluentdElasticsearchConf := index $fluentdConf "elasticsearch" -}} - {{- $fluentdFlush := index $fluentdElasticsearchConf "flush_interval" -}} - fluentdFlush={{$fluentdFlush}} - - totalFlush=$(($fluentBitFlush + $fluentdFlush)) - sleep $totalFlush - - {{/* - Iterate over Fluentd's config and for each Elasticsearch output, determine - the logstash index prefix and check Elasticsearch for that index - */}} - {{ range $key, $config := .Values.conf.td_agent -}} - - {{/* Get list of keys to determine config header to index on */}} - {{- $keyList := keys $config -}} - {{- $configSection := first $keyList -}} - - {{/* Index config section dictionary */}} - {{- $configEntry := index $config $configSection -}} - - {{- if hasKey $configEntry "type" -}} - {{- $type := index $configEntry "type" -}} - {{- if eq $type "elasticsearch" -}} - {{- if hasKey $configEntry "logstash_prefix" -}} - {{- $logstashPrefix := index $configEntry "logstash_prefix" }} - {{$logstashPrefix}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ - -XGET "${ELASTICSEARCH_ENDPOINT}/{{$logstashPrefix}}-*/_search?pretty" -H 'Content-Type: application/json' \ - | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']") - if [ "${{$logstashPrefix}}_total_hits" -gt 0 ]; then - echo "PASS: Successful hits on {{$logstashPrefix}}-* index!" - else - echo "FAIL: No hits on query for {{$logstashPrefix}}-* index! Exiting"; - exit 1; - fi - {{ else }} - logstash_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ - -XGET "${ELASTICSEARCH_ENDPOINT}/logstash-*/_search?pretty" -H 'Content-Type: application/json' \ - | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']") - if [ "$logstash_total_hits" -gt 0 ]; then - echo "PASS: Successful hits on logstash-* index!" - else - echo "FAIL: No hits on query for logstash-* index! Exiting"; - exit 1; - fi - {{ end }} - {{- end }} - {{- end }} - {{- end -}} -} - {{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }} # Tests whether fluent-logging has successfully generated the elasticsearch index mapping # templates defined by values.yaml @@ -106,4 +39,3 @@ function check_templates () { {{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }} check_templates {{ end }} -check_output_indexes_exist diff --git a/fluent-logging/templates/configmap-etc.yaml b/fluent-logging/templates/configmap-etc.yaml index 80c3f964d..fae65dcac 100644 --- a/fluent-logging/templates/configmap-etc.yaml +++ b/fluent-logging/templates/configmap-etc.yaml @@ -23,9 +23,9 @@ metadata: name: fluent-logging-etc type: Opaque data: - fluent-bit.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.fluentbit | b64enc }} - parsers.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.parsers | b64enc }} - fluent.conf: {{ include "fluent_logging.utils.to_fluentd_conf" .Values.conf.fluentd | b64enc }} + fluent-bit.conf: {{ .Values.conf.fluentbit.template | b64enc }} + parsers.conf: {{ .Values.conf.parsers.template | b64enc }} + fluent.conf: {{ .Values.conf.fluentd.template | b64enc }} {{ range $template, $fields := .Values.conf.templates }} {{ $template }}.json: {{ toJson $fields | b64enc }} {{ end }} diff --git a/fluent-logging/templates/utils/_to_fluentbit_conf.tpl b/fluent-logging/templates/utils/_to_fluentbit_conf.tpl deleted file mode 100644 index c32f82e01..000000000 --- a/fluent-logging/templates/utils/_to_fluentbit_conf.tpl +++ /dev/null @@ -1,44 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -# This function generates fluentbit configuration files with entries in the -# fluent-logging values.yaml. It results in a configuration section with the -# following format (for as many key/value pairs defined in values for a section): -# [HEADER] -# key value -# key value -# key value -# The configuration schema can be found here: -# http://fluentbit.io/documentation/0.12/configuration/schema.html - -{{- define "fluent_logging.utils.to_fluentbit_conf" -}} -{{- range $values := . -}} -{{- range $section := . -}} -{{- $header := pick . "header" -}} -{{- $config := omit . "header" }} -[{{$header.header | upper }}] -{{range $key, $value := $config -}} -{{ if eq $key "Rename" }} -{{- range $original, $new := $value -}} -{{ printf "Rename %s %s" $original $new | indent 4 }} -{{end}} -{{- else -}} -{{ $key | indent 4 }} {{ $value }} -{{end -}} -{{- end -}} -{{- end -}} -{{- end -}} -{{- end -}} diff --git a/fluent-logging/templates/utils/_to_fluentd_conf.tpl b/fluent-logging/templates/utils/_to_fluentd_conf.tpl deleted file mode 100644 index de9672c32..000000000 --- a/fluent-logging/templates/utils/_to_fluentd_conf.tpl +++ /dev/null @@ -1,90 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -# This function generates fluentd configuration files with entries in the -# fluent-logging values.yaml. It results in a configuration section with either -# of the following formats (for as many key/value pairs defined in values for a -section): -#
-# key value -# key value -# key value -#
-# or -#
-# key value -# -# key value -# -#
-# The configuration schema can be found here: -# https://docs.fluentd.org/v0.12/articles/config-file - -{{- define "fluent_logging.utils.to_fluentd_conf" -}} -{{- range $values := . -}} -{{- range $section := . -}} -{{- $header := pick . "header" -}} -{{- $config := omit . "header" "expression" -}} -{{- if hasKey . "expression" -}} -{{ $regex := pick . "expression" }} -{{ printf "<%s %s>" $header.header $regex.expression }} -{{- else }} -{{ printf "<%s>" $header.header }} -{{- end }} -{{- range $key, $value := $config -}} -{{- if kindIs "slice" $value }} -{{- range $value := . -}} -{{- range $innerSection := . -}} -{{- $innerHeader := pick . "header" -}} -{{- $innerConfig := omit . "header" "expression" -}} -{{- if hasKey . "expression" -}} -{{ $innerRegex := pick . "expression" }} -{{ printf "<%s %s>" $innerHeader.header $innerRegex.expression | indent 2 }} -{{- else }} -{{ printf "<%s>" $innerHeader.header | indent 2 }} -{{- end }} -{{- range $innerKey, $innerValue := $innerConfig -}} -{{- if eq $innerKey "type" -}} -{{ $type := list "@" "type" | join "" }} -{{ $type | indent 4 }} {{ $innerValue }} -{{- else if contains "ENV" ($innerValue | quote) }} -{{ $innerKey | indent 4 }} {{ $innerValue | quote }} -{{- else if eq $innerKey "flush_interval" }} -{{ $innerKey | indent 4 }} {{ printf "%ss" $innerValue }} -{{- else }} -{{ $innerKey | indent 4 }} {{ $innerValue }} -{{- end }} -{{- end }} -{{ printf "" $innerHeader.header | indent 2 }} -{{- end -}} -{{ end -}} -{{- else }} -{{- if eq $key "type" -}} -{{ $type := list "@" "type" | join "" }} -{{ $type | indent 2 }} {{ $value }} -{{- else if contains "ENV" ($value | quote) }} -{{ $key | indent 2 }} {{ $value | quote }} -{{- else if eq $key "flush_interval" }} -{{ $key | indent 2 }} {{ printf "%ss" $value }} -{{- else }} -{{ $key | indent 2 }} {{ $value }} -{{- end -}} -{{- end -}} -{{- end }} -{{ printf "" $header.header }} -{{- end }} -{{ end -}} -{{- end -}} diff --git a/fluent-logging/values.yaml b/fluent-logging/values.yaml index e1c01703d..d020c6508 100644 --- a/fluent-logging/values.yaml +++ b/fluent-logging/values.yaml @@ -103,347 +103,340 @@ dependencies: service: fluentd conf: - fluentbit: - - service: - header: service - Flush: 30 - Daemon: Off - Log_Level: info - Parsers_File: parsers.conf - - kernel_messages: - header: input - Name: tail - Tag: kernel - Path: /var/log/kern.log - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - containers_tail: - header: input - Name: tail - Tag: kube.* - Path: /var/log/containers/*.log - Parser: docker - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - libvirt: - header: input - Name: tail - Tag: libvirt - Path: /var/log/libvirt/libvirtd.log - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - qemu: - header: input - Name: tail - Tag: qemu - Path: /var/log/libvirt/qemu/*.log - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - kubelet: - header: input - Name: systemd - Tag: journal.* - Path: ${JOURNAL_PATH} - Systemd_Filter: _SYSTEMD_UNIT=kubelet.service - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - docker_daemon: - header: input - Name: systemd - Tag: journal.* - Path: ${JOURNAL_PATH} - Systemd_Filter: _SYSTEMD_UNIT=docker.service - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - throttle_filter: - header: filter - Name: throttle - Match: "**" - Rate: 1000 - Window: 300 - Interval: 1s - - libvirt_record_modifier: - header: filter - Name: record_modifier - Match: libvirt - Record: hostname ${HOSTNAME} - - qemu_record_modifier: - header: filter - Name: record_modifier - Match: qemu - Record: hostname ${HOSTNAME} - - kernel_record_modifier: - header: filter - Name: record_modifier - Match: kernel - Record: hostname ${HOSTNAME} - - systemd_modify_fields: - header: filter - Name: modify - Match: journal.** - Rename: - _BOOT_ID: BOOT_ID - _CAP_EFFECTIVE: CAP_EFFECTIVE - _CMDLINE: CMDLINE - _COMM: COMM - _EXE: EXE - _GID: GID - _HOSTNAME: HOSTNAME - _MACHINE_ID: MACHINE_ID - _PID: PID - _SYSTEMD_CGROUP: SYSTEMD_CGROUP - _SYSTEMD_SLICE: SYSTEMD_SLICE - _SYSTEMD_UNIT: SYSTEMD_UNIT - _UID: UID - _TRANSPORT: TRANSPORT - - drop_fluentd_logs: - header: output - Name: "null" - Match: "**.fluentd**" - - kube_filter: - header: filter - Name: kubernetes - Match: kube.* - Merge_JSON_Log: On - - fluentd_output: - header: output - Name: forward - Match: "*" - Host: ${FLUENTD_HOST} - Port: ${FLUENTD_PORT} - parsers: - - docker: - header: parser - Name: docker - Format: json - Time_Key: time - Time_Format: "%Y-%m-%dT%H:%M:%S.%L" - Time_Keep: On - Decode_Field_As: escaped_utf8 log fluentd: - - metrics_agent: - header: source - type: monitor_agent - bind: 0.0.0.0 - port: 24220 - - fluentbit_forward: - header: source - type: forward - port: "#{ENV['FLUENTD_PORT']}" - bind: 0.0.0.0 - - filter_fluentd_logs: - header: match - expression: "fluent.**" - type: "null" - # NOTE(srwilkers): Look for specific keywords in the log key to determine - # log level of event - - tag_kubernetes_log_level: - header: match - type: rewrite_tag_filter - expression: "kube.var.log.containers.**.log" - rule: - - - - header: rule - key: log - pattern: /info/i - tag: info.${tag} - - header: rule - key: log - pattern: /warn/i - tag: warn.${tag} - - header: rule - key: log - pattern: /error/i - tag: error.${tag} - - header: rule - key: log - pattern: /critical/i - tag: critical.${tag} - - header: rule - key: log - pattern: (.+) - tag: info.${tag} - # NOTE(srwilkers): Create new key for log level, and use the tag prefix - # added previously - - add_kubernetes_log_level_and_application_key: - header: filter - type: record_transformer - enable_ruby: true - expression: "**.kube.var.log.containers.**.log" - record: - - - - header: record - level: ${tag_parts[0]} - application: ${record["kubernetes"]["labels"]["application"]} - - add_openstack_application_key: - header: filter - type: record_transformer - expression: "openstack.**" - record: - - - - header: record - application: ${tag_parts[1]} - #NOTE(srwilkers): This prefixes the tag for oslo.log entries from the - # fluent handler/formatter with the log level, allowing for lookups on - # openstack logs with a particular log level (ie: error.openstack.keystone) - - tag_openstack_log_level: - header: match - type: rewrite_tag_filter - expression: "openstack.**" - rule: - - - - header: rule - key: level - pattern: INFO - tag: info.${tag} - - header: rule - key: level - pattern: WARN - tag: warn.${tag} - - header: rule - key: level - pattern: ERROR - tag: error.${tag} - - header: rule - key: level - pattern: CRITICAL - tag: critical.${tag} - #NOTE(tp6510): This prefixes the tag for auth entries - # it allows for lookups on openstack logs with - # a particular auth log (ie: auth.openstack.keystone) - - tag_auth_log: - header: match - type: rewrite_tag_filter - expression: "*.openstack.**" - rule: - - - - header: rule - key: application - pattern: keystone - tag: auth.${tag} - - header: rule - key: application - pattern: horizon - tag: auth.${tag} - - header: rule - key: application - pattern: mariadb - tag: auth.${tag} - - header: rule - key: application - pattern: memcached - tag: auth.${tag} - - header: rule - key: application - pattern: rabbitmq - tag: auth.${tag} - - libvirt_elasticsearch: - header: match - type: elasticsearch - user: "#{ENV['ELASTICSEARCH_USERNAME']}" - password: "#{ENV['ELASTICSEARCH_PASSWORD']}" - expression: "libvirt" - include_tag_key: true - host: "#{ENV['ELASTICSEARCH_HOST']}" - port: "#{ENV['ELASTICSEARCH_PORT']}" - logstash_format: true - logstash_prefix: libvirt - buffer: - - - - header: buffer - flush_thread_count: 8 - flush_interval: "15" - chunk_limit_size: 8MB - queue_limit_length: 256 - retry_max_interval: 30 - retry_forever: false - - qemu_elasticsearch: - header: match - type: elasticsearch - user: "#{ENV['ELASTICSEARCH_USERNAME']}" - password: "#{ENV['ELASTICSEARCH_PASSWORD']}" - expression: "qemu" - include_tag_key: true - host: "#{ENV['ELASTICSEARCH_HOST']}" - port: "#{ENV['ELASTICSEARCH_PORT']}" - logstash_format: true - logstash_prefix: qemu - buffer: - - - - header: buffer - flush_thread_count: 8 - flush_interval: "15" - chunk_limit_size: 8MB - queue_limit_length: 256 - retry_max_interval: 30 - retry_forever: false - - journal_elasticsearch: - header: match - type: elasticsearch - user: "#{ENV['ELASTICSEARCH_USERNAME']}" - password: "#{ENV['ELASTICSEARCH_PASSWORD']}" - expression: "journal.**" - include_tag_key: true - host: "#{ENV['ELASTICSEARCH_HOST']}" - port: "#{ENV['ELASTICSEARCH_PORT']}" - logstash_format: true - logstash_prefix: journal - buffer: - - - - header: buffer - flush_thread_count: 8 - flush_interval: "15" - chunk_limit_size: 8MB - queue_limit_length: 256 - retry_max_interval: 30 - retry_forever: false - - kernel_elasticsearch: - header: match - type: elasticsearch - user: "#{ENV['ELASTICSEARCH_USERNAME']}" - password: "#{ENV['ELASTICSEARCH_PASSWORD']}" - expression: "kernel" - include_tag_key: true - host: "#{ENV['ELASTICSEARCH_HOST']}" - port: "#{ENV['ELASTICSEARCH_PORT']}" - logstash_format: true - logstash_prefix: kernel - buffer: - - - - header: buffer - flush_thread_count: 8 - flush_interval: "15" - chunk_limit_size: 8MB - queue_limit_length: 256 - retry_max_interval: 30 - retry_forever: false - - elasticsearch: - header: match - type: elasticsearch - user: "#{ENV['ELASTICSEARCH_USERNAME']}" - password: "#{ENV['ELASTICSEARCH_PASSWORD']}" - expression: "**" - include_tag_key: true - host: "#{ENV['ELASTICSEARCH_HOST']}" - port: "#{ENV['ELASTICSEARCH_PORT']}" - logstash_format: true - type_name: fluent - buffer: - - - - header: buffer - flush_thread_count: 8 - flush_interval: "15" - chunk_limit_size: 8MB - queue_limit_length: 256 - retry_max_interval: 30 - retry_forever: false - flush_interval: "15" + template: | + + bind 0.0.0.0 + port 24220 + @type monitor_agent + + + + bind 0.0.0.0 + port "#{ENV['FLUENTD_PORT']}" + @type forward + + + + @type null + + + + + key log + pattern /info/i + tag info.${tag} + + + key log + pattern /warn/i + tag warn.${tag} + + + key log + pattern /error/i + tag error.${tag} + + + key log + pattern /critical/i + tag critical.${tag} + + + key log + pattern (.+) + tag info.${tag} + + @type rewrite_tag_filter + + + + enable_ruby true + + application ${record["kubernetes"]["labels"]["application"]} + level ${tag_parts[0]} + + @type record_transformer + + + + + application ${tag_parts[1]} + + @type record_transformer + + + + + key level + pattern INFO + tag info.${tag} + + + key level + pattern WARN + tag warn.${tag} + + + key level + pattern ERROR + tag error.${tag} + + + key level + pattern CRITICAL + tag critical.${tag} + + @type rewrite_tag_filter + + + + + key application + pattern keystone + tag auth.${tag} + + + key application + pattern horizon + tag auth.${tag} + + + key application + pattern mariadb + tag auth.${tag} + + + key application + pattern memcached + tag auth.${tag} + + + key application + pattern rabbitmq + tag auth.${tag} + + @type rewrite_tag_filter + + + + + chunk_limit_size 8MB + flush_interval 15s + flush_thread_count 8 + queue_limit_length 256 + retry_forever false + retry_max_interval 30 + + host "#{ENV['ELASTICSEARCH_HOST']}" + include_tag_key true + logstash_format true + logstash_prefix libvirt + password "#{ENV['ELASTICSEARCH_PASSWORD']}" + port "#{ENV['ELASTICSEARCH_PORT']}" + @type elasticsearch + user "#{ENV['ELASTICSEARCH_USERNAME']}" + + + + + chunk_limit_size 8MB + flush_interval 15s + flush_thread_count 8 + queue_limit_length 256 + retry_forever false + retry_max_interval 30 + + host "#{ENV['ELASTICSEARCH_HOST']}" + include_tag_key true + logstash_format true + logstash_prefix qemu + password "#{ENV['ELASTICSEARCH_PASSWORD']}" + port "#{ENV['ELASTICSEARCH_PORT']}" + @type elasticsearch + user "#{ENV['ELASTICSEARCH_USERNAME']}" + + + + + chunk_limit_size 8MB + flush_interval 15s + flush_thread_count 8 + queue_limit_length 256 + retry_forever false + retry_max_interval 30 + + host "#{ENV['ELASTICSEARCH_HOST']}" + include_tag_key true + logstash_format true + logstash_prefix journal + password "#{ENV['ELASTICSEARCH_PASSWORD']}" + port "#{ENV['ELASTICSEARCH_PORT']}" + @type elasticsearch + user "#{ENV['ELASTICSEARCH_USERNAME']}" + + + + + chunk_limit_size 8MB + flush_interval 15s + flush_thread_count 8 + queue_limit_length 256 + retry_forever false + retry_max_interval 30 + + host "#{ENV['ELASTICSEARCH_HOST']}" + include_tag_key true + logstash_format true + logstash_prefix kernel + password "#{ENV['ELASTICSEARCH_PASSWORD']}" + port "#{ENV['ELASTICSEARCH_PORT']}" + @type elasticsearch + user "#{ENV['ELASTICSEARCH_USERNAME']}" + + + + + chunk_limit_size 8MB + flush_interval 15s + flush_thread_count 8 + queue_limit_length 256 + retry_forever false + retry_max_interval 30 + + flush_interval 15s + host "#{ENV['ELASTICSEARCH_HOST']}" + include_tag_key true + logstash_format true + password "#{ENV['ELASTICSEARCH_PASSWORD']}" + port "#{ENV['ELASTICSEARCH_PORT']}" + @type elasticsearch + type_name fluent + user "#{ENV['ELASTICSEARCH_USERNAME']}" + + fluentbit: + template: | + [SERVICE] + Daemon false + Flush 30 + Log_Level info + Parsers_File parsers.conf + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Path /var/log/kern.log + Tag kernel + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Parser docker + Path /var/log/containers/*.log + Tag kube.* + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Path /var/log/libvirt/libvirtd.log + Tag libvirt + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Path /var/log/libvirt/qemu/*.log + Tag qemu + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name systemd + Path ${JOURNAL_PATH} + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + Tag journal.* + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name systemd + Path ${JOURNAL_PATH} + Systemd_Filter _SYSTEMD_UNIT=docker.service + Tag journal.* + + [FILTER] + Interval 1s + Match ** + Name throttle + Rate 1000 + Window 300 + + [FILTER] + Match libvirt + Name record_modifier + Record hostname ${HOSTNAME} + + [FILTER] + Match qemu + Name record_modifier + Record hostname ${HOSTNAME} + + [FILTER] + Match kernel + Name record_modifier + Record hostname ${HOSTNAME} + + [FILTER] + Match journal.** + Name modify + Rename _BOOT_ID BOOT_ID + Rename _CAP_EFFECTIVE CAP_EFFECTIVE + Rename _CMDLINE CMDLINE + Rename _COMM COMM + Rename _EXE EXE + Rename _GID GID + Rename _HOSTNAME HOSTNAME + Rename _MACHINE_ID MACHINE_ID + Rename _PID PID + Rename _SYSTEMD_CGROUP SYSTEMD_CGROUP + Rename _SYSTEMD_SLICE SYSTEMD_SLICE + Rename _SYSTEMD_UNIT SYSTEMD_UNIT + Rename _TRANSPORT TRANSPORT + Rename _UID UID + + [OUTPUT] + Match **.fluentd** + Name null + + [FILTER] + Match kube.* + Merge_JSON_Log true + Name kubernetes + + [OUTPUT] + Host ${FLUENTD_HOST} + Match * + Name forward + Port ${FLUENTD_PORT} + parsers: + template: | + [PARSER] + Decode_Field_As escaped_utf8 log + Format json + Name docker + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep true + Time_Key time + fluentd_exporter: log: format: "logger:stdout?json=true" diff --git a/tools/deployment/armada/manifests/armada-lma.yaml b/tools/deployment/armada/manifests/armada-lma.yaml index 6d7d50a40..968dff685 100644 --- a/tools/deployment/armada/manifests/armada-lma.yaml +++ b/tools/deployment/armada/manifests/armada-lma.yaml @@ -369,461 +369,6 @@ data: component: test values: release_uuid: ${RELEASE_UUID} - conf: - fluentbit: - - service: - header: service - Flush: 30 - Daemon: Off - Log_Level: info - Parsers_File: parsers.conf - - ceph_cluster_logs: - header: input - Name: tail - Tag: ceph.cluster.* - Path: /var/log/ceph/ceph.log - Parsers: syslog - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - ceph_audit_logs: - header: input - Name: tail - Tag: ceph.audit.* - Path: /var/log/ceph/ceph.audit.log - Parsers: syslog - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - ceph_mon_logs: - header: input - Name: tail - Tag: ceph.mon.* - Path: /var/log/ceph/ceph-mon**.log - Parsers: syslog - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - ceph_osd_logs: - header: input - Name: tail - Tag: ceph.osd.* - Path: /var/log/ceph/ceph-osd**.log - Parsers: syslog - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - kernel_messages: - header: input - Name: tail - Tag: kernel - Path: /var/log/kern.log - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - kubelet: - header: input - Name: systemd - Tag: journal.* - Path: ${JOURNAL_PATH} - Systemd_Filter: _SYSTEMD_UNIT=kubelet.service - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - libvirt: - header: input - Name: tail - Tag: libvirt - Path: /var/log/libvirt/libvirtd.log - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - qemu: - header: input - Name: tail - Tag: qemu - Path: /var/log/libvirt/qemu/*.log - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - docker_daemon: - header: input - Name: systemd - Tag: journal.* - Path: ${JOURNAL_PATH} - Systemd_Filter: _SYSTEMD_UNIT=docker.service - Mem_Buf_Limit: 5MB - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - throttle_filter: - header: filter - Name: throttle - Match: "**" - Rate: 1000 - Window: 300 - Interval: 1s - - libvirt_record_modifier: - header: filter - Name: record_modifier - Match: libvirt - Record: hostname ${HOSTNAME} - - qemu_record_modifier: - header: filter - Name: record_modifier - Match: qemu - Record: hostname ${HOSTNAME} - - kernel_record_modifier: - header: filter - Name: record_modifier - Match: kernel - Record: hostname ${HOSTNAME} - - systemd_modify_fields: - header: filter - Name: modify - Match: journal.** - Rename: - _BOOT_ID: BOOT_ID - _CAP_EFFECTIVE: CAP_EFFECTIVE - _CMDLINE: CMDLINE - _COMM: COMM - _EXE: EXE - _GID: GID - _HOSTNAME: HOSTNAME - _MACHINE_ID: MACHINE_ID - _PID: PID - _SYSTEMD_CGROUP: SYSTEMD_CGROUP - _SYSTEMD_SLICE: SYSTEMD_SLICE - _SYSTEMD_UNIT: SYSTEMD_UNIT - _UID: UID - _TRANSPORT: TRANSPORT - - containers_tail: - header: input - Name: tail - Tag: kube.* - Path: /var/log/containers/*.log - Parser: docker - DB: /var/log/flb_kube.db - Mem_Buf_Limit: 5MB - DB.Sync: Normal - Buffer_Chunk_Size: 1M - Buffer_Max_Size: 1M - - drop_fluentd_logs: - header: output - Name: "null" - Match: "**.fluentd**" - - kube_filter: - header: filter - Name: kubernetes - Match: kube.* - Merge_JSON_Log: On - - fluentd_output: - header: output - Name: forward - Match: "*" - Host: ${FLUENTD_HOST} - Port: ${FLUENTD_PORT} - parsers: - - docker: - header: parser - Name: docker - Format: json - Time_Key: time - Time_Format: "%Y-%m-%dT%H:%M:%S.%L" - Time_Keep: On - - syslog: - header: parser - Name: syslog - Format: regex - Regex: '^(?