openstack/openstack-helm-infra
Code Issues Proposed changes

Fluentd: Allow for Multiple Config Files

Browse Source 
Change-Id: I2eb6ff2a599ee7bc479f9fc4955016b18902c879
This commit is contained in:
Steven Fitzpatrick 2020-07-27 08:57:39 -05:00
parent 959417f321
commit 4e73195bc2
5 changed files with 201 additions and 446 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
fluentd/templates/bin/_fluentd.sh.tpl
View File

@ -18,7 +18,7 @@ set -ex
COMMAND="${@:-start}" COMMAND="${@:-start}"
function start () { function start () {
exec fluentd -c /fluentd/etc/fluent.conf exec fluentd -c /fluentd/etc/main.conf
} }
function stop () { function stop () {

17
fluentd/templates/configmap-etc.yaml
View File

@ -12,8 +12,15 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/}} */}}
{{- define "fluentd_main" }}
{{- $path := .Values.conf.fluentd.path}}
{{- range $name, $conf := .Values.conf.fluentd.conf }}
{{ printf "%s %s/%s.conf" "@include" $path $name | indent 4}}
{{- end }}
{{- end }}
{{- if .Values.manifests.configmap_etc }} {{- if .Values.manifests.configmap_etc }}
{{- $envAll := . }} {{ $envAll := .}}
--- ---
apiVersion: v1 apiVersion: v1
kind: Secret kind: Secret
@ -22,6 +29,12 @@ metadata:
annotations: annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
type: Opaque type: Opaque
stringData:
main.conf: |
{{- template "fluentd_main" . }}
data: data:
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.fluentd.template "key" "fluent.conf" "format" "Secret") | indent 2 }} {{- range $name, $config := .Values.conf.fluentd.conf }}
{{- $filename := printf "%s.conf" $name}}
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" $config "key" $filename "format" "Secret") | indent 2 }}
{{- end }}
{{- end }} {{- end }}

11
fluentd/templates/daemonset.yaml
View File

@ -15,6 +15,7 @@ limitations under the License.
{{- if .Values.manifests.daemonset }} {{- if .Values.manifests.daemonset }}
{{- $envAll := . }} {{- $envAll := . }}
{{- $config_path := .Values.conf.fluentd.path }}
{{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }} {{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }}
{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.fluentd }} {{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.fluentd }}
@ -183,9 +184,15 @@ spec:
- name: pod-etc-fluentd - name: pod-etc-fluentd
mountPath: /fluentd/etc mountPath: /fluentd/etc
- name: fluentd-etc - name: fluentd-etc
mountPath: /fluentd/etc/fluent.conf mountPath: {{ printf "%s/%s.conf" $config_path "main" }}
subPath: fluent.conf subPath: {{ printf "%s.conf" "main"}}
readOnly: true readOnly: true
{{- range $name, $config := .Values.conf.fluentd.conf }}
- name: fluentd-etc
mountPath: {{ printf "%s/%s.conf" $config_path $name }}
subPath: {{ printf "%s.conf" $name }}
readOnly: true
{{- end }}
- name: fluentd-bin - name: fluentd-bin
mountPath: /tmp/fluentd.sh mountPath: /tmp/fluentd.sh
subPath: fluentd.sh subPath: fluentd.sh

282
fluentd/values.yaml
View File

@ -54,243 +54,51 @@ dependencies:
conf: conf:
fluentd: fluentd:
# This field is now rendered as a helm template! path: /fluentd/etc
template: | conf:
<source> input: |
@type prometheus <source>
port {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} bind 0.0.0.0
</source> port "#{ENV['FLUENTD_PORT']}"
@type forward
<source> </source>
@type prometheus_monitor <source>
</source> <parse>
time_format %Y-%m-%dT%H:%M:%S.%NZ
<source> @type json
@type prometheus_output_monitor </parse>
</source> path /var/log/containers/*.log
read_from_head true
<source> tag kubernetes.*
@type prometheus_tail_monitor @type tail
</source> </source>
<match **>
<source> @type relabel
bind 0.0.0.0 @label @output
port "#{ENV['FLUENTD_PORT']}" </match>
@type forward output: |
</source> <label @output>
<match **>
<match fluent.**> <buffer>
@type null chunk_limit_size 512K
</match> flush_interval 5s
flush_thread_count 8
<match kube.var.log.containers.**.log> queue_limit_length 32
<rule> retry_forever false
key log retry_max_interval 30
pattern /info/i </buffer>
tag info.${tag} host "#{ENV['ELASTICSEARCH_HOST']}"
</rule> reload_connections false
<rule> reconnect_on_error true
key log reload_on_failure true
pattern /warn/i include_tag_key true
tag warn.${tag} logstash_format true
</rule> password "#{ENV['ELASTICSEARCH_PASSWORD']}"
<rule> port "#{ENV['ELASTICSEARCH_PORT']}"
key log @type elasticsearch
pattern /error/i user "#{ENV['ELASTICSEARCH_USERNAME']}"
tag error.${tag} </match>
</rule> </label>
<rule>
key log
pattern /critical/i
tag critical.${tag}
</rule>
<rule>
key log
pattern (.+)
tag info.${tag}
</rule>
@type rewrite_tag_filter
</match>
<filter **.kube.var.log.containers.**.log>
enable_ruby true
<record>
application ${record["kubernetes"]["labels"]["application"]}
level ${tag_parts[0]}
</record>
@type record_transformer
</filter>
<filter openstack.**>
<record>
application ${tag_parts[1]}
</record>
@type record_transformer
</filter>
<match openstack.**>
<rule>
key level
pattern INFO
tag info.${tag}
</rule>
<rule>
key level
pattern WARN
tag warn.${tag}
</rule>
<rule>
key level
pattern ERROR
tag error.${tag}
</rule>
<rule>
key level
pattern CRITICAL
tag critical.${tag}
</rule>
@type rewrite_tag_filter
</match>
<match *.openstack.**>
<rule>
key application
pattern keystone
tag auth.${tag}
</rule>
<rule>
key application
pattern horizon
tag auth.${tag}
</rule>
<rule>
key application
pattern mariadb
tag auth.${tag}
</rule>
<rule>
key application
pattern memcached
tag auth.${tag}
</rule>
<rule>
key application
pattern rabbitmq
tag auth.${tag}
</rule>
@type rewrite_tag_filter
</match>
<match libvirt>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix libvirt
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match qemu>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix qemu
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match journal.**>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix journal
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match kernel>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix kernel
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match **>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
type_name fluent
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
endpoints: endpoints:
cluster_domain_suffix: cluster.local cluster_domain_suffix: cluster.local
local_image_registry: local_image_registry:

335
tools/deployment/common/fluentd.sh
View File

@ -28,228 +28,155 @@ pod:
MY_TEST_SECRET: BAR MY_TEST_SECRET: BAR
conf: conf:
fluentd: fluentd:
# This field is now rendered as a helm template! conf:
template: | # These fields are rendered as helm templates
<source> input: |
@type prometheus <source>
port {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} @type prometheus
</source> port {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
</source>
<source> <source>
@type prometheus_monitor @type prometheus_monitor
</source> </source>
<source> <source>
@type prometheus_output_monitor @type prometheus_output_monitor
</source> </source>
<source> <source>
@type prometheus_tail_monitor @type prometheus_tail_monitor
</source> </source>
<source> <source>
bind 0.0.0.0 bind 0.0.0.0
port "#{ENV['FLUENTD_PORT']}" port "#{ENV['FLUENTD_PORT']}"
@type forward @type forward
</source> </source>
<source> <source>
<parse> <parse>
time_format %Y-%m-%dT%H:%M:%S.%NZ time_format %Y-%m-%dT%H:%M:%S.%NZ
@type json @type json
</parse> </parse>
path /var/log/containers/*.log path /var/log/containers/*.log
read_from_head true read_from_head true
tag kubernetes.* tag kubernetes.*
@type tail @type tail
</source> </source>
<source> <source>
@type tail @type tail
tag libvirt.* tag libvirt.*
path /var/log/libvirt/**.log path /var/log/libvirt/**.log
read_from_head true read_from_head true
<parse> <parse>
@type none @type none
</parse> </parse>
</source> </source>
<source> <source>
@type systemd @type systemd
tag auth tag auth
path /var/log/journal path /var/log/journal
matches [{ "SYSLOG_FACILITY":"10" }] matches [{ "SYSLOG_FACILITY":"10" }]
read_from_head true read_from_head true
<entry> <entry>
fields_strip_underscores true fields_strip_underscores true
fields_lowercase true fields_lowercase true
</entry> </entry>
</source> </source>
<source> <source>
@type systemd @type systemd
tag journal.* tag journal.*
path /var/log/journal path /var/log/journal
matches [{ "_SYSTEMD_UNIT": "docker.service" }] matches [{ "_SYSTEMD_UNIT": "docker.service" }]
read_from_head true read_from_head true
<entry> <entry>
fields_strip_underscores true fields_strip_underscores true
fields_lowercase true fields_lowercase true
</entry> </entry>
</source> </source>
<source> <source>
@type systemd @type systemd
tag journal.* tag journal.*
path /var/log/journal path /var/log/journal
matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
read_from_head true read_from_head true
<entry> <entry>
fields_strip_underscores true fields_strip_underscores true
fields_lowercase true fields_lowercase true
</entry> </entry>
</source> </source>
<source> <source>
@type systemd @type systemd
tag kernel tag kernel
path /var/log/journal path /var/log/journal
matches [{ "_TRANSPORT": "kernel" }] matches [{ "_TRANSPORT": "kernel" }]
read_from_head true read_from_head true
<entry> <entry>
fields_strip_underscores true fields_strip_underscores true
fields_lowercase true fields_lowercase true
</entry> </entry>
</source> </source>
<filter kubernetes.**> <match **>
@type kubernetes_metadata @type relabel
</filter> @label @filter
</match>
<filter libvirt.**> filter: |
@type record_transformer <label @filter>
<record> <filter kubernetes.**>
hostname "#{ENV['NODE_NAME']}" @type kubernetes_metadata
fluentd_pod "#{ENV['POD_NAME']}" </filter>
</record>
</filter>
<match fluent.**> <filter libvirt.**>
@type null @type record_transformer
</match> <record>
hostname "#{ENV['NODE_NAME']}"
fluentd_pod "#{ENV['POD_NAME']}"
</record>
</filter>
<match **>
@type relabel
@label @output
</match>
</label>
output: |
<label @output>
<match fluent.**>
@type null
</match>
<match libvirt.**> <match **>
<buffer> <buffer>
chunk_limit_size 512K chunk_limit_size 512K
flush_interval 5s flush_interval 5s
flush_thread_count 8 flush_thread_count 8
queue_limit_length 32 queue_limit_length 32
retry_forever false retry_forever false
retry_max_interval 30 retry_max_interval 30
</buffer> </buffer>
host "#{ENV['ELASTICSEARCH_HOST']}" host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false reload_connections false
reconnect_on_error true reconnect_on_error true
reload_on_failure true reload_on_failure true
include_tag_key true include_tag_key true
logstash_format true logstash_format true
logstash_prefix libvirt password "#{ENV['ELASTICSEARCH_PASSWORD']}"
password "#{ENV['ELASTICSEARCH_PASSWORD']}" port "#{ENV['ELASTICSEARCH_PORT']}"
port "#{ENV['ELASTICSEARCH_PORT']}" @type elasticsearch
@type elasticsearch user "#{ENV['ELASTICSEARCH_USERNAME']}"
user "#{ENV['ELASTICSEARCH_USERNAME']}" </match>
</match> </label>
<match kernel>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
disable_chunk_backup true
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix kernel
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match auth>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix auth
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match journal.**>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix journal
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match **>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
EOF EOF
helm upgrade --install fluentd ./fluentd \ helm upgrade --install fluentd ./fluentd \
--namespace=osh-infra \ --namespace=osh-infra \