Fluentbit: Remove database used in tail inputs

This updates the fluentbit configuration for tail inputs to remove
the values for utilizing mysqlite databases to track its location
in each file it's configured to tail.  This is intended to reduce
the pressure fluentbit exerts on the host through writing to
/var/log/foo.db. To help mitigate large amounts of traffic
sent from fluentbit to fluentd upon a pod restart, this also
adds a throttle filter to fluentbit.

As a result, Fluentbit no longer needs a writable mount to its
hostPath on /var/log on the host.  Thus, this change includes
updating the Fluentbit daemonset's mount on /var/log to be
readOnly

Change-Id: If4381f4ff47e887f3ea10beded4f6172edaf08ba
This commit is contained in:
Steve Wilkerson 2019-01-17 11:38:29 -06:00
parent 25e4e5662e
commit 3614d025dc
2 changed files with 17 additions and 21 deletions

View File

@ -122,6 +122,7 @@ spec:
readOnly: true readOnly: true
- name: varlog - name: varlog
mountPath: /var/log mountPath: /var/log
readOnly: true
- name: varlibdockercontainers - name: varlibdockercontainers
mountPath: /var/lib/docker/containers mountPath: /var/lib/docker/containers
readOnly: true readOnly: true

View File

@ -115,9 +115,16 @@ conf:
Name: tail Name: tail
Tag: kernel Tag: kernel
Path: /var/log/kern.log Path: /var/log/kern.log
DB: /var/log/kern.db
Mem_Buf_Limit: 5MB Mem_Buf_Limit: 5MB
DB.Sync: Normal Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- containers_tail:
header: input
Name: tail
Tag: kube.*
Path: /var/log/containers/*.log
Parser: docker
Mem_Buf_Limit: 5MB
Buffer_Chunk_Size: 1M Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M Buffer_Max_Size: 1M
- libvirt: - libvirt:
@ -125,9 +132,7 @@ conf:
Name: tail Name: tail
Tag: libvirt Tag: libvirt
Path: /var/log/libvirt/libvirtd.log Path: /var/log/libvirt/libvirtd.log
DB: /var/log/libvirt.db
Mem_Buf_Limit: 5MB Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M Buffer_Max_Size: 1M
- qemu: - qemu:
@ -135,9 +140,7 @@ conf:
Name: tail Name: tail
Tag: qemu Tag: qemu
Path: /var/log/libvirt/qemu/*.log Path: /var/log/libvirt/qemu/*.log
DB: /var/log/qemu.db
Mem_Buf_Limit: 5MB Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M Buffer_Max_Size: 1M
- kubelet: - kubelet:
@ -146,9 +149,7 @@ conf:
Tag: journal.* Tag: journal.*
Path: ${JOURNAL_PATH} Path: ${JOURNAL_PATH}
Systemd_Filter: _SYSTEMD_UNIT=kubelet.service Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
DB: /var/log/kubelet.db
Mem_Buf_Limit: 5MB Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M Buffer_Max_Size: 1M
- docker_daemon: - docker_daemon:
@ -157,11 +158,16 @@ conf:
Tag: journal.* Tag: journal.*
Path: ${JOURNAL_PATH} Path: ${JOURNAL_PATH}
Systemd_Filter: _SYSTEMD_UNIT=docker.service Systemd_Filter: _SYSTEMD_UNIT=docker.service
DB: /var/log/docker.db
Mem_Buf_Limit: 5MB Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M Buffer_Max_Size: 1M
- throttle_filter:
header: filter
Name: throttle
Match: "**"
Rate: 1000
Window: 300
Interval: 1s
- libvirt_record_modifier: - libvirt_record_modifier:
header: filter header: filter
Name: record_modifier Name: record_modifier
@ -196,17 +202,6 @@ conf:
_SYSTEMD_UNIT: SYSTEMD_UNIT _SYSTEMD_UNIT: SYSTEMD_UNIT
_UID: UID _UID: UID
_TRANSPORT: TRANSPORT _TRANSPORT: TRANSPORT
- containers_tail:
header: input
Name: tail
Tag: kube.*
Path: /var/log/containers/*.log
Parser: docker
DB: /var/log/flb_kube.db
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- drop_fluentd_logs: - drop_fluentd_logs:
header: output header: output
Name: "null" Name: "null"