Fluentbit: Remove database used in tail inputs
This updates the fluentbit configuration for tail inputs to remove the values for utilizing mysqlite databases to track its location in each file it's configured to tail. This is intended to reduce the pressure fluentbit exerts on the host through writing to /var/log/foo.db. To help mitigate large amounts of traffic sent from fluentbit to fluentd upon a pod restart, this also adds a throttle filter to fluentbit. As a result, Fluentbit no longer needs a writable mount to its hostPath on /var/log on the host. Thus, this change includes updating the Fluentbit daemonset's mount on /var/log to be readOnly Change-Id: If4381f4ff47e887f3ea10beded4f6172edaf08ba
This commit is contained in:
parent
25e4e5662e
commit
3614d025dc
@ -122,6 +122,7 @@ spec:
|
|||||||
readOnly: true
|
readOnly: true
|
||||||
- name: varlog
|
- name: varlog
|
||||||
mountPath: /var/log
|
mountPath: /var/log
|
||||||
|
readOnly: true
|
||||||
- name: varlibdockercontainers
|
- name: varlibdockercontainers
|
||||||
mountPath: /var/lib/docker/containers
|
mountPath: /var/lib/docker/containers
|
||||||
readOnly: true
|
readOnly: true
|
||||||
|
@ -115,9 +115,16 @@ conf:
|
|||||||
Name: tail
|
Name: tail
|
||||||
Tag: kernel
|
Tag: kernel
|
||||||
Path: /var/log/kern.log
|
Path: /var/log/kern.log
|
||||||
DB: /var/log/kern.db
|
|
||||||
Mem_Buf_Limit: 5MB
|
Mem_Buf_Limit: 5MB
|
||||||
DB.Sync: Normal
|
Buffer_Chunk_Size: 1M
|
||||||
|
Buffer_Max_Size: 1M
|
||||||
|
- containers_tail:
|
||||||
|
header: input
|
||||||
|
Name: tail
|
||||||
|
Tag: kube.*
|
||||||
|
Path: /var/log/containers/*.log
|
||||||
|
Parser: docker
|
||||||
|
Mem_Buf_Limit: 5MB
|
||||||
Buffer_Chunk_Size: 1M
|
Buffer_Chunk_Size: 1M
|
||||||
Buffer_Max_Size: 1M
|
Buffer_Max_Size: 1M
|
||||||
- libvirt:
|
- libvirt:
|
||||||
@ -125,9 +132,7 @@ conf:
|
|||||||
Name: tail
|
Name: tail
|
||||||
Tag: libvirt
|
Tag: libvirt
|
||||||
Path: /var/log/libvirt/libvirtd.log
|
Path: /var/log/libvirt/libvirtd.log
|
||||||
DB: /var/log/libvirt.db
|
|
||||||
Mem_Buf_Limit: 5MB
|
Mem_Buf_Limit: 5MB
|
||||||
DB.Sync: Normal
|
|
||||||
Buffer_Chunk_Size: 1M
|
Buffer_Chunk_Size: 1M
|
||||||
Buffer_Max_Size: 1M
|
Buffer_Max_Size: 1M
|
||||||
- qemu:
|
- qemu:
|
||||||
@ -135,9 +140,7 @@ conf:
|
|||||||
Name: tail
|
Name: tail
|
||||||
Tag: qemu
|
Tag: qemu
|
||||||
Path: /var/log/libvirt/qemu/*.log
|
Path: /var/log/libvirt/qemu/*.log
|
||||||
DB: /var/log/qemu.db
|
|
||||||
Mem_Buf_Limit: 5MB
|
Mem_Buf_Limit: 5MB
|
||||||
DB.Sync: Normal
|
|
||||||
Buffer_Chunk_Size: 1M
|
Buffer_Chunk_Size: 1M
|
||||||
Buffer_Max_Size: 1M
|
Buffer_Max_Size: 1M
|
||||||
- kubelet:
|
- kubelet:
|
||||||
@ -146,9 +149,7 @@ conf:
|
|||||||
Tag: journal.*
|
Tag: journal.*
|
||||||
Path: ${JOURNAL_PATH}
|
Path: ${JOURNAL_PATH}
|
||||||
Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
|
Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
|
||||||
DB: /var/log/kubelet.db
|
|
||||||
Mem_Buf_Limit: 5MB
|
Mem_Buf_Limit: 5MB
|
||||||
DB.Sync: Normal
|
|
||||||
Buffer_Chunk_Size: 1M
|
Buffer_Chunk_Size: 1M
|
||||||
Buffer_Max_Size: 1M
|
Buffer_Max_Size: 1M
|
||||||
- docker_daemon:
|
- docker_daemon:
|
||||||
@ -157,11 +158,16 @@ conf:
|
|||||||
Tag: journal.*
|
Tag: journal.*
|
||||||
Path: ${JOURNAL_PATH}
|
Path: ${JOURNAL_PATH}
|
||||||
Systemd_Filter: _SYSTEMD_UNIT=docker.service
|
Systemd_Filter: _SYSTEMD_UNIT=docker.service
|
||||||
DB: /var/log/docker.db
|
|
||||||
Mem_Buf_Limit: 5MB
|
Mem_Buf_Limit: 5MB
|
||||||
DB.Sync: Normal
|
|
||||||
Buffer_Chunk_Size: 1M
|
Buffer_Chunk_Size: 1M
|
||||||
Buffer_Max_Size: 1M
|
Buffer_Max_Size: 1M
|
||||||
|
- throttle_filter:
|
||||||
|
header: filter
|
||||||
|
Name: throttle
|
||||||
|
Match: "**"
|
||||||
|
Rate: 1000
|
||||||
|
Window: 300
|
||||||
|
Interval: 1s
|
||||||
- libvirt_record_modifier:
|
- libvirt_record_modifier:
|
||||||
header: filter
|
header: filter
|
||||||
Name: record_modifier
|
Name: record_modifier
|
||||||
@ -196,17 +202,6 @@ conf:
|
|||||||
_SYSTEMD_UNIT: SYSTEMD_UNIT
|
_SYSTEMD_UNIT: SYSTEMD_UNIT
|
||||||
_UID: UID
|
_UID: UID
|
||||||
_TRANSPORT: TRANSPORT
|
_TRANSPORT: TRANSPORT
|
||||||
- containers_tail:
|
|
||||||
header: input
|
|
||||||
Name: tail
|
|
||||||
Tag: kube.*
|
|
||||||
Path: /var/log/containers/*.log
|
|
||||||
Parser: docker
|
|
||||||
DB: /var/log/flb_kube.db
|
|
||||||
Mem_Buf_Limit: 5MB
|
|
||||||
DB.Sync: Normal
|
|
||||||
Buffer_Chunk_Size: 1M
|
|
||||||
Buffer_Max_Size: 1M
|
|
||||||
- drop_fluentd_logs:
|
- drop_fluentd_logs:
|
||||||
header: output
|
header: output
|
||||||
Name: "null"
|
Name: "null"
|
||||||
|
Loading…
Reference in New Issue
Block a user