Add fluentd_enable_watch_timer

By default, the watch timer in Fluentd is set to True.
To save CPU and I/O consumption this can be set to False, which
kolla-ansible has been hardcoding so far.

When the watch timer is disabled, in_tail relies entirely
on inotify. In certain constellations, this may not work
reliably. In these cases, the watch timer needs to be activated, so this
change adds a variable to make the setting configurable.

Change-Id: Ic8ce6fbc3ed8f31d5d090e114b35703532679729

ansible/roles/common/defaults/main.yml

@@ -127,6 +127,9 @@ cron_logrotate_schedule: "daily"
 # Fluentd
 ####################
 
+# Enable the additional watch timer
+fluentd_enable_watch_timer: "false"
+
 fluentd_input_openstack_services:
   - name: aodh
     enabled: "{{ enable_aodh | bool }}"

ansible/roles/common/templates/conf/input/00-global.conf.j2

@@ -14,7 +14,7 @@
   pos_file /var/run/td-agent/kolla-openstack.pos
   tag kolla.*
   ignore_repeated_permission_error true
-  enable_watch_timer false
+  enable_watch_timer {{ fluentd_enable_watch_timer }}
   <parse>
     @type multiline
     format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/

ansible/roles/common/templates/conf/input/02-mariadb.conf.j2

@@ -5,7 +5,7 @@
   path /var/log/kolla/mariadb/mariadb.log
   pos_file /var/run/td-agent/mariadb.pos
   tag infra.mariadb
-  enable_watch_timer false
+  enable_watch_timer {{ fluentd_enable_watch_timer }}
   <parse>
     @type multiline
     format_firstline /^(\d{4}-\d{2}-\d{2}|\d{6}) /
@@ -18,7 +18,7 @@
   pos_file /var/run/td-agent/mariadb-xinetd.pos
   tag infra.mariadb-xinetd
   ignore_repeated_permission_error true
-  enable_watch_timer false
+  enable_watch_timer {{ fluentd_enable_watch_timer }}
   <parse>
     @type multiline
     format_firstline /^\d{2}/\d{1,2}/\d{1,2}@\d{1,2}:\d{1,2}:\d{1,2}\: (START|EXIT)\: /

ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2

@@ -3,7 +3,7 @@
   path /var/log/kolla/rabbitmq/rabbit@{{ ansible_facts.hostname }}.log
   pos_file /var/run/td-agent/rabbit.pos
   tag infra.rabbit
-  enable_watch_timer false
+  enable_watch_timer {{ fluentd_enable_watch_timer }}
   <parse>
     @type multiline
     format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}/

ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2

@@ -4,7 +4,7 @@
   path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log,/var/log/kolla/*/*_access.log,/var/log/kolla/*/*_error.log
   pos_file /var/run/td-agent/kolla-openstack-wsgi.pos
   tag kolla.*
-  enable_watch_timer false
+  enable_watch_timer {{ fluentd_enable_watch_timer }}
   <parse>
     @type regexp
     expression /^(?<Payload>.*)$/

ansible/roles/common/templates/conf/input/05-libvirt.conf.j2

@@ -3,7 +3,7 @@
   path /var/log/kolla/libvirt/libvirtd.log
   pos_file /var/run/td-agent/libvirt.pos
   tag infra.libvirt
-  enable_watch_timer false
+  enable_watch_timer {{ fluentd_enable_watch_timer }}
   <parse>
     @type regexp
     expression /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?<Pid>\d+): (?<log_level>\S+) : (?<Payload>.*)?$/

ansible/roles/common/templates/conf/input/10-openvswitch.conf.j2

@@ -3,7 +3,7 @@
   path /var/log/kolla/openvswitch/ovs-vswitchd.log
   pos_file /var/run/td-agent/openvswitch.pos
   tag infra.openvswitch
-  enable_watch_timer false
+  enable_watch_timer {{ fluentd_enable_watch_timer }}
   <parse>
     @type multiline
     format_firstline /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}/
@@ -18,7 +18,7 @@
   path /var/log/kolla/openvswitch/ovsdb-server.log
   pos_file /var/run/td-agent/openvswitchdb.pos
   tag infra.openvswitchdb
-  enable_watch_timer false
+  enable_watch_timer {{ fluentd_enable_watch_timer }}
   <parse>
     @type multiline
     format_firstline /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}/

releasenotes/notes/add-parameter-fluentd-enable-watch-timer-08fbe65ba2b549e8.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    With the boolean parameter ``fluentd_enable_watch_timer`` it is
+    now possible to enable the additional watch timer of Fluentd.
+
+    The default value of ``fluentd_enable_watch_timer`` is set
+    to ``false``.
+
+    More details about the watch timer in Fluentd can be found at
+    https://docs.fluentd.org/input/tail#enable_watch_timer.