fluentd: remove deprecated parameter

In fluentd v1.0, "format" became a deprecated parameter,
replaced by "<parse>"

Change-Id: I0825e81fcd90fbc7f64c8df3ea9ae199ad79294a

ansible/roles/common/templates/conf/input/00-global.conf.j2

@@ -13,13 +13,15 @@
                 "/var/log/kolla/*/*_error.log"]
   pos_file /var/run/td-agent/kolla-openstack.pos
   tag kolla.*
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
   ignore_repeated_permission_error true
   enable_watch_timer false
+  <parse>
+    @type multiline
+    format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
+    format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
+    time_key Timestamp
+    keep_time_key true
+    time_format %F %T.%L
+  </parse>
 </source>
 {%- endif %}

ansible/roles/common/templates/conf/input/01-syslog.conf.j2

@@ -5,5 +5,8 @@
   tag syslog
   severity_key log_level
   source_hostname_key Hostname
-  format /^(?<Payload>.*)$/
+  <parse>
+    @type regexp
+    expression /^(?<Payload>.*)$/
+  </parse>
 </source>

ansible/roles/common/templates/conf/input/02-mariadb.conf.j2

@@ -5,22 +5,26 @@
   path /var/log/kolla/mariadb/mariadb.log
   pos_file /var/run/td-agent/mariadb.pos
   tag infra.mariadb
-  format multiline
-  format_firstline /^(\d{4}-\d{2}-\d{2}|\d{6}) /
-  format1 /^(?<Payload>.*)/
   enable_watch_timer false
+  <parse>
+    @type multiline
+    format_firstline /^(\d{4}-\d{2}-\d{2}|\d{6}) /
+    format1 /^(?<Payload>.*)/
+  </parse>
 </source>
 <source>
   @type tail
   path /var/log/kolla/mariadb/xinetd.log
   pos_file /var/run/td-agent/mariadb-xinetd.pos
   tag infra.mariadb-xinetd
-  format multiline
-  format_firstline /^\d{2}/\d{1,2}/\d{1,2}@\d{1,2}:\d{1,2}:\d{1,2}\: (START|EXIT)\: /
-  format1 /^(?<Timestamp>\S+) (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %y/%m/%d@%T
   ignore_repeated_permission_error true
   enable_watch_timer false
+  <parse>
+    @type multiline
+    format_firstline /^\d{2}/\d{1,2}/\d{1,2}@\d{1,2}:\d{1,2}:\d{1,2}\: (START|EXIT)\: /
+    format1 /^(?<Timestamp>\S+) (?<Payload>.*)?$/
+    time_key Timestamp
+    keep_time_key true
+    time_format %y/%m/%d@%T
+  </parse>
 </source>

ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2

@@ -3,8 +3,10 @@
   path /var/log/kolla/rabbitmq/rabbit@{{ ansible_facts.hostname }}.log
   pos_file /var/run/td-agent/rabbit.pos
   tag infra.rabbit
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}/
-  format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \[(?<log_level>\w+)\] (?<Payload>.*)/
   enable_watch_timer false
+  <parse>
+    @type multiline
+    format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}/
+    format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \[(?<log_level>\w+)\] (?<Payload>.*)/
+  </parse>
 </source>

ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2

@@ -4,6 +4,9 @@
   path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log,/var/log/kolla/*/*_access.log,/var/log/kolla/*/*_error.log
   pos_file /var/run/td-agent/kolla-openstack-wsgi.pos
   tag kolla.*
-  format /^(?<Payload>.*)$/
   enable_watch_timer false
+  <parse>
+    @type regexp
+    expression /^(?<Payload>.*)$/
+  </parse>
 </source>

ansible/roles/common/templates/conf/input/05-libvirt.conf.j2

@@ -3,8 +3,11 @@
   path /var/log/kolla/libvirt/libvirtd.log
   pos_file /var/run/td-agent/libvirt.pos
   tag infra.libvirt
-  format /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?<Pid>\d+): (?<log_level>\S+) : (?<Payload>.*)?$/
-  time_key Timestamp
-  time_format %F %T.%L%z
   enable_watch_timer false
+  <parse>
+    @type regexp
+    expression /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?<Pid>\d+): (?<log_level>\S+) : (?<Payload>.*)?$/
+    time_key Timestamp
+    time_format %F %T.%L%z
+  </parse>
 </source>

ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2

@@ -3,8 +3,10 @@
   path /var/log/kolla/zookeeper/zookeeper.log
   pos_file /var/run/td-agent/zookeeper.pos
   tag infra.*
-  format multiline
+  <parse>
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/
+    @type multiline
-  format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) \[(?<server_id>\S+)\] \S+ (?<log_level>\S+) (?<Payload>.*)$/
+    format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/
-  time_key Timestamp
+    format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) \[(?<server_id>\S+)\] \S+ (?<log_level>\S+) (?<Payload>.*)$/
+    time_key Timestamp
+  </parse>
 </source>

ansible/roles/common/templates/conf/input/07-kafka.conf.j2

@@ -3,8 +3,10 @@
   path /var/log/kolla/kafka/controller.log, /var/log/kolla/kafka/server.log, /var/log/kolla/kafka/state-change.log
   pos_file /var/run/td-agent/kafka.pos
   tag infra.*
-  format multiline
+  <parse>
-  format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/
+    @type multiline
-  format1 /^\[(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\] (?<log_level>\S+) (?<Payload>.*)$/
+    format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/
-  time_key Timestamp
+    format1 /^\[(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\] (?<log_level>\S+) (?<Payload>.*)$/
+    time_key Timestamp
+  </parse>
 </source>

ansible/roles/common/templates/conf/input/09-monasca.conf.j2

@@ -3,8 +3,10 @@
   path /var/log/kolla/monasca/agent*.log
   pos_file /var/run/td-agent/monasca-agent.pos
   tag kolla.*
-  format multiline
+  <parse>
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/
+    @type multiline
-  format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+) \| (?<log_level>\S+) \| (?<programname>\S+) \| (?<Payload>.*)$/
+    format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/
-  time_key Timestamp
+    format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+) \| (?<log_level>\S+) \| (?<programname>\S+) \| (?<Payload>.*)$/
+    time_key Timestamp
+  </parse>
 </source>

ansible/roles/common/templates/conf/input/10-openvswitch.conf.j2

@@ -3,12 +3,14 @@
   path /var/log/kolla/openvswitch/ovs-vswitchd.log
   pos_file /var/run/td-agent/openvswitch.pos
   tag infra.openvswitch
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}/
-  format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})Z\|\d{5,}\|(?<module>\S+)\|(?<log_level>\S+)\|(?<Payload>.*)/
-  time_key Timestamp
-  time_format %FT%T.%L
   enable_watch_timer false
+  <parse>
+    @type multiline
+    format_firstline /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}/
+    format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})Z\|\d{5,}\|(?<module>\S+)\|(?<log_level>\S+)\|(?<Payload>.*)/
+    time_key Timestamp
+    time_format %FT%T.%L
+  </parse>
 </source>
 
 <source>
@@ -16,10 +18,12 @@
   path /var/log/kolla/openvswitch/ovsdb-server.log
   pos_file /var/run/td-agent/openvswitchdb.pos
   tag infra.openvswitchdb
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}/
-  format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})Z\|\d{5,}\|(?<module>\S+)\|(?<log_level>\S+)\|(?<Payload>.*)/
-  time_key Timestamp
-  time_format %FT%T.%L
   enable_watch_timer false
+  <parse>
+    @type multiline
+    format_firstline /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}/
+    format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})Z\|\d{5,}\|(?<module>\S+)\|(?<log_level>\S+)\|(?<Payload>.*)/
+    time_key Timestamp
+    time_format %FT%T.%L
+  </parse>
 </source>

ansible/roles/common/templates/conf/output/00-local.conf.j2

@@ -36,12 +36,12 @@
 {% endif %}
        logstash_format true
        logstash_prefix {{ kibana_log_prefix }}
-       flush_interval 15s
        reconnect_on_error true
        suppress_type_name true
        <buffer>
          @type file
          path /var/lib/fluentd/data/elasticsearch.buffer/{{ item.facility }}.*
+         flush_interval 15s
        </buffer>
   </store>
 {% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}

ansible/roles/common/templates/conf/output/01-es.conf.j2

@@ -21,12 +21,12 @@
 {% endif %}
        logstash_format true
        logstash_prefix {{ kibana_log_prefix }}
-       flush_interval 15s
        reconnect_on_error true
        suppress_type_name true
        <buffer>
          @type file
          path /var/lib/fluentd/data/elasticsearch.buffer/openstack.*
+         flush_interval 15s
        </buffer>
     </store>
 </match>