diff --git a/ansible/roles/common/templates/conf/input/00-global.conf.j2 b/ansible/roles/common/templates/conf/input/00-global.conf.j2 index d4469c71b7..af6a4fedb8 100644 --- a/ansible/roles/common/templates/conf/input/00-global.conf.j2 +++ b/ansible/roles/common/templates/conf/input/00-global.conf.j2 @@ -13,13 +13,15 @@ "/var/log/kolla/*/*_error.log"] pos_file /var/run/td-agent/kolla-openstack.pos tag kolla.* - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L ignore_repeated_permission_error true enable_watch_timer false + + @type multiline + format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ + format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ + time_key Timestamp + keep_time_key true + time_format %F %T.%L + {%- endif %} diff --git a/ansible/roles/common/templates/conf/input/01-syslog.conf.j2 b/ansible/roles/common/templates/conf/input/01-syslog.conf.j2 index 4f00792f10..237905cc39 100644 --- a/ansible/roles/common/templates/conf/input/01-syslog.conf.j2 +++ b/ansible/roles/common/templates/conf/input/01-syslog.conf.j2 @@ -5,5 +5,8 @@ tag syslog severity_key log_level source_hostname_key Hostname - format /^(?.*)$/ + + @type regexp + expression /^(?.*)$/ + diff --git a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 index 03726607a5..f4ed045f81 100644 --- a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 +++ b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 @@ -5,22 +5,26 @@ path /var/log/kolla/mariadb/mariadb.log pos_file /var/run/td-agent/mariadb.pos tag infra.mariadb - format multiline - format_firstline /^(\d{4}-\d{2}-\d{2}|\d{6}) / - format1 /^(?.*)/ enable_watch_timer false + + @type multiline + format_firstline /^(\d{4}-\d{2}-\d{2}|\d{6}) / + format1 /^(?.*)/ + @type tail path /var/log/kolla/mariadb/xinetd.log pos_file /var/run/td-agent/mariadb-xinetd.pos tag infra.mariadb-xinetd - format multiline - format_firstline /^\d{2}/\d{1,2}/\d{1,2}@\d{1,2}:\d{1,2}:\d{1,2}\: (START|EXIT)\: / - format1 /^(?\S+) (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %y/%m/%d@%T ignore_repeated_permission_error true enable_watch_timer false + + @type multiline + format_firstline /^\d{2}/\d{1,2}/\d{1,2}@\d{1,2}:\d{1,2}:\d{1,2}\: (START|EXIT)\: / + format1 /^(?\S+) (?.*)?$/ + time_key Timestamp + keep_time_key true + time_format %y/%m/%d@%T + diff --git a/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 b/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 index 5846547079..e0e74b1fea 100644 --- a/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 +++ b/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 @@ -3,8 +3,10 @@ path /var/log/kolla/rabbitmq/rabbit@{{ ansible_facts.hostname }}.log pos_file /var/run/td-agent/rabbit.pos tag infra.rabbit - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}/ - format1 /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \[(?\w+)\] (?.*)/ enable_watch_timer false + + @type multiline + format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}/ + format1 /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \[(?\w+)\] (?.*)/ + diff --git a/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 b/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 index 73977d8cc0..f94edf8a6a 100644 --- a/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 +++ b/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 @@ -4,6 +4,9 @@ path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log,/var/log/kolla/*/*_access.log,/var/log/kolla/*/*_error.log pos_file /var/run/td-agent/kolla-openstack-wsgi.pos tag kolla.* - format /^(?.*)$/ enable_watch_timer false + + @type regexp + expression /^(?.*)$/ + diff --git a/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 b/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 index b9713df78a..249e52b214 100644 --- a/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 +++ b/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 @@ -3,8 +3,11 @@ path /var/log/kolla/libvirt/libvirtd.log pos_file /var/run/td-agent/libvirt.pos tag infra.libvirt - format /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?\d+): (?\S+) : (?.*)?$/ - time_key Timestamp - time_format %F %T.%L%z enable_watch_timer false + + @type regexp + expression /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?\d+): (?\S+) : (?.*)?$/ + time_key Timestamp + time_format %F %T.%L%z + diff --git a/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 index 061dad8dd6..3db0ea0a22 100644 --- a/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 +++ b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 @@ -3,8 +3,10 @@ path /var/log/kolla/zookeeper/zookeeper.log pos_file /var/run/td-agent/zookeeper.pos tag infra.* - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/ - format1 /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) \[(?\S+)\] \S+ (?\S+) (?.*)$/ - time_key Timestamp + + @type multiline + format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/ + format1 /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) \[(?\S+)\] \S+ (?\S+) (?.*)$/ + time_key Timestamp + diff --git a/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 b/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 index a72db20a4d..a80dd0a00c 100644 --- a/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 +++ b/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 @@ -3,8 +3,10 @@ path /var/log/kolla/kafka/controller.log, /var/log/kolla/kafka/server.log, /var/log/kolla/kafka/state-change.log pos_file /var/run/td-agent/kafka.pos tag infra.* - format multiline - format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/ - format1 /^\[(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\] (?\S+) (?.*)$/ - time_key Timestamp + + @type multiline + format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/ + format1 /^\[(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\] (?\S+) (?.*)$/ + time_key Timestamp + diff --git a/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 b/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 index cc58d1b0d2..ede130cabb 100644 --- a/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 +++ b/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 @@ -3,8 +3,10 @@ path /var/log/kolla/monasca/agent*.log pos_file /var/run/td-agent/monasca-agent.pos tag kolla.* - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/ - format1 /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+) \| (?\S+) \| (?\S+) \| (?.*)$/ - time_key Timestamp + + @type multiline + format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/ + format1 /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+) \| (?\S+) \| (?\S+) \| (?.*)$/ + time_key Timestamp + diff --git a/ansible/roles/common/templates/conf/input/10-openvswitch.conf.j2 b/ansible/roles/common/templates/conf/input/10-openvswitch.conf.j2 index 41b640a651..f08272bdf9 100644 --- a/ansible/roles/common/templates/conf/input/10-openvswitch.conf.j2 +++ b/ansible/roles/common/templates/conf/input/10-openvswitch.conf.j2 @@ -3,12 +3,14 @@ path /var/log/kolla/openvswitch/ovs-vswitchd.log pos_file /var/run/td-agent/openvswitch.pos tag infra.openvswitch - format multiline - format_firstline /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}/ - format1 /^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})Z\|\d{5,}\|(?\S+)\|(?\S+)\|(?.*)/ - time_key Timestamp - time_format %FT%T.%L enable_watch_timer false + + @type multiline + format_firstline /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}/ + format1 /^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})Z\|\d{5,}\|(?\S+)\|(?\S+)\|(?.*)/ + time_key Timestamp + time_format %FT%T.%L + @@ -16,10 +18,12 @@ path /var/log/kolla/openvswitch/ovsdb-server.log pos_file /var/run/td-agent/openvswitchdb.pos tag infra.openvswitchdb - format multiline - format_firstline /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}/ - format1 /^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})Z\|\d{5,}\|(?\S+)\|(?\S+)\|(?.*)/ - time_key Timestamp - time_format %FT%T.%L enable_watch_timer false + + @type multiline + format_firstline /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}/ + format1 /^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})Z\|\d{5,}\|(?\S+)\|(?\S+)\|(?.*)/ + time_key Timestamp + time_format %FT%T.%L + diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2 index 2f628f8111..78ce43177c 100644 --- a/ansible/roles/common/templates/conf/output/00-local.conf.j2 +++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2 @@ -36,12 +36,12 @@ {% endif %} logstash_format true logstash_prefix {{ kibana_log_prefix }} - flush_interval 15s reconnect_on_error true suppress_type_name true @type file path /var/lib/fluentd/data/elasticsearch.buffer/{{ item.facility }}.* + flush_interval 15s {% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %} diff --git a/ansible/roles/common/templates/conf/output/01-es.conf.j2 b/ansible/roles/common/templates/conf/output/01-es.conf.j2 index dd653d0830..3affbf8f1a 100644 --- a/ansible/roles/common/templates/conf/output/01-es.conf.j2 +++ b/ansible/roles/common/templates/conf/output/01-es.conf.j2 @@ -21,12 +21,12 @@ {% endif %} logstash_format true logstash_prefix {{ kibana_log_prefix }} - flush_interval 15s reconnect_on_error true suppress_type_name true @type file path /var/lib/fluentd/data/elasticsearch.buffer/openstack.* + flush_interval 15s