From 9b52728e4fb0224ccaa475d4e665d6c2041d0e98 Mon Sep 17 00:00:00 2001 From: jiasirui Date: Fri, 9 Sep 2022 16:23:29 +0800 Subject: [PATCH] depandance the module collect logs Change-Id: If03b294d682c384d0d6482fa64cc1df2e6b05052 --- .../filter/00-record_transformer.conf | 9 +---- devstack/fluentd-conf/input/barbican.conf | 32 +++++++++++++----- devstack/fluentd-conf/input/cinder.conf | 31 ++++++++++++----- devstack/fluentd-conf/input/cyborg.conf | 32 +++++++++++++----- devstack/fluentd-conf/input/designate.conf | 31 ++++++++++++----- devstack/fluentd-conf/input/glance.conf | 32 +++++++++++++----- devstack/fluentd-conf/input/heat.conf | 31 ++++++++++++----- devstack/fluentd-conf/input/ironic.conf | 32 +++++++++++++----- devstack/fluentd-conf/input/keystone.conf | 31 ++++++++++++----- devstack/fluentd-conf/input/magnum.conf | 30 +++++++++++++---- devstack/fluentd-conf/input/manila.conf | 31 ++++++++++++----- devstack/fluentd-conf/input/neutron.conf | 31 ++++++++++++----- devstack/fluentd-conf/input/nova.conf | 32 +++++++++++++----- devstack/fluentd-conf/input/octavia.conf | 31 ++++++++++++----- devstack/fluentd-conf/input/palcement.conf | 31 ++++++++++++----- devstack/fluentd-conf/input/senlin.conf | 32 +++++++++++++----- devstack/fluentd-conf/input/swift.conf | 32 +++++++++++++----- devstack/fluentd-conf/input/syslog.conf | 2 +- devstack/fluentd-conf/input/venus.conf | 30 +++++++++++++++++ devstack/fluentd-conf/input/vitrage.conf | 31 ++++++++++++----- devstack/fluentd-conf/input/zun.conf | 33 ++++++++++++++----- .../{00-local.conf.back => 00-local.conf} | 0 devstack/fluentd-conf/output/01-es.conf | 32 ++++++++++-------- requirements.txt | 1 + 24 files changed, 474 insertions(+), 166 deletions(-) create mode 100644 devstack/fluentd-conf/input/venus.conf rename devstack/fluentd-conf/output/{00-local.conf.back => 00-local.conf} (100%) diff --git a/devstack/fluentd-conf/filter/00-record_transformer.conf b/devstack/fluentd-conf/filter/00-record_transformer.conf index 34f316d..139597f 100644 --- a/devstack/fluentd-conf/filter/00-record_transformer.conf +++ b/devstack/fluentd-conf/filter/00-record_transformer.conf @@ -1,9 +1,2 @@ - - @type record_transformer - - Hostname ${hostname} - Logger openstack.${tag_parts[4]} - programname ${tag_parts[5]} - - + diff --git a/devstack/fluentd-conf/input/barbican.conf b/devstack/fluentd-conf/input/barbican.conf index 94323fc..f9f3c54 100644 --- a/devstack/fluentd-conf/input/barbican.conf +++ b/devstack/fluentd-conf/input/barbican.conf @@ -2,13 +2,29 @@ @type tail path /var/log/barbican/*.log pos_file /var/run/td-agent/devstack-openstack_barbican.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-barbican + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - \ No newline at end of file + + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger barbican + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/cinder.conf b/devstack/fluentd-conf/input/cinder.conf index c22e728..3d87c7e 100644 --- a/devstack/fluentd-conf/input/cinder.conf +++ b/devstack/fluentd-conf/input/cinder.conf @@ -2,14 +2,29 @@ @type tail path /var/log/cinder/*.log pos_file /var/run/td-agent/devstack-openstack_cinder.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-cinder + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger cinder + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/cyborg.conf b/devstack/fluentd-conf/input/cyborg.conf index dbfe877..4a06ca0 100644 --- a/devstack/fluentd-conf/input/cyborg.conf +++ b/devstack/fluentd-conf/input/cyborg.conf @@ -2,13 +2,29 @@ @type tail path /var/log/cyborg/*.log pos_file /var/run/td-agent/devstack-openstack_cyborg.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-cyborg + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - \ No newline at end of file + + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger cyborg + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/designate.conf b/devstack/fluentd-conf/input/designate.conf index 88de798..665c705 100644 --- a/devstack/fluentd-conf/input/designate.conf +++ b/devstack/fluentd-conf/input/designate.conf @@ -2,14 +2,29 @@ @type tail path /var/log/designate/*.log pos_file /var/run/td-agent/devstack-openstack_designate.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-designate + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger designate + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/glance.conf b/devstack/fluentd-conf/input/glance.conf index 4bed3e8..81fc1e7 100644 --- a/devstack/fluentd-conf/input/glance.conf +++ b/devstack/fluentd-conf/input/glance.conf @@ -2,13 +2,29 @@ @type tail path /var/log/glance/*.log pos_file /var/run/td-agent/devstack-openstack_glance.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-glance + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - \ No newline at end of file + + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger glance + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/heat.conf b/devstack/fluentd-conf/input/heat.conf index 66731ae..7152786 100644 --- a/devstack/fluentd-conf/input/heat.conf +++ b/devstack/fluentd-conf/input/heat.conf @@ -2,14 +2,29 @@ @type tail path /var/log/heat/*.log pos_file /var/run/td-agent/devstack-openstack_heat.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-heat + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger heat + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/ironic.conf b/devstack/fluentd-conf/input/ironic.conf index 99bfd87..eccc51e 100644 --- a/devstack/fluentd-conf/input/ironic.conf +++ b/devstack/fluentd-conf/input/ironic.conf @@ -2,13 +2,29 @@ @type tail path /var/log/ironic/*.log pos_file /var/run/td-agent/devstack-openstack_ironic.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-ironic + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - \ No newline at end of file + + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger ironic + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/keystone.conf b/devstack/fluentd-conf/input/keystone.conf index 8cab93e..d1b68aa 100644 --- a/devstack/fluentd-conf/input/keystone.conf +++ b/devstack/fluentd-conf/input/keystone.conf @@ -2,14 +2,29 @@ @type tail path /var/log/keystone/*.log pos_file /var/run/td-agent/devstack-openstack_keystone.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-keystone + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger keystone + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/magnum.conf b/devstack/fluentd-conf/input/magnum.conf index 47b3f2e..0feab98 100644 --- a/devstack/fluentd-conf/input/magnum.conf +++ b/devstack/fluentd-conf/input/magnum.conf @@ -2,13 +2,29 @@ @type tail path /var/log/magnum/*.log pos_file /var/run/td-agent/devstack-openstack_magnum.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-magnum + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger magnum + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/manila.conf b/devstack/fluentd-conf/input/manila.conf index 74d6453..023b57c 100644 --- a/devstack/fluentd-conf/input/manila.conf +++ b/devstack/fluentd-conf/input/manila.conf @@ -2,14 +2,29 @@ @type tail path /var/log/manila/*.log pos_file /var/run/td-agent/devstack-openstack_manila.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-manila + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger manila + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/neutron.conf b/devstack/fluentd-conf/input/neutron.conf index 4defc94..912a79f 100644 --- a/devstack/fluentd-conf/input/neutron.conf +++ b/devstack/fluentd-conf/input/neutron.conf @@ -2,14 +2,29 @@ @type tail path /var/log/neutron/*.log pos_file /var/run/td-agent/devstack-openstack_neutron.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-neutron + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger neutron + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/nova.conf b/devstack/fluentd-conf/input/nova.conf index 948ff36..f6b549a 100644 --- a/devstack/fluentd-conf/input/nova.conf +++ b/devstack/fluentd-conf/input/nova.conf @@ -2,13 +2,29 @@ @type tail path /var/log/nova/*.log pos_file /var/run/td-agent/devstack-openstack_nova.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-nova + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - \ No newline at end of file + + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger nova + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/octavia.conf b/devstack/fluentd-conf/input/octavia.conf index 9302444..7493824 100644 --- a/devstack/fluentd-conf/input/octavia.conf +++ b/devstack/fluentd-conf/input/octavia.conf @@ -2,14 +2,29 @@ @type tail path /var/log/octavia/*.log pos_file /var/run/td-agent/devstack-openstack_octavia.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-octavia + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger octavia + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/palcement.conf b/devstack/fluentd-conf/input/palcement.conf index 43446d5..0417c6d 100644 --- a/devstack/fluentd-conf/input/palcement.conf +++ b/devstack/fluentd-conf/input/palcement.conf @@ -2,14 +2,29 @@ @type tail path /var/log/placement/*.log pos_file /var/run/td-agent/devstack-openstack_placement.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-placement + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger placement + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/senlin.conf b/devstack/fluentd-conf/input/senlin.conf index 05726fb..342db9f 100644 --- a/devstack/fluentd-conf/input/senlin.conf +++ b/devstack/fluentd-conf/input/senlin.conf @@ -2,13 +2,29 @@ @type tail path /var/log/senlin/*.log pos_file /var/run/td-agent/devstack-openstack_senlin.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-senlin + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - \ No newline at end of file + + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger senlin + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/swift.conf b/devstack/fluentd-conf/input/swift.conf index de99069..51e7f3a 100644 --- a/devstack/fluentd-conf/input/swift.conf +++ b/devstack/fluentd-conf/input/swift.conf @@ -2,13 +2,29 @@ @type tail path /var/log/swift/*.log pos_file /var/run/td-agent/devstack-openstack_swift.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-swift + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - \ No newline at end of file + + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger swift + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/syslog.conf b/devstack/fluentd-conf/input/syslog.conf index 7e5d365..f76b986 100644 --- a/devstack/fluentd-conf/input/syslog.conf +++ b/devstack/fluentd-conf/input/syslog.conf @@ -1,8 +1,8 @@ @type tail + tag syslog.* path /var/log/bootstrap.log,/var/log/btmp,/var/log/syslog pos_file /var/run/td-agent/devstack-syslog.pos - tag syslog.* source_hostname_key Hostname format /^(?.*)$/ enable_watch_timer false diff --git a/devstack/fluentd-conf/input/venus.conf b/devstack/fluentd-conf/input/venus.conf new file mode 100644 index 0000000..09566aa --- /dev/null +++ b/devstack/fluentd-conf/input/venus.conf @@ -0,0 +1,30 @@ + + @type tail + path /var/log/venus/*.log + pos_file /var/run/td-agent/devstack-openstack_venus.pos + tag devstack-venus + path_key module + + @type none + + read_from_head true + ignore_repeated_permission_error true + enable_watch_timer false + + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger venus + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/vitrage.conf b/devstack/fluentd-conf/input/vitrage.conf index e36722a..cae95f7 100644 --- a/devstack/fluentd-conf/input/vitrage.conf +++ b/devstack/fluentd-conf/input/vitrage.conf @@ -2,14 +2,29 @@ @type tail path /var/log/vitrage/*.log pos_file /var/run/td-agent/devstack-openstack_vitrage.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-vitrage + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger vitrage + programname ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/input/zun.conf b/devstack/fluentd-conf/input/zun.conf index 4196c0c..8432152 100644 --- a/devstack/fluentd-conf/input/zun.conf +++ b/devstack/fluentd-conf/input/zun.conf @@ -2,13 +2,30 @@ @type tail path /var/log/zun/*.log pos_file /var/run/td-agent/devstack-openstack_zun.pos - tag devstack - format multiline - format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ - format1 /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (\[(?\S+) (req-(?\S+) (?\S+) (?\S+) (?\S+) (?\S+) (?\S+)|-)\])? (?.*)?$/ - time_key Timestamp - keep_time_key true - time_format %F %T.%L + tag devstack-zun + path_key module + + @type none + + read_from_head true ignore_repeated_permission_error true enable_watch_timer false - \ No newline at end of file + + + @type record_transformer + enable_ruby true + + Hostname ${hostname} + Logger zun + programname zun + Logfile ${record['module']} + + + + @type parser + key_name message + format /^(?\S+ \S+) (?\d+) (?\S+) (?\S+) (?.*)?$/ + time_format %F %T.%L + reserve_data true + emit_invalid_record_to_error false + \ No newline at end of file diff --git a/devstack/fluentd-conf/output/00-local.conf.back b/devstack/fluentd-conf/output/00-local.conf similarity index 100% rename from devstack/fluentd-conf/output/00-local.conf.back rename to devstack/fluentd-conf/output/00-local.conf diff --git a/devstack/fluentd-conf/output/01-es.conf b/devstack/fluentd-conf/output/01-es.conf index 516905d..94226c1 100644 --- a/devstack/fluentd-conf/output/01-es.conf +++ b/devstack/fluentd-conf/output/01-es.conf @@ -1,18 +1,22 @@ - + @type copy - @type elasticsearch - host localhost - port 9200 - logstash_format true - logstash_prefix flog - buffer_type memory - buffer_chunk_limit 8m - buffer_queue_limit 8 - buffer_queue_full_action drop_oldest_chunk - flush_interval 30s - num_threads 4 - reconnect_on_error true - resurrect_after 5s + @type elasticsearch + host localhost + port 9200 + logstash_format true + logstash_prefix "flog" + flush_interval 3s + reconnect_on_error true + resurrect_after 5s + + @type memory + flush_thread_count 4 + flush_interval 30s + chunk_limit_size 2m + queue_limit_length 8 + overflow_action drop_oldest_chunk + + diff --git a/requirements.txt b/requirements.txt index af7dac4..83305e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,3 +33,4 @@ openstacksdk>=0.46.0 # Apache-2.0 elasticsearch>=2.0.0,<3.0.0 # Apache-2.0 apscheduler>=3.6.0 # MIT License retrying>=1.3.3 +lxml>=4.6.2