Merge "depandance the module collect logs"

devstack/fluentd-conf/filter/00-record_transformer.conf

@@ -1,9 +1,2 @@
-<filter devstack>
-    @type record_transformer
-    <record>
-        Hostname ${hostname}
-        Logger openstack.${tag_parts[4]}
-        programname ${tag_parts[5]}
-    </record>
-</filter>
+
 

devstack/fluentd-conf/input/barbican.conf

@@ -2,13 +2,29 @@
   @type tail
   path /var/log/barbican/*.log
   pos_file /var/run/td-agent/devstack-openstack_barbican.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-barbican
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
-</source>
+</source>
+<filter devstack-barbican>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger barbican
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-barbican>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/cinder.conf

@@ -2,14 +2,29 @@
   @type tail
   path /var/log/cinder/*.log
   pos_file /var/run/td-agent/devstack-openstack_cinder.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-cinder
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
 </source>
-
+<filter devstack-cinder>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger cinder
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-cinder>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/cyborg.conf

@@ -2,13 +2,29 @@
   @type tail
   path /var/log/cyborg/*.log
   pos_file /var/run/td-agent/devstack-openstack_cyborg.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-cyborg
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
-</source>
+</source>
+<filter devstack-cyborg>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger cyborg
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-cyborg>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/designate.conf

@@ -2,14 +2,29 @@
   @type tail
   path /var/log/designate/*.log
   pos_file /var/run/td-agent/devstack-openstack_designate.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-designate
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
 </source>
-
+<filter devstack-designate>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger designate
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-designate>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/glance.conf

@@ -2,13 +2,29 @@
   @type tail
   path /var/log/glance/*.log
   pos_file /var/run/td-agent/devstack-openstack_glance.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-glance
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
-</source>
+</source>
+<filter devstack-glance>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger glance
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-glance>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/heat.conf

@@ -2,14 +2,29 @@
   @type tail
   path /var/log/heat/*.log
   pos_file /var/run/td-agent/devstack-openstack_heat.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-heat
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
 </source>
-
+<filter devstack-heat>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger heat
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-heat>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/ironic.conf

@@ -2,13 +2,29 @@
   @type tail
   path /var/log/ironic/*.log
   pos_file /var/run/td-agent/devstack-openstack_ironic.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-ironic
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
-</source>
+</source>
+<filter devstack-ironic>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger ironic
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-ironic>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/keystone.conf

@@ -2,14 +2,29 @@
   @type tail
   path /var/log/keystone/*.log
   pos_file /var/run/td-agent/devstack-openstack_keystone.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-keystone
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
 </source>
-
+<filter devstack-keystone>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger keystone
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-keystone>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/magnum.conf

@@ -2,13 +2,29 @@
   @type tail
   path /var/log/magnum/*.log
   pos_file /var/run/td-agent/devstack-openstack_magnum.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-magnum
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
 </source>
+<filter devstack-magnum>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger magnum
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-magnum>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/manila.conf

@@ -2,14 +2,29 @@
   @type tail
   path /var/log/manila/*.log
   pos_file /var/run/td-agent/devstack-openstack_manila.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-manila
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
 </source>
-
+<filter devstack-manila>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger manila
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-manila>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/neutron.conf

@@ -2,14 +2,29 @@
   @type tail
   path /var/log/neutron/*.log
   pos_file /var/run/td-agent/devstack-openstack_neutron.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-neutron
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
 </source>
-
+<filter devstack-neutron>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger neutron
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-neutron>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/nova.conf

@@ -2,13 +2,29 @@
   @type tail
   path /var/log/nova/*.log
   pos_file /var/run/td-agent/devstack-openstack_nova.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-nova
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
-</source>
+</source>
+<filter devstack-nova>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger nova
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-nova>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/octavia.conf

@@ -2,14 +2,29 @@
   @type tail
   path /var/log/octavia/*.log
   pos_file /var/run/td-agent/devstack-openstack_octavia.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-octavia
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
 </source>
-
+<filter devstack-octavia>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger octavia
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-octavia>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/palcement.conf

@@ -2,14 +2,29 @@
   @type tail
   path /var/log/placement/*.log
   pos_file /var/run/td-agent/devstack-openstack_placement.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-placement
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
 </source>
-
+<filter devstack-placement>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger placement
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-placement>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/senlin.conf

@@ -2,13 +2,29 @@
   @type tail
   path /var/log/senlin/*.log
   pos_file /var/run/td-agent/devstack-openstack_senlin.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-senlin
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
-</source>
+</source>
+<filter devstack-senlin>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger senlin
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-senlin>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/swift.conf

@@ -2,13 +2,29 @@
   @type tail
   path /var/log/swift/*.log
   pos_file /var/run/td-agent/devstack-openstack_swift.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-swift
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
-</source>
+</source>
+<filter devstack-swift>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger swift
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-swift>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/syslog.conf

@@ -1,8 +1,8 @@
 <source>
   @type tail
+  tag syslog.*
   path /var/log/bootstrap.log,/var/log/btmp,/var/log/syslog
   pos_file /var/run/td-agent/devstack-syslog.pos
-  tag syslog.*
   source_hostname_key Hostname
   format /^(?<Payload>.*)$/
   enable_watch_timer false

devstack/fluentd-conf/input/venus.conf

@@ -0,0 +1,30 @@
+<source>
+  @type tail
+  path /var/log/venus/*.log
+  pos_file /var/run/td-agent/devstack-openstack_venus.pos
+  tag devstack-venus
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
+  ignore_repeated_permission_error true
+  enable_watch_timer false
+</source>
+<filter devstack-venus>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger venus
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-venus>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/vitrage.conf

@@ -2,14 +2,29 @@
   @type tail
   path /var/log/vitrage/*.log
   pos_file /var/run/td-agent/devstack-openstack_vitrage.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-vitrage
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
 </source>
-
+<filter devstack-vitrage>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger vitrage
+        programname ${record['module']}
+    </record>
+</filter>
+<filter devstack-vitrage>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/input/zun.conf

@@ -2,13 +2,30 @@
   @type tail
   path /var/log/zun/*.log
   pos_file /var/run/td-agent/devstack-openstack_zun.pos
-  tag devstack
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
-  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
-  time_key Timestamp
-  keep_time_key true
-  time_format %F %T.%L
+  tag devstack-zun
+  path_key module
+  <parse>
+     @type none
+  </parse>
+  read_from_head true
   ignore_repeated_permission_error true
   enable_watch_timer false
-</source>
+</source>
+<filter devstack-zun>
+    @type record_transformer
+    enable_ruby true
+    <record>
+        Hostname ${hostname}
+        Logger zun
+        programname zun
+        Logfile ${record['module']}
+    </record>
+</filter>
+<filter devstack-zun>
+    @type parser
+    key_name message
+    format /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (?<Payload>.*)?$/
+    time_format %F %T.%L
+    reserve_data true
+    emit_invalid_record_to_error false
+</filter>

devstack/fluentd-conf/output/01-es.conf

@@ -1,18 +1,22 @@
-<match devstack>
+<match devstack*>
     @type copy
     <store>
-       @type elasticsearch
-       host localhost
-       port 9200
-       logstash_format true
-       logstash_prefix flog
-       buffer_type memory
-       buffer_chunk_limit 8m
-       buffer_queue_limit 8
-       buffer_queue_full_action drop_oldest_chunk
-       flush_interval 30s
-       num_threads 4
-       reconnect_on_error true
-       resurrect_after 5s
+      @type elasticsearch
+      host localhost
+      port 9200
+      logstash_format true
+      logstash_prefix "flog"
+      flush_interval 3s
+      reconnect_on_error true
+      resurrect_after 5s
+      <buffer>
+          @type memory
+          flush_thread_count 4
+          flush_interval 30s
+          chunk_limit_size 2m
+          queue_limit_length 8
+          overflow_action drop_oldest_chunk
+      </buffer>
     </store>
 </match>
+

requirements.txt

@@ -33,3 +33,4 @@ openstacksdk>=0.46.0 # Apache-2.0
 elasticsearch>=2.0.0,<3.0.0  # Apache-2.0
 apscheduler>=3.6.0 # MIT License
 retrying>=1.3.3
+lxml>=4.6.2