From 38a6d30411be2b8bb116bfb765b054309a3a9c91 Mon Sep 17 00:00:00 2001
From: Bertrand Lallau <bertrand.lallau@thalesgroup.com>
Date: Thu, 17 Aug 2017 18:05:20 +0200
Subject: [PATCH] Fluentd: manage multiline log pattern

Actual Fluentd config doesn't manage multiline log pattern like:

2017-08-16 21:41:03.097 7 WARNING oslo_config.cfg [-] Option "firewall_driver" from group "DEFAULT" is deprecated for removal (
nova-network is deprecated, as are any related configuration options.
). Its value may be silently ignored in the future.

This log will generate 1 Fluentd event for each line (in previous
example 3 events).
This is really a wrong behaviour and made Kibana interface not friendly.
This patch uses a multiline parser.

Change-Id: Iee337645d1dc42876ec79be08ee3715f11a53d87
Closes-Bug: #1711399
---
 ansible/roles/common/tasks/config.yml                      | 1 -
 .../roles/common/templates/conf/input/00-global.conf.j2    | 7 ++++++-
 ansible/roles/common/templates/fluentd.json.j2             | 6 ------
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index 27024db028..8eccc94f20 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -60,7 +60,6 @@
   with_items:
     - "apache_access"
     - "wsgi_access"
-    - "wsgi_python"
   notify:
     - Restart fluentd container
 
diff --git a/ansible/roles/common/templates/conf/input/00-global.conf.j2 b/ansible/roles/common/templates/conf/input/00-global.conf.j2
index b1fa69d92f..013785250a 100644
--- a/ansible/roles/common/templates/conf/input/00-global.conf.j2
+++ b/ansible/roles/common/templates/conf/input/00-global.conf.j2
@@ -45,5 +45,10 @@
                 "/var/log/kolla/*/*-error.log"]
   pos_file /var/run/{{ fluentd_dir }}/kolla-openstack.pos
   tag kolla.*
-  format /^(?<message>.*)$/
+  format multiline
+  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
+  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
+  time_key Timestamp
+  time_format %F %T.%L
+  ignore_repeated_permission_error true
 </source>
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index 95cacc5def..b72dd226c8 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -64,12 +64,6 @@
             "dest": "{{ fluentd_dir }}/format/wsgi_access.conf",
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/format/wsgi_python.conf",
-            "dest": "{{ fluentd_dir }}/format/wsgi_python.conf",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
         },{% if enable_elasticsearch | bool or
                 ( elasticsearch_address != kolla_internal_vip_address ) | bool %}
         {