From eab91d428813b91bae37876b4fb8ada68e371271 Mon Sep 17 00:00:00 2001
From: Bertrand Lallau <bertrand.lallau@thalesgroup.com>
Date: Thu, 17 Aug 2017 17:13:38 +0200
Subject: [PATCH] Fluentd: rework and fix tail source config

Actually a unique 'tail' source is used for all logs files in
kolla/*/*.log.

So log files from services like: chrony, elasticsearch, etcd, grafana,
influxdb, kibana, mongodb... are actually processed by our Fluentd
config, but should not cause all following config will failed to parse
them. It just uses CPU for nothing cause they will
never match the following filters:
* record_transformer => add Hostname, Logger, programname field
* rewrite_tag_filter => rewrite 'programname' TAG if needed

Hence these filters should only be applied to Openstack services logs to
avoid CPU usage.

Furthermore WGI and Apache logs files must be treated differently.

Change-Id: I425979160291e1123e3775e39243bca06ed22c2f
Closes-Bug: #1711390
---
 ansible/roles/common/tasks/config.yml         |  1 +
 .../templates/conf/input/00-global.conf.j2    | 46 +++++++++++++++++--
 .../conf/input/04-openstack-wsgi.conf.j2      |  9 ++++
 .../roles/common/templates/fluentd.json.j2    | 10 +++-
 4 files changed, 61 insertions(+), 5 deletions(-)
 create mode 100644 ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2

diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index 3e10add019..27024db028 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -33,6 +33,7 @@
     - "01-syslog"
     - "02-mariadb"
     - "03-rabbitmq"
+    - "04-openstack-wsgi"
   notify:
     - Restart fluentd container
 
diff --git a/ansible/roles/common/templates/conf/input/00-global.conf.j2 b/ansible/roles/common/templates/conf/input/00-global.conf.j2
index a338ddf776..b1fa69d92f 100644
--- a/ansible/roles/common/templates/conf/input/00-global.conf.j2
+++ b/ansible/roles/common/templates/conf/input/00-global.conf.j2
@@ -1,9 +1,49 @@
+#jinja2: trim_blocks: False
 {% set fluentd_dir = 'td-agent' if kolla_base_distro in ['ubuntu', 'debian'] else 'fluentd' %}
+{% set services = [
+    ( 'aodh', enable_aodh ),
+    ( 'barbican', enable_barbican ),
+    ( 'ceilometer', enable_ceilometer ),
+    ( 'cinder', enable_cinder ),
+    ( 'cloudkitty', enable_cloudkitty ),
+    ( 'congress', enable_congress ),
+    ( 'designate', enable_designate ),
+    ( 'freezer', enable_freezer ),
+    ( 'glance', enable_glance ),
+    ( 'gnocchi', enable_gnocchi ),
+    ( 'heat', enable_heat ),
+    ( 'horizon', enable_horizon ),
+    ( 'ironic', enable_ironic ),
+    ( 'karbor', enable_karbor ),
+    ( 'keystone', enable_keystone ),
+    ( 'kuryr', enable_kuryr ),
+    ( 'magnum', enable_magnum ),
+    ( 'manila', enable_manila ),
+    ( 'mistral', enable_mistral ),
+    ( 'mongodb', enable_mongodb ),
+    ( 'murano', enable_murano ),
+    ( 'neutron', enable_neutron ),
+    ( 'nova', enable_nova ),
+    ( 'octavia', enable_octavia ),
+    ( 'opendaylight', enable_opendaylight ),
+    ( 'panko', enable_panko ),
+    ( 'rally', enable_rally ),
+    ( 'sahara', enable_sahara ),
+    ( 'searchlight', enable_searchlight ),
+    ( 'senlin', enable_senlin ),
+    ( 'solum', enable_solum ),
+    ( 'tacker', enable_tacker ),
+    ( 'trove', enable_trove ),
+    ( 'watcher', enable_watcher ),
+    ( 'zun', enable_zun )
+] %}
 <source>
   @type tail
-  path /var/log/kolla/*/*.log
-  exclude_path ["/var/log/kolla/rabbitmq/*.log", "/var/log/kolla/mariadb/mariadb.log", "/var/log/kolla/haproxy/*.log", "/var/log/kolla/swift/*.log"]
-  pos_file /var/run/{{ fluentd_dir }}/kolla.pos
+  path {% for service, enabled in services if enabled | bool %}/var/log/kolla/{{ service }}/*.log{% if not loop.last %},{% endif %}{% endfor %}
+  exclude_path ["/var/log/kolla/neutron/dnsmasq.log",
+                "/var/log/kolla/*/*-access.log",
+                "/var/log/kolla/*/*-error.log"]
+  pos_file /var/run/{{ fluentd_dir }}/kolla-openstack.pos
   tag kolla.*
   format /^(?<message>.*)$/
 </source>
diff --git a/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 b/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2
new file mode 100644
index 0000000000..10410f2715
--- /dev/null
+++ b/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2
@@ -0,0 +1,9 @@
+# Note (blallau): to manage Apache and WSGI log files
+{% set fluentd_dir = 'td-agent' if kolla_base_distro in ['ubuntu', 'debian'] else 'fluentd' %}
+<source>
+  @type tail
+  path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log
+  pos_file /var/run/{{ fluentd_dir }}/kolla-openstack-wsgi.pos
+  tag kolla.*
+  format /^(?<message>.*)$/
+</source>
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index e1c341b510..95cacc5def 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -17,6 +17,12 @@
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },
+        {
+            "source": "{{ container_config_directory }}/input/01-syslog.conf",
+            "dest": "{{ fluentd_dir }}/input/01-syslog.conf",
+            "owner": "{{ fluentd_user }}",
+            "perm": "0600"
+        },
         {
             "source": "{{ container_config_directory }}/input/02-mariadb.conf",
             "dest": "{{ fluentd_dir }}/input/02-mariadb.conf",
@@ -30,8 +36,8 @@
             "perm": "0600"
         },
         {
-            "source": "{{ container_config_directory }}/input/01-syslog.conf",
-            "dest": "{{ fluentd_dir }}/input/01-syslog.conf",
+            "source": "{{ container_config_directory }}/input/04-openstack-wsgi.conf",
+            "dest": "{{ fluentd_dir }}/input/04-openstack-wsgi.conf",
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },