Merge "Support customising Fluentd formatting"

ansible/roles/common/handlers/main.yml

@@ -23,6 +23,7 @@
       or fluentd_output.changed | bool
       or fluentd_output_custom.changed | bool
       or fluentd_format.changed | bool
+      or fluentd_format_custom.changed | bool
       or fluentd_filter.changed | bool
       or fluentd_filter_custom.changed | bool
       or fluentd_td_agent.changed | bool

ansible/roles/common/tasks/config.yml

@@ -158,6 +158,27 @@
   notify:
     - Restart fluentd container
 
+- name: Find custom fluentd format config files
+  local_action:
+    module: find
+    path: "{{ node_custom_config }}/fluentd/format"
+    pattern: "*.conf"
+  run_once: True
+  register: find_custom_fluentd_format
+  when:
+    - enable_fluentd | bool
+
+- name: Copying over custom fluentd format config files
+  template:
+    src: "{{ item.path }}"
+    dest: "{{ node_config_directory }}/fluentd/format/{{ item.path | basename }}"
+  register: fluentd_format_custom
+  when:
+    - enable_fluentd | bool
+  with_items: "{{ find_custom_fluentd_format.files }}"
+  notify:
+    - Restart fluentd container
+
 - name: Copying over fluentd filter config files
   template:
     src: "conf/filter/{{ item.src }}.conf.j2"

ansible/roles/common/templates/fluentd.json.j2

@@ -27,15 +27,11 @@
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },
+        {# Copy all configuration files in format/ directory to include #}
+        {# custom format configs. #}
         {
-            "source": "{{ container_config_directory }}/format/apache_access.conf",
+            "source": "{{ container_config_directory }}/format/*.conf",
-            "dest": "{{ fluentd_dir }}/format/apache_access.conf",
+            "dest": "{{ fluentd_dir }}/format/",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/format/wsgi_access.conf",
-            "dest": "{{ fluentd_dir }}/format/wsgi_access.conf",
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },

doc/source/reference/logging-and-monitoring/central-logging-guide.rst

@@ -182,6 +182,17 @@ Configuration of custom fluentd filters is possible by placing filter
 configuration files in ``/etc/kolla/config/fluentd/filter/*.conf`` on the
 control host.
 
+Custom log formatting
+---------------------
+
+In some scenarios it may be useful to perform custom formatting of logs before
+forwarding them. For example, the JSON formatter plugin can be used to convert
+an event to JSON.
+
+Configuration of custom fluentd formatting is possible by placing filter
+configuration files in ``/etc/kolla/config/fluentd/format/*.conf`` on the
+control host.
+
 Custom log forwarding
 ---------------------
 

releasenotes/notes/fluentd-custom-format-config-b6fc3a4883efc213.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Adds ability to configure custom fluentd formatting.
+
+    In some scenarios it may be useful to configure custom fluentd formatting
+    to, for example, convert events to JSON.
+
+    Configuration of custom fluentd formatting is possible by placing
+    output configuration files in /etc/kolla/config/fluentd/format/\*.conf.