Merge "Fluentd: Allow for Multiple Config Files"

fluentd/templates/bin/_fluentd.sh.tpl

@@ -18,7 +18,7 @@ set -ex
 COMMAND="${@:-start}"
 
 function start () {
-  exec fluentd -c /fluentd/etc/fluent.conf
+  exec fluentd -c /fluentd/etc/main.conf
 }
 
 function stop () {

fluentd/templates/configmap-etc.yaml

@@ -12,8 +12,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- define "fluentd_main" }}
+{{- $path := .Values.conf.fluentd.path}}
+{{- range $name, $conf := .Values.conf.fluentd.conf }}
+{{ printf "%s %s/%s.conf" "@include" $path $name | indent 4}}
+{{- end }}
+{{- end }}
+
 {{- if .Values.manifests.configmap_etc }}
-{{- $envAll := . }}
+{{ $envAll := .}}
 ---
 apiVersion: v1
 kind: Secret
@@ -22,6 +29,12 @@ metadata:
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
 type: Opaque
+stringData:
+  main.conf: |
+{{- template "fluentd_main" . }}
 data:
-{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.fluentd.template "key" "fluent.conf" "format" "Secret") | indent 2 }}
+{{- range $name, $config := .Values.conf.fluentd.conf }}
+{{- $filename := printf "%s.conf" $name}}
+{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" $config "key" $filename "format" "Secret") | indent 2 }}
+{{- end }}
 {{- end }}

fluentd/templates/daemonset.yaml

@@ -15,6 +15,7 @@ limitations under the License.
 {{- if .Values.manifests.daemonset }}
 {{- $envAll := . }}
 
+{{- $config_path := .Values.conf.fluentd.path }}
 {{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }}
 {{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.fluentd }}
 
@@ -183,9 +184,15 @@ spec:
             - name: pod-etc-fluentd
               mountPath: /fluentd/etc
             - name: fluentd-etc
-              mountPath: /fluentd/etc/fluent.conf
+              mountPath: {{ printf "%s/%s.conf" $config_path "main" }}
-              subPath: fluent.conf
+              subPath: {{ printf "%s.conf" "main"}}
               readOnly: true
+{{- range $name, $config := .Values.conf.fluentd.conf }}
+            - name: fluentd-etc
+              mountPath: {{ printf "%s/%s.conf" $config_path $name }}
+              subPath: {{ printf "%s.conf" $name }}
+              readOnly: true
+{{- end }}
             - name: fluentd-bin
               mountPath: /tmp/fluentd.sh
               subPath: fluentd.sh

fluentd/values.yaml

@@ -54,227 +54,36 @@ dependencies:
 
 conf:
   fluentd:
-    # This field is now rendered as a helm template!
+    path: /fluentd/etc
-    template: |
+    conf:
-      <source>
+      input: |
-        @type prometheus
-        port {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-      </source>
-
-      <source>
-        @type prometheus_monitor
-      </source>
-
-      <source>
-        @type prometheus_output_monitor
-      </source>
-
-      <source>
-        @type prometheus_tail_monitor
-      </source>
-
         <source>
           bind 0.0.0.0
           port "#{ENV['FLUENTD_PORT']}"
           @type forward
         </source>
-
+        <source>
-      <match fluent.**>
+          <parse>
-        @type null
+            time_format %Y-%m-%dT%H:%M:%S.%NZ
+            @type json
+          </parse>
+          path /var/log/containers/*.log
+          read_from_head true
+          tag kubernetes.*
+          @type tail
+        </source>
+        <match **>
+          @type relabel
+          @label @output
         </match>
-
+      output: |
-      <match kube.var.log.containers.**.log>
+        <label @output>
-        <rule>
-          key log
-          pattern /info/i
-          tag info.${tag}
-        </rule>
-        <rule>
-          key log
-          pattern /warn/i
-          tag warn.${tag}
-        </rule>
-        <rule>
-          key log
-          pattern /error/i
-          tag error.${tag}
-        </rule>
-        <rule>
-          key log
-          pattern /critical/i
-          tag critical.${tag}
-        </rule>
-        <rule>
-          key log
-          pattern (.+)
-          tag info.${tag}
-        </rule>
-        @type rewrite_tag_filter
-      </match>
-
-      <filter **.kube.var.log.containers.**.log>
-        enable_ruby true
-        <record>
-          application ${record["kubernetes"]["labels"]["application"]}
-          level ${tag_parts[0]}
-        </record>
-        @type record_transformer
-      </filter>
-
-      <filter openstack.**>
-        <record>
-          application ${tag_parts[1]}
-        </record>
-        @type record_transformer
-      </filter>
-
-      <match openstack.**>
-        <rule>
-          key level
-          pattern INFO
-          tag info.${tag}
-        </rule>
-        <rule>
-          key level
-          pattern WARN
-          tag warn.${tag}
-        </rule>
-        <rule>
-          key level
-          pattern ERROR
-          tag error.${tag}
-        </rule>
-        <rule>
-          key level
-          pattern CRITICAL
-          tag critical.${tag}
-        </rule>
-        @type rewrite_tag_filter
-      </match>
-
-      <match *.openstack.**>
-        <rule>
-          key application
-          pattern keystone
-          tag auth.${tag}
-        </rule>
-        <rule>
-          key application
-          pattern horizon
-          tag auth.${tag}
-        </rule>
-        <rule>
-          key application
-          pattern mariadb
-          tag auth.${tag}
-        </rule>
-        <rule>
-          key application
-          pattern memcached
-          tag auth.${tag}
-        </rule>
-        <rule>
-          key application
-          pattern rabbitmq
-          tag auth.${tag}
-        </rule>
-        @type rewrite_tag_filter
-      </match>
-
-      <match libvirt>
-        <buffer>
-          chunk_limit_size 8MB
-          flush_interval 15s
-          flush_thread_count 8
-          queue_limit_length 256
-          retry_forever false
-          retry_max_interval 30
-        </buffer>
-        host "#{ENV['ELASTICSEARCH_HOST']}"
-        reload_connections false
-        reconnect_on_error true
-        reload_on_failure true
-        include_tag_key true
-        logstash_format true
-        logstash_prefix libvirt
-        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
-        port "#{ENV['ELASTICSEARCH_PORT']}"
-        @type elasticsearch
-        user "#{ENV['ELASTICSEARCH_USERNAME']}"
-      </match>
-
-      <match qemu>
-        <buffer>
-          chunk_limit_size 8MB
-          flush_interval 15s
-          flush_thread_count 8
-          queue_limit_length 256
-          retry_forever false
-          retry_max_interval 30
-        </buffer>
-        host "#{ENV['ELASTICSEARCH_HOST']}"
-        reload_connections false
-        reconnect_on_error true
-        reload_on_failure true
-        include_tag_key true
-        logstash_format true
-        logstash_prefix qemu
-        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
-        port "#{ENV['ELASTICSEARCH_PORT']}"
-        @type elasticsearch
-        user "#{ENV['ELASTICSEARCH_USERNAME']}"
-      </match>
-
-      <match journal.**>
-        <buffer>
-          chunk_limit_size 8MB
-          flush_interval 15s
-          flush_thread_count 8
-          queue_limit_length 256
-          retry_forever false
-          retry_max_interval 30
-        </buffer>
-        host "#{ENV['ELASTICSEARCH_HOST']}"
-        reload_connections false
-        reconnect_on_error true
-        reload_on_failure true
-        include_tag_key true
-        logstash_format true
-        logstash_prefix journal
-        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
-        port "#{ENV['ELASTICSEARCH_PORT']}"
-        @type elasticsearch
-        user "#{ENV['ELASTICSEARCH_USERNAME']}"
-      </match>
-
-      <match kernel>
-        <buffer>
-          chunk_limit_size 8MB
-          flush_interval 15s
-          flush_thread_count 8
-          queue_limit_length 256
-          retry_forever false
-          retry_max_interval 30
-        </buffer>
-        host "#{ENV['ELASTICSEARCH_HOST']}"
-        reload_connections false
-        reconnect_on_error true
-        reload_on_failure true
-        include_tag_key true
-        logstash_format true
-        logstash_prefix kernel
-        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
-        port "#{ENV['ELASTICSEARCH_PORT']}"
-        @type elasticsearch
-        user "#{ENV['ELASTICSEARCH_USERNAME']}"
-      </match>
-
           <match **>
             <buffer>
-          chunk_limit_size 8MB
+              chunk_limit_size 512K
-          flush_interval 15s
+              flush_interval 5s
               flush_thread_count 8
-          queue_limit_length 256
+              queue_limit_length 32
               retry_forever false
               retry_max_interval 30
             </buffer>
@@ -287,10 +96,9 @@ conf:
             password "#{ENV['ELASTICSEARCH_PASSWORD']}"
             port "#{ENV['ELASTICSEARCH_PORT']}"
             @type elasticsearch
-        type_name fluent
             user "#{ENV['ELASTICSEARCH_USERNAME']}"
           </match>
-
+        </label>
 endpoints:
   cluster_domain_suffix: cluster.local
   local_image_registry:

tools/deployment/common/fluentd.sh

@@ -28,8 +28,9 @@ pod:
         MY_TEST_SECRET: BAR
 conf:
   fluentd:
-    # This field is now rendered as a helm template!
+    conf:
-    template: |
+      # These fields are rendered as helm templates
+      input: |
         <source>
           @type prometheus
           port {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
@@ -126,6 +127,13 @@ conf:
           </entry>
         </source>
 
+        <match **>
+          @type relabel
+          @label @filter
+        </match>
+
+      filter: |
+        <label @filter>
           <filter kubernetes.**>
             @type kubernetes_metadata
           </filter>
@@ -137,99 +145,17 @@ conf:
               fluentd_pod "#{ENV['POD_NAME']}"
             </record>
           </filter>
-
+          <match **>
+            @type relabel
+            @label @output
+          </match>
+        </label>
+      output: |
+        <label @output>
           <match fluent.**>
             @type null
           </match>
 
-      <match libvirt.**>
-        <buffer>
-          chunk_limit_size 512K
-          flush_interval 5s
-          flush_thread_count 8
-          queue_limit_length 32
-          retry_forever false
-          retry_max_interval 30
-        </buffer>
-        host "#{ENV['ELASTICSEARCH_HOST']}"
-        reload_connections false
-        reconnect_on_error true
-        reload_on_failure true
-        include_tag_key true
-        logstash_format true
-        logstash_prefix libvirt
-        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
-        port "#{ENV['ELASTICSEARCH_PORT']}"
-        @type elasticsearch
-        user "#{ENV['ELASTICSEARCH_USERNAME']}"
-      </match>
-
-      <match kernel>
-        <buffer>
-          chunk_limit_size 512K
-          flush_interval 5s
-          flush_thread_count 8
-          queue_limit_length 32
-          retry_forever false
-          disable_chunk_backup true
-        </buffer>
-        host "#{ENV['ELASTICSEARCH_HOST']}"
-        reload_connections false
-        reconnect_on_error true
-        reload_on_failure true
-        include_tag_key true
-        logstash_format true
-        logstash_prefix kernel
-        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
-        port "#{ENV['ELASTICSEARCH_PORT']}"
-        @type elasticsearch
-        user "#{ENV['ELASTICSEARCH_USERNAME']}"
-      </match>
-
-      <match auth>
-        <buffer>
-          chunk_limit_size 512K
-          flush_interval 5s
-          flush_thread_count 8
-          queue_limit_length 32
-          retry_forever false
-          retry_max_interval 30
-        </buffer>
-        host "#{ENV['ELASTICSEARCH_HOST']}"
-        reload_connections false
-        reconnect_on_error true
-        reload_on_failure true
-        include_tag_key true
-        logstash_format true
-        logstash_prefix auth
-        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
-        port "#{ENV['ELASTICSEARCH_PORT']}"
-        @type elasticsearch
-        user "#{ENV['ELASTICSEARCH_USERNAME']}"
-      </match>
-
-      <match journal.**>
-        <buffer>
-          chunk_limit_size 512K
-          flush_interval 5s
-          flush_thread_count 8
-          queue_limit_length 32
-          retry_forever false
-          retry_max_interval 30
-        </buffer>
-        host "#{ENV['ELASTICSEARCH_HOST']}"
-        reload_connections false
-        reconnect_on_error true
-        reload_on_failure true
-        include_tag_key true
-        logstash_format true
-        logstash_prefix journal
-        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
-        port "#{ENV['ELASTICSEARCH_PORT']}"
-        @type elasticsearch
-        user "#{ENV['ELASTICSEARCH_USERNAME']}"
-      </match>
-
           <match **>
             <buffer>
               chunk_limit_size 512K
@@ -250,6 +176,7 @@ conf:
             @type elasticsearch
             user "#{ENV['ELASTICSEARCH_USERNAME']}"
           </match>
+        </label>
 EOF
 helm upgrade --install fluentd ./fluentd \
     --namespace=osh-infra \