Move fluentbit and fluentd configs to values.yaml

Defines configuration files for fluentbit and fluentd via the
values.yaml file for fluent-logging. This provides flexibility in
defining parsers and routes for log gathering and routing

This functionality is added via helm-toolkit helper functions for
both fluentd and fluentbit to make the values configuration
cleaner

Change-Id: I8a43f36e487e651561bec8abf7752c8fac68aefc

fluent-logging/templates/_helpers.tpl

@@ -0,0 +1,110 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+# This function generates fluentd configuration files with entries in the
+# fluent-logging values.yaml.  It results in a configuration section with either
+# of the following formats (for as many key/value pairs defined in values for a
+section):
+# <HEADER>
+#   key value
+#   key value
+#   key value
+# </HEADER>
+# or
+# <HEADER>
+#   key value
+#   <INNER_HEADER>
+#     key value
+#   </INNER_HEADER>
+# </HEADER>
+# The configuration schema can be found here:
+# https://docs.fluentd.org/v0.12/articles/config-file
+
+{{- define "fluent_logging.to_fluentd_conf" -}}
+{{- range $values := . -}}
+{{- range $section := . -}}
+{{- $header := pick . "header" -}}
+{{- $config := omit . "header" "expression" -}}
+{{- if hasKey . "expression" -}}
+{{ $regex := pick . "expression" }}
+{{ printf "<%s %s>" $header.header $regex.expression }}
+{{- else }}
+{{ printf "<%s>" $header.header }}
+{{- end }}
+{{- range $key, $value := $config -}}
+{{- if kindIs "slice" $value }}
+{{- range $value := . -}}
+{{- range $innerSection := . -}}
+{{- $innerHeader := pick . "header" -}}
+{{- $innerConfig := omit . "header" "expression" -}}
+{{- if hasKey . "expression" -}}
+{{ $innerRegex := pick . "expression" }}
+{{ printf "<%s %s>" $innerHeader.header $innerRegex.expression | indent 2 }}
+{{- else }}
+{{ printf "<%s>" $innerHeader.header | indent 2 }}
+{{- end }}
+{{- range $innerKey, $innerValue := $innerConfig -}}
+{{- if eq $innerKey "type" -}}
+{{ $type := list "@" "type" | join "" }}
+{{ $type | indent 4 }} {{ $innerValue }}
+{{- else if contains "ENV" ($innerValue | quote) }}
+{{ $innerKey | indent 4 }} {{ $innerValue | quote }}
+{{- else }}
+{{ $innerKey | indent 4 }} {{ $innerValue }}
+{{- end }}
+{{- end }}
+{{ printf "</%s>" $innerHeader.header | indent 2 }}
+{{- end -}}
+{{ end -}}
+{{- else }}
+{{- if eq $key "type" -}}
+{{ $type := list "@" "type" | join "" }}
+{{ $type | indent 2 }} {{ $value }}
+{{- else if contains "ENV" ($value | quote) }}
+{{ $key | indent 2 }} {{ $value | quote }}
+{{- else }}
+{{ $key | indent 2 }} {{ $value }}
+{{- end -}}
+{{- end -}}
+{{- end }}
+{{ printf "</%s>" $header.header }}
+{{- end }}
+{{ end -}}
+{{- end -}}
+
+
+# This function generates fluentbit configuration files with entries in the
+# fluent-logging values.yaml.  It results in a configuration section with the
+# following format (for as many key/value pairs defined in values for a section):
+# [HEADER]
+#     key value
+#     key value
+#     key value
+# The configuration schema can be found here:
+# http://fluentbit.io/documentation/0.12/configuration/schema.html
+
+{{- define "fluent_logging.to_fluentbit_conf" -}}
+{{- range $values := . -}}
+{{- range $section := . -}}
+{{- $header := pick . "header" -}}
+{{- $config := omit . "header" }}
+[{{$header.header | upper }}]
+{{range $key, $value := $config -}}
+{{ $key | indent 4 }} {{ $value }}
+{{end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}

fluent-logging/templates/configmap-etc.yaml

@@ -23,9 +23,9 @@ metadata:
   name: fluent-logging-etc
 data:
   fluent-bit.conf: |+
-{{- tuple .Values.conf.fluentbit "etc/_fluent-bit.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }}
+{{ include "fluent_logging.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
   parsers.conf: |+
-{{- tuple .Values.conf.parsers "etc/_parsers.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }}
+{{ include "fluent_logging.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
   td-agent.conf: |+
-{{- tuple .Values.conf.td_agent "etc/_td-agent.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }}
+{{ include "fluent_logging.to_fluentd_conf" .Values.conf.td_agent | indent 4 }}
 {{- end }}

fluent-logging/templates/daemonset-fluent-bit.yaml

@@ -57,6 +57,11 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           command:
             - /tmp/fluent-bit.sh
+          env:
+            - name: FLUENTD_HOST
+              value: {{ tuple "fluentd" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote}}
+            - name: FLUENTD_PORT
+              value: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
           volumeMounts:
             - name: fluent-logging-bin
               mountPath: /tmp/fluent-bit.sh

fluent-logging/templates/deployment-fluentd.yaml

@@ -37,17 +37,17 @@ spec:
   template:
     metadata:
       labels:
-{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
       annotations:
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
     spec:
       serviceAccountName: {{ $serviceAccountName }}
       affinity:
-{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
       nodeSelector:
         {{ .Values.labels.fluentd.node_selector_key }}: {{ .Values.labels.fluentd.node_selector_value }}
-      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.fluentd_aggregator.timeout | default "30" }}
+      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.fluentd.timeout | default "30" }}
       initContainers:
 {{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
@@ -59,7 +59,15 @@ spec:
             - /tmp/fluentd.sh
             - start
           ports:
-            - containerPort: {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: forward
+              containerPort: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+          env:
+            - name: FLUENTD_PORT
+              value: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: ELASTICSEARCH_HOST
+              value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
+            - name: ELASTICSEARCH_PORT
+              value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
           volumeMounts:
             - name: pod-etc-fluentd
               mountPath: /etc/td-agent

fluent-logging/templates/etc/_fluent-bit.conf.tpl

@@ -1,19 +0,0 @@
-[SERVICE]
-    Flush          1
-    Daemon         Off
-    Log_Level      {{ .Values.conf.fluentbit.service.log_level }}
-    Parsers_File   parsers.conf
-
-[INPUT]
-    Name           tail
-    Tag            kube.*
-    Path           /var/log/containers/*.log
-    Parser         docker
-    DB             /var/log/flb_kube.db
-    Mem_Buf_Limit  {{ .Values.conf.fluentbit.input.mem_buf_limit }}
-
-[OUTPUT]
-    Name          forward
-    Match         *
-    Host          {{ tuple "aggregator" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
-    Port          {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}

fluent-logging/templates/etc/_parsers.conf.tpl

@@ -1,6 +0,0 @@
-[PARSER]
-    Name        docker
-    Format      json
-    Time_Key    time
-    Time_Format %Y-%m-%dT%H:%M:%S.%L
-    Time_Keep   On

fluent-logging/templates/etc/_td-agent.conf.tpl

@@ -1,83 +0,0 @@
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-   http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-<source>
-  @type forward
-  port {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-  bind 0.0.0.0
-</source>
-
-<filter kube.**>
-  type kubernetes_metadata
-</filter>
-
-<match ** >
-{{ if .Values.conf.fluentd.kafka.enabled }}
-  @type copy
-
-  <store>
-    @type kafka_buffered
-
-    # list of seed brokers
-    brokers {{ tuple "kafka" "public"  . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}:{{ tuple "kafka" "public" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-
-    # buffer settings
-    buffer_type file
-    buffer_path /var/log/td-agent/buffer/td
-    flush_interval {{ .Values.conf.fluentd.kafka.flush_interval }}
-
-    # topic settings
-    default_topic  {{ .Values.conf.fluentd.kafka.topic_name }}
-
-    # data type settings
-    output_data_type {{ .Values.conf.fluentd.kafka.output_data_type }}
-    compression_codec gzip
-
-    # producer settings
-    max_send_retries 1
-    required_acks -1
-  </store>
-
-  <store>
-{{- end }}
-    @type elasticsearch
-    include_tag_key true
-    host {{ tuple "elasticsearch" "internal"  . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
-    port {{ tuple "elasticsearch" "internal"  "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-    logstash_format {{ .Values.conf.fluentd.elasticsearch.logstash }}
-
-    # Set the chunk limit the same as for fluentd-gcp.
-    buffer_chunk_limit {{ .Values.conf.fluentd.elasticsearch.buffer_chunk_limit }}
-
-    # Cap buffer memory usage to 2MiB/chunk * 32 chunks = 64 MiB
-    buffer_queue_limit {{ .Values.conf.fluentd.elasticsearch.buffer_queue_limit }}
-
-    # Flush buffer every 30s to write to Elasticsearch
-    flush_interval {{ .Values.conf.fluentd.elasticsearch.flush_interval }}
-
-    # Never wait longer than 5 minutes between retries.
-    max_retry_wait {{ .Values.conf.fluentd.elasticsearch.max_retry_wait }}
-
-{{- if .Values.conf.fluentd.elasticsearch.disable_retry_limit }}
-
-    # Disable the limit on the number of retries (retry forever).
-    disable_retry_limit
-{{- end }}
-
-    # Use multiple threads for processing.
-    num_threads {{ .Values.conf.fluentd.elasticsearch.num_threads }}
-{{ if .Values.conf.fluentd.kafka.enabled }}
-  </store>
-{{- end }}
-
-</match>

fluent-logging/templates/service-fluentd.yaml

@@ -20,18 +20,17 @@ limitations under the License.
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ tuple "aggregator" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  name: {{ tuple "fluentd" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 spec:
   ports:
-  - name: aggregator
+  - name: forward
     port:  {{ .Values.network.fluentd.port }}
     {{ if .Values.network.fluentd.node_port.enabled }}
     nodePort: {{ .Values.network.fluentd.node_port.port }}
     {{ end }}
   selector:
-{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
   {{ if .Values.network.fluentd.node_port.enabled }}
   type: NodePort
   {{ end }}
 {{- end }}
-

fluent-logging/values.yaml

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Default values for fluentbit.
+# Default values for fluentbit and fluentd.
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
@@ -57,13 +57,13 @@ dependencies:
       endpoint: public
   fluentbit:
     services:
-    - service: aggregator
+    - service: fluentd
       endpoint: internal
   tests:
     services:
     - service: elasticsearch
       endpoint: internal
-    - service: aggregator
+    - service: fluentd
       endpoint: internal
 
 conditional_dependencies:
@@ -78,29 +78,62 @@ conditional_dependencies:
       - service: kafka
         endpoint: public
 
-
 conf:
   fluentbit:
-    service:
+    - service:
-      log_level: info
+        header: service
-    input:
+        Flush: 1
-      mem_buf_limit: 5MB
+        Daemon: Off
-    override:
+        Log_Level: info
-  fluentd:
+        Parsers_File: parsers.conf
-    kafka:
+    - containers_tail:
-      enabled: false
+        header: input
-      topic_name: logs
+        Name: tail
-      flush_interval: 3s
+        Tag: kube.*
-      output_data_type: json
+        Path: /var/log/containers/*.log
-    elasticsearch:
+        Parser: docker
-      logstash: true
+        DB: /var/log/flb_kube.db
+        Mem_Buf_Limit: 5MB
+    - kube_filter:
+        header: filter
+        Name: kubernetes
+        Match: kube.*
+        Merge_JSON_Log: On
+    - fluentd_output:
+        header: output
+        Name: forward
+        Match: "*"
+        Host: ${FLUENTD_HOST}
+        Port: ${FLUENTD_PORT}
+  parsers:
+    - docker:
+        header: parser
+        Name: docker
+        Format: json
+        Time_Key: time
+        Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
+        Time_Keep: On
+  td_agent:
+    - fluentbit_forward:
+        header: source
+        type: forward
+        port: "#{ENV['FLUENTD_PORT']}"
+        bind: 0.0.0.0
+    - elasticsearch:
+        header: match
+        type: elasticsearch
+        expression: "**"
+        include_tag_key: true
+        host: "#{ENV['ELASTICSEARCH_HOST']}"
+        port: "#{ENV['ELASTICSEARCH_PORT']}"
+        logstash_format: true
         buffer_chunk_limit: 10M
         buffer_queue_limit: 32
-      flush_interval: 15s
+        flush_interval: 20s
         max_retry_wait: 300
-      disable_retry_limit: true
+        disable_retry_limit: ""
         num_threads: 8
-    override:
+
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -129,26 +162,29 @@ endpoints:
     hosts:
       default: kafka-logging
       public: kafka
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
     scheme:
       default: http
-      public: http
     port:
       service:
         default: 9092
-  aggregator:
+  fluentd:
     namespace: null
     name: fluentd
     hosts:
       default: fluentd-logging
-      internal: fluentd-logging
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
     scheme:
       default: http
     port:
       service:
         default: 24224
-        internal: 24224
-    host_fqdn_override:
-      default: null
 
 network:
   fluentd:
@@ -179,8 +215,10 @@ pod:
           max_unavailable: 1
           max_surge: 3
     termination_grace_period:
-      fluentd_aggregator:
+      fluentd:
         timeout: 30
+  replicas:
+    fluentd: 3
   resources:
     fluentbit:
       enabled: false
@@ -199,15 +237,21 @@ pod:
         memory: '128Mi'
         cpu: '500m'
     jobs:
-      tests:
+      image_repo_sync:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
         limits:
-          memory: '1024Mi'
+          memory: "1024Mi"
-          cpu: '2000m'
+          cpu: "2000m"
+      tests:
         requests:
           memory: '128Mi'
           cpu: '100m'
-  replicas:
+        limits:
-    fluentd: 3
+          memory: '1024Mi'
+          cpu: '2000m'
+
   mounts:
     fluentd:
       fluentd: