Move fluentbit and fluentd configs to values.yaml

Defines configuration files for fluentbit and fluentd via the values.yaml file for fluent-logging. This provides flexibility in defining parsers and routes for log gathering and routing This functionality is added via helm-toolkit helper functions for both fluentd and fluentbit to make the values configuration cleaner Change-Id: I8a43f36e487e651561bec8abf7752c8fac68aefc
2017-12-12 12:23:29 -06:00 · 2017-12-12 12:23:29 -06:00 · 09939a04de
commit 09939a04de
parent 558ed8cd27
9 changed files with 213 additions and 155 deletions
--- a/fluent-logging/templates/_helpers.tpl
+++ b/fluent-logging/templates/_helpers.tpl
@ -0,0 +1,110 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+# This function generates fluentd configuration files with entries in the
+# fluent-logging values.yaml.  It results in a configuration section with either
+# of the following formats (for as many key/value pairs defined in values for a
+section):
+# <HEADER>
+#   key value
+#   key value
+#   key value
+# </HEADER>
+# or
+# <HEADER>
+#   key value
+#   <INNER_HEADER>
+#     key value
+#   </INNER_HEADER>
+# </HEADER>
+# The configuration schema can be found here:
+# https://docs.fluentd.org/v0.12/articles/config-file
+
+{{- define "fluent_logging.to_fluentd_conf" -}}
+{{- range $values := . -}}
+{{- range $section := . -}}
+{{- $header := pick . "header" -}}
+{{- $config := omit . "header" "expression" -}}
+{{- if hasKey . "expression" -}}
+{{ $regex := pick . "expression" }}
+{{ printf "<%s %s>" $header.header $regex.expression }}
+{{- else }}
+{{ printf "<%s>" $header.header }}
+{{- end }}
+{{- range $key, $value := $config -}}
+{{- if kindIs "slice" $value }}
+{{- range $value := . -}}
+{{- range $innerSection := . -}}
+{{- $innerHeader := pick . "header" -}}
+{{- $innerConfig := omit . "header" "expression" -}}
+{{- if hasKey . "expression" -}}
+{{ $innerRegex := pick . "expression" }}
+{{ printf "<%s %s>" $innerHeader.header $innerRegex.expression | indent 2 }}
+{{- else }}
+{{ printf "<%s>" $innerHeader.header | indent 2 }}
+{{- end }}
+{{- range $innerKey, $innerValue := $innerConfig -}}
+{{- if eq $innerKey "type" -}}
+{{ $type := list "@" "type" | join "" }}
+{{ $type | indent 4 }} {{ $innerValue }}
+{{- else if contains "ENV" ($innerValue | quote) }}
+{{ $innerKey | indent 4 }} {{ $innerValue | quote }}
+{{- else }}
+{{ $innerKey | indent 4 }} {{ $innerValue }}
+{{- end }}
+{{- end }}
+{{ printf "</%s>" $innerHeader.header | indent 2 }}
+{{- end -}}
+{{ end -}}
+{{- else }}
+{{- if eq $key "type" -}}
+{{ $type := list "@" "type" | join "" }}
+{{ $type | indent 2 }} {{ $value }}
+{{- else if contains "ENV" ($value | quote) }}
+{{ $key | indent 2 }} {{ $value | quote }}
+{{- else }}
+{{ $key | indent 2 }} {{ $value }}
+{{- end -}}
+{{- end -}}
+{{- end }}
+{{ printf "</%s>" $header.header }}
+{{- end }}
+{{ end -}}
+{{- end -}}
+
+
+# This function generates fluentbit configuration files with entries in the
+# fluent-logging values.yaml.  It results in a configuration section with the
+# following format (for as many key/value pairs defined in values for a section):
+# [HEADER]
+#     key value
+#     key value
+#     key value
+# The configuration schema can be found here:
+# http://fluentbit.io/documentation/0.12/configuration/schema.html
+
+{{- define "fluent_logging.to_fluentbit_conf" -}}
+{{- range $values := . -}}
+{{- range $section := . -}}
+{{- $header := pick . "header" -}}
+{{- $config := omit . "header" }}
+[{{$header.header | upper }}]
+{{range $key, $value := $config -}}
+{{ $key | indent 4 }} {{ $value }}
+{{end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
--- a/fluent-logging/templates/configmap-etc.yaml
+++ b/fluent-logging/templates/configmap-etc.yaml
@ -23,9 +23,9 @@ metadata:
  name: fluent-logging-etc
 data:
  fluent-bit.conf: |+
-{{- tuple .Values.conf.fluentbit "etc/_fluent-bit.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }}
+{{ include "fluent_logging.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
  parsers.conf: |+
-{{- tuple .Values.conf.parsers "etc/_parsers.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }}
+{{ include "fluent_logging.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
  td-agent.conf: |+
-{{- tuple .Values.conf.td_agent "etc/_td-agent.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }}
+{{ include "fluent_logging.to_fluentd_conf" .Values.conf.td_agent | indent 4 }}
 {{- end }}
--- a/fluent-logging/templates/daemonset-fluent-bit.yaml
+++ b/fluent-logging/templates/daemonset-fluent-bit.yaml
@ -57,6 +57,11 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
          command:
            - /tmp/fluent-bit.sh
+          env:
+            - name: FLUENTD_HOST
+              value: {{ tuple "fluentd" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote}}
+            - name: FLUENTD_PORT
+              value: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
          volumeMounts:
            - name: fluent-logging-bin
              mountPath: /tmp/fluent-bit.sh
--- a/fluent-logging/templates/deployment-fluentd.yaml
+++ b/fluent-logging/templates/deployment-fluentd.yaml
@ -37,17 +37,17 @@ spec:
  template:
    metadata:
      labels:
-{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
      annotations:
        configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
        configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
    spec:
      serviceAccountName: {{ $serviceAccountName }}
      affinity:
-{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
      nodeSelector:
        {{ .Values.labels.fluentd.node_selector_key }}: {{ .Values.labels.fluentd.node_selector_value }}
-      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.fluentd_aggregator.timeout | default "30" }}
+      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.fluentd.timeout | default "30" }}
      initContainers:
 {{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
      containers:
@ -59,7 +59,15 @@ spec:
            - /tmp/fluentd.sh
            - start
          ports:
-            - containerPort: {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: forward
+              containerPort: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+          env:
+            - name: FLUENTD_PORT
+              value: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: ELASTICSEARCH_HOST
+              value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
+            - name: ELASTICSEARCH_PORT
+              value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
          volumeMounts:
            - name: pod-etc-fluentd
              mountPath: /etc/td-agent
--- a/fluent-logging/templates/etc/_fluent-bit.conf.tpl
+++ b/fluent-logging/templates/etc/_fluent-bit.conf.tpl
@ -1,19 +0,0 @@
-[SERVICE]
-    Flush          1
-    Daemon         Off
-    Log_Level      {{ .Values.conf.fluentbit.service.log_level }}
-    Parsers_File   parsers.conf
-
-[INPUT]
-    Name           tail
-    Tag            kube.*
-    Path           /var/log/containers/*.log
-    Parser         docker
-    DB             /var/log/flb_kube.db
-    Mem_Buf_Limit  {{ .Values.conf.fluentbit.input.mem_buf_limit }}
-
-[OUTPUT]
-    Name          forward
-    Match         *
-    Host          {{ tuple "aggregator" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
-    Port          {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
--- a/fluent-logging/templates/etc/_parsers.conf.tpl
+++ b/fluent-logging/templates/etc/_parsers.conf.tpl
@ -1,6 +0,0 @@
-[PARSER]
-    Name        docker
-    Format      json
-    Time_Key    time
-    Time_Format %Y-%m-%dT%H:%M:%S.%L
-    Time_Keep   On
--- a/fluent-logging/templates/etc/_td-agent.conf.tpl
+++ b/fluent-logging/templates/etc/_td-agent.conf.tpl
@ -1,83 +0,0 @@
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-   http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-<source>
-  @type forward
-  port {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-  bind 0.0.0.0
-</source>
-
-<filter kube.**>
-  type kubernetes_metadata
-</filter>
-
-<match ** >
-{{ if .Values.conf.fluentd.kafka.enabled }}
-  @type copy
-
-  <store>
-    @type kafka_buffered
-
-    # list of seed brokers
-    brokers {{ tuple "kafka" "public"  . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}:{{ tuple "kafka" "public" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-
-    # buffer settings
-    buffer_type file
-    buffer_path /var/log/td-agent/buffer/td
-    flush_interval {{ .Values.conf.fluentd.kafka.flush_interval }}
-
-    # topic settings
-    default_topic  {{ .Values.conf.fluentd.kafka.topic_name }}
-
-    # data type settings
-    output_data_type {{ .Values.conf.fluentd.kafka.output_data_type }}
-    compression_codec gzip
-
-    # producer settings
-    max_send_retries 1
-    required_acks -1
-  </store>
-
-  <store>
-{{- end }}
-    @type elasticsearch
-    include_tag_key true
-    host {{ tuple "elasticsearch" "internal"  . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
-    port {{ tuple "elasticsearch" "internal"  "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-    logstash_format {{ .Values.conf.fluentd.elasticsearch.logstash }}
-
-    # Set the chunk limit the same as for fluentd-gcp.
-    buffer_chunk_limit {{ .Values.conf.fluentd.elasticsearch.buffer_chunk_limit }}
-
-    # Cap buffer memory usage to 2MiB/chunk * 32 chunks = 64 MiB
-    buffer_queue_limit {{ .Values.conf.fluentd.elasticsearch.buffer_queue_limit }}
-
-    # Flush buffer every 30s to write to Elasticsearch
-    flush_interval {{ .Values.conf.fluentd.elasticsearch.flush_interval }}
-
-    # Never wait longer than 5 minutes between retries.
-    max_retry_wait {{ .Values.conf.fluentd.elasticsearch.max_retry_wait }}
-
-{{- if .Values.conf.fluentd.elasticsearch.disable_retry_limit }}
-
-    # Disable the limit on the number of retries (retry forever).
-    disable_retry_limit
-{{- end }}
-
-    # Use multiple threads for processing.
-    num_threads {{ .Values.conf.fluentd.elasticsearch.num_threads }}
-{{ if .Values.conf.fluentd.kafka.enabled }}
-  </store>
-{{- end }}
-
-</match>
--- a/fluent-logging/templates/service-fluentd.yaml
+++ b/fluent-logging/templates/service-fluentd.yaml
@ -20,18 +20,17 @@ limitations under the License.
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ tuple "aggregator" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  name: {{ tuple "fluentd" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 spec:
  ports:
-  - name: aggregator
+  - name: forward
    port:  {{ .Values.network.fluentd.port }}
    {{ if .Values.network.fluentd.node_port.enabled }}
    nodePort: {{ .Values.network.fluentd.node_port.port }}
    {{ end }}
  selector:
-{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
  {{ if .Values.network.fluentd.node_port.enabled }}
  type: NodePort
  {{ end }}
 {{- end }}
-
--- a/fluent-logging/values.yaml
+++ b/fluent-logging/values.yaml
@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-# Default values for fluentbit.
+# Default values for fluentbit and fluentd.
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.

@ -57,13 +57,13 @@ dependencies:
      endpoint: public
  fluentbit:
    services:
-    - service: aggregator
+    - service: fluentd
      endpoint: internal
  tests:
    services:
    - service: elasticsearch
      endpoint: internal
-    - service: aggregator
+    - service: fluentd
      endpoint: internal

 conditional_dependencies:
@ -78,29 +78,62 @@ conditional_dependencies:
      - service: kafka
        endpoint: public

-
 conf:
  fluentbit:
-    service:
-      log_level: info
-    input:
-      mem_buf_limit: 5MB
-    override:
-  fluentd:
-    kafka:
-      enabled: false
-      topic_name: logs
-      flush_interval: 3s
-      output_data_type: json
-    elasticsearch:
-      logstash: true
-      buffer_chunk_limit: 10M
-      buffer_queue_limit: 32
-      flush_interval: 15s
-      max_retry_wait: 300
-      disable_retry_limit: true
-      num_threads: 8
-    override:
+    - service:
+        header: service
+        Flush: 1
+        Daemon: Off
+        Log_Level: info
+        Parsers_File: parsers.conf
+    - containers_tail:
+        header: input
+        Name: tail
+        Tag: kube.*
+        Path: /var/log/containers/*.log
+        Parser: docker
+        DB: /var/log/flb_kube.db
+        Mem_Buf_Limit: 5MB
+    - kube_filter:
+        header: filter
+        Name: kubernetes
+        Match: kube.*
+        Merge_JSON_Log: On
+    - fluentd_output:
+        header: output
+        Name: forward
+        Match: "*"
+        Host: ${FLUENTD_HOST}
+        Port: ${FLUENTD_PORT}
+  parsers:
+    - docker:
+        header: parser
+        Name: docker
+        Format: json
+        Time_Key: time
+        Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
+        Time_Keep: On
+  td_agent:
+    - fluentbit_forward:
+        header: source
+        type: forward
+        port: "#{ENV['FLUENTD_PORT']}"
+        bind: 0.0.0.0
+    - elasticsearch:
+        header: match
+        type: elasticsearch
+        expression: "**"
+        include_tag_key: true
+        host: "#{ENV['ELASTICSEARCH_HOST']}"
+        port: "#{ENV['ELASTICSEARCH_PORT']}"
+        logstash_format: true
+        buffer_chunk_limit: 10M
+        buffer_queue_limit: 32
+        flush_interval: 20s
+        max_retry_wait: 300
+        disable_retry_limit: ""
+        num_threads: 8
+

 endpoints:
  cluster_domain_suffix: cluster.local
@ -129,26 +162,29 @@ endpoints:
    hosts:
      default: kafka-logging
      public: kafka
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
    scheme:
      default: http
-      public: http
    port:
      service:
        default: 9092
-  aggregator:
+  fluentd:
    namespace: null
    name: fluentd
    hosts:
      default: fluentd-logging
-      internal: fluentd-logging
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
    scheme:
      default: http
    port:
      service:
        default: 24224
-        internal: 24224
-    host_fqdn_override:
-      default: null

 network:
  fluentd:
@ -179,8 +215,10 @@ pod:
          max_unavailable: 1
          max_surge: 3
    termination_grace_period:
-      fluentd_aggregator:
+      fluentd:
        timeout: 30
+  replicas:
+    fluentd: 3
  resources:
    fluentbit:
      enabled: false
@ -199,15 +237,21 @@ pod:
        memory: '128Mi'
        cpu: '500m'
    jobs:
-      tests:
+      image_repo_sync:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
        limits:
-          memory: '1024Mi'
-          cpu: '2000m'
+          memory: "1024Mi"
+          cpu: "2000m"
+      tests:
        requests:
          memory: '128Mi'
          cpu: '100m'
-  replicas:
-    fluentd: 3
+        limits:
+          memory: '1024Mi'
+          cpu: '2000m'
+
  mounts:
    fluentd:
      fluentd: