diff --git a/fluent-logging/templates/_helpers.tpl b/fluent-logging/templates/_helpers.tpl new file mode 100644 index 000000000..6cbf26ca9 --- /dev/null +++ b/fluent-logging/templates/_helpers.tpl @@ -0,0 +1,110 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function generates fluentd configuration files with entries in the +# fluent-logging values.yaml. It results in a configuration section with either +# of the following formats (for as many key/value pairs defined in values for a +section): +#
+# key value +# key value +# key value +#
+# or +#
+# key value +# +# key value +# +#
+# The configuration schema can be found here: +# https://docs.fluentd.org/v0.12/articles/config-file + +{{- define "fluent_logging.to_fluentd_conf" -}} +{{- range $values := . -}} +{{- range $section := . -}} +{{- $header := pick . "header" -}} +{{- $config := omit . "header" "expression" -}} +{{- if hasKey . "expression" -}} +{{ $regex := pick . "expression" }} +{{ printf "<%s %s>" $header.header $regex.expression }} +{{- else }} +{{ printf "<%s>" $header.header }} +{{- end }} +{{- range $key, $value := $config -}} +{{- if kindIs "slice" $value }} +{{- range $value := . -}} +{{- range $innerSection := . -}} +{{- $innerHeader := pick . "header" -}} +{{- $innerConfig := omit . "header" "expression" -}} +{{- if hasKey . "expression" -}} +{{ $innerRegex := pick . "expression" }} +{{ printf "<%s %s>" $innerHeader.header $innerRegex.expression | indent 2 }} +{{- else }} +{{ printf "<%s>" $innerHeader.header | indent 2 }} +{{- end }} +{{- range $innerKey, $innerValue := $innerConfig -}} +{{- if eq $innerKey "type" -}} +{{ $type := list "@" "type" | join "" }} +{{ $type | indent 4 }} {{ $innerValue }} +{{- else if contains "ENV" ($innerValue | quote) }} +{{ $innerKey | indent 4 }} {{ $innerValue | quote }} +{{- else }} +{{ $innerKey | indent 4 }} {{ $innerValue }} +{{- end }} +{{- end }} +{{ printf "" $innerHeader.header | indent 2 }} +{{- end -}} +{{ end -}} +{{- else }} +{{- if eq $key "type" -}} +{{ $type := list "@" "type" | join "" }} +{{ $type | indent 2 }} {{ $value }} +{{- else if contains "ENV" ($value | quote) }} +{{ $key | indent 2 }} {{ $value | quote }} +{{- else }} +{{ $key | indent 2 }} {{ $value }} +{{- end -}} +{{- end -}} +{{- end }} +{{ printf "" $header.header }} +{{- end }} +{{ end -}} +{{- end -}} + + +# This function generates fluentbit configuration files with entries in the +# fluent-logging values.yaml. It results in a configuration section with the +# following format (for as many key/value pairs defined in values for a section): +# [HEADER] +# key value +# key value +# key value +# The configuration schema can be found here: +# http://fluentbit.io/documentation/0.12/configuration/schema.html + +{{- define "fluent_logging.to_fluentbit_conf" -}} +{{- range $values := . -}} +{{- range $section := . -}} +{{- $header := pick . "header" -}} +{{- $config := omit . "header" }} +[{{$header.header | upper }}] +{{range $key, $value := $config -}} +{{ $key | indent 4 }} {{ $value }} +{{end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/fluent-logging/templates/configmap-etc.yaml b/fluent-logging/templates/configmap-etc.yaml index 75f46b8a6..63d892909 100644 --- a/fluent-logging/templates/configmap-etc.yaml +++ b/fluent-logging/templates/configmap-etc.yaml @@ -23,9 +23,9 @@ metadata: name: fluent-logging-etc data: fluent-bit.conf: |+ -{{- tuple .Values.conf.fluentbit "etc/_fluent-bit.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }} +{{ include "fluent_logging.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }} parsers.conf: |+ -{{- tuple .Values.conf.parsers "etc/_parsers.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }} +{{ include "fluent_logging.to_fluentbit_conf" .Values.conf.parsers | indent 4 }} td-agent.conf: |+ -{{- tuple .Values.conf.td_agent "etc/_td-agent.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }} +{{ include "fluent_logging.to_fluentd_conf" .Values.conf.td_agent | indent 4 }} {{- end }} diff --git a/fluent-logging/templates/daemonset-fluent-bit.yaml b/fluent-logging/templates/daemonset-fluent-bit.yaml index 5a86f2a3c..b53afe288 100644 --- a/fluent-logging/templates/daemonset-fluent-bit.yaml +++ b/fluent-logging/templates/daemonset-fluent-bit.yaml @@ -57,6 +57,11 @@ spec: {{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} command: - /tmp/fluent-bit.sh + env: + - name: FLUENTD_HOST + value: {{ tuple "fluentd" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote}} + - name: FLUENTD_PORT + value: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} volumeMounts: - name: fluent-logging-bin mountPath: /tmp/fluent-bit.sh diff --git a/fluent-logging/templates/deployment-fluentd.yaml b/fluent-logging/templates/deployment-fluentd.yaml index 0d9c18407..2119d1eed 100644 --- a/fluent-logging/templates/deployment-fluentd.yaml +++ b/fluent-logging/templates/deployment-fluentd.yaml @@ -37,17 +37,17 @@ spec: template: metadata: labels: -{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} annotations: configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} spec: serviceAccountName: {{ $serviceAccountName }} affinity: -{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} +{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} nodeSelector: {{ .Values.labels.fluentd.node_selector_key }}: {{ .Values.labels.fluentd.node_selector_value }} - terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.fluentd_aggregator.timeout | default "30" }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.fluentd.timeout | default "30" }} initContainers: {{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: @@ -59,7 +59,15 @@ spec: - /tmp/fluentd.sh - start ports: - - containerPort: {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: forward + containerPort: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: FLUENTD_PORT + value: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} volumeMounts: - name: pod-etc-fluentd mountPath: /etc/td-agent diff --git a/fluent-logging/templates/etc/_fluent-bit.conf.tpl b/fluent-logging/templates/etc/_fluent-bit.conf.tpl deleted file mode 100644 index 7b09615d9..000000000 --- a/fluent-logging/templates/etc/_fluent-bit.conf.tpl +++ /dev/null @@ -1,19 +0,0 @@ -[SERVICE] - Flush 1 - Daemon Off - Log_Level {{ .Values.conf.fluentbit.service.log_level }} - Parsers_File parsers.conf - -[INPUT] - Name tail - Tag kube.* - Path /var/log/containers/*.log - Parser docker - DB /var/log/flb_kube.db - Mem_Buf_Limit {{ .Values.conf.fluentbit.input.mem_buf_limit }} - -[OUTPUT] - Name forward - Match * - Host {{ tuple "aggregator" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} - Port {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} diff --git a/fluent-logging/templates/etc/_parsers.conf.tpl b/fluent-logging/templates/etc/_parsers.conf.tpl deleted file mode 100644 index 9f3b6b331..000000000 --- a/fluent-logging/templates/etc/_parsers.conf.tpl +++ /dev/null @@ -1,6 +0,0 @@ -[PARSER] - Name docker - Format json - Time_Key time - Time_Format %Y-%m-%dT%H:%M:%S.%L - Time_Keep On diff --git a/fluent-logging/templates/etc/_td-agent.conf.tpl b/fluent-logging/templates/etc/_td-agent.conf.tpl deleted file mode 100644 index b9d78bbb2..000000000 --- a/fluent-logging/templates/etc/_td-agent.conf.tpl +++ /dev/null @@ -1,83 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - - - @type forward - port {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - bind 0.0.0.0 - - - - type kubernetes_metadata - - - -{{ if .Values.conf.fluentd.kafka.enabled }} - @type copy - - - @type kafka_buffered - - # list of seed brokers - brokers {{ tuple "kafka" "public" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}:{{ tuple "kafka" "public" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - - # buffer settings - buffer_type file - buffer_path /var/log/td-agent/buffer/td - flush_interval {{ .Values.conf.fluentd.kafka.flush_interval }} - - # topic settings - default_topic {{ .Values.conf.fluentd.kafka.topic_name }} - - # data type settings - output_data_type {{ .Values.conf.fluentd.kafka.output_data_type }} - compression_codec gzip - - # producer settings - max_send_retries 1 - required_acks -1 - - - -{{- end }} - @type elasticsearch - include_tag_key true - host {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} - port {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - logstash_format {{ .Values.conf.fluentd.elasticsearch.logstash }} - - # Set the chunk limit the same as for fluentd-gcp. - buffer_chunk_limit {{ .Values.conf.fluentd.elasticsearch.buffer_chunk_limit }} - - # Cap buffer memory usage to 2MiB/chunk * 32 chunks = 64 MiB - buffer_queue_limit {{ .Values.conf.fluentd.elasticsearch.buffer_queue_limit }} - - # Flush buffer every 30s to write to Elasticsearch - flush_interval {{ .Values.conf.fluentd.elasticsearch.flush_interval }} - - # Never wait longer than 5 minutes between retries. - max_retry_wait {{ .Values.conf.fluentd.elasticsearch.max_retry_wait }} - -{{- if .Values.conf.fluentd.elasticsearch.disable_retry_limit }} - - # Disable the limit on the number of retries (retry forever). - disable_retry_limit -{{- end }} - - # Use multiple threads for processing. - num_threads {{ .Values.conf.fluentd.elasticsearch.num_threads }} -{{ if .Values.conf.fluentd.kafka.enabled }} - -{{- end }} - - diff --git a/fluent-logging/templates/service-fluentd.yaml b/fluent-logging/templates/service-fluentd.yaml index 4a3aa63bb..a2b606bb4 100644 --- a/fluent-logging/templates/service-fluentd.yaml +++ b/fluent-logging/templates/service-fluentd.yaml @@ -20,18 +20,17 @@ limitations under the License. apiVersion: v1 kind: Service metadata: - name: {{ tuple "aggregator" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + name: {{ tuple "fluentd" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: ports: - - name: aggregator + - name: forward port: {{ .Values.network.fluentd.port }} {{ if .Values.network.fluentd.node_port.enabled }} nodePort: {{ .Values.network.fluentd.node_port.port }} {{ end }} selector: -{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} {{ if .Values.network.fluentd.node_port.enabled }} type: NodePort {{ end }} {{- end }} - diff --git a/fluent-logging/values.yaml b/fluent-logging/values.yaml index 2a05a66c6..caec3235f 100644 --- a/fluent-logging/values.yaml +++ b/fluent-logging/values.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Default values for fluentbit. +# Default values for fluentbit and fluentd. # This is a YAML-formatted file. # Declare variables to be passed into your templates. @@ -57,13 +57,13 @@ dependencies: endpoint: public fluentbit: services: - - service: aggregator + - service: fluentd endpoint: internal tests: services: - service: elasticsearch endpoint: internal - - service: aggregator + - service: fluentd endpoint: internal conditional_dependencies: @@ -78,29 +78,62 @@ conditional_dependencies: - service: kafka endpoint: public - conf: fluentbit: - service: - log_level: info - input: - mem_buf_limit: 5MB - override: - fluentd: - kafka: - enabled: false - topic_name: logs - flush_interval: 3s - output_data_type: json - elasticsearch: - logstash: true - buffer_chunk_limit: 10M - buffer_queue_limit: 32 - flush_interval: 15s - max_retry_wait: 300 - disable_retry_limit: true - num_threads: 8 - override: + - service: + header: service + Flush: 1 + Daemon: Off + Log_Level: info + Parsers_File: parsers.conf + - containers_tail: + header: input + Name: tail + Tag: kube.* + Path: /var/log/containers/*.log + Parser: docker + DB: /var/log/flb_kube.db + Mem_Buf_Limit: 5MB + - kube_filter: + header: filter + Name: kubernetes + Match: kube.* + Merge_JSON_Log: On + - fluentd_output: + header: output + Name: forward + Match: "*" + Host: ${FLUENTD_HOST} + Port: ${FLUENTD_PORT} + parsers: + - docker: + header: parser + Name: docker + Format: json + Time_Key: time + Time_Format: "%Y-%m-%dT%H:%M:%S.%L" + Time_Keep: On + td_agent: + - fluentbit_forward: + header: source + type: forward + port: "#{ENV['FLUENTD_PORT']}" + bind: 0.0.0.0 + - elasticsearch: + header: match + type: elasticsearch + expression: "**" + include_tag_key: true + host: "#{ENV['ELASTICSEARCH_HOST']}" + port: "#{ENV['ELASTICSEARCH_PORT']}" + logstash_format: true + buffer_chunk_limit: 10M + buffer_queue_limit: 32 + flush_interval: 20s + max_retry_wait: 300 + disable_retry_limit: "" + num_threads: 8 + endpoints: cluster_domain_suffix: cluster.local @@ -129,26 +162,29 @@ endpoints: hosts: default: kafka-logging public: kafka + host_fqdn_override: + default: null + path: + default: null scheme: default: http - public: http port: service: default: 9092 - aggregator: + fluentd: namespace: null name: fluentd hosts: default: fluentd-logging - internal: fluentd-logging + host_fqdn_override: + default: null + path: + default: null scheme: default: http port: service: default: 24224 - internal: 24224 - host_fqdn_override: - default: null network: fluentd: @@ -179,8 +215,10 @@ pod: max_unavailable: 1 max_surge: 3 termination_grace_period: - fluentd_aggregator: + fluentd: timeout: 30 + replicas: + fluentd: 3 resources: fluentbit: enabled: false @@ -199,15 +237,21 @@ pod: memory: '128Mi' cpu: '500m' jobs: - tests: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" limits: - memory: '1024Mi' - cpu: '2000m' + memory: "1024Mi" + cpu: "2000m" + tests: requests: memory: '128Mi' cpu: '100m' - replicas: - fluentd: 3 + limits: + memory: '1024Mi' + cpu: '2000m' + mounts: fluentd: fluentd: