From 4c29bafcbc8adf5a673b8951c4674999535b6b8d Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Fri, 19 Oct 2018 07:57:35 -0500 Subject: [PATCH] Gates: Update fluent-logging/elasticsearch configurations This updates the fluentd buffer output configurations to account for the restraints of the jobs deploying fluentd. This also renames the fluentd configuration key from td_agent to fluentd to reflect the fact we're no longer deploying td-agent This also updates the Elasticsearch default replicas and overrides the replica counts in each Elasticsearch deployment to account for resource constraints Change-Id: I55dee410eced99c3e1645f7452e4306ad646e601 --- elasticsearch/values.yaml | 2 +- fluent-logging/templates/bin/_helm-tests.sh.tpl | 2 +- fluent-logging/templates/configmap-etc.yaml | 2 +- fluent-logging/values.yaml | 8 +------- tools/deployment/multinode/120-elasticsearch.sh | 4 ++++ tools/deployment/network-policy/120-elasticsearch.sh | 1 + tools/deployment/osh-infra-logging/050-elasticsearch.sh | 1 + 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 7939fd953..6a36e6adc 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -106,7 +106,7 @@ pod: default: kubernetes.io/hostname replicas: master: 3 - data: 2 + data: 3 client: 3 lifecycle: upgrades: diff --git a/fluent-logging/templates/bin/_helm-tests.sh.tpl b/fluent-logging/templates/bin/_helm-tests.sh.tpl index e30b9ae8f..12bbf3786 100644 --- a/fluent-logging/templates/bin/_helm-tests.sh.tpl +++ b/fluent-logging/templates/bin/_helm-tests.sh.tpl @@ -34,7 +34,7 @@ function check_output_indexes_exist () { Elasticsearch outputs should match. This means we can safely use the last item in fluentd's configuration to get the Fluentd flush output interval */}} - {{- $fluentdConf := last .Values.conf.td_agent -}} + {{- $fluentdConf := last .Values.conf.fluentd -}} {{- $fluentdElasticsearchConf := index $fluentdConf "elasticsearch" -}} {{- $fluentdFlush := index $fluentdElasticsearchConf "flush_interval" -}} fluentdFlush={{$fluentdFlush}} diff --git a/fluent-logging/templates/configmap-etc.yaml b/fluent-logging/templates/configmap-etc.yaml index 723526ecb..80c3f964d 100644 --- a/fluent-logging/templates/configmap-etc.yaml +++ b/fluent-logging/templates/configmap-etc.yaml @@ -25,7 +25,7 @@ type: Opaque data: fluent-bit.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.fluentbit | b64enc }} parsers.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.parsers | b64enc }} - fluent.conf: {{ include "fluent_logging.utils.to_fluentd_conf" .Values.conf.td_agent | b64enc }} + fluent.conf: {{ include "fluent_logging.utils.to_fluentd_conf" .Values.conf.fluentd | b64enc }} {{ range $template, $fields := .Values.conf.templates }} {{ $template }}.json: {{ toJson $fields | b64enc }} {{ end }} diff --git a/fluent-logging/values.yaml b/fluent-logging/values.yaml index ae30b78cb..3124f6a70 100644 --- a/fluent-logging/values.yaml +++ b/fluent-logging/values.yaml @@ -202,7 +202,7 @@ conf: Time_Key: time Time_Format: "%Y-%m-%dT%H:%M:%S.%L" Time_Keep: On - td_agent: + fluentd: - metrics_agent: header: source type: monitor_agent @@ -304,10 +304,8 @@ conf: buffer_chunk_limit: 2M buffer_queue_limit: 8 flush_interval: "5" - flush_thread_count: 2 max_retry_wait: 300 disable_retry_limit: "" - num_threads: 8 - kernel_elasticsearch: header: match type: elasticsearch @@ -322,10 +320,8 @@ conf: buffer_chunk_limit: 2M buffer_queue_limit: 8 flush_interval: "5" - flush_thread_count: 2 max_retry_wait: 300 disable_retry_limit: "" - num_threads: 8 # NOTE(srwilkers): This configuration entry should always be the last output # defined, as it is used to determine the total flush cycle time for fluentbit # and fluentd @@ -342,10 +338,8 @@ conf: buffer_chunk_limit: 2M buffer_queue_limit: 8 flush_interval: "5" - flush_thread_count: 2 max_retry_wait: 300 disable_retry_limit: "" - num_threads: 8 type_name: fluent fluentd_exporter: log: diff --git a/tools/deployment/multinode/120-elasticsearch.sh b/tools/deployment/multinode/120-elasticsearch.sh index c35224ff5..a0ae6d18a 100755 --- a/tools/deployment/multinode/120-elasticsearch.sh +++ b/tools/deployment/multinode/120-elasticsearch.sh @@ -21,6 +21,10 @@ make elasticsearch #NOTE: Deploy command tee /tmp/elasticsearch.yaml << EOF +pod: + replicas: + data: 1 + master: 2 conf: elasticsearch: env: diff --git a/tools/deployment/network-policy/120-elasticsearch.sh b/tools/deployment/network-policy/120-elasticsearch.sh index 9161d9496..02a408a04 100755 --- a/tools/deployment/network-policy/120-elasticsearch.sh +++ b/tools/deployment/network-policy/120-elasticsearch.sh @@ -27,6 +27,7 @@ monitoring: pod: replicas: data: 1 + master: 2 manifests: network_policy: true network_policy: diff --git a/tools/deployment/osh-infra-logging/050-elasticsearch.sh b/tools/deployment/osh-infra-logging/050-elasticsearch.sh index 8ec6c1d23..3f6da7010 100755 --- a/tools/deployment/osh-infra-logging/050-elasticsearch.sh +++ b/tools/deployment/osh-infra-logging/050-elasticsearch.sh @@ -27,6 +27,7 @@ monitoring: pod: replicas: data: 1 + master: 2 conf: elasticsearch: snapshots: