From ffb4767c05050b2078b56fea69d8f1c2d2182aff Mon Sep 17 00:00:00 2001 From: Stig Telfer Date: Thu, 28 Jul 2022 14:54:12 +0100 Subject: [PATCH] Increase the Fluentd request timeout for ES Fluentd has a default timeout of 5s for flushing data to ElasticSearch. If there is a significant backlog of unsent log messages, this timeout can be exceeded, resulting in Fluentd failing to make further progress. Raise the default timeout to 60s. This patch adopts the configuration parameters previously proposed by Krzysztof Klimonda. Closes-Bug: #1983031 Closes-Bug: #1896611 Change-Id: I1aaab654a5a0752fccef2cfb8cc0bde4a0ee2562 --- ansible/roles/common/defaults/main.yml | 1 + .../common/templates/conf/output/00-local.conf.j2 | 1 + .../roles/common/templates/conf/output/01-es.conf.j2 | 1 + ...elasticsearch-request-timeout-f110883dd4412f65.yaml | 10 ++++++++++ 4 files changed, 13 insertions(+) create mode 100644 releasenotes/notes/fluentd-elasticsearch-request-timeout-f110883dd4412f65.yaml diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml index a6fe1e7c7f..72b51b25f7 100644 --- a/ansible/roles/common/defaults/main.yml +++ b/ansible/roles/common/defaults/main.yml @@ -43,6 +43,7 @@ fluentd_elasticsearch_password: "" fluentd_elasticsearch_ssl_version: "TLSv1_2" fluentd_elasticsearch_ssl_verify: "true" fluentd_elasticsearch_cacert: "{{ openstack_cacert }}" +fluentd_elasticsearch_request_timeout: "60s" #################### # Docker diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2 index 78ce43177c..d336230e7c 100644 --- a/ansible/roles/common/templates/conf/output/00-local.conf.j2 +++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2 @@ -37,6 +37,7 @@ logstash_format true logstash_prefix {{ kibana_log_prefix }} reconnect_on_error true + request_timeout {{ fluentd_elasticsearch_request_timeout }} suppress_type_name true @type file diff --git a/ansible/roles/common/templates/conf/output/01-es.conf.j2 b/ansible/roles/common/templates/conf/output/01-es.conf.j2 index 3affbf8f1a..3056547801 100644 --- a/ansible/roles/common/templates/conf/output/01-es.conf.j2 +++ b/ansible/roles/common/templates/conf/output/01-es.conf.j2 @@ -22,6 +22,7 @@ logstash_format true logstash_prefix {{ kibana_log_prefix }} reconnect_on_error true + request_timeout {{ fluentd_elasticsearch_request_timeout }} suppress_type_name true @type file diff --git a/releasenotes/notes/fluentd-elasticsearch-request-timeout-f110883dd4412f65.yaml b/releasenotes/notes/fluentd-elasticsearch-request-timeout-f110883dd4412f65.yaml new file mode 100644 index 0000000000..c319141ff3 --- /dev/null +++ b/releasenotes/notes/fluentd-elasticsearch-request-timeout-f110883dd4412f65.yaml @@ -0,0 +1,10 @@ +--- +fixes: + - | + Under circumstances of extended disruption to the Fluentd-ElasticSearch + central logging pipeline, it is possible to generate a sufficient buffer + of unsent log data that takes longer than the default Fluentd request + timeout (default 5 seconds) to transfer the buffer. The default request + timeout value is raised to ``60s``, and made configurable using new + parameter ``fluentd_elasticsearch_request_timeout``. `LP#1983031 + `__