From a192fb31292e6f79009804b0d322846cd4ffe9e8 Mon Sep 17 00:00:00 2001 From: Duncan Martin Walker Date: Wed, 22 Jan 2020 09:58:48 +0000 Subject: [PATCH] Elastic beat index template configuration Templates for the beat config files have been updated to allow more configuration options to be passed through to the associated Elasticsearch index templates. In particular, one can optionally specify values of index_template_max_docvalue_search to be set at the creation of the beat index template. This can prevent shard failure errors when viewing output in Kibana relating to "Trying to retrieve too many docvalue_fields". Any similar config options can in future be passed into the template via the elastic_beat_settings object. Change-Id: Ic9136c8e063bbd231ed280bb446661b251879407 --- elk_metrics_7x/createElasticIndexes.yml | 10 +++++----- .../elastic_apm_server/templates/apm-server.yml.j2 | 2 +- .../roles/elastic_auditbeat/templates/auditbeat.yml.j2 | 2 +- .../roles/elastic_data_hosts/defaults/main.yml | 4 ++++ .../elastic_data_hosts/vars/data-node-variables.yml | 8 ++++++++ .../roles/elastic_filebeat/templates/filebeat.yml.j2 | 2 +- .../roles/elastic_heartbeat/templates/heartbeat.yml.j2 | 2 +- .../elastic_journalbeat/templates/journalbeat.yml.j2 | 2 +- .../elastic_metricbeat/templates/metricbeat.yml.j2 | 4 ++-- .../elastic_packetbeat/templates/packetbeat.yml.j2 | 2 +- elk_metrics_7x/templates/_macros.j2 | 8 ++++++-- 11 files changed, 31 insertions(+), 15 deletions(-) diff --git a/elk_metrics_7x/createElasticIndexes.yml b/elk_metrics_7x/createElasticIndexes.yml index 12a9bc58..4af8faac 100644 --- a/elk_metrics_7x/createElasticIndexes.yml +++ b/elk_metrics_7x/createElasticIndexes.yml @@ -19,7 +19,7 @@ - role: elastic_data_hosts vars: - _elastic_refresh_interval: "{{ (elasticsearch_number_of_replicas | int) * 5 }}" + _elastic_refresh_interval: "{{ (elasticsearch_beat_settings.number_of_replicas | int) * 5 }}" elastic_refresh_interval: "{{ (_elastic_refresh_interval > 0) | ternary(30, _elastic_refresh_interval) }}" vars_files: @@ -83,7 +83,7 @@ indices.queries.cache.size: "5%" - name: "_all/_settings" index_options: - index.number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}" + index.number_of_replicas: "{{ elasticsearch_beat_settings.number_of_replicas | int }}" index.translog.durability: "async" index.refresh_interval: "{{ ((elastic_refresh_interval | int) > 30) | ternary(30, elastic_refresh_interval) }}s" @@ -129,7 +129,7 @@ | list }} settings: - number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}" + number_of_replicas: "{{ elasticsearch_beat_settings.number_of_replicas | int }}" index: mapping: total_fields: @@ -151,7 +151,7 @@ template: ".monitoring*" order: 1 settings: - number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}" + number_of_replicas: "{{ elasticsearch_beat_settings.number_of_replicas | int }}" number_of_shards: 1 - name: Create custom skydive index template @@ -170,7 +170,7 @@ template: "skydive*" order: 1 settings: - number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}" + number_of_replicas: "{{ elasticsearch_beat_settings.number_of_replicas | int }}" number_of_shards: 1 diff --git a/elk_metrics_7x/roles/elastic_apm_server/templates/apm-server.yml.j2 b/elk_metrics_7x/roles/elastic_apm_server/templates/apm-server.yml.j2 index b0dfaf3f..764ab3d5 100644 --- a/elk_metrics_7x/roles/elastic_apm_server/templates/apm-server.yml.j2 +++ b/elk_metrics_7x/roles/elastic_apm_server/templates/apm-server.yml.j2 @@ -134,7 +134,7 @@ apm-server: {{ elk_macros.setup_dashboards('apm') }} #=============================== Template ====================================== -{{ elk_macros.setup_template('apm', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +{{ elk_macros.setup_template('apm', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 %} diff --git a/elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 b/elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 index c8954ad5..b8affa06 100644 --- a/elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 @@ -811,7 +811,7 @@ processors: {{ elk_macros.setup_dashboards('auditbeat') }} #=============================== Template ====================================== -{{ elk_macros.setup_template('auditbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +{{ elk_macros.setup_template('auditbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 %} diff --git a/elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml b/elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml index b8f8897d..5c31b2b3 100644 --- a/elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml @@ -30,3 +30,7 @@ elastic_data_interface: "{{ ansible_default_ipv4['alias'] }}" # to override the default of ansible_host:logstash_beat_input_port # for each node, or to provide different coordinators when necessary # elasticsearch_coordination_node_socket_addresses: [] + +## Elasticsearch index template settings +# Set the default max number of fields used in a query +# elastic_max_docvalue_fields_search: "100" diff --git a/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml b/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml index 572fbe1f..aad576af 100644 --- a/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml +++ b/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml @@ -205,12 +205,20 @@ elastic_thread_pool_size: "{{ ((ansible_processor_count | int) >= 24) | ternary( # Set a data node facts. The data nodes, in the case of elasticsearch are also # ingest nodes. elasticsearch_number_of_replicas: "{{ ((data_nodes | length) > 2) | ternary('2', ((data_nodes | length) > 1) | ternary('1', '0')) }}" + +# Input data for the beat config templates +elasticsearch_beat_settings: + number_of_replicas: "{{ elasticsearch_number_of_replicas }}" + max_docvalue_fields_search: "{{ elastic_max_docvalue_fields_search | default('100') }}" + + elasticsearch_data_hosts: |- {% set data_hosts = elasticsearch_data_node_details | shuffle(seed=inventory_hostname) %} {% if inventory_hostname in data_nodes %} {% set _ = data_hosts.insert(0, '127.0.0.1:' ~ elastic_port) %} {% endif %} {{ data_hosts }} + logstash_data_hosts: |- {% set data_hosts = logstash_data_node_details | shuffle(seed=inventory_hostname) %} {% if inventory_hostname in data_nodes %} diff --git a/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 b/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 index d3413f7c..a3da522c 100644 --- a/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 @@ -1349,7 +1349,7 @@ processors: {{ elk_macros.setup_dashboards('filebeat') }} #============================== Template ====================================== -{{ elk_macros.setup_template('filebeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +{{ elk_macros.setup_template('filebeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Setup ILM ===================================== diff --git a/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 b/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 index fa574cc1..c714e21a 100644 --- a/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 @@ -994,7 +994,7 @@ processors: {{ elk_macros.setup_dashboards('heartbeat') }} #============================== Template ===================================== -{{ elk_macros.setup_template('heartbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +{{ elk_macros.setup_template('heartbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Setup ILM ===================================== diff --git a/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 b/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 index 1d1807f5..864bd7b2 100644 --- a/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 @@ -749,7 +749,7 @@ processors: {{ elk_macros.setup_dashboards('journalbeat') }} #============================== Template ===================================== -{{ elk_macros.setup_template('journalbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +{{ elk_macros.setup_template('journalbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Setup ILM ===================================== diff --git a/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 b/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 index e9ab2bd4..52e959e9 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 @@ -1460,8 +1460,8 @@ processors: {{ elk_macros.setup_dashboards('metricbeat') }} #============================== Template ===================================== -{{ elk_macros.setup_template('metricbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} - +{{ elk_macros.setup_template('metricbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} + #============================== Setup ILM ===================================== # Configure Index Lifecycle Management Index Lifecycle Management creates a diff --git a/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 b/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 index dc947f4c..8ae35f18 100644 --- a/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 @@ -1185,7 +1185,7 @@ processors: {{ elk_macros.setup_dashboards('packetbeat') }} #============================== Template ===================================== -{{ elk_macros.setup_template('packetbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +{{ elk_macros.setup_template('packetbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Setup ILM ===================================== diff --git a/elk_metrics_7x/templates/_macros.j2 b/elk_metrics_7x/templates/_macros.j2 index 9147af4a..2f52352c 100644 --- a/elk_metrics_7x/templates/_macros.j2 +++ b/elk_metrics_7x/templates/_macros.j2 @@ -249,7 +249,7 @@ setup.dashboards.enabled: false #setup.dashboards.retry.maximum: 0 {%- endmacro %} -{% macro setup_template(beat_name, host, data_nodes, elasticsearch_replicas) -%} +{% macro setup_template(beat_name, host, data_nodes, elasticsearch_beat_settings={}) -%} # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. # These settings can be adjusted to load your own template or overwrite existing ones. @@ -296,7 +296,11 @@ setup.template.settings: number_of_routing_shards: {{ (shards | int) * 2 }} # The default number of replicas will be based on the number of data nodes # within the environment with a limit of 2 replicas. - number_of_replicas: {{ elasticsearch_replicas | int }} + number_of_replicas: {{ elasticsearch_beat_settings.number_of_replicas | int }} + # Maximum doc values allowed for default kibana search +{% if 'max_docvalue_fields_search' in elasticsearch_beat_settings %} + max_docvalue_fields_search: {{ elasticsearch_beat_settings.max_docvalue_fields_search | int }} +{% endif %} # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html