From 0cd1f76757cf96fbe3490dd8255f2619643a43b1 Mon Sep 17 00:00:00 2001 From: Duncan Martin Walker Date: Thu, 30 Apr 2020 14:11:27 +0100 Subject: [PATCH] Alter default shard count in index templates Currently, we use only a single primary shard for each beat index, as configured by the index templates. This is not usually an optimal setup, as all index searching is handled on a single node. A more efficient configuration for a small number of different indices is to balance, with around one primary shard per node. This commit exposes the config option to set the number of primary shards, which was previously hardcoded to 1. This now defaults to one shard per data node in the cluster. Change-Id: Icacf76c2270c98f8676d57952c00773646ca7b51 --- elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml | 3 +++ .../roles/elastic_data_hosts/vars/data-node-variables.yml | 1 + elk_metrics_7x/templates/_macros.j2 | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml b/elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml index edc8bba0..1a1db482 100644 --- a/elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml @@ -34,6 +34,9 @@ elastic_data_interface: "{{ ansible_default_ipv4['alias'] }}" ## Elasticsearch index template settings # Set the default max number of fields used in a query # elastic_max_docvalue_fields_search: "100" +# Set the number of primary shards for each index +# (defaults to the number of Elasticsearch data nodes) +# elastic_primary_shard_count: 1 # The set of nodes used for elasticsearch discovery # elasticsearch_discovery_seed_hosts: "{{ (elastic_hosts | union(groups['kibana'])) | map('extract', hostvars, 'ansible_host') | list }}" diff --git a/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml b/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml index 3bc23c57..b8958476 100644 --- a/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml +++ b/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml @@ -169,6 +169,7 @@ elasticsearch_number_of_replicas: "{{ ((data_nodes | length) > 2) | ternary('2', elasticsearch_beat_settings: number_of_replicas: "{{ elasticsearch_number_of_replicas }}" max_docvalue_fields_search: "{{ elastic_max_docvalue_fields_search | default('100') }}" + shard_count: "{{ elastic_primary_shard_count | default(elasticsearch_data_node_details | length) }}" # Shuffled elasticsearch endpoints (with localhost if relevant) for use in beat config files elasticsearch_data_hosts: |- diff --git a/elk_metrics_7x/templates/_macros.j2 b/elk_metrics_7x/templates/_macros.j2 index 325eb449..b6ba9940 100644 --- a/elk_metrics_7x/templates/_macros.j2 +++ b/elk_metrics_7x/templates/_macros.j2 @@ -281,7 +281,7 @@ setup.template.fields: "${path.config}/fields.yml" # Overwrite existing template setup.template.overwrite: {{ host == data_nodes[0] | default(false)}} -{% set shards = 1 %} +{% set shards = elasticsearch_beat_settings.shard_count | int %} # Elasticsearch template settings setup.template.settings: