From 6ebca3d78ffc856c9ba7f6a423354978556aeafd Mon Sep 17 00:00:00 2001
From: Kevin Carter <kevin.carter@rackspace.com>
Date: Wed, 2 May 2018 15:31:54 -0500
Subject: [PATCH] Correct cluster properties

When running a multi-node ELK stack (5+ nodes) the elasticsearch server
needs to be able to mark and identify nodes that will be data and master
nodes. This change ensures that the first set of available master nodes
is mark accordingly as both data and master and that all other nodes are
marked as cluster members with alternating nodes being marked as data
nodes. With this change the cluster will be able to grow as well as
handle large data sets more efficiently.

This change ensures that sharding is correctly started when the cluster
is deployed and that the beat services are using the recommended
sharding practices.

Change-Id: Id555132199f268b21aaa939a84760c744238dd2b
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
---
 elk_metrics_6x/installElastic.yml             |  2 +
 elk_metrics_6x/installKibana.yml              |  2 +-
 elk_metrics_6x/installLogstash.yml            |  2 +-
 elk_metrics_6x/readme.rst                     |  1 +
 .../templates/_include_setup_template.yml.j2  | 18 ++++
 elk_metrics_6x/templates/apm-server.yml.j2    | 17 +---
 elk_metrics_6x/templates/auditbeat.yml.j2     | 17 +---
 elk_metrics_6x/templates/elasticsearch.yml.j2 | 23 +++--
 .../templates/es-log4j2.properties.j2         | 85 +++++++++++++++++++
 elk_metrics_6x/templates/filebeat.yml.j2      | 17 +---
 elk_metrics_6x/templates/heartbeat.yml.j2     | 17 +---
 elk_metrics_6x/templates/journalbeat.yml.j2   | 17 +---
 elk_metrics_6x/templates/metricbeat.yml.j2    | 19 +----
 elk_metrics_6x/templates/packetbeat.yml.j2    | 17 +---
 14 files changed, 131 insertions(+), 123 deletions(-)
 create mode 100644 elk_metrics_6x/templates/_include_setup_template.yml.j2
 create mode 100644 elk_metrics_6x/templates/es-log4j2.properties.j2

diff --git a/elk_metrics_6x/installElastic.yml b/elk_metrics_6x/installElastic.yml
index fee27dc1..f2e7905a 100644
--- a/elk_metrics_6x/installElastic.yml
+++ b/elk_metrics_6x/installElastic.yml
@@ -67,6 +67,8 @@
           dest: /etc/elasticsearch/elasticsearch.yml
         - src: templates/jvm.options.j2
           dest: /etc/elasticsearch/jvm.options
+        - src: templates/es-log4j2.properties.j2
+          dest: /etc/elasticsearch/log4j2.properties
       tags:
         - config
 
diff --git a/elk_metrics_6x/installKibana.yml b/elk_metrics_6x/installKibana.yml
index 96d21664..3e00a426 100644
--- a/elk_metrics_6x/installKibana.yml
+++ b/elk_metrics_6x/installKibana.yml
@@ -36,7 +36,7 @@
         enabled: true
         state: restarted
 
-    - name: Ensure kibana  is installed.
+    - name: Ensure kibana  is installed
       apt:
         name: kibana
         state: present
diff --git a/elk_metrics_6x/installLogstash.yml b/elk_metrics_6x/installLogstash.yml
index 5f16c6b2..9edb925f 100644
--- a/elk_metrics_6x/installLogstash.yml
+++ b/elk_metrics_6x/installLogstash.yml
@@ -52,7 +52,7 @@
         install_recommends: yes
         update_cache: yes
 
-    - name: Ensure Logstash is installed.
+    - name: Ensure Logstash is installed
       apt:
         name: logstash
         state: present
diff --git a/elk_metrics_6x/readme.rst b/elk_metrics_6x/readme.rst
index 1ee32af9..58144c15 100644
--- a/elk_metrics_6x/readme.rst
+++ b/elk_metrics_6x/readme.rst
@@ -70,6 +70,7 @@ lines.
           haproxy_backend_port: 9200  # This is set using the "elastic_port" variable
           haproxy_balance_type: tcp
 
+
 Optional | run the haproxy-install playbook
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/elk_metrics_6x/templates/_include_setup_template.yml.j2 b/elk_metrics_6x/templates/_include_setup_template.yml.j2
new file mode 100644
index 00000000..84820180
--- /dev/null
+++ b/elk_metrics_6x/templates/_include_setup_template.yml.j2
@@ -0,0 +1,18 @@
+{% set shards = ((groups["elastic-logstash"] | length) * 3) | int %}
+
+# Elasticsearch template settings
+setup.template.settings:
+
+  # A dictionary of settings to place into the settings.index dictionary
+  # of the Elasticsearch template. For more details, please check
+  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
+  index:
+    number_of_shards: {{ shards }}
+    codec: best_compression
+    # This provides for an index split of up to 2 times the number of available shards
+    number_of_routing_shards: {{ (shards | int) * 2 * (groups["elastic-logstash"] | length) }}
+
+  # A dictionary of settings for the _source field. For more details, please check
+  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
+  #_source:
+    #enabled: false
diff --git a/elk_metrics_6x/templates/apm-server.yml.j2 b/elk_metrics_6x/templates/apm-server.yml.j2
index 78751537..9b17a0db 100644
--- a/elk_metrics_6x/templates/apm-server.yml.j2
+++ b/elk_metrics_6x/templates/apm-server.yml.j2
@@ -275,22 +275,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}
diff --git a/elk_metrics_6x/templates/auditbeat.yml.j2 b/elk_metrics_6x/templates/auditbeat.yml.j2
index 6e1de8e8..d85c5032 100644
--- a/elk_metrics_6x/templates/auditbeat.yml.j2
+++ b/elk_metrics_6x/templates/auditbeat.yml.j2
@@ -716,22 +716,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}
diff --git a/elk_metrics_6x/templates/elasticsearch.yml.j2 b/elk_metrics_6x/templates/elasticsearch.yml.j2
index 517cdb01..10af56cd 100644
--- a/elk_metrics_6x/templates/elasticsearch.yml.j2
+++ b/elk_metrics_6x/templates/elasticsearch.yml.j2
@@ -43,17 +43,24 @@ http.port: {{ elastic_port }}
 #
 {% set IP_ARR=[] %}
 {% for host in groups['elastic-logstash'] %}
-{%   if IP_ARR.insert(loop.index,hostvars[host]['ansible_host']) %}
+{%   if IP_ARR.extend([hostvars[host]['ansible_host'] | string]) %}
 {%   endif %}
 {% endfor %}
-discovery.zen.ping.unicast.hosts: [{{ IP_ARR | join(', ') }}]
-node.master: {{ (inventory_hostname == groups['elastic-logstash'][0]) | ternary(true, false) }}
-node.data: true
+{% set available_nodes = (groups['elastic-logstash'] | length) %}
+{# the master node count takes half the available nodes or sets it's self as 1 #}
+{% set _master_node_count = (available_nodes > 1) | ternary(((available_nodes // 2) | int), 1) %}
+{# if the master node count is even, add one to it otherwise use the provided value #}
+{% set master_node_count = ((_master_node_count | int) % 2 != 0) | ternary(_master_node_count, (_master_node_count + 1)) %}
+
+discovery.zen.ping.unicast.hosts: {{ IP_ARR }}
 #
 # Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
-#
-# discovery.zen.minimum_master_nodes: 3
-#
+discovery.zen.minimum_master_nodes: {{ master_node_count | int }}
+# The first set of nodes in the master_node_count are marked as such
+node.master: {{ (inventory_hostname in groups['elastic-logstash'][:master_node_count | int]) | ternary(true, false) }}
+# Every node in the master list and every other node after will be a data node
+node.data: {{ (inventory_hostname in (groups['elastic-logstash'][:master_node_count| int] + groups['elastic-logstash'][master_node_count | int::2])) | ternary(true, false) }}
+
 # For more information, see the documentation at:
 # <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
 #
@@ -61,7 +68,7 @@ node.data: true
 #
 # Block initial recovery after a full cluster restart until N nodes are started:
 #
-# gateway.recover_after_nodes: 3
+gateway.recover_after_nodes: {{ master_node_count }}
 #
 # For more information, see the documentation at:
 # <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>
diff --git a/elk_metrics_6x/templates/es-log4j2.properties.j2 b/elk_metrics_6x/templates/es-log4j2.properties.j2
new file mode 100644
index 00000000..7091d480
--- /dev/null
+++ b/elk_metrics_6x/templates/es-log4j2.properties.j2
@@ -0,0 +1,85 @@
+status = error
+
+# log action execution errors for easier debugging
+logger.action.name = org.elasticsearch.action
+logger.action.level = debug
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+
+appender.rolling.type = RollingFile
+appender.rolling.name = rolling
+appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz
+appender.rolling.policies.type = Policies
+appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval = 1
+appender.rolling.policies.time.modulate = true
+appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size = 128MB
+appender.rolling.strategy.type = DefaultRolloverStrategy
+appender.rolling.strategy.fileIndex = nomax
+appender.rolling.strategy.action.type = Delete
+appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path}
+appender.rolling.strategy.action.condition.type = IfFileName
+appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-*
+appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
+appender.rolling.strategy.action.condition.nested_condition.age = {{ elastic_logstash_retention }}D
+
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.rolling.ref = rolling
+
+appender.deprecation_rolling.type = RollingFile
+appender.deprecation_rolling.name = deprecation_rolling
+appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
+appender.deprecation_rolling.layout.type = PatternLayout
+appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
+appender.deprecation_rolling.policies.type = Policies
+appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.deprecation_rolling.policies.size.size = 128MB
+appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
+appender.deprecation_rolling.strategy.max = 2
+
+logger.deprecation.name = org.elasticsearch.deprecation
+logger.deprecation.level = warn
+logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
+logger.deprecation.additivity = false
+
+appender.index_search_slowlog_rolling.type = RollingFile
+appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
+appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
+appender.index_search_slowlog_rolling.layout.type = PatternLayout
+appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
+appender.index_search_slowlog_rolling.policies.type = Policies
+appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_search_slowlog_rolling.policies.time.interval = 1
+appender.index_search_slowlog_rolling.policies.time.modulate = true
+
+logger.index_search_slowlog_rolling.name = index.search.slowlog
+logger.index_search_slowlog_rolling.level = trace
+logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
+logger.index_search_slowlog_rolling.additivity = false
+
+appender.index_indexing_slowlog_rolling.type = RollingFile
+appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
+appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
+appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
+appender.index_indexing_slowlog_rolling.policies.type = Policies
+appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_indexing_slowlog_rolling.policies.time.interval = 1
+appender.index_indexing_slowlog_rolling.policies.time.modulate = true
+
+logger.index_indexing_slowlog.name = index.indexing.slowlog.index
+logger.index_indexing_slowlog.level = trace
+logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
+logger.index_indexing_slowlog.additivity = false
diff --git a/elk_metrics_6x/templates/filebeat.yml.j2 b/elk_metrics_6x/templates/filebeat.yml.j2
index b6c8e64e..1721ab2d 100644
--- a/elk_metrics_6x/templates/filebeat.yml.j2
+++ b/elk_metrics_6x/templates/filebeat.yml.j2
@@ -1764,22 +1764,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}
diff --git a/elk_metrics_6x/templates/heartbeat.yml.j2 b/elk_metrics_6x/templates/heartbeat.yml.j2
index 7e211fd3..f3840670 100644
--- a/elk_metrics_6x/templates/heartbeat.yml.j2
+++ b/elk_metrics_6x/templates/heartbeat.yml.j2
@@ -825,22 +825,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}
diff --git a/elk_metrics_6x/templates/journalbeat.yml.j2 b/elk_metrics_6x/templates/journalbeat.yml.j2
index 0445a313..00e5e84a 100644
--- a/elk_metrics_6x/templates/journalbeat.yml.j2
+++ b/elk_metrics_6x/templates/journalbeat.yml.j2
@@ -631,22 +631,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}
diff --git a/elk_metrics_6x/templates/metricbeat.yml.j2 b/elk_metrics_6x/templates/metricbeat.yml.j2
index 8d3b0c9b..fd48861b 100644
--- a/elk_metrics_6x/templates/metricbeat.yml.j2
+++ b/elk_metrics_6x/templates/metricbeat.yml.j2
@@ -104,7 +104,7 @@ metricbeat.modules:
 
   # Include the cumulative CPU tick values with the process metrics. Defaults
   # to false.
-  process.include_cpu_ticks: {{ inventory_hostname == physical_host }}
+  #process.include_cpu_ticks: false
 
   # Configure reverse DNS lookup on remote IP addresses in the socket metricset.
   socket.reverse_lookup.enabled: true
@@ -1085,22 +1085,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}
diff --git a/elk_metrics_6x/templates/packetbeat.yml.j2 b/elk_metrics_6x/templates/packetbeat.yml.j2
index 090fff2e..b2ffc80f 100644
--- a/elk_metrics_6x/templates/packetbeat.yml.j2
+++ b/elk_metrics_6x/templates/packetbeat.yml.j2
@@ -1104,22 +1104,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}