Add additional ES cluster tuning

The following options will reduce cluster pressure and generally
improve search performance.

Change-Id: I1619680db1fd595503f0845b182d6f6ce4c59f3c
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/common_task_data_node_hosts.yml

@@ -49,6 +49,39 @@
       {%     set _ = nodes.insert(loop.index, (hostvars[host]['ansible_host'] | string)) %}
       {%   endfor %}
       {{ nodes }}
+    elasticserch_interface_speed: |-
+      {% set default_interface_fact = hostvars[inventory_hostname]['ansible_' + (elastic_data_interface | replace('-', '_'))] %}
+      {% set speeds = [] %}
+      {% if default_interface_fact['type'] == 'bridge' %}
+      {%   for interface in default_interface_fact['interfaces'] %}
+      {%     set interface_fact = hostvars[inventory_hostname]['ansible_' + (interface | replace('-', '_'))] %}
+      {%     if 'speed' in interface_fact %}
+      {%       set speed = (interface_fact['speed'] | default(1000)) | string %}
+      {%       if speed == "-1" %}
+      {%         set _ = speeds.append(1000) %}
+      {%       else %}
+      {%         set _ = speeds.append(speed | int) %}
+      {%       endif %}
+      {%       if 'module' in interface_fact %}
+      {%         set _ = speeds.append((interface_fact['speed'] | default(1000)) | int) %}
+      {%       else %}
+      {%         set _ = speeds.append(1000) %}
+      {%       endif %}
+      {%     endif %}
+      {%   endfor %}
+      {% else %}
+      {%   if ('module' in default_interface_fact) or (default_interface_fact['type'] == 'bond') %}
+      {%     set speed = (default_interface_fact['speed'] | default(1000)) | string %}
+      {%     if speed == "-1" %}
+      {%       set _ = speeds.append(1000) %}
+      {%     else %}
+      {%       set _ = speeds.append(speed | int) %}
+      {%     endif %}
+      {%   else %}
+      {%     set _ = speeds.append(1000) %}
+      {%   endif %}
+      {% endif %}
+      {{ ((speeds | min) * 0.75) | int }}
   tags:
     - always
 

elk_metrics_6x/createElasticIndexes.yml

@@ -1,5 +1,5 @@
 ---
-- name: Create known indexes in Elasticsearch
+- name: Create/Setup known indexes in Elasticsearch
   hosts: "elastic-logstash[0]"
   become: true
 
@@ -31,3 +31,9 @@
                     limit: "10000"
                 refresh_interval: "5s"
                 number_of_replicas: "1"
+        - name: "_all/_settings?preserve_existing=true"
+          index_options:
+            index.refresh_interval: "1m"
+        - name: "_all/_settings?preserve_existing=true"
+          index_options:
+            index.queries.cache.enabled: "false"

elk_metrics_6x/installElastic.yml

@@ -6,6 +6,9 @@
   vars_files:
     - vars/variables.yml
 
+  vars:
+    temp_dir: /var/lib/elasticsearch/tmp
+
   environment: "{{ deployment_environment_variables | default({}) }}"
 
   pre_tasks:
@@ -143,6 +146,14 @@
       tags:
         - config
 
+    - name: Ensure logstash tmp dir
+      file:
+        path: "/var/lib/elasticsearch/tmp"
+        state: directory
+        owner: "elasticsearch"
+        group: "elasticsearch"
+        mode: "0750"
+
   handlers:
     - name: Enable and restart elastic
       systemd:

elk_metrics_6x/installLogstash.yml

@@ -5,6 +5,9 @@
   vars_files:
     - vars/variables.yml
 
+  vars:
+    temp_dir: /var/lib/logstash/tmp
+
   environment: "{{ deployment_environment_variables | default({}) }}"
 
   pre_tasks:
@@ -215,6 +218,14 @@
       retries: 3
       delay: 1
 
+    - name: Ensure logstash tmp dir
+      file:
+        path: "/var/lib/logstash/tmp"
+        state: directory
+        owner: "logstash"
+        group: "logstash"
+        mode: "0750"
+
   handlers:
     - name: Enable and restart logstash
       systemd:

elk_metrics_6x/site-elka.yml

@@ -18,4 +18,3 @@
 - import_playbook: installCurator.yml
 - import_playbook: installKibana.yml
 - import_playbook: installAPMserver.yml
-- import_playbook: createElasticIndexes.yml

elk_metrics_6x/site.yml

@@ -16,3 +16,4 @@
 - import_playbook: site-elka.yml
 - import_playbook: site-beats-core.yml
 - import_playbook: site-beats-community.yml
+- import_playbook: createElasticIndexes.yml

elk_metrics_6x/templates/elasticsearch.yml.j2

@@ -105,3 +105,12 @@ thread_pool:
   bulk:
     size: {{ thread_pool_size }}
     queue_size: {{ thread_pool_size * 512 }}
+
+# Accepts either a percentage or a byte size value. Set to 30%, meaning that 30%
+# of the total heap allocated to a node will be used as the indexing buffer size
+# shared across all shards.
+indices.memory.index_buffer_size: 30%
+
+# Connection throttling on recovery is limited to 75% of the detected interface
+# speed. This will improce search speeds and reduce general cluster pressure.
+indices.recovery.max_bytes_per_sec: {{ elasticserch_interface_speed }}mb

elk_metrics_6x/templates/jvm.options.j2

@@ -45,7 +45,7 @@
 ## basic
 
 # set the I/O temp directory
-#-Djava.io.tmpdir=$HOME
+-Djava.io.tmpdir={{ temp_dir }}
 
 # set to headless, just in case
 -Djava.awt.headless=true

elk_metrics_6x/vars/variables.yml

@@ -13,6 +13,8 @@ elastic_hap_port: 9201
 cluster_name: openstack_elk
 node_name: ${HOSTNAME}
 
+elastic_data_interface: "{{ ansible_default_ipv4['alias'] }}"
+
 ### Elastic curator variables
 ## Default retention policy options. All retention options are in days.
 #  elastic_logstash_retention: 1