Add options for better memory management

This change allows EL to have customizable heap sizes which will
automatically set themselves to the recommended size as advised by the
elastic documentation.

https://www.elastic.co/guide/en/elasticsearch/reference/6.2/heap-size.html

Change-Id: I55d014dab37bf5e5a4806bd19d4e07e072f9fa7e
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/installElastic.yml

@@ -5,6 +5,16 @@
   vars_files:
     - vars/variables.yml
   tasks:
+    - name: Set half memory fact
+      set_fact:
+        h_mem: "{{ ansible_memtotal_mb // 2 }}"
+
+    - name: Set logstash facts
+      set_fact:
+        elastic_heap_size: "{{ ((h_mem | int) > 30720) | ternary(30720, h_mem) }}"
+      when:
+        - elastic_heap_size is undefined
+
     - name: Configure systcl vm.max_map_count=262144 on container hosts
       sysctl:
         name: "vm.max_map_count"
@@ -48,10 +58,15 @@
         state: present
         update_cache: yes
 
-    - name: Drop elastic search conf file
+    - name: Drop elasticsearch conf file
       template:
-        src: templates/elasticsearch.yml.j2
-        dest: /etc/elasticsearch/elasticsearch.yml
+        src: "{{ item.src }}"
+        dest: "{{ item.dest }}"
+      with_items:
+        - src: templates/elasticsearch.yml.j2
+          dest: /etc/elasticsearch/elasticsearch.yml
+        - src: templates/jvm.options.j2
+          dest: /etc/elasticsearch/jvm.options
       tags:
         - config
 

elk_metrics_6x/installKibana.yml

@@ -7,7 +7,7 @@
   tasks:
     - include_tasks: common_task_install_elk_repo.yml
 
-    - name: Ensure Nginx is installed.
+    - name: Ensure Nginx is installed
       apt:
         name: "{{ item }}"
         state: present

elk_metrics_6x/installLogstash.yml

@@ -5,6 +5,16 @@
   vars_files:
     - vars/variables.yml
   tasks:
+    - name: Set quarter memory fact
+      set_fact:
+        q_mem: "{{ ansible_memtotal_mb // 4 }}"
+
+    - name: Set logstash facts
+      set_fact:
+        elastic_heap_size: "{{ ((q_mem | int) > 30720) | ternary(30720, q_mem) }}"
+      when:
+        - elastic_heap_size is undefined
+
     - include_tasks: common_task_install_elk_repo.yml
 
     - name: Configure systcl vm.max_map_count=262144 on container hosts
@@ -63,6 +73,16 @@
         src: templates/30-elasticsearch-output.conf.j2
         dest: /etc/logstash/conf.d/30-elasticsearch-output.conf
 
+    - name: Drop elasticsearch conf file
+      template:
+        src: "{{ item.src }}"
+        dest: "{{ item.dest }}"
+      with_items:
+        - src: templates/jvm.options.j2
+          dest: /etc/logstash/jvm.options
+      tags:
+        - config
+
     - name: Ensure logstash ownership
       file:
         path: /var/lib/logstash
@@ -70,15 +90,6 @@
         group: logstash
         recurse: true
 
-    - name: Load logstash config
-      command: "/usr/share/logstash/bin/logstash -t --path.settings /etc/logstash"
-      register: conf_success
-      become: yes
-      become_user: logstash
-
-    - name: Print config output
-      debug: var=conf_success
-
     - name: Enable and restart logstash
       systemd:
         name: "logstash"

elk_metrics_6x/readme.rst

@@ -99,7 +99,7 @@ Create the containers
 
 .. code-block:: bash
 
-   cd /opt/openstack-ansible-playbooks
+   cd /opt/openstack-ansible/playbooks
    openstack-ansible lxc-containers-create.yml -e 'container_group=elastic-logstash:kibana'
 
 install master/data elasticsearch nodes on the elastic-logstash containers

elk_metrics_6x/templates/elasticsearch.yml.j2

@@ -22,7 +22,7 @@ path.logs: /var/log/elasticsearch/
 #
 # Lock the memory on startup:
 #
-# bootstrap.memory_lock: true
+bootstrap.memory_lock: false
 #
 # Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory
 # available on the system and that the owner of the process is allowed to use this limit.

elk_metrics_6x/templates/jvm.options.j2

@@ -0,0 +1,74 @@
+## JVM configuration
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms{{ elastic_heap_size }}m
+-Xmx{{ elastic_heap_size }}m
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseParNewGC
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## optimizations
+
+# disable calls to System#gc
+-XX:+DisableExplicitGC
+
+## locale
+# Set the locale language
+#-Duser.language=en
+
+# Set the locale country
+#-Duser.country=US
+
+# Set the locale variant, if any
+#-Duser.variant=
+
+## basic
+
+# set the I/O temp directory
+#-Djava.io.tmpdir=$HOME
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+#-Djna.nosys=true
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof
+
+## GC logging
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${LS_GC_LOG_FILE}