Tune ElasticSearch settings.

* modules/logstash/templates/elasticsearch.yml.erb:
Give 33% of memory to indexing instead of 40%.

* modules/logstash/files/es-logstash-template.json:
Flush the translog every 50k operations instead of the default 5k
(we can do 5k ops every second or two so the flushing was happening too
often). Limit the total number of shards per index per node to 3. We
have 5 nodes and 10 shards per index, 5*3 > 10 so we should be fine.
Set the field cache to soft type so that its entries can be garbage
collected in memory constricted situations.

* modules/logstash/manifests/elasticsearch.pp:
Apply es-logstash-template.json to the elasticsearch servers.

Change-Id: I2337fc41998fd00e090b0acfd29f007dfb6ec8df

modules/logstash/files/es-logstash-template.json

@@ -0,0 +1,10 @@
+{
+    "logstash_settings" : {
+        "template" : "logstash*",
+        "settings" : {
+            "index.routing.allocation.total_shards_per_node" : "3",
+            "index.translog.flush_threshold_ops" : "50000",
+            "index.cache.field.type" : "soft"
+        }
+    }
+}

modules/logstash/manifests/elasticsearch.pp

@@ -50,6 +50,24 @@ class logstash::elasticsearch (
     require => Package['elasticsearch'],
   }
 
+  file { '/etc/elasticsearch/templates':
+    ensure  => directory,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    require => Package['elasticsearch'],
+  }
+
+  file { '/etc/elasticsearch/templates/logstash_settings.json':
+    ensure  => present,
+    source  => 'puppet:///modules/logstash/es-logstash-template.json',
+    replace => true,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    require => File['/etc/elasticsearch/templates'],
+  }
+
   file { '/etc/elasticsearch/default-mapping.json':
     ensure  => present,
     source  => 'puppet:///modules/logstash/elasticsearch.mapping.json',

modules/logstash/templates/elasticsearch.yml.erb

@@ -139,7 +139,7 @@ node.name: "<%= scope.lookupvar("::hostname") %>"
 index.store.compress.stored: true
 index.store.compress.tv: true
 
-indices.memory.index_buffer_size: "40%"
+indices.memory.index_buffer_size: "33%"
 
 #################################### Paths ####################################