Use local ES balancers on logstash workers

Set up small non master non data elasticsearch daemons on logstash
workers to act as local load balancers for the elasticsearch http
protocol.

Change-Id: Ie3729f851ebef3331a6b69f718e57d663209bfc2

modules/elasticsearch/manifests/init.pp

@@ -20,15 +20,19 @@ class elasticsearch (
   $es_template_config = {}
 ) {
   # install java runtime
+  if ! defined(Package['openjdk-7-jre-headless']) {
     package { 'openjdk-7-jre-headless':
       ensure => present,
     }
+  }
 
   # Curl is handy for talking to the ES API on localhost. Allows for
   # querying cluster state and deleting indexes and so on.
+  if ! defined(Package['curl']) {
     package { 'curl':
       ensure => present,
     }
+  }
 
   exec { 'get_elasticsearch_deb':
     command => "wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${version}.deb -O /tmp/elasticsearch-${version}.deb",

modules/elasticsearch/templates/elasticsearch.yml.erb

@@ -86,6 +86,13 @@ node.name: "<%= scope.lookupvar("::hostname") %>"
 # to disable it, set the following:
 # node.max_local_storage_nodes: 1
 
+<% if es_template_config.has_key?('node.master') then -%>
+node.master: <%= es_template_config['node.master'] %>
+<% end -%>
+<% if es_template_config.has_key?('node.data') then -%>
+node.data: <%= es_template_config['node.data'] %>
+<% end -%>
+
 
 #################################### Index ####################################
 

modules/logstash/manifests/init.pp

@@ -75,7 +75,9 @@ class logstash {
     mode   => '0644',
   }
 
+  if ! defined(Package['openjdk-7-jre-headless']) {
     package { 'openjdk-7-jre-headless':
       ensure => present,
     }
   }
+}

modules/logstash/manifests/watchdog.pp

@@ -23,9 +23,11 @@ class logstash::watchdog (
   package { 'jq':
     ensure => present,
   }
+  if ! defined(Package['curl']) {
     package { 'curl':
       ensure => present,
     }
+  }
 
   file { '/usr/local/bin/logstash-watchdog':
     ensure  => present,

modules/openstack_project/manifests/logstash_worker.pp

@@ -57,4 +57,19 @@ class openstack_project::logstash_worker (
   log_processor::worker { 'D':
     config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
   }
+
+  class { '::elasticsearch':
+    es_template_config => {
+      'gateway.recover_after_nodes'          => '5',
+      'gateway.recover_after_time'           => '5m',
+      'gateway.expected_nodes'               => '6',
+      'discovery.zen.minimum_master_nodes'   => '5',
+      'discovery.zen.ping.multicast.enabled' => false,
+      'discovery.zen.ping.unicast.hosts'     => $elasticsearch_nodes,
+      'node.master'                          => false,
+      'node.data'                            => false,
+    },
+    heap_size          => '1g',
+    version            => '0.90.9',
+  }
 }

modules/openstack_project/templates/logstash/indexer.conf.erb

@@ -125,8 +125,8 @@ filter {
 
 output {
   elasticsearch_http {
-    host => "<%= scope.lookupvar("::openstack_project::logstash_worker::discover_node") %>"
+    host => "localhost"
     manage_template => false
-    flush_size => 512
+    flush_size => 1024
   }
 }