From 5a8351635d9842aa9f518d51a22d47a47ddb47cd Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Tue, 11 Feb 2014 17:03:14 -0800
Subject: [PATCH] Upgrade to logstash 1.3.3 and ElasticSearch 0.90.9

Upgrade to logstash 1.3.3 and ElasticSearch 0.90.9. Logstash 1.3.3 comes
with many performance improvements and bug fixes and needs ElasticSearch
0.90.9 to use the ElasticSearch output so upgrade ES as well. These
upgrades come with config changes for logstash as well to use new
features and disable others that are not wanted.

Change-Id: Ia180c5edfa755214a801f2cad06f091125814b0e
---
 modules/logstash/manifests/init.pp                     | 10 +++++-----
 .../openstack_project/manifests/elasticsearch_node.pp  |  2 +-
 .../templates/logstash/indexer.conf.erb                | 10 +++-------
 3 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/modules/logstash/manifests/init.pp b/modules/logstash/manifests/init.pp
index 08513299ad..94a1c9f76f 100644
--- a/modules/logstash/manifests/init.pp
+++ b/modules/logstash/manifests/init.pp
@@ -38,13 +38,13 @@ class logstash {
   }
 
   exec { 'get_logstash_jar':
-    command => 'wget https://download.elasticsearch.org/logstash/logstash/logstash-1.2.1-flatjar.jar -O /opt/logstash/logstash-1.2.1-flatjar.jar',
+    command => 'wget https://download.elasticsearch.org/logstash/logstash/logstash-1.3.3-flatjar.jar -O /opt/logstash/logstash-1.3.3-flatjar.jar',
     path    => '/bin:/usr/bin',
-    creates => '/opt/logstash/logstash-1.2.1-flatjar.jar',
+    creates => '/opt/logstash/logstash-1.3.3-flatjar.jar',
     require => File['/opt/logstash'],
   }
 
-  file { '/opt/logstash/logstash-1.2.1-flatjar.jar':
+  file { '/opt/logstash/logstash-1.3.3-flatjar.jar':
     ensure  => present,
     owner   => 'logstash',
     group   => 'logstash',
@@ -57,8 +57,8 @@ class logstash {
 
   file { '/opt/logstash/logstash.jar':
     ensure  => link,
-    target  => '/opt/logstash/logstash-1.2.1-flatjar.jar',
-    require => File['/opt/logstash/logstash-1.2.1-flatjar.jar'],
+    target  => '/opt/logstash/logstash-1.3.3-flatjar.jar',
+    require => File['/opt/logstash/logstash-1.3.3-flatjar.jar'],
   }
 
   file { '/var/log/logstash':
diff --git a/modules/openstack_project/manifests/elasticsearch_node.pp b/modules/openstack_project/manifests/elasticsearch_node.pp
index 76b6d1bc09..fd1efff87c 100644
--- a/modules/openstack_project/manifests/elasticsearch_node.pp
+++ b/modules/openstack_project/manifests/elasticsearch_node.pp
@@ -45,7 +45,7 @@ class openstack_project::elasticsearch_node (
       'discovery.zen.ping.multicast.enabled' => false,
       'discovery.zen.ping.unicast.hosts'     => $discover_nodes,
     },
-    version            => '0.90.3',
+    version            => '0.90.9',
   }
 
   cron { 'delete_old_es_indices':
diff --git a/modules/openstack_project/templates/logstash/indexer.conf.erb b/modules/openstack_project/templates/logstash/indexer.conf.erb
index 69a564e87c..3ad2c3e7ee 100644
--- a/modules/openstack_project/templates/logstash/indexer.conf.erb
+++ b/modules/openstack_project/templates/logstash/indexer.conf.erb
@@ -2,19 +2,13 @@ input {
   tcp {
     host => "localhost"
     port => 9999
-    codec => line {}
+    codec => json_lines {}
     type => "jenkins"
   }
 }
 
 # You can check grok patterns at http://grokdebug.herokuapp.com/
 filter {
-  # This is a work around for a bug. We should be able to set the tcp
-  # input codec to json, but that codec doesn't support streaming.
-  # Convert to json here instead.
-  json {
-    source => "message"
-  }
   if "screen" in [tags] and [message] =~ "^\+ " {
     drop {}
   }
@@ -118,5 +112,7 @@ output {
   elasticsearch {
     host => "<%= scope.lookupvar("::openstack_project::logstash_worker::discover_node") %>"
     node_name => "<%= scope.lookupvar("::hostname") %>"
+    manage_template => false
+    flush_size => 512
   }
 }