From c385590720a1ecb9cac6f8cb96d600d68ecd8ae4 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Mon, 25 Apr 2016 13:04:45 -0700
Subject: [PATCH] Use logstash filters from filters repo

This uses new logstash conf.d features to properly link in the
configuration from the logstash-filters repo. This should make
configuring logstash far more flexible and reconsumable.

Change-Id: Ia304eb7e73c82ca5ce85967fbf442bb4bc5f8b7a
Depends-On: Icbca7a6ba0c5a94a273ef158f707311b588483fd
---
 .../manifests/logstash_worker.pp              | 27 ++++++++++++++++---
 .../templates/logstash/input.conf.erb         |  8 ++++++
 .../templates/logstash/output.conf.erb        |  7 +++++
 3 files changed, 39 insertions(+), 3 deletions(-)
 create mode 100644 modules/openstack_project/templates/logstash/input.conf.erb
 create mode 100644 modules/openstack_project/templates/logstash/output.conf.erb

diff --git a/modules/openstack_project/manifests/logstash_worker.pp b/modules/openstack_project/manifests/logstash_worker.pp
index 1180f4d217..ad8f494a14 100644
--- a/modules/openstack_project/manifests/logstash_worker.pp
+++ b/modules/openstack_project/manifests/logstash_worker.pp
@@ -25,11 +25,32 @@ class openstack_project::logstash_worker (
     source => 'puppet:///modules/openstack_project/logstash/logstash-indexer.default',
   }
 
-  class { 'logstash::indexer':
-    conf_template => 'openstack_project/logstash/indexer.conf.erb',
+  vcsrepo { '/opt/logstash-filters':
+    ensure   => latest,
+    provider => git,
+    revision => 'master',
+    source   => 'https://git.openstack.org/openstack-infra/logstash-filters',
   }
 
-  include log_processor
+  include ::logstash
+
+  logstash::filter { 'openstack-logstash-filters':
+    level   => '50',
+    target  => '/opt/logstash-filters/filters/openstack-filters.conf',
+    require => [
+      Class['::logstash'],
+      Vcsrepo['/opt/logstash-filters'],
+    ],
+    notify  => Service['logstash'],
+  }
+
+  class { '::logstash::indexer':
+    input_template  => 'openstack_project/logstash/input.conf.erb',
+    output_template => 'openstack_project/logstash/output.conf.erb',
+    require         => Logstash::Filter['openstack-logstash-filters'],
+  }
+
+  include ::log_processor
   log_processor::worker { 'A':
     config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
   }
diff --git a/modules/openstack_project/templates/logstash/input.conf.erb b/modules/openstack_project/templates/logstash/input.conf.erb
new file mode 100644
index 0000000000..7e748a26d7
--- /dev/null
+++ b/modules/openstack_project/templates/logstash/input.conf.erb
@@ -0,0 +1,8 @@
+input {
+  tcp {
+    host => "localhost"
+    port => 9999
+    codec => json_lines {}
+    type => "jenkins"
+  }
+}
diff --git a/modules/openstack_project/templates/logstash/output.conf.erb b/modules/openstack_project/templates/logstash/output.conf.erb
new file mode 100644
index 0000000000..d189ccfeac
--- /dev/null
+++ b/modules/openstack_project/templates/logstash/output.conf.erb
@@ -0,0 +1,7 @@
+output {
+  elasticsearch {
+    hosts => <%= @elasticsearch_nodes.map { |node| node + ":9200" }.inspect %>
+    manage_template => false
+    flush_size => 1024
+  }
+}