Use logstash filters from filters repo

This uses new logstash conf.d features to properly link in the
configuration from the logstash-filters repo. This should make
configuring logstash far more flexible and reconsumable.

Change-Id: Ia304eb7e73c82ca5ce85967fbf442bb4bc5f8b7a
Depends-On: Icbca7a6ba0c5a94a273ef158f707311b588483fd

modules/openstack_project/manifests/logstash_worker.pp

@@ -25,11 +25,32 @@ class openstack_project::logstash_worker (
     source => 'puppet:///modules/openstack_project/logstash/logstash-indexer.default',
   }
 
-  class { 'logstash::indexer':
-    conf_template => 'openstack_project/logstash/indexer.conf.erb',
+  vcsrepo { '/opt/logstash-filters':
+    ensure   => latest,
+    provider => git,
+    revision => 'master',
+    source   => 'https://git.openstack.org/openstack-infra/logstash-filters',
   }
 
-  include log_processor
+  include ::logstash
+
+  logstash::filter { 'openstack-logstash-filters':
+    level   => '50',
+    target  => '/opt/logstash-filters/filters/openstack-filters.conf',
+    require => [
+      Class['::logstash'],
+      Vcsrepo['/opt/logstash-filters'],
+    ],
+    notify  => Service['logstash'],
+  }
+
+  class { '::logstash::indexer':
+    input_template  => 'openstack_project/logstash/input.conf.erb',
+    output_template => 'openstack_project/logstash/output.conf.erb',
+    require         => Logstash::Filter['openstack-logstash-filters'],
+  }
+
+  include ::log_processor
   log_processor::worker { 'A':
     config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
   }

modules/openstack_project/templates/logstash/input.conf.erb

@@ -0,0 +1,8 @@
+input {
+  tcp {
+    host => "localhost"
+    port => 9999
+    codec => json_lines {}
+    type => "jenkins"
+  }
+}

modules/openstack_project/templates/logstash/output.conf.erb

@@ -0,0 +1,7 @@
+output {
+  elasticsearch {
+    hosts => <%= @elasticsearch_nodes.map { |node| node + ":9200" }.inspect %>
+    manage_template => false
+    flush_size => 1024
+  }
+}