Enable mqtt notifications from logstash gearman worker

This commit enables the mqtt notifications in the gearman worker. Most
of the setup was already there from our previous failed attempts to
leverage the logstash plugin directly. This just pivots that to use the
gearman worker instead.

Change-Id: I6becec12604c61fe50d3e6b9c7ed9f9e9be311ae
Depends-On: I0a17444cce18dd4b63f1f924e393483f6d8fe8eb
Depends-On: I43be3562780c61591ebede61f3a8929e8217f199

modules/openstack_project/files/logstash/jenkins-log-worker.yaml

@@ -1,7 +0,0 @@
-gearman-host: logstash.openstack.org
-gearman-port: 4730
-output-host: localhost
-output-port: 9999
-output-mode: tcp
-crm114-script: /usr/local/bin/classify-log.crm
-crm114-data: /var/lib/crm114

modules/openstack_project/manifests/logstash_worker.pp

@@ -18,7 +18,7 @@ class openstack_project::logstash_worker (
   $discover_node = 'elasticsearch01.openstack.org',
   $filter_rev    = 'master',
   $filter_source = 'https://git.openstack.org/openstack-infra/logstash-filters',
-  $enable_mqtt = false,
+  $enable_mqtt = true,
   $mqtt_hostname = 'firehose.openstack.org',
   $mqtt_port = 8883,
   $mqtt_topic = "logstash/${::hostname}",
@@ -26,6 +26,15 @@ class openstack_project::logstash_worker (
   $mqtt_password = undef,
   $mqtt_ca_cert_contents = undef,
 ) {
+
+  file { '/etc/logprocessor/worker.yaml':
+    ensure => present,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    content => template('openstack_project/logstash/jenkins-log-worker.yaml.erb'),
+  }
+
   file { '/etc/default/logstash-indexer':
     ensure => present,
     owner  => 'root',
@@ -53,26 +62,33 @@ class openstack_project::logstash_worker (
     notify  => Service['logstash'],
   }
 
+  file { '/etc/logstash/mqtt-root-CA.pem.crt':
+      ensure  => present,
+      content => $mqtt_ca_cert_contents,
+      replace => true,
+      owner   => 'subunit',
+      group   => 'subunit',
+      mode    => '0555',
+  }
+
   validate_array($elasticsearch_nodes)  # needed by output.conf.erb
   class { '::logstash::indexer':
     input_template         => 'openstack_project/logstash/input.conf.erb',
     output_template        => 'openstack_project/logstash/output.conf.erb',
-    enable_mqtt            => $enable_mqtt,
-    mqtt_ca_cert_contents  => $mqtt_ca_cert_contents,
     require                => Logstash::Filter['openstack-logstash-filters'],
   }
 
   include ::log_processor
   log_processor::worker { 'A':
-    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
+    config_file => '/etc/logprocessor/worker.yaml',
   }
   log_processor::worker { 'B':
-    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
+    config_file => '/etc/logprocessor/worker.yaml',
   }
   log_processor::worker { 'C':
-    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
+    config_file => '/etc/logprocessor/worker.yaml',
   }
   log_processor::worker { 'D':
-    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
+    config_file => '/etc/logprocessor/worker.yaml',
   }
 }

modules/openstack_project/templates/logstash/jenkins-log-worker.yaml.erb

@@ -0,0 +1,13 @@
+gearman-host: logstash.openstack.org
+gearman-port: 4730
+output-host: localhost
+output-port: 9999
+output-mode: tcp
+crm114-script: /usr/local/bin/classify-log.crm
+crm114-data: /var/lib/crm114
+mqtt-host: <%= @mqtt_hostname %>
+mqtt-port: <%= @mqtt_port %>
+mqtt-topic: gearman-logstash/<%= @hostname %>
+mqtt-user: <%= @mqtt_user %>
+mqtt-pass: <%= @mqtt_pass %>
+mqtt-ca-certs: /etc/logstash/mqtt-root-CA.pem.crt

modules/openstack_project/templates/logstash/output.conf.erb

@@ -4,16 +4,5 @@ output {
     manage_template => false
     flush_size => 1024
   }
-<% if @enable_mqtt -%>
-  mqtt {
-    host => "<%= @mqtt_hostname %>"
-    port => <%= @mqtt_port %>
-    topic => "<%= @mqtt_topic %>"
-    username => "<%= @mqtt_username %>"
-    password => "<%= @mqtt_password %>"
-    ssl => true
-    ca_file => "/etc/logstash/mqtt-root-CA.pem.crt"
-  }
-<% end -%>
 
 }