From 9f6089824b584f97fe234414af515cadec58977c Mon Sep 17 00:00:00 2001
From: Matthew Treinish <mtreinish@kortar.org>
Date: Thu, 27 Apr 2017 15:57:09 -0400
Subject: [PATCH] Enable mqtt notifications from logstash gearman worker

This commit enables the mqtt notifications in the gearman worker. Most
of the setup was already there from our previous failed attempts to
leverage the logstash plugin directly. This just pivots that to use the
gearman worker instead.

Change-Id: I6becec12604c61fe50d3e6b9c7ed9f9e9be311ae
Depends-On: I0a17444cce18dd4b63f1f924e393483f6d8fe8eb
Depends-On: I43be3562780c61591ebede61f3a8929e8217f199
---
 .../files/logstash/jenkins-log-worker.yaml    |  7 -----
 .../manifests/logstash_worker.pp              | 30 ++++++++++++++-----
 .../logstash/jenkins-log-worker.yaml.erb      | 13 ++++++++
 .../templates/logstash/output.conf.erb        | 11 -------
 4 files changed, 36 insertions(+), 25 deletions(-)
 delete mode 100644 modules/openstack_project/files/logstash/jenkins-log-worker.yaml
 create mode 100644 modules/openstack_project/templates/logstash/jenkins-log-worker.yaml.erb

diff --git a/modules/openstack_project/files/logstash/jenkins-log-worker.yaml b/modules/openstack_project/files/logstash/jenkins-log-worker.yaml
deleted file mode 100644
index 6702f1b972..0000000000
--- a/modules/openstack_project/files/logstash/jenkins-log-worker.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-gearman-host: logstash.openstack.org
-gearman-port: 4730
-output-host: localhost
-output-port: 9999
-output-mode: tcp
-crm114-script: /usr/local/bin/classify-log.crm
-crm114-data: /var/lib/crm114
diff --git a/modules/openstack_project/manifests/logstash_worker.pp b/modules/openstack_project/manifests/logstash_worker.pp
index c6abdc1337..8894107e93 100644
--- a/modules/openstack_project/manifests/logstash_worker.pp
+++ b/modules/openstack_project/manifests/logstash_worker.pp
@@ -18,7 +18,7 @@ class openstack_project::logstash_worker (
   $discover_node = 'elasticsearch01.openstack.org',
   $filter_rev    = 'master',
   $filter_source = 'https://git.openstack.org/openstack-infra/logstash-filters',
-  $enable_mqtt = false,
+  $enable_mqtt = true,
   $mqtt_hostname = 'firehose.openstack.org',
   $mqtt_port = 8883,
   $mqtt_topic = "logstash/${::hostname}",
@@ -26,6 +26,15 @@ class openstack_project::logstash_worker (
   $mqtt_password = undef,
   $mqtt_ca_cert_contents = undef,
 ) {
+
+  file { '/etc/logprocessor/worker.yaml':
+    ensure => present,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    content => template('openstack_project/logstash/jenkins-log-worker.yaml.erb'),
+  }
+
   file { '/etc/default/logstash-indexer':
     ensure => present,
     owner  => 'root',
@@ -53,26 +62,33 @@ class openstack_project::logstash_worker (
     notify  => Service['logstash'],
   }
 
+  file { '/etc/logstash/mqtt-root-CA.pem.crt':
+      ensure  => present,
+      content => $mqtt_ca_cert_contents,
+      replace => true,
+      owner   => 'subunit',
+      group   => 'subunit',
+      mode    => '0555',
+  }
+
   validate_array($elasticsearch_nodes)  # needed by output.conf.erb
   class { '::logstash::indexer':
     input_template         => 'openstack_project/logstash/input.conf.erb',
     output_template        => 'openstack_project/logstash/output.conf.erb',
-    enable_mqtt            => $enable_mqtt,
-    mqtt_ca_cert_contents  => $mqtt_ca_cert_contents,
     require                => Logstash::Filter['openstack-logstash-filters'],
   }
 
   include ::log_processor
   log_processor::worker { 'A':
-    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
+    config_file => '/etc/logprocessor/worker.yaml',
   }
   log_processor::worker { 'B':
-    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
+    config_file => '/etc/logprocessor/worker.yaml',
   }
   log_processor::worker { 'C':
-    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
+    config_file => '/etc/logprocessor/worker.yaml',
   }
   log_processor::worker { 'D':
-    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
+    config_file => '/etc/logprocessor/worker.yaml',
   }
 }
diff --git a/modules/openstack_project/templates/logstash/jenkins-log-worker.yaml.erb b/modules/openstack_project/templates/logstash/jenkins-log-worker.yaml.erb
new file mode 100644
index 0000000000..c744e6bc63
--- /dev/null
+++ b/modules/openstack_project/templates/logstash/jenkins-log-worker.yaml.erb
@@ -0,0 +1,13 @@
+gearman-host: logstash.openstack.org
+gearman-port: 4730
+output-host: localhost
+output-port: 9999
+output-mode: tcp
+crm114-script: /usr/local/bin/classify-log.crm
+crm114-data: /var/lib/crm114
+mqtt-host: <%= @mqtt_hostname %>
+mqtt-port: <%= @mqtt_port %>
+mqtt-topic: gearman-logstash/<%= @hostname %>
+mqtt-user: <%= @mqtt_user %>
+mqtt-pass: <%= @mqtt_pass %>
+mqtt-ca-certs: /etc/logstash/mqtt-root-CA.pem.crt
diff --git a/modules/openstack_project/templates/logstash/output.conf.erb b/modules/openstack_project/templates/logstash/output.conf.erb
index fed1eb2243..7c4f4aa08e 100644
--- a/modules/openstack_project/templates/logstash/output.conf.erb
+++ b/modules/openstack_project/templates/logstash/output.conf.erb
@@ -4,16 +4,5 @@ output {
     manage_template => false
     flush_size => 1024
   }
-<% if @enable_mqtt -%>
-  mqtt {
-    host => "<%= @mqtt_hostname %>"
-    port => <%= @mqtt_port %>
-    topic => "<%= @mqtt_topic %>"
-    username => "<%= @mqtt_username %>"
-    password => "<%= @mqtt_password %>"
-    ssl => true
-    ca_file => "/etc/logstash/mqtt-root-CA.pem.crt"
-  }
-<% end -%>
 
 }