From 5dd697f3c4b0f477e704338122a534618d790c9e Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Thu, 23 May 2013 15:35:35 -0700
Subject: [PATCH] Fix elasticsearch connectivity on logstash.o.o.

The new elasticsearch server needs to be able to connect back to
logstash over the elasticsearch ports. Also make kibana talk to the new
elasticsearch server.

Change-Id: I88af40c7885b263fcef5ff16210f073cd0a696a5
Reviewed-on: https://review.openstack.org/30354
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins
---
 modules/kibana/manifests/init.pp                           | 7 +++++--
 .../kibana/{files/config.rb => templates/config.rb.erb}    | 2 +-
 modules/logstash/manifests/web.pp                          | 7 +++++--
 modules/openstack_project/manifests/logstash.pp            | 6 +++++-
 .../openstack_project/templates/logstash/indexer.conf.erb  | 2 +-
 5 files changed, 17 insertions(+), 7 deletions(-)
 rename modules/kibana/{files/config.rb => templates/config.rb.erb} (98%)

diff --git a/modules/kibana/manifests/init.pp b/modules/kibana/manifests/init.pp
index 7eebb50f83..2987eaf115 100644
--- a/modules/kibana/manifests/init.pp
+++ b/modules/kibana/manifests/init.pp
@@ -14,7 +14,9 @@
 #
 # Class to install kibana frontend to logstash.
 #
-class kibana {
+class kibana (
+  $elasticsearch_host = 'localhost'
+) {
 
   group { 'kibana':
     ensure => present,
@@ -67,7 +69,8 @@ class kibana {
 
   file { '/opt/kibana/kibana/KibanaConfig.rb':
     ensure  => present,
-    source  => 'puppet:///modules/kibana/config.rb',
+    content => template('kibana/config.rb.erb'),
+    replace => true,
     owner   => 'kibana',
     group   => 'kibana',
     require => Vcsrepo['/opt/kibana/kibana'],
diff --git a/modules/kibana/files/config.rb b/modules/kibana/templates/config.rb.erb
similarity index 98%
rename from modules/kibana/files/config.rb
rename to modules/kibana/templates/config.rb.erb
index e45fee27b9..98f0d9fc12 100755
--- a/modules/kibana/files/config.rb
+++ b/modules/kibana/templates/config.rb.erb
@@ -7,7 +7,7 @@ module KibanaConfig
   # Your elastic search server(s). This may be set as an array for round robin
   # load balancing
   # Elasticsearch = ["elasticsearch1:9200","elasticsearch2:9200"]
-  Elasticsearch = "localhost:9200"
+  Elasticsearch = "<%= scope.lookupvar("::kibana::elasticsearch_host") %>:9200"
 
   #Set the Net::HTTP read/open timeouts for the connection to the ES backend
   ElasticsearchTimeout = 500
diff --git a/modules/logstash/manifests/web.pp b/modules/logstash/manifests/web.pp
index f853717008..23fa18597f 100644
--- a/modules/logstash/manifests/web.pp
+++ b/modules/logstash/manifests/web.pp
@@ -17,7 +17,8 @@
 class logstash::web (
   $vhost_name = $::fqdn,
   $serveradmin = "webmaster@${::fqdn}",
-  $frontend = 'internal'
+  $frontend = 'internal',
+  $elasticsearch_host = 'localhost',
 ) {
   include apache
   a2mod { 'rewrite':
@@ -54,7 +55,9 @@ class logstash::web (
     }
 
     'kibana': {
-      include kibana
+      class { 'kibana':
+        elasticsearch_host => $elasticsearch_host,
+      }
       $vhost = 'logstash/kibana.vhost.erb'
     }
 
diff --git a/modules/openstack_project/manifests/logstash.pp b/modules/openstack_project/manifests/logstash.pp
index 33e8555674..87197b2a17 100644
--- a/modules/openstack_project/manifests/logstash.pp
+++ b/modules/openstack_project/manifests/logstash.pp
@@ -17,8 +17,11 @@
 class openstack_project::logstash (
   $sysadmins = []
 ) {
+  $iptables_rule = '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s elasticsearch.openstack.org -j ACCEPT'
   class { 'openstack_project::server':
     iptables_public_tcp_ports => [22, 80],
+    iptables_rules6           => $iptables_rule,
+    iptables_rules4           => $iptables_rule,
     sysadmins                 => $sysadmins,
   }
 
@@ -26,7 +29,8 @@ class openstack_project::logstash (
     conf_template => 'openstack_project/logstash/indexer.conf.erb',
   }
   class { 'logstash::web':
-    frontend => 'kibana',
+    frontend           => 'kibana',
+    elasticsearch_host => 'elasticsearch.openstack.org',
   }
 
   package { 'python3':
diff --git a/modules/openstack_project/templates/logstash/indexer.conf.erb b/modules/openstack_project/templates/logstash/indexer.conf.erb
index 940f77a8d1..83fd5c31a6 100644
--- a/modules/openstack_project/templates/logstash/indexer.conf.erb
+++ b/modules/openstack_project/templates/logstash/indexer.conf.erb
@@ -54,7 +54,7 @@ filter {
     # Do multiline matching as the above mutliline filter may add newlines
     # to the log messages.
     # TODO move the LOGLEVELs into a proper grok pattern.
-    pattern => [ "(?m)^%{DATESTAMP:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" ]
+    pattern => [ "(?m)^%{DATESTAMP:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" ]
     add_field => [ "received_at", "%{@timestamp}" ]
   }
   date {