opendev/system-config
Code Issues Proposed changes

Fix elasticsearch connectivity on logstash.o.o.

Browse Source 
The new elasticsearch server needs to be able to connect back to
logstash over the elasticsearch ports. Also make kibana talk to the new
elasticsearch server.

Change-Id: I88af40c7885b263fcef5ff16210f073cd0a696a5
Reviewed-on: https://review.openstack.org/30354
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins
This commit is contained in:
Clark Boylan 2013-05-23 15:35:35 -07:00 committed by Jenkins
parent 6d0c0d307a
commit 5dd697f3c4
5 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
modules/kibana/manifests/init.pp
View File

@ -14,7 +14,9 @@
# #
# Class to install kibana frontend to logstash. # Class to install kibana frontend to logstash.
# #
class kibana { class kibana (
$elasticsearch_host = 'localhost'
) {
group { 'kibana': group { 'kibana':
ensure => present, ensure => present,
@ -67,7 +69,8 @@ class kibana {
file { '/opt/kibana/kibana/KibanaConfig.rb': file { '/opt/kibana/kibana/KibanaConfig.rb':
ensure => present, ensure => present,
source => 'puppet:///modules/kibana/config.rb', content => template('kibana/config.rb.erb'),
replace => true,
owner => 'kibana', owner => 'kibana',
group => 'kibana', group => 'kibana',
require => Vcsrepo['/opt/kibana/kibana'], require => Vcsrepo['/opt/kibana/kibana'],

2
modules/kibana/files/config.rb → modules/kibana/templates/config.rb.erb
View File

@ -7,7 +7,7 @@ module KibanaConfig
# Your elastic search server(s). This may be set as an array for round robin # Your elastic search server(s). This may be set as an array for round robin
# load balancing # load balancing
# Elasticsearch = ["elasticsearch1:9200","elasticsearch2:9200"] # Elasticsearch = ["elasticsearch1:9200","elasticsearch2:9200"]
Elasticsearch = "localhost:9200" Elasticsearch = "<%= scope.lookupvar("::kibana::elasticsearch_host") %>:9200"
#Set the Net::HTTP read/open timeouts for the connection to the ES backend #Set the Net::HTTP read/open timeouts for the connection to the ES backend
ElasticsearchTimeout = 500 ElasticsearchTimeout = 500

7
modules/logstash/manifests/web.pp
View File

@ -17,7 +17,8 @@
class logstash::web ( class logstash::web (
$vhost_name = $::fqdn, $vhost_name = $::fqdn,
$serveradmin = "webmaster@${::fqdn}", $serveradmin = "webmaster@${::fqdn}",
$frontend = 'internal' $frontend = 'internal',
$elasticsearch_host = 'localhost',
) { ) {
include apache include apache
a2mod { 'rewrite': a2mod { 'rewrite':
@ -54,7 +55,9 @@ class logstash::web (
} }
'kibana': { 'kibana': {
include kibana class { 'kibana':
elasticsearch_host => $elasticsearch_host,
}
$vhost = 'logstash/kibana.vhost.erb' $vhost = 'logstash/kibana.vhost.erb'
} }

6
modules/openstack_project/manifests/logstash.pp
View File

@ -17,8 +17,11 @@
class openstack_project::logstash ( class openstack_project::logstash (
$sysadmins = [] $sysadmins = []
) { ) {
$iptables_rule = '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s elasticsearch.openstack.org -j ACCEPT'
class { 'openstack_project::server': class { 'openstack_project::server':
iptables_public_tcp_ports => [22, 80], iptables_public_tcp_ports => [22, 80],
iptables_rules6 => $iptables_rule,
iptables_rules4 => $iptables_rule,
sysadmins => $sysadmins, sysadmins => $sysadmins,
} }
@ -26,7 +29,8 @@ class openstack_project::logstash (
conf_template => 'openstack_project/logstash/indexer.conf.erb', conf_template => 'openstack_project/logstash/indexer.conf.erb',
} }
class { 'logstash::web': class { 'logstash::web':
frontend => 'kibana', frontend => 'kibana',
elasticsearch_host => 'elasticsearch.openstack.org',
} }
package { 'python3': package { 'python3':

2
modules/openstack_project/templates/logstash/indexer.conf.erb
View File

@ -54,7 +54,7 @@ filter {
# Do multiline matching as the above mutliline filter may add newlines # Do multiline matching as the above mutliline filter may add newlines
# to the log messages. # to the log messages.
# TODO move the LOGLEVELs into a proper grok pattern. # TODO move the LOGLEVELs into a proper grok pattern.
pattern => [ "(?m)^%{DATESTAMP:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" ] pattern => [ "(?m)^%{DATESTAMP:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" ]
add_field => [ "received_at", "%{@timestamp}" ] add_field => [ "received_at", "%{@timestamp}" ]
} }
date { date {