From 901e706aea3f729b0dfaa5ec1f4b25049bbee307 Mon Sep 17 00:00:00 2001 From: K Jonathan Harker Date: Wed, 17 Apr 2013 22:15:20 +0000 Subject: [PATCH] Use kibana as the web frontend to logstash. Add a kibana module that installs kibana and flat configuration files, and also disables the previous boring bland web frontend. Change-Id: I772c66f07d2bc3a88128e101074be9d4162e3f8a Reviewed-on: https://review.openstack.org/27089 Reviewed-by: Jesse Keating Reviewed-by: Clark Boylan Reviewed-by: Jeremy Stanley Approved: Clark Boylan Tested-by: Jenkins --- modules/kibana/files/config.rb | 140 ++++++++++++++++++ modules/kibana/files/kibana.init | 22 +++ modules/kibana/manifests/init.pp | 90 +++++++++++ modules/logstash/manifests/web.pp | 55 ++++--- modules/logstash/templates/kibana.vhost.erb | 13 ++ .../openstack_project/manifests/logstash.pp | 4 +- 6 files changed, 304 insertions(+), 20 deletions(-) create mode 100644 modules/kibana/files/config.rb create mode 100644 modules/kibana/files/kibana.init create mode 100644 modules/kibana/manifests/init.pp create mode 100644 modules/logstash/templates/kibana.vhost.erb diff --git a/modules/kibana/files/config.rb b/modules/kibana/files/config.rb new file mode 100644 index 0000000000..7d4b800c97 --- /dev/null +++ b/modules/kibana/files/config.rb @@ -0,0 +1,140 @@ +module KibanaConfig + + # A Note: While the only option you really have to set is "Elasticsearch" it + # is HIGHLY recommended you glance over every option. I personally consider + # 'Facet_index_limit' really important. + + # Your elastic search server(s). This may be set as an array for round robin + # load balancing + # Elasticsearch = ["elasticsearch1:9200","elasticsearch2:9200"] + Elasticsearch = "localhost:9200" + + #Set the Net::HTTP read/open timeouts for the connection to the ES backend + ElasticsearchTimeout = 500 + + # The port Kibana should listen on + KibanaPort = 5601 + + # The adress ip Kibana should listen on. Comment out or set to + # 0.0.0.0 to listen on all interfaces. + KibanaHost = '127.0.0.1' + + # The record type as defined in your logstash configuration. + # Seperate multiple types with a comma, no spaces. Leave blank + # for all. + Type = '' + + # Results to show per page + Per_page = 100 + + # Timezone. Leave this set to 'user' to have the user's browser autocorrect. + # Otherwise, set a timezone string + # Examples: 'UTC', 'America/Phoenix', 'Europe/Athens', MST + # You can use `date +%Z` on linux to get your timezone string + Timezone = 'UTC' + + # Format for timestamps. Defaults to mm/dd HH:MM:ss. + # For syntax see: http://blog.stevenlevithan.com/archives/date-time-format + # Time_format = 'isoDateTime' + Time_format = 'isoDateTime' + + # Change which fields are shown by default. Must be set as an array + # Default_fields = ['@fields.vhost','@fields.response','@fields.request'] + Default_fields = ['@message'] + + # If set to true, Kibana will use the Highlight feature of Elasticsearch to + # display highlighted search results + Highlight_results = true + + # A field needs to be specified for the highlight feature. By default, + # Elasticsearch doesn't allow highlighting on _all because the field has to + # be either stored or part of the _source field. + Highlighted_field = "@message" + + # Make URLs clickable in detailed view + Clickable_URLs = true + + # The default operator used if no explicit operator is specified. + # For example, with a default operator of OR, the query capital of + # Hungary is translated to capital OR of OR Hungary, and with default + # operator of AND, the same query is translated to capital AND of AND + # Hungary. The default value is OR. + Default_operator = 'OR' + + # When using analyze, use this many of the most recent + # results for user's query + Analyze_limit = 2000 + + # Show this many results in analyze/trend/terms/stats modes + Analyze_show = 25 + + # Show this many results in an rss feed + Rss_show = 25 + + # Show this many results in an exported file + Export_show = 2000 + + # Delimit exported file fields with what? + # You may want to change this to something like "\t" (tab) if you have + # commas in your logs + Export_delimiter = "," + + # You may wish to insert a default search which all user searches + # must match. For example @source_host:www1 might only show results + # from www1. + Filter = '' + + # When searching, Kibana will attempt to only search indices + # that match your timeframe, to make searches faster. You can + # turn this behavior off if you use something other than daily + # indexing + Smart_index = true + + # You can define your custom pattern here for index names if you + # use something other than daily indexing. Pattern needs to have + # date formatting like '%Y.%m.%d'. Will accept an array of smart + # indexes. + # Smart_index_pattern = ['logstash-web-%Y.%m.%d', 'logstash-mail-%Y.%m.%d'] + Smart_index_pattern = 'logstash-%Y.%m.%d' + + # Number of seconds between each index. 86400 = 1 day. + Smart_index_step = 86400 + + # ElasticSearch has a default limit on URL size for REST calls, + # so Kibana will fall back to _all if a search spans too many + # indices. Use this to set that 'too many' number. By default this + # is set really high, ES might not like this + Smart_index_limit = 150 + + # Elasticsearch has an internal mechanism called "faceting" for performing + # analysis that we use for the "Stats" and "Terms" modes. However, on large + # data sets/queries facetting can cause ES to crash if there isn't enough + # memory available. It is suggested that you limit the number of indices that + # Kibana will use for the "Stats" and "Terms" to prevent ES crashes. For very + # large data sets and undersized ES clusers, a limit of 1 is not unreasonable. + # Default is 0 (unlimited) + Facet_index_limit = 0 + + # You probably don't want to touch anything below this line + # unless you really know what you're doing + + # Primary field. By default Elastic Search has a special + # field called _all that is searched when no field is specified. + # Dropping _all can reduce index size significantly. If you do that + # you'll need to change primary_field to be '@message' + Primary_field = '@message' + + # Default Elastic Search index to query + Default_index = '@message' + + # TODO: This isn't functional yet + # Prevent wildcard search terms which result in extremely slow queries + # See: http:#www.elasticsearch.org/guide/reference/query-dsl/wildcard-query.html + Disable_fullscan = false + + # Set headers to allow kibana to be loaded in an iframe from a different origin. + Allow_iframed = false + + # Use this interval as fallback. + Fallback_interval = 900 +end diff --git a/modules/kibana/files/kibana.init b/modules/kibana/files/kibana.init new file mode 100644 index 0000000000..9203f9234b --- /dev/null +++ b/modules/kibana/files/kibana.init @@ -0,0 +1,22 @@ +# kibana - web instance +# +# Copied from http://cookbook.logstash.net/recipes/using-upstart/ + +description "kibana web instance" + +start on virtual-filesystems +stop on runlevel [06] + +# Respawn it if the process exits +respawn +respawn limit 5 30 +expect fork + +chdir /opt/kibana/kibana + +script + + su -s /bin/sh -c 'exec "$0" "$@"' kibana -- /usr/bin/ruby /opt/kibana/kibana/kibana.rb & + emit kibana-running +end script + diff --git a/modules/kibana/manifests/init.pp b/modules/kibana/manifests/init.pp new file mode 100644 index 0000000000..7eebb50f83 --- /dev/null +++ b/modules/kibana/manifests/init.pp @@ -0,0 +1,90 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Class to install kibana frontend to logstash. +# +class kibana { + + group { 'kibana': + ensure => present, + } + + user { 'kibana': + ensure => present, + comment => 'Kibana User', + home => '/opt/kibana', + gid => 'kibana', + shell => '/bin/bash', + membership => 'minimum', + require => Group['kibana'], + } + + file { '/opt/kibana': + ensure => directory, + owner => 'kibana', + group => 'kibana', + mode => '0644', + recurse => true, + require => User['kibana'], + } + + vcsrepo { '/opt/kibana/kibana': + ensure => latest, + provider => git, + source => 'https://github.com/rashidkpc/Kibana.git', + revision => 'v0.2.0', + require => File['/opt/kibana'], + } + + package { 'bundler': + ensure => latest, + provider => 'gem', + } + + exec { 'install_kibana': + command => 'bundle install', + path => ['/usr/bin', '/usr/local/bin'], + cwd => '/opt/kibana/kibana', + logoutput => true, + refreshonly => true, + subscribe => Vcsrepo['/opt/kibana/kibana'], + require => [ + User['kibana'], + Package['bundler'], + ], + } + + file { '/opt/kibana/kibana/KibanaConfig.rb': + ensure => present, + source => 'puppet:///modules/kibana/config.rb', + owner => 'kibana', + group => 'kibana', + require => Vcsrepo['/opt/kibana/kibana'], + } + + file { '/etc/init/kibana.conf': + ensure => present, + source => 'puppet:///modules/kibana/kibana.init', + } + + service { 'kibana': + ensure => running, + require => [ + File['/etc/init/kibana.conf'], + File['/opt/kibana/kibana/KibanaConfig.rb'], + Exec['install_kibana'], + ], + } + +} diff --git a/modules/logstash/manifests/web.pp b/modules/logstash/manifests/web.pp index cd6ac361cb..f853717008 100644 --- a/modules/logstash/manifests/web.pp +++ b/modules/logstash/manifests/web.pp @@ -16,15 +16,10 @@ # class logstash::web ( $vhost_name = $::fqdn, - $serveradmin = "webmaster@${::fqdn}" + $serveradmin = "webmaster@${::fqdn}", + $frontend = 'internal' ) { include apache - apache::vhost { $vhost_name: - port => 80, - docroot => 'MEANINGLESS ARGUMENT', - priority => '50', - template => 'logstash/logstash.vhost.erb', - } a2mod { 'rewrite': ensure => present, } @@ -37,19 +32,41 @@ class logstash::web ( include logstash - file { '/etc/init/logstash-web.conf': - ensure => present, - source => 'puppet:///modules/logstash/logstash-web.conf', - replace => true, - owner => 'root', + case $frontend { + 'internal': { + file { '/etc/init/logstash-web.conf': + ensure => present, + source => 'puppet:///modules/logstash/logstash-web.conf', + replace => true, + owner => 'root', + } + + service { 'logstash-web': + ensure => running, + enable => true, + require => [ + Class['logstash'], + File['/etc/init/logstash-web.conf'], + ], + } + + $vhost = 'logstash/logstash.vhost.erb' + } + + 'kibana': { + include kibana + $vhost = 'logstash/kibana.vhost.erb' + } + + default: { + fail("Unknown frontend to logstash: ${frontend}.") + } } - service { 'logstash-web': - ensure => running, - enable => true, - require => [ - Class['logstash'], - File['/etc/init/logstash-web.conf'], - ] + apache::vhost { $vhost_name: + port => 80, + docroot => 'MEANINGLESS ARGUMENT', + priority => '50', + template => $vhost, } } diff --git a/modules/logstash/templates/kibana.vhost.erb b/modules/logstash/templates/kibana.vhost.erb new file mode 100644 index 0000000000..eb4093285e --- /dev/null +++ b/modules/logstash/templates/kibana.vhost.erb @@ -0,0 +1,13 @@ +:80> + ServerName <%= scope.lookupvar("::logstash::web::vhost_name") %> + ServerAdmin <%= scope.lookupvar("::logstash::web::serveradmin") %> + + ErrorLog ${APACHE_LOG_DIR}/<%= scope.lookupvar("::logstash::web::vhost_name") %>-error.log + + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/<%= scope.lookupvar("::logstash::web::vhost_name") %>-access.log combined + + ProxyPass / http://127.0.0.1:5601/ retry=0 + ProxyPassReverse / http://127.0.0.1:5601/ + diff --git a/modules/openstack_project/manifests/logstash.pp b/modules/openstack_project/manifests/logstash.pp index f72676d7be..f080d8d10c 100644 --- a/modules/openstack_project/manifests/logstash.pp +++ b/modules/openstack_project/manifests/logstash.pp @@ -25,8 +25,10 @@ class openstack_project::logstash ( class { 'logstash::indexer': conf_template => 'openstack_project/logstash/indexer.conf.erb', } + class { 'logstash::web': + frontend => 'kibana', + } include logstash::elasticsearch - include logstash::web package { 'redis-server': ensure => 'absent',