diff --git a/modules/openstack_project/manifests/logstash.pp b/modules/openstack_project/manifests/logstash.pp index 0a0b7947a9..f72676d7be 100644 --- a/modules/openstack_project/manifests/logstash.pp +++ b/modules/openstack_project/manifests/logstash.pp @@ -22,16 +22,16 @@ class openstack_project::logstash ( sysadmins => $sysadmins, } - class { 'logstash::agent': - conf_template => 'openstack_project/logstash/agent.conf.erb', - } class { 'logstash::indexer': conf_template => 'openstack_project/logstash/indexer.conf.erb', } - include logstash::redis include logstash::elasticsearch include logstash::web + package { 'redis-server': + ensure => 'absent', + } + package { 'python3': ensure => 'present', } diff --git a/modules/openstack_project/templates/logstash/indexer.conf.erb b/modules/openstack_project/templates/logstash/indexer.conf.erb index 386dbd378d..5ab682591a 100644 --- a/modules/openstack_project/templates/logstash/indexer.conf.erb +++ b/modules/openstack_project/templates/logstash/indexer.conf.erb @@ -1,13 +1,46 @@ input { - redis { - host => "127.0.0.1" - type => "redis-input" - # these settings should match the output of the agent - data_type => "list" - key => "logstash" + pipe { + command => "python3 /usr/local/bin/log-pusher.py -r -z tcp://jenkins.openstack.org:8888 -l http://logs.openstack.org -f console.html" + format => "json" + message_format => "%{event_message}" + tags => ["jenkins", "console"] + type => "jenkins_console" + } +} - # We use json_event here since the sender is a logstash agent - format => "json_event" +# You can check grok patterns at http://grokdebug.herokuapp.com/ +filter { + grep { + type => "jenkins_console" + # Drop matches. + negate => true + match => ["@message", "^$"] + } + multiline { + type => "jenkins_console" + negate => true + pattern => "^%{DATESTAMP} \|" + what => "previous" + } + grok { + type => "jenkins_console" + pattern => [ "^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ] + add_field => [ "received_at", "%{@timestamp}" ] + } + date { + type => "jenkins_console" + exclude_tags => "_grokparsefailure" + match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ] + } + mutate { + type => "jenkins_console" + exclude_tags => "_grokparsefailure" + replace => [ "@message", "%{logmessage}" ] + } + mutate { + type => "jenkins_console" + exclude_tags => "_grokparsefailure" + remove => [ "logdate", "logmessage" ] } }