opendev/system-config
Code Issues Proposed changes

Use log-pusher injected tags during logstash grok

Browse Source 
* modules/openstack_project/templates/logstash/indexer.conf.erb: The
log-pusher.py script tags events with the filename of the log generating
the event. Use these values instead of different types to differentiate
Jenkins console logs from nova logs and so on.

Note that filters must match all of the values in the tags array (they
are ANDed together). This may mean that as the logstash filter rulesets
grow we will need a set of initial mutates to convert
'logs/screen-n-api.txt' and 'logs/screen-n-cpu.txt' tags to 'nova' to
allow for common grok and parsing filters.

Change-Id: I2769bc05a2e9cc7e8dbc46849e052146b9fee75e
Reviewed-on: https://review.openstack.org/28119
Reviewed-by: James E. Blair <corvus@inaugust.com>
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: Jeremy Stanley <fungi@yuggoth.org>
Tested-by: Jenkins
This commit is contained in:
Clark Boylan 2013-05-02 16:02:19 -07:00 committed by Jenkins
parent f07756b5bd
commit 104c5ef5a8
1 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
modules/openstack_project/templates/logstash/indexer.conf.erb
View File

@ -4,42 +4,44 @@ input {
port => 9999
format => "json"
message_format => "%{event_message}"
tags => ["jenkins", "console"]
type => "jenkins_console"
type => "jenkins"
}
}
# You can check grok patterns at http://grokdebug.herokuapp.com/
filter {
grep {
type => "jenkins_console"
type => "jenkins"
tags => ["console.html"]
# Drop matches.
negate => true
match => ["@message", "^</?pre>$"]
}
multiline {
type => "jenkins_console"
type => "jenkins"
tags => ["console.html"]
negate => true
pattern => "^%{DATESTAMP} \|"
what => "previous"
}
grok {
type => "jenkins_console"
type => "jenkins"
tags => ["console.html"]
pattern => [ "^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
add_field => [ "received_at", "%{@timestamp}" ]
}
date {
type => "jenkins_console"
type => "jenkins"
exclude_tags => "_grokparsefailure"
match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ]
}
mutate {
type => "jenkins_console"
type => "jenkins"
exclude_tags => "_grokparsefailure"
replace => [ "@message", "%{logmessage}" ]
}
mutate {
type => "jenkins_console"
type => "jenkins"
exclude_tags => "_grokparsefailure"
remove => [ "logdate", "logmessage" ]
}