diff --git a/doc/source/logstash.rst b/doc/source/logstash.rst index 83c4b75a17..50db5842fc 100644 --- a/doc/source/logstash.rst +++ b/doc/source/logstash.rst @@ -172,23 +172,23 @@ schema. The config file that tells Logstash how to do this flattening can be found at -:cgit_file:`modules/openstack_project/templates/logstash/indexer.conf.erb` +https://git.openstack.org/cgit/openstack-infra/logstash-filters/tree/filters/openstack-filters.conf This works via the tags that are associated with a given message. The tags in -:cgit_file:`modules/openstack_project/templates/logstash/indexer.conf.erb` +https://git.openstack.org/cgit/openstack-infra/logstash-filters/tree/filters/openstack-filters.conf are used to tell logstash how to parse a given file's messages, based on the file's message format. When adding a new file to be indexed to http://git.openstack.org/cgit/openstack-infra/project-config/tree/roles/submit-logstash-jobs/defaults/main.yaml -at least one tag from the indexer.conf.erb file should be associated +at least one tag from the openstack-filters.conf file should be associated with the new file. One can expect to see '{%logmessage%}' instead of actual message data if indexing is not working properly. In the event a new file's format is not covered, a patch for -:cgit_file:`modules/openstack_project/templates/logstash/indexer.conf.erb` +https://git.openstack.org/cgit/openstack-infra/logstash-filters/tree/filters/openstack-filters.conf should be submitted with an appropriate parsing pattern. ElasticSearch diff --git a/modules/openstack_project/templates/logstash/agent.conf.erb b/modules/openstack_project/templates/logstash/agent.conf.erb deleted file mode 100644 index ba67e52504..0000000000 --- a/modules/openstack_project/templates/logstash/agent.conf.erb +++ /dev/null @@ -1,47 +0,0 @@ -input { - pipe { - command => "python3 /usr/local/bin/log-pusher.py -r -z tcp://jenkins.openstack.org:8888 -l http://logs.openstack.org -f console.html" - format => "json" - message_format => "%{event_message}" - tags => ["jenkins", "console"] - type => "jenkins_console" - } -} - -# You can check grok patterns at http://grokdebug.herokuapp.com/ -filter { - grep { - type => "jenkins_console" - # Drop matches. - negate => true - match => ["@message", "^$"] - } - multiline { - type => "jenkins_console" - negate => true - pattern => "^%{DATESTAMP} \|" - what => "previous" - } - grok { - type => "jenkins_console" - pattern => [ "^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ] - add_field => [ "received_at", "%{@timestamp}" ] - } - date { - type => "jenkins_console" - exclude_tags => "_grokparsefailure" - match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ] - } - mutate { - type => "jenkins_console" - exclude_tags => "_grokparsefailure" - replace => [ "@message", "%{logmessage}" ] - } - mutate { - type => "jenkins_console" - exclude_tags => "_grokparsefailure" - remove => [ "logdate", "logmessage" ] - } -} - -<%= scope.function_template(['openstack_project/logstash/redis-output.conf.erb']) %> diff --git a/modules/openstack_project/templates/logstash/indexer.conf.erb b/modules/openstack_project/templates/logstash/indexer.conf.erb deleted file mode 100644 index 6065d44190..0000000000 --- a/modules/openstack_project/templates/logstash/indexer.conf.erb +++ /dev/null @@ -1,124 +0,0 @@ -input { - tcp { - host => "localhost" - port => 9999 - codec => json_lines {} - type => "jenkins" - } -} - -# You can check grok patterns at http://grokdebug.herokuapp.com/ -filter { - if "screen" in [tags] and [message] =~ "^\+ " { - drop {} - } - if "console" in [tags] or "console.html" in [tags] { - if [message] == "
" or [message] == "
" { - drop {} - } - multiline { - negate => true - pattern => "^%{TIMESTAMP_ISO8601} \|" - what => "previous" - stream_identity => "%{host}.%{filename}" - } - grok { - # Do multiline matching as the above mutliline filter may add newlines - # to the log messages. - match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate} \| %{GREEDYDATA:logmessage}" } - add_field => { "received_at" => "%{@timestamp}" } - } - } else if "oslofmt" in [tags] { - multiline { - negate => true - pattern => "^(%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP}) " - what => "previous" - stream_identity => "%{host}.%{filename}" - } - multiline { - negate => false - # NOTE(mriedem): oslo.log 1.2.0 changed the logging_exception_prefix - # config option from using TRACE to ERROR so we have to handle both. - # - # NOTE(sdague): stack traces always include process id, so - # NUMBER being required element here is important, otherwise - # ERROR messages just fold into the previous messages, which are - # typically INFO. - pattern => "^(%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP})%{SPACE}%{NUMBER}%{SPACE}(TRACE|ERROR)" - what => "previous" - stream_identity => "%{host}.%{filename}" - } - grok { - # Do multiline matching as the above mutliline filter may add newlines - # to the log messages. - # TODO move the LOGLEVELs into a proper grok pattern. - match => { "message" => "(?m)^(%{TIMESTAMP_ISO8601:logdate}|%{SYSLOGTIMESTAMP:logdate})%{SPACE}(%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:|%{NUMBER:pid})?%{SPACE}?(?AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" } - add_field => { "received_at" => "%{@timestamp}" } - } - } else if "apachecombined" in [tags] { - grok { - match => { "message" => "%{COMBINEDAPACHELOG}" } - add_field => { "received_at" => "%{@timestamp}" } - add_field => { "logdate" => "%{timestamp}" } - add_field => { "logmessage" => "%{verb} %{request} %{response}" } - } - } else if "apacheerror" in [tags] { - grok { - match => { "message" => "\[(?%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}%{SPACE}%{TZ}?)\]%{SPACE}\[%{LOGLEVEL:loglevel}\]%{SPACE}%{GREEDYDATA:logmessage}" } - add_field => { "received_at" => "%{@timestamp}" } - } - } else if "libvirt" in [tags] { - grok { - # libvirtd grok filter adapted from - # https://github.com/OpenStratus/openstack-logstash/blob/master/agent.conf - match => { "message" => "%{TIMESTAMP_ISO8601:logdate}:%{SPACE}%{NUMBER:pid}:%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}:%{SPACE}%{GREEDYDATA:logmessage}" } - add_field => { "received_at" => "%{@timestamp}" } - } - } else if "syslog" in [tags] { - grok { - # Syslog grok filter adapted from - # http://cookbook.logstash.net/recipes/syslog-pri/syslog.conf - match => { "message" => "%{SYSLOGTIMESTAMP:logdate}%{SPACE}%{SYSLOGHOST:syslog_host}?%{SPACE}%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:? %{GREEDYDATA:logmessage}" } - add_field => { "received_at" => "%{@timestamp}" } - } - } - - # Filters below here should be consistent for all Jenkins log formats. - # Remove DEBUG logs to reduce the amount of data that needs to be processed. - if [loglevel] == "DEBUG" { - drop {} - } - - if ! ("_grokparsefailure" in [tags]) { - date { - match => [ "logdate", - "yyyy-MM-dd HH:mm:ss.SSS", - "yyyy-MM-dd HH:mm:ss.SSSSSS", - "yyyy-MM-dd HH:mm:ss,SSS", - "yyyy-MM-dd HH:mm:ss", - "MMM d HH:mm:ss", - "MMM dd HH:mm:ss", - "dd/MMM/yyyy:HH:mm:ss Z", - "yyyy-MM-dd HH:mm:ss.SSSZ", - "E MMM dd HH:mm:ss yyyy Z", - "E MMM dd HH:mm:ss yyyy", - "ISO8601" - ] - timezone => "UTC" - } - mutate { - replace => { "message" => "%{logmessage}" } - } - mutate { - remove_field => [ "logdate", "logmessage" ] - } - } -} - -output { - elasticsearch { - hosts => <%= @elasticsearch_nodes.map { |node| node + ":9200" }.inspect %> - manage_template => false - flush_size => 1024 - } -}