Merge "Add logstash filters"

This commit is contained in:
Zuul 2018-06-08 03:32:31 +00:00 committed by Gerrit Code Review
commit 3a2328af00
23 changed files with 464 additions and 1 deletions

View File

@ -86,6 +86,51 @@
tags:
- config
- name: Logstash Extra Patterns
template:
src: "{{ item }}"
dest: "/opt/logstash/patterns/{{ item }}"
owner: "logstash"
group: "logstash"
with_items:
- extras
when:
- logstash_deploy_filters
tags:
- logstash-filters
- config
- name: Deploy Logstash configuration files
template:
src: "{{ item }}"
dest: "/etc/logstash/conf.d/{{ item }}"
with_items:
- 02-general.conf
- 03-nova.conf
- 04-neutron.conf
- 05-glance.conf
- 06-cinder.conf
- 07-libvirt.conf
- 08-apache.conf
- 09-heat.conf
- 10-mysql.conf
- 11-auth.conf
- 12-logstash.conf
- 13-swift.conf
- 14-keystone.conf
- 16-elasticsearch.conf
- 17-rabbitmq.conf
- 18-ceph.conf
- 19-nginx.conf
- 20-magnum.conf
- 21-octavia.conf
- 98-traceback.conf
when:
- logstash_deploy_filters
tags:
- logstash-filters
- config
- name: Ensure logstash ownership
file:
path: /var/lib/logstash

View File

@ -0,0 +1,16 @@
filter {
if "oslofmt" in [tags] or "openstack" in [tags] {
if "Can not find policy directory: policy.d" in [message] {
drop{}
}
grok {
match => {
"message" => [
"^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?",
"^%{CISCOTIMESTAMP:journalddate}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{SYSLOGPROG:prog}%{SPACE}%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}%{SPACE}%{NOTSPACE:loglevel}%{SPACE}%{NOTSPACE:module}%{SPACE}%{GREEDYDATA:logmessage}"
]
}
add_field => { "received_at" => "%{@timestamp}" }
}
}
}

View File

@ -0,0 +1,23 @@
filter {
if "nova" in [tags] {
mutate {
gsub => ['logmessage',"\"",""]
}
if [module] == "nova.osapi_compute.wsgi.server" {
grok {
match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes:int} time\: %{BASE10NUM:httptime:float}" }
add_tag => ["apimetrics"]
}
} else if [module] == "nova.api.ec2" {
grok {
match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NUMBER:seconds}s %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} None\:None %{NUMBER:response} %{GREEDYDATA:user_agent}" }
add_tag => ["apimetrics"]
}
} else if [module] == "nova.metadata.wsgi.server" {
grok {
match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes} time\: %{NUMBER:seconds}" }
add_tag => ["apimetrics"]
}
}
}
}

View File

@ -0,0 +1,28 @@
filter {
if "neutron" in [tags] {
if [module] == "neutron.wsgi" {
if "accepted" not in [logmessage] {
mutate {
gsub => ['logmessage',"\"",""]
}
grok {
match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" }
add_tag => ["apimetrics"]
}
}
} else if "neutron-ha-tool" in [source] {
mutate {
add_tag => ["neutron-ha-tool"]
remove_tag => ["_grokparsefailure"]
}
}
if "starting" in [message] and "_grokparsefailure" in [tags] {
grok {
match => { "logmessage" => "\[(%{NOTSPACE:requestid}|\-)\](%{SPACE}\(%{NUMBER:pid}\)) %{GREEDYDATA:servicemessage}" }
}
mutate {
remove_tag => ["_grokparsefailure"]
}
}
}
}

View File

@ -0,0 +1,16 @@
filter {
if "glance" in [tags] {
if [module] == "eventlet.wsgi.server" {
mutate {
gsub => ['logmessage',"\"",""]
}
grok {
match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" }
add_tag => ["apimetrics"]
}
mutate {
replace => { "module" => "glance.%{module}" }
}
}
}
}

View File

@ -0,0 +1,18 @@
filter {
if "cinder" in [tags] {
if [module] == "cinder.eventlet.wsgi.server" {
if "accepted" not in [logmessage] {
mutate {
gsub => ['logmessage',"\"",""]
}
grok {
match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" }
add_tag => ["apimetrics"]
}
}
mutate {
replace => { "module" => "cinder.%{module}" }
}
}
}
}

View File

@ -0,0 +1,11 @@
filter {
if "libvirt" in [tags] {
grok {
match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}:%{SPACE}%{NUMBER:code}:?%{SPACE}\[?\b%{NOTSPACE:loglevel}\b\]?%{SPACE}?:?%{SPACE}\[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
add_field => { "received_at" => "%{@timestamp}"}
}
mutate {
uppercase => [ "loglevel" ]
}
}
}

View File

@ -0,0 +1,29 @@
filter {
if "horizon" in [tags] {
grok {
patterns_dir => ["/opt/logstash/patterns"]
match => {
"message" => [
"%{COMMONAPACHELOG}",
"\[%{APACHE_ERROR_TIMESTAMP:timestamp}\] \[%{DATA:module}:%{DATA:loglevel}\] \[pid %{POSINT:apache_pid}\:tid %{POSINT:apache_tid}\] ?(?:\[client %{IP:clientip}:%{POSINT:clientport}\] )?%{GREEDYDATA:logmessage}",
"%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{PROG:prog}%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NUMBER}%{SPACE}%{QS}%{SPACE}%{QS}"
]
}
}
if ![loglevel] {
mutate {
add_field => { "logmessage" => "%{request}" }
add_field => { "module" => "horizon.access" }
add_field => { "loglevel" => "INFO" }
add_tag => [ "apache-access" ]
}
} else {
mutate {
replace => { "module" => "horizon.error.%{module}" }
add_tag => [ "apache-error" ]
uppercase => [ "loglevel" ]
}
}
}
}

View File

@ -0,0 +1,23 @@
filter {
if "heat" in [tags] {
if [module] == "eventlet.wsgi.server" {
if "accepted" not in [logmessage] {
mutate {
gsub => ['logmessage',"\"",""]
}
grok {
match => { "logmessage" => "\[%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} %{NOTSPACE} %{NOTSPACE} %{NOTSPACE}\] %{NOTSPACE:requesterip} %{NOTSPACE} %{NOTSPACE} \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes} %{BASE10NUM:httptime}" }
add_tag => ["apimetrics"]
}
}
mutate {
replace => { "module" => "heat.%{module}" }
}
} else if [module] == "heat.engine.service" {
grok {
match => { "logmessage" => "\[%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} %{NOTSPACE} %{NOTSPACE} %{NOTSPACE} %{GREEDYDATA:servicemessage}" }
add_tag => ["apimetrics"]
}
}
}
}

View File

@ -0,0 +1,33 @@
filter {
if "mysql" in [tags] {
grok {
match => { "message" => "# User@Host: %{WORD:user}\[%{WORD}\] @ (%{HOSTNAME:client_hostname}|) \[(%{IP:client_ip}|)\]" }
}
grok {
match => { "message" => "# Thread_id: %{NUMBER:thread_id:int} \s*Schema: (%{WORD:schema}| ) \s*QC_hit: %{WORD:qc_hit}" }
}
grok {
match => { "message" => "# Query_time: %{NUMBER:query_time:float} \s*Lock_time: %{NUMBER:lock_time:float} \s*Rows_sent: %{NUMBER:rows_sent:int} \s*Rows_examined: %{NUMBER:rows_examined:int}" }
}
grok {
match => { "message" => "(?m)SET timestamp=%{NUMBER:timestamp};%{GREEDYDATA:logmessage}" }
}
date {
match => [ "timestamp", "UNIX" ]
}
mutate {
remove_field => "timestamp"
}
mutate {
gsub => [ "logmessage", "^\n", "" ]
add_field => { "module" => "mysql" }
add_field => { "loglevel" => "WARNING" }
}
}
}

View File

@ -0,0 +1,10 @@
filter {
if "auth" in [tags] {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{NOTSPACE:logsource} %{SYSLOGPROG}: (?:%{SPACE})?%{GREEDYDATA:logmessage}" }
}
mutate {
add_field => { "module" => "auth" }
}
}
}

View File

@ -0,0 +1,25 @@
filter {
if "logstash" in [tags] {
grok {
match => {
"message" => "\{\:timestamp=>\"%{TIMESTAMP_ISO8601:timestamp}\", \:message=>\"%{DATA:logmessage}\"(;|)(, \:address=>\"%{URIHOST:address}\", \:exception=>#<%{DATA:exception}>, \:backtrace=>\[%{DATA:backtrace}\]|)(, \:level=>:%{LOGLEVEL:loglevel}|)\}"
}
}
mutate {
add_field => { "module" => "logstash" }
uppercase => [ "loglevel" ]
}
if [loglevel] == "WARN" {
mutate {
replace => { "loglevel" => "WARNING" }
}
} else if ![loglevel] {
mutate {
add_field => { "loglevel" => "ERROR" }
}
}
}
}

View File

@ -0,0 +1,56 @@
filter {
if "swift-container" in [tags] {
grok {
match => {
"message" => "%{CISCOTIMESTAMP}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{CISCOTIMESTAMP}%{SPACE}%{HOSTNAME}%{SPACE}%{PROG}%{SPACE}%{USER}%{SPACE}%{USERNAME}%{SPACE}%{NOTSPACE}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{HTTPDUSER}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{INT}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SECOND}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}"
}
}
}
if "swift-account" in [tags] {
grok {
match => {
"message" => "%{SYSLOGTIMESTAMP}%{SPACE}%{HOSTNAME}%{SPACE}%{PROG}%{SPACE}%{SYSLOGTIMESTAMP}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{IP}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{POSINT}%{SPACE}%{NOTSPACE}%{SPACE}%{QS}%{SPACE}%{QS}%{SPACE}%{QS}%{SPACE}%{SECOND}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NOTSPACE}"
}
}
}
if "swift" in [tags] {
grok {
match => {
"message" => "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{NOTSPACE:logsource} %{SYSLOGPROG:module}: (?:%{SPACE})?%{GREEDYDATA:logmessage}"
}
}
grok {
patterns_dir => ['/opt/logstash/patterns']
match => {
"logmessage" => [
"%{COMBINEDAPACHELOG}",
"%{SWIFTPROXY_ACCESS}",
"%{GREEDYDATA:logmessage} \(txn\: %{DATA:swift_txn}\)"
]
}
tag_on_failure => []
overwrite => [ "logmessage" ]
}
if [request] {
mutate {
replace => { "logmessage" => "%{request}" }
}
}
mutate {
replace => { "module" => "swift.%{module}" }
}
if [file] =~ "error.log$" {
mutate {
add_field => { "loglevel" => "NOTICE" }
}
} else {
mutate {
add_field => { "loglevel" => "INFO" }
}
}
}
}

View File

@ -0,0 +1,28 @@
filter {
if "keystone-access" in [tags] {
grok {
match => { "message" => "%{CISCOTIMESTAMP:keystone_access_timestamp}%{SPACE}%{SYSLOGHOST:log_host}%{SPACE}%{SYSLOGPROG:prog}%{SPACE}%{TIMESTAMP_ISO8601:keystone_timestmp}%{SPACE}%{NUMBER:pid}%{SPACE}%{NOTSPACE:loglevel}%{SPACE}%{NOTSPACE:module}%{SPACE}%{SYSLOG5424SD:requestid}%{SPACE}%{WORD:verb}%{SPACE}%{NOTSPACE:request}" }
}
}
if "keystone" in [tags] {
if "apache-access" in [tags] {
grok {
match => { "message" => "%{COMMONAPACHELOG}" }
}
mutate {
add_field => { "logmessage" => "%{request}" }
add_field => { "module" => "keystone.access" }
add_field => { "loglevel" => "INFO" }
}
} else if "apache-error" in [tags] {
grok {
patterns_dir => ["/opt/logstash/patterns"]
match => { "message" => "%{KEYSTONE_SUBSECOND_TIMESTAMP:keystone_subsecond_timestamp} %{STANDARD_TIMESTAMP:standard_timestamp} %{NUMBER:pid} %{DATA:loglevel} %{DATA:module} \[%{DATA:requestid}\] %{WORD:verb} %{NOTSPACE:request}" }
}
mutate {
replace => { "module" => "keystone.error.%{module}" }
uppercase => [ "loglevel" ]
}
}
}
}

View File

@ -0,0 +1,10 @@
filter {
if "elasticsearch" in [tags] {
grok {
match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{LOGLEVEL:loglevel}\s*\]\[%{NOTSPACE:module}\s*\] %{GREEDYDATA:logmessage}" }
}
mutate {
replace => { "module" => "elasticsearch.%{module}" }
}
}
}

View File

@ -0,0 +1,20 @@
filter {
if "rabbitmq" in [tags] {
if [message] == "" {
drop { }
}
grok {
match => { "message" => "^\=%{LOGLEVEL:loglevel} REPORT\=\=\=\= %{MONTHDAY:event_day}\-%{MONTH:event_month}\-%{YEAR:event_year}\:\:%{TIME:event_time} \=\=\=\n%{GREEDYDATA:logmessage}" }
}
mutate {
replace => { "module" => "rabbitmq" }
add_field => { "timestamp" => "%{event_day} %{event_month} %{event_year} %{event_time}" }
}
date {
match => [ "timestamp", "dd MMM YYYY HH:mm:ss" ]
remove_field => [ "event_day", "event_month", "event_year", "event_time", "timestamp" ]
}
}
}

View File

@ -0,0 +1,12 @@
filter {
if "ceph" in [tags] {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:date} %{NOTSPACE:osd_epoch} ?%{SPACE}?%{NOTSPACE:error_bool} %{GREEDYDATA:logmessage}" }
}
}
if "ceph-osd" in [tags] {
grok {
match => { "message" => "-- (?<src_host>(%{IPORHOST}\:%{POSINT}/%{POSINT})) (?:[<|>]){1,2} (?<dst_host>(%{IPORHOST}\:%{POSINT}/%{POSINT}))" }
}
}
}

View File

@ -0,0 +1,20 @@
filter {
if "nginx" in [tags] {
if "nginx-access" in [tags] {
grok {
patterns_dir => ['/opt/logstash/patterns']
match => {
"message" => "%{IP:client_ip} - %{USER:client_user} \[%{NGINX_TIMESTAMP:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{INT:response_code} %{INT:bytes} %{QUOTEDSTRING:referer} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:gzip_ratio}"
}
}
}
if "nginx-error" in [tags] {
grok {
patterns_dir => ['/opt/logstash/patterns']
match => {
"message" => "%{NGINX_ERROR_TIMESTAMP:timestamp} \[%{LOGLEVEL:loglevel}\] %{GREEDYDATA:error_msg}"
}
}
}
}
}

View File

@ -0,0 +1,16 @@
filter {
if "magnum" in [tags] {
if [module] == "eventlet.wsgi.server" {
mutate {
gsub => ['logmessage',"\"",""]
}
grok {
match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" }
add_tag => ["apimetrics"]
}
mutate {
replace => { "module" => "magnum.%{module}" }
}
}
}
}

View File

@ -0,0 +1,16 @@
filter {
if "octavia" in [tags] {
if [module] == "eventlet.wsgi.server" {
mutate {
gsub => ['logmessage',"\"",""]
}
grok {
match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" }
add_tag => ["apimetrics"]
}
mutate {
replace => { "module" => "octavia.%{module}" }
}
}
}
}

View File

@ -0,0 +1,8 @@
filter {
if "Traceback" in [message] {
mutate {
add_tag => ["traceback"]
remove_tag => ["_grokparsefailure"]
}
}
}

View File

@ -35,7 +35,7 @@ kibana_server_name: "{{ ansible_hostname }}"
# logstash vars
logstash_beat_input_port: 5044
logstash_deploy_filters: true
# Beat options
heartbeat_services: