From b6343c57a43a4d021c462bb54e7cbda4d165d5d7 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 26 Jul 2018 19:20:57 -0500 Subject: [PATCH] Convert logstash groks to a multi-pipeline setup The logstash groks were running in line using the legacy method which uses lexical sorting of all logstash filter files and loads them in order. While this works it makes it so all data has to travel through all filters. This change makes use of the logstash multi-pipeline capabilities using a distributor and fork pattern. This allows data to flow through logstash more quickly and not block whenever there's an issue with an output plugin. Finger-prints using SHA1 when there's a message and UUID when not. This will ensure we're duplicating log entries which will help speed up transations and further reduce the storage required. Change-Id: I38268e33b370da0f1e186ecf65911d4a312c3e6a Signed-off-by: Kevin Carter --- .../common_task_data_node_hosts.yml | 1 + elk_metrics_6x/createElasticIndexes.yml | 9 +- elk_metrics_6x/files/02-general.conf | 16 - elk_metrics_6x/files/02-journald.conf | 22 - elk_metrics_6x/files/03-nova.conf | 23 - elk_metrics_6x/files/04-neutron.conf | 28 - elk_metrics_6x/files/05-glance.conf | 16 - elk_metrics_6x/files/06-cinder.conf | 18 - elk_metrics_6x/files/07-libvirt.conf | 11 - elk_metrics_6x/files/08-apache.conf | 31 - elk_metrics_6x/files/09-heat.conf | 23 - elk_metrics_6x/files/10-mysql.conf | 37 - elk_metrics_6x/files/10-syslog-filter.conf | 13 - elk_metrics_6x/files/11-auth.conf | 10 - elk_metrics_6x/files/12-logstash.conf | 25 - elk_metrics_6x/files/13-swift.conf | 56 -- elk_metrics_6x/files/14-keystone.conf | 28 - elk_metrics_6x/files/16-elasticsearch.conf | 10 - elk_metrics_6x/files/17-rabbitmq.conf | 20 - elk_metrics_6x/files/18-ceph.conf | 12 - elk_metrics_6x/files/19-nginx.conf | 23 - elk_metrics_6x/files/20-magnum.conf | 16 - elk_metrics_6x/files/21-octavia.conf | 16 - elk_metrics_6x/files/98-traceback.conf | 8 - elk_metrics_6x/installLogstash.yml | 87 +-- .../templates/02-beats-input.conf.j2 | 17 - .../templates/99-elasticsearch-output.conf.j2 | 28 - .../templates/99-kafka-output.conf.j2 | 13 - .../templates/_include_setup_template.yml.j2 | 3 + .../templates/curator-actions.yml.j2 | 2 +- elk_metrics_6x/templates/filebeat.yml.j2 | 3 +- .../templates/logstash-pipelines.yml.j2 | 739 ++++++++++++++++++ elk_metrics_6x/templates/metricbeat.yml.j2 | 4 +- 33 files changed, 777 insertions(+), 591 deletions(-) delete mode 100644 elk_metrics_6x/files/02-general.conf delete mode 100644 elk_metrics_6x/files/02-journald.conf delete mode 100644 elk_metrics_6x/files/03-nova.conf delete mode 100644 elk_metrics_6x/files/04-neutron.conf delete mode 100644 elk_metrics_6x/files/05-glance.conf delete mode 100644 elk_metrics_6x/files/06-cinder.conf delete mode 100644 elk_metrics_6x/files/07-libvirt.conf delete mode 100644 elk_metrics_6x/files/08-apache.conf delete mode 100644 elk_metrics_6x/files/09-heat.conf delete mode 100644 elk_metrics_6x/files/10-mysql.conf delete mode 100644 elk_metrics_6x/files/10-syslog-filter.conf delete mode 100644 elk_metrics_6x/files/11-auth.conf delete mode 100644 elk_metrics_6x/files/12-logstash.conf delete mode 100644 elk_metrics_6x/files/13-swift.conf delete mode 100644 elk_metrics_6x/files/14-keystone.conf delete mode 100644 elk_metrics_6x/files/16-elasticsearch.conf delete mode 100644 elk_metrics_6x/files/17-rabbitmq.conf delete mode 100644 elk_metrics_6x/files/18-ceph.conf delete mode 100644 elk_metrics_6x/files/19-nginx.conf delete mode 100644 elk_metrics_6x/files/20-magnum.conf delete mode 100644 elk_metrics_6x/files/21-octavia.conf delete mode 100644 elk_metrics_6x/files/98-traceback.conf delete mode 100644 elk_metrics_6x/templates/02-beats-input.conf.j2 delete mode 100644 elk_metrics_6x/templates/99-elasticsearch-output.conf.j2 delete mode 100644 elk_metrics_6x/templates/99-kafka-output.conf.j2 create mode 100644 elk_metrics_6x/templates/logstash-pipelines.yml.j2 diff --git a/elk_metrics_6x/common_task_data_node_hosts.yml b/elk_metrics_6x/common_task_data_node_hosts.yml index ca60efbc..186453c7 100644 --- a/elk_metrics_6x/common_task_data_node_hosts.yml +++ b/elk_metrics_6x/common_task_data_node_hosts.yml @@ -112,6 +112,7 @@ # ingest nodes. - name: Set data nodes set_fact: + elasticsearch_number_of_replicas: "{{ ((data_nodes | length) > 1) | ternary(((data_nodes | length) > 2 | ternary(2, 1)), 0) }}" elasticsearch_data_hosts: |- {% set nodes = elasticsearch_data_node_details %} {% if inventory_hostname in data_nodes %} diff --git a/elk_metrics_6x/createElasticIndexes.yml b/elk_metrics_6x/createElasticIndexes.yml index c26d21a6..bfe8f40b 100644 --- a/elk_metrics_6x/createElasticIndexes.yml +++ b/elk_metrics_6x/createElasticIndexes.yml @@ -8,6 +8,11 @@ environment: "{{ deployment_environment_variables | default({}) }}" + pre_tasks: + - include_tasks: common_task_data_node_hosts.yml + tags: + - always + tasks: - name: Create basic indexes uri: @@ -30,7 +35,6 @@ total_fields: limit: "10000" refresh_interval: "5s" - number_of_replicas: "1" - name: "_all/_settings?preserve_existing=true" index_options: index.refresh_interval: "10s" @@ -38,3 +42,6 @@ index_options: index.queries.cache.enabled: "true" indices.queries.cache.size: "5%" + - name: "_all/_settings" + index_options: + index.number_of_replicas: "{{ elasticsearch_number_of_replicas }}" diff --git a/elk_metrics_6x/files/02-general.conf b/elk_metrics_6x/files/02-general.conf deleted file mode 100644 index 16bba6bf..00000000 --- a/elk_metrics_6x/files/02-general.conf +++ /dev/null @@ -1,16 +0,0 @@ -filter { - if "oslofmt" in [tags] or "openstack" in [tags] { - if "Can not find policy directory: policy.d" in [message] { - drop{} - } - grok { - match => { - "message" => [ - "^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?", - "^%{CISCOTIMESTAMP:journalddate}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{SYSLOGPROG:prog}%{SPACE}%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}%{SPACE}%{NOTSPACE:loglevel}%{SPACE}%{NOTSPACE:module}%{SPACE}%{GREEDYDATA:logmessage}" - ] - } - add_field => { "received_at" => "%{@timestamp}" } - } - } -} diff --git a/elk_metrics_6x/files/02-journald.conf b/elk_metrics_6x/files/02-journald.conf deleted file mode 100644 index 6822f565..00000000 --- a/elk_metrics_6x/files/02-journald.conf +++ /dev/null @@ -1,22 +0,0 @@ -filter { - if "journald" in [tags] { - if [systemd_slice] { - mutate { - copy => { "systemd_slice" => "systemd_slice_tag" } - } - mutate { - gsub => [ "systemd_slice_tag", ".slice", "" ] - } - if [systemd_slice_tag] != "-" { - mutate { - add_tag => [ - "%{systemd_slice_tag}" - ] - } - } - mutate { - remove_field => [ "%{systemd_slice_tag}" ] - } - } - } -} diff --git a/elk_metrics_6x/files/03-nova.conf b/elk_metrics_6x/files/03-nova.conf deleted file mode 100644 index c020994c..00000000 --- a/elk_metrics_6x/files/03-nova.conf +++ /dev/null @@ -1,23 +0,0 @@ -filter { - if "nova" in [tags] { - mutate { - gsub => ['logmessage',"\"",""] - } - if [module] == "nova.osapi_compute.wsgi.server" { - grok { - match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes:int} time\: %{BASE10NUM:httptime:float}" } - add_tag => ["apimetrics"] - } - } else if [module] == "nova.api.ec2" { - grok { - match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NUMBER:seconds}s %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} None\:None %{NUMBER:response} %{GREEDYDATA:user_agent}" } - add_tag => ["apimetrics"] - } - } else if [module] == "nova.metadata.wsgi.server" { - grok { - match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes} time\: %{NUMBER:seconds}" } - add_tag => ["apimetrics"] - } - } - } -} diff --git a/elk_metrics_6x/files/04-neutron.conf b/elk_metrics_6x/files/04-neutron.conf deleted file mode 100644 index 13acfdfc..00000000 --- a/elk_metrics_6x/files/04-neutron.conf +++ /dev/null @@ -1,28 +0,0 @@ -filter { - if "neutron" in [tags] { - if [module] == "neutron.wsgi" { - if "accepted" not in [logmessage] { - mutate { - gsub => ['logmessage',"\"",""] - } - grok { - match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } - add_tag => ["apimetrics"] - } - } - } else if "neutron-ha-tool" in [source] { - mutate { - add_tag => ["neutron-ha-tool"] - remove_tag => ["_grokparsefailure"] - } - } - if "starting" in [message] and "_grokparsefailure" in [tags] { - grok { - match => { "logmessage" => "\[(%{NOTSPACE:requestid}|\-)\](%{SPACE}\(%{NUMBER:pid}\)) %{GREEDYDATA:servicemessage}" } - } - mutate { - remove_tag => ["_grokparsefailure"] - } - } - } -} diff --git a/elk_metrics_6x/files/05-glance.conf b/elk_metrics_6x/files/05-glance.conf deleted file mode 100644 index 2eaec5dd..00000000 --- a/elk_metrics_6x/files/05-glance.conf +++ /dev/null @@ -1,16 +0,0 @@ -filter { - if "glance" in [tags] { - if [module] == "eventlet.wsgi.server" { - mutate { - gsub => ['logmessage',"\"",""] - } - grok { - match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } - add_tag => ["apimetrics"] - } - mutate { - replace => { "module" => "glance.%{module}" } - } - } - } -} diff --git a/elk_metrics_6x/files/06-cinder.conf b/elk_metrics_6x/files/06-cinder.conf deleted file mode 100644 index 09c62fe9..00000000 --- a/elk_metrics_6x/files/06-cinder.conf +++ /dev/null @@ -1,18 +0,0 @@ -filter { - if "cinder" in [tags] { - if [module] == "cinder.eventlet.wsgi.server" { - if "accepted" not in [logmessage] { - mutate { - gsub => ['logmessage',"\"",""] - } - grok { - match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } - add_tag => ["apimetrics"] - } - } - mutate { - replace => { "module" => "cinder.%{module}" } - } - } - } -} diff --git a/elk_metrics_6x/files/07-libvirt.conf b/elk_metrics_6x/files/07-libvirt.conf deleted file mode 100644 index c417c155..00000000 --- a/elk_metrics_6x/files/07-libvirt.conf +++ /dev/null @@ -1,11 +0,0 @@ -filter { - if "libvirt" in [tags] { - grok { - match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}:%{SPACE}%{NUMBER:code}:?%{SPACE}\[?\b%{NOTSPACE:loglevel}\b\]?%{SPACE}?:?%{SPACE}\[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" } - add_field => { "received_at" => "%{@timestamp}"} - } - mutate { - uppercase => [ "loglevel" ] - } - } -} diff --git a/elk_metrics_6x/files/08-apache.conf b/elk_metrics_6x/files/08-apache.conf deleted file mode 100644 index 59b0de89..00000000 --- a/elk_metrics_6x/files/08-apache.conf +++ /dev/null @@ -1,31 +0,0 @@ -filter { - if "horizon" in [tags] { - grok { - patterns_dir => ["/opt/logstash/patterns"] - match => { - "message" => [ - "%{COMMONAPACHELOG}", - "\[%{APACHE_ERROR_TIMESTAMP:timestamp}\] \[%{DATA:module}:%{DATA:loglevel}\] \[pid %{POSINT:apache_pid}\:tid %{POSINT:apache_tid}\] ?(?:\[client %{IP:clientip}:%{POSINT:clientport}\] )?%{GREEDYDATA:logmessage}", - "%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{PROG:prog}%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NUMBER}%{SPACE}%{QS}%{SPACE}%{QS}" - ] - } - } - geoip { - source => "clientip" - } - if ![loglevel] { - mutate { - add_field => { "logmessage" => "%{request}" } - add_field => { "module" => "horizon.access" } - add_field => { "loglevel" => "INFO" } - add_tag => [ "apache-access" ] - } - } else { - mutate { - replace => { "module" => "horizon.error.%{module}" } - add_tag => [ "apache-error" ] - uppercase => [ "loglevel" ] - } - } - } -} diff --git a/elk_metrics_6x/files/09-heat.conf b/elk_metrics_6x/files/09-heat.conf deleted file mode 100644 index 8c22e39a..00000000 --- a/elk_metrics_6x/files/09-heat.conf +++ /dev/null @@ -1,23 +0,0 @@ -filter { - if "heat" in [tags] { - if [module] == "eventlet.wsgi.server" { - if "accepted" not in [logmessage] { - mutate { - gsub => ['logmessage',"\"",""] - } - grok { - match => { "logmessage" => "\[%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} %{NOTSPACE} %{NOTSPACE} %{NOTSPACE}\] %{NOTSPACE:requesterip} %{NOTSPACE} %{NOTSPACE} \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes} %{BASE10NUM:httptime}" } - add_tag => ["apimetrics"] - } - } - mutate { - replace => { "module" => "heat.%{module}" } - } - } else if [module] == "heat.engine.service" { - grok { - match => { "logmessage" => "\[%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} %{NOTSPACE} %{NOTSPACE} %{NOTSPACE} %{GREEDYDATA:servicemessage}" } - add_tag => ["apimetrics"] - } - } - } -} diff --git a/elk_metrics_6x/files/10-mysql.conf b/elk_metrics_6x/files/10-mysql.conf deleted file mode 100644 index c977c005..00000000 --- a/elk_metrics_6x/files/10-mysql.conf +++ /dev/null @@ -1,37 +0,0 @@ -filter { - if "mysql" in [tags] { - grok { - match => { "message" => "# User@Host: %{WORD:user}\[%{WORD}\] @ (%{HOSTNAME:client_hostname}|) \[(%{IP:client_ip}|)\]" } - } - - grok { - match => { "message" => "# Thread_id: %{NUMBER:thread_id:int} \s*Schema: (%{WORD:schema}| ) \s*QC_hit: %{WORD:qc_hit}" } - } - - grok { - match => { "message" => "# Query_time: %{NUMBER:query_time:float} \s*Lock_time: %{NUMBER:lock_time:float} \s*Rows_sent: %{NUMBER:rows_sent:int} \s*Rows_examined: %{NUMBER:rows_examined:int}" } - } - - grok { - match => { "message" => "(?m)SET timestamp=%{NUMBER:timestamp};%{GREEDYDATA:logmessage}" } - } - - geoip { - source => "clientip" - } - - date { - match => [ "timestamp", "UNIX" ] - } - - mutate { - remove_field => "timestamp" - } - - mutate { - gsub => [ "logmessage", "^\n", "" ] - add_field => { "module" => "mysql" } - add_field => { "loglevel" => "WARNING" } - } - } -} diff --git a/elk_metrics_6x/files/10-syslog-filter.conf b/elk_metrics_6x/files/10-syslog-filter.conf deleted file mode 100644 index acce463c..00000000 --- a/elk_metrics_6x/files/10-syslog-filter.conf +++ /dev/null @@ -1,13 +0,0 @@ -filter { - if [type] == "syslog" { - grok { - match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } - add_field => [ "received_at", "%{@timestamp}" ] - add_field => [ "received_from", "%{host}" ] - } - syslog_pri { } - date { - match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] - } - } -} diff --git a/elk_metrics_6x/files/11-auth.conf b/elk_metrics_6x/files/11-auth.conf deleted file mode 100644 index d9d1ee45..00000000 --- a/elk_metrics_6x/files/11-auth.conf +++ /dev/null @@ -1,10 +0,0 @@ -filter { - if "auth" in [tags] { - grok { - match => { "message" => "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{NOTSPACE:logsource} %{SYSLOGPROG}: (?:%{SPACE})?%{GREEDYDATA:logmessage}" } - } - mutate { - add_field => { "module" => "auth" } - } - } -} diff --git a/elk_metrics_6x/files/12-logstash.conf b/elk_metrics_6x/files/12-logstash.conf deleted file mode 100644 index e8ff8d3e..00000000 --- a/elk_metrics_6x/files/12-logstash.conf +++ /dev/null @@ -1,25 +0,0 @@ -filter { - if "logstash" in [tags] { - grok { - match => { - "message" => "\{\:timestamp=>\"%{TIMESTAMP_ISO8601:timestamp}\", \:message=>\"%{DATA:logmessage}\"(;|)(, \:address=>\"%{URIHOST:address}\", \:exception=>#<%{DATA:exception}>, \:backtrace=>\[%{DATA:backtrace}\]|)(, \:level=>:%{LOGLEVEL:loglevel}|)\}" - } - } - - mutate { - add_field => { "module" => "logstash" } - uppercase => [ "loglevel" ] - } - - if [loglevel] == "WARN" { - mutate { - replace => { "loglevel" => "WARNING" } - } - } else if ![loglevel] { - mutate { - add_field => { "loglevel" => "ERROR" } - } - } - - } -} diff --git a/elk_metrics_6x/files/13-swift.conf b/elk_metrics_6x/files/13-swift.conf deleted file mode 100644 index aaa577d5..00000000 --- a/elk_metrics_6x/files/13-swift.conf +++ /dev/null @@ -1,56 +0,0 @@ -filter { - if "swift-container" in [tags] { - grok { - match => { - "message" => "%{CISCOTIMESTAMP}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{CISCOTIMESTAMP}%{SPACE}%{HOSTNAME}%{SPACE}%{PROG}%{SPACE}%{USER}%{SPACE}%{USERNAME}%{SPACE}%{NOTSPACE}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{HTTPDUSER}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{INT}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SECOND}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}" - } - } - } - if "swift-account" in [tags] { - grok { - match => { - "message" => "%{SYSLOGTIMESTAMP}%{SPACE}%{HOSTNAME}%{SPACE}%{PROG}%{SPACE}%{SYSLOGTIMESTAMP}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{IP}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{POSINT}%{SPACE}%{NOTSPACE}%{SPACE}%{QS}%{SPACE}%{QS}%{SPACE}%{QS}%{SPACE}%{SECOND}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NOTSPACE}" - } - } - } - if "swift" in [tags] { - grok { - match => { - "message" => "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{NOTSPACE:logsource} %{SYSLOGPROG:module}: (?:%{SPACE})?%{GREEDYDATA:logmessage}" - } - } - - grok { - patterns_dir => ['/opt/logstash/patterns'] - match => { - "logmessage" => [ - "%{COMBINEDAPACHELOG}", - "%{SWIFTPROXY_ACCESS}", - "%{GREEDYDATA:logmessage} \(txn\: %{DATA:swift_txn}\)" - ] - } - tag_on_failure => [] - overwrite => [ "logmessage" ] - } - - if [request] { - mutate { - replace => { "logmessage" => "%{request}" } - } - } - - mutate { - replace => { "module" => "swift.%{module}" } - } - - if [file] =~ "error.log$" { - mutate { - add_field => { "loglevel" => "NOTICE" } - } - } else { - mutate { - add_field => { "loglevel" => "INFO" } - } - } - } -} diff --git a/elk_metrics_6x/files/14-keystone.conf b/elk_metrics_6x/files/14-keystone.conf deleted file mode 100644 index e8cc6eca..00000000 --- a/elk_metrics_6x/files/14-keystone.conf +++ /dev/null @@ -1,28 +0,0 @@ -filter { - if "keystone-access" in [tags] { - grok { - match => { "message" => "%{CISCOTIMESTAMP:keystone_access_timestamp}%{SPACE}%{SYSLOGHOST:log_host}%{SPACE}%{SYSLOGPROG:prog}%{SPACE}%{TIMESTAMP_ISO8601:keystone_timestmp}%{SPACE}%{NUMBER:pid}%{SPACE}%{NOTSPACE:loglevel}%{SPACE}%{NOTSPACE:module}%{SPACE}%{SYSLOG5424SD:requestid}%{SPACE}%{WORD:verb}%{SPACE}%{NOTSPACE:request}" } - } - } - if "keystone" in [tags] { - if "apache-access" in [tags] { - grok { - match => { "message" => "%{COMMONAPACHELOG}" } - } - mutate { - add_field => { "logmessage" => "%{request}" } - add_field => { "module" => "keystone.access" } - add_field => { "loglevel" => "INFO" } - } - } else if "apache-error" in [tags] { - grok { - patterns_dir => ["/opt/logstash/patterns"] - match => { "message" => "%{KEYSTONE_SUBSECOND_TIMESTAMP:keystone_subsecond_timestamp} %{STANDARD_TIMESTAMP:standard_timestamp} %{NUMBER:pid} %{DATA:loglevel} %{DATA:module} \[%{DATA:requestid}\] %{WORD:verb} %{NOTSPACE:request}" } - } - mutate { - replace => { "module" => "keystone.error.%{module}" } - uppercase => [ "loglevel" ] - } - } - } -} diff --git a/elk_metrics_6x/files/16-elasticsearch.conf b/elk_metrics_6x/files/16-elasticsearch.conf deleted file mode 100644 index 8ca5890d..00000000 --- a/elk_metrics_6x/files/16-elasticsearch.conf +++ /dev/null @@ -1,10 +0,0 @@ -filter { - if "elasticsearch" in [tags] { - grok { - match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{LOGLEVEL:loglevel}\s*\]\[%{NOTSPACE:module}\s*\] %{GREEDYDATA:logmessage}" } - } - mutate { - replace => { "module" => "elasticsearch.%{module}" } - } - } -} diff --git a/elk_metrics_6x/files/17-rabbitmq.conf b/elk_metrics_6x/files/17-rabbitmq.conf deleted file mode 100644 index c46f2551..00000000 --- a/elk_metrics_6x/files/17-rabbitmq.conf +++ /dev/null @@ -1,20 +0,0 @@ -filter { - if "rabbitmq" in [tags] { - if [message] == "" { - drop { } - } - grok { - match => { "message" => "^\=%{LOGLEVEL:loglevel} REPORT\=\=\=\= %{MONTHDAY:event_day}\-%{MONTH:event_month}\-%{YEAR:event_year}\:\:%{TIME:event_time} \=\=\=\n%{GREEDYDATA:logmessage}" } - } - - mutate { - replace => { "module" => "rabbitmq" } - add_field => { "timestamp" => "%{event_day} %{event_month} %{event_year} %{event_time}" } - } - - date { - match => [ "timestamp", "dd MMM YYYY HH:mm:ss" ] - remove_field => [ "event_day", "event_month", "event_year", "event_time", "timestamp" ] - } - } -} diff --git a/elk_metrics_6x/files/18-ceph.conf b/elk_metrics_6x/files/18-ceph.conf deleted file mode 100644 index 5eb0d700..00000000 --- a/elk_metrics_6x/files/18-ceph.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - if "ceph" in [tags] { - grok { - match => { "message" => "%{TIMESTAMP_ISO8601:date} %{NOTSPACE:osd_epoch} ?%{SPACE}?%{NOTSPACE:error_bool} %{GREEDYDATA:logmessage}" } - } - } - if "ceph-osd" in [tags] { - grok { - match => { "message" => "-- (?(%{IPORHOST}\:%{POSINT}/%{POSINT})) (?:[<|>]){1,2} (?(%{IPORHOST}\:%{POSINT}/%{POSINT}))" } - } - } -} diff --git a/elk_metrics_6x/files/19-nginx.conf b/elk_metrics_6x/files/19-nginx.conf deleted file mode 100644 index 8a68484c..00000000 --- a/elk_metrics_6x/files/19-nginx.conf +++ /dev/null @@ -1,23 +0,0 @@ -filter { - if "nginx" in [tags] { - if "nginx-access" in [tags] { - grok { - patterns_dir => ['/opt/logstash/patterns'] - match => { - "message" => "%{IP:client_ip} - %{USER:client_user} \[%{NGINX_TIMESTAMP:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{INT:response_code} %{INT:bytes} %{QUOTEDSTRING:referer} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:gzip_ratio}" - } - } - geoip { - source => "clientip" - } - } - if "nginx-error" in [tags] { - grok { - patterns_dir => ['/opt/logstash/patterns'] - match => { - "message" => "%{NGINX_ERROR_TIMESTAMP:timestamp} \[%{LOGLEVEL:loglevel}\] %{GREEDYDATA:error_msg}" - } - } - } - } -} diff --git a/elk_metrics_6x/files/20-magnum.conf b/elk_metrics_6x/files/20-magnum.conf deleted file mode 100644 index 0b827b0a..00000000 --- a/elk_metrics_6x/files/20-magnum.conf +++ /dev/null @@ -1,16 +0,0 @@ -filter { - if "magnum" in [tags] { - if [module] == "eventlet.wsgi.server" { - mutate { - gsub => ['logmessage',"\"",""] - } - grok { - match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } - add_tag => ["apimetrics"] - } - mutate { - replace => { "module" => "magnum.%{module}" } - } - } - } -} diff --git a/elk_metrics_6x/files/21-octavia.conf b/elk_metrics_6x/files/21-octavia.conf deleted file mode 100644 index e797f6e1..00000000 --- a/elk_metrics_6x/files/21-octavia.conf +++ /dev/null @@ -1,16 +0,0 @@ -filter { - if "octavia" in [tags] { - if [module] == "eventlet.wsgi.server" { - mutate { - gsub => ['logmessage',"\"",""] - } - grok { - match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } - add_tag => ["apimetrics"] - } - mutate { - replace => { "module" => "octavia.%{module}" } - } - } - } -} diff --git a/elk_metrics_6x/files/98-traceback.conf b/elk_metrics_6x/files/98-traceback.conf deleted file mode 100644 index 102dd3b6..00000000 --- a/elk_metrics_6x/files/98-traceback.conf +++ /dev/null @@ -1,8 +0,0 @@ -filter { - if "Traceback" in [message] { - mutate { - add_tag => ["traceback"] - remove_tag => ["_grokparsefailure"] - } - } -} diff --git a/elk_metrics_6x/installLogstash.yml b/elk_metrics_6x/installLogstash.yml index 40379a30..1c1e128e 100644 --- a/elk_metrics_6x/installLogstash.yml +++ b/elk_metrics_6x/installLogstash.yml @@ -8,6 +8,7 @@ vars: temp_dir: /var/lib/logstash/tmp + logstash_pipelines: "{{lookup('template', 'templates/logstash-pipelines.yml.j2') }}" environment: "{{ deployment_environment_variables | default({}) }}" @@ -27,7 +28,7 @@ - name: Set processor cores fact set_fact: - q_storage: "{{ (ansible_processor_cores | int) * 2 }}" + q_storage: "{{ (ansible_processor_cores | int) * (ansible_processor_threads_per_core | int) * 2 }}" when: - q_storage is not defined tags: @@ -36,7 +37,7 @@ - name: Set logstash facts set_fact: elastic_heap_size: "{{ ((q_mem | int) > 30720) | ternary(30720, q_mem) }}" - logstash_queue_size: "{{ (((q_storage | int) > 16) | ternary(16, q_storage) | int) * 1024 }}" + logstash_queue_size: "{{ ((((q_storage | int) >= 2) | ternary(q_storage, 2) | int) * 1024) // ((logstash_pipelines | from_yaml) | length) }}" elastic_log_rotate_path: "/var/log/logstash" tags: - always @@ -125,24 +126,6 @@ notify: - Enable and restart logstash - - name: Drop logstash conf file(s) - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - with_items: - - src: templates/jvm.options.j2 - dest: /etc/logstash/jvm.options - - src: templates/99-elasticsearch-output.conf.j2 - dest: /etc/logstash/conf.d/99-elasticsearch-output.conf - - src: templates/02-beats-input.conf.j2 - dest: /etc/logstash/conf.d/02-beats-input.conf - - src: templates/logstash.yml.j2 - dest: /etc/logstash/logstash.yml - notify: - - Enable and restart logstash - tags: - - config - - name: Create patterns directory file: name: "/opt/logstash/patterns" @@ -168,41 +151,6 @@ - logstash-filters - config - - name: Deploy Logstash configuration files - copy: - src: "files/{{ item }}" - dest: "/etc/logstash/conf.d/{{ item }}" - with_items: - - 02-general.conf - - 02-journald.conf - - 03-nova.conf - - 04-neutron.conf - - 05-glance.conf - - 06-cinder.conf - - 07-libvirt.conf - - 08-apache.conf - - 09-heat.conf - - 10-mysql.conf - - 10-syslog-filter.conf - - 11-auth.conf - - 12-logstash.conf - - 13-swift.conf - - 14-keystone.conf - - 16-elasticsearch.conf - - 17-rabbitmq.conf - - 18-ceph.conf - - 19-nginx.conf - - 20-magnum.conf - - 21-octavia.conf - - 98-traceback.conf - when: - - logstash_deploy_filters - notify: - - Enable and restart logstash - tags: - - logstash-filters - - config - - name: Run kafka output block block: - name: Copy kafka keystore into place @@ -219,15 +167,27 @@ when: - logstash_kafka_ssl_truststore_location is defined - - name: Deploy Logstash kafka configuration files - template: - src: "templates/99-kafka-output.conf.j2" - dest: "/etc/logstash/conf.d/99-kafka-output.conf" - notify: - - Enable and restart logstash when: - logstash_kafka_options is defined + - name: Drop logstash conf file(s) + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: + - src: templates/jvm.options.j2 + dest: /etc/logstash/jvm.options + - src: templates/logstash.yml.j2 + dest: /etc/logstash/logstash.yml + - src: templates/logstash-pipelines.yml.j2 + dest: /etc/logstash/pipelines.yml + - src: "templates/logrotate.j2" + dest: "/etc/logrotate.d/logstash" + notify: + - Enable and restart logstash + tags: + - config + - name: Ensure logstash ownership file: path: /var/lib/logstash @@ -247,11 +207,6 @@ group: "logstash" mode: "0750" - - name: Create logrotate config - template: - src: "templates/logrotate.j2" - dest: "/etc/logrotate.d/logstash" - - name: Run arcsight output block block: - name: Initialise arcsight local facts diff --git a/elk_metrics_6x/templates/02-beats-input.conf.j2 b/elk_metrics_6x/templates/02-beats-input.conf.j2 deleted file mode 100644 index 6c979811..00000000 --- a/elk_metrics_6x/templates/02-beats-input.conf.j2 +++ /dev/null @@ -1,17 +0,0 @@ -input { - beats { - port => {{ logstash_beat_input_port }} - } -} - -filter { - if [source.ip] { - geoip { - source => "source.ip" - } - } else if [ip] { - geoip { - source => "ip" - } - } -} diff --git a/elk_metrics_6x/templates/99-elasticsearch-output.conf.j2 b/elk_metrics_6x/templates/99-elasticsearch-output.conf.j2 deleted file mode 100644 index bbc7046f..00000000 --- a/elk_metrics_6x/templates/99-elasticsearch-output.conf.j2 +++ /dev/null @@ -1,28 +0,0 @@ -filter { - fingerprint { - source => "message" - target => "[@metadata][fingerprint]" - method => "SHA1" - key => "{{ cluster_name | replace(' ', '_') }}" - base64encode => true - } -} -output { - if [@metadata][version] { - elasticsearch { - document_id => "%{[@metadata][fingerprint]}" - hosts => {{ elasticsearch_data_hosts | shuffle(seed=inventory_hostname) | to_json }} - sniffing => {{ (not data_node | bool) | lower }} - manage_template => {{ (data_node | bool) | lower }} - index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" - } - } else { - elasticsearch { - document_id => "%{[@metadata][fingerprint]}" - hosts => {{ elasticsearch_data_hosts | shuffle(seed=inventory_hostname) | to_json }} - sniffing => {{ (not data_node | bool) | lower }} - manage_template => {{ (data_node | bool) | lower }} - index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" - } - } -} diff --git a/elk_metrics_6x/templates/99-kafka-output.conf.j2 b/elk_metrics_6x/templates/99-kafka-output.conf.j2 deleted file mode 100644 index 315b4d2a..00000000 --- a/elk_metrics_6x/templates/99-kafka-output.conf.j2 +++ /dev/null @@ -1,13 +0,0 @@ -output { - kafka { -{% for key, value in logstash_kafka_options.items() %} -{% if value is number %} - {{ key }} => {{ value }} -{% elif value is iterable and value is not string %} - {{ key }} => "{{ value | join(',') }}" -{% else %} - {{ key }} => "{{ value }}" -{% endif %} -{% endfor %} - } -} diff --git a/elk_metrics_6x/templates/_include_setup_template.yml.j2 b/elk_metrics_6x/templates/_include_setup_template.yml.j2 index 2da9661f..93bd7e74 100644 --- a/elk_metrics_6x/templates/_include_setup_template.yml.j2 +++ b/elk_metrics_6x/templates/_include_setup_template.yml.j2 @@ -33,6 +33,9 @@ setup.template.settings: codec: best_compression # This provides for an index split of up to 2 times the number of available shards number_of_routing_shards: {{ (shards | int) * 2 }} + # The default number of replicas will be based on the number of data nodes + # within the environment with a limit of 2 replicas. + number_of_replicas: {{ elasticsearch_number_of_replicas }} # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/elk_metrics_6x/templates/curator-actions.yml.j2 b/elk_metrics_6x/templates/curator-actions.yml.j2 index 097e7fcf..2628bdd2 100644 --- a/elk_metrics_6x/templates/curator-actions.yml.j2 +++ b/elk_metrics_6x/templates/curator-actions.yml.j2 @@ -68,7 +68,7 @@ "exclude_nodes": (groups['kibana'] | map('extract', hostvars, 'ansible_host') | list) }, "number_of_shards": 1, - "number_of_replicas": 1, + "number_of_replicas": elasticsearch_number_of_replicas, "shrink_suffix": '-shrink', "copy_aliases": true, "delete_after": true, diff --git a/elk_metrics_6x/templates/filebeat.yml.j2 b/elk_metrics_6x/templates/filebeat.yml.j2 index 927755d8..9844a6cf 100644 --- a/elk_metrics_6x/templates/filebeat.yml.j2 +++ b/elk_metrics_6x/templates/filebeat.yml.j2 @@ -1310,7 +1310,8 @@ filebeat.prospectors: # The tags of the shipper are included in their own field with each # transaction published. Tags make it easy to group servers by different # logical properties. -#tags: ["service-X", "web-tier"] +tags: + - filebeat # Optional fields that you can specify to add additional information to the # output. Fields can be scalar values, arrays, dictionaries, or any nested diff --git a/elk_metrics_6x/templates/logstash-pipelines.yml.j2 b/elk_metrics_6x/templates/logstash-pipelines.yml.j2 new file mode 100644 index 00000000..7ed5405e --- /dev/null +++ b/elk_metrics_6x/templates/logstash-pipelines.yml.j2 @@ -0,0 +1,739 @@ +# This file is where you define your pipelines. You can define multiple. +# For more information on multiple pipelines, see the documentation: +# https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html + +{% set output_pipeline = ["es_local"] %} +{% if logstash_kafka_options is defined %} +{% set _ = output_pipeline.append('kafka_remote') %} +{% endif %} +{% set output_pipeline = output_pipeline | to_json %} + +- pipeline.id: "beats-intake" + queue.type: persisted + config.string: | + input { + beats { + id => "inputBeats" + port => {{ logstash_beat_input_port }} + } + } + output { + pipeline { + id => "sendDistributorPipeline" + send_to => [distributor] + } + } + +- pipeline.id: "general-distributor" + config.string: | + input { + pipeline { + id => "inputDistribute" + address => distributor + } + } + output { + if "filebeat" in [tags] { + pipeline { + id => "sendFilebeatPipeline" + send_to => [filebeat] + } + } else if "journald" in [tags] { + pipeline { + id => "sendJournalbeatPipeline" + send_to => [journalbeat] + } + } else { + pipeline { + id => "sendOutputPipeline" + send_to => {{ output_pipeline }} + } + } + } + +- pipeline.id: "parse-journalbeat" + path.config: "/etc/logstash/conf.d/02-journald.conf" + config.string: | + input { + pipeline { + id => "inputJournalbeat" + address => journalbeat + } + } + filter { + if [systemd_slice] { + mutate { + copy => { "systemd_slice" => "systemd_slice_tag" } + } + mutate { + gsub => [ "systemd_slice_tag", ".slice", "" ] + } + if [systemd_slice_tag] != "-" { + mutate { + add_tag => [ + "%{systemd_slice_tag}" + ] + } + } + mutate { + remove_field => [ "%{systemd_slice_tag}" ] + } + } + } + output { + pipeline { + id => "sendFilebeat" + send_to => [filebeat] + } + } + +- pipeline.id: "parse-filebeat" + config.string: | + input { + pipeline { + id => "inputFilebeat" + address => filebeat + } + } + filter { + if "Traceback" in [message] { + mutate { + add_tag => ["traceback"] + remove_tag => ["_grokparsefailure"] + } + } + } + output { + if "auth" in [tags] { + pipeline { + id => "sendAuthLog" + send_to => [auth] + } + } else if "elasticsearch" in [tags] { + pipeline { + id => "sendElasticsearch" + send_to => [elasticsearch] + } + } else if "ceph" in [tags] { + pipeline { + id => "sendCeph" + send_to => [ceph] + } + } else if "libvirt" in [tags] { + pipeline { + id => "sendLibvirt" + send_to => [libvirt] + } + } else if "logstash" in [tags] { + pipeline { + id => "sendLogstash" + send_to => [logstash] + } + } else if "mysql" in [tags] { + pipeline { + id => "sendMysql" + send_to => [mysql] + } + } else if "nginx" in [tags] { + pipeline { + id => "sendNginx" + send_to => [nginx] + } + } else if "openstack" in [tags] { + pipeline { + id => "sendOpenstack" + send_to => [openstack] + } + } else if "rabbitmq" in [tags] { + pipeline { + id => "sendRabbitmq" + send_to => [rabbitmq] + } + } else { + pipeline { + id => "sendOutputPipeline" + send_to => {{ output_pipeline }} + } + } + } + +- pipeline.id: "parse-auth" + config.string: | + input { + pipeline { + id => "inputAuthLog" + address => auth + } + } + filter { + grok { + match => { "message" => "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{NOTSPACE:logsource} %{SYSLOGPROG}: (?:%{SPACE})?%{GREEDYDATA:logmessage}" } + } + mutate { + add_field => { "module" => "auth" } + } + } + output { + pipeline { + id => "sendOutputPipeline" + send_to => {{ output_pipeline }} + } + } + +- pipeline.id: "parse-ceph" + config.string: | + input { + pipeline { + id => "inputCeph" + address => ceph + } + } + filter { + grok { + match => { "message" => "%{TIMESTAMP_ISO8601:date} %{NOTSPACE:osd_epoch} ?%{SPACE}?%{NOTSPACE:error_bool} %{GREEDYDATA:logmessage}" } + } + if "ceph-osd" in [tags] { + grok { + match => { "message" => "-- (?(%{IPORHOST}\:%{POSINT}/%{POSINT})) (?:[<|>]){1,2} (?(%{IPORHOST}\:%{POSINT}/%{POSINT}))" } + } + } + } + output { + pipeline { + id => "sendOutputPipeline" + send_to => {{ output_pipeline }} + } + } + +- pipeline.id: "parse-elasticsearch" + config.string: | + input { + pipeline { + id => "inputElasticsearch" + address => elasticsearch + } + } + filter { + grok { + match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{LOGLEVEL:loglevel}\s*\]\[%{NOTSPACE:module}\s*\] %{GREEDYDATA:logmessage}" } + } + mutate { + replace => { "module" => "elasticsearch.%{module}" } + } + } + output { + pipeline { + id => "sendOutputPipeline" + send_to => {{ output_pipeline }} + } + } + +- pipeline.id: "parse-libvirt" + config.string: | + input { + pipeline { + id => "inputLibvirt" + address => libvirt + } + } + filter { + grok { + match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}:%{SPACE}%{NUMBER:code}:?%{SPACE}\[?\b%{NOTSPACE:loglevel}\b\]?%{SPACE}?:?%{SPACE}\[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" } + add_field => { "received_at" => "%{@timestamp}"} + } + mutate { + uppercase => [ "loglevel" ] + } + } + output { + pipeline { + id => "sendOutputPipeline" + send_to => {{ output_pipeline }} + } + } + +- pipeline.id: "parse-logstash" + config.string: | + input { + pipeline { + id => "inputLogstash" + address => logstash + } + } + filter { + grok { + match => { + "message" => "\{\:timestamp=>\"%{TIMESTAMP_ISO8601:timestamp}\", \:message=>\"%{DATA:logmessage}\"(;|)(, \:address=>\"%{URIHOST:address}\", \:exception=>#<\"%{DATA:exception}\">, \:backtrace=>\[%{DATA:backtrace}\]|)(, \:level=>:\"%{LOGLEVEL:loglevel}\"|)\}" + } + } + mutate { + add_field => { "module" => "logstash" } + uppercase => [ "loglevel" ] + } + if [loglevel] == "WARN" { + mutate { + replace => { "loglevel" => "WARNING" } + } + } else if ![loglevel] { + mutate { + add_field => { "loglevel" => "ERROR" } + } + } + } + output { + pipeline { + id => "sendOutputPipeline" + send_to => {{ output_pipeline }} + } + } + +- pipeline.id: "parse-mysql" + path.config: "/etc/logstash/conf.d/10-mysql.conf" + config.string: | + input { + pipeline { + id => "inputMysql" + address => mysql + } + } + filter { + grok { + match => { "message" => "# User@Host: %{WORD:user}\[%{WORD}\] @ (%{HOSTNAME:client_hostname}|) \[(%{IP:client_ip}|)\]" } + } + + grok { + match => { "message" => "# Thread_id: %{NUMBER:thread_id:int} \s*Schema: (%{WORD:schema}| ) \s*QC_hit: %{WORD:qc_hit}" } + } + + grok { + match => { "message" => "# Query_time: %{NUMBER:query_time:float} \s*Lock_time: %{NUMBER:lock_time:float} \s*Rows_sent: %{NUMBER:rows_sent:int} \s*Rows_examined: %{NUMBER:rows_examined:int}" } + } + + grok { + match => { "message" => "(?m)SET timestamp=%{NUMBER:timestamp};%{GREEDYDATA:logmessage}" } + } + + geoip { + source => "clientip" + } + + date { + match => [ "timestamp", "UNIX" ] + } + + mutate { + remove_field => "timestamp" + } + + mutate { + gsub => [ "logmessage", "^\n", "" ] + add_field => { "module" => "mysql" } + add_field => { "loglevel" => "WARNING" } + } + } + output { + pipeline { + id => "sendOutputPipeline" + send_to => {{ output_pipeline }} + } + } + +- pipeline.id: "parse-nginx" + config.string: | + input { + pipeline { + id => "inputNginx" + address => nginx + } + } + filter { + if "nginx-access" in [tags] { + grok { + patterns_dir => ["/opt/logstash/patterns"] + match => { + "message" => "%{IP:client_ip} - %{USER:client_user} \[%{NGINX_TIMESTAMP:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{INT:response_code} %{INT:bytes} %{QUOTEDSTRING:referer} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:gzip_ratio}" + } + } + geoip { + source => "clientip" + } + } + if "nginx-error" in [tags] { + grok { + patterns_dir => ["/opt/logstash/patterns"] + match => { + "message" => "%{NGINX_ERROR_TIMESTAMP:timestamp} \[%{LOGLEVEL:loglevel}\] %{GREEDYDATA:error_msg}" + } + } + } + } + output { + pipeline { + id => "sendOutputPipeline" + send_to => {{ output_pipeline }} + } + } + +- pipeline.id: "parse-openstack" + config.string: | + input { + pipeline { + id => "inputOpenstack" + address => openstack + } + } + filter { + if "oslofmt" in [tags] { + if "Can not find policy directory: policy.d" in [message] { + drop { } + } + grok { + match => { + "message" => [ + "^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?", + "^%{CISCOTIMESTAMP:journalddate}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{SYSLOGPROG:prog}%{SPACE}%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}%{SPACE}%{NOTSPACE:loglevel}%{SPACE}%{NOTSPACE:module}%{SPACE}%{GREEDYDATA:logmessage}" + ] + } + add_field => { "received_at" => "%{@timestamp}" } + } + } + if "nova" in [tags] { + mutate { + gsub => ["logmessage","\"",""] + } + if [module] == "nova.osapi_compute.wsgi.server" { + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes:int} time\: %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + } else if [module] == "nova.api.ec2" { + grok { + match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NUMBER:seconds}s %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} None\:None %{NUMBER:response} %{GREEDYDATA:user_agent}" } + add_tag => ["apimetrics"] + } + } else if [module] == "nova.metadata.wsgi.server" { + grok { + match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes} time\: %{NUMBER:seconds}" } + add_tag => ["apimetrics"] + } + } + } else if "neutron" in [tags] { + if [module] == "neutron.wsgi" { + if "accepted" not in [logmessage] { + mutate { + gsub => ["logmessage","\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + } + } else if "neutron-ha-tool" in [source] { + mutate { + add_tag => ["neutron-ha-tool"] + remove_tag => ["_grokparsefailure"] + } + } + if "starting" in [message] and "_grokparsefailure" in [tags] { + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid}|\-)\](%{SPACE}\(%{NUMBER:pid}\)) %{GREEDYDATA:servicemessage}" } + } + mutate { + remove_tag => ["_grokparsefailure"] + } + } + } else if "glance" in [tags] { + if [module] == "eventlet.wsgi.server" { + mutate { + gsub => ["logmessage","\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + mutate { + replace => { "module" => "glance.%{module}" } + } + } + } else if "cinder" in [tags] { + if [module] == "cinder.eventlet.wsgi.server" { + if "accepted" not in [logmessage] { + mutate { + gsub => ["logmessage","\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + } + mutate { + replace => { "module" => "cinder.%{module}" } + } + } + } else if "horizon" in [tags] { + grok { + patterns_dir => ["/opt/logstash/patterns"] + match => { + "message" => [ + "%{COMMONAPACHELOG}", + "\[%{APACHE_ERROR_TIMESTAMP:timestamp}\] \[%{DATA:module}:%{DATA:loglevel}\] \[pid %{POSINT:apache_pid}\:tid %{POSINT:apache_tid}\] ?(?:\[client %{IP:clientip}:%{POSINT:clientport}\] )?%{GREEDYDATA:logmessage}", + "%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{PROG:prog}%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NUMBER}%{SPACE}%{QS}%{SPACE}%{QS}" + ] + } + } + geoip { + source => "clientip" + } + if ![loglevel] { + mutate { + add_field => { "logmessage" => "%{request}" } + add_field => { "module" => "horizon.access" } + add_field => { "loglevel" => "INFO" } + add_tag => [ "apache-access" ] + } + } else { + mutate { + replace => { "module" => "horizon.error.%{module}" } + add_tag => [ "apache-error" ] + uppercase => [ "loglevel" ] + } + } + } else if "heat" in [tags] { + if [module] == "eventlet.wsgi.server" { + if "accepted" not in [logmessage] { + mutate { + gsub => ["logmessage","\"",""] + } + grok { + match => { "logmessage" => "\[%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} %{NOTSPACE} %{NOTSPACE} %{NOTSPACE}\] %{NOTSPACE:requesterip} %{NOTSPACE} %{NOTSPACE} \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes} %{BASE10NUM:httptime}" } + add_tag => ["apimetrics"] + } + } + mutate { + replace => { "module" => "heat.%{module}" } + } + } else if [module] == "heat.engine.service" { + grok { + match => { "logmessage" => "\[%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} %{NOTSPACE} %{NOTSPACE} %{NOTSPACE} %{GREEDYDATA:servicemessage}" } + add_tag => ["apimetrics"] + } + } + } else if "swift-container" in [tags] { + grok { + match => { + "message" => "%{CISCOTIMESTAMP}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{CISCOTIMESTAMP}%{SPACE}%{HOSTNAME}%{SPACE}%{PROG}%{SPACE}%{USER}%{SPACE}%{USERNAME}%{SPACE}%{NOTSPACE}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{HTTPDUSER}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{INT}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SECOND}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}" + } + } + } else if "swift-account" in [tags] { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP}%{SPACE}%{HOSTNAME}%{SPACE}%{PROG}%{SPACE}%{SYSLOGTIMESTAMP}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{IP}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{POSINT}%{SPACE}%{NOTSPACE}%{SPACE}%{QS}%{SPACE}%{QS}%{SPACE}%{QS}%{SPACE}%{SECOND}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NOTSPACE}" + } + } + } else if "swift" in [tags] { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{NOTSPACE:logsource} %{SYSLOGPROG:module}: (?:%{SPACE})?%{GREEDYDATA:logmessage}" + } + } + grok { + patterns_dir => ["/opt/logstash/patterns"] + match => { + "logmessage" => [ + "%{COMBINEDAPACHELOG}", + "%{SWIFTPROXY_ACCESS}", + "%{GREEDYDATA:logmessage} \(txn\: %{DATA:swift_txn}\)" + ] + } + tag_on_failure => [] + overwrite => [ "logmessage" ] + } + + if [request] { + mutate { + replace => { "logmessage" => "%{request}" } + } + } + + mutate { + replace => { "module" => "swift.%{module}" } + } + + if [file] =~ "error.log$" { + mutate { + add_field => { "loglevel" => "NOTICE" } + } + } else { + mutate { + add_field => { "loglevel" => "INFO" } + } + } + } else if "keystone-access" in [tags] { + grok { + match => { "message" => "%{CISCOTIMESTAMP:keystone_access_timestamp}%{SPACE}%{SYSLOGHOST:log_host}%{SPACE}%{SYSLOGPROG:prog}%{SPACE}%{TIMESTAMP_ISO8601:keystone_timestmp}%{SPACE}%{NUMBER:pid}%{SPACE}%{NOTSPACE:loglevel}%{SPACE}%{NOTSPACE:module}%{SPACE}%{SYSLOG5424SD:requestid}%{SPACE}%{WORD:verb}%{SPACE}%{NOTSPACE:request}" } + } + } else if "keystone" in [tags] { + if "apache-access" in [tags] { + grok { + match => { "message" => "%{COMMONAPACHELOG}" } + } + mutate { + add_field => { "logmessage" => "%{request}" } + add_field => { "module" => "keystone.access" } + add_field => { "loglevel" => "INFO" } + } + } else if "apache-error" in [tags] { + grok { + patterns_dir => ["/opt/logstash/patterns"] + match => { "message" => "%{KEYSTONE_SUBSECOND_TIMESTAMP:keystone_subsecond_timestamp} %{STANDARD_TIMESTAMP:standard_timestamp} %{NUMBER:pid} %{DATA:loglevel} %{DATA:module} \[%{DATA:requestid}\] %{WORD:verb} %{NOTSPACE:request}" } + } + mutate { + replace => { "module" => "keystone.error.%{module}" } + uppercase => [ "loglevel" ] + } + } + } else if "magnum" in [tags] { + if [module] == "eventlet.wsgi.server" { + mutate { + gsub => ["logmessage","\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + mutate { + replace => { "module" => "magnum.%{module}" } + } + } + } else if "octavia" in [tags] { + if [module] == "eventlet.wsgi.server" { + mutate { + gsub => ["logmessage","\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + mutate { + replace => { "module" => "octavia.%{module}" } + } + } + } + } + output { + pipeline { + id => "sendOutputPipeline" + send_to => {{ output_pipeline }} + } + } + +- pipeline.id: "parse-rabbitmq" + config.string: | + input { + pipeline { + id => "inputRabbitmq" + address => rabbitmq + } + } + filter { + if [message] == "" { + drop { } + } + grok { + match => { "message" => "^\=%{LOGLEVEL:loglevel} REPORT\=\=\=\= %{MONTHDAY:event_day}\-%{MONTH:event_month}\-%{YEAR:event_year}\:\:%{TIME:event_time} \=\=\=\n%{GREEDYDATA:logmessage}" } + } + mutate { + replace => { "module" => "rabbitmq" } + add_field => { "timestamp" => "%{event_day} %{event_month} %{event_year} %{event_time}" } + } + date { + match => [ "timestamp", "dd MMM YYYY HH:mm:ss" ] + remove_field => [ "event_day", "event_month", "event_year", "event_time", "timestamp" ] + } + } + output { + pipeline { + id => "sendOutputPipeline" + send_to => {{ output_pipeline }} + } + } + +- pipeline.id: "local-elasticsearch" + config.string: | + input { + pipeline { + id => "inputElasticsearchPipeline" + address => es_local + } + } + filter { + if [source.ip] { + geoip { + id => "setGeoIpSource" + source => "source.ip" + } + } else if [ip] { + geoip { + id => "setGeoIp" + source => "ip" + } + } + if [message] { + fingerprint { + id => "setSHA1" + target => "[@metadata][fingerprint]" + method => "SHA1" + key => "{{ inventory_hostname | to_uuid }}" + } + } else { + fingerprint { + target => "[@metadata][fingerprint]" + method => "UUID" + } + } + } + output { + if [@metadata][version] { + elasticsearch { + id => "elasticsearchOutputPipeline" + document_id => "%{[@metadata][fingerprint]}" + hosts => {{ elasticsearch_data_hosts | shuffle(seed=inventory_hostname) | to_json }} + sniffing => {{ (not data_node | bool) | lower }} + manage_template => {{ (data_node | bool) | lower }} + index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" + } + } else { + elasticsearch { + id => "elasticsearchLegacyOutputPipeline" + document_id => "%{[@metadata][fingerprint]}" + hosts => {{ elasticsearch_data_hosts | shuffle(seed=inventory_hostname) | to_json }} + sniffing => {{ (not data_node | bool) | lower }} + manage_template => {{ (data_node | bool) | lower }} + index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" + } + } + } + +{% if logstash_kafka_options is defined %} +- pipeline.id: "remote-kafka" + config.string: | + input { + pipeline { + id => "inputKafkaPipeline" + address => kafka_remote + } + } + output { + kafka { + {% for key, value in logstash_kafka_options.items() %} + {% if value is number %} + {{ key }} => {{ value }} + {% elif value is iterable and value is not string %} + {{ key }} => "{{ value | join(',') }}" + {% else %} + {{ key }} => "{{ value }}" + {% endif %} + {% endfor %} + } + } +{% endif %} diff --git a/elk_metrics_6x/templates/metricbeat.yml.j2 b/elk_metrics_6x/templates/metricbeat.yml.j2 index 206135c8..a2765e67 100644 --- a/elk_metrics_6x/templates/metricbeat.yml.j2 +++ b/elk_metrics_6x/templates/metricbeat.yml.j2 @@ -46,7 +46,7 @@ metricbeat.max_start_delay: 10s #========================== Modules configuration ============================ metricbeat.modules: - +{% if (not physical_host is defined) or (physical_host is defined and physical_host == inventory_hostname) %} #------------------------------- System Module ------------------------------- - module: system metricsets: @@ -111,7 +111,7 @@ metricbeat.modules: socket.reverse_lookup.enabled: true socket.reverse_lookup.success_ttl: 60s socket.reverse_lookup.failure_ttl: 60s - +{% endif %} ##------------------------------ Aerospike Module ----------------------------- #- module: aerospike # metricsets: ["namespace"]