openstack/kolla-ansible
Code Issues Proposed changes

Support deploying Monasca Log Metrics

Browse Source 
The log metrics service generates metrics from log messages
which allows further analysis and alerting to be performed
on them. Basic configuration is provided so that metrics
are generated for high level warning logs such as error, or
warning.

Change-Id: I45cc17817c716296451f620f304c0b1108162a56
Partially-Implements: blueprint monasca-roles
This commit is contained in:
Doug Szumski 2018-06-08 10:10:56 +01:00
parent 01da938412
commit 1ae10375f7
9 changed files with 159 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ansible/inventory/all-in-one
View File

@ -455,6 +455,9 @@ monasca
[monasca-log-persister:children]
monasca
[monasca-log-metrics:children]
monasca
# Ironic
[ironic-api:children]
ironic

3
ansible/inventory/multinode
View File

@ -464,6 +464,9 @@ monasca
[monasca-log-persister:children]
monasca
[monasca-log-metrics:children]
monasca
# Ironic
[ironic-api:children]
ironic

11
ansible/roles/monasca/defaults/main.yml
View File

@ -40,6 +40,16 @@ monasca_services:
- "/etc/localtime:/etc/localtime:ro"
- "kolla_logs:/var/log/kolla"
dimensions: "{{ monasca_log_persister_dimensions }}"
monasca-log-metrics:
container_name: monasca_log_metrics
group: monasca-log-metrics
enabled: true
image: "{{ monasca_logstash_image_full }}"
volumes:
- "{{ node_config_directory }}/monasca-log-metrics/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "kolla_logs:/var/log/kolla"
dimensions: "{{ monasca_log_metrics_dimensions }}"
####################
# Databases
@ -94,6 +104,7 @@ monasca_api_dimensions: "{{ default_container_dimensions }}"
monasca_log_api_dimensions: "{{ default_container_dimensions }}"
monasca_log_transformer_dimensions: "{{ default_container_dimensions }}"
monasca_log_persister_dimensions: "{{ default_container_dimensions }}"
monasca_log_metrics_dimensions: "{{ default_container_dimensions }}"
####################

21
ansible/roles/monasca/handlers/main.yml
View File

@ -87,3 +87,24 @@
or monasca_log_persister_confs.changed | bool
or monasca_log_persister_elasticsearch_template.changed | bool
or monasca_log_persister_container.changed | bool
- name: Restart monasca-log-metrics container
vars:
service_name: "monasca-log-metrics"
service: "{{ monasca_services[service_name] }}"
config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
monasca_log_metrics_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
volumes: "{{ service.volumes }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- inventory_hostname in groups[service.group]
- service.enabled | bool
- config_json.changed | bool
or monasca_log_metrics_confs.changed | bool
or monasca_log_metrics_container.changed | bool

18
ansible/roles/monasca/tasks/config.yml
View File

@ -156,6 +156,24 @@
notify:
- Restart monasca-log-persister container
- name: Copying over monasca-log-metrics config
vars:
service: "{{ monasca_services['monasca-log-metrics'] }}"
template:
src: "{{ item }}"
dest: "{{ node_config_directory }}/monasca-log-metrics/log-metrics.conf"
mode: "0660"
become: true
register: monasca_log_metrics_confs
with_first_found:
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-metrics.conf"
- "{{ node_custom_config }}/monasca/log-metrics.conf"
- "{{ role_path }}/templates/monasca-log-metrics/log-metrics.conf.j2"
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-log-metrics container
- name: Check monasca containers
become: true

6
ansible/roles/monasca/tasks/deploy.yml
View File

@ -7,7 +7,8 @@
when: inventory_hostname in groups['monasca-api'] or
inventory_hostname in groups['monasca-log-api'] or
inventory_hostname in groups['monasca-log-transformer'] or
inventory_hostname in groups['monasca-log-persister']
inventory_hostname in groups['monasca-log-persister'] or
inventory_hostname in groups['monasca-log-metrics']
- include_tasks: bootstrap.yml
when: inventory_hostname in groups['monasca-api']
@ -19,4 +20,5 @@
when: inventory_hostname in groups['monasca-api'] or
inventory_hostname in groups['monasca-log-api'] or
inventory_hostname in groups['monasca-log-transformer'] or
inventory_hostname in groups['monasca-log-persister']
inventory_hostname in groups['monasca-log-persister'] or
inventory_hostname in groups['monasca-log-metrics']

76
ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 Normal file
View File

@ -0,0 +1,76 @@
# This config file is used to generate Monasca metrics from logs ingested
# by the Monasca Log API. Alarms and notifications can then be set to alert
# users when a particular log message is ingested. This example config file
# generates metrics for all logs which have a log level which isn't
# debug, trace or info level. A user may want to apply additional logic to
# generate special metrics when particular log messages are ingested. For
# example, if HAProxy fails over, file system corruption is detected or
# other scenarios of interest.
input {
kafka {
zk_connect => "{{ monasca_zookeeper_servers }}"
topic_id => "{{ monasca_transformed_logs_topic }}"
group_id => "log_metrics"
consumer_id => "log_metrics_{{ ansible_hostname }}"
consumer_threads => "{{ monasca_log_pipeline_threads }}"
}
}
filter {
# Drop everything we don't want to create metrics for.
if ![log][dimensions][log_level] or [log][dimensions][log_level] in [ "debug", "trace", "info" ] {
drop {
}
}
# Generate a metric name based on the program and log level
mutate {
add_field => { "[metric][name]" => "log.%{[log][dimensions][programname]}.%{[log][dimensions][log_level]}" }
}
# Form the metric structure.
mutate {
add_field => { "[metric][value]" => 1 }
rename => { "[log][dimensions]" => "[metric][dimensions]" }
rename => { "[metric][dimensions][Hostname]" => "[metric][dimensions][hostname]" }
rename => { "[metric][dimensions][programname]" => "[metric][dimensions][service]" }
}
mutate {
convert => { "[metric][value]" => "float" }
}
# Convert the timestamp of the event to milliseconds since epoch.
ruby {
code => "event['metric']['timestamp'] = event['@timestamp'].to_i * 1000"
}
# Clean up any fields which aren't required from the new metric to save space
mutate {
remove_field => ["[metric][dimensions][log_level]",
"[metric][dimensions][domain_id]",
"[metric][dimensions][user_id]",
"[metric][dimensions][tenant_id]",
"[metric][dimensions][project_domain]",
"[metric][dimensions][tag]",
"[metric][dimensions][Logger]",
"[metric][dimensions][Pid]",
"[metric][dimensions][user_domain]",
"[metric][dimensions][request_id]",
"[metric][dimensions][python_module]",
"[metric][meta]",
"creation_time",
"log",
"@version",
"@timestamp"]
}
}
output {
kafka {
bootstrap_servers => "{{ monasca_kafka_servers }}"
topic_id => "{{ monasca_metrics_topic }}"
client_id => "log_metrics_{{ ansible_hostname }}"
workers => "{{ monasca_log_pipeline_threads }}"
}
}

18
ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 Normal file
View File

@ -0,0 +1,18 @@
{
"command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-metrics.log -f /etc/logstash/conf.d/log-metrics.conf",
"config_files": [
{
"source": "{{ container_config_directory }}/log-metrics.conf",
"dest": "/etc/logstash/conf.d/log-metrics.conf",
"owner": "logstash",
"perm": "0600"
}
],
"permissions": [
{
"path": "/var/log/kolla/logstash",
"owner": "logstash:kolla",
"recurse": true
}
]
}

5
releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- |
Add the Monasca Log Metrics service. This service is responsible for
generating metrics from log files.