diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index c6907097ab..25dd8dc1a4 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -455,6 +455,9 @@ monasca [monasca-log-persister:children] monasca +[monasca-log-metrics:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index b796c2986a..c64e74cdab 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -464,6 +464,9 @@ monasca [monasca-log-persister:children] monasca +[monasca-log-metrics:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index 55c2f125dc..309d545dd3 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -40,6 +40,16 @@ monasca_services: - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla" dimensions: "{{ monasca_log_persister_dimensions }}" + monasca-log-metrics: + container_name: monasca_log_metrics + group: monasca-log-metrics + enabled: true + image: "{{ monasca_logstash_image_full }}" + volumes: + - "{{ node_config_directory }}/monasca-log-metrics/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla" + dimensions: "{{ monasca_log_metrics_dimensions }}" #################### # Databases @@ -94,6 +104,7 @@ monasca_api_dimensions: "{{ default_container_dimensions }}" monasca_log_api_dimensions: "{{ default_container_dimensions }}" monasca_log_transformer_dimensions: "{{ default_container_dimensions }}" monasca_log_persister_dimensions: "{{ default_container_dimensions }}" +monasca_log_metrics_dimensions: "{{ default_container_dimensions }}" #################### diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index d20d442741..e47955b1e0 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -87,3 +87,24 @@ or monasca_log_persister_confs.changed | bool or monasca_log_persister_elasticsearch_template.changed | bool or monasca_log_persister_container.changed | bool + +- name: Restart monasca-log-metrics container + vars: + service_name: "monasca-log-metrics" + service: "{{ monasca_services[service_name] }}" + config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + monasca_log_metrics_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or monasca_log_metrics_confs.changed | bool + or monasca_log_metrics_container.changed | bool diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index 454452270e..4c07bd31b7 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -156,6 +156,24 @@ notify: - Restart monasca-log-persister container +- name: Copying over monasca-log-metrics config + vars: + service: "{{ monasca_services['monasca-log-metrics'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-log-metrics/log-metrics.conf" + mode: "0660" + become: true + register: monasca_log_metrics_confs + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-metrics.conf" + - "{{ node_custom_config }}/monasca/log-metrics.conf" + - "{{ role_path }}/templates/monasca-log-metrics/log-metrics.conf.j2" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-log-metrics container - name: Check monasca containers become: true diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml index 15dbbbdecc..aa233dc2c5 100644 --- a/ansible/roles/monasca/tasks/deploy.yml +++ b/ansible/roles/monasca/tasks/deploy.yml @@ -7,7 +7,8 @@ when: inventory_hostname in groups['monasca-api'] or inventory_hostname in groups['monasca-log-api'] or inventory_hostname in groups['monasca-log-transformer'] or - inventory_hostname in groups['monasca-log-persister'] + inventory_hostname in groups['monasca-log-persister'] or + inventory_hostname in groups['monasca-log-metrics'] - include_tasks: bootstrap.yml when: inventory_hostname in groups['monasca-api'] @@ -19,4 +20,5 @@ when: inventory_hostname in groups['monasca-api'] or inventory_hostname in groups['monasca-log-api'] or inventory_hostname in groups['monasca-log-transformer'] or - inventory_hostname in groups['monasca-log-persister'] + inventory_hostname in groups['monasca-log-persister'] or + inventory_hostname in groups['monasca-log-metrics'] diff --git a/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 new file mode 100644 index 0000000000..805164541e --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 @@ -0,0 +1,76 @@ +# This config file is used to generate Monasca metrics from logs ingested +# by the Monasca Log API. Alarms and notifications can then be set to alert +# users when a particular log message is ingested. This example config file +# generates metrics for all logs which have a log level which isn't +# debug, trace or info level. A user may want to apply additional logic to +# generate special metrics when particular log messages are ingested. For +# example, if HAProxy fails over, file system corruption is detected or +# other scenarios of interest. + +input { + kafka { + zk_connect => "{{ monasca_zookeeper_servers }}" + topic_id => "{{ monasca_transformed_logs_topic }}" + group_id => "log_metrics" + consumer_id => "log_metrics_{{ ansible_hostname }}" + consumer_threads => "{{ monasca_log_pipeline_threads }}" + } +} + +filter { + # Drop everything we don't want to create metrics for. + if ![log][dimensions][log_level] or [log][dimensions][log_level] in [ "debug", "trace", "info" ] { + drop { + } + } + + # Generate a metric name based on the program and log level + mutate { + add_field => { "[metric][name]" => "log.%{[log][dimensions][programname]}.%{[log][dimensions][log_level]}" } + } + + # Form the metric structure. + mutate { + add_field => { "[metric][value]" => 1 } + rename => { "[log][dimensions]" => "[metric][dimensions]" } + rename => { "[metric][dimensions][Hostname]" => "[metric][dimensions][hostname]" } + rename => { "[metric][dimensions][programname]" => "[metric][dimensions][service]" } + } + mutate { + convert => { "[metric][value]" => "float" } + } + + # Convert the timestamp of the event to milliseconds since epoch. + ruby { + code => "event['metric']['timestamp'] = event['@timestamp'].to_i * 1000" + } + + # Clean up any fields which aren't required from the new metric to save space + mutate { + remove_field => ["[metric][dimensions][log_level]", + "[metric][dimensions][domain_id]", + "[metric][dimensions][user_id]", + "[metric][dimensions][tenant_id]", + "[metric][dimensions][project_domain]", + "[metric][dimensions][tag]", + "[metric][dimensions][Logger]", + "[metric][dimensions][Pid]", + "[metric][dimensions][user_domain]", + "[metric][dimensions][request_id]", + "[metric][dimensions][python_module]", + "[metric][meta]", + "creation_time", + "log", + "@version", + "@timestamp"] + } +} + +output { + kafka { + bootstrap_servers => "{{ monasca_kafka_servers }}" + topic_id => "{{ monasca_metrics_topic }}" + client_id => "log_metrics_{{ ansible_hostname }}" + workers => "{{ monasca_log_pipeline_threads }}" + } +} diff --git a/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 b/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 new file mode 100644 index 0000000000..287ffcd975 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 @@ -0,0 +1,18 @@ +{ + "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-metrics.log -f /etc/logstash/conf.d/log-metrics.conf", + "config_files": [ + { + "source": "{{ container_config_directory }}/log-metrics.conf", + "dest": "/etc/logstash/conf.d/log-metrics.conf", + "owner": "logstash", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/logstash", + "owner": "logstash:kolla", + "recurse": true + } + ] +} diff --git a/releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml b/releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml new file mode 100644 index 0000000000..8abda7ab34 --- /dev/null +++ b/releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add the Monasca Log Metrics service. This service is responsible for + generating metrics from log files.