Merge "Support deploying Monasca Log Metrics"

ansible/inventory/all-in-one

@@ -454,6 +454,9 @@ monasca
 [monasca-log-persister:children]
 monasca
 
+[monasca-log-metrics:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/inventory/multinode

@@ -463,6 +463,9 @@ monasca
 [monasca-log-persister:children]
 monasca
 
+[monasca-log-metrics:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/roles/monasca/defaults/main.yml

@@ -40,6 +40,16 @@ monasca_services:
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla"
     dimensions: "{{ monasca_log_persister_dimensions }}"
+  monasca-log-metrics:
+    container_name: monasca_log_metrics
+    group: monasca-log-metrics
+    enabled: true
+    image: "{{ monasca_logstash_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/monasca-log-metrics/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
+    dimensions: "{{ monasca_log_metrics_dimensions }}"
 
 ####################
 # Databases
@@ -94,6 +104,7 @@ monasca_api_dimensions: "{{ default_container_dimensions }}"
 monasca_log_api_dimensions: "{{ default_container_dimensions }}"
 monasca_log_transformer_dimensions: "{{ default_container_dimensions }}"
 monasca_log_persister_dimensions: "{{ default_container_dimensions }}"
+monasca_log_metrics_dimensions: "{{ default_container_dimensions }}"
 
 
 ####################

ansible/roles/monasca/handlers/main.yml

@@ -87,3 +87,24 @@
       or monasca_log_persister_confs.changed | bool
       or monasca_log_persister_elasticsearch_template.changed | bool
       or monasca_log_persister_container.changed | bool
+
+- name: Restart monasca-log-metrics container
+  vars:
+    service_name: "monasca-log-metrics"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_log_metrics_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_log_metrics_confs.changed | bool
+      or monasca_log_metrics_container.changed | bool

ansible/roles/monasca/tasks/config.yml

@@ -156,6 +156,24 @@
   notify:
     - Restart monasca-log-persister container
 
+- name: Copying over monasca-log-metrics config
+  vars:
+    service: "{{ monasca_services['monasca-log-metrics'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-log-metrics/log-metrics.conf"
+    mode: "0660"
+  become: true
+  register: monasca_log_metrics_confs
+  with_first_found:
+    - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-metrics.conf"
+    - "{{ node_custom_config }}/monasca/log-metrics.conf"
+    - "{{ role_path }}/templates/monasca-log-metrics/log-metrics.conf.j2"
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-log-metrics container
 
 - name: Check monasca containers
   become: true

ansible/roles/monasca/tasks/deploy.yml

@@ -7,7 +7,8 @@
   when: inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api'] or
         inventory_hostname in groups['monasca-log-transformer'] or
-        inventory_hostname in groups['monasca-log-persister']
+        inventory_hostname in groups['monasca-log-persister'] or
+        inventory_hostname in groups['monasca-log-metrics']
 
 - include_tasks: bootstrap.yml
   when: inventory_hostname in groups['monasca-api']
@@ -19,4 +20,5 @@
   when: inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api'] or
         inventory_hostname in groups['monasca-log-transformer'] or
-        inventory_hostname in groups['monasca-log-persister']
+        inventory_hostname in groups['monasca-log-persister'] or
+        inventory_hostname in groups['monasca-log-metrics']

ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2

@@ -0,0 +1,76 @@
+# This config file is used to generate Monasca metrics from logs ingested
+# by the Monasca Log API. Alarms and notifications can then be set to alert
+# users when a particular log message is ingested. This example config file
+# generates metrics for all logs which have a log level which isn't
+# debug, trace or info level. A user may want to apply additional logic to
+# generate special metrics when particular log messages are ingested. For
+# example, if HAProxy fails over, file system corruption is detected or
+# other scenarios of interest.
+
+input {
+    kafka {
+        zk_connect => "{{ monasca_zookeeper_servers }}"
+        topic_id => "{{ monasca_transformed_logs_topic }}"
+        group_id => "log_metrics"
+        consumer_id => "log_metrics_{{ ansible_hostname }}"
+        consumer_threads => "{{ monasca_log_pipeline_threads }}"
+    }
+}
+
+filter {
+    # Drop everything we don't want to create metrics for.
+    if ![log][dimensions][log_level] or [log][dimensions][log_level] in [ "debug", "trace", "info" ] {
+        drop {
+        }
+    }
+
+    # Generate a metric name based on the program and log level
+    mutate {
+        add_field => { "[metric][name]" => "log.%{[log][dimensions][programname]}.%{[log][dimensions][log_level]}" }
+    }
+
+    # Form the metric structure.
+    mutate {
+        add_field => { "[metric][value]" => 1 }
+        rename => { "[log][dimensions]" => "[metric][dimensions]" }
+        rename => { "[metric][dimensions][Hostname]" => "[metric][dimensions][hostname]" }
+        rename => { "[metric][dimensions][programname]" => "[metric][dimensions][service]" }
+    }
+    mutate {
+         convert => { "[metric][value]" => "float" }
+    }
+
+    # Convert the timestamp of the event to milliseconds since epoch.
+    ruby {
+        code => "event['metric']['timestamp'] = event['@timestamp'].to_i * 1000"
+    }
+
+    # Clean up any fields which aren't required from the new metric to save space
+    mutate {
+        remove_field => ["[metric][dimensions][log_level]",
+                         "[metric][dimensions][domain_id]",
+                         "[metric][dimensions][user_id]",
+                         "[metric][dimensions][tenant_id]",
+                         "[metric][dimensions][project_domain]",
+                         "[metric][dimensions][tag]",
+                         "[metric][dimensions][Logger]",
+                         "[metric][dimensions][Pid]",
+                         "[metric][dimensions][user_domain]",
+                         "[metric][dimensions][request_id]",
+                         "[metric][dimensions][python_module]",
+                         "[metric][meta]",
+                         "creation_time",
+                         "log",
+                         "@version",
+                         "@timestamp"]
+    }
+}
+
+output {
+    kafka {
+        bootstrap_servers => "{{ monasca_kafka_servers }}"
+        topic_id => "{{ monasca_metrics_topic }}"
+        client_id => "log_metrics_{{ ansible_hostname }}"
+        workers => "{{ monasca_log_pipeline_threads }}"
+    }
+}

ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-metrics.log -f /etc/logstash/conf.d/log-metrics.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/log-metrics.conf",
+            "dest": "/etc/logstash/conf.d/log-metrics.conf",
+            "owner": "logstash",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/logstash",
+            "owner": "logstash:kolla",
+            "recurse": true
+        }
+    ]
+}

releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add the Monasca Log Metrics service. This service is responsible for
+    generating metrics from log files.