Merge "Support deploying Monasca Log Transformer"

ansible/inventory/all-in-one

@@ -446,6 +446,9 @@ monasca
 [monasca-log-api:children]
 monasca
 
+[monasca-log-transformer:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/inventory/multinode

@@ -455,6 +455,9 @@ monasca
 [monasca-log-api:children]
 monasca
 
+[monasca-log-transformer:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/roles/monasca/defaults/main.yml

@@ -18,6 +18,15 @@ monasca_services:
       - "{{ node_config_directory }}/monasca-log-api/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla"
+  monasca-log-transformer:
+    container_name: monasca_log_transformer
+    group: monasca-log-transformer
+    enabled: true
+    image: "{{ monasca_logstash_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
 
 ####################
 # Databases
@@ -34,10 +43,12 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}"
 # Monasca
 ####################
 monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}"
+monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 
 monasca_metrics_topic: "metrics"
 monasca_raw_logs_topic: "logs"
+monasca_transformed_logs_topic: "transformed-logs"
 
 ####################
 # Docker
@@ -53,6 +64,10 @@ monasca_log_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{
 monasca_log_api_tag: "{{ monasca_tag }}"
 monasca_log_api_image_full: "{{ monasca_log_api_image }}:{{ monasca_log_api_tag }}"
 
+monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-logstash"
+monasca_logstash_tag: "{{ monasca_tag }}"
+monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}"
+
 ####################
 # OpenStack
 ####################

ansible/roles/monasca/handlers/main.yml

@@ -42,3 +42,23 @@
       or monasca_log_api_confs.changed | bool
       or monasca_log_api_wsgi_confs.changed | bool
       or monasca_log_api_container.changed | bool
+
+- name: Restart monasca-log-transformer container
+  vars:
+    service_name: "monasca-log-transformer"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_log_transformer_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+  when:
+    - action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_log_transformer_confs.changed | bool
+      or monasca_log_transformer_container.changed | bool

ansible/roles/monasca/tasks/config.yml

@@ -25,8 +25,7 @@
     - item.value.enabled | bool
   with_dict: "{{ monasca_services }}"
   notify:
-    - Restart monasca-api container
-    - Restart monasca-log-api container
+    - "Restart {{ item.key }} container"
 
 - name: Copying over monasca-api config
   vars:
@@ -100,6 +99,25 @@
   notify:
     - Restart monasca-log-api container
 
+- name: Copying over monasca-log-transformer config
+  vars:
+    service: "{{ monasca_services['monasca-log-transformer'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-log-transformer/log-transformer.conf"
+    mode: "0660"
+  become: true
+  register: monasca_log_transformer_confs
+  with_first_found:
+    - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-transformer.conf"
+    - "{{ node_custom_config }}/monasca/log-transformer.conf"
+    - "{{ role_path }}/templates/monasca-log-transformer/log-transformer.conf.j2"
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-log-transformer container
+
 - name: Check monasca containers
   become: true
   kolla_docker:
@@ -115,5 +133,4 @@
     - item.value.enabled | bool
   with_dict: "{{ monasca_services }}"
   notify:
-    - Restart monasca-api container
-    - Restart monasca-log-api container
+    - "Restart {{ item.key }} container"

ansible/roles/monasca/tasks/deploy.yml

@@ -5,7 +5,8 @@
 
 - include: config.yml
   when: inventory_hostname in groups['monasca-api'] or
-        inventory_hostname in groups['monasca-log-api']
+        inventory_hostname in groups['monasca-log-api'] or
+        inventory_hostname in groups['monasca-log-transformer']
 
 - include: bootstrap.yml
   when: inventory_hostname in groups['monasca-api']
@@ -15,4 +16,5 @@
 
 - include: check.yml
   when: inventory_hostname in groups['monasca-api'] or
-        inventory_hostname in groups['monasca-log-api']
+        inventory_hostname in groups['monasca-log-api'] or
+        inventory_hostname in groups['monasca-log-transformer']

ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2

@@ -0,0 +1,25 @@
+# Provide input/output streams for transforming Monasca logs.
+# Filters should be provided in other configuration files.
+
+input {
+    kafka {
+        zk_connect => "{{ monasca_zookeeper_servers }}"
+        topic_id => "{{ monasca_raw_logs_topic }}"
+        group_id => "transformer-logstash-consumer"
+    }
+}
+
+filter {
+    # Update the timestamp of the event based on the time in the message.
+    date {
+        match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss +0000", "ISO8601"]
+        remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ]
+    }
+}
+
+output {
+    kafka {
+        bootstrap_servers => "{{ monasca_kafka_servers }}"
+        topic_id => "{{ monasca_transformed_logs_topic }}"
+    }
+}

ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/log-transformer.conf",
+            "dest": "/etc/logstash/conf.d/log-transformer.conf",
+            "owner": "logstash",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/logstash",
+            "owner": "logstash:kolla",
+            "recurse": true
+        }
+    ]
+}

releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add support for deploying the Monasca Log Transformer for
+    providing log standardisation in Monasca.