Merge "Support deploying Monasca Log Persister"

ansible/inventory/all-in-one

@@ -449,6 +449,9 @@ monasca
 [monasca-log-transformer:children]
 monasca
 
+[monasca-log-persister:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/inventory/multinode

@@ -458,6 +458,9 @@ monasca
 [monasca-log-transformer:children]
 monasca
 
+[monasca-log-persister:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/roles/monasca/defaults/main.yml

@@ -27,6 +27,15 @@ monasca_services:
       - "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla"
+  monasca-log-persister:
+    container_name: monasca_log_persister
+    group: monasca-log-persister
+    enabled: true
+    image: "{{ monasca_logstash_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/monasca-log-persister/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
 
 ####################
 # Databases
@@ -45,6 +54,7 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}"
 monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}"
+monasca_elasticsearch_servers: "{% for host in groups['elasticsearch'] %}'{{ internal_protocol }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }}'{% if not loop.last %},{% endif %}{% endfor %}"
 
 monasca_metrics_topic: "metrics"
 monasca_raw_logs_topic: "logs"

ansible/roles/monasca/handlers/main.yml

@@ -62,3 +62,24 @@
     - config_json.changed | bool
       or monasca_log_transformer_confs.changed | bool
       or monasca_log_transformer_container.changed | bool
+
+- name: Restart monasca-log-persister container
+  vars:
+    service_name: "monasca-log-persister"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_log_persister_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+  when:
+    - action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_log_persister_confs.changed | bool
+      or monasca_log_persister_elasticsearch_template.changed | bool
+      or monasca_log_persister_container.changed | bool

ansible/roles/monasca/tasks/config.yml

@@ -118,6 +118,45 @@
   notify:
     - Restart monasca-log-transformer container
 
+- name: Copying over monasca-log-persister config
+  vars:
+    service: "{{ monasca_services['monasca-log-persister'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-log-persister/log-persister.conf"
+    mode: "0660"
+  become: true
+  register: monasca_log_persister_confs
+  with_first_found:
+    - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-persister.conf"
+    - "{{ node_custom_config }}/monasca/log-persister.conf"
+    - "{{ role_path }}/templates/monasca-log-persister/log-persister.conf.j2"
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-log-persister container
+
+- name: Copying over monasca-log-persister elasticsearch template
+  vars:
+    service: "{{ monasca_services['monasca-log-persister'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-log-persister/elasticsearch-template.json"
+    mode: "0660"
+  become: true
+  register: monasca_log_persister_elasticsearch_template
+  with_first_found:
+    - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/elasticsearch-template.json"
+    - "{{ node_custom_config }}/monasca/elasticsearch-template.json"
+    - "{{ role_path }}/templates/monasca-log-persister/elasticsearch-template.json"
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-log-persister container
+
+
 - name: Check monasca containers
   become: true
   kolla_docker:

ansible/roles/monasca/tasks/deploy.yml

@@ -6,7 +6,8 @@
 - include: config.yml
   when: inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api'] or
-        inventory_hostname in groups['monasca-log-transformer']
+        inventory_hostname in groups['monasca-log-transformer'] or
+        inventory_hostname in groups['monasca-log-persister']
 
 - include: bootstrap.yml
   when: inventory_hostname in groups['monasca-api']
@@ -17,4 +18,5 @@
 - include: check.yml
   when: inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api'] or
-        inventory_hostname in groups['monasca-log-transformer']
+        inventory_hostname in groups['monasca-log-transformer'] or
+        inventory_hostname in groups['monasca-log-persister']

ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json

@@ -0,0 +1,56 @@
+{
+  "aliases": {},
+  "mappings": {
+    "log": {
+      "_all": {
+        "enabled": true,
+        "omit_norms": true
+      },
+      "dynamic_templates": [
+        {
+          "message_field": {
+            "mapping": {
+              "fielddata": {
+                "format": "disabled"
+              },
+              "index": "analyzed",
+              "omit_norms": true,
+              "type": "string"
+            },
+            "match": "message",
+            "match_mapping_type": "string"
+          }
+        },
+        {
+          "other_fields": {
+            "mapping": {
+              "index": "not_analyzed",
+              "type": "string"
+            },
+            "match": "*",
+            "match_mapping_type": "string"
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "@version": {
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "creation_time": {
+          "type": "date"
+        }
+      }
+    }
+  },
+  "order": 0,
+  "settings": {
+    "index": {
+      "refresh_interval": "5s"
+    }
+  },
+  "template": "monasca-*"
+}

ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2

@@ -0,0 +1,19 @@
+# Persist transformed logs to Elasticsearch
+
+input {
+    kafka {
+        zk_connect => "{{ monasca_zookeeper_servers }}"
+        topic_id => "{{ monasca_transformed_logs_topic }}"
+        group_id => "transformer-logstash-consumer"
+    }
+}
+
+output {
+    elasticsearch {
+        index => "monasca-%{[meta][tenantId]}-%{+YYYY.MM.dd}"
+        hosts => [{{ monasca_elasticsearch_servers }}]
+        document_type => "log"
+        template_name => "monasca"
+        template => "/etc/logstash/elasticsearch-template.json"
+    }
+}

ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2

@@ -0,0 +1,24 @@
+{
+    "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/monasca/monasca-log-persister.log -f /etc/logstash/conf.d/log-persister.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/log-persister.conf",
+            "dest": "/etc/logstash/conf.d/log-persister.conf",
+            "owner": "logstash",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/elasticsearch-template.json",
+            "dest": "/etc/logstash/elasticsearch-template.json",
+            "owner": "logstash",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/monasca",
+            "owner": "logstash:kolla",
+            "recurse": true
+        }
+    ]
+}

releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Add support for deploying the Monasca Log Persister. The Log
+    Persister is responsible for reading logs from the Kafka processed
+    logs topic and writing them to Elasticsearch.