From 5441963c9a25257e67d63762d993aaf07d9a1b4c Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Thu, 7 Jun 2018 15:29:24 +0100 Subject: [PATCH] Support deploying Monasca Log Persister This is a Logstash component which reads processed logs from Kafka and writes them to Elasticsearch (or some other backend supported by Logstash). Ingesting the logs from this service with Fluentd will be covered under a different commit. Change-Id: I2d722991ab2072c54c4715507b19a4c9279f921b Partially-Implements: blueprint monasca-roles --- ansible/inventory/all-in-one | 3 + ansible/inventory/multinode | 3 + ansible/roles/monasca/defaults/main.yml | 10 ++++ ansible/roles/monasca/handlers/main.yml | 21 +++++++ ansible/roles/monasca/tasks/config.yml | 39 +++++++++++++ ansible/roles/monasca/tasks/deploy.yml | 6 +- .../elasticsearch-template.json | 56 +++++++++++++++++++ .../log-persister.conf.j2 | 19 +++++++ .../monasca-log-persister.json.j2 | 24 ++++++++ ...onasca-log-persister-f4da4370a0c5777e.yaml | 6 ++ 10 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json create mode 100644 ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 create mode 100644 ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 create mode 100644 releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index ee1da6efe5..12c845f67d 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -449,6 +449,9 @@ monasca [monasca-log-transformer:children] monasca +[monasca-log-persister:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 1caccd0709..c4c9848654 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -458,6 +458,9 @@ monasca [monasca-log-transformer:children] monasca +[monasca-log-persister:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index 4db2feb1c6..c8dc213438 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -27,6 +27,15 @@ monasca_services: - "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla" + monasca-log-persister: + container_name: monasca_log_persister + group: monasca-log-persister + enabled: true + image: "{{ monasca_logstash_image_full }}" + volumes: + - "{{ node_config_directory }}/monasca-log-persister/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla" #################### # Databases @@ -45,6 +54,7 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}" monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}" +monasca_elasticsearch_servers: "{% for host in groups['elasticsearch'] %}'{{ internal_protocol }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }}'{% if not loop.last %},{% endif %}{% endfor %}" monasca_metrics_topic: "metrics" monasca_raw_logs_topic: "logs" diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index 6caadb6df0..c9db3faf06 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -62,3 +62,24 @@ - config_json.changed | bool or monasca_log_transformer_confs.changed | bool or monasca_log_transformer_container.changed | bool + +- name: Restart monasca-log-persister container + vars: + service_name: "monasca-log-persister" + service: "{{ monasca_services[service_name] }}" + config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + monasca_log_persister_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or monasca_log_persister_confs.changed | bool + or monasca_log_persister_elasticsearch_template.changed | bool + or monasca_log_persister_container.changed | bool diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index 42c40f38dc..66217b5124 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -118,6 +118,45 @@ notify: - Restart monasca-log-transformer container +- name: Copying over monasca-log-persister config + vars: + service: "{{ monasca_services['monasca-log-persister'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-log-persister/log-persister.conf" + mode: "0660" + become: true + register: monasca_log_persister_confs + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-persister.conf" + - "{{ node_custom_config }}/monasca/log-persister.conf" + - "{{ role_path }}/templates/monasca-log-persister/log-persister.conf.j2" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-log-persister container + +- name: Copying over monasca-log-persister elasticsearch template + vars: + service: "{{ monasca_services['monasca-log-persister'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-log-persister/elasticsearch-template.json" + mode: "0660" + become: true + register: monasca_log_persister_elasticsearch_template + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/elasticsearch-template.json" + - "{{ node_custom_config }}/monasca/elasticsearch-template.json" + - "{{ role_path }}/templates/monasca-log-persister/elasticsearch-template.json" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-log-persister container + + - name: Check monasca containers become: true kolla_docker: diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml index b386d15e3f..d470d08cfa 100644 --- a/ansible/roles/monasca/tasks/deploy.yml +++ b/ansible/roles/monasca/tasks/deploy.yml @@ -6,7 +6,8 @@ - include: config.yml when: inventory_hostname in groups['monasca-api'] or inventory_hostname in groups['monasca-log-api'] or - inventory_hostname in groups['monasca-log-transformer'] + inventory_hostname in groups['monasca-log-transformer'] or + inventory_hostname in groups['monasca-log-persister'] - include: bootstrap.yml when: inventory_hostname in groups['monasca-api'] @@ -17,4 +18,5 @@ - include: check.yml when: inventory_hostname in groups['monasca-api'] or inventory_hostname in groups['monasca-log-api'] or - inventory_hostname in groups['monasca-log-transformer'] + inventory_hostname in groups['monasca-log-transformer'] or + inventory_hostname in groups['monasca-log-persister'] diff --git a/ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json b/ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json new file mode 100644 index 0000000000..c882454cb7 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json @@ -0,0 +1,56 @@ +{ + "aliases": {}, + "mappings": { + "log": { + "_all": { + "enabled": true, + "omit_norms": true + }, + "dynamic_templates": [ + { + "message_field": { + "mapping": { + "fielddata": { + "format": "disabled" + }, + "index": "analyzed", + "omit_norms": true, + "type": "string" + }, + "match": "message", + "match_mapping_type": "string" + } + }, + { + "other_fields": { + "mapping": { + "index": "not_analyzed", + "type": "string" + }, + "match": "*", + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "@version": { + "index": "not_analyzed", + "type": "string" + }, + "creation_time": { + "type": "date" + } + } + } + }, + "order": 0, + "settings": { + "index": { + "refresh_interval": "5s" + } + }, + "template": "monasca-*" +} diff --git a/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 new file mode 100644 index 0000000000..31b08d3c5f --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 @@ -0,0 +1,19 @@ +# Persist transformed logs to Elasticsearch + +input { + kafka { + zk_connect => "{{ monasca_zookeeper_servers }}" + topic_id => "{{ monasca_transformed_logs_topic }}" + group_id => "transformer-logstash-consumer" + } +} + +output { + elasticsearch { + index => "monasca-%{[meta][tenantId]}-%{+YYYY.MM.dd}" + hosts => [{{ monasca_elasticsearch_servers }}] + document_type => "log" + template_name => "monasca" + template => "/etc/logstash/elasticsearch-template.json" + } +} diff --git a/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 b/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 new file mode 100644 index 0000000000..3eb19fa42d --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 @@ -0,0 +1,24 @@ +{ + "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/monasca/monasca-log-persister.log -f /etc/logstash/conf.d/log-persister.conf", + "config_files": [ + { + "source": "{{ container_config_directory }}/log-persister.conf", + "dest": "/etc/logstash/conf.d/log-persister.conf", + "owner": "logstash", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/elasticsearch-template.json", + "dest": "/etc/logstash/elasticsearch-template.json", + "owner": "logstash", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/monasca", + "owner": "logstash:kolla", + "recurse": true + } + ] +} diff --git a/releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml b/releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml new file mode 100644 index 0000000000..cc0a4a5c5b --- /dev/null +++ b/releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add support for deploying the Monasca Log Persister. The Log + Persister is responsible for reading logs from the Kafka processed + logs topic and writing them to Elasticsearch.