From 9c88262ad93f224c2fd057373a491362f7176ba0 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Mon, 4 Jun 2018 15:34:38 +0100 Subject: [PATCH] Support deploying Monasca Log Transformer The Monasca Log Transformer takes raw, unstandardised logs from one Kafka topic, standardises them with whatever rules the operator wants to use, and then writes them to a standardised logs topic in Kafka. It is currently implemented as a Logstash config file. Since Kolla does a fairly good job of standardising logs, this service does very little processing. However, when other sources of logs are used, it may be useful to add rules to the Transformer, particularly if it's not possible to standardise the logs at source. Ingesting the logs from this service with Fluentd will be covered under a different commit. Change-Id: I31cbb7e9a40a848391f517a56a67e3fd5bc12529 Partially-Implements: blueprint monasca-roles --- ansible/inventory/all-in-one | 3 +++ ansible/inventory/multinode | 3 +++ ansible/roles/monasca/defaults/main.yml | 15 +++++++++++ ansible/roles/monasca/handlers/main.yml | 20 +++++++++++++++ ansible/roles/monasca/tasks/config.yml | 25 ++++++++++++++++--- ansible/roles/monasca/tasks/deploy.yml | 6 +++-- .../log-transformer.conf.j2 | 25 +++++++++++++++++++ .../monasca-log-transformer.json.j2 | 18 +++++++++++++ ...asca-log-transformer-80d985fc77603478.yaml | 5 ++++ 9 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 create mode 100644 ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 create mode 100644 releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 0eb6b6641d..845b5c03ad 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -446,6 +446,9 @@ monasca [monasca-log-api:children] monasca +[monasca-log-transformer:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 1a223722c5..1466025b75 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -455,6 +455,9 @@ monasca [monasca-log-api:children] monasca +[monasca-log-transformer:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index b5fc75a695..4db2feb1c6 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -18,6 +18,15 @@ monasca_services: - "{{ node_config_directory }}/monasca-log-api/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla" + monasca-log-transformer: + container_name: monasca_log_transformer + group: monasca-log-transformer + enabled: true + image: "{{ monasca_logstash_image_full }}" + volumes: + - "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla" #################### # Databases @@ -34,10 +43,12 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}" # Monasca #################### monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}" +monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_metrics_topic: "metrics" monasca_raw_logs_topic: "logs" +monasca_transformed_logs_topic: "transformed-logs" #################### # Docker @@ -53,6 +64,10 @@ monasca_log_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ monasca_log_api_tag: "{{ monasca_tag }}" monasca_log_api_image_full: "{{ monasca_log_api_image }}:{{ monasca_log_api_tag }}" +monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-logstash" +monasca_logstash_tag: "{{ monasca_tag }}" +monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}" + #################### # OpenStack #################### diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index dffdbc7d59..6caadb6df0 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -42,3 +42,23 @@ or monasca_log_api_confs.changed | bool or monasca_log_api_wsgi_confs.changed | bool or monasca_log_api_container.changed | bool + +- name: Restart monasca-log-transformer container + vars: + service_name: "monasca-log-transformer" + service: "{{ monasca_services[service_name] }}" + config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + monasca_log_transformer_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or monasca_log_transformer_confs.changed | bool + or monasca_log_transformer_container.changed | bool diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index 164b970868..42c40f38dc 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -25,8 +25,7 @@ - item.value.enabled | bool with_dict: "{{ monasca_services }}" notify: - - Restart monasca-api container - - Restart monasca-log-api container + - "Restart {{ item.key }} container" - name: Copying over monasca-api config vars: @@ -100,6 +99,25 @@ notify: - Restart monasca-log-api container +- name: Copying over monasca-log-transformer config + vars: + service: "{{ monasca_services['monasca-log-transformer'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-log-transformer/log-transformer.conf" + mode: "0660" + become: true + register: monasca_log_transformer_confs + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-transformer.conf" + - "{{ node_custom_config }}/monasca/log-transformer.conf" + - "{{ role_path }}/templates/monasca-log-transformer/log-transformer.conf.j2" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-log-transformer container + - name: Check monasca containers become: true kolla_docker: @@ -115,5 +133,4 @@ - item.value.enabled | bool with_dict: "{{ monasca_services }}" notify: - - Restart monasca-api container - - Restart monasca-log-api container + - "Restart {{ item.key }} container" diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml index decfa5ff9f..b386d15e3f 100644 --- a/ansible/roles/monasca/tasks/deploy.yml +++ b/ansible/roles/monasca/tasks/deploy.yml @@ -5,7 +5,8 @@ - include: config.yml when: inventory_hostname in groups['monasca-api'] or - inventory_hostname in groups['monasca-log-api'] + inventory_hostname in groups['monasca-log-api'] or + inventory_hostname in groups['monasca-log-transformer'] - include: bootstrap.yml when: inventory_hostname in groups['monasca-api'] @@ -15,4 +16,5 @@ - include: check.yml when: inventory_hostname in groups['monasca-api'] or - inventory_hostname in groups['monasca-log-api'] + inventory_hostname in groups['monasca-log-api'] or + inventory_hostname in groups['monasca-log-transformer'] diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 new file mode 100644 index 0000000000..c571623317 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 @@ -0,0 +1,25 @@ +# Provide input/output streams for transforming Monasca logs. +# Filters should be provided in other configuration files. + +input { + kafka { + zk_connect => "{{ monasca_zookeeper_servers }}" + topic_id => "{{ monasca_raw_logs_topic }}" + group_id => "transformer-logstash-consumer" + } +} + +filter { + # Update the timestamp of the event based on the time in the message. + date { + match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss +0000", "ISO8601"] + remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ] + } +} + +output { + kafka { + bootstrap_servers => "{{ monasca_kafka_servers }}" + topic_id => "{{ monasca_transformed_logs_topic }}" + } +} diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 new file mode 100644 index 0000000000..94cb1cf375 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 @@ -0,0 +1,18 @@ +{ + "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf", + "config_files": [ + { + "source": "{{ container_config_directory }}/log-transformer.conf", + "dest": "/etc/logstash/conf.d/log-transformer.conf", + "owner": "logstash", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/logstash", + "owner": "logstash:kolla", + "recurse": true + } + ] +} diff --git a/releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml b/releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml new file mode 100644 index 0000000000..3e1eb09639 --- /dev/null +++ b/releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add support for deploying the Monasca Log Transformer for + providing log standardisation in Monasca.