diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index 18c5624a28..2b0f5f45a2 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -251,16 +251,19 @@ monasca_log_transformer_default_volumes: - "/etc/localtime:/etc/localtime:ro" - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}" - "kolla_logs:/var/log/kolla" + - "monasca_log_transformer_data:/var/lib/logstash" monasca_log_persister_default_volumes: - "{{ node_config_directory }}/monasca-log-persister/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}" - "kolla_logs:/var/log/kolla" + - "monasca_log_persister_data:/var/lib/logstash" monasca_log_metrics_default_volumes: - "{{ node_config_directory }}/monasca-log-metrics/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}" - "kolla_logs:/var/log/kolla" + - "monasca_log_metrics_data:/var/lib/logstash" monasca_thresh_default_volumes: - "{{ node_config_directory }}/monasca-thresh/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" diff --git a/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 index a237d61a7b..bb05e9d488 100644 --- a/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 +++ b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 @@ -9,11 +9,11 @@ input { kafka { - zk_connect => "{{ monasca_zookeeper_servers }}" - topic_id => "{{ monasca_transformed_logs_topic }}" + bootstrap_servers => "{{ monasca_kafka_servers }}" + topics => ["{{ monasca_transformed_logs_topic }}"] group_id => "log_metrics" - consumer_id => "log_metrics_{{ ansible_hostname }}" consumer_threads => "{{ monasca_log_pipeline_threads }}" + codec => json } } @@ -42,7 +42,7 @@ filter { # Convert the timestamp of the event to milliseconds since epoch. ruby { - code => "event['metric']['timestamp'] = event['@timestamp'].to_i * 1000" + code => "event.set('[metric][timestamp]', event.get('[@timestamp]').to_i*1000)" } # Clean up any fields which aren't required from the new metric to save space @@ -68,9 +68,8 @@ filter { output { kafka { + codec => json bootstrap_servers => "{{ monasca_kafka_servers }}" topic_id => "{{ monasca_metrics_topic }}" - client_id => "log_metrics_{{ ansible_hostname }}" - workers => {{ monasca_log_pipeline_threads|int }} } } diff --git a/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 b/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 index 287ffcd975..7c605aeab7 100644 --- a/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 +++ b/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 @@ -1,5 +1,5 @@ { - "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-metrics.log -f /etc/logstash/conf.d/log-metrics.conf", + "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-metrics -f /etc/logstash/conf.d/log-metrics.conf", "config_files": [ { "source": "{{ container_config_directory }}/log-metrics.conf", diff --git a/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 index 6eb9219b4c..732c07e5a0 100644 --- a/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 +++ b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 @@ -2,11 +2,11 @@ input { kafka { - zk_connect => "{{ monasca_zookeeper_servers }}" - topic_id => "{{ monasca_transformed_logs_topic }}" + bootstrap_servers => "{{ monasca_kafka_servers }}" + topics => ["{{ monasca_transformed_logs_topic }}"] group_id => "log_persister" - consumer_id => "log_persister_{{ ansible_hostname }}" consumer_threads => "{{ monasca_log_pipeline_threads }}" + codec => json } } diff --git a/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 b/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 index fa99a35d20..365d293dde 100644 --- a/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 +++ b/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 @@ -1,5 +1,5 @@ { - "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-persister.log -f /etc/logstash/conf.d/log-persister.conf", + "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-persister -f /etc/logstash/conf.d/log-persister.conf", "config_files": [ { "source": "{{ container_config_directory }}/log-persister.conf", diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 index 494bf5257f..b637925375 100644 --- a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 +++ b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 @@ -3,11 +3,11 @@ input { kafka { - zk_connect => "{{ monasca_zookeeper_servers }}" - topic_id => "{{ monasca_raw_logs_topic }}" + bootstrap_servers => "{{ monasca_kafka_servers }}" + topics => ["{{ monasca_raw_logs_topic }}"] group_id => "log_transformer" - consumer_id => "log_transformer_{{ ansible_hostname }}" consumer_threads => "{{ monasca_log_pipeline_threads }}" + codec => json } } @@ -39,9 +39,8 @@ filter { output { kafka { + codec => json bootstrap_servers => "{{ monasca_kafka_servers }}" topic_id => "{{ monasca_transformed_logs_topic }}" - client_id => "log_transformer_{{ ansible_hostname }}" - workers => {{ monasca_log_pipeline_threads|int }} } } diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 index 733eeedad9..6d5cee36ee 100644 --- a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 +++ b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 @@ -1,5 +1,5 @@ { - "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf", + "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-transformer -f /etc/logstash/conf.d/log-transformer.conf", "config_files": [ { "source": "{{ container_config_directory }}/log-transformer.conf", diff --git a/releasenotes/notes/bug-1884090-c40ff140d5514108.yaml b/releasenotes/notes/bug-1884090-c40ff140d5514108.yaml new file mode 100644 index 0000000000..8d9d6128d1 --- /dev/null +++ b/releasenotes/notes/bug-1884090-c40ff140d5514108.yaml @@ -0,0 +1,12 @@ +--- +upgrade: + - | + When deploying Monasca with Logstash 6, any + custom Logstash 2 configuration for Monasca will need to be updated to + work with Logstash 6. Please consult the `documentation + `__. +fixes: + - | + When deploying Elasticsearch 6, Logstash 2 was + deployed by default which is not compatible with Elasticsearch 6. Logstash + 6 is now deployed by default.