Logstash 6 support

Co-Authored-By: Doug Szumski <doug@stackhpc.com>
Closes-Bug: #1884090
Depends-On: https://review.opendev.org/#/c/736768

Change-Id: If2d0dd1739e484b14e3c15a185a236918737b0ab

ansible/roles/monasca/defaults/main.yml

@@ -251,16 +251,19 @@ monasca_log_transformer_default_volumes:
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
   - "kolla_logs:/var/log/kolla"
+  - "monasca_log_transformer_data:/var/lib/logstash"
 monasca_log_persister_default_volumes:
   - "{{ node_config_directory }}/monasca-log-persister/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
   - "kolla_logs:/var/log/kolla"
+  - "monasca_log_persister_data:/var/lib/logstash"
 monasca_log_metrics_default_volumes:
   - "{{ node_config_directory }}/monasca-log-metrics/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
   - "kolla_logs:/var/log/kolla"
+  - "monasca_log_metrics_data:/var/lib/logstash"
 monasca_thresh_default_volumes:
   - "{{ node_config_directory }}/monasca-thresh/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"

ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2

@@ -9,11 +9,11 @@
 
 input {
     kafka {
-        zk_connect => "{{ monasca_zookeeper_servers }}"
+        bootstrap_servers => "{{ monasca_kafka_servers }}"
-        topic_id => "{{ monasca_transformed_logs_topic }}"
+        topics => ["{{ monasca_transformed_logs_topic }}"]
         group_id => "log_metrics"
-        consumer_id => "log_metrics_{{ ansible_hostname }}"
         consumer_threads => "{{ monasca_log_pipeline_threads }}"
+        codec => json
     }
 }
 
@@ -42,7 +42,7 @@ filter {
 
     # Convert the timestamp of the event to milliseconds since epoch.
     ruby {
-        code => "event['metric']['timestamp'] = event['@timestamp'].to_i * 1000"
+        code => "event.set('[metric][timestamp]', event.get('[@timestamp]').to_i*1000)"
     }
 
     # Clean up any fields which aren't required from the new metric to save space
@@ -68,9 +68,8 @@ filter {
 
 output {
     kafka {
+        codec => json
         bootstrap_servers => "{{ monasca_kafka_servers }}"
         topic_id => "{{ monasca_metrics_topic }}"
-        client_id => "log_metrics_{{ ansible_hostname }}"
-        workers => {{ monasca_log_pipeline_threads|int }}
     }
 }

ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2

@@ -1,5 +1,5 @@
 {
-    "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-metrics.log -f /etc/logstash/conf.d/log-metrics.conf",
+    "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-metrics -f /etc/logstash/conf.d/log-metrics.conf",
     "config_files": [
         {
             "source": "{{ container_config_directory }}/log-metrics.conf",

ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2

@@ -2,11 +2,11 @@
 
 input {
     kafka {
-        zk_connect => "{{ monasca_zookeeper_servers }}"
+        bootstrap_servers => "{{ monasca_kafka_servers }}"
-        topic_id => "{{ monasca_transformed_logs_topic }}"
+        topics => ["{{ monasca_transformed_logs_topic }}"]
         group_id => "log_persister"
-        consumer_id => "log_persister_{{ ansible_hostname }}"
         consumer_threads => "{{ monasca_log_pipeline_threads }}"
+        codec => json
     }
 }
 

ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2

@@ -1,5 +1,5 @@
 {
-    "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-persister.log -f /etc/logstash/conf.d/log-persister.conf",
+    "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-persister -f /etc/logstash/conf.d/log-persister.conf",
     "config_files": [
         {
             "source": "{{ container_config_directory }}/log-persister.conf",

ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2

@@ -3,11 +3,11 @@
 
 input {
     kafka {
-        zk_connect => "{{ monasca_zookeeper_servers }}"
+        bootstrap_servers => "{{ monasca_kafka_servers }}"
-        topic_id => "{{ monasca_raw_logs_topic }}"
+        topics => ["{{ monasca_raw_logs_topic }}"]
         group_id => "log_transformer"
-        consumer_id => "log_transformer_{{ ansible_hostname }}"
         consumer_threads => "{{ monasca_log_pipeline_threads }}"
+        codec => json
     }
 }
 
@@ -39,9 +39,8 @@ filter {
 
 output {
     kafka {
+        codec => json
         bootstrap_servers => "{{ monasca_kafka_servers }}"
         topic_id => "{{ monasca_transformed_logs_topic }}"
-        client_id => "log_transformer_{{ ansible_hostname }}"
-        workers => {{ monasca_log_pipeline_threads|int }}
     }
 }

ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2

@@ -1,5 +1,5 @@
 {
-    "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf",
+    "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-transformer -f /etc/logstash/conf.d/log-transformer.conf",
     "config_files": [
         {
             "source": "{{ container_config_directory }}/log-transformer.conf",

releasenotes/notes/bug-1884090-c40ff140d5514108.yaml

@@ -0,0 +1,12 @@
+---
+upgrade:
+  - |
+    When deploying Monasca with Logstash 6, any
+    custom Logstash 2 configuration for Monasca will need to be updated to
+    work with Logstash 6. Please consult the `documentation
+    <https://www.elastic.co/guide/en/logstash/master/upgrading-logstash.html>`__.
+fixes:
+  - |
+    When deploying Elasticsearch 6, Logstash 2 was
+    deployed by default which is not compatible with Elasticsearch 6. Logstash
+    6 is now deployed by default.