openstack/kolla-ansible
Code Issues Proposed changes

Logstash 6 support

Browse Source 
Co-Authored-By: Doug Szumski <doug@stackhpc.com>
Closes-Bug: #1884090
Depends-On: https://review.opendev.org/#/c/736768

Change-Id: If2d0dd1739e484b14e3c15a185a236918737b0ab
This commit is contained in:
Bartosz Bezak 2020-07-07 13:43:18 +00:00 committed by Doug Szumski
parent f81aee5094
commit 17d8332604
8 changed files with 30 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ansible/roles/monasca/defaults/main.yml
View File

@ -251,16 +251,19 @@ monasca_log_transformer_default_volumes:
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
- "kolla_logs:/var/log/kolla"
- "monasca_log_transformer_data:/var/lib/logstash"
monasca_log_persister_default_volumes:
- "{{ node_config_directory }}/monasca-log-persister/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
- "kolla_logs:/var/log/kolla"
- "monasca_log_persister_data:/var/lib/logstash"
monasca_log_metrics_default_volumes:
- "{{ node_config_directory }}/monasca-log-metrics/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
- "kolla_logs:/var/log/kolla"
- "monasca_log_metrics_data:/var/lib/logstash"
monasca_thresh_default_volumes:
- "{{ node_config_directory }}/monasca-thresh/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"

11
ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2
View File

@ -9,11 +9,11 @@
input {
kafka {
zk_connect => "{{ monasca_zookeeper_servers }}"
topic_id => "{{ monasca_transformed_logs_topic }}"
bootstrap_servers => "{{ monasca_kafka_servers }}"
topics => ["{{ monasca_transformed_logs_topic }}"]
group_id => "log_metrics"
consumer_id => "log_metrics_{{ ansible_hostname }}"
consumer_threads => "{{ monasca_log_pipeline_threads }}"
codec => json
}
}
@ -42,7 +42,7 @@ filter {
# Convert the timestamp of the event to milliseconds since epoch.
ruby {
code => "event['metric']['timestamp'] = event['@timestamp'].to_i * 1000"
code => "event.set('[metric][timestamp]', event.get('[@timestamp]').to_i*1000)"
}
# Clean up any fields which aren't required from the new metric to save space
@ -68,9 +68,8 @@ filter {
output {
kafka {
codec => json
bootstrap_servers => "{{ monasca_kafka_servers }}"
topic_id => "{{ monasca_metrics_topic }}"
client_id => "log_metrics_{{ ansible_hostname }}"
workers => {{ monasca_log_pipeline_threads|int }}
}
}

2
ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2
View File

@ -1,5 +1,5 @@
{
"command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-metrics.log -f /etc/logstash/conf.d/log-metrics.conf",
"command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-metrics -f /etc/logstash/conf.d/log-metrics.conf",
"config_files": [
{
"source": "{{ container_config_directory }}/log-metrics.conf",

6
ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2
View File

@ -2,11 +2,11 @@
input {
kafka {
zk_connect => "{{ monasca_zookeeper_servers }}"
topic_id => "{{ monasca_transformed_logs_topic }}"
bootstrap_servers => "{{ monasca_kafka_servers }}"
topics => ["{{ monasca_transformed_logs_topic }}"]
group_id => "log_persister"
consumer_id => "log_persister_{{ ansible_hostname }}"
consumer_threads => "{{ monasca_log_pipeline_threads }}"
codec => json
}
}

2
ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2
View File

@ -1,5 +1,5 @@
{
"command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-persister.log -f /etc/logstash/conf.d/log-persister.conf",
"command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-persister -f /etc/logstash/conf.d/log-persister.conf",
"config_files": [
{
"source": "{{ container_config_directory }}/log-persister.conf",

9
ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2
View File

@ -3,11 +3,11 @@
input {
kafka {
zk_connect => "{{ monasca_zookeeper_servers }}"
topic_id => "{{ monasca_raw_logs_topic }}"
bootstrap_servers => "{{ monasca_kafka_servers }}"
topics => ["{{ monasca_raw_logs_topic }}"]
group_id => "log_transformer"
consumer_id => "log_transformer_{{ ansible_hostname }}"
consumer_threads => "{{ monasca_log_pipeline_threads }}"
codec => json
}
}
@ -39,9 +39,8 @@ filter {
output {
kafka {
codec => json
bootstrap_servers => "{{ monasca_kafka_servers }}"
topic_id => "{{ monasca_transformed_logs_topic }}"
client_id => "log_transformer_{{ ansible_hostname }}"
workers => {{ monasca_log_pipeline_threads|int }}
}
}

2
ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2
View File

@ -1,5 +1,5 @@
{
"command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf",
"command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-transformer -f /etc/logstash/conf.d/log-transformer.conf",
"config_files": [
{
"source": "{{ container_config_directory }}/log-transformer.conf",

12
releasenotes/notes/bug-1884090-c40ff140d5514108.yaml Normal file
View File

@ -0,0 +1,12 @@
---
upgrade:
- |
When deploying Monasca with Logstash 6, any
custom Logstash 2 configuration for Monasca will need to be updated to
work with Logstash 6. Please consult the `documentation
<https://www.elastic.co/guide/en/logstash/master/upgrading-logstash.html>`__.
fixes:
- |
When deploying Elasticsearch 6, Logstash 2 was
deployed by default which is not compatible with Elasticsearch 6. Logstash
6 is now deployed by default.