From 0743a9bf4b8bdce0aec8ee8640574ce3ee6a6b35 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Thu, 7 Jan 2021 17:14:15 +0000 Subject: [PATCH] Remove Monasca Log Transformer Historically Monasca Log Transformer has been for log standardisation and processing. For example, logs from different sources may use slightly different error levels such as WARN, 5, or WARNING. Monasca Log Transformer is a place where these could be 'squashed' into a single error level to simplify log searches based on labels such as these. However, in Kolla Ansible, we do this processing in Fluentd so that the simpler Fluentd -> Elastic -> Kibana pipeline also benefits. This helps to avoid spreading out log parsing configuration over many services, with the Fluentd Monasca output plugin being yet another potential place for processing (which should be avoided). It therefore makes sense to remove this service entirely, and squash any existing configuration which can't be moved to Fluentd into the Log Perister service. I.e. by removing this pipeline, we don't loose any functionality, we encourage log processing to take place in Fluentd, or at least outside of Monasca, and we make significant gains in efficiency by removing a topic from Kafka which contains a copy of all logs in transit. Finally, users forwarding logs from outside the control plane, eg. from tenant instances, should be encouraged to process the logs at the point of sending using whichever framework they are forwarding them with. This makes sense, because all Logstash configuration in Monasca is only accessible by control plane admins. A user can't typically do any processing inside Monasca, with or without this change. Change-Id: I65c76d0d1cd488725e4233b7e75a11d03866095c --- ansible/monasca_cleanup.yml | 14 ++++++ ansible/roles/monasca/defaults/main.yml | 6 ++- ansible/roles/monasca/handlers/main.yml | 15 ------ ansible/roles/monasca/tasks/cleanup.yml | 22 +++++++++ ansible/roles/monasca/tasks/config.yml | 28 ++--------- ansible/roles/monasca/tasks/upgrade.yml | 2 + .../monasca-log-metrics/log-metrics.conf.j2 | 2 +- .../log-persister.conf.j2 | 30 +++++++++++- .../log-transformer.conf.j2 | 46 ------------------- .../monasca-log-transformer.json.j2 | 25 ---------- .../logging-and-monitoring/monasca-guide.rst | 39 +++++++++++++++- ...asca-log-transformer-bdcb3ef1cdbc1611.yaml | 24 ++++++++++ tools/kolla-ansible | 7 +++ 13 files changed, 144 insertions(+), 116 deletions(-) create mode 100644 ansible/monasca_cleanup.yml create mode 100644 ansible/roles/monasca/tasks/cleanup.yml delete mode 100644 ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 delete mode 100644 ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 create mode 100644 releasenotes/notes/remove-monasca-log-transformer-bdcb3ef1cdbc1611.yaml diff --git a/ansible/monasca_cleanup.yml b/ansible/monasca_cleanup.yml new file mode 100644 index 0000000000..25b1243a9c --- /dev/null +++ b/ansible/monasca_cleanup.yml @@ -0,0 +1,14 @@ +--- +- name: Cleanup unused Monasca services + hosts: + - monasca-api + - monasca-grafana + - monasca-log-transformer + - monasca-log-persister + - monasca-log-metrics + - monasca-thresh + - monasca-notification + - monasca-persister + roles: + - { role: monasca, + tags: monasca } diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index 5a4d506bb9..ab219b832c 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -20,10 +20,12 @@ monasca_services: mode: "http" external: true port: "{{ monasca_api_port }}" + # NOTE(dszumski): We can remove log_transformer and all other references after the + # Wallaby release monasca-log-transformer: container_name: monasca_log_transformer group: monasca-log-transformer - enabled: true + enabled: false image: "{{ monasca_logstash_image_full }}" volumes: "{{ monasca_log_transformer_default_volumes + monasca_log_transformer_extra_volumes }}" dimensions: "{{ monasca_log_transformer_dimensions }}" @@ -175,7 +177,7 @@ monasca_all_topics: - name: "{{ monasca_transformed_logs_topic }}" partitions: "{{ monasca_default_topic_partitions }}" replication_factor: "{{ monasca_default_topic_replication_factor }}" - enabled: True + enabled: False - name: "{{ monasca_events_topic }}" partitions: "{{ monasca_default_topic_partitions }}" replication_factor: "{{ monasca_default_topic_replication_factor }}" diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index 7eb6162825..1346e5356b 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -14,21 +14,6 @@ when: - kolla_action != "config" -- name: Restart monasca-log-transformer container - vars: - service_name: "monasca-log-transformer" - service: "{{ monasca_services[service_name] }}" - become: true - kolla_docker: - action: "recreate_or_restart_container" - common_options: "{{ docker_common_options }}" - name: "{{ service.container_name }}" - image: "{{ service.image }}" - volumes: "{{ service.volumes }}" - dimensions: "{{ service.dimensions }}" - when: - - kolla_action != "config" - - name: Restart monasca-log-persister container vars: service_name: "monasca-log-persister" diff --git a/ansible/roles/monasca/tasks/cleanup.yml b/ansible/roles/monasca/tasks/cleanup.yml new file mode 100644 index 0000000000..dd23dbc153 --- /dev/null +++ b/ansible/roles/monasca/tasks/cleanup.yml @@ -0,0 +1,22 @@ +--- +- name: Stop and remove containers for disabled monasca services + become: true + kolla_docker: + action: "stop_and_remove_container" + name: "{{ item.value.container_name }}" + when: + - inventory_hostname in groups[item.value.group] + - not item.value.enabled | bool + with_dict: "{{ monasca_services }}" + +- name: Removing config for any disabled services + file: + path: "{{ node_config_directory }}/{{ item.key }}" + state: "absent" + become: true + when: + - inventory_hostname in groups[item.value.group] + - not item.value.enabled | bool + with_dict: "{{ monasca_services }}" + +# NOTE(dszumski): Docker volume removal is currently a manual procedure diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index ed1f6919bd..41f3a0869c 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -160,29 +160,11 @@ notify: - Restart monasca-api container -- name: Copying over monasca-log-transformer config - vars: - service: "{{ monasca_services['monasca-log-transformer'] }}" - template: - src: "{{ item }}" - dest: "{{ node_config_directory }}/monasca-log-transformer/log-transformer.conf" - mode: "0660" - become: true - with_first_found: - - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-transformer.conf" - - "{{ node_custom_config }}/monasca/log-transformer.conf" - - "{{ role_path }}/templates/monasca-log-transformer/log-transformer.conf.j2" - when: - - inventory_hostname in groups[service['group']] - - service.enabled | bool - notify: - - Restart monasca-log-transformer container - - name: Ensuring logstash patterns folder exists vars: - service: "{{ monasca_services['monasca-log-transformer'] }}" + service: "{{ monasca_services['monasca-log-persister'] }}" file: - path: "{{ node_config_directory }}/monasca-log-transformer/logstash_patterns" + path: "{{ node_config_directory }}/monasca-log-persister/logstash_patterns" state: "directory" mode: "0770" become: true @@ -200,10 +182,10 @@ - name: Copying over custom logstash patterns vars: - service: "{{ monasca_services['monasca-log-transformer'] }}" + service: "{{ monasca_services['monasca-log-persister'] }}" template: src: "{{ item.path }}" - dest: "{{ node_config_directory }}/monasca-log-transformer/logstash_patterns/{{ item.path | basename }}" + dest: "{{ node_config_directory }}/monasca-log-persister/logstash_patterns/{{ item.path | basename }}" mode: "0660" with_items: "{{ monasca_custom_logstash_patterns.files }}" become: true @@ -211,7 +193,7 @@ - inventory_hostname in groups[service['group']] - service.enabled | bool notify: - - Restart monasca-log-transformer container + - Restart monasca-log-persister container - name: Copying over monasca-log-persister config vars: diff --git a/ansible/roles/monasca/tasks/upgrade.yml b/ansible/roles/monasca/tasks/upgrade.yml index d42398e4ab..c5b8e5bbe2 100644 --- a/ansible/roles/monasca/tasks/upgrade.yml +++ b/ansible/roles/monasca/tasks/upgrade.yml @@ -14,6 +14,8 @@ - import_tasks: config.yml +- import_tasks: cleanup.yml + - import_tasks: check-containers.yml # NOTE(dszumski): We don't want old Grafana instances running after diff --git a/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 index ad651a20c2..4ab974be4d 100644 --- a/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 +++ b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 @@ -10,7 +10,7 @@ input { kafka { bootstrap_servers => "{{ monasca_kafka_servers }}" - topics => ["{{ monasca_transformed_logs_topic }}"] + topics => ["{{ monasca_raw_logs_topic }}"] group_id => "log_metrics" consumer_threads => "{{ monasca_log_pipeline_threads }}" codec => json diff --git a/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 index 732c07e5a0..c4e6994776 100644 --- a/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 +++ b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 @@ -1,15 +1,41 @@ -# Persist transformed logs to Elasticsearch +# Persist logs to Elasticsearch. input { kafka { bootstrap_servers => "{{ monasca_kafka_servers }}" - topics => ["{{ monasca_transformed_logs_topic }}"] + topics => ["{{ monasca_raw_logs_topic }}"] group_id => "log_persister" consumer_threads => "{{ monasca_log_pipeline_threads }}" codec => json } } +filter { + # Update the timestamp of the event based on the time in the message. + date { + match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss Z", "ISO8601"] + remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ] + } + + # Monasca Log API adds a timestamp when it processes a log entry. This + # timestamp needs to be converted from seconds since the epoch for + # Elasticsearch to parse it correctly. Here we make that conversion. + date { + match => ["creation_time", "UNIX"] + target => "creation_time" + } + + # OpenStack log levels are uppercase, and syslog are lowercase. + # Furthermore, syslog has more log levels that OpenStack. To avoid + # mapping syslog log levels to OpenStack log levels, we standardise + # on the syslog style here. + if [log][dimensions][log_level] { + mutate { + lowercase => [ "[log][dimensions][log_level]" ] + } + } +} + output { elasticsearch { index => "monasca-%{[meta][tenantId]}-%{+YYYY.MM.dd}" diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 deleted file mode 100644 index b637925375..0000000000 --- a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 +++ /dev/null @@ -1,46 +0,0 @@ -# Provide input/output streams for transforming Monasca logs. -# Filters should be provided in other configuration files. - -input { - kafka { - bootstrap_servers => "{{ monasca_kafka_servers }}" - topics => ["{{ monasca_raw_logs_topic }}"] - group_id => "log_transformer" - consumer_threads => "{{ monasca_log_pipeline_threads }}" - codec => json - } -} - -filter { - # Update the timestamp of the event based on the time in the message. - date { - match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss Z", "ISO8601"] - remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ] - } - - # Monasca Log API adds a timestamp when it processes a log entry. This - # timestamp needs to be converted from seconds since the epoch for - # Elasticsearch to parse it correctly. Here we make that conversion. - date { - match => ["creation_time", "UNIX"] - target => "creation_time" - } - - # OpenStack log levels are uppercase, and syslog are lowercase. - # Furthermore, syslog has more log levels that OpenStack. To avoid - # mapping syslog log levels to OpenStack log levels, we standardise - # on the syslog style here. - if [log][dimensions][log_level] { - mutate { - lowercase => [ "[log][dimensions][log_level]" ] - } - } -} - -output { - kafka { - codec => json - bootstrap_servers => "{{ monasca_kafka_servers }}" - topic_id => "{{ monasca_transformed_logs_topic }}" - } -} diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 deleted file mode 100644 index 6d5cee36ee..0000000000 --- a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 +++ /dev/null @@ -1,25 +0,0 @@ -{ - "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-transformer -f /etc/logstash/conf.d/log-transformer.conf", - "config_files": [ - { - "source": "{{ container_config_directory }}/log-transformer.conf", - "dest": "/etc/logstash/conf.d/log-transformer.conf", - "owner": "logstash", - "perm": "0600" - }, - { - "source": "{{ container_config_directory }}/logstash_patterns/*", - "dest": "/etc/logstash/conf.d/patterns/", - "owner": "logstash", - "perm": "0600", - "optional": true - } - ], - "permissions": [ - { - "path": "/var/log/kolla/logstash", - "owner": "logstash:kolla", - "recurse": true - } - ] -} diff --git a/doc/source/reference/logging-and-monitoring/monasca-guide.rst b/doc/source/reference/logging-and-monitoring/monasca-guide.rst index a73d4a4254..afd876beac 100644 --- a/doc/source/reference/logging-and-monitoring/monasca-guide.rst +++ b/doc/source/reference/logging-and-monitoring/monasca-guide.rst @@ -318,6 +318,42 @@ Apply the password changes by running the following command: kolla-ansible reconfigure -t monasca +Cleanup +~~~~~~~ + +From time-to-time it may be necessary to manually invoke the Monasca cleanup +command. If this is required during an upgrade it will be mentioned in the +release notes. It may also be necessary to run the cleanup command when +disabling certain parts of the Monasca pipeline. A full list of scenarios in +which you must run the cleanup command is given below: + +- Upgrading from Victoria to Wallaby to remove the unused Monasca Log + Transformer service + +The cleanup command can be invoked from the Kolla Ansible CLI, for example: + +.. code-block:: console + + kolla-ansible monasca_cleanup + +Following cleanup, you may also choose to remove unused container volumes. +It is recommended to run this manually on each Monasca service host. Note +that `docker prune` will indiscriminately remove all unused volumes, +which may not always be what you want. If you wish to keep a subset of +unused volumes, you can remove them individually. + +To remove all unused volumes on a host: + +.. code-block:: console + + docker prune + +To remove a single unused volume, run for example: + +.. code-block:: console + + docker volume rm monasca_log_transformer_data + System requirements and performance impact ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -335,9 +371,8 @@ Monasca will deploy the following Docker containers: * Monasca Agent Statsd * Monasca API * Monasca Log API -* Monasca Log Transformer (Logstash) * Monasca Log Metrics (Logstash) -* Monasca Log Perister (Logstash) +* Monasca Log Persister (Logstash) * Monasca Notification * Monasca Persister * Monasca Thresh (Apache Storm topology) diff --git a/releasenotes/notes/remove-monasca-log-transformer-bdcb3ef1cdbc1611.yaml b/releasenotes/notes/remove-monasca-log-transformer-bdcb3ef1cdbc1611.yaml new file mode 100644 index 0000000000..4770a85e3c --- /dev/null +++ b/releasenotes/notes/remove-monasca-log-transformer-bdcb3ef1cdbc1611.yaml @@ -0,0 +1,24 @@ +--- +upgrade: + - | + Monasca Log Transformer has been merged with Monasca Log Persister to + improve performance and reduce resource consumption. Any custom Monasca + Log Transformer configuration should be either merged into Monasca Log + Persister configuration, or moved outside of the Monasca pipeline, for + example, to Fluentd. Any custom Monasca Log Metrics config will also + need to be updated to read from the raw logs pipeline, rather than the + transformed logs pipeline. The transformed logs pipeline will be + removed from Kafka automatically, as will any log transformer containers. + There will be a short interruption to logging services whilst the + pipeline is updated. During this time it's likely that a small window + of logs will be lost from the transformed logs Kafka queue. If this + is a problem, the Monasca API should be stopped on all nodes prior to + upgrading Monasca. This will allow the transformed logs topic to drain + into Elasticsearch before the pipeline is reconfigured. Services such + as Fluentd, which post logs to the Monasca API, should buffer logs + whilst this happens up to the maximum configured buffer. Note that + there may be other services forwarding logs, and these will need to be + inspected independently. The Log Transformer volumes will remain on + the monitoring nodes and can be manually removed as described in the + `documentation + `__. diff --git a/tools/kolla-ansible b/tools/kolla-ansible index 56e675e8ae..bca3869f04 100755 --- a/tools/kolla-ansible +++ b/tools/kolla-ansible @@ -133,6 +133,7 @@ Commands: mariadb_backup Take a backup of MariaDB databases --full (default) --incremental + monasca_cleanup Remove unused containers for the Monasca service bootstrap-servers Bootstrap servers with kolla deploy dependencies destroy Destroy Kolla containers, volumes and host configuration --include-images to also destroy Kolla images @@ -175,6 +176,7 @@ prechecks check mariadb_recovery mariadb_backup +monasca_cleanup bootstrap-servers destroy deploy @@ -359,6 +361,11 @@ case "$1" in EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=backup -e mariadb_backup_type=${BACKUP_TYPE}" PLAYBOOK="${BASEDIR}/ansible/mariadb_backup.yml" ;; +(monasca_cleanup) + ACTION="Cleanup unused Monasca services" + EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=cleanup" + PLAYBOOK="${BASEDIR}/ansible/monasca_cleanup.yml" + ;; (destroy) ACTION="Destroy Kolla containers, volumes and host configuration" PLAYBOOK="${BASEDIR}/ansible/destroy.yml"