From 444097848c38cfcf661f5203ffef9c6bfc25cc37 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Tue, 9 Feb 2021 11:57:43 +0000 Subject: [PATCH] Support disabling Monasca alerting pipeline The Monasca alerting pipeline provides multi-tenancy alerts and notifications. It runs as an Apache Storm topology and generally places a significant memory and CPU burden on monitoring hosts, particularly when there are lot of metrics. This is fine if the alerting service is in use, but sometimes it is not. For example you may use Prometheus for monitoring the control plane, and wish to offer tenants a monitoring service via Monasca without alerting and notification functionality. In this case it makes sense to disable this part of the Monasca pipeline and this patch adds support for that. If the service is ever re-enabled, all alerts and notifications should spawn back automatically since they are persisted in the central mysql database cluster. Change-Id: I84aa04125c621712f805f41c8efbc92c8e156db9 --- ansible/group_vars/all.yml | 4 +++- ansible/monasca_cleanup.yml | 8 +++++++ ansible/roles/monasca/defaults/main.yml | 14 ++++++------ .../monasca-persister/persister.conf.j2 | 4 ++++ ansible/roles/storm/defaults/main.yml | 4 ++-- ansible/roles/storm/tasks/cleanup.yml | 22 +++++++++++++++++++ .../logging-and-monitoring/monasca-guide.rst | 20 ++++++++++++++--- ...sca-alerting-service-92d49ead068447f8.yaml | 6 +++++ 8 files changed, 69 insertions(+), 13 deletions(-) create mode 100644 ansible/roles/storm/tasks/cleanup.yml create mode 100644 releasenotes/notes/support-disabling-monasca-alerting-service-92d49ead068447f8.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 1454e826e7..ce1a41b201 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -673,7 +673,7 @@ enable_sahara: "no" enable_senlin: "no" enable_skydive: "no" enable_solum: "no" -enable_storm: "{{ enable_monasca | bool }}" +enable_storm: "{{ enable_monasca | bool and monasca_enable_alerting_pipeline | bool }}" enable_swift: "no" enable_swift_s3api: "no" enable_swift_recon: "no" @@ -707,6 +707,8 @@ enable_destroy_images: "no" #################### # Monasca options #################### +monasca_enable_alerting_pipeline: True + monasca_api_admin_base_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_api_port }}" monasca_api_internal_base_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_api_port }}" monasca_api_public_base_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn | put_address_in_context('url') }}:{{ monasca_api_port }}" diff --git a/ansible/monasca_cleanup.yml b/ansible/monasca_cleanup.yml index 25b1243a9c..300fd641bc 100644 --- a/ansible/monasca_cleanup.yml +++ b/ansible/monasca_cleanup.yml @@ -12,3 +12,11 @@ roles: - { role: monasca, tags: monasca } + +- name: Cleanup unused Storm services + hosts: + - storm-worker + - storm-nimbus + roles: + - { role: storm, + tags: storm } diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index 9ea240e01c..4755be9df4 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -46,14 +46,14 @@ monasca_services: monasca-thresh: container_name: monasca_thresh group: monasca-thresh - enabled: true + enabled: "{{ monasca_enable_alerting_pipeline | bool }}" image: "{{ monasca_thresh_image_full }}" volumes: "{{ monasca_thresh_default_volumes + monasca_thresh_extra_volumes }}" dimensions: "{{ monasca_thresh_dimensions }}" monasca-notification: container_name: monasca_notification group: monasca-notification - enabled: true + enabled: "{{ monasca_enable_alerting_pipeline | bool }}" image: "{{ monasca_notification_image_full }}" volumes: "{{ monasca_notification_default_volumes + monasca_notification_extra_volumes }}" dimensions: "{{ monasca_notification_dimensions }}" @@ -186,23 +186,23 @@ monasca_all_topics: - name: "{{ monasca_events_topic }}" partitions: "{{ monasca_default_topic_partitions }}" replication_factor: "{{ monasca_default_topic_replication_factor }}" - enabled: True + enabled: "{{ monasca_enable_alerting_pipeline | bool }}" - name: "{{ monasca_alarm_state_transitions_topic }}" partitions: "{{ monasca_default_topic_partitions }}" replication_factor: "{{ monasca_default_topic_replication_factor }}" - enabled: True + enabled: "{{ monasca_enable_alerting_pipeline | bool }}" - name: "{{ monasca_alarm_notifications_topic }}" partitions: "{{ monasca_default_topic_partitions }}" replication_factor: "{{ monasca_default_topic_replication_factor }}" - enabled: True + enabled: "{{ monasca_enable_alerting_pipeline | bool }}" - name: "{{ monasca_alarm_notifications_retry_topic }}" partitions: "{{ monasca_default_topic_partitions }}" replication_factor: "{{ monasca_default_topic_replication_factor }}" - enabled: True + enabled: "{{ monasca_enable_alerting_pipeline | bool }}" - name: "{{ monasca_periodic_notifications_topic }}" partitions: "{{ monasca_default_topic_partitions }}" replication_factor: "{{ monasca_default_topic_replication_factor }}" - enabled: True + enabled: "{{ monasca_enable_alerting_pipeline | bool }}" # NOTE(dszumski): Due to the way monasca-notification is currently # implemented it is not recommended to change this period. diff --git a/ansible/roles/monasca/templates/monasca-persister/persister.conf.j2 b/ansible/roles/monasca/templates/monasca-persister/persister.conf.j2 index fb7cbc9dd2..7f0d229694 100644 --- a/ansible/roles/monasca/templates/monasca-persister/persister.conf.j2 +++ b/ansible/roles/monasca/templates/monasca-persister/persister.conf.j2 @@ -15,9 +15,13 @@ port = {{ monasca_influxdb_http_port }} legacy_kafka_client_enabled = False [kafka_alarm_history] +{% if not monasca_enable_alerting_pipeline | bool %} +enabled = False +{% else %} uri = {{ monasca_kafka_servers }} topic = {{ monasca_alarm_state_transitions_topic }} num_processors = 1 +{% endif %} [kafka_metrics] uri = {{ monasca_kafka_servers }} diff --git a/ansible/roles/storm/defaults/main.yml b/ansible/roles/storm/defaults/main.yml index efadad23cd..f343af3d8d 100644 --- a/ansible/roles/storm/defaults/main.yml +++ b/ansible/roles/storm/defaults/main.yml @@ -5,7 +5,7 @@ storm_services: storm-worker: container_name: storm_worker group: storm-worker - enabled: true + enabled: "{{ enable_storm | bool }}" image: "{{ storm_image_full }}" environment: STORM_LOG_DIR: /var/log/kolla/storm @@ -15,7 +15,7 @@ storm_services: storm-nimbus: container_name: storm_nimbus group: storm-nimbus - enabled: true + enabled: "{{ enable_storm | bool }}" image: "{{ storm_image_full }}" environment: STORM_LOG_DIR: /var/log/kolla/storm diff --git a/ansible/roles/storm/tasks/cleanup.yml b/ansible/roles/storm/tasks/cleanup.yml new file mode 100644 index 0000000000..4d0a5a81c8 --- /dev/null +++ b/ansible/roles/storm/tasks/cleanup.yml @@ -0,0 +1,22 @@ +--- +- name: Stop and remove containers for Storm services + become: true + kolla_docker: + action: "stop_and_remove_container" + name: "{{ item.value.container_name }}" + when: + - inventory_hostname in groups[item.value.group] + - not item.value.enabled | bool + with_dict: "{{ storm_services }}" + +- name: Removing config for any disabled services + file: + path: "{{ node_config_directory }}/{{ item.key }}" + state: "absent" + become: true + when: + - inventory_hostname in groups[item.value.group] + - not item.value.enabled | bool + with_dict: "{{ storm_services }}" + +# NOTE(dszumski): Docker volume removal is currently a manual procedure diff --git a/doc/source/reference/logging-and-monitoring/monasca-guide.rst b/doc/source/reference/logging-and-monitoring/monasca-guide.rst index 365c952c5d..947265011c 100644 --- a/doc/source/reference/logging-and-monitoring/monasca-guide.rst +++ b/doc/source/reference/logging-and-monitoring/monasca-guide.rst @@ -42,6 +42,13 @@ Enable Monasca in ``/etc/kolla/globals.yml``: enable_monasca: "yes" +If you wish to disable the alerting and notification pipeline to reduce +resource usage you can set ``/etc/kolla/globals.yml``: + +.. code-block:: yaml + + monasca_enable_alerting_pipeline: "no" + Currently Monasca is only supported using the ``source`` install type Kolla images. If you are using the ``binary`` install type you should set the following override in ``/etc/kolla/globals.yml``: @@ -334,6 +341,9 @@ which you must run the cleanup command is given below: Wallaby. - Upgrading from Wallaby to Xena to remove the Monasca Log Metrics service if the option to disable it by default was overridden in Wallaby. +- If you have disabled the alerting pipeline via the + `monasca_enable_alerting_pipeline` flag after you have deployed the alerting + services. The cleanup command can be invoked from the Kolla Ansible CLI, for example: @@ -365,7 +375,7 @@ System requirements and performance impact Monasca will deploy the following Docker containers: * Apache Kafka -* Apache Storm +* Apache Storm (optional) * Apache Zookeeper * Elasticsearch * Grafana @@ -378,9 +388,9 @@ Monasca will deploy the following Docker containers: * Monasca Log API * Monasca Log Metrics (Logstash, optional, deprecated) * Monasca Log Persister (Logstash) -* Monasca Notification +* Monasca Notification (optional) * Monasca Persister -* Monasca Thresh (Apache Storm topology) +* Monasca Thresh (Apache Storm topology, optional) In addition to these, Monasca will also utilise Kolla deployed MariaDB, Keystone, Memcached and HAProxy/Keepalived. The Monasca Agent containers @@ -397,6 +407,10 @@ multi-core CPU. You will also need enough space to store metrics and logs, and to buffer these in Kafka. Whilst Kafka is happy with spinning disks, you will likely want to use SSDs to back InfluxDB and Elasticsearch. +If resources are tight, it is possible to disable the alerting and +notification pipeline which removes the need for Apache Storm, Monasca +Thresh and Monasca Notification. This can have a significant effect. + .. _Security impact: Security impact diff --git a/releasenotes/notes/support-disabling-monasca-alerting-service-92d49ead068447f8.yaml b/releasenotes/notes/support-disabling-monasca-alerting-service-92d49ead068447f8.yaml new file mode 100644 index 0000000000..3f4017713b --- /dev/null +++ b/releasenotes/notes/support-disabling-monasca-alerting-service-92d49ead068447f8.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Support has been added to optionally disable the Monasca alerting + pipeline. This can be helpful to reduce resource consumption on + Monasca service hosts if the alerting pipeline is not in use.