From b6cce3e3f3d777e9b06377c56f4cd95764210ab2 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Wed, 13 Jun 2018 16:26:06 +0100 Subject: [PATCH] Support deploying Monasca Thresh Monasca Thresh is a Storm topology which generates alerts from metric streams according to alarms defined via the Monasca API. This change runs the thresholder in local mode, which means that the log output for the topology is directed to stdout and the topology is restarted if the container is restarted. A future change will improve the log collection and introduce a better way of the checking the topology is running for multi-node clusters. Change-Id: I063dca5eead15f3cec009df62f0fc5d857dd4bb0 Partially-Implements: blueprint monasca-roles --- ansible/inventory/all-in-one | 3 + ansible/inventory/multinode | 3 + ansible/roles/monasca/defaults/main.yml | 20 +++ ansible/roles/monasca/handlers/main.yml | 22 +++ ansible/roles/monasca/tasks/config.yml | 41 ++++++ ansible/roles/monasca/tasks/deploy.yml | 6 +- .../monasca-thresh/monasca-thresh.json.j2 | 29 ++++ .../templates/monasca-thresh/storm.yml.j2 | 9 ++ .../monasca-thresh/thresh-config.yml.j2 | 132 ++++++++++++++++++ .../add-monasca-thresh-f7a860ce996684f9.yaml | 5 + 10 files changed, 268 insertions(+), 2 deletions(-) create mode 100644 ansible/roles/monasca/templates/monasca-thresh/monasca-thresh.json.j2 create mode 100644 ansible/roles/monasca/templates/monasca-thresh/storm.yml.j2 create mode 100644 ansible/roles/monasca/templates/monasca-thresh/thresh-config.yml.j2 create mode 100644 releasenotes/notes/add-monasca-thresh-f7a860ce996684f9.yaml diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 9ed937f82e..e72a49ddd5 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -461,6 +461,9 @@ monasca [monasca-log-metrics:children] monasca +[monasca-thresh:children] +monasca + # Storm [storm-worker:children] storm diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 1b2f4cfb27..a7969b9342 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -470,6 +470,9 @@ monasca [monasca-log-metrics:children] monasca +[monasca-thresh:children] +monasca + # Storm [storm-worker:children] storm diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index 309d545dd3..547180e45d 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -50,6 +50,17 @@ monasca_services: - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla" dimensions: "{{ monasca_log_metrics_dimensions }}" + monasca-thresh: + container_name: monasca_thresh + group: monasca-thresh + enabled: true + image: "{{ monasca_thresh_image_full }}" + volumes: + - "{{ node_config_directory }}/monasca-thresh/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "monasca_thresh:/var/lib/monasca-thresh/" + - "kolla_logs:/var/log/kolla" + dimensions: "{{ monasca_thresh_dimensions }}" #################### # Databases @@ -69,10 +80,14 @@ monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansi monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_elasticsearch_servers: "{% for host in groups['elasticsearch'] %}'{{ internal_protocol }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }}'{% if not loop.last %},{% endif %}{% endfor %}" +monasca_storm_nimbus_servers: "{% for host in groups['storm-nimbus'] %}'{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}'{% if not loop.last %},{% endif %}{% endfor %}" +# Kafka topics used by Monasca services monasca_metrics_topic: "metrics" monasca_raw_logs_topic: "logs" monasca_transformed_logs_topic: "transformed-logs" +monasca_events_topic: "events" +monasca_alarm_state_transitions_topic: "alarm-state-transitions" # Processing pipeline threads. In a large scale deployment you will likely # want to tune these with finer precision. For example, if you have a very @@ -100,11 +115,16 @@ monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{ monasca_logstash_tag: "{{ monasca_tag }}" monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}" +monasca_thresh_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-thresh" +monasca_thresh_tag: "{{ monasca_tag }}" +monasca_thresh_image_full: "{{ monasca_thresh_image }}:{{ monasca_thresh_tag }}" + monasca_api_dimensions: "{{ default_container_dimensions }}" monasca_log_api_dimensions: "{{ default_container_dimensions }}" monasca_log_transformer_dimensions: "{{ default_container_dimensions }}" monasca_log_persister_dimensions: "{{ default_container_dimensions }}" monasca_log_metrics_dimensions: "{{ default_container_dimensions }}" +monasca_thresh_dimensions: "{{ default_container_dimensions }}" #################### diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index e47955b1e0..e02ad4aad1 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -108,3 +108,25 @@ - config_json.changed | bool or monasca_log_metrics_confs.changed | bool or monasca_log_metrics_container.changed | bool + +- name: Restart monasca-thresh container + vars: + service_name: "monasca-thresh" + service: "{{ monasca_services[service_name] }}" + config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + monasca_thresh_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or monasca_thresh_confs.changed | bool + or monasca_thresh_storm_conf.changed | bool + or monasca_thresh_container.changed | bool diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index 4c07bd31b7..dc070b4481 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -175,6 +175,47 @@ notify: - Restart monasca-log-metrics container +- name: Copying over monasca-thresh config + vars: + service: "{{ monasca_services['monasca-thresh'] }}" + # NOTE(dszumski): We can't use merge_yaml since it replaces empty values + # with `null`. This breaks the thresholder config file parsing (which should + # probably be more robust). + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-thresh/thresh-config.yml" + mode: "0660" + become: true + register: monasca_thresh_confs + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/thresh-config.yml" + - "{{ node_custom_config }}/monasca/thresh-config.yml" + - "{{ role_path }}/templates/monasca-thresh/thresh-config.yml.j2" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-thresh container + +- name: Copying over monasca-thresh storm config + vars: + service: "{{ monasca_services['monasca-thresh'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-thresh/storm.yml" + mode: "0660" + become: true + register: monasca_thresh_storm_conf + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/storm.yml" + - "{{ node_custom_config }}/monasca/storm.yml" + - "{{ role_path }}/templates/monasca-thresh/storm.yml.j2" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-thresh container + - name: Check monasca containers become: true kolla_docker: diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml index aa233dc2c5..3732a6418d 100644 --- a/ansible/roles/monasca/tasks/deploy.yml +++ b/ansible/roles/monasca/tasks/deploy.yml @@ -8,7 +8,8 @@ inventory_hostname in groups['monasca-log-api'] or inventory_hostname in groups['monasca-log-transformer'] or inventory_hostname in groups['monasca-log-persister'] or - inventory_hostname in groups['monasca-log-metrics'] + inventory_hostname in groups['monasca-log-metrics'] or + inventory_hostname in groups['monasca-thresh'] - include_tasks: bootstrap.yml when: inventory_hostname in groups['monasca-api'] @@ -21,4 +22,5 @@ inventory_hostname in groups['monasca-log-api'] or inventory_hostname in groups['monasca-log-transformer'] or inventory_hostname in groups['monasca-log-persister'] or - inventory_hostname in groups['monasca-log-metrics'] + inventory_hostname in groups['monasca-log-metrics'] or + inventory_hostname in groups['monasca-thresh'] diff --git a/ansible/roles/monasca/templates/monasca-thresh/monasca-thresh.json.j2 b/ansible/roles/monasca/templates/monasca-thresh/monasca-thresh.json.j2 new file mode 100644 index 0000000000..b733ddf698 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-thresh/monasca-thresh.json.j2 @@ -0,0 +1,29 @@ +{ + "command": "/opt/storm/bin/storm jar /monasca-thresh-source/monasca-thresh-*/thresh/target/monasca-thresh-*-SNAPSHOT-shaded.jar monasca.thresh.ThresholdingEngine /etc/monasca/thresh-config.yml monasca-thresh local", + "config_files": [ + { + "source": "{{ container_config_directory }}/thresh-config.yml", + "dest": "/etc/monasca/thresh-config.yml", + "owner": "monasca", + "perm": "0600" + }, + { + "source": "/var/lib/kolla/config_files/storm.yml", + "dest": "/opt/storm/conf/storm.yaml", + "owner": "monasca", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/monasca", + "owner": "monasca:kolla", + "recurse": true + }, + { + "path": "/var/lib/monasca-thresh", + "owner": "monasca:kolla", + "recurse": true + } + ] +} diff --git a/ansible/roles/monasca/templates/monasca-thresh/storm.yml.j2 b/ansible/roles/monasca/templates/monasca-thresh/storm.yml.j2 new file mode 100644 index 0000000000..ee48f60e72 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-thresh/storm.yml.j2 @@ -0,0 +1,9 @@ +storm.local.dir: "/var/lib/monasca-thresh/data" +storm.log.dir: "/var/log/kolla/storm" +storm.workers.artifacts.dir: "/var/lib/monasca-thresh/worker-artifacts" +nimbus.seeds: [{{ monasca_storm_nimbus_servers }}] +storm.zookeeper.port: {{ zookeeper_client_port }} +storm.zookeeper.servers: +{% for host in groups['zookeeper'] %} + - "{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}" +{% endfor %} diff --git a/ansible/roles/monasca/templates/monasca-thresh/thresh-config.yml.j2 b/ansible/roles/monasca/templates/monasca-thresh/thresh-config.yml.j2 new file mode 100644 index 0000000000..7a35efa9fe --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-thresh/thresh-config.yml.j2 @@ -0,0 +1,132 @@ +# +# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP +# Copyright 2017 Fujitsu LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#    http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +metricSpoutThreads: 2 +metricSpoutTasks: 2 + +#statsdConfig: +# host: "127.0.0.1" +# port: %MONASCA_STATSD_PORT% +# prefix: monasca.storm. +# dimensions: !!map +# service : monitoring +# component : storm + + +metricSpoutConfig: + kafkaConsumerConfiguration: + # See http://kafka.apache.org/documentation.html#api for semantics and defaults. + topic: "{{ monasca_metrics_topic }}" + numThreads: 1 + groupId: "thresh-metric" + zookeeperConnect: "{{ monasca_zookeeper_servers }}" + consumerId: 1 + socketTimeoutMs: 30000 + socketReceiveBufferBytes: 65536 + fetchMessageMaxBytes: 1048576 + autoCommitEnable: true + autoCommitIntervalMs: 60000 + queuedMaxMessageChunks: 10 + rebalanceMaxRetries: 4 + fetchMinBytes: 1 + fetchWaitMaxMs: 100 + rebalanceBackoffMs: 2000 + refreshLeaderBackoffMs: 200 + autoOffsetReset: largest + consumerTimeoutMs: -1 + clientId: 1 + zookeeperSessionTimeoutMs: 60000 + zookeeperConnectionTimeoutMs: 60000 + zookeeperSyncTimeMs: 2000 + + +eventSpoutConfig: + kafkaConsumerConfiguration: + # See http://kafka.apache.org/documentation.html#api for semantics and defaults. + topic: "{{ monasca_events_topic }}" + numThreads: 1 + groupId: "thresh-event" + zookeeperConnect: "{{ monasca_zookeeper_servers }}" + consumerId: 1 + socketTimeoutMs: 30000 + socketReceiveBufferBytes: 65536 + fetchMessageMaxBytes: 1048576 + autoCommitEnable: true + autoCommitIntervalMs: 60000 + queuedMaxMessageChunks: 10 + rebalanceMaxRetries: 4 + fetchMinBytes: 1 + fetchWaitMaxMs: 100 + rebalanceBackoffMs: 2000 + refreshLeaderBackoffMs: 200 + autoOffsetReset: largest + consumerTimeoutMs: -1 + clientId: 1 + zookeeperSessionTimeoutMs: 60000 + zookeeperConnectionTimeoutMs: 60000 + zookeeperSyncTimeMs: 2000 + + +kafkaProducerConfig: + # See http://kafka.apache.org/documentation.html#api for semantics and defaults. + topic: "{{ monasca_alarm_state_transitions_topic }}" + metadataBrokerList: "{{ monasca_kafka_servers }}" + serializerClass: kafka.serializer.StringEncoder + partitionerClass: + requestRequiredAcks: 1 + requestTimeoutMs: 10000 + producerType: sync + keySerializerClass: + compressionCodec: none + compressedTopics: + messageSendMaxRetries: 3 + retryBackoffMs: 100 + topicMetadataRefreshIntervalMs: 600000 + queueBufferingMaxMs: 5000 + queueBufferingMaxMessages: 10000 + queueEnqueueTimeoutMs: -1 + batchNumMessages: 200 + sendBufferBytes: 102400 + clientId: Threshold_Engine + + +sporadicMetricNamespaces: + - foo + +database: + driverClass: org.drizzle.jdbc.DrizzleDriver + url: "jdbc:drizzle://{{ monasca_database_address }}/{{ monasca_database_name }}" + user: "{{ monasca_database_user }}" + password: "{{ monasca_database_password }}" + properties: + ssl: false + # the maximum amount of time to wait on an empty pool before throwing an exception + maxWaitForConnection: 1s + # the SQL query to run when validating a connection's liveness TODO FIXME + validationQuery: "/* MyService Health Check */ SELECT 1" + # the minimum number of connections to keep open + minSize: 8 + # the maximum number of connections to keep open + maxSize: 41 + hibernateSupport: false + # hibernate provider class + providerClass: com.zaxxer.hikari.hibernate.HikariConnectionProvider + databaseName: "{{ monasca_database_name }}" + serverName: "{{ database_address }}" + portNumber: "{{ database_port }}" + # hibernate auto configuration parameter + autoConfig: validate diff --git a/releasenotes/notes/add-monasca-thresh-f7a860ce996684f9.yaml b/releasenotes/notes/add-monasca-thresh-f7a860ce996684f9.yaml new file mode 100644 index 0000000000..e4cbf47083 --- /dev/null +++ b/releasenotes/notes/add-monasca-thresh-f7a860ce996684f9.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add support for deploying the Monasca thresh service, an Apache Storm + topology for alerting.