Support deploying Monasca Thresh

Monasca Thresh is a Storm topology which generates alerts from
metric streams according to alarms defined via the Monasca API.

This change runs the thresholder in local mode, which means that
the log output for the topology is directed to stdout and the
topology is restarted if the container is restarted. A future
change will improve the log collection and introduce a better
way of the checking the topology is running for multi-node
clusters.

Change-Id: I063dca5eead15f3cec009df62f0fc5d857dd4bb0
Partially-Implements: blueprint monasca-roles

ansible/inventory/all-in-one

@@ -461,6 +461,9 @@ monasca
 [monasca-log-metrics:children]
 monasca
 
+[monasca-thresh:children]
+monasca
+
 # Storm
 [storm-worker:children]
 storm

ansible/inventory/multinode

@@ -470,6 +470,9 @@ monasca
 [monasca-log-metrics:children]
 monasca
 
+[monasca-thresh:children]
+monasca
+
 # Storm
 [storm-worker:children]
 storm

ansible/roles/monasca/defaults/main.yml

@@ -50,6 +50,17 @@ monasca_services:
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla"
     dimensions: "{{ monasca_log_metrics_dimensions }}"
+  monasca-thresh:
+    container_name: monasca_thresh
+    group: monasca-thresh
+    enabled: true
+    image: "{{ monasca_thresh_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/monasca-thresh/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "monasca_thresh:/var/lib/monasca-thresh/"
+      - "kolla_logs:/var/log/kolla"
+    dimensions: "{{ monasca_thresh_dimensions }}"
 
 ####################
 # Databases
@@ -69,10 +80,14 @@ monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansi
 monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 monasca_elasticsearch_servers: "{% for host in groups['elasticsearch'] %}'{{ internal_protocol }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }}'{% if not loop.last %},{% endif %}{% endfor %}"
+monasca_storm_nimbus_servers: "{% for host in groups['storm-nimbus'] %}'{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}'{% if not loop.last %},{% endif %}{% endfor %}"
 
+# Kafka topics used by Monasca services
 monasca_metrics_topic: "metrics"
 monasca_raw_logs_topic: "logs"
 monasca_transformed_logs_topic: "transformed-logs"
+monasca_events_topic: "events"
+monasca_alarm_state_transitions_topic: "alarm-state-transitions"
 
 # Processing pipeline threads. In a large scale deployment you will likely
 # want to tune these with finer precision. For example, if you have a very
@@ -100,11 +115,16 @@ monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{
 monasca_logstash_tag: "{{ monasca_tag }}"
 monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}"
 
+monasca_thresh_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-thresh"
+monasca_thresh_tag: "{{ monasca_tag }}"
+monasca_thresh_image_full: "{{ monasca_thresh_image }}:{{ monasca_thresh_tag }}"
+
 monasca_api_dimensions: "{{ default_container_dimensions }}"
 monasca_log_api_dimensions: "{{ default_container_dimensions }}"
 monasca_log_transformer_dimensions: "{{ default_container_dimensions }}"
 monasca_log_persister_dimensions: "{{ default_container_dimensions }}"
 monasca_log_metrics_dimensions: "{{ default_container_dimensions }}"
+monasca_thresh_dimensions: "{{ default_container_dimensions }}"
 
 
 ####################

ansible/roles/monasca/handlers/main.yml

@@ -108,3 +108,25 @@
     - config_json.changed | bool
       or monasca_log_metrics_confs.changed | bool
       or monasca_log_metrics_container.changed | bool
+
+- name: Restart monasca-thresh container
+  vars:
+    service_name: "monasca-thresh"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_thresh_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_thresh_confs.changed | bool
+      or monasca_thresh_storm_conf.changed | bool
+      or monasca_thresh_container.changed | bool

ansible/roles/monasca/tasks/config.yml

@@ -175,6 +175,47 @@
   notify:
     - Restart monasca-log-metrics container
 
+- name: Copying over monasca-thresh config
+  vars:
+    service: "{{ monasca_services['monasca-thresh'] }}"
+  # NOTE(dszumski): We can't use merge_yaml since it replaces empty values
+  # with `null`. This breaks the thresholder config file parsing (which should
+  # probably be more robust).
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-thresh/thresh-config.yml"
+    mode: "0660"
+  become: true
+  register: monasca_thresh_confs
+  with_first_found:
+    - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/thresh-config.yml"
+    - "{{ node_custom_config }}/monasca/thresh-config.yml"
+    - "{{ role_path }}/templates/monasca-thresh/thresh-config.yml.j2"
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-thresh container
+
+- name: Copying over monasca-thresh storm config
+  vars:
+    service: "{{ monasca_services['monasca-thresh'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-thresh/storm.yml"
+    mode: "0660"
+  become: true
+  register: monasca_thresh_storm_conf
+  with_first_found:
+    - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/storm.yml"
+    - "{{ node_custom_config }}/monasca/storm.yml"
+    - "{{ role_path }}/templates/monasca-thresh/storm.yml.j2"
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-thresh container
+
 - name: Check monasca containers
   become: true
   kolla_docker:

ansible/roles/monasca/tasks/deploy.yml

@@ -8,7 +8,8 @@
         inventory_hostname in groups['monasca-log-api'] or
         inventory_hostname in groups['monasca-log-transformer'] or
         inventory_hostname in groups['monasca-log-persister'] or
-        inventory_hostname in groups['monasca-log-metrics']
+        inventory_hostname in groups['monasca-log-metrics'] or
+        inventory_hostname in groups['monasca-thresh']
 
 - include_tasks: bootstrap.yml
   when: inventory_hostname in groups['monasca-api']
@@ -21,4 +22,5 @@
         inventory_hostname in groups['monasca-log-api'] or
         inventory_hostname in groups['monasca-log-transformer'] or
         inventory_hostname in groups['monasca-log-persister'] or
-        inventory_hostname in groups['monasca-log-metrics']
+        inventory_hostname in groups['monasca-log-metrics'] or
+        inventory_hostname in groups['monasca-thresh']

ansible/roles/monasca/templates/monasca-thresh/monasca-thresh.json.j2

@@ -0,0 +1,29 @@
+{
+    "command": "/opt/storm/bin/storm jar /monasca-thresh-source/monasca-thresh-*/thresh/target/monasca-thresh-*-SNAPSHOT-shaded.jar monasca.thresh.ThresholdingEngine /etc/monasca/thresh-config.yml monasca-thresh local",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/thresh-config.yml",
+            "dest": "/etc/monasca/thresh-config.yml",
+            "owner": "monasca",
+            "perm": "0600"
+        },
+        {
+            "source": "/var/lib/kolla/config_files/storm.yml",
+            "dest": "/opt/storm/conf/storm.yaml",
+            "owner": "monasca",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/monasca",
+            "owner": "monasca:kolla",
+            "recurse": true
+        },
+        {
+            "path": "/var/lib/monasca-thresh",
+            "owner": "monasca:kolla",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/monasca/templates/monasca-thresh/storm.yml.j2

@@ -0,0 +1,9 @@
+storm.local.dir: "/var/lib/monasca-thresh/data"
+storm.log.dir: "/var/log/kolla/storm"
+storm.workers.artifacts.dir: "/var/lib/monasca-thresh/worker-artifacts"
+nimbus.seeds: [{{ monasca_storm_nimbus_servers }}]
+storm.zookeeper.port: {{ zookeeper_client_port }}
+storm.zookeeper.servers:
+{% for host in groups['zookeeper'] %}
+  - "{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}"
+{% endfor %}

ansible/roles/monasca/templates/monasca-thresh/thresh-config.yml.j2

@@ -0,0 +1,132 @@
+#
+# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP
+# Copyright 2017 Fujitsu LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+metricSpoutThreads: 2
+metricSpoutTasks: 2
+
+#statsdConfig:
+#  host: "127.0.0.1"
+#  port: %MONASCA_STATSD_PORT%
+#  prefix: monasca.storm.
+#  dimensions: !!map
+#    service : monitoring
+#    component : storm
+
+
+metricSpoutConfig:
+  kafkaConsumerConfiguration:
+  # See http://kafka.apache.org/documentation.html#api for semantics and defaults.
+    topic: "{{ monasca_metrics_topic }}"
+    numThreads: 1
+    groupId: "thresh-metric"
+    zookeeperConnect: "{{ monasca_zookeeper_servers }}"
+    consumerId: 1
+    socketTimeoutMs: 30000
+    socketReceiveBufferBytes: 65536
+    fetchMessageMaxBytes: 1048576
+    autoCommitEnable: true
+    autoCommitIntervalMs: 60000
+    queuedMaxMessageChunks: 10
+    rebalanceMaxRetries: 4
+    fetchMinBytes: 1
+    fetchWaitMaxMs: 100
+    rebalanceBackoffMs: 2000
+    refreshLeaderBackoffMs: 200
+    autoOffsetReset: largest
+    consumerTimeoutMs: -1
+    clientId: 1
+    zookeeperSessionTimeoutMs: 60000
+    zookeeperConnectionTimeoutMs: 60000
+    zookeeperSyncTimeMs: 2000
+
+
+eventSpoutConfig:
+  kafkaConsumerConfiguration:
+  # See http://kafka.apache.org/documentation.html#api for semantics and defaults.
+    topic: "{{ monasca_events_topic }}"
+    numThreads: 1
+    groupId: "thresh-event"
+    zookeeperConnect: "{{ monasca_zookeeper_servers }}"
+    consumerId: 1
+    socketTimeoutMs: 30000
+    socketReceiveBufferBytes: 65536
+    fetchMessageMaxBytes: 1048576
+    autoCommitEnable: true
+    autoCommitIntervalMs: 60000
+    queuedMaxMessageChunks: 10
+    rebalanceMaxRetries: 4
+    fetchMinBytes: 1
+    fetchWaitMaxMs: 100
+    rebalanceBackoffMs: 2000
+    refreshLeaderBackoffMs: 200
+    autoOffsetReset: largest
+    consumerTimeoutMs: -1
+    clientId: 1
+    zookeeperSessionTimeoutMs: 60000
+    zookeeperConnectionTimeoutMs: 60000
+    zookeeperSyncTimeMs: 2000
+
+
+kafkaProducerConfig:
+  # See http://kafka.apache.org/documentation.html#api for semantics and defaults.
+  topic: "{{ monasca_alarm_state_transitions_topic }}"
+  metadataBrokerList: "{{ monasca_kafka_servers }}"
+  serializerClass: kafka.serializer.StringEncoder
+  partitionerClass:
+  requestRequiredAcks: 1
+  requestTimeoutMs: 10000
+  producerType: sync
+  keySerializerClass:
+  compressionCodec: none
+  compressedTopics:
+  messageSendMaxRetries: 3
+  retryBackoffMs: 100
+  topicMetadataRefreshIntervalMs: 600000
+  queueBufferingMaxMs: 5000
+  queueBufferingMaxMessages: 10000
+  queueEnqueueTimeoutMs: -1
+  batchNumMessages: 200
+  sendBufferBytes: 102400
+  clientId: Threshold_Engine
+
+
+sporadicMetricNamespaces:
+  - foo
+
+database:
+  driverClass: org.drizzle.jdbc.DrizzleDriver
+  url: "jdbc:drizzle://{{ monasca_database_address }}/{{ monasca_database_name }}"
+  user: "{{ monasca_database_user }}"
+  password: "{{ monasca_database_password }}"
+  properties:
+      ssl: false
+  # the maximum amount of time to wait on an empty pool before throwing an exception
+  maxWaitForConnection: 1s
+  # the SQL query to run when validating a connection's liveness TODO FIXME
+  validationQuery: "/* MyService Health Check */ SELECT 1"
+  # the minimum number of connections to keep open
+  minSize: 8
+  # the maximum number of connections to keep open
+  maxSize: 41
+  hibernateSupport: false
+  # hibernate provider class
+  providerClass: com.zaxxer.hikari.hibernate.HikariConnectionProvider
+  databaseName: "{{ monasca_database_name }}"
+  serverName: "{{ database_address }}"
+  portNumber: "{{ database_port }}"
+  # hibernate auto configuration parameter
+  autoConfig: validate

releasenotes/notes/add-monasca-thresh-f7a860ce996684f9.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add support for deploying the Monasca thresh service, an Apache Storm
+    topology for alerting.