Add support for deploying prometheus-msteams

This can be used to forward Prometheus Alertmanager notifications to
Microsoft Teams.

Change-Id: I563f2438b3cb0895606b029b5269ce2e50c413e3
Depends-On: https://review.opendev.org/c/openstack/kolla/+/812678

ansible/group_vars/all.yml

@@ -462,6 +462,9 @@ prometheus_etcd_integration_port: "{{ etcd_client_port }}"
 prometheus_alertmanager_port: "9093"
 prometheus_alertmanager_cluster_port: "9094"
 
+# Prometheus MSTeams port
+prometheus_msteams_port: "9095"
+
 # Prometheus openstack-exporter ports
 prometheus_openstack_exporter_port: "9198"
 prometheus_elasticsearch_exporter_port: "9108"
@@ -1128,6 +1131,7 @@ enable_prometheus_blackbox_exporter: "{{ enable_prometheus | bool }}"
 enable_prometheus_rabbitmq_exporter: "{{ enable_prometheus | bool and enable_rabbitmq | bool }}"
 enable_prometheus_libvirt_exporter: "{{ enable_prometheus | bool and enable_nova | bool and nova_compute_virt_type in ['kvm', 'qemu'] }}"
 enable_prometheus_etcd_integration: "{{ enable_prometheus | bool and enable_etcd | bool }}"
+enable_prometheus_msteams: "no"
 
 prometheus_alertmanager_user: "admin"
 prometheus_scrape_interval: "60s"
@@ -1139,6 +1143,7 @@ prometheus_ceph_mgr_exporter_endpoints: []
 prometheus_openstack_exporter_endpoint_type: "internal"
 prometheus_openstack_exporter_compute_api_version: "latest"
 prometheus_libvirt_exporter_interval: "60s"
+prometheus_msteams_webhook_url:
 
 ############
 # Vitrage

ansible/inventory/all-in-one

@@ -700,6 +700,9 @@ monitoring
 [prometheus-libvirt-exporter:children]
 compute
 
+[prometheus-msteams:children]
+prometheus-alertmanager
+
 [masakari-api:children]
 control
 

ansible/inventory/multinode

@@ -718,6 +718,9 @@ monitoring
 [prometheus-libvirt-exporter:children]
 compute
 
+[prometheus-msteams:children]
+prometheus-alertmanager
+
 [masakari-api:children]
 control
 

ansible/roles/prometheus/defaults/main.yml

@@ -108,6 +108,14 @@ prometheus_services:
     image: "{{ prometheus_libvirt_exporter_image_full }}"
     volumes: "{{ prometheus_libvirt_exporter_default_volumes + prometheus_libvirt_exporter_extra_volumes }}"
     dimensions: "{{ prometheus_libvirt_exporter_dimensions }}"
+  prometheus-msteams:
+    container_name: "prometheus_msteams"
+    group: "prometheus-msteams"
+    enabled: "{{ enable_prometheus_msteams | bool }}"
+    environment: "{{ prometheus_msteams_container_proxy }}"
+    image: "{{ prometheus_msteams_image_full }}"
+    volumes: "{{ prometheus_msteams_default_volumes + prometheus_msteams_extra_volumes }}"
+    dimensions: "{{ prometheus_msteams_dimensions }}"
 
 ####################
 # Prometheus Server
@@ -189,6 +197,10 @@ prometheus_libvirt_exporter_image: "{{ docker_registry ~ '/' if docker_registry
 prometheus_libvirt_exporter_tag: "{{ prometheus_tag }}"
 prometheus_libvirt_exporter_image_full: "{{ prometheus_libvirt_exporter_image }}:{{ prometheus_libvirt_exporter_tag }}"
 
+prometheus_msteams_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/prometheus-msteams"
+prometheus_msteams_tag: "{{ prometheus_tag }}"
+prometheus_msteams_image_full: "{{ prometheus_msteams_image }}:{{ prometheus_msteams_tag }}"
+
 prometheus_server_dimensions: "{{ default_container_dimensions }}"
 prometheus_haproxy_exporter_dimensions: "{{ default_container_dimensions }}"
 prometheus_mysqld_exporter_dimensions: "{{ default_container_dimensions }}"
@@ -200,6 +212,7 @@ prometheus_openstack_exporter_dimensions: "{{ default_container_dimensions }}"
 prometheus_elasticsearch_exporter_dimensions: "{{ default_container_dimensions }}"
 prometheus_blackbox_exporter_dimensions: "{{ default_container_dimensions }}"
 prometheus_libvirt_exporter_dimensions: "{{ default_container_dimensions }}"
+prometheus_msteams_dimensions: "{{ default_container_dimensions }}"
 
 prometheus_server_default_volumes:
   - "{{ node_config_directory }}/prometheus-server/:{{ container_config_directory }}/:ro"
@@ -265,6 +278,11 @@ prometheus_libvirt_exporter_default_volumes:
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "/run/libvirt:/run/libvirt:ro"
+prometheus_msteams_default_volumes:
+  - "{{ node_config_directory }}/prometheus-msteams/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
+  - "kolla_logs:/var/log/kolla/"
 
 prometheus_extra_volumes: "{{ default_extra_volumes }}"
 prometheus_server_extra_volumes: "{{ prometheus_extra_volumes }}"
@@ -278,6 +296,9 @@ prometheus_openstack_exporter_extra_volumes: "{{ prometheus_extra_volumes }}"
 prometheus_elasticsearch_exporter_extra_volumes: "{{ prometheus_extra_volumes }}"
 prometheus_blackbox_exporter_extra_volumes: "{{ prometheus_extra_volumes }}"
 prometheus_libvirt_exporter_extra_volumes: "{{ prometheus_extra_volumes }}"
+prometheus_msteams_extra_volumes: "{{ prometheus_extra_volumes }}"
+
+prometheus_msteams_container_proxy: "{{ container_proxy }}"
 
 prometheus_openstack_exporter_disabled_volume: "{{ '--disable-service.volume' if not enable_cinder | bool else '' }}"
 prometheus_openstack_exporter_disabled_dns: "{{ '--disable-service.dns' if not enable_designate | bool else '' }}"

ansible/roles/prometheus/handlers/main.yml

@@ -165,3 +165,19 @@
     dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
+
+- name: Restart prometheus-msteams container
+  vars:
+    service_name: "prometheus-msteams"
+    service: "{{ prometheus_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    environment: "{{ service.environment }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"

ansible/roles/prometheus/tasks/config.yml

@@ -237,3 +237,37 @@
   when:
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
+
+- name: Copying over prometheus msteams config file
+  vars:
+    service: "{{ prometheus_services['prometheus-msteams'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/prometheus-msteams/msteams.yml"
+  become: true
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+  with_first_found:
+    - "{{ node_custom_config }}/prometheus/{{ inventory_hostname }}/prometheus-msteams.yml"
+    - "{{ node_custom_config }}/prometheus/prometheus-msteams.yml"
+    - "{{ role_path }}/templates/prometheus-msteams.yml.j2"
+  notify:
+    - Restart prometheus-msteams container
+
+- name: Copying over prometheus msteams template file
+  vars:
+    service: "{{ prometheus_services['prometheus-msteams'] }}"
+  copy:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/prometheus-msteams/msteams.tmpl"
+  become: true
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+  with_first_found:
+    - "{{ node_custom_config }}/prometheus/{{ inventory_hostname }}/prometheus-msteams.tmpl"
+    - "{{ node_custom_config }}/prometheus/prometheus-msteams.tmpl"
+    - "{{ role_path }}/templates/prometheus-msteams.tmpl"
+  notify:
+    - Restart prometheus-msteams container

ansible/roles/prometheus/tasks/precheck.yml

@@ -164,3 +164,17 @@
     - enable_prometheus_libvirt_exporter | bool
   with_items:
     - "{{ prometheus_libvirt_exporter_port }}"
+
+- name: Checking free ports for Prometheus msteams
+  wait_for:
+    host: "{{ 'api' | kolla_address }}"
+    port: "{{ item }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - container_facts['prometheus_msteams'] is not defined
+    - inventory_hostname in groups['prometheus-msteams']
+    - enable_prometheus_msteams | bool
+  with_items:
+    - "{{ prometheus_msteams_port }}"

ansible/roles/prometheus/templates/prometheus-alertmanager.yml.j2

@@ -6,6 +6,10 @@ route:
   group_wait: 10s
   group_interval: 5m
   repeat_interval: 3h
+{% if enable_prometheus_msteams | bool %}
+  routes:
+    - receiver: 'prometheus-msteams'
+{% endif %}
 receivers:
   - name: default-receiver
 {% if enable_vitrage | bool and enable_vitrage_prometheus_datasource | bool %}
@@ -17,5 +21,11 @@ receivers:
             username: '{{ keystone_admin_user }}'
             password: '{{ keystone_admin_password }}'
 {% endif %}
+{% if enable_prometheus_msteams | bool %}
+  - name: 'prometheus-msteams'
+    webhook_configs:
+    - send_resolved: true
+      url: 'http://localhost:{{ prometheus_msteams_port }}/alertmanager'
+{% endif %}
 templates:
   - '/etc/prometheus/*.tmpl'

ansible/roles/prometheus/templates/prometheus-msteams.json.j2

@@ -0,0 +1,24 @@
+{
+    "command": "/opt/prometheus-msteams -http-addr localhost:{{ prometheus_msteams_port }} -config-file /etc/msteams/msteams.yml -template-file /etc/msteams/msteams.tmpl",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/msteams.yml",
+            "dest": "/etc/msteams/msteams.yml",
+            "owner": "prometheus",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/msteams.tmpl",
+            "dest": "/etc/msteams/msteams.tmpl",
+            "owner": "prometheus",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/prometheus",
+            "owner": "prometheus:kolla",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/prometheus/templates/prometheus-msteams.tmpl

@@ -0,0 +1,50 @@
+{{ define "teams.card" }}
+{
+  "@type": "MessageCard",
+  "@context": "http://schema.org/extensions",
+  "themeColor": "{{- if eq .Status "resolved" -}}2DC72D
+                 {{- else if eq .Status "firing" -}}
+                    {{- if eq .CommonLabels.severity "critical" -}}8C1A1A
+                    {{- else if eq .CommonLabels.severity "warning" -}}FFA500
+                    {{- else -}}808080{{- end -}}
+                 {{- else -}}808080{{- end -}}",
+  "summary": "{{- if eq .CommonAnnotations.summary "" -}}
+                  {{- if eq .CommonAnnotations.message "" -}}
+                    {{- if eq .CommonLabels.alertname "" -}}
+                      Prometheus Alert
+                    {{- else -}}
+                      {{- .CommonLabels.alertname -}}
+                    {{- end -}}
+                  {{- else -}}
+                    {{- .CommonAnnotations.message -}}
+                  {{- end -}}
+              {{- else -}}
+                  {{- .CommonAnnotations.summary -}}
+              {{- end -}}",
+  "title": "Prometheus Alert ({{ .Status | title }})",
+  "sections": [ {{$externalUrl := .ExternalURL}}
+  {{- range $index, $alert := .Alerts }}{{- if $index }},{{- end }}
+    {
+      "activityTitle": "[{{ $alert.Annotations.description }}]({{ $externalUrl }})",
+      "facts": [
+        {{- range $key, $value := $alert.Annotations }}
+        {
+          {{- if ne $key "description" -}}
+            "name": "{{ $key }}",
+            "value": "{{ $value }}"
+          {{- end -}}
+        },
+        {{- end -}}
+        {{$c := counter}}{{ range $key, $value := $alert.Labels }}{{if call $c}},{{ end }}
+        {
+          "name": "{{ $key }}",
+          "value": "{{ $value }}"
+        }
+        {{- end }}
+      ],
+      "markdown": true
+    }
+    {{- end }}
+  ]
+}
+{{ end }}

ansible/roles/prometheus/templates/prometheus-msteams.yml.j2

@@ -0,0 +1,2 @@
+connectors:
+  - alertmanager: "{{ prometheus_msteams_webhook_url }}"

etc/kolla/globals.yml

@@ -718,6 +718,7 @@ workaround_ansible_issue_8743: yes
 #enable_prometheus_blackbox_exporter: "{{ enable_prometheus | bool }}"
 #enable_prometheus_libvirt_exporter: "{{ enable_prometheus | bool and enable_nova | bool and nova_compute_virt_type in ['kvm', 'qemu'] }}"
 #enable_prometheus_etcd_integration: "{{ enable_prometheus | bool and enable_etcd | bool }}"
+#enable_prometheus_msteams: "no"
 
 # The labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager).
 # prometheus_external_labels:

releasenotes/notes/prometheus-msteams-361a2b76300e7921.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds support for deploying ``prometheus-msteams``, which can be used to
+    forward Prometheus Alertmanager notifications to Microsoft Teams. It is
+    enabled by setting ``enable_prometheus_msteams`` to ``true``.

tests/templates/inventory.j2

@@ -771,6 +771,9 @@ monitoring
 [prometheus-libvirt-exporter:children]
 compute
 
+[prometheus-msteams:children]
+prometheus-alertmanager
+
 # NOTE(yoctozepto): In CI we want to test Masakari HA but not of other services,
 # to conserve the resources. Hence, we set Masakari groups to use both
 # primary and secondary while the parent group (control) uses only primary.