From 2b1f91d45d8ab6e2362dcd79958a515758fc5edc Mon Sep 17 00:00:00 2001 From: Muhamad Najjar Date: Wed, 13 Feb 2019 08:45:46 +0000 Subject: [PATCH] Prometheus get_all support Change-Id: I991951c150b369266fc1fa5a5811b28ab54ac7b0 Story: 2004988 Task: 29457 --- .../contributor/prometheus-datasource.rst | 13 ++- ...metheus-enhancement-b9dc2ef5705d1000.yaml} | 1 + vitrage/datasources/prometheus/__init__.py | 7 ++ vitrage/datasources/prometheus/driver.py | 81 +++++++++++++++---- vitrage/datasources/prometheus/properties.py | 8 ++ vitrage/datasources/prometheus/transformer.py | 13 ++- .../prometheus/test_prometheus_driver.py | 14 ++-- 7 files changed, 111 insertions(+), 26 deletions(-) rename releasenotes/notes/{prometheus-configuration-b9dc2ef5705d1000.yaml => prometheus-enhancement-b9dc2ef5705d1000.yaml} (75%) diff --git a/doc/source/contributor/prometheus-datasource.rst b/doc/source/contributor/prometheus-datasource.rst index cd1e2bba2..0ac6fa73c 100644 --- a/doc/source/contributor/prometheus-datasource.rst +++ b/doc/source/contributor/prometheus-datasource.rst @@ -46,6 +46,16 @@ In ``/etc/vitrage/vitrage.conf`` add ``prometheus`` to the list of active dataso types = nova.host,nova.instance,nova.zone,aodh,static,cinder.volume,neutron.network,neutron.port,prometheus +Add the http url of Prometheus Alertmanager api for alerts and the receiver name +from the previous step under ``[prometheus]`` section:: + + [prometheus] + alertmanager_url = http://localhost:9093/api/v2/alerts + receiver = vitrage + + +Note: Both v1 and v2 Alertmanager apis are supported + Step 3 - Map Prometheus alerts to Vitrage resources --------------------------------------------------- @@ -53,7 +63,8 @@ Step 3 - Map Prometheus alerts to Vitrage resources A configuration file that maps the Prometheus alert labels to a corresponding Vitrage resource with specific properties (id or other unique properties). The mapping will most likely be defined by the alert name and other fields. -Set the location of the alerts mapping in ``/etc/vitrage/vitrage.conf`` :: +Set the location of the alerts mapping in ``/etc/vitrage/vitrage.conf`` +under ``[prometheus]`` section:: [prometheus] config_file = /path/to/alert/mapping diff --git a/releasenotes/notes/prometheus-configuration-b9dc2ef5705d1000.yaml b/releasenotes/notes/prometheus-enhancement-b9dc2ef5705d1000.yaml similarity index 75% rename from releasenotes/notes/prometheus-configuration-b9dc2ef5705d1000.yaml rename to releasenotes/notes/prometheus-enhancement-b9dc2ef5705d1000.yaml index b3e7fbfa8..c05cd141c 100644 --- a/releasenotes/notes/prometheus-configuration-b9dc2ef5705d1000.yaml +++ b/releasenotes/notes/prometheus-enhancement-b9dc2ef5705d1000.yaml @@ -3,3 +3,4 @@ features: - Added support for a yaml configuration file that maps the Prometheus alert labels to a corresponding Vitrage resource with specific properties (id or other unique properties). + - Added support for get_all alerts from Prometheus Alertmanager. diff --git a/vitrage/datasources/prometheus/__init__.py b/vitrage/datasources/prometheus/__init__.py index f96b788a1..5f45427fd 100644 --- a/vitrage/datasources/prometheus/__init__.py +++ b/vitrage/datasources/prometheus/__init__.py @@ -15,6 +15,8 @@ from oslo_config import cfg from vitrage.common.constants import DatasourceOpts as DSOpts from vitrage.common.constants import UpdateMethod +from vitrage.datasources.prometheus.properties \ + import PrometheusGetAllProperties as PGAProps PROMETHEUS_DATASOURCE = 'prometheus' @@ -38,4 +40,9 @@ OPTS = [ required=True), cfg.StrOpt(DSOpts.CONFIG_FILE, default='/etc/vitrage/prometheus_conf.yaml', help='Prometheus configuration file'), + cfg.StrOpt(PGAProps.ALERTMANAGER_URL, + help='Prometheus Alertmanager http api url to get alerts'), + cfg.StrOpt(PGAProps.RECEIVER, + help='Receiver configured in Prometheus Alertmanager to send ' + 'alerts to Vitrage'), ] diff --git a/vitrage/datasources/prometheus/driver.py b/vitrage/datasources/prometheus/driver.py index 568b470b1..1b7811a2c 100644 --- a/vitrage/datasources/prometheus/driver.py +++ b/vitrage/datasources/prometheus/driver.py @@ -11,11 +11,13 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import json import socket from collections import namedtuple from ipaddress import ip_address from oslo_log import log +import requests import six import six.moves.urllib.parse as urlparse @@ -37,6 +39,8 @@ from vitrage.datasources.prometheus.properties \ import PrometheusConfigFileProperties as PCFProps from vitrage.datasources.prometheus.properties \ import PrometheusDatasourceProperties as PDProps +from vitrage.datasources.prometheus.properties \ + import PrometheusGetAllProperties as PGAProps from vitrage.datasources.prometheus.properties \ import PrometheusProperties as PProps from vitrage import os_clients @@ -163,8 +167,44 @@ class PrometheusDriver(AlarmDriverBase): old_alarm.get(PAlertProps.STATUS) def _get_all_alarms(self): + alertmanager_url = self.conf.prometheus.alertmanager_url + receiver = self.conf.prometheus.receiver + if not alertmanager_url: + LOG.warning('Alertmanager url is not defined') + return [] + + if not receiver: + LOG.warning('Receiver is not defined') + return [] + + payload = {PGAProps.ACTIVE: 'true', + PGAProps.RECEIVER: receiver} + + session = requests.Session() + + response = session.get(alertmanager_url, + params=payload) + + if response.status_code == requests.codes.ok: + if 'v1' in alertmanager_url: + alerts = json.loads(response.text)[PGAProps.DATA] + else: + alerts = json.loads(response.text) + self._modify_alert_status(alerts) + alarms = self._enrich_alerts(alerts, PROMETHEUS_EVENT_TYPE) + return alarms + else: + LOG.error('Failed to get Alertmanager data. Response code: %s', + response.status_code) return [] + @staticmethod + def _modify_alert_status(alerts): + for alert in alerts: + if alert.get(PAlertProps.STATUS).get(PGAProps.STATE) == \ + PGAProps.ACTIVE: + alert[PAlertProps.STATUS] = PAlertStatus.FIRING + def _get_changed_alarms(self): return [] @@ -222,27 +262,38 @@ class PrometheusDriver(AlarmDriverBase): alarms = [] details = event.get(EProps.DETAILS) if details: - for alert in details.get(PProps.ALERTS, []): - alert[DSProps.EVENT_TYPE] = event_type - vitrage_entity_unique_props = \ - self._calculate_vitrage_entity_unique_props(alert) - - alert[PDProps.ENTITY_UNIQUE_PROPS] = \ - vitrage_entity_unique_props - old_alarm = self._old_alarm(alert) - alert = self._filter_and_cache_alarm( - alert, old_alarm, - self._filter_get_erroneous, - get_alarm_update_time(alert)) - - if alert: - alarms.append(alert) + alarms = self._enrich_alerts(details.get(PProps.ALERTS, []), + event_type) LOG.debug('Enriched event. Created alert events: %s', str(alarms)) return self.make_pickleable(alarms, PROMETHEUS_DATASOURCE, DatasourceAction.UPDATE) + def _enrich_alerts(self, alerts, event_type): + return [self._enrich_alert(alert, event_type) for alert in alerts] + + def _enrich_alert(self, alert, event_type): + """Enrich prometheus alert. + + Adding fields to prometheus alert in order to map it to vitrage entity. + + :param alert: Prometheus alert + :param event_type: The type of the event. Always 'prometheus.alert'. + :return: Enriched prometheus alert + """ + alert[DSProps.EVENT_TYPE] = event_type + vitrage_entity_unique_props = \ + self._calculate_vitrage_entity_unique_props(alert) + alert[PDProps.ENTITY_UNIQUE_PROPS] = \ + vitrage_entity_unique_props + old_alarm = self._old_alarm(alert) + alert = self._filter_and_cache_alarm( + alert, old_alarm, + self._filter_get_erroneous, + get_alarm_update_time(alert)) + return alert + def _calculate_vitrage_entity_unique_props(self, alert): """Build a vitrage entity unique props. diff --git a/vitrage/datasources/prometheus/properties.py b/vitrage/datasources/prometheus/properties.py index ead1348dd..68cc8728f 100644 --- a/vitrage/datasources/prometheus/properties.py +++ b/vitrage/datasources/prometheus/properties.py @@ -47,6 +47,14 @@ class PrometheusDatasourceProperties(object): ENTITY_UNIQUE_PROPS = 'vitrage_entity_unique_props' +class PrometheusGetAllProperties(object): + ALERTMANAGER_URL = 'alertmanager_url' + RECEIVER = 'receiver' + STATE = 'state' + ACTIVE = 'active' + DATA = 'data' + + def get_alarm_update_time(alarm): if PrometheusAlertStatus.FIRING == \ alarm.get(PrometheusAlertProperties.STATUS): diff --git a/vitrage/datasources/prometheus/transformer.py b/vitrage/datasources/prometheus/transformer.py index cb900d098..fa5d689cb 100644 --- a/vitrage/datasources/prometheus/transformer.py +++ b/vitrage/datasources/prometheus/transformer.py @@ -42,16 +42,17 @@ class PrometheusTransformer(AlarmTransformerBase): super(PrometheusTransformer, self).__init__(transformers, conf) def _create_snapshot_entity_vertex(self, entity_event): - # TODO(iafek): should be implemented - return None + return self._create_vertex(entity_event) def _create_update_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_vertex(self, entity_event): metadata = { VProps.NAME: get_label(entity_event, PAlertLabels.ALERT_NAME), VProps.SEVERITY: get_label(entity_event, PAlertLabels.SEVERITY), PProps.STATUS: entity_event.get(PProps.STATUS), } - return graph_utils.create_vertex( self._create_entity_key(entity_event), vitrage_category=ECategory.ALARM, @@ -62,7 +63,13 @@ class PrometheusTransformer(AlarmTransformerBase): metadata=metadata ) + def _create_snapshot_neighbors(self, entity_event): + return self._create_prometheus_neighbors(entity_event) + def _create_update_neighbors(self, entity_event): + return self._create_prometheus_neighbors(entity_event) + + def _create_prometheus_neighbors(self, entity_event): graph_neighbors = entity_event.get(self.QUERY_RESULT, []) return [self._create_neighbor(entity_event, graph_neighbor[VProps.ID], diff --git a/vitrage/tests/unit/datasources/prometheus/test_prometheus_driver.py b/vitrage/tests/unit/datasources/prometheus/test_prometheus_driver.py index 48151c008..7b5e70f43 100644 --- a/vitrage/tests/unit/datasources/prometheus/test_prometheus_driver.py +++ b/vitrage/tests/unit/datasources/prometheus/test_prometheus_driver.py @@ -201,13 +201,6 @@ class PrometheusDriverTest(base.BaseTest): # Test assertions self._assert_event_equal(created_events, PROMETHEUS_EVENT_TYPE) - @staticmethod - def _generate_event(update_vals=None): - generators = mock_driver.simple_prometheus_alarm_generators( - update_vals=update_vals) - - return mock_driver.generate_sequential_events_list(generators)[0] - def _assert_event_equal(self, created_events, expected_event_type): @@ -223,3 +216,10 @@ class PrometheusDriverTest(base.BaseTest): event = self._generate_event() details = event[EProps.DETAILS] return details[PProps.ALERTS] + + @staticmethod + def _generate_event(update_vals=None): + generators = mock_driver.simple_prometheus_alarm_generators( + update_vals=update_vals) + + return mock_driver.generate_sequential_events_list(generators)[0]