Add Monasca datasource

Introduces Monasca datasource enabling Monasca alerts in Vitrage
Entity Graph.

Change-Id: Ib3549c404c84e549f66e6057368c3f227bfe806a
Signed-off-by: Bartosz Zurkowski <b.zurkowski@samsung.com>

devstack/plugin.sh

@@ -189,6 +189,11 @@ function configure_vitrage {
         disable_vitrage_datasource trove.instance trove.cluster
     fi
 
+    # remove monasca vitrage datasource if nagios datasource not installed
+    if ! is_service_enabled monasca; then
+        disable_vitrage_datasource monasca
+    fi
+
     # remove nagios vitrage datasource if nagios datasource not installed
     if [[ "$VITRAGE_USE_NAGIOS" == "False" ]]; then
         disable_vitrage_datasource nagios
@@ -342,6 +347,12 @@ function modify_heat_global_index_policy_rule {
     fi
 }
 
+function add_monasca_rule_to_vitrage {
+    if is_service_enabled monasca; then
+       get_or_add_user_project_role "monasca-read-only-user" "vitrage" "admin"
+    fi
+}
+
 # This is the main for plugin.sh
 if is_service_enabled vitrage; then
     if [[ "$1" == "stack" && "$2" == "pre-install" ]]; then
@@ -361,6 +372,9 @@ if is_service_enabled vitrage; then
         modify_heat_global_index_policy_rule
         # Tidy base for vitrage
         init_vitrage
+        # add monasca-read-only-user role to vitrage user so
+        # vitrage user can read monasca alarms
+        add_monasca_rule_to_vitrage
         # Start the services
         start_vitrage
     fi

devstack/settings

@@ -36,7 +36,7 @@ VITRAGE_USE_STATIC=$(trueorfalse False VITRAGE_USE_STATIC)
 VITRAGE_USE_DOCTOR=$(trueorfalse False VITRAGE_USE_DOCTOR)
 VITRAGE_USE_PROMETHEUS=$(trueorfalse False VITRAGE_USE_PROMETHEUS)
 
-VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static,aodh,cinder.volume,neutron.network,neutron.port,heat.stack,doctor,prometheus,trove.instance,trove.cluster}
+VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static,aodh,cinder.volume,neutron.network,neutron.port,heat.stack,doctor,prometheus,trove.instance,trove.cluster,monasca}
 
 # for now dont use pip install for the client
 LIBS_FROM_GIT=python-vitrageclient

doc/requirements.txt

@@ -3,6 +3,7 @@
 # process, which may cause wedges in the gate later.
 oslo.config>=5.2.0 # Apache-2.0
 openstackdocstheme>=1.24.0 # Apache-2.0
-sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD
+sphinx!=1.6.6,!=1.6.7,>=1.6.2,<2.0.0;python_version=='2.7'  # BSD
+sphinx!=1.6.6,!=1.6.7,>=1.6.2;python_version>='3.4'  # BSD
 reno>=2.7.0 # Apache-2.0
 docutils>=0.11  # OSI-Approved Open Source, Public Domain

etc/vitrage/datasources_values/monasca.yaml

@@ -0,0 +1,17 @@
+category: ALARM
+values:
+  - aggregated values:
+      priority: 30
+      original values:
+        - name: ALARM
+          operational_value: CRITICAL
+  - aggregated values:
+      priority: 20
+      original values:
+        - name: UNDETERMINED
+          operational_value: N/A
+  - aggregated values:
+      priority: 10
+      original values:
+        - name: OK
+          operational_value: OK

lower-constraints.txt

@@ -50,7 +50,8 @@ msgpack==0.5.6
 munch==2.2.0
 netaddr==0.7.19
 netifaces==0.10.6
-networkx==2.0
+networkx==2.3;python_version>='3.4'
+networkx==2.0;python_version<'3.0'
 openstacksdk==0.12.0
 os-client-config==1.29.0
 os-service-types==1.2.0
@@ -98,6 +99,7 @@ python-heatclient==1.14.0
 python-keystoneclient==3.15.0
 python-mimeparse==1.6.0
 python-mistralclient==3.3.0
+python-monascaclient==1.7.1
 python-neutronclient==6.7.0
 python-novaclient==10.1.0
 python-openstackclient==3.12.0

releasenotes/notes/monasca-datasource-9ca61922ef14c2a8.yaml

@@ -0,0 +1,20 @@
+---
+features:
+  - A new ``Monasca Datasource`` has been introduced to include Monasca alarms
+    in Vitrage Entity Graph. Monasca is Monitoring as a Service solution
+    offering centralized monitoring sink for metrics gathered by Monasca Agents
+    at many infrastructure levels. Moreover it provides alarm management API
+    that enables defining alarms based on collected metrics.
+
+    This change is the first stage of integration with Monasca. At this point,
+    Monasca entities are extracted using PULL approach, based on periodical
+    snapshot-query to Monasca Alarm API for the current list of alarm entities.
+    In the future, PUSH approach based on Monasca notifications will be
+    implemented.
+
+    Current implementation requires that the metrics associated with the given
+    alarm contain information about resource type and ID - required for
+    associating alarms with entities in Vitrage Entity Graph. This additional
+    information should be included in the form of metric dimensions, precisely
+    ``resource_type`` and ``resource_id``. Dimensions can be defined in Monasca
+    agent configuration.

requirements.txt

@@ -15,11 +15,13 @@ python-novaclient>=10.1.0 # Apache-2.0
 python-heatclient>=1.14.0 # Apache-2.0
 python-mistralclient>=3.3.0 # Apache-2.0
 python-openstackclient>=3.12.0 # Apache-2.0
+python-monascaclient>=1.7.1 # Apache-2.0
 python-troveclient>=2.2.0 # Apache-2.0
 python-zaqarclient >=1.2.0
 gnocchiclient>=3.3.1 # Apache-2.0
 pyzabbix>=0.7.4 # LGPL
-networkx>=2.0 # BSD
+networkx<2.3,>=2.0;python_version<'3.0'  # BSD
+networkx>=2.3;python_version>='3.4'  # BSD
 oslo.config>=5.2.0 # Apache-2.0
 oslo.context>=2.20.0 # Apache-2.0
 oslo.db>=4.35.0 # Apache-2.0

vitrage/datasources/monasca/__init__.py

@@ -0,0 +1,45 @@
+# Copyright 2018 Samsung Electronics
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from vitrage.common.constants import DatasourceOpts as DSOpts
+from vitrage.common.constants import UpdateMethod
+
+MONASCA_DATASOURCE = 'monasca'
+
+OPTS = [
+    cfg.StrOpt(DSOpts.TRANSFORMER,
+               default='vitrage.datasources.monasca.transformer.'
+                       'MonascaTransformer',
+               help='Monasca transformer class path.',
+               required=True),
+    cfg.StrOpt(DSOpts.DRIVER,
+               default='vitrage.datasources.monasca.driver.'
+                       'MonascaDriver',
+               help='Monasca driver class path.',
+               required=True),
+    cfg.StrOpt(DSOpts.UPDATE_METHOD,
+               default=UpdateMethod.PULL,
+               help='None: updates only via Vitrage periodic snapshots.'
+                    'Pull: updates periodically.'
+                    'Push: updates by getting notifications from the'
+                    ' datasource itself.',
+               required=True),
+    cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
+               default=30,
+               min=10,
+               help='Interval in seconds between checking changes in Monasca'
+                    'datasource.'),
+    ]

vitrage/datasources/monasca/driver.py

@@ -0,0 +1,76 @@
+# Copyright 2018 Samsung Electronics
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_log import log
+
+from vitrage.datasources.alarm_driver_base import AlarmDriverBase
+from vitrage.datasources.monasca import MONASCA_DATASOURCE
+from vitrage.datasources.monasca.properties import (
+    MonascaAlarmStatuses as MAlarmStatuses)
+from vitrage.datasources.monasca.properties import MonascaProperties as MProps
+from vitrage.datasources.transformer_base import extract_field_value
+from vitrage import os_clients
+
+LOG = log.getLogger(__name__)
+
+
+class MonascaDriver(AlarmDriverBase):
+
+    def __init__(self, conf):
+        super(MonascaDriver, self).__init__()
+        self.conf = conf
+        self.__client = None
+
+    @property
+    def client(self):
+        if not self.__client:
+            self.__client = os_clients.monasca_client(self.conf)
+        return self.__client
+
+    def _vitrage_type(self):
+        return MONASCA_DATASOURCE
+
+    def _alarm_key(self, alarm):
+        return alarm[MProps.ID]
+
+    def _get_alarms(self):
+        try:
+            return self.client.alarms.list()
+        except Exception:
+            LOG.exception("Failed to fetch Monasca alarms.")
+        return []
+
+    def _enrich_alarms(self, alarms):
+        for alarm in alarms:
+            alarm[MProps.RESOURCE_TYPE] = extract_field_value(
+                alarm, 'metrics', 0, 'dimensions', 'resource_type')
+            alarm[MProps.RESOURCE_ID] = extract_field_value(
+                alarm, 'metrics', 0, 'dimensions', 'resource_id')
+
+    def _is_erroneous(self, alarm):
+        return alarm and alarm[MProps.STATUS] == MAlarmStatuses.ALARM
+
+    def _status_changed(self, new_alarm, old_alarm):
+        return new_alarm and old_alarm and \
+            new_alarm[MProps.STATUS] != old_alarm[MProps.STATUS]
+
+    def _is_valid(self, alarm):
+        return alarm and \
+            alarm[MProps.RESOURCE_TYPE] is not None and \
+            alarm[MProps.RESOURCE_ID] is not None
+
+    @staticmethod
+    def should_delete_outdated_entities():
+        return True

vitrage/datasources/monasca/properties.py

@@ -0,0 +1,30 @@
+# Copyright 2018 Samsung Electronics
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+class MonascaProperties(object):
+
+    ID = 'id'
+    NAME = 'name'
+    RESOURCE_TYPE = 'resource_type'
+    RESOURCE_ID = 'resource_id'
+    STATUS = 'state'
+    UPDATE_TIMESTAMP = 'updated_timestamp'
+
+
+class MonascaAlarmStatuses(object):
+
+    OK = 'OK'
+    ALARM = 'ALARM'

vitrage/datasources/monasca/transformer.py

@@ -0,0 +1,85 @@
+# Copyright 2018 Samsung Electronics
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from vitrage.common.constants import DatasourceProperties as DSProps
+from vitrage.common.constants import EdgeLabel
+from vitrage.common.constants import EntityCategory
+from vitrage.common.constants import VertexProperties as VProps
+from vitrage.datasources.alarm_transformer_base import AlarmTransformerBase
+from vitrage.datasources.monasca import MONASCA_DATASOURCE
+from vitrage.datasources.monasca.properties import (
+    MonascaAlarmStatuses as MAlarmStatuses)
+from vitrage.datasources.monasca.properties import MonascaProperties as MProps
+from vitrage.datasources import transformer_base as tbase
+from vitrage.datasources.transformer_base import extract_field_value
+import vitrage.graph.utils as graph_utils
+
+
+class MonascaTransformer(AlarmTransformerBase):
+
+    def _create_snapshot_entity_vertex(self, entity_event):
+        return self._create_vertex(entity_event)
+
+    def _create_update_entity_vertex(self, entity_event):
+        return self._create_vertex(entity_event)
+
+    def _create_vertex(self, entity_event):
+        update_timestamp = entity_event[MProps.UPDATE_TIMESTAMP]
+        sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
+
+        name = extract_field_value(
+            entity_event, 'alarm_definition', 'name')
+
+        metadata = {
+            VProps.NAME: name,
+            VProps.VITRAGE_RESOURCE_ID: entity_event[MProps.RESOURCE_ID],
+            VProps.VITRAGE_RESOURCE_TYPE: entity_event[MProps.RESOURCE_TYPE],
+            VProps.SEVERITY: entity_event[MProps.STATUS]
+        }
+
+        return graph_utils.create_vertex(
+            self._create_entity_key(entity_event),
+            vitrage_category=EntityCategory.ALARM,
+            vitrage_type=MONASCA_DATASOURCE,
+            vitrage_sample_timestamp=sample_timestamp,
+            entity_id=entity_event[MProps.ID],
+            entity_state=self._get_alarm_state(entity_event),
+            update_timestamp=update_timestamp,
+            metadata=metadata)
+
+    def _create_snapshot_neighbors(self, entity_event):
+        return self._create_monasca_neighbors(entity_event)
+
+    def _create_update_neighbors(self, entity_event):
+        return self._create_monasca_neighbors(entity_event)
+
+    def _create_monasca_neighbors(self, entity_event):
+        return [self._create_neighbor(
+            entity_event,
+            entity_event[MProps.RESOURCE_ID],
+            entity_event[MProps.RESOURCE_TYPE],
+            EdgeLabel.ON,
+            neighbor_category=EntityCategory.RESOURCE)]
+
+    def _ok_status(self, entity_event):
+        return entity_event[MProps.STATUS] == MAlarmStatuses.OK
+
+    def _create_entity_key(self, entity_event):
+        entity_id = entity_event[MProps.ID]
+        key_fields = self._key_values(MONASCA_DATASOURCE, entity_id)
+        return tbase.build_key(key_fields)
+
+    def get_vitrage_type(self):
+        return MONASCA_DATASOURCE

vitrage/os_clients.py

@@ -18,6 +18,7 @@ from oslo_utils import importutils as utils
 
 from vitrage import keystone_client
 
+
 LOG = log.getLogger(__name__)
 
 OPTS = [
@@ -30,6 +31,7 @@ OPTS = [
     cfg.StrOpt('mistral_version', default='2', help='Mistral version'),
     cfg.StrOpt('gnocchi_version', default='1', help='Gnocchi version'),
     cfg.StrOpt('trove_version', default='1', help='Trove version'),
+    cfg.StrOpt('monasca_version', default='2_0', help='Monasca version'),
     cfg.BoolOpt('use_nova_versioned_notifications',
                 default=True,
                 help='Indicates whether to use Nova versioned notifications.'
@@ -49,7 +51,8 @@ _client_modules = {
     'heat': 'heatclient.client',
     'mistral': 'mistralclient.api.v2.client',
     'gnocchi': 'gnocchiclient.v1.client',
-    'trove': 'troveclient.v1.client'
+    'trove': 'troveclient.v1.client',
+    'monasca': 'monascaclient.client'
 }
 
 
@@ -205,3 +208,22 @@ def zaqar_client(conf):
         return client
     except Exception:
         LOG.exception('Create Zaqar client - Got Exception.')
+
+
+def monasca_client(conf):
+    """Get an instance of Monasca client"""
+    try:
+        mon_client = driver_module('monasca')
+
+        session = keystone_client.get_session(conf)
+        endpoint = session.get_endpoint(service_type='monitoring',
+                                        interface='publicURL')
+        client = mon_client.Client(
+            api_version=conf.monasca_version,
+            session=session,
+            endpoint=endpoint
+        )
+        LOG.info('Monasca client created')
+        return client
+    except Exception:
+        LOG.exception('Create Monasca client - Got Exception.')