Added TMF API 639 Datasource

Change-Id: I48ccc6202738b52e80b64b965fe4993d141061f8

doc/source/contributor/configuration.rst

@@ -29,6 +29,7 @@ Datasources
    nova-config
    prometheus-datasource
    kapacitor-datasource
+   tmfapi639-datasource
 
 Notifiers
 ---------

doc/source/contributor/tmfapi639-datasource.rst

@@ -0,0 +1,37 @@
+TMF API 639 - Vitrage
+=====================
+
+This datasource loads to Vitrage topologies exposed in TMF API 639 Resource Inventory Management.
+https://www.tmforum.org/resources/specification/tmf639-resource-inventory-management-api-rest-specification-r17-0-1/
+
+The fields used to define the topology will be:
+- id
+- name
+- @type
+- resourceRelationship : [resource: id]
+
+Configuration
+-------------
+
+
+1. Create file ``tmfapi639_conf.yaml`` on your vitrage folder (generally: /etc/vitrage/) according to the following template:
+
+
+      | -endpoint:
+      |     snapshot: URL CONTAINING COMPLETE TOPOLOGY
+      |     update: OPTIONAL URL CONTAINING NOTIFICATIONS FOR TOPOLOGY CHANGES 
+
+You may allow as many endpoints as you desire.
+
+
+2. Add tmfapi639 to list of datasources in ``/etc/vitrage/vitrage.conf``
+
+.. code::
+
+    [datasources]
+    types = ...,tmfapi639,...
+
+
+3. Restart vitrage service in devstack/openstack
+
+**Warning:** due to limitations on TMF API definition, topology changes will require all parents all the way to the root to be defined in order to be correctly represented.

releasenotes/notes/tmfapi639_datasource-bcae9474430ae5a0.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - New TMF API 639 datasource added capable of both handling
+    topology snapshots and further updates. All described within
+    the TMF's API 639 specification.

vitrage/datasources/tmfapi639/__init__.py

@@ -0,0 +1,51 @@
+# Copyright 2020
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from vitrage.common.constants import DatasourceOpts as DSOpts
+from vitrage.common.constants import UpdateMethod
+
+TMFAPI639_DATASOURCE = 'tmfapi639'
+
+OPTS = [
+    cfg.StrOpt(DSOpts.TRANSFORMER,
+               default='vitrage.datasources.tmfapi639.transformer.'
+                       'TmfApi639Transformer',
+               help='TmfApi639 transformer class path',
+               required=True),
+    cfg.StrOpt(DSOpts.DRIVER,
+               default='vitrage.datasources.tmfapi639.driver.'
+                       'TmfApi639Driver',
+               help='TmfApi639 driver class path',
+               required=True),
+    cfg.StrOpt(DSOpts.UPDATE_METHOD,
+               default=UpdateMethod.PULL,
+               help='None: updates only via Vitrage periodic snapshots.'
+                    'Pull: updates periodically.'
+                    'Push: updates by getting notifications from the'
+                    ' datasource itself.',
+               required=True),
+    cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
+               default=30,
+               min=10,
+               help='interval in seconds between checking changes in the'
+                    ' TmfApi 639 interface'),
+    cfg.StrOpt(DSOpts.CONFIG_FILE, default='/etc/vitrage/tmfapi639_conf.yaml',
+               help='TmfApi639 configuration file'
+               )]
+
+
+class TmfApi639Fields(object):
+    TYPE = 'type'
+    ID = 'id'

vitrage/datasources/tmfapi639/config.py

@@ -0,0 +1,51 @@
+# Copyright 2020
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from oslo_log import log
+
+from vitrage.common.constants import DatasourceOpts as DSOpts
+from vitrage.utils import file as file_utils
+
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
+
+
+class TmfApi639Config(object):
+    def __init__(self):
+        try:
+            tmfapi639_config_file = CONF.tmfapi639[DSOpts.CONFIG_FILE]
+            tmfapi639_config = file_utils.load_yaml_file(tmfapi639_config_file)
+            self.endpoints = self._create_mapping(tmfapi639_config)
+        except Exception as e:
+            LOG.error("Failed initialization: " + str(e))
+            self.endpoints = []
+
+    @staticmethod
+    def _create_mapping(config):
+        """Read URL list from config dictionary"""
+        LOG.debug(config)
+        endpoint_list = []
+        # Tuple list containing either 1 or 2 elements (Endpoint and updates)
+        for e in config:
+            snapshot_url = e["endpoint"]["snapshot"]
+            update_url = ""
+            if "update" in e["endpoint"]:
+                update_url = e["endpoint"]["update"]
+            if update_url != "":
+                endpoint_list.append((snapshot_url, update_url))
+            else:
+                endpoint_list.append(snapshot_url)
+        LOG.info("Finished reading endpoints file")
+        return endpoint_list

vitrage/datasources/tmfapi639/driver.py

@@ -0,0 +1,96 @@
+# Copyright 2020
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log
+
+from vitrage.datasources.driver_base import DriverBase
+from vitrage.datasources.tmfapi639 import TMFAPI639_DATASOURCE
+
+from vitrage.datasources.tmfapi639.config import TmfApi639Config
+
+import json
+import requests
+
+
+LOG = log.getLogger(__name__)
+
+
+class TmfApi639Driver(DriverBase):
+
+    def __init__(self):
+        super(TmfApi639Driver, self).__init__()
+        self.config = TmfApi639Config()
+        self.endpoints = self.config.endpoints
+        self.event_lambda = 0
+
+    @staticmethod
+    def get_event_types():
+        return ['tmfapi639.instance.create',
+                'tmfapi639.instance.update',
+                'tmfapi639.instance.delete']
+
+    def enrich_event(self, event, event_type):
+        pass
+
+    def get_all(self, datasource_action):
+        """Query all entities and send events to the vitrage events queue.
+
+        When done for the first time, send an "end" event to inform it has
+        finished the get_all for the datasource (because it is done
+        asynchronously).
+        """
+        return self.make_pickleable(self._get_all_entities(),
+                                    TMFAPI639_DATASOURCE,
+                                    datasource_action)
+
+    def get_changes(self, datasource_action):
+        """Send an event to the vitrage events queue upon any change."""
+        return self.make_pickleable(self._get_changes_entities(),
+                                    TMFAPI639_DATASOURCE,
+                                    datasource_action)
+
+    def _get_all_entities(self):
+        total = []
+        for pairs in self.endpoints:
+            try:
+                if type(pairs) is tuple:  # Contains an update URL
+                    LOG.info("Connecting to " + pairs[0] +
+                             "with updates in " + pairs[1])
+                    r = requests.get(pairs[0])
+                elif type(pairs) is str:  # Doesn't contain update URL
+                    LOG.info("Connecting to " + pairs)
+                    r = requests.get(pairs)
+                r_dict = json.loads(r.text)
+                total += r_dict
+            except Exception as e:
+                LOG.error("Couldn't establish connection:" + str(e))
+        return total
+
+    def _get_changes_entities(self):  # Called by get changes
+        total = []
+        for pairs in self.endpoints:
+            try:
+                if type(pairs) is tuple:  # Contains an update URL
+                    LOG.info("Connecting to " + pairs[0] +
+                             "with updates in " + pairs[1])
+                    r = requests.get(pairs[1])
+                    r_dict = json.loads(r.text)
+                    for e in r_dict:
+                        if e["eventId"] < self.event_lambda:
+                            continue
+                        total.append(e["event"]["resource"])
+                        self.event_lambda = e["eventId"]
+            except Exception as e:
+                LOG.error("Couldn't establish connection:" + str(e))
+        return total

vitrage/datasources/tmfapi639/transformer.py

@@ -0,0 +1,97 @@
+# Copyright 2020
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log as logging
+
+from vitrage.common.constants import EntityCategory
+from vitrage.common.constants import VertexProperties as VProps
+from vitrage.datasources.resource_transformer_base \
+    import ResourceTransformerBase
+from vitrage.datasources.tmfapi639 import TMFAPI639_DATASOURCE
+from vitrage.datasources import transformer_base
+import vitrage.graph.utils as graph_utils
+
+from datetime import datetime
+
+LOG = logging.getLogger(__name__)
+
+
+class TmfApi639Transformer(ResourceTransformerBase):
+
+    def __init__(self, transformers):
+        super(TmfApi639Transformer, self).__init__(transformers)
+
+    def _create_snapshot_entity_vertex(self, entity_event):
+        return self._create_vertex(entity_event)
+
+    def _create_update_entity_vertex(self, entity_event):
+        return self._create_vertex(entity_event)
+
+    def _create_snapshot_neighbors(self, entity_event):
+        return self._create_tmfapi639_neighbors(entity_event)
+
+    def _create_update_neighbors(self, entity_event):
+        return self._create_tmfapi639_neighbors(entity_event)
+
+    def _create_entity_key(self, entity_event):
+        """the unique key of this entity"""
+        entity_id = entity_event["id"]
+        entity_type = TMFAPI639_DATASOURCE
+        key_fields = self._key_values(entity_type, entity_id)
+        return transformer_base.build_key(key_fields)
+
+    @staticmethod
+    def get_vitrage_type():
+        return TMFAPI639_DATASOURCE
+
+    def _create_vertex(self, entity_event):
+        """Camps used from the received JSON:
+
+        {id, name, @type ,resourceRelationship : [type, resource: id]}
+
+        The TMF 639 API REST Endpoint can contain more information
+        but we only use this one for topology.
+        """
+        sample_timestamp = \
+            datetime.now().strftime(transformer_base.TIMESTAMP_FORMAT)
+        update_timestamp = self._format_update_timestamp(
+            update_timestamp=None,
+            sample_timestamp=sample_timestamp)
+
+        metadata = {
+            VProps.NAME: entity_event["name"],
+        }
+
+        return graph_utils.create_vertex(
+            self._create_entity_key(entity_event),
+            vitrage_category=EntityCategory.RESOURCE,
+            vitrage_type=TMFAPI639_DATASOURCE,
+            vitrage_sample_timestamp=sample_timestamp,
+            entity_id=entity_event["id"],
+            update_timestamp=update_timestamp,
+            entity_state='available',
+            metadata=metadata)
+
+    def _create_tmfapi639_neighbors(self, entity_event):
+        neighbors_list = []
+        for n in entity_event["resourceRelationship"]:
+            # create placeholder vertex
+            neigh = self._create_neighbor(
+                entity_event,
+                n["resource"]["id"],
+                TMFAPI639_DATASOURCE,
+                n["type"],
+                is_entity_source=True)
+            neighbors_list.append(neigh)
+        return neighbors_list

vitrage/tests/unit/datasources/tmfapi639/test_tmfapi639_transformer.py

@@ -0,0 +1,142 @@
+# Copyright 2020
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from testtools import matchers
+
+from vitrage.common.constants import DatasourceAction
+from vitrage.common.constants import DatasourceOpts as DSOpts
+from vitrage.common.constants import DatasourceProperties as DSProps
+from vitrage.common.constants import UpdateMethod
+
+from vitrage.datasources.tmfapi639 import TMFAPI639_DATASOURCE
+from vitrage.datasources.tmfapi639.transformer import TmfApi639Transformer
+from vitrage.datasources import transformer_base
+from vitrage.datasources.transformer_base import TransformerBase
+
+from vitrage.tests.unit.datasources.test_alarm_transformer_base import \
+    BaseAlarmTransformerTest
+
+from datetime import datetime
+from json import loads
+
+LOG = logging.getLogger(__name__)
+
+message = '[{"id":"1","name":"Host-1","@type":"Host",\
+    "resourceRelationship":[{"type":"parent","resource":{"id":"1"}}]},\
+            {"id":"2","name":"Host-2","@type":"Host",\
+            "resourceRelationship":[{"type":"parent","resource":{"id":"1"}}]}]'
+
+
+# noinspection PyProtectedMember
+class TestTmfApi639Transformer(BaseAlarmTransformerTest):
+
+    OPTS = [
+        cfg.StrOpt(DSOpts.UPDATE_METHOD,
+                   default=UpdateMethod.PULL),
+    ]
+
+    # noinspection PyAttributeOutsideInit,PyPep8Naming
+    @classmethod
+    def setUpClass(cls):
+        super(TestTmfApi639Transformer, cls).setUpClass()
+        cls.transformers = {}
+        cls.conf = cfg.ConfigOpts()
+        cls.conf.register_opts(cls.OPTS, group=TMFAPI639_DATASOURCE)
+        cls.transformer = TmfApi639Transformer(cls.transformers)
+        cls.transformers[TMFAPI639_DATASOURCE] = cls.transformer
+
+    # noinspection PyAttributeOutsideInit
+    def setUp(self):
+        super(TestTmfApi639Transformer, self).setUp()
+        # self.entity_type = TMFAPI639_DATASOURCE
+        # self.entity_id = '12345'
+        self.timestamp = datetime.utcnow()
+
+    def test_create_entity_key(self):
+        event = loads(message)[0]
+        self.assertIsNotNone(event)
+
+        transformer = TmfApi639Transformer(self.transformers)
+        observed_key = transformer._create_entity_key(event)
+
+        entity_type = TMFAPI639_DATASOURCE
+        entity_id = event["id"]
+
+        # Test assertions
+        observed_key_fields = observed_key.split(
+            TransformerBase.KEY_SEPARATOR)
+
+        self.assertEqual(entity_type, observed_key_fields[1])
+        self.assertEqual(entity_id, observed_key_fields[2])
+
+    # Transformer tests:
+    # - Vertex creation
+    # - Neighbor link
+
+    def test_topology(self):
+
+        sample_timestamp = \
+            datetime.now().strftime(transformer_base.TIMESTAMP_FORMAT)
+        update_timestamp = TransformerBase._format_update_timestamp(
+            update_timestamp=None,
+            sample_timestamp=sample_timestamp)
+
+        transformer = self.transformers[TMFAPI639_DATASOURCE]
+
+        # Create 1 vertex
+        event1 = loads(message)[0]
+        event1[DSProps.DATASOURCE_ACTION] = DatasourceAction.SNAPSHOT
+        event1[DSProps.SAMPLE_DATE] = update_timestamp
+        self.assertIsNotNone(event1)
+
+        # Create vertex 1
+        wrapper1 = transformer.transform(event1)
+        # Assertion
+        self._validate_base_vertex_props(
+            wrapper1.vertex,
+            event1["name"],
+            TMFAPI639_DATASOURCE
+        )
+
+        # Create 2nd vertex
+        event2 = loads(message)[1]
+        event2[DSProps.DATASOURCE_ACTION] = DatasourceAction.SNAPSHOT
+        event2[DSProps.SAMPLE_DATE] = update_timestamp
+        self.assertIsNotNone(event2)
+
+        # Create vertex 2
+        wrapper2 = transformer.transform(event2)
+        # Assertion
+        self._validate_base_vertex_props(
+            wrapper2.vertex,
+            event2["name"],
+            TMFAPI639_DATASOURCE
+        )
+
+        # Test whether they are linked
+        self.assertThat(wrapper2.neighbors, matchers.HasLength(1))
+
+        parent_id = transformer._create_entity_key(event1)
+        parent_uuid = \
+            transformer.uuid_from_deprecated_vitrage_id(parent_id)
+
+        child_id = transformer._create_entity_key(event2)
+        child_uuid = \
+            transformer.uuid_from_deprecated_vitrage_id(child_id)
+
+        self.assertEqual(wrapper2.neighbors[0].edge.source_id, child_uuid)
+        self.assertEqual(wrapper2.neighbors[0].edge.target_id, parent_uuid)