From b801bbcdbe64070d7dcfb28209bd5bda8eaf2e4b Mon Sep 17 00:00:00 2001 From: Dinis Canastro Date: Fri, 3 Apr 2020 19:20:54 +0100 Subject: [PATCH] Added TMF API 639 Datasource Change-Id: I48ccc6202738b52e80b64b965fe4993d141061f8 --- doc/source/contributor/configuration.rst | 1 + .../contributor/tmfapi639-datasource.rst | 37 +++++ ...tmfapi639_datasource-bcae9474430ae5a0.yaml | 5 + vitrage/datasources/tmfapi639/__init__.py | 51 +++++++ vitrage/datasources/tmfapi639/config.py | 51 +++++++ vitrage/datasources/tmfapi639/driver.py | 96 ++++++++++++ vitrage/datasources/tmfapi639/transformer.py | 97 ++++++++++++ .../unit/datasources/tmfapi639/__init__.py | 0 .../tmfapi639/test_tmfapi639_transformer.py | 142 ++++++++++++++++++ 9 files changed, 480 insertions(+) create mode 100644 doc/source/contributor/tmfapi639-datasource.rst create mode 100644 releasenotes/notes/tmfapi639_datasource-bcae9474430ae5a0.yaml create mode 100644 vitrage/datasources/tmfapi639/__init__.py create mode 100644 vitrage/datasources/tmfapi639/config.py create mode 100644 vitrage/datasources/tmfapi639/driver.py create mode 100644 vitrage/datasources/tmfapi639/transformer.py create mode 100644 vitrage/tests/unit/datasources/tmfapi639/__init__.py create mode 100644 vitrage/tests/unit/datasources/tmfapi639/test_tmfapi639_transformer.py diff --git a/doc/source/contributor/configuration.rst b/doc/source/contributor/configuration.rst index 554b0a94a..f5f04b9df 100644 --- a/doc/source/contributor/configuration.rst +++ b/doc/source/contributor/configuration.rst @@ -29,6 +29,7 @@ Datasources nova-config prometheus-datasource kapacitor-datasource + tmfapi639-datasource Notifiers --------- diff --git a/doc/source/contributor/tmfapi639-datasource.rst b/doc/source/contributor/tmfapi639-datasource.rst new file mode 100644 index 000000000..1f38f3b50 --- /dev/null +++ b/doc/source/contributor/tmfapi639-datasource.rst @@ -0,0 +1,37 @@ +TMF API 639 - Vitrage +===================== + +This datasource loads to Vitrage topologies exposed in TMF API 639 Resource Inventory Management. +https://www.tmforum.org/resources/specification/tmf639-resource-inventory-management-api-rest-specification-r17-0-1/ + +The fields used to define the topology will be: +- id +- name +- @type +- resourceRelationship : [resource: id] + +Configuration +------------- + + +1. Create file ``tmfapi639_conf.yaml`` on your vitrage folder (generally: /etc/vitrage/) according to the following template: + + + | -endpoint: + | snapshot: URL CONTAINING COMPLETE TOPOLOGY + | update: OPTIONAL URL CONTAINING NOTIFICATIONS FOR TOPOLOGY CHANGES + +You may allow as many endpoints as you desire. + + +2. Add tmfapi639 to list of datasources in ``/etc/vitrage/vitrage.conf`` + +.. code:: + + [datasources] + types = ...,tmfapi639,... + + +3. Restart vitrage service in devstack/openstack + +**Warning:** due to limitations on TMF API definition, topology changes will require all parents all the way to the root to be defined in order to be correctly represented. diff --git a/releasenotes/notes/tmfapi639_datasource-bcae9474430ae5a0.yaml b/releasenotes/notes/tmfapi639_datasource-bcae9474430ae5a0.yaml new file mode 100644 index 000000000..faa0991b1 --- /dev/null +++ b/releasenotes/notes/tmfapi639_datasource-bcae9474430ae5a0.yaml @@ -0,0 +1,5 @@ +--- +features: + - New TMF API 639 datasource added capable of both handling + topology snapshots and further updates. All described within + the TMF's API 639 specification. \ No newline at end of file diff --git a/vitrage/datasources/tmfapi639/__init__.py b/vitrage/datasources/tmfapi639/__init__.py new file mode 100644 index 000000000..d18a8db8e --- /dev/null +++ b/vitrage/datasources/tmfapi639/__init__.py @@ -0,0 +1,51 @@ +# Copyright 2020 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from vitrage.common.constants import DatasourceOpts as DSOpts +from vitrage.common.constants import UpdateMethod + +TMFAPI639_DATASOURCE = 'tmfapi639' + +OPTS = [ + cfg.StrOpt(DSOpts.TRANSFORMER, + default='vitrage.datasources.tmfapi639.transformer.' + 'TmfApi639Transformer', + help='TmfApi639 transformer class path', + required=True), + cfg.StrOpt(DSOpts.DRIVER, + default='vitrage.datasources.tmfapi639.driver.' + 'TmfApi639Driver', + help='TmfApi639 driver class path', + required=True), + cfg.StrOpt(DSOpts.UPDATE_METHOD, + default=UpdateMethod.PULL, + help='None: updates only via Vitrage periodic snapshots.' + 'Pull: updates periodically.' + 'Push: updates by getting notifications from the' + ' datasource itself.', + required=True), + cfg.IntOpt(DSOpts.CHANGES_INTERVAL, + default=30, + min=10, + help='interval in seconds between checking changes in the' + ' TmfApi 639 interface'), + cfg.StrOpt(DSOpts.CONFIG_FILE, default='/etc/vitrage/tmfapi639_conf.yaml', + help='TmfApi639 configuration file' + )] + + +class TmfApi639Fields(object): + TYPE = 'type' + ID = 'id' diff --git a/vitrage/datasources/tmfapi639/config.py b/vitrage/datasources/tmfapi639/config.py new file mode 100644 index 000000000..81935ecb4 --- /dev/null +++ b/vitrage/datasources/tmfapi639/config.py @@ -0,0 +1,51 @@ +# Copyright 2020 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_log import log + +from vitrage.common.constants import DatasourceOpts as DSOpts +from vitrage.utils import file as file_utils + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + + +class TmfApi639Config(object): + def __init__(self): + try: + tmfapi639_config_file = CONF.tmfapi639[DSOpts.CONFIG_FILE] + tmfapi639_config = file_utils.load_yaml_file(tmfapi639_config_file) + self.endpoints = self._create_mapping(tmfapi639_config) + except Exception as e: + LOG.error("Failed initialization: " + str(e)) + self.endpoints = [] + + @staticmethod + def _create_mapping(config): + """Read URL list from config dictionary""" + LOG.debug(config) + endpoint_list = [] + # Tuple list containing either 1 or 2 elements (Endpoint and updates) + for e in config: + snapshot_url = e["endpoint"]["snapshot"] + update_url = "" + if "update" in e["endpoint"]: + update_url = e["endpoint"]["update"] + if update_url != "": + endpoint_list.append((snapshot_url, update_url)) + else: + endpoint_list.append(snapshot_url) + LOG.info("Finished reading endpoints file") + return endpoint_list diff --git a/vitrage/datasources/tmfapi639/driver.py b/vitrage/datasources/tmfapi639/driver.py new file mode 100644 index 000000000..f552dd9a8 --- /dev/null +++ b/vitrage/datasources/tmfapi639/driver.py @@ -0,0 +1,96 @@ +# Copyright 2020 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log + +from vitrage.datasources.driver_base import DriverBase +from vitrage.datasources.tmfapi639 import TMFAPI639_DATASOURCE + +from vitrage.datasources.tmfapi639.config import TmfApi639Config + +import json +import requests + + +LOG = log.getLogger(__name__) + + +class TmfApi639Driver(DriverBase): + + def __init__(self): + super(TmfApi639Driver, self).__init__() + self.config = TmfApi639Config() + self.endpoints = self.config.endpoints + self.event_lambda = 0 + + @staticmethod + def get_event_types(): + return ['tmfapi639.instance.create', + 'tmfapi639.instance.update', + 'tmfapi639.instance.delete'] + + def enrich_event(self, event, event_type): + pass + + def get_all(self, datasource_action): + """Query all entities and send events to the vitrage events queue. + + When done for the first time, send an "end" event to inform it has + finished the get_all for the datasource (because it is done + asynchronously). + """ + return self.make_pickleable(self._get_all_entities(), + TMFAPI639_DATASOURCE, + datasource_action) + + def get_changes(self, datasource_action): + """Send an event to the vitrage events queue upon any change.""" + return self.make_pickleable(self._get_changes_entities(), + TMFAPI639_DATASOURCE, + datasource_action) + + def _get_all_entities(self): + total = [] + for pairs in self.endpoints: + try: + if type(pairs) is tuple: # Contains an update URL + LOG.info("Connecting to " + pairs[0] + + "with updates in " + pairs[1]) + r = requests.get(pairs[0]) + elif type(pairs) is str: # Doesn't contain update URL + LOG.info("Connecting to " + pairs) + r = requests.get(pairs) + r_dict = json.loads(r.text) + total += r_dict + except Exception as e: + LOG.error("Couldn't establish connection:" + str(e)) + return total + + def _get_changes_entities(self): # Called by get changes + total = [] + for pairs in self.endpoints: + try: + if type(pairs) is tuple: # Contains an update URL + LOG.info("Connecting to " + pairs[0] + + "with updates in " + pairs[1]) + r = requests.get(pairs[1]) + r_dict = json.loads(r.text) + for e in r_dict: + if e["eventId"] < self.event_lambda: + continue + total.append(e["event"]["resource"]) + self.event_lambda = e["eventId"] + except Exception as e: + LOG.error("Couldn't establish connection:" + str(e)) + return total diff --git a/vitrage/datasources/tmfapi639/transformer.py b/vitrage/datasources/tmfapi639/transformer.py new file mode 100644 index 000000000..3c3ee40ed --- /dev/null +++ b/vitrage/datasources/tmfapi639/transformer.py @@ -0,0 +1,97 @@ +# Copyright 2020 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging + +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources.resource_transformer_base \ + import ResourceTransformerBase +from vitrage.datasources.tmfapi639 import TMFAPI639_DATASOURCE +from vitrage.datasources import transformer_base +import vitrage.graph.utils as graph_utils + +from datetime import datetime + +LOG = logging.getLogger(__name__) + + +class TmfApi639Transformer(ResourceTransformerBase): + + def __init__(self, transformers): + super(TmfApi639Transformer, self).__init__(transformers) + + def _create_snapshot_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_update_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_snapshot_neighbors(self, entity_event): + return self._create_tmfapi639_neighbors(entity_event) + + def _create_update_neighbors(self, entity_event): + return self._create_tmfapi639_neighbors(entity_event) + + def _create_entity_key(self, entity_event): + """the unique key of this entity""" + entity_id = entity_event["id"] + entity_type = TMFAPI639_DATASOURCE + key_fields = self._key_values(entity_type, entity_id) + return transformer_base.build_key(key_fields) + + @staticmethod + def get_vitrage_type(): + return TMFAPI639_DATASOURCE + + def _create_vertex(self, entity_event): + """Camps used from the received JSON: + + {id, name, @type ,resourceRelationship : [type, resource: id]} + + The TMF 639 API REST Endpoint can contain more information + but we only use this one for topology. + """ + sample_timestamp = \ + datetime.now().strftime(transformer_base.TIMESTAMP_FORMAT) + update_timestamp = self._format_update_timestamp( + update_timestamp=None, + sample_timestamp=sample_timestamp) + + metadata = { + VProps.NAME: entity_event["name"], + } + + return graph_utils.create_vertex( + self._create_entity_key(entity_event), + vitrage_category=EntityCategory.RESOURCE, + vitrage_type=TMFAPI639_DATASOURCE, + vitrage_sample_timestamp=sample_timestamp, + entity_id=entity_event["id"], + update_timestamp=update_timestamp, + entity_state='available', + metadata=metadata) + + def _create_tmfapi639_neighbors(self, entity_event): + neighbors_list = [] + for n in entity_event["resourceRelationship"]: + # create placeholder vertex + neigh = self._create_neighbor( + entity_event, + n["resource"]["id"], + TMFAPI639_DATASOURCE, + n["type"], + is_entity_source=True) + neighbors_list.append(neigh) + return neighbors_list diff --git a/vitrage/tests/unit/datasources/tmfapi639/__init__.py b/vitrage/tests/unit/datasources/tmfapi639/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vitrage/tests/unit/datasources/tmfapi639/test_tmfapi639_transformer.py b/vitrage/tests/unit/datasources/tmfapi639/test_tmfapi639_transformer.py new file mode 100644 index 000000000..5a6b3d69d --- /dev/null +++ b/vitrage/tests/unit/datasources/tmfapi639/test_tmfapi639_transformer.py @@ -0,0 +1,142 @@ +# Copyright 2020 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_log import log as logging + +from testtools import matchers + +from vitrage.common.constants import DatasourceAction +from vitrage.common.constants import DatasourceOpts as DSOpts +from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.common.constants import UpdateMethod + +from vitrage.datasources.tmfapi639 import TMFAPI639_DATASOURCE +from vitrage.datasources.tmfapi639.transformer import TmfApi639Transformer +from vitrage.datasources import transformer_base +from vitrage.datasources.transformer_base import TransformerBase + +from vitrage.tests.unit.datasources.test_alarm_transformer_base import \ + BaseAlarmTransformerTest + +from datetime import datetime +from json import loads + +LOG = logging.getLogger(__name__) + +message = '[{"id":"1","name":"Host-1","@type":"Host",\ + "resourceRelationship":[{"type":"parent","resource":{"id":"1"}}]},\ + {"id":"2","name":"Host-2","@type":"Host",\ + "resourceRelationship":[{"type":"parent","resource":{"id":"1"}}]}]' + + +# noinspection PyProtectedMember +class TestTmfApi639Transformer(BaseAlarmTransformerTest): + + OPTS = [ + cfg.StrOpt(DSOpts.UPDATE_METHOD, + default=UpdateMethod.PULL), + ] + + # noinspection PyAttributeOutsideInit,PyPep8Naming + @classmethod + def setUpClass(cls): + super(TestTmfApi639Transformer, cls).setUpClass() + cls.transformers = {} + cls.conf = cfg.ConfigOpts() + cls.conf.register_opts(cls.OPTS, group=TMFAPI639_DATASOURCE) + cls.transformer = TmfApi639Transformer(cls.transformers) + cls.transformers[TMFAPI639_DATASOURCE] = cls.transformer + + # noinspection PyAttributeOutsideInit + def setUp(self): + super(TestTmfApi639Transformer, self).setUp() + # self.entity_type = TMFAPI639_DATASOURCE + # self.entity_id = '12345' + self.timestamp = datetime.utcnow() + + def test_create_entity_key(self): + event = loads(message)[0] + self.assertIsNotNone(event) + + transformer = TmfApi639Transformer(self.transformers) + observed_key = transformer._create_entity_key(event) + + entity_type = TMFAPI639_DATASOURCE + entity_id = event["id"] + + # Test assertions + observed_key_fields = observed_key.split( + TransformerBase.KEY_SEPARATOR) + + self.assertEqual(entity_type, observed_key_fields[1]) + self.assertEqual(entity_id, observed_key_fields[2]) + + # Transformer tests: + # - Vertex creation + # - Neighbor link + + def test_topology(self): + + sample_timestamp = \ + datetime.now().strftime(transformer_base.TIMESTAMP_FORMAT) + update_timestamp = TransformerBase._format_update_timestamp( + update_timestamp=None, + sample_timestamp=sample_timestamp) + + transformer = self.transformers[TMFAPI639_DATASOURCE] + + # Create 1 vertex + event1 = loads(message)[0] + event1[DSProps.DATASOURCE_ACTION] = DatasourceAction.SNAPSHOT + event1[DSProps.SAMPLE_DATE] = update_timestamp + self.assertIsNotNone(event1) + + # Create vertex 1 + wrapper1 = transformer.transform(event1) + # Assertion + self._validate_base_vertex_props( + wrapper1.vertex, + event1["name"], + TMFAPI639_DATASOURCE + ) + + # Create 2nd vertex + event2 = loads(message)[1] + event2[DSProps.DATASOURCE_ACTION] = DatasourceAction.SNAPSHOT + event2[DSProps.SAMPLE_DATE] = update_timestamp + self.assertIsNotNone(event2) + + # Create vertex 2 + wrapper2 = transformer.transform(event2) + # Assertion + self._validate_base_vertex_props( + wrapper2.vertex, + event2["name"], + TMFAPI639_DATASOURCE + ) + + # Test whether they are linked + self.assertThat(wrapper2.neighbors, matchers.HasLength(1)) + + parent_id = transformer._create_entity_key(event1) + parent_uuid = \ + transformer.uuid_from_deprecated_vitrage_id(parent_id) + + child_id = transformer._create_entity_key(event2) + child_uuid = \ + transformer.uuid_from_deprecated_vitrage_id(child_id) + + self.assertEqual(wrapper2.neighbors[0].edge.source_id, child_uuid) + self.assertEqual(wrapper2.neighbors[0].edge.target_id, parent_uuid)