From 149045edbcdaaede210fa01e36e023c9f7b59f4d Mon Sep 17 00:00:00 2001 From: Bartosz Zurkowski Date: Fri, 12 Oct 2018 14:56:31 +0200 Subject: [PATCH] Add Trove datasource Intorduces Trove datasource enabling Trove instances and clusters in Vitrage Entity Graph. Change-Id: I4c42c577216a6d1ef911c60b151f1e9fe6d729dd Signed-off-by: Bartosz Zurkowski --- devstack/plugin.sh | 5 + devstack/settings | 2 +- .../datasources_values/trove.cluster.yaml | 31 ++++ .../datasources_values/trove.instance.yaml | 47 ++++++ lower-constraints.txt | 1 + .../trove-datasource-2aa7a88ff20aff8c.yaml | 14 ++ requirements.txt | 3 +- vitrage/datasources/trove/__init__.py | 0 vitrage/datasources/trove/cluster/__init__.py | 44 +++++ vitrage/datasources/trove/cluster/driver.py | 51 ++++++ .../datasources/trove/cluster/transformer.py | 84 ++++++++++ .../datasources/trove/instance/__init__.py | 44 +++++ vitrage/datasources/trove/instance/driver.py | 49 ++++++ .../datasources/trove/instance/transformer.py | 81 ++++++++++ vitrage/datasources/trove/properties.py | 34 ++++ .../datasources/trove/trove_driver_base.py | 104 ++++++++++++ vitrage/os_clients.py | 18 ++- .../functional/datasources/trove/__init__.py | 0 .../datasources/trove/test_trove_instance.py | 92 +++++++++++ vitrage/tests/mocks/mock_driver.py | 54 +++++++ vitrage/tests/mocks/trace_generator.py | 45 +++++- ...driver_trove_cluster_snapshot_dynamic.json | 19 +++ ...river_trove_instance_snapshot_dynamic.json | 13 ++ .../tests/unit/datasources/trove/__init__.py | 0 .../trove/test_trove_cluster_transformer.py | 151 ++++++++++++++++++ .../trove/test_trove_instance_transformer.py | 151 ++++++++++++++++++ 26 files changed, 1131 insertions(+), 6 deletions(-) create mode 100644 etc/vitrage/datasources_values/trove.cluster.yaml create mode 100644 etc/vitrage/datasources_values/trove.instance.yaml create mode 100644 releasenotes/notes/trove-datasource-2aa7a88ff20aff8c.yaml create mode 100644 vitrage/datasources/trove/__init__.py create mode 100644 vitrage/datasources/trove/cluster/__init__.py create mode 100644 vitrage/datasources/trove/cluster/driver.py create mode 100644 vitrage/datasources/trove/cluster/transformer.py create mode 100644 vitrage/datasources/trove/instance/__init__.py create mode 100644 vitrage/datasources/trove/instance/driver.py create mode 100644 vitrage/datasources/trove/instance/transformer.py create mode 100644 vitrage/datasources/trove/properties.py create mode 100644 vitrage/datasources/trove/trove_driver_base.py create mode 100644 vitrage/tests/functional/datasources/trove/__init__.py create mode 100644 vitrage/tests/functional/datasources/trove/test_trove_instance.py create mode 100644 vitrage/tests/resources/mock_configurations/driver/driver_trove_cluster_snapshot_dynamic.json create mode 100644 vitrage/tests/resources/mock_configurations/driver/driver_trove_instance_snapshot_dynamic.json create mode 100644 vitrage/tests/unit/datasources/trove/__init__.py create mode 100644 vitrage/tests/unit/datasources/trove/test_trove_cluster_transformer.py create mode 100644 vitrage/tests/unit/datasources/trove/test_trove_instance_transformer.py diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 070c56fbf..cd988e98c 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -188,6 +188,11 @@ function configure_vitrage { disable_vitrage_datasource heat.stack fi + # remove trove vitrage datasource if trove datasource not installed + if ! is_service_enabled trove; then + disable_vitrage_datasource trove.instance trove.cluster + fi + # remove nagios vitrage datasource if nagios datasource not installed if [ "$VITRAGE_USE_NAGIOS" == "False" ]; then disable_vitrage_datasource nagios diff --git a/devstack/settings b/devstack/settings index 115392506..a46aa9ffd 100644 --- a/devstack/settings +++ b/devstack/settings @@ -37,7 +37,7 @@ VITRAGE_DEPLOY=${VITRAGE_DEPLOY} # Toggle for deploying Vitrage with/without nagios VITRAGE_USE_NAGIOS=$(trueorfalse False VITRAGE_USE_NAGIOS) -VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static,aodh,cinder.volume,neutron.network,neutron.port,heat.stack,doctor,prometheus} +VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static,aodh,cinder.volume,neutron.network,neutron.port,heat.stack,doctor,prometheus,trove.instance,trove.cluster} # for now dont use pip install for the client LIBS_FROM_GIT=python-vitrageclient diff --git a/etc/vitrage/datasources_values/trove.cluster.yaml b/etc/vitrage/datasources_values/trove.cluster.yaml new file mode 100644 index 000000000..0d70c0917 --- /dev/null +++ b/etc/vitrage/datasources_values/trove.cluster.yaml @@ -0,0 +1,31 @@ +category: RESOURCE +values: + - aggregated values: + priority: 50 + original values: + - name: DELETED + operational_value: DELETED + - aggregated values: + priority: 40 + original values: + - name: ERROR + operational_value: ERROR + - aggregated values: + priority: 30 + original values: + - name: GROWING + operational_value: TRANSIENT + - name: SHRINKING + operational_value: TRANSIENT + - name: UPGRADING + operational_value: TRANSIENT + - aggregated values: + priority: 20 + original values: + - name: SUBOPTIMAL + operational_value: SUBOPTIMAL + - aggregated values: + priority: 10 + original values: + - name: NONE + operational_value: OK diff --git a/etc/vitrage/datasources_values/trove.instance.yaml b/etc/vitrage/datasources_values/trove.instance.yaml new file mode 100644 index 000000000..9c0a60d19 --- /dev/null +++ b/etc/vitrage/datasources_values/trove.instance.yaml @@ -0,0 +1,47 @@ +category: RESOURCE +values: + - aggregated values: + priority: 70 + original values: + - name: DELETED + operational_value: DELETED + - aggregated values: + priority: 60 + original values: + - name: FAILED + operational_value: ERROR + - name: ERROR + operational_value: ERROR + - aggregated values: + priority: 50 + original values: + - name: SHUTDOWN + operational_value: SUBOPTIMAL + - name: BLOCKED + operational_value: SUBOPTIMAL + - aggregated values: + priority: 40 + original values: + - name: RESIZING + operational_value: TRANSIENT + - name: UPGRADING + operational_value: TRANSIENT + - name: REBOOTING + operational_value: TRANSIENT + - aggregated values: + priority: 30 + original values: + - name: NEW + operational_value: TRANSIENT + - name: BUILDING + operational_value: TRANSIENT + - aggregated values: + priority: 20 + original values: + - name: SUBOPTIMAL + operational_value: SUBOPTIMAL + - aggregated values: + priority: 10 + original values: + - name: ACTIVE + operational_value: OK diff --git a/lower-constraints.txt b/lower-constraints.txt index 6c491594f..d124e30b7 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -103,6 +103,7 @@ python-novaclient==10.1.0 python-openstackclient==3.12.0 python-subunit==1.2.0 python-swiftclient==3.5.0 +python-troveclient==2.2.0 pytz==2013.6 PyYAML==3.12 pyzabbix==0.7.4 diff --git a/releasenotes/notes/trove-datasource-2aa7a88ff20aff8c.yaml b/releasenotes/notes/trove-datasource-2aa7a88ff20aff8c.yaml new file mode 100644 index 000000000..cab33b599 --- /dev/null +++ b/releasenotes/notes/trove-datasource-2aa7a88ff20aff8c.yaml @@ -0,0 +1,14 @@ +--- +features: + - A new ``Trove Datasource`` has been introduced to include Trove entities + (database instances and clusters) in Vitrage Entity Graph. Trove is Database + as a Service solution offering database lifecycle management (automated + provisioning, configuration, backups, clustering etc.). Adding the + datasource to Vitrage enables detecting problems at lower levels of + infrastructure that may affect functioning of running databases, and react + in response to identified issues e.g. scale the database up/out or + live-migrate virtual machines from failed compute. This change is the + first stage of integration with Trove. At this point, Trove entities are + extracted using PULL approach, based on periodical snapshot-query to Trove + API for the current list of Trove entities. In the future, PUSH approach + based on Trove notifications will be implemented. diff --git a/requirements.txt b/requirements.txt index ba81131c9..bee35a9b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ python-novaclient>=10.1.0 # Apache-2.0 python-heatclient>=1.14.0 # Apache-2.0 python-mistralclient>=3.3.0 # Apache-2.0 python-openstackclient>=3.12.0 # Apache-2.0 +python-troveclient>=2.2.0 # Apache-2.0 gnocchiclient>=3.3.1 # Apache-2.0 pyzabbix>=0.7.4 # LGPL networkx>=2.0 # BSD @@ -49,4 +50,4 @@ debtcollector>=1.19.0 # Apache-2.0 cotyledon>=1.6.8 # Apache-2.0 futures>=3.0.0;python_version=='2.7' or python_version=='2.6' # BSD pytz>=2013.6 # MIT -tenacity>=4.9.0 \ No newline at end of file +tenacity>=4.9.0 diff --git a/vitrage/datasources/trove/__init__.py b/vitrage/datasources/trove/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vitrage/datasources/trove/cluster/__init__.py b/vitrage/datasources/trove/cluster/__init__.py new file mode 100644 index 000000000..2011536ea --- /dev/null +++ b/vitrage/datasources/trove/cluster/__init__.py @@ -0,0 +1,44 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from vitrage.common.constants import DatasourceOpts as DSOpts +from vitrage.common.constants import UpdateMethod + +TROVE_CLUSTER_DATASOURCE = 'trove.cluster' + +OPTS = [ + cfg.StrOpt(DSOpts.TRANSFORMER, + default='vitrage.datasources.trove.cluster.transformer.' + 'TroveClusterTransformer', + help='Trove transformer class path.', + required=True), + cfg.StrOpt(DSOpts.DRIVER, + default='vitrage.datasources.trove.cluster.driver.' + 'TroveClusterDriver', + help='Trove driver class path.', + required=True), + cfg.StrOpt(DSOpts.UPDATE_METHOD, + default=UpdateMethod.PULL, + help='None: updates only via Vitrage periodic snapshots.' + 'Pull: updates periodically.' + 'Push: updates by getting notifications from the' + ' datasource itself.', + required=True), + cfg.IntOpt(DSOpts.CHANGES_INTERVAL, + default=30, + min=10, + help='Interval in seconds between checking changes in Trove' + 'cluster datasource.')] diff --git a/vitrage/datasources/trove/cluster/driver.py b/vitrage/datasources/trove/cluster/driver.py new file mode 100644 index 000000000..5835b8424 --- /dev/null +++ b/vitrage/datasources/trove/cluster/driver.py @@ -0,0 +1,51 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log + +from vitrage.datasources.transformer_base import extract_field_value +from vitrage.datasources.trove.cluster import TROVE_CLUSTER_DATASOURCE +from vitrage.datasources.trove.properties import \ + TroveClusterProperties as TProps +from vitrage.datasources.trove.trove_driver_base import TroveDriverBase + + +LOG = log.getLogger(__name__) + + +class TroveClusterDriver(TroveDriverBase): + + def __init__(self, conf): + super(TroveClusterDriver, self).__init__(conf) + self._cached_entities = [] + + def _get_vitrage_type(self): + return TROVE_CLUSTER_DATASOURCE + + def _get_all_entities(self): + # TODO(bzurkowski): Add all_tenants option to Trove client + return self.extract_entities(self.client.clusters.list()) + + def _find_entity(self, search_entity, entities): + for entity in entities: + if entity[TProps.ID] == search_entity[TProps.ID]: + return entity + + def _equal_entities(self, old_entity, new_entity): + old_state = extract_field_value(old_entity, *TProps.STATE) + new_state = extract_field_value(old_entity, *TProps.STATE) + return old_entity[TProps.ID] == new_entity[TProps.ID] and \ + old_entity[TProps.NAME] == new_entity[TProps.NAME] and \ + old_state == new_state diff --git a/vitrage/datasources/trove/cluster/transformer.py b/vitrage/datasources/trove/cluster/transformer.py new file mode 100644 index 000000000..fce24723e --- /dev/null +++ b/vitrage/datasources/trove/cluster/transformer.py @@ -0,0 +1,84 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.common.constants import EdgeLabel +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources.resource_transformer_base import \ + ResourceTransformerBase +from vitrage.datasources import transformer_base as tbase +from vitrage.datasources.transformer_base import extract_field_value +from vitrage.datasources.trove.cluster import TROVE_CLUSTER_DATASOURCE +from vitrage.datasources.trove.instance import TROVE_INSTANCE_DATASOURCE +from vitrage.datasources.trove.properties import \ + TroveClusterProperties as TProps +import vitrage.graph.utils as graph_utils + + +class TroveClusterTransformer(ResourceTransformerBase): + + def _create_snapshot_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_update_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_vertex(self, entity_event): + # TODO(bzurkowski): Add project ID + entity_id = entity_event[TProps.ID] + name = entity_event[TProps.NAME] + state = extract_field_value(entity_event, *TProps.STATE) + update_timestamp = entity_event[TProps.UPDATE_TIMESTAMP] + sample_timestamp = entity_event[DSProps.SAMPLE_DATE] + metadata = { + VProps.NAME: name + } + return graph_utils.create_vertex( + self._create_entity_key(entity_event), + vitrage_category=EntityCategory.RESOURCE, + vitrage_type=TROVE_CLUSTER_DATASOURCE, + vitrage_sample_timestamp=sample_timestamp, + entity_id=entity_id, + update_timestamp=update_timestamp, + entity_state=state, + metadata=metadata) + + def _create_snapshot_neighbors(self, entity_event): + return self._create_entity_neighbours(entity_event) + + def _create_update_neighbors(self, entity_event): + return self._create_entity_neighbours(entity_event) + + def _create_entity_neighbours(self, entity_event): + neighbours = [] + for instance in entity_event[TProps.INSTANCES]: + instance_neighbour = self._create_neighbor( + entity_event, + instance[TProps.ID], + TROVE_INSTANCE_DATASOURCE, + EdgeLabel.CONTAINS, + is_entity_source=True) + neighbours.append(instance_neighbour) + return neighbours + + def _create_entity_key(self, entity_event): + entity_id = entity_event[TProps.ID] + key_fields = self._key_values(TROVE_CLUSTER_DATASOURCE, entity_id) + return tbase.build_key(key_fields) + + @staticmethod + def get_vitrage_type(): + return TROVE_CLUSTER_DATASOURCE diff --git a/vitrage/datasources/trove/instance/__init__.py b/vitrage/datasources/trove/instance/__init__.py new file mode 100644 index 000000000..63c433171 --- /dev/null +++ b/vitrage/datasources/trove/instance/__init__.py @@ -0,0 +1,44 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from vitrage.common.constants import DatasourceOpts as DSOpts +from vitrage.common.constants import UpdateMethod + +TROVE_INSTANCE_DATASOURCE = 'trove.instance' + +OPTS = [ + cfg.StrOpt(DSOpts.TRANSFORMER, + default='vitrage.datasources.trove.instance.transformer.' + 'TroveInstanceTransformer', + help='Trove transformer class path.', + required=True), + cfg.StrOpt(DSOpts.DRIVER, + default='vitrage.datasources.trove.instance.driver.' + 'TroveInstanceDriver', + help='Trove driver class path.', + required=True), + cfg.StrOpt(DSOpts.UPDATE_METHOD, + default=UpdateMethod.PULL, + help='None: updates only via Vitrage periodic snapshots.' + 'Pull: updates periodically.' + 'Push: updates by getting notifications from the' + ' datasource itself.', + required=True), + cfg.IntOpt(DSOpts.CHANGES_INTERVAL, + default=30, + min=10, + help='Interval in seconds between checking changes in Trove' + 'instance datasource.')] diff --git a/vitrage/datasources/trove/instance/driver.py b/vitrage/datasources/trove/instance/driver.py new file mode 100644 index 000000000..a1e66cbe5 --- /dev/null +++ b/vitrage/datasources/trove/instance/driver.py @@ -0,0 +1,49 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log + +from vitrage.datasources.trove.instance import TROVE_INSTANCE_DATASOURCE +from vitrage.datasources.trove.properties import \ + TroveInstanceProperties as TProps +from vitrage.datasources.trove.trove_driver_base import TroveDriverBase + + +LOG = log.getLogger(__name__) + + +class TroveInstanceDriver(TroveDriverBase): + + def __init__(self, conf): + super(TroveInstanceDriver, self).__init__(conf) + self._cached_entities = [] + + def _get_vitrage_type(self): + return TROVE_INSTANCE_DATASOURCE + + def _get_all_entities(self): + # TODO(bzurkowski): Add all_tenants option to Trove client + return self.extract_entities( + self.client.instances.list(include_clustered=True, detailed=True)) + + def _find_entity(self, search_entity, entities): + for entity in entities: + if entity[TProps.ID] == search_entity[TProps.ID]: + return entity + + def _equal_entities(self, old_entity, new_entity): + return old_entity[TProps.ID] == new_entity[TProps.ID] and \ + old_entity[TProps.NAME] == new_entity[TProps.NAME] and \ + old_entity[TProps.STATE] == new_entity[TProps.STATE] diff --git a/vitrage/datasources/trove/instance/transformer.py b/vitrage/datasources/trove/instance/transformer.py new file mode 100644 index 000000000..757a7aed7 --- /dev/null +++ b/vitrage/datasources/trove/instance/transformer.py @@ -0,0 +1,81 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.common.constants import EdgeLabel +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE +from vitrage.datasources.resource_transformer_base import \ + ResourceTransformerBase +from vitrage.datasources import transformer_base as tbase +from vitrage.datasources.trove.instance import TROVE_INSTANCE_DATASOURCE +from vitrage.datasources.trove.properties import \ + TroveInstanceProperties as TProps +import vitrage.graph.utils as graph_utils + + +class TroveInstanceTransformer(ResourceTransformerBase): + + def _create_snapshot_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_update_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_vertex(self, entity_event): + entity_id = entity_event[TProps.ID] + name = entity_event[TProps.NAME] + state = entity_event[TProps.STATE] + project_id = entity_event[TProps.PROJECT_ID] + update_timestamp = entity_event[TProps.UPDATE_TIMESTAMP] + sample_timestamp = entity_event[DSProps.SAMPLE_DATE] + metadata = { + VProps.NAME: name, + VProps.PROJECT_ID: project_id + } + return graph_utils.create_vertex( + self._create_entity_key(entity_event), + vitrage_category=EntityCategory.RESOURCE, + vitrage_type=TROVE_INSTANCE_DATASOURCE, + vitrage_sample_timestamp=sample_timestamp, + entity_id=entity_id, + update_timestamp=update_timestamp, + entity_state=state, + metadata=metadata) + + def _create_snapshot_neighbors(self, entity_event): + return self._create_entity_neighbours(entity_event) + + def _create_update_neighbors(self, entity_event): + return self._create_entity_neighbours(entity_event) + + def _create_entity_neighbours(self, entity_event): + server_neighbour = self._create_neighbor( + entity_event, + entity_event[TProps.SERVER_ID], + NOVA_INSTANCE_DATASOURCE, + EdgeLabel.CONTAINS, + is_entity_source=True) + return [server_neighbour] + + def _create_entity_key(self, entity_event): + entity_id = entity_event[TProps.ID] + key_fields = self._key_values(TROVE_INSTANCE_DATASOURCE, entity_id) + return tbase.build_key(key_fields) + + @staticmethod + def get_vitrage_type(): + return TROVE_INSTANCE_DATASOURCE diff --git a/vitrage/datasources/trove/properties.py b/vitrage/datasources/trove/properties.py new file mode 100644 index 000000000..56b128709 --- /dev/null +++ b/vitrage/datasources/trove/properties.py @@ -0,0 +1,34 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class TroveBaseProperties(object): + + ID = 'id' + NAME = 'name' + UPDATE_TIMESTAMP = 'updated' + PROJECT_ID = 'tenant_id' + + +class TroveInstanceProperties(TroveBaseProperties): + + STATE = 'status' + SERVER_ID = 'server_id' + + +class TroveClusterProperties(TroveBaseProperties): + + STATE = ('task', 'name') + INSTANCES = 'instances' diff --git a/vitrage/datasources/trove/trove_driver_base.py b/vitrage/datasources/trove/trove_driver_base.py new file mode 100644 index 000000000..34a8f5996 --- /dev/null +++ b/vitrage/datasources/trove/trove_driver_base.py @@ -0,0 +1,104 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc + +from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.common.constants import GraphAction +from vitrage.datasources.driver_base import DriverBase +from vitrage import os_clients + + +class TroveDriverBase(DriverBase): + + def __init__(self, conf): + super(TroveDriverBase, self).__init__() + self.conf = conf + self.__client = None + self.__cached_entities = [] + + @property + def client(self): + if not self.__client: + self.__client = os_clients.trove_client(self.conf) + return self.__client + + def get_all(self, datasource_action): + return self.make_pickleable(self._get_and_cache_all_entities(), + self._get_vitrage_type(), + datasource_action, + *self.properties_to_filter_out()) + + def get_changes(self, datasource_action): + return self.make_pickleable(self._get_changed_entities(), + self._get_vitrage_type(), + datasource_action, + *self.properties_to_filter_out()) + + def _get_and_cache_all_entities(self): + self.__cached_entities = self._get_all_entities() + return self.__cached_entities + + def _get_changed_entities(self): + actual_entities = self._get_all_entities() + changed_entities = [] + + for actual_entity in actual_entities: + cached_entity = self._find_entity(actual_entity, + self.__cached_entities) + if cached_entity: + # Add modified entities + if not self._equal_entities(actual_entity, cached_entity): + changed_entities.append(actual_entity) + else: + # Add new entities + changed_entities.append(actual_entity) + + # Delete removed entities + for cached_entity in self.__cached_entities: + if not self._find_entity(cached_entity, actual_entities): + cached_entity[DSProps.EVENT_TYPE] = GraphAction.DELETE_ENTITY + changed_entities.append(cached_entity) + + self.__cached_entities = actual_entities + return changed_entities + + @abc.abstractmethod + def _get_vitrage_type(self): + pass + + @abc.abstractmethod + def _get_all_entities(self): + pass + + @abc.abstractmethod + def _find_entity(self, search_entity, entities): + pass + + @abc.abstractmethod + def _equal_entities(self, old_entity, new_entity): + pass + + @staticmethod + def properties_to_filter_out(): + return ['manager', '_info'] + + @staticmethod + def should_delete_outdated_entities(): + return True + + @staticmethod + def extract_entities(entities): + return [entity.to_dict() for entity in entities] diff --git a/vitrage/os_clients.py b/vitrage/os_clients.py index 06fbf8ccc..2405ef13b 100644 --- a/vitrage/os_clients.py +++ b/vitrage/os_clients.py @@ -29,6 +29,7 @@ OPTS = [ cfg.StrOpt('heat_version', default='1', help='Heat version'), cfg.StrOpt('mistral_version', default='2', help='Mistral version'), cfg.StrOpt('gnocchi_version', default='1', help='Gnocchi version'), + cfg.StrOpt('trove_version', default='1', help='Trove version'), cfg.BoolOpt('use_nova_versioned_notifications', default=True, help='Indicates whether to use Nova versioned notifications.' @@ -47,7 +48,8 @@ _client_modules = { 'neutron': 'neutronclient.v2_0.client', 'heat': 'heatclient.client', 'mistral': 'mistralclient.api.v2.client', - 'gnocchi': 'gnocchiclient.v1.client' + 'gnocchi': 'gnocchiclient.v1.client', + 'trove': 'troveclient.v1.client' } @@ -110,6 +112,20 @@ def nova_client(conf): LOG.exception('Create Nova client - Got Exception.') +def trove_client(conf): + """Get an instance of trove client""" + try: + tr_client = driver_module('trove') + client = tr_client.Client( + version=conf.trove_version, + session=keystone_client.get_session(conf), + ) + LOG.info('Trove client created') + return client + except Exception: + LOG.exception('Create Trove client - Got Exception.') + + def cinder_client(conf): """Get an instance of cinder client""" try: diff --git a/vitrage/tests/functional/datasources/trove/__init__.py b/vitrage/tests/functional/datasources/trove/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vitrage/tests/functional/datasources/trove/test_trove_instance.py b/vitrage/tests/functional/datasources/trove/test_trove_instance.py new file mode 100644 index 000000000..70cae9f25 --- /dev/null +++ b/vitrage/tests/functional/datasources/trove/test_trove_instance.py @@ -0,0 +1,92 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from testtools import matchers + +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources.nagios import NAGIOS_DATASOURCE +from vitrage.datasources import NOVA_HOST_DATASOURCE +from vitrage.datasources import NOVA_INSTANCE_DATASOURCE +from vitrage.datasources import NOVA_ZONE_DATASOURCE +from vitrage.datasources.trove.instance import TROVE_INSTANCE_DATASOURCE +from vitrage.tests.functional.datasources.base import TestDataSourcesBase +from vitrage.tests.mocks import mock_driver + + +class TestTroveInstance(TestDataSourcesBase): + + DATASOURCES_OPTS = [ + cfg.ListOpt('types', + default=[NAGIOS_DATASOURCE, + NOVA_HOST_DATASOURCE, + NOVA_INSTANCE_DATASOURCE, + NOVA_ZONE_DATASOURCE, + TROVE_INSTANCE_DATASOURCE], + help='Names of supported driver data sources'), + + cfg.ListOpt('path', + default=['vitrage.datasources'], + help='Base path for data sources') + ] + + # noinspection PyPep8Naming + @classmethod + def setUpClass(cls): + super(TestTroveInstance, cls).setUpClass() + cls.conf = cfg.ConfigOpts() + cls.conf.register_opts(cls.PROCESSOR_OPTS, group='entity_graph') + cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources') + cls.load_datasources(cls.conf) + + def test_trove_instance_validity(self): + # Setup + processor = self._create_processor_with_graph(self.conf) + self.assertThat(processor.entity_graph, + matchers.HasLength( + self._num_total_expected_vertices()) + ) + + spec_list = mock_driver.simple_trove_instance_generators( + inst_num=1, + snapshot_events=1) + static_events = mock_driver.generate_random_events_list(spec_list) + trove_instance_event = static_events[0] + trove_instance_event['server_id'] = \ + self._find_entity_id_by_type(processor.entity_graph, + NOVA_INSTANCE_DATASOURCE) + + # Action + processor.process_event(trove_instance_event) + + # Test assertions + self.assertThat(processor.entity_graph, + matchers.HasLength( + self._num_total_expected_vertices() + 1) + ) + + trove_vertices = processor.entity_graph.get_vertices( + vertex_attr_filter={ + VProps.VITRAGE_CATEGORY: EntityCategory.RESOURCE, + VProps.VITRAGE_TYPE: TROVE_INSTANCE_DATASOURCE + }) + self.assertThat(trove_vertices, matchers.HasLength(1)) + + trove_neighbors = processor.entity_graph.neighbors( + trove_vertices[0].vertex_id) + self.assertThat(trove_neighbors, matchers.HasLength(1)) + self.assertEqual(NOVA_INSTANCE_DATASOURCE, + trove_neighbors[0][VProps.VITRAGE_TYPE]) diff --git a/vitrage/tests/mocks/mock_driver.py b/vitrage/tests/mocks/mock_driver.py index 81384acfa..e7eb86a90 100644 --- a/vitrage/tests/mocks/mock_driver.py +++ b/vitrage/tests/mocks/mock_driver.py @@ -203,6 +203,60 @@ def simple_zone_generators(zone_num, host_num, snapshot_events=0, return tg.get_trace_generators(test_entity_spec_list) +def simple_trove_instance_generators(inst_num, snapshot_events=0, + snap_vals=None): + """A function for returning Trove instance generators. + + Returns generators for a given number of Trove instances. + + :param inst_num: number of instances + :return: generators for inst_num instances as specified + """ + + mapping = [('tr-instance-{0}'.format(idx), 'vm-{0}'.format(idx)) + for idx in range(inst_num)] + + test_entity_spec_list = [ + {tg.DYNAMIC_INFO_FKEY: tg.DRIVER_TROVE_INSTANCE_SNAPSHOT_D, + tg.STATIC_INFO_FKEY: None, + tg.MAPPING_KEY: mapping, + tg.EXTERNAL_INFO_KEY: snap_vals, + tg.NAME_KEY: 'Database instance snapshot generator', + tg.NUM_EVENTS: snapshot_events + } + ] + + return tg.get_trace_generators(test_entity_spec_list) + + +def simple_trove_cluster_generators(clust_num, inst_num, snapshot_events=0, + snap_vals=None): + """A function for returning Trove cluster generators. + + Returns generators for a given number of Trove clusters. + + :param clust_num: number of clusters + :param inst_num: number of instances + :return: generators for clust_num clusters as specified + """ + + mapping = [('tr-cluster-{0}'.format(idx % clust_num), + 'tr-inst-{0}'.format(idx)) + for idx in range(inst_num)] + + test_entity_spec_list = [ + {tg.DYNAMIC_INFO_FKEY: tg.DRIVER_TROVE_CLUSTER_SNAPSHOT_D, + tg.STATIC_INFO_FKEY: None, + tg.MAPPING_KEY: mapping, + tg.EXTERNAL_INFO_KEY: snap_vals, + tg.NAME_KEY: 'Database cluster snapshot generator', + tg.NUM_EVENTS: snapshot_events + } + ] + + return tg.get_trace_generators(test_entity_spec_list) + + def simple_volume_generators(volume_num, instance_num, snapshot_events=0, update_events=0, snap_vals=None, update_vals=None): diff --git a/vitrage/tests/mocks/trace_generator.py b/vitrage/tests/mocks/trace_generator.py index 9da2d3132..fe874e2b8 100644 --- a/vitrage/tests/mocks/trace_generator.py +++ b/vitrage/tests/mocks/trace_generator.py @@ -68,7 +68,10 @@ DRIVER_STACK_SNAPSHOT_D = 'driver_stack_snapshot_dynamic.json' DRIVER_CONSISTENCY_UPDATE_D = 'driver_consistency_update_dynamic.json' DRIVER_ZONE_SNAPSHOT_D = 'driver_zone_snapshot_dynamic.json' DRIVER_KUBE_SNAPSHOT_D = 'driver_kubernetes_snapshot_dynamic.json' - +DRIVER_TROVE_INSTANCE_SNAPSHOT_D = \ + 'driver_trove_instance_snapshot_dynamic.json' +DRIVER_TROVE_CLUSTER_SNAPSHOT_D = \ + 'driver_trove_cluster_snapshot_dynamic.json' # Mock transformer Specs (i.e., what the transformer outputs) MOCK_TRANSFORMER_PATH = '%s/mock_configurations/transformer' % \ @@ -79,7 +82,6 @@ TRANS_DOCTOR_UPDATE_D = 'transformer_doctor_update_dynamic.json' TRANS_COLLECTD_UPDATE_D = 'transformer_collectd_update_dynamic.json' TRANS_PROMETHEUS_UPDATE_D = 'transformer_prometheus_update_dynamic.json' TRANS_INST_SNAPSHOT_D = 'transformer_inst_snapshot_dynamic.json' -TRANS_INST_SNAPSHOT_S = 'transformer_inst_snapshot_static.json' TRANS_HOST_SNAPSHOT_D = 'transformer_host_snapshot_dynamic.json' TRANS_HOST_SNAPSHOT_S = 'transformer_host_snapshot_static.json' TRANS_ZONE_SNAPSHOT_D = 'transformer_zone_snapshot_dynamic.json' @@ -139,7 +141,10 @@ class EventTraceGenerator(object): DRIVER_CONSISTENCY_UPDATE_D: _get_consistency_update_driver_values, DRIVER_PROMETHEUS_UPDATE_D: _get_simple_update_driver_values, - + DRIVER_TROVE_INSTANCE_SNAPSHOT_D: + _get_trove_instance_snapshot_driver_values, + DRIVER_TROVE_CLUSTER_SNAPSHOT_D: + _get_trove_cluster_snapshot_driver_values, TRANS_AODH_SNAPSHOT_D: _get_trans_aodh_alarm_snapshot_values, TRANS_AODH_UPDATE_D: _get_trans_aodh_alarm_snapshot_values, TRANS_DOCTOR_UPDATE_D: _get_simple_trans_alarm_update_values, @@ -664,6 +669,40 @@ def _get_zabbix_alarm_driver_values(spec): return static_values +def _get_trove_instance_snapshot_driver_values(spec): + inst_srv_mapping = spec[MAPPING_KEY] + static_info = None + if spec[STATIC_INFO_FKEY] is not None: + static_info = utils.load_specs(spec[STATIC_INFO_FKEY]) + static_values = [] + + for inst_name, srv_name in inst_srv_mapping: + mapping = {'id': inst_name, + 'name': inst_name, + 'server_id': srv_name} + static_values.append(combine_data( + static_info, mapping, spec.get(EXTERNAL_INFO_KEY, None))) + return static_values + + +def _get_trove_cluster_snapshot_driver_values(spec): + clust_inst_mapping = spec[MAPPING_KEY] + static_info = None + if spec[STATIC_INFO_FKEY] is not None: + static_info = utils.load_specs(spec[STATIC_INFO_FKEY]) + static_values = [] + + for clust_name, inst_name in clust_inst_mapping: + mapping = {'id': clust_name, + 'name': clust_name, + 'instances': [ + {'id': inst_name, 'name': inst_name} + ]} + static_values.append(combine_data( + static_info, mapping, spec.get(EXTERNAL_INFO_KEY, None))) + return static_values + + def _get_trans_host_snapshot_values(spec): """Generates the static driver values for each host. diff --git a/vitrage/tests/resources/mock_configurations/driver/driver_trove_cluster_snapshot_dynamic.json b/vitrage/tests/resources/mock_configurations/driver/driver_trove_cluster_snapshot_dynamic.json new file mode 100644 index 000000000..9252283ed --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/driver/driver_trove_cluster_snapshot_dynamic.json @@ -0,0 +1,19 @@ +{ + "id": "tr-clust-0", + "name": "tr-clust-0", + "task": { + "id": 1, + "name": "NONE" + }, + "instances": [ + { + "id": "tr-inst-0", + "name": "tr-inst-0" + } + ], + "created": "2018-10-01T12:45:30Z", + "updated": "2018-10-01T12:46:00Z", + "vitrage_entity_type": "trove.cluster", + "vitrage_datasource_action": "snapshot", + "vitrage_sample_date": "2018-10-01 12:50:00.000000+00:00" +} diff --git a/vitrage/tests/resources/mock_configurations/driver/driver_trove_instance_snapshot_dynamic.json b/vitrage/tests/resources/mock_configurations/driver/driver_trove_instance_snapshot_dynamic.json new file mode 100644 index 000000000..61843246d --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/driver/driver_trove_instance_snapshot_dynamic.json @@ -0,0 +1,13 @@ +{ + "id": "tr-inst-0", + "name": "tr-inst-0", + "status": "ACTIVE", + "server_id": "vm-0", + "volume_id": "volume-0", + "tenant_id": "tenant-0", + "created": "2018-10-01T12:45:30Z", + "updated": "2018-10-01T12:46:00Z", + "vitrage_entity_type": "trove.instance", + "vitrage_datasource_action": "snapshot", + "vitrage_sample_date": "2018-10-01 12:50:00.000000+00:00" +} diff --git a/vitrage/tests/unit/datasources/trove/__init__.py b/vitrage/tests/unit/datasources/trove/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vitrage/tests/unit/datasources/trove/test_trove_cluster_transformer.py b/vitrage/tests/unit/datasources/trove/test_trove_cluster_transformer.py new file mode 100644 index 000000000..8354b1c70 --- /dev/null +++ b/vitrage/tests/unit/datasources/trove/test_trove_cluster_transformer.py @@ -0,0 +1,151 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime + +from oslo_config import cfg +from oslo_log import log as logging +from testtools import matchers + +from vitrage.common.constants import DatasourceOpts as DSOpts +from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.common.constants import EdgeLabel +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import UpdateMethod +from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources import transformer_base as tb +from vitrage.datasources.transformer_base import TransformerBase +from vitrage.datasources.trove.cluster.transformer import \ + TroveClusterTransformer +from vitrage.datasources.trove.cluster import TROVE_CLUSTER_DATASOURCE +from vitrage.datasources.trove.instance import TROVE_INSTANCE_DATASOURCE +from vitrage.tests import base +from vitrage.tests.mocks import mock_driver as mock_sync + +LOG = logging.getLogger(__name__) + + +class TroveClusterTransformerTest(base.BaseTest): + + OPTS = [ + cfg.StrOpt(DSOpts.UPDATE_METHOD, default=UpdateMethod.PULL), + ] + + @classmethod + def setUpClass(cls): + super(TroveClusterTransformerTest, cls).setUpClass() + cls.transformers = {} + cls.conf = cfg.ConfigOpts() + cls.conf.register_opts(cls.OPTS, group=TROVE_CLUSTER_DATASOURCE) + cls.transformers[TROVE_CLUSTER_DATASOURCE] = \ + TroveClusterTransformer(cls.transformers, cls.conf) + + def test_create_placeholder_vertex(self): + # Tests setup + cluster_id = 'tr-cluster-0' + timestamp = datetime.datetime.utcnow() + + properties = { + VProps.ID: cluster_id, + VProps.VITRAGE_TYPE: TROVE_CLUSTER_DATASOURCE, + VProps.VITRAGE_CATEGORY: EntityCategory.RESOURCE, + VProps.VITRAGE_SAMPLE_TIMESTAMP: timestamp + } + + transformer = self.transformers[TROVE_CLUSTER_DATASOURCE] + + # Test action + placeholder = transformer.create_neighbor_placeholder_vertex( + **properties) + + # Test assertions + expected_key = tb.build_key( + transformer._key_values(TROVE_CLUSTER_DATASOURCE, cluster_id)) + expected_uuid = TransformerBase.uuid_from_deprecated_vitrage_id( + expected_key) + self.assertEqual(expected_uuid, placeholder.vertex_id) + + self.assertEqual(timestamp, + placeholder.get(VProps.VITRAGE_SAMPLE_TIMESTAMP)) + + self.assertEqual(TROVE_CLUSTER_DATASOURCE, + placeholder.get(VProps.VITRAGE_TYPE)) + + self.assertEqual(cluster_id, placeholder.get(VProps.ID)) + + self.assertEqual(EntityCategory.RESOURCE, + placeholder.get(VProps.VITRAGE_CATEGORY)) + + self.assertTrue(placeholder.get(VProps.VITRAGE_IS_PLACEHOLDER)) + + def test_snapshot_event_transform(self): + # Test setup + spec_list = mock_sync.simple_trove_cluster_generators( + clust_num=1, inst_num=1, snapshot_events=10) + events = mock_sync.generate_random_events_list(spec_list) + + for event in events: + # Test action + transformer = self.transformers[TROVE_CLUSTER_DATASOURCE] + wrapper = transformer.transform(event) + + # Test assertions + vertex = wrapper.vertex + self._validate_vertex_props(vertex, event) + + neighbours = wrapper.neighbors + self.assertThat(neighbours, matchers.HasLength(1)) + self._validate_server_neighbour(neighbours[0], vertex.vertex_id, + event) + + def _validate_vertex_props(self, vertex, event): + self.assertEqual(event['id'], vertex[VProps.ID]) + + self.assertEqual(event['name'], vertex[VProps.NAME]) + + self.assertEqual(event['task']['name'], vertex[VProps.STATE]) + + self.assertEqual(event[DSProps.SAMPLE_DATE], + vertex[VProps.VITRAGE_SAMPLE_TIMESTAMP]) + + self.assertEqual(EntityCategory.RESOURCE, + vertex[VProps.VITRAGE_CATEGORY]) + + self.assertEqual(TROVE_CLUSTER_DATASOURCE, + vertex[VProps.VITRAGE_TYPE]) + + self.assertFalse(vertex[VProps.VITRAGE_IS_PLACEHOLDER]) + self.assertFalse(vertex[VProps.VITRAGE_IS_DELETED]) + + def _validate_server_neighbour(self, neighbour, cluster_id, event): + vertex, edge = neighbour.vertex, neighbour.edge + + # Validate neighbor vertex + self.assertEqual(EntityCategory.RESOURCE, + vertex[VProps.VITRAGE_CATEGORY]) + + self.assertEqual(TROVE_INSTANCE_DATASOURCE, + vertex[VProps.VITRAGE_TYPE]) + + instance_id = event['instances'][0]['id'] + self.assertEqual(instance_id, vertex[VProps.ID]) + + self.assertTrue(vertex[VProps.VITRAGE_IS_PLACEHOLDER]) + self.assertFalse(vertex[VProps.VITRAGE_IS_DELETED]) + + # Validate neighbor edge + self.assertEqual(edge.target_id, vertex.vertex_id) + self.assertEqual(edge.source_id, cluster_id) + self.assertEqual(edge.label, EdgeLabel.CONTAINS) diff --git a/vitrage/tests/unit/datasources/trove/test_trove_instance_transformer.py b/vitrage/tests/unit/datasources/trove/test_trove_instance_transformer.py new file mode 100644 index 000000000..45e18c744 --- /dev/null +++ b/vitrage/tests/unit/datasources/trove/test_trove_instance_transformer.py @@ -0,0 +1,151 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime + +from oslo_config import cfg +from oslo_log import log as logging +from testtools import matchers + +from vitrage.common.constants import DatasourceOpts as DSOpts +from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.common.constants import EdgeLabel +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import UpdateMethod +from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE +from vitrage.datasources import transformer_base as tb +from vitrage.datasources.transformer_base import TransformerBase +from vitrage.datasources.trove.instance.transformer import \ + TroveInstanceTransformer +from vitrage.datasources.trove.instance import TROVE_INSTANCE_DATASOURCE +from vitrage.tests import base +from vitrage.tests.mocks import mock_driver as mock_sync + +LOG = logging.getLogger(__name__) + + +class TroveInstanceTransformerTest(base.BaseTest): + + OPTS = [ + cfg.StrOpt(DSOpts.UPDATE_METHOD, default=UpdateMethod.PULL), + ] + + @classmethod + def setUpClass(cls): + super(TroveInstanceTransformerTest, cls).setUpClass() + cls.transformers = {} + cls.conf = cfg.ConfigOpts() + cls.conf.register_opts(cls.OPTS, group=TROVE_INSTANCE_DATASOURCE) + cls.transformers[TROVE_INSTANCE_DATASOURCE] = \ + TroveInstanceTransformer(cls.transformers, cls.conf) + + def test_create_placeholder_vertex(self): + # Tests setup + instance_id = 'tr-instance-0' + timestamp = datetime.datetime.utcnow() + + properties = { + VProps.ID: instance_id, + VProps.VITRAGE_TYPE: TROVE_INSTANCE_DATASOURCE, + VProps.VITRAGE_CATEGORY: EntityCategory.RESOURCE, + VProps.VITRAGE_SAMPLE_TIMESTAMP: timestamp + } + + transformer = self.transformers[TROVE_INSTANCE_DATASOURCE] + + # Test action + placeholder = transformer.create_neighbor_placeholder_vertex( + **properties) + + # Test assertions + expected_key = tb.build_key( + transformer._key_values(TROVE_INSTANCE_DATASOURCE, instance_id)) + expected_uuid = TransformerBase.uuid_from_deprecated_vitrage_id( + expected_key) + self.assertEqual(expected_uuid, placeholder.vertex_id) + + self.assertEqual(timestamp, + placeholder.get(VProps.VITRAGE_SAMPLE_TIMESTAMP)) + + self.assertEqual(TROVE_INSTANCE_DATASOURCE, + placeholder.get(VProps.VITRAGE_TYPE)) + + self.assertEqual(instance_id, placeholder.get(VProps.ID)) + + self.assertEqual(EntityCategory.RESOURCE, + placeholder.get(VProps.VITRAGE_CATEGORY)) + + self.assertTrue(placeholder.get(VProps.VITRAGE_IS_PLACEHOLDER)) + + def test_snapshot_event_transform(self): + # Test setup + spec_list = mock_sync.simple_trove_instance_generators( + inst_num=1, snapshot_events=10) + events = mock_sync.generate_random_events_list(spec_list) + + for event in events: + # Test action + transformer = self.transformers[TROVE_INSTANCE_DATASOURCE] + wrapper = transformer.transform(event) + + # Test assertions + vertex = wrapper.vertex + self._validate_vertex_props(vertex, event) + + neighbours = wrapper.neighbors + self.assertThat(neighbours, matchers.HasLength(1)) + self._validate_server_neighbour(neighbours[0], vertex.vertex_id, + event) + + def _validate_vertex_props(self, vertex, event): + self.assertEqual(event['id'], vertex[VProps.ID]) + + self.assertEqual(event['name'], vertex[VProps.NAME]) + + self.assertEqual(event['status'], vertex[VProps.STATE]) + + self.assertEqual(event['tenant_id'], vertex[VProps.PROJECT_ID]) + + self.assertEqual(event[DSProps.SAMPLE_DATE], + vertex[VProps.VITRAGE_SAMPLE_TIMESTAMP]) + + self.assertEqual(EntityCategory.RESOURCE, + vertex[VProps.VITRAGE_CATEGORY]) + + self.assertEqual(TROVE_INSTANCE_DATASOURCE, + vertex[VProps.VITRAGE_TYPE]) + + self.assertFalse(vertex[VProps.VITRAGE_IS_PLACEHOLDER]) + self.assertFalse(vertex[VProps.VITRAGE_IS_DELETED]) + + def _validate_server_neighbour(self, neighbour, instance_id, event): + vertex, edge = neighbour.vertex, neighbour.edge + + # Validate neighbor vertex + self.assertEqual(EntityCategory.RESOURCE, + vertex[VProps.VITRAGE_CATEGORY]) + + self.assertEqual(NOVA_INSTANCE_DATASOURCE, vertex[VProps.VITRAGE_TYPE]) + + self.assertEqual(event['server_id'], vertex[VProps.ID]) + + self.assertTrue(vertex[VProps.VITRAGE_IS_PLACEHOLDER]) + self.assertFalse(vertex[VProps.VITRAGE_IS_DELETED]) + + # Validate neighbor edge + self.assertEqual(edge.target_id, vertex.vertex_id) + self.assertEqual(edge.source_id, instance_id) + self.assertEqual(edge.label, EdgeLabel.CONTAINS)