diff --git a/devstack/gate_hook.sh b/devstack/gate_hook.sh index 2d2b85d9d..675dac6b8 100644 --- a/devstack/gate_hook.sh +++ b/devstack/gate_hook.sh @@ -64,9 +64,6 @@ topics = notifications, vitrage_notifications [DEFAULT] notifiers = mistral,nova,webhook -[static_physical] -changes_interval = 5 - [datasources] snapshots_interval = 120 EOF diff --git a/devstack/settings b/devstack/settings index c6ce20d35..115392506 100644 --- a/devstack/settings +++ b/devstack/settings @@ -37,7 +37,7 @@ VITRAGE_DEPLOY=${VITRAGE_DEPLOY} # Toggle for deploying Vitrage with/without nagios VITRAGE_USE_NAGIOS=$(trueorfalse False VITRAGE_USE_NAGIOS) -VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static,static_physical,aodh,cinder.volume,neutron.network,neutron.port,heat.stack,doctor,prometheus} +VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static,aodh,cinder.volume,neutron.network,neutron.port,heat.stack,doctor,prometheus} # for now dont use pip install for the client LIBS_FROM_GIT=python-vitrageclient diff --git a/doc/source/contributor/configuration.rst b/doc/source/contributor/configuration.rst index d26883200..49c6d8e9b 100644 --- a/doc/source/contributor/configuration.rst +++ b/doc/source/contributor/configuration.rst @@ -24,7 +24,6 @@ Datasources nagios-config static-config - static-physical-config zabbix_vitrage k8s_datasource diff --git a/doc/source/contributor/static-config.rst b/doc/source/contributor/static-config.rst index d3c75e68a..9d25abe78 100644 --- a/doc/source/contributor/static-config.rst +++ b/doc/source/contributor/static-config.rst @@ -5,14 +5,15 @@ Static Datasource Configuration Overview -------- -The static datasource allows users to integrate the **unmanaged** resources and topology into Vitrage. Unmanaged means -the resource, relationship or property can not be retrieved from any API or database, except static configuration file. +The static datasource allows users to integrate **unmanaged** resources and +topology into Vitrage. Unmanaged means the resource, relationship or property +can not be retrieved from any API or database, except static configuration +file. The static configuration may include physical, virtual or application +resources. -This datasource is static - pre-configured in a file. This is sufficient in many cases where the resources and -relationship is relatively unchanging. - -Static datasource suppresses the legacy static physical datasource. Theoretically both physical and virtual resources -and relationship between them can be configured in it. +This datasource is static. It is configured in a file that is being reloaded +periodically, based on the configuration. This is sufficient in many cases +where the resources and relationship is relatively unchanging. Configure Access to Static -------------------------- @@ -20,13 +21,13 @@ Configure Access to Static The following should be set in **/etc/vitrage/vitrage.conf**, under ``[static]`` section: -+------------------+---------------------------------------------------------+----------------------------------+ -| Name | Description | Default Value | -+==================+=========================================================+==================================+ -| directory | Directory path from where to load the configurations | /etc/vitrage/static_datasources/ | -+------------------+---------------------------------------------------------+----------------------------------+ -| changes_interval | Interval of checking changes in the configuration files | 30 seconds | -+------------------+---------------------------------------------------------+----------------------------------+ ++------------------+--------------------------------------------------------+----------------------------------+ +| Name | Description | Default Value | ++==================+========================================================+==================================+ +| directory | Directory path from where to load the configurations | /etc/vitrage/static_datasources/ | ++------------------+--------------------------------------------------------+----------------------------------+ +| changes_interval | Interval of checking changes in the configuration file | 30 seconds | ++------------------+--------------------------------------------------------+----------------------------------+ Configure Static Mapping ------------------------ @@ -45,11 +46,6 @@ Static datasource use the same semantics as Vitrage template, except for the fol There may be more than one configuration file. All files will be read from ``/etc/vitrage/static_datasources/``. See previous section on how to configure this location. -Notes: - - Static datasource shares the same configuration folder as legacy static physical datasource. - - Both static configuration and legacy static physical configuration will be loaded in Ocata release. - - The format is distinguished by checking existence of ``metadata`` key which is only available in static datasource. - Example +++++++ diff --git a/doc/source/contributor/static-physical-config.rst b/doc/source/contributor/static-physical-config.rst deleted file mode 100644 index 9c3fb85ec..000000000 --- a/doc/source/contributor/static-physical-config.rst +++ /dev/null @@ -1,98 +0,0 @@ -==================================================== -DEPRECATED: static physical datasource configuration -==================================================== - -**The static physical datasource is deprecated and will be removed in Stein -release.** - -**Please use the static datasource instead.** - - -Overview --------- - -The Static Physical datasource allows users to integrate the physical topology -into Vitrage. Physical topology includes switches and their connection to -other switches and physical hosts. - -This datasource is static - pre-configured in a file. This is sufficient in -many cases where the physical topology is relatively unchanging. - -Configure Access to Static Physical ------------------------------------ - -The following should be set in **/etc/vitrage/vitrage.conf**, under -[static_physical] section: - -+------------------+---------------------------------------------------------+----------------------------------+ -| Name | Description | Default Value | -+==================+=========================================================+==================================+ -| directory | Directory path from where to load the configurations | /etc/vitrage/static_datasources/ | -+------------------+---------------------------------------------------------+----------------------------------+ -| changes_interval | Interval of checking changes in the configuration files | 30 seconds | -+------------------+---------------------------------------------------------+----------------------------------+ -| entities | Static physical entity types list | switch | -+------------------+---------------------------------------------------------+----------------------------------+ - - -Configure Static Physical Mapping ---------------------------------- - -Physical configuration is made for configuring statically physical entities, -and their relationships to other entities in the topology. - -Some physical entities, such as switches, can not be retrieved from OpenStack, -and so are defined here. - -There may be more than one configuration file. All files will be read from -*/etc/vitrage/static_datasources/*. See previous section on how to configure this -location. - -Format -++++++ - -.. code:: - - entities: - - name: - id: - type: - state: - relationships: - - type: - name: - id: - relation_type: - - type: ... - ... - - -Notes: - - The "type" key must match the name of a type from an existing datasource. - - Type names appear, for each datasource, in its __init__.py file. - - For example see */workspace/dev/vitrage/vitrage/datasources/nova/host/__init__.py* - - -Example -+++++++ - -The following will define a switch that is attached to host-1 and is a backup -of switch-2 - -.. code:: - - entities: - - type: switch - name: switch-1 - id: switch-1 # should be same as name - state: available - relationships: - - type: nova.host - name: host-1 - id: host-1 # should be same as name - relation_type: attached - - type: switch - name: switch-2 - id: switch-2 # should be same as name - relation_type: backup - diff --git a/releasenotes/notes/remove-static-physical-datasource-e8e70158f453cff1.yaml b/releasenotes/notes/remove-static-physical-datasource-e8e70158f453cff1.yaml new file mode 100644 index 000000000..5968fcb93 --- /dev/null +++ b/releasenotes/notes/remove-static-physical-datasource-e8e70158f453cff1.yaml @@ -0,0 +1,4 @@ +--- +deprecations: + - The static_physical datasource was removed. Please use the static + datasource instead. diff --git a/vitrage/datasources/static/__init__.py b/vitrage/datasources/static/__init__.py index e1b197294..ca02539e5 100644 --- a/vitrage/datasources/static/__init__.py +++ b/vitrage/datasources/static/__init__.py @@ -41,9 +41,6 @@ OPTS = [ min=10, help='interval in seconds between checking changes in the' 'static configuration files'), - # NOTE: This folder is already used by static_physical datasource. Legacy - # configuration files will NOT be converted automatically. But user will - # receive deprecation warnings. cfg.StrOpt('directory', default='/etc/vitrage/static_datasources', help='static data sources configuration directory')] diff --git a/vitrage/datasources/static_physical/__init__.py b/vitrage/datasources/static_physical/__init__.py deleted file mode 100644 index ce4ad2207..000000000 --- a/vitrage/datasources/static_physical/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2016 - Nokia -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from oslo_config import cfg -from oslo_log import versionutils - -from vitrage.common.constants import DatasourceOpts as DSOpts -from vitrage.common.constants import UpdateMethod -from vitrage.common import utils - -STATIC_PHYSICAL_DATASOURCE = 'static_physical' -SWITCH = 'switch' - -_DEPRECATED_MSG = utils.fmt(""" -`Static_physical` was deprecated in Queens and and will be removed in Stein. -Please use static Datasource. -""") - -OPTS = [ - cfg.StrOpt(DSOpts.TRANSFORMER, - default='vitrage.datasources.static_physical.transformer.' - 'StaticPhysicalTransformer', - help='Static physical transformer class path', - deprecated_for_removal=True, - deprecated_reason=_DEPRECATED_MSG, - deprecated_since=versionutils.deprecated.QUEENS, - required=True), - cfg.StrOpt(DSOpts.DRIVER, - default='vitrage.datasources.static_physical.driver.' - 'StaticPhysicalDriver', - help='Static physical driver class path', - required=True), - cfg.StrOpt(DSOpts.UPDATE_METHOD, - default=UpdateMethod.PULL, - help='None: updates only via Vitrage periodic snapshots.' - 'Pull: updates every [changes_interval] seconds.' - 'Push: updates by getting notifications from the' - ' datasource itself.', - deprecated_for_removal=True, - deprecated_reason=_DEPRECATED_MSG, - deprecated_since=versionutils.deprecated.QUEENS, - required=True), - cfg.IntOpt(DSOpts.CHANGES_INTERVAL, - default=20, - min=5, - help='interval between checking changes in the configuration ' - 'files of the physical topology data sources', - deprecated_for_removal=True, - deprecated_reason=_DEPRECATED_MSG, - deprecated_since=versionutils.deprecated.QUEENS), - - cfg.StrOpt('directory', default='/etc/vitrage/static_datasources', - help='Static physical data sources directory', - deprecated_for_removal=True, - deprecated_reason=_DEPRECATED_MSG, - deprecated_since=versionutils.deprecated.QUEENS), - cfg.ListOpt('entities', - default=[SWITCH], - help='Static physical entity types list', - deprecated_for_removal=True, - deprecated_reason=_DEPRECATED_MSG, - deprecated_since=versionutils.deprecated.QUEENS) -] diff --git a/vitrage/datasources/static_physical/driver.py b/vitrage/datasources/static_physical/driver.py deleted file mode 100644 index f89a2dd9e..000000000 --- a/vitrage/datasources/static_physical/driver.py +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright 2016 - Nokia -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy - -from debtcollector import removals -from oslo_log import log - -from vitrage.common.constants import DatasourceProperties as DSProps -from vitrage.common.constants import GraphAction -from vitrage.datasources.driver_base import DriverBase -from vitrage.datasources.static.driver import StaticDriver -from vitrage.datasources.static import StaticFields -from vitrage.datasources.static_physical import STATIC_PHYSICAL_DATASOURCE -from vitrage.utils import file as file_utils - -LOG = log.getLogger(__name__) - - -class StaticPhysicalDriver(DriverBase): - @staticmethod - def get_event_types(): - return [] - - def enrich_event(self, event, event_type): - pass - - ENTITIES_SECTION = 'entities' - - def __init__(self, conf): - removals.removed_module(__name__, "datasources.static") - super(StaticPhysicalDriver, self).__init__() - self.cfg = conf - self.cache = {} - - def get_all(self, datasource_action): - return self.make_pickleable(self._get_all_entities(), - STATIC_PHYSICAL_DATASOURCE, - datasource_action) - - def get_changes(self, datasource_action): - return self.make_pickleable(self._get_changes_entities(), - STATIC_PHYSICAL_DATASOURCE, - datasource_action) - - def _get_all_entities(self): - static_entities = [] - - files = file_utils.list_files( - self.cfg.static_physical.directory, '.yaml') - - for file_ in files: - full_path = self.cfg.static_physical.directory \ - + '/' + file_ - static_entities += self._get_entities_from_file(file_, - full_path) - - return static_entities - - def _get_entities_from_file(self, file_, path): - static_entities = [] - config = file_utils.load_yaml_file(path) - - if StaticDriver._is_valid_config(config): - LOG.warning("Skipped config of new static datasource: {}" - .format(file_)) - return [] - - for entity in config[self.ENTITIES_SECTION]: - static_entities.append(entity.copy()) - - self.cache[file_] = config - - return static_entities - - def _get_changes_entities(self): - - entities_updates = [] - files = file_utils.list_files( - self.cfg.static_physical.directory, '.yaml') - - for file_ in files: - full_path = self.cfg.static_physical.directory +\ - '/' + file_ - config = file_utils.load_yaml_file(full_path) - - if StaticDriver._is_valid_config(config): - LOG.warning("Skipped config of new static datasource: {}" - .format(file_)) - return [] - - if config: - if file_ in self.cache: - if str(config) != str(self.cache[file_]): - # TODO(alexey_weyl): need also to remove deleted - # files from cache - old_config = copy.deepcopy(config) - - self._update_on_existing_entities( - self.cache[file_][self.ENTITIES_SECTION], - config[self.ENTITIES_SECTION], - entities_updates) - - self._update_on_new_entities( - config[self.ENTITIES_SECTION], - self.cache[file_][self.ENTITIES_SECTION], - entities_updates) - - self.cache[file_] = old_config - else: - self.cache[file_] = config - entities_updates += \ - self._get_entities_from_file(file_, full_path) - - # iterate over deleted files - deleted_files = set(self.cache.keys()) - set(files) - for file_ in deleted_files: - self._update_on_existing_entities( - self.cache[file_][self.ENTITIES_SECTION], - {}, - entities_updates) - del self.cache[file_] - - return entities_updates - - def _update_on_existing_entities(self, - old_entities, - new_entities, - updates): - for old_entity in old_entities: - if not new_entities or old_entity not in new_entities: - new_entity = self._find_entity(old_entity, new_entities) - if not new_entity: - self._set_event_type(old_entity, GraphAction.DELETE_ENTITY) - updates.append(old_entity.copy()) - else: - self._set_event_type(new_entity, GraphAction.UPDATE_ENTITY) - updates.append(new_entity.copy()) - - @staticmethod - def _find_entity(new_entity, entities): - for entity in entities: - if entity[StaticFields.TYPE] == new_entity[StaticFields.TYPE] \ - and entity[StaticFields.ID] == new_entity[StaticFields.ID]: - return entity - return None - - @staticmethod - def _update_on_new_entities(new_entities, old_entities, updates): - for entity in new_entities: - if entity not in updates and entity not in old_entities: - updates.append(entity.copy()) - - @staticmethod - def _set_event_type(entity, event_type): - entity[DSProps.EVENT_TYPE] = event_type diff --git a/vitrage/datasources/static_physical/transformer.py b/vitrage/datasources/static_physical/transformer.py deleted file mode 100644 index be9122c0e..000000000 --- a/vitrage/datasources/static_physical/transformer.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2016 - Nokia -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from debtcollector import removals -from vitrage.common.constants import DatasourceProperties as DSProps -from vitrage.common.constants import EntityCategory -from vitrage.common.constants import GraphAction -from vitrage.common.constants import VertexProperties as VProps -from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE -from vitrage.datasources.resource_transformer_base import \ - ResourceTransformerBase -from vitrage.datasources.static import StaticFields -from vitrage.datasources.static_physical import STATIC_PHYSICAL_DATASOURCE -from vitrage.datasources.static_physical import SWITCH -from vitrage.datasources import transformer_base -import vitrage.graph.utils as graph_utils - - -class StaticPhysicalTransformer(ResourceTransformerBase): - - RELATION_TYPE = 'relation_type' - RELATIONSHIPS_SECTION = 'relationships' - - # graph actions which need to refer them differently - GRAPH_ACTION_MAPPING = { - GraphAction.DELETE_ENTITY: GraphAction.DELETE_ENTITY - } - - def __init__(self, transformers, conf): - removals.removed_module(__name__, "datasources.static") - super(StaticPhysicalTransformer, self).__init__(transformers, conf) - self._register_relations_direction() - - def _create_snapshot_entity_vertex(self, entity_event): - return self._create_vertex(entity_event) - - def _create_update_entity_vertex(self, entity_event): - return self._create_vertex(entity_event) - - def _create_vertex(self, entity_event): - - entity_type = entity_event[StaticFields.TYPE] - entity_id = entity_event[VProps.ID] - vitrage_sample_timestamp = entity_event[DSProps.SAMPLE_DATE] - update_timestamp = self._format_update_timestamp( - update_timestamp=None, - sample_timestamp=vitrage_sample_timestamp) - state = entity_event[VProps.STATE] - entity_key = self._create_entity_key(entity_event) - metadata = self._extract_metadata(entity_event) - - return graph_utils.create_vertex( - entity_key, - vitrage_category=EntityCategory.RESOURCE, - vitrage_type=entity_type, - vitrage_sample_timestamp=vitrage_sample_timestamp, - entity_id=entity_id, - update_timestamp=update_timestamp, - entity_state=state, - metadata=metadata) - - def _create_snapshot_neighbors(self, entity_event): - return self._create_static_physical_neighbors(entity_event) - - def _create_update_neighbors(self, entity_event): - return self._create_static_physical_neighbors(entity_event) - - def _create_static_physical_neighbors(self, entity_event): - neighbors = [] - entity_type = entity_event[StaticFields.TYPE] - - for neighbor_details in entity_event.get( - self.RELATIONSHIPS_SECTION, {}): - # TODO(alexey): need to decide what to do if one of the entities - # fails - neighbor_id = neighbor_details[VProps.ID] - neighbor_type = neighbor_details[StaticFields.TYPE] - relation_type = neighbor_details[self.RELATION_TYPE] - is_entity_source = not self._find_relation_direction_source( - entity_type, neighbor_type) - neighbor = self._create_neighbor(entity_event, - neighbor_id, - neighbor_type, - relation_type, - is_entity_source=is_entity_source) - if neighbor is not None: - neighbors.append(neighbor) - - return neighbors - - def _create_entity_key(self, entity_event): - entity_id = entity_event[VProps.ID] - entity_type = entity_event[StaticFields.TYPE] - key_fields = self._key_values(entity_type, entity_id) - return transformer_base.build_key(key_fields) - - @staticmethod - def _extract_metadata(entity_event): - metadata = {VProps.NAME: entity_event[VProps.NAME]} - return metadata - - def _find_relation_direction_source(self, entity_type, neighbor_type): - # TODO(alexey): maybe check if this type exists, because it throws - # exception if it doesn't - return self.relation_direction[(entity_type, neighbor_type)] - - def _register_relations_direction(self): - self.relation_direction = {} - - relationship = (SWITCH, NOVA_HOST_DATASOURCE) - self.relation_direction[relationship] = True - - relationship = (SWITCH, SWITCH) - self.relation_direction[relationship] = True - - def get_vitrage_type(self): - return STATIC_PHYSICAL_DATASOURCE diff --git a/vitrage/tests/functional/datasources/static_physical/__init__.py b/vitrage/tests/functional/datasources/static_physical/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/vitrage/tests/functional/datasources/static_physical/test_static_physical.py b/vitrage/tests/functional/datasources/static_physical/test_static_physical.py deleted file mode 100644 index 2028ada43..000000000 --- a/vitrage/tests/functional/datasources/static_physical/test_static_physical.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2016 - Nokia -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg -from testtools import matchers - -from vitrage.common.constants import DatasourceProperties as DSProps -from vitrage.common.constants import EntityCategory -from vitrage.common.constants import VertexProperties as VProps -from vitrage.datasources.nagios import NAGIOS_DATASOURCE -from vitrage.datasources import NOVA_HOST_DATASOURCE -from vitrage.datasources import NOVA_INSTANCE_DATASOURCE -from vitrage.datasources import NOVA_ZONE_DATASOURCE -from vitrage.datasources.static_physical import STATIC_PHYSICAL_DATASOURCE -from vitrage.datasources.static_physical import SWITCH -from vitrage.tests.functional.datasources.base import \ - TestDataSourcesBase -from vitrage.tests.mocks import mock_driver - - -class TestStaticPhysical(TestDataSourcesBase): - - DATASOURCES_OPTS = [ - cfg.ListOpt('types', - default=[NAGIOS_DATASOURCE, - NOVA_HOST_DATASOURCE, - NOVA_INSTANCE_DATASOURCE, - NOVA_ZONE_DATASOURCE, - STATIC_PHYSICAL_DATASOURCE], - help='Names of supported driver data sources'), - - cfg.ListOpt('path', - default=['vitrage.datasources'], - help='base path for data sources') - ] - - # noinspection PyPep8Naming - @classmethod - def setUpClass(cls): - super(TestStaticPhysical, cls).setUpClass() - cls.conf = cfg.ConfigOpts() - cls.conf.register_opts(cls.PROCESSOR_OPTS, group='entity_graph') - cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources') - cls.load_datasources(cls.conf) - - def test_static_physical_validity(self): - # Setup - processor = self._create_processor_with_graph(self.conf) - transformers = processor.transformer_manager.transformers - transformers[SWITCH] = transformers[STATIC_PHYSICAL_DATASOURCE] - self.assertThat(processor.entity_graph, - matchers.HasLength( - self._num_total_expected_vertices()) - ) - - spec_list = mock_driver.simple_switch_generators( - switch_num=1, - host_num=1, - snapshot_events=1) - static_events = mock_driver.generate_random_events_list(spec_list) - static_physical_event = static_events[0] - static_physical_event[DSProps.ENTITY_TYPE] = SWITCH - static_physical_event['relationships'][0]['name'] = \ - self._find_entity_id_by_type(processor.entity_graph, - NOVA_HOST_DATASOURCE) - - # Action - processor.process_event(static_physical_event) - - # Test assertions - self.assertThat(processor.entity_graph, - matchers.HasLength( - self._num_total_expected_vertices() + 1) - ) - - static_physical_vertices = processor.entity_graph.get_vertices( - vertex_attr_filter={ - VProps.VITRAGE_CATEGORY: EntityCategory.RESOURCE, - VProps.VITRAGE_TYPE: SWITCH - }) - self.assertThat(static_physical_vertices, matchers.HasLength(1)) - - static_physical_neighbors = processor.entity_graph.neighbors( - static_physical_vertices[0].vertex_id) - self.assertThat(static_physical_neighbors, matchers.HasLength(1)) - - self.assertEqual(NOVA_HOST_DATASOURCE, - static_physical_neighbors[0][VProps.VITRAGE_TYPE]) diff --git a/vitrage/tests/mocks/mock_driver.py b/vitrage/tests/mocks/mock_driver.py index f81d86f9d..c6b9552c2 100644 --- a/vitrage/tests/mocks/mock_driver.py +++ b/vitrage/tests/mocks/mock_driver.py @@ -317,56 +317,6 @@ def simple_consistency_generators(consistency_num, update_events=0, return tg.get_trace_generators(test_entity_spec_list) -def simple_switch_generators(switch_num, host_num, - snapshot_events=0, snap_vals=None, - update_events=0, update_vals=None): - """A function for returning switch events generators. - - Returns generators for a given number of switches and hosts. - Hosts will be distributed across switches in round-robin style. - Switches are interconnected in a line. - - :param update_vals: number of events from update event - :param update_events: number of values from update event - :param switch_num: number of zones - :param host_num: number of hosts - :param snapshot_events: number of snapshot events per zone - :param snap_vals: preset values for ALL snapshot events - :return: generators for switch events as specified - """ - - mapping = [('host-{0}'.format(index), 'switch-{0}'.format(index % - switch_num)) - for index in range(host_num) - ] - - test_entity_spec_list = [] - if snapshot_events: - test_entity_spec_list.append( - {tg.DYNAMIC_INFO_FKEY: tg.DRIVER_SWITCH_SNAPSHOT_D, - tg.STATIC_INFO_FKEY: None, - tg.EXTERNAL_INFO_KEY: snap_vals, - tg.MAPPING_KEY: mapping, - tg.NAME_KEY: 'Switch snapshot generator', - tg.NUM_EVENTS: snapshot_events - } - ) - if update_events: - update_vals = {} if not update_vals else update_vals - update_vals[DSProps.DATASOURCE_ACTION] = \ - DatasourceAction.UPDATE - test_entity_spec_list.append( - {tg.DYNAMIC_INFO_FKEY: tg.DRIVER_SWITCH_SNAPSHOT_D, - tg.STATIC_INFO_FKEY: None, - tg.EXTERNAL_INFO_KEY: update_vals, - tg.MAPPING_KEY: mapping, - tg.NAME_KEY: 'Switch update generator', - tg.NUM_EVENTS: update_events - } - ) - return tg.get_trace_generators(test_entity_spec_list) - - def simple_static_generators(switch_num=2, host_num=10, snapshot_events=0, snap_vals=None, update_events=0, update_vals=None): diff --git a/vitrage/tests/mocks/trace_generator.py b/vitrage/tests/mocks/trace_generator.py index cfbeebefb..63ee34633 100644 --- a/vitrage/tests/mocks/trace_generator.py +++ b/vitrage/tests/mocks/trace_generator.py @@ -58,7 +58,6 @@ DRIVER_NAGIOS_SNAPSHOT_D = 'driver_nagios_snapshot_dynamic.json' DRIVER_NAGIOS_SNAPSHOT_S = 'driver_nagios_snapshot_static.json' DRIVER_PROMETHEUS_UPDATE_D = 'driver_prometheus_update_dynamic.json' DRIVER_ZABBIX_SNAPSHOT_D = 'driver_zabbix_snapshot_dynamic.json' -DRIVER_SWITCH_SNAPSHOT_D = 'driver_switch_snapshot_dynamic.json' DRIVER_STATIC_SNAPSHOT_D = 'driver_static_snapshot_dynamic.json' DRIVER_STATIC_SNAPSHOT_S = 'driver_static_snapshot_static.json' DRIVER_VOLUME_UPDATE_D = 'driver_volume_update_dynamic.json' @@ -131,7 +130,6 @@ class EventTraceGenerator(object): DRIVER_VOLUME_UPDATE_D: _get_volume_update_driver_values, DRIVER_STACK_SNAPSHOT_D: _get_stack_snapshot_driver_values, DRIVER_STACK_UPDATE_D: _get_stack_update_driver_values, - DRIVER_SWITCH_SNAPSHOT_D: _get_switch_snapshot_driver_values, DRIVER_STATIC_SNAPSHOT_D: _get_static_snapshot_driver_values, DRIVER_NAGIOS_SNAPSHOT_D: _get_nagios_alarm_driver_values, DRIVER_ZABBIX_SNAPSHOT_D: _get_zabbix_alarm_driver_values, @@ -513,46 +511,6 @@ def _get_vm_update_driver_values(spec): return static_values -def _get_switch_snapshot_driver_values(spec): - """Generates the static driver values for each zone. - - :param spec: specification of event generation. - :type spec: dict - :return: list of static driver values for each zone. - :rtype: list - """ - - host_switch_mapping = spec[MAPPING_KEY] - static_info = None - if spec[STATIC_INFO_FKEY] is not None: - static_info = utils.load_specs(spec[STATIC_INFO_FKEY]) - - static_values = [] - - switches_info = {} - for host_name, switch_name in host_switch_mapping: - switch_info = switches_info.get(switch_name, []) - - relationship_info = {"type": NOVA_HOST_DATASOURCE, - "name": host_name, - "id": host_name, - "relation_type": "contains" - } - - switch_info.append(relationship_info) - switches_info[switch_name] = switch_info - - for host_name, switch_name in host_switch_mapping: - mapping = {'name': switch_name, - 'id': switch_name, - 'relationships': switches_info[switch_name] - } - static_values.append(combine_data(static_info, - mapping, - spec.get(EXTERNAL_INFO_KEY, None))) - return static_values - - def _get_static_snapshot_driver_values(spec): """Generates the static driver values for static datasource. diff --git a/vitrage/tests/resources/mock_configurations/driver/driver_switch_snapshot_dynamic.json b/vitrage/tests/resources/mock_configurations/driver/driver_switch_snapshot_dynamic.json deleted file mode 100644 index 9d23d22db..000000000 --- a/vitrage/tests/resources/mock_configurations/driver/driver_switch_snapshot_dynamic.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "relationships": [ - {"type": "nova.host", - "name": "host-[1-9]", - "id": "[1-9]{5}", - "relation_type": "contains" - } - ], - "name": "switch-[0-9]", - "id": "[1-9]{5}", - "vitrage_sample_date": "2015-12-01T12:46:41Z", - "type": "switch", - "vitrage_datasource_action": "snapshot", - "state": "available", - "vitrage_event_type": "entity_update" -} diff --git a/vitrage/tests/unit/datasources/static_physical/__init__.py b/vitrage/tests/unit/datasources/static_physical/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/vitrage/tests/unit/datasources/static_physical/test_static_physical_driver.py b/vitrage/tests/unit/datasources/static_physical/test_static_physical_driver.py deleted file mode 100644 index 6d9fd0d3f..000000000 --- a/vitrage/tests/unit/datasources/static_physical/test_static_physical_driver.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2016 - Nokia -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - -from oslo_config import cfg -from testtools import matchers - -from vitrage.common.constants import DatasourceAction -from vitrage.common.constants import DatasourceOpts as DSOpts -from vitrage.common.constants import DatasourceProperties as DSProps -from vitrage.common.constants import GraphAction -from vitrage.datasources.static import StaticFields -from vitrage.datasources.static_physical import driver -from vitrage.datasources.static_physical import STATIC_PHYSICAL_DATASOURCE -from vitrage.datasources.static_physical import SWITCH -from vitrage.tests import base -from vitrage.tests.base import IsEmpty -from vitrage.tests.mocks import utils -from vitrage.utils import file as file_utils - - -class TestStaticPhysicalDriver(base.BaseTest): - - OPTS = [ - cfg.StrOpt(DSOpts.TRANSFORMER, - default='vitrage.datasources.static_physical.transformer.' - 'StaticPhysicalTransformer'), - cfg.StrOpt(DSOpts.DRIVER, - default='vitrage.datasources.static_physical.driver.' - 'StaticPhysicalDriver'), - cfg.IntOpt(DSOpts.CHANGES_INTERVAL, - default=30, - min=30, - help='interval between checking changes in the ' - 'configuration files of the physical topology plugin'), - cfg.StrOpt('directory', - default=utils.get_resources_dir() + '/static_datasources'), - cfg.ListOpt('entities', - default=[SWITCH]) - ] - - CHANGES_OPTS = [ - cfg.StrOpt(DSOpts.TRANSFORMER, - default='vitrage.datasources.static_physical.transformer.' - 'StaticPhysicalTransformer'), - cfg.StrOpt(DSOpts.DRIVER, - default='vitrage.datasources.static_physical.driver.' - 'StaticPhysicalDriver'), - cfg.IntOpt(DSOpts.CHANGES_INTERVAL, - default=30, - min=30, - help='interval between checking changes in the ' - 'configuration files of the physical topology plugin'), - cfg.StrOpt('directory', - default=utils.get_resources_dir() + - '/static_datasources/changes_datasources'), - ] - - # noinspection PyAttributeOutsideInit,PyPep8Naming - @classmethod - def setUpClass(cls): - super(TestStaticPhysicalDriver, cls).setUpClass() - cls.conf = cfg.ConfigOpts() - cls.conf.register_opts(cls.OPTS, group=STATIC_PHYSICAL_DATASOURCE) - cls.static_physical_driver = driver.StaticPhysicalDriver(cls.conf) - - def test_static_datasources_loader(self): - # Setup - total_static_datasources = \ - os.listdir(self.conf.static_physical.directory) - - # Action - static_configs = file_utils.load_yaml_files( - self.conf.static_physical.directory) - - # Test assertions - # -1 is because there are 2 files and a folder in static_datasource_dir - self.assertEqual(len(total_static_datasources) - 1, - len(static_configs)) - - def test_get_all(self): - # Action - static_entities = \ - self.static_physical_driver.get_all(DatasourceAction.UPDATE) - - # Test assertions - self.assertThat(static_entities, matchers.HasLength(5)) - - # noinspection PyAttributeOutsideInit - def test_get_changes(self): - # Setup - entities = self.static_physical_driver.get_all(DatasourceAction.UPDATE) - self.assertThat(entities, matchers.HasLength(5)) - - self.conf = cfg.ConfigOpts() - self.conf.register_opts(self.CHANGES_OPTS, - group=STATIC_PHYSICAL_DATASOURCE) - self.static_physical_driver.cfg = self.conf - - # Action - changes = self.static_physical_driver.get_changes( - GraphAction.UPDATE_ENTITY) - - # Test Assertions - status = any(change[StaticFields.TYPE] == SWITCH and - change[StaticFields.ID] == '12345' for change in changes) - self.assertFalse(status) - - status = any(change[StaticFields.TYPE] == SWITCH and - change[StaticFields.ID] == '23456' and - change[DSProps.EVENT_TYPE] == GraphAction.DELETE_ENTITY - for change in changes) - self.assertTrue(status) - - status = any(change[StaticFields.TYPE] == SWITCH and - change[StaticFields.ID] == '34567' for change in changes) - self.assertTrue(status) - - status = any(change[StaticFields.TYPE] == SWITCH and - change[StaticFields.ID] == '45678' for change in changes) - self.assertTrue(status) - status = any(change[StaticFields.TYPE] == SWITCH and - change[StaticFields.ID] == '56789' for change in changes) - self.assertTrue(status) - - self.assertThat(changes, matchers.HasLength(4)) - - # Action - changes = self.static_physical_driver.get_changes( - GraphAction.UPDATE_ENTITY) - - # Test Assertions - self.assertThat(changes, IsEmpty()) diff --git a/vitrage/tests/unit/datasources/static_physical/test_static_physical_transformer.py b/vitrage/tests/unit/datasources/static_physical/test_static_physical_transformer.py deleted file mode 100644 index 1ebcb8725..000000000 --- a/vitrage/tests/unit/datasources/static_physical/test_static_physical_transformer.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright 2016 - Nokia -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime - -from oslo_config import cfg -from oslo_log import log as logging - -from vitrage.common.constants import DatasourceOpts as DSOpts -from vitrage.common.constants import DatasourceProperties as DSProps -from vitrage.common.constants import EdgeLabel -from vitrage.common.constants import EntityCategory -from vitrage.common.constants import UpdateMethod -from vitrage.common.constants import VertexProperties as VProps -from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE -from vitrage.datasources.nova.host.transformer import HostTransformer -from vitrage.datasources.static import StaticFields -from vitrage.datasources.static_physical import STATIC_PHYSICAL_DATASOURCE -from vitrage.datasources.static_physical.transformer \ - import StaticPhysicalTransformer -from vitrage.datasources import transformer_base as tbase -from vitrage.datasources.transformer_base import TransformerBase -from vitrage.tests import base -from vitrage.tests.mocks import mock_driver as mock_sync - -LOG = logging.getLogger(__name__) - - -class TestStaticPhysicalTransformer(base.BaseTest): - - OPTS = [ - cfg.StrOpt(DSOpts.UPDATE_METHOD, - default=UpdateMethod.PULL), - ] - - # noinspection PyAttributeOutsideInit,PyPep8Naming - @classmethod - def setUpClass(cls): - super(TestStaticPhysicalTransformer, cls).setUpClass() - cls.transformers = {} - cls.conf = cfg.ConfigOpts() - cls.conf.register_opts(cls.OPTS, group=STATIC_PHYSICAL_DATASOURCE) - cls.transformers[NOVA_HOST_DATASOURCE] = HostTransformer( - cls.transformers, cls.conf) - cls.transformers[STATIC_PHYSICAL_DATASOURCE] = \ - StaticPhysicalTransformer(cls.transformers, cls.conf) - - def test_create_placeholder_vertex(self): - - LOG.debug('Static Physical transformer test: Create placeholder ' - 'vertex') - - # Test setup - switch_type = STATIC_PHYSICAL_DATASOURCE - switch_name = 'switch-1' - timestamp = datetime.datetime.utcnow() - static_transformer = self.transformers[STATIC_PHYSICAL_DATASOURCE] - - # Test action - properties = { - VProps.VITRAGE_TYPE: switch_type, - VProps.ID: switch_name, - VProps.VITRAGE_CATEGORY: EntityCategory.RESOURCE, - VProps.VITRAGE_SAMPLE_TIMESTAMP: timestamp - } - placeholder = \ - static_transformer.create_neighbor_placeholder_vertex(**properties) - - # Test assertions - observed_uuid = placeholder.vertex_id - expected_key = tbase.build_key(static_transformer._key_values( - switch_type, switch_name)) - expected_uuid = \ - TransformerBase.uuid_from_deprecated_vitrage_id(expected_key) - self.assertEqual(expected_uuid, observed_uuid) - - observed_time = placeholder.get(VProps.VITRAGE_SAMPLE_TIMESTAMP) - self.assertEqual(timestamp, observed_time) - - observed_subtype = placeholder.get(VProps.VITRAGE_TYPE) - self.assertEqual(switch_type, observed_subtype) - - observed_entity_id = placeholder.get(VProps.ID) - self.assertEqual(switch_name, observed_entity_id) - - observed_vitrage_category = placeholder.get(VProps.VITRAGE_CATEGORY) - self.assertEqual(EntityCategory.RESOURCE, observed_vitrage_category) - - vitrage_is_placeholder = placeholder.get(VProps.VITRAGE_IS_PLACEHOLDER) - self.assertTrue(vitrage_is_placeholder) - - def test_key_values(self): - LOG.debug('Static Physical transformer test: get key values') - - # Test setup - switch_type = STATIC_PHYSICAL_DATASOURCE - switch_name = 'switch-1' - static_transformer = self.transformers[STATIC_PHYSICAL_DATASOURCE] - - # Test action - observed_key_fields = static_transformer._key_values(switch_type, - switch_name) - - # Test assertions - self.assertEqual(EntityCategory.RESOURCE, observed_key_fields[0]) - self.assertEqual(STATIC_PHYSICAL_DATASOURCE, observed_key_fields[1]) - self.assertEqual(switch_name, observed_key_fields[2]) - - def test_snapshot_transform(self): - LOG.debug('Test transform entity snapshot/snapshot_init event') - - # Test setup - spec_list = mock_sync.simple_switch_generators(2, 10, 10) - static_events = mock_sync.generate_random_events_list(spec_list) - self._events_transform_test(static_events) - - def test_update_transform(self): - LOG.debug('Test transform entity update event') - - # Test setup - spec_list = mock_sync.simple_switch_generators(2, 10, 0, None, 10) - static_events = mock_sync.generate_random_events_list(spec_list) - self._events_transform_test(static_events) - - def _events_transform_test(self, events): - - for event in events: - # Test action - wrapper = self.transformers[STATIC_PHYSICAL_DATASOURCE].\ - transform(event) - - # Test assertions - vertex = wrapper.vertex - self._validate_switch_vertex_props(vertex, event) - - neighbors = wrapper.neighbors - self._validate_neighbors(neighbors, vertex.vertex_id, event) - - def _validate_neighbors(self, neighbors, switch_vertex_id, event): - host_counter = 0 - - for neighbor in neighbors: - self._validate_host_neighbor(neighbor, - event['relationships'][host_counter], - switch_vertex_id) - host_counter += 1 - - self.assertEqual(5, - host_counter, - 'Zone can belongs to only one Cluster') - - def _validate_host_neighbor(self, - host_neighbor, - host_event, - switch_vertex_id): - # validate neighbor vertex - self._validate_host_vertex_props(host_neighbor.vertex, host_event) - - # Validate neighbor edge - edge = host_neighbor.edge - self.assertEqual(edge.target_id, switch_vertex_id) - self.assertEqual(edge.source_id, host_neighbor.vertex.vertex_id) - self.assertEqual(edge.label, EdgeLabel.CONTAINS) - - def _validate_common_vertex_props(self, vertex, event): - self.assertEqual(EntityCategory.RESOURCE, - vertex[VProps.VITRAGE_CATEGORY]) - self.assertEqual(event[StaticFields.TYPE], - vertex[VProps.VITRAGE_TYPE]) - self.assertEqual(event[StaticFields.ID], vertex[VProps.ID]) - - def _validate_switch_vertex_props(self, vertex, event): - self._validate_common_vertex_props(vertex, event) - self.assertEqual(event[DSProps.SAMPLE_DATE], - vertex[VProps.VITRAGE_SAMPLE_TIMESTAMP]) - self.assertEqual(event[VProps.NAME], vertex[VProps.NAME]) - self.assertEqual(event[VProps.STATE], vertex[VProps.STATE]) - self.assertFalse(vertex[VProps.VITRAGE_IS_PLACEHOLDER]) - self.assertFalse(vertex[VProps.VITRAGE_IS_DELETED]) - - def _validate_host_vertex_props(self, vertex, event): - self._validate_common_vertex_props(vertex, event) - self.assertTrue(vertex[VProps.VITRAGE_IS_PLACEHOLDER]) - self.assertFalse(vertex[VProps.VITRAGE_IS_DELETED]) diff --git a/vitrage/tests/unit/graph/base.py b/vitrage/tests/unit/graph/base.py index 69695e41e..c258514da 100644 --- a/vitrage/tests/unit/graph/base.py +++ b/vitrage/tests/unit/graph/base.py @@ -31,7 +31,6 @@ from vitrage.common.exception import VitrageError from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE from vitrage.datasources import OPENSTACK_CLUSTER -from vitrage.datasources.static_physical import SWITCH from vitrage.datasources.transformer_base import CLUSTER_ID from vitrage.graph.driver.networkx_graph import NXGraph from vitrage.graph import utils as graph_utils @@ -80,9 +79,9 @@ v_alarm = graph_utils.create_vertex( metadata={VProps.RESOURCE_ID: '333333333333', VProps.NAME: 'anotheralarm'}) v_switch = graph_utils.create_vertex( - vitrage_id=SWITCH + '1212121212', + vitrage_id='switch1212121212', vitrage_category=RESOURCE, - vitrage_type=SWITCH, + vitrage_type='switch', entity_id='1212121212') e_node_to_host = graph_utils.create_edge( diff --git a/vitrage/tests/unit/graph/test_graph_algo.py b/vitrage/tests/unit/graph/test_graph_algo.py index 07ff827c5..59c0fc497 100644 --- a/vitrage/tests/unit/graph/test_graph_algo.py +++ b/vitrage/tests/unit/graph/test_graph_algo.py @@ -111,14 +111,14 @@ class GraphAlgorithmTest(GraphTestBase): query = { 'or': [ - {'==': {VProps.VITRAGE_TYPE: SWITCH}}, + {'==': {VProps.VITRAGE_TYPE: 'switch'}}, {'==': {VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE}}, ] } subgraph = ga.graph_query_vertices( root_id=first_host_id, query_dict=query, depth=1) self.assertEqual( - 1, # For SWITCH + 1, # For 'switch' subgraph.num_edges(), 'num of BOTH edges Host (depth 1)') subgraph = ga.graph_query_vertices(root_id=first_host_id, depth=2) @@ -266,7 +266,7 @@ class GraphAlgorithmTest(GraphTestBase): t_v_vm_alarm = graph_utils.create_vertex( vitrage_id='4', vitrage_category=ALARM, vitrage_type=ALARM_ON_VM) t_v_switch = graph_utils.create_vertex( - vitrage_id='5', vitrage_category=RESOURCE, vitrage_type=SWITCH) + vitrage_id='5', vitrage_category=RESOURCE, vitrage_type='switch') t_v_node = graph_utils.create_vertex( vitrage_id='6', vitrage_category=RESOURCE,