From c771ad1a1b945269651bd208deab3c93369889e7 Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Fri, 13 Jan 2017 16:26:33 +0800 Subject: [PATCH] Neighbor validation for static datasource transformer Implement: blueprint static-datasource-config-format Change-Id: Ic1a30e0048708b6a43f2248ffab979dd0b738367 --- vitrage/common/constants.py | 1 + vitrage/tests/mocks/mock_driver.py | 12 ++- vitrage/tests/mocks/trace_generator.py | 97 ++++++++++--------- .../driver_static_snapshot_dynamic.json | 8 +- .../driver/driver_static_snapshot_static.json | 6 ++ .../static/test_static_transformer.py | 48 +++++---- 6 files changed, 97 insertions(+), 75 deletions(-) create mode 100644 vitrage/tests/resources/mock_configurations/driver/driver_static_snapshot_static.json diff --git a/vitrage/common/constants.py b/vitrage/common/constants.py index e34a06fa1..c28d47cc6 100644 --- a/vitrage/common/constants.py +++ b/vitrage/common/constants.py @@ -122,6 +122,7 @@ class TopologyFields(object): ENTITIES = 'entities' ENTITY = 'entity' TYPE = 'type' + ID = 'id' RELATIONSHIPS = 'relationships' RELATIONSHIP = 'relationship' diff --git a/vitrage/tests/mocks/mock_driver.py b/vitrage/tests/mocks/mock_driver.py index dfca22f23..d614d8225 100644 --- a/vitrage/tests/mocks/mock_driver.py +++ b/vitrage/tests/mocks/mock_driver.py @@ -35,8 +35,10 @@ usage example: import random +from vitrage.common.constants import DatasourceAction from vitrage.common.constants import DatasourceProperties as DSProps import vitrage.tests.mocks.trace_generator as tg +from vitrage.utils.datetime import utcnow def generate_random_events_list(generator_spec_list): @@ -391,10 +393,12 @@ def simple_static_generators(switch_num=2, host_num=10, if snapshot_events > 0: if snap_vals is None: snap_vals = {} - snap_vals[DSProps.DATASOURCE_ACTION] = 'update' + snap_vals.update({ + DSProps.DATASOURCE_ACTION: DatasourceAction.SNAPSHOT, + DSProps.SAMPLE_DATE: utcnow()}) test_entity_spec_list.append( {tg.DYNAMIC_INFO_FKEY: tg.DRIVER_STATIC_SNAPSHOT_D, - tg.STATIC_INFO_FKEY: None, + tg.STATIC_INFO_FKEY: tg.DRIVER_STATIC_SNAPSHOT_S, tg.EXTERNAL_INFO_KEY: snap_vals, tg.MAPPING_KEY: mapping, tg.NAME_KEY: 'Static snapshot generator', @@ -404,7 +408,9 @@ def simple_static_generators(switch_num=2, host_num=10, if update_events > 0: if update_vals is None: update_vals = {} - update_vals[DSProps.DATASOURCE_ACTION] = 'update' + update_vals.update({ + DSProps.DATASOURCE_ACTION: DatasourceAction.UPDATE, + DSProps.SAMPLE_DATE: utcnow()}) test_entity_spec_list.append( {tg.DYNAMIC_INFO_FKEY: tg.DRIVER_STATIC_SNAPSHOT_D, tg.STATIC_INFO_FKEY: None, diff --git a/vitrage/tests/mocks/trace_generator.py b/vitrage/tests/mocks/trace_generator.py index 0069b3fa7..e168c3aaf 100644 --- a/vitrage/tests/mocks/trace_generator.py +++ b/vitrage/tests/mocks/trace_generator.py @@ -26,7 +26,10 @@ from collections import defaultdict from random import randint # noinspection PyPep8Naming +from vitrage.common.constants import EdgeLabel +from vitrage.common.constants import TopologyFields from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE +from vitrage.datasources.static import StaticFields from vitrage.tests.mocks.entity_model import BasicEntityModel as Bem import vitrage.tests.mocks.utils as utils @@ -57,6 +60,7 @@ DRIVER_NAGIOS_SNAPSHOT_S = 'driver_nagios_snapshot_static.json' DRIVER_ZABBIX_SNAPSHOT_D = 'driver_zabbix_snapshot_dynamic.json' DRIVER_SWITCH_SNAPSHOT_D = 'driver_switch_snapshot_dynamic.json' DRIVER_STATIC_SNAPSHOT_D = 'driver_static_snapshot_dynamic.json' +DRIVER_STATIC_SNAPSHOT_S = 'driver_static_snapshot_static.json' DRIVER_VOLUME_UPDATE_D = 'driver_volume_update_dynamic.json' DRIVER_VOLUME_SNAPSHOT_D = 'driver_volume_snapshot_dynamic.json' DRIVER_STACK_UPDATE_D = 'driver_stack_update_dynamic.json' @@ -547,81 +551,84 @@ def _get_static_snapshot_driver_values(spec): """ host_switch_mapping = spec[MAPPING_KEY] - static_info_spec = None + if spec[STATIC_INFO_FKEY] is not None: static_info_spec = utils.load_specs(spec[STATIC_INFO_FKEY]) + else: + static_info_spec = None static_values = [] + # use defaultdict to create placeholder relationships = defaultdict(lambda: []) entities = defaultdict(lambda: {}) + touched = set({}) for host_index, switch_index in host_switch_mapping: host_id = "h{}".format(host_index) switch_id = "s{}".format(switch_index) relationship = { - "source": switch_id, - "target": host_id, - "relationship_type": "attached" + TopologyFields.SOURCE: switch_id, + TopologyFields.TARGET: host_id, + TopologyFields.RELATIONSHIP_TYPE: EdgeLabel.ATTACHED } - host_rel = relationship.copy() - host_rel['source'] = entities[switch_id] - relationships[host_id].append(host_rel) - - switch_rel = relationship.copy() - switch_rel['target'] = entities[host_id] - relationships[switch_id].append(switch_rel) + rel = relationship.copy() + rel[TopologyFields.TARGET] = entities[host_id] + relationships[switch_id].append(rel) for host_index, switch_index in host_switch_mapping: - mapping = {} - switch_id = "s{}".format(switch_index) - if switch_id not in entities: + if switch_id not in touched: switch_name = "switch-{}".format(switch_index) - mapping = { - 'name': switch_name, - 'config_id': switch_id, - 'id': str(randint(0, 100000)), - 'type': 'switch', - 'relationships': relationships[switch_id] + vals = { + StaticFields.CONFIG_ID: switch_id, + StaticFields.TYPE: 'switch', + StaticFields.ID: str(randint(0, 100000)), + StaticFields.NAME: switch_name, + StaticFields.RELATIONSHIPS: relationships[switch_id] } - entities[switch_id].update(**mapping) + entities[switch_id].update(**vals) + touched.add(switch_id) host_id = "h{}".format(host_index) - if host_id not in entities: - mapping = { - 'config_id': host_id, - 'type': 'nova.host', - 'id': str(randint(0, 100000)), - 'relationships': relationships[host_id] + if host_id not in touched: + vals = { + StaticFields.CONFIG_ID: host_id, + StaticFields.TYPE: NOVA_HOST_DATASOURCE, + StaticFields.ID: str(randint(0, 100000)), + StaticFields.RELATIONSHIPS: relationships[host_id] } - entities[host_id].update(**mapping) + entities[host_id].update(**vals) + touched.add(host_id) + for vals in entities.values(): static_values.append(combine_data(static_info_spec, - mapping, + vals, spec.get(EXTERNAL_INFO_KEY, None))) - for index in range(10): - custom_id = 'c{}'.format(index) - # self-pointing relationship - relationships = [ - { - "source": custom_id, - "target": custom_id, - "relationship_type": "custom" - } - ] - mapping = { - 'config_id': custom_id, - 'type': 'custom', - 'id': str(randint(0, 100000)), - 'relationships': relationships + custom_num = 10 + for index in range(custom_num): + source_id = 'c{}'.format(index) + target_id = 'c{}'.format(custom_num - 1 - index) + source_name = 'custom-{}'.format(source_id) + vals = { + StaticFields.CONFIG_ID: source_id, + StaticFields.TYPE: 'custom', + StaticFields.ID: str(randint(0, 100000)), + StaticFields.NAME: source_name, + StaticFields.RELATIONSHIPS: [{ + StaticFields.SOURCE: source_id, + StaticFields.TARGET: entities[target_id], + StaticFields.RELATIONSHIP_TYPE: 'custom'}] } + entities[source_id].update(**vals) static_values.append(combine_data(static_info_spec, - mapping, + vals, spec.get(EXTERNAL_INFO_KEY, None))) + # TODO(yujunz) verify self-pointing relationship + return static_values diff --git a/vitrage/tests/resources/mock_configurations/driver/driver_static_snapshot_dynamic.json b/vitrage/tests/resources/mock_configurations/driver/driver_static_snapshot_dynamic.json index bcc7c6099..a120ff30c 100644 --- a/vitrage/tests/resources/mock_configurations/driver/driver_static_snapshot_dynamic.json +++ b/vitrage/tests/resources/mock_configurations/driver/driver_static_snapshot_dynamic.json @@ -7,11 +7,5 @@ "vitrage_datasource_action": "snapshot", "vitrage_sample_date": "2015-12-01T12:46:41Z", "vitrage_event_type": "entity_update", - "relationships": [ - { - "source": "[sc][1-9]", - "target": "[hc][1-9]", - "relationship_type": "attached|custom" - } - ] + "relationships": [] } diff --git a/vitrage/tests/resources/mock_configurations/driver/driver_static_snapshot_static.json b/vitrage/tests/resources/mock_configurations/driver/driver_static_snapshot_static.json new file mode 100644 index 000000000..f8e3f957f --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/driver/driver_static_snapshot_static.json @@ -0,0 +1,6 @@ +{ + "state": "available", + "vitrage_datasource_action": "snapshot", + "vitrage_sample_date": "2015-12-01T12:46:41Z", + "vitrage_event_type": "entity_update" +} diff --git a/vitrage/tests/unit/datasources/static/test_static_transformer.py b/vitrage/tests/unit/datasources/static/test_static_transformer.py index d073248e4..4788f29db 100644 --- a/vitrage/tests/unit/datasources/static/test_static_transformer.py +++ b/vitrage/tests/unit/datasources/static/test_static_transformer.py @@ -19,6 +19,7 @@ from oslo_log import log as logging from vitrage.common.constants import DatasourceProperties as DSProps from vitrage.common.constants import EntityCategory +from vitrage.common.constants import TopologyFields from vitrage.common.constants import UpdateMethod from vitrage.common.constants import VertexProperties as VProps from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE @@ -86,13 +87,13 @@ class TestStaticTransformer(base.BaseTest): self.assertEqual(is_placeholder, True) def test_snapshot_transform(self): - spec_list = mock_driver.simple_static_generators(snapshot_events=10) - events = mock_driver.generate_random_events_list(spec_list) + vals_list = mock_driver.simple_static_generators(snapshot_events=1) + events = mock_driver.generate_random_events_list(vals_list) self._event_transform_test(events) def test_update_transform(self): - spec_list = mock_driver.simple_static_generators(update_events=10) - events = mock_driver.generate_random_events_list(spec_list) + vals_list = mock_driver.simple_static_generators(update_events=1) + events = mock_driver.generate_random_events_list(vals_list) self._event_transform_test(events) def _event_transform_test(self, events): @@ -106,28 +107,35 @@ class TestStaticTransformer(base.BaseTest): self._validate_neighbors(neighbors, vertex.vertex_id, event) def _validate_vertex(self, vertex, event): + self._validate_common_props(vertex, event) + self.assertEqual(vertex[VProps.SAMPLE_TIMESTAMP], + event[DSProps.SAMPLE_DATE]) + + for k, v in event.get(TopologyFields.METADATA, {}): + self.assertEqual(vertex[k], v) + + def _validate_common_props(self, vertex, event): self.assertEqual(vertex[VProps.CATEGORY], EntityCategory.RESOURCE) self.assertEqual(vertex[VProps.TYPE], event[VProps.TYPE]) self.assertEqual(vertex[VProps.ID], event[VProps.ID]) - self.assertEqual(vertex[VProps.SAMPLE_TIMESTAMP], - event[DSProps.SAMPLE_DATE]) self.assertFalse(vertex[VProps.IS_DELETED]) def _validate_neighbors(self, neighbors, vertex_id, event): for i in range(len(neighbors)): - if event['type'] == 'nova.host': - self._validate_switch_neighbor(neighbors[i], - event['relationships'][i], - vertex_id) - elif event['type'] == 'switch': - self._validate_host_neighbor(neighbors[i], - event['relationships'][i], - vertex_id) + self._validate_neighbor(neighbors[i], + event[TopologyFields.RELATIONSHIPS][i], + vertex_id) - def _validate_switch_neighbor(self, neighbor, rel, host_vertex_id): - # TODO(yujunz) - pass + def _validate_neighbor(self, neighbor, rel, vertex_id): + vertex = neighbor.vertex + self._validate_neighbor_vertex_props(vertex, + rel[TopologyFields.TARGET]) - def _validate_host_neighbor(self, neighbor, rel, switch_vertex_id): - # TODO(yujunz) - pass + edge = neighbor.edge + self.assertEqual(edge.source_id, vertex_id) + self.assertEqual(edge.target_id, neighbor.vertex.vertex_id) + self.assertEqual(edge.label, rel[TopologyFields.RELATIONSHIP_TYPE]) + + def _validate_neighbor_vertex_props(self, vertex, event): + self._validate_common_props(vertex, event) + self.assertTrue(vertex[VProps.IS_PLACEHOLDER])