From c7e0b55d3514ccbcd969e6c1755c2f7a6efbb633 Mon Sep 17 00:00:00 2001 From: Liat Har-Tal Date: Wed, 30 Dec 2015 13:56:53 +0000 Subject: [PATCH] nova transformer - transform instance update events Change-Id: I683ef276851a11704b576da5af3488881cc7d20e --- vitrage/entity_graph/transformer/base.py | 35 +++- .../transformer/nova_transformer.py | 135 ++++++++----- .../tests/unit/processor/test_processor.py | 2 + .../transformers/test_nova_transformers.py | 187 +++++++++++------- 4 files changed, 228 insertions(+), 131 deletions(-) diff --git a/vitrage/entity_graph/transformer/base.py b/vitrage/entity_graph/transformer/base.py index f07ecd124..c81509958 100644 --- a/vitrage/entity_graph/transformer/base.py +++ b/vitrage/entity_graph/transformer/base.py @@ -29,6 +29,15 @@ EntityWrapper = \ Neighbor = namedtuple('Neighbor', ['vertex', 'edge']) +def extract_field_value(entity_event, key_names): + + value = entity_event + for key in key_names: + value = value[key] + + return value + + @six.add_metaclass(abc.ABCMeta) class Transformer(object): @@ -36,20 +45,36 @@ class Transformer(object): @abc.abstractmethod def transform(self, entity_event): - """Transforms an entity event into entity wrapper + """Transforms an entity event into entity wrapper. - :return: An EntityWrapper. EntityWrapper is namedTuple that contains - an entity vertex and a list of vertex and an edge pair that describe - the entity's neighbors. + :return: An EntityWrapper. EntityWrapper - a namedTuple that contains + three fields: + 1. Vertex - Entity vertex + 2. Neighbors - vertex and an edge pairs + 3. Action - CREATE/UPDATE/DELETE :rtype: EntityWrapper """ pass - def key_fields(self): + @staticmethod + def key_fields(): + """Returns a field list that are must to create the entity key. + + The field order is important to. + :return: field list + """ return [cons.VertexProperties.TYPE, cons.VertexProperties.SUB_TYPE, cons.VertexProperties.ID] @abc.abstractmethod def extract_key(self, entity_event): + """Extract entity key from given event + + By given an entity event, return a entity key which + consisting key fields + + :param entity_event: event that returns from the synchronizer + :return: key + """ pass diff --git a/vitrage/entity_graph/transformer/nova_transformer.py b/vitrage/entity_graph/transformer/nova_transformer.py index afb902590..75decde98 100644 --- a/vitrage/entity_graph/transformer/nova_transformer.py +++ b/vitrage/entity_graph/transformer/nova_transformer.py @@ -15,6 +15,7 @@ from oslo_log import log as logging import vitrage.common.constants as cons +from vitrage.common.exception import VitrageTransformerError from vitrage.entity_graph.transformer import base import vitrage.graph.utils as graph_utils @@ -26,27 +27,48 @@ HOST_SUBTYPE = 'nova.host' class InstanceTransformer(base.Transformer): - # # Fields returned from Nova Instance snapshot - SNAPSHOT_INSTANCE_ID = 'id' - UPDATE_INSTANCE_ID = 'instance_id' + # Fields returned from Nova Instance snapshot + INSTANCE_ID = { + cons.SyncMode.SNAPSHOT: ('id',), + cons.SyncMode.INIT_SNAPSHOT: ('id',), + cons.SyncMode.UPDATE: ('payload', 'instance_id') + } - SNAPSHOT_INSTANCE_STATE = 'status' - UPDATE_INSTANCE_STATE = 'state' + INSTANCE_STATE = { + cons.SyncMode.SNAPSHOT: ('status',), + cons.SyncMode.INIT_SNAPSHOT: ('status',), + cons.SyncMode.UPDATE: ('payload', 'state') + } - SNAPSHOT_TIMESTAMP = 'updated' - UPDATE_TIMESTAMP = 'metadata;timestamp' + TIMESTAMP = { + cons.SyncMode.SNAPSHOT: ('updated',), + cons.SyncMode.INIT_SNAPSHOT: ('updated',), + cons.SyncMode.UPDATE: ('metadata', 'timestamp') + } - PROJECT_ID = 'tenant_id' - INSTANCE_NAME = 'name' - HOST_NAME = 'OS-EXT-SRV-ATTR:host' + HOST_NAME = { + cons.SyncMode.SNAPSHOT: ('OS-EXT-SRV-ATTR:host',), + cons.SyncMode.INIT_SNAPSHOT: ('OS-EXT-SRV-ATTR:host',), + cons.SyncMode.UPDATE: ('payload', 'host') + } - def __init__(self): + PROJECT_ID = { + cons.SyncMode.SNAPSHOT: ('tenant_id',), + cons.SyncMode.INIT_SNAPSHOT: ('tenant_id',), + cons.SyncMode.UPDATE: ('payload', 'tenant_id') + } - self.transform_methods = { - cons.SyncMode.SNAPSHOT: self._transform_snapshot_event, - cons.SyncMode.INIT_SNAPSHOT: self._transform_init_snapshot_event, - cons.SyncMode.UPDATE: self._transform_update_event - } + INSTANCE_NAME = { + cons.SyncMode.SNAPSHOT: ('name',), + cons.SyncMode.INIT_SNAPSHOT: ('name',), + cons.SyncMode.UPDATE: ('payload', 'hostname') + } + + # Event types which need to refer them differently + INSTANCE_EVENT_TYPES = { + 'compute.instance.delete.end': cons.EventAction.DELETE, + 'compute.instance.create.start': cons.EventAction.CREATE + } def transform(self, entity_event): """Transform an entity event into entity wrapper. @@ -62,64 +84,73 @@ class InstanceTransformer(base.Transformer): :rtype:EntityWrapper """ sync_mode = entity_event['sync_mode'] - return self.transform_methods[sync_mode](entity_event) - def _transform_snapshot_event(self, entity_event): + field_value = base.extract_field_value + + metadata = { + cons.VertexProperties.NAME: field_value( + entity_event, + self.INSTANCE_NAME[sync_mode] + ), + cons.VertexProperties.IS_PLACEHOLDER: False + } entity_key = self.extract_key(entity_event) - metadata = { - cons.VertexProperties.NAME: entity_event[self.INSTANCE_NAME] - } + + entity_id = field_value(entity_event, self.INSTANCE_ID[sync_mode]) + project = field_value(entity_event, self.PROJECT_ID[sync_mode]) + state = field_value(entity_event, self.INSTANCE_STATE[sync_mode]) + update_timestamp = field_value( + entity_event, + self.TIMESTAMP[sync_mode] + ) entity_vertex = graph_utils.create_vertex( entity_key, - entity_id=entity_event[self.SNAPSHOT_INSTANCE_ID], + entity_id=entity_id, entity_type=cons.EntityTypes.RESOURCE, entity_subtype=INSTANCE_SUBTYPE, - entity_project=entity_event[self.PROJECT_ID], - entity_state=entity_event[self.SNAPSHOT_INSTANCE_STATE], - update_timestamp=entity_event[self.SNAPSHOT_TIMESTAMP], - is_placeholder=False, + entity_project=project, + entity_state=state, + update_timestamp=update_timestamp, metadata=metadata ) host_neighbor = self.create_host_neighbor( entity_vertex.vertex_id, - entity_event[self.HOST_NAME]) + field_value(entity_event, self.HOST_NAME[sync_mode]) + ) return base.EntityWrapper( entity_vertex, [host_neighbor], - cons.EventAction.UPDATE) + self._extract_action_type(entity_event)) - def _transform_init_snapshot_event(self, entity_event): - - entity_wrapper = self._transform_snapshot_event(entity_event) - # TODO(Liat): check why set is forbidden - # entity_wrapper.action = cons.EventAction.CREATE - entity_wrapper = base.EntityWrapper(entity_wrapper.vertex, - entity_wrapper.neighbors, - cons.EventAction.CREATE) - return entity_wrapper - - def _transform_update_event(self, entity_event): - pass - - # def key_fields(self): - # return [cons.VertexProperties.TYPE, - # cons.VertexProperties.SUB_TYPE, - # cons.VertexProperties.ID] - - def extract_key(self, entity_event): + def _extract_action_type(self, entity_event): sync_mode = entity_event['sync_mode'] - if sync_mode == cons.SyncMode.UPDATE: - event_id = entity_event[self.UPDATE_INSTANCE_ID] - else: - event_id = entity_event[self.SNAPSHOT_INSTANCE_ID] + if cons.SyncMode.UPDATE == sync_mode: + return self.INSTANCE_EVENT_TYPES.get( + entity_event['sync_mode'], + cons.EventAction.UPDATE) - return self.build_instance_key(event_id) + if cons.SyncMode.SNAPSHOT == sync_mode: + return cons.EventAction.CREATE + + if cons.SyncMode.INIT_SNAPSHOT == sync_mode: + return cons.EventAction.UPDATE + + raise VitrageTransformerError( + 'Invalid sync mode: (%s)' % sync_mode) + return None + + def extract_key(self, entity_event): + + instance_id = base.extract_field_value( + entity_event, + self.INSTANCE_ID[entity_event['sync_mode']]) + return self.build_instance_key(instance_id) @staticmethod def build_instance_key(instance_id): diff --git a/vitrage/tests/unit/processor/test_processor.py b/vitrage/tests/unit/processor/test_processor.py index 7a4472a64..911d1eeb5 100644 --- a/vitrage/tests/unit/processor/test_processor.py +++ b/vitrage/tests/unit/processor/test_processor.py @@ -68,6 +68,8 @@ class TestProcessor(base.BaseTest): self.NUM_EDGES_AFTER_CREATION) # check update instance even + # TODO(Alexey): Create an event in update event structure + # (update snapshot fields won't work) event['sync_mode'] = SyncMode.UPDATE event['event_type'] = 'compute.instance.volume.attach' event['hostname'] = 'new_host' diff --git a/vitrage/tests/unit/transformers/test_nova_transformers.py b/vitrage/tests/unit/transformers/test_nova_transformers.py index fc4c2680e..0bf89085e 100644 --- a/vitrage/tests/unit/transformers/test_nova_transformers.py +++ b/vitrage/tests/unit/transformers/test_nova_transformers.py @@ -42,74 +42,6 @@ def get_instance_entity_spec_list(config_file_path, number_of_instances): class NovaInstanceTransformerTest(base.BaseTest): - def test_transform_snapshot_event(self): - LOG.debug('Test transform snapshot event') - - transformer = get_nova_instance_transformer() - - spec_list = mock_sync.simple_instance_generators(1, 2, 10) - instance_events = mock_sync.generate_random_events_list(spec_list) - - for event in instance_events: - entity_wrapper = transformer._transform_snapshot_event(event) - - self.assertEqual(cons.EventAction.UPDATE, - entity_wrapper.action) - - expected_key = transformer.extract_key(event) - self.assertEqual(expected_key, entity_wrapper.vertex.vertex_id) - - self._validate_snapshot_vertex_props(entity_wrapper.vertex, event) - - self.assertEqual(1, entity_wrapper.neighbors.__len__()) - self._validate_host_neighbor( - entity_wrapper.neighbors[0], - event[transformer.HOST_NAME]) - - def _validate_host_neighbor(self, h_neighbor, host_name): - expected_neighbor = nt.HostTransformer.\ - create_placeholder_vertex(host_name) - self.assertEqual(expected_neighbor, h_neighbor.vertex) - - def _validate_snapshot_vertex_props(self, vertex, event): - - # properties = vertex.properties - self.assertEqual(9, vertex.properties.__len__()) - - expected_id = event[nt.InstanceTransformer.SNAPSHOT_INSTANCE_ID] - observed_id = vertex.get(cons.VertexProperties.ID) - self.assertEqual(expected_id, observed_id) - - self.assertEqual(cons.EntityTypes.RESOURCE, - vertex.get(cons.VertexProperties.TYPE)) - - self.assertEqual(nt.INSTANCE_SUBTYPE, - vertex.get(cons.VertexProperties.SUB_TYPE)) - - expected_subtype = event[nt.InstanceTransformer.SNAPSHOT_INSTANCE_ID] - observed_subtype = vertex.get(cons.VertexProperties.ID) - self.assertEqual(expected_subtype, observed_subtype) - - expected_project = event[nt.InstanceTransformer.PROJECT_ID] - observed_project = vertex.get(cons.VertexProperties.PROJECT) - self.assertEqual(expected_project, observed_project) - - expected_state = event[nt.InstanceTransformer.SNAPSHOT_INSTANCE_STATE] - observed_state = vertex.get(cons.VertexProperties.STATE) - self.assertEqual(expected_state, observed_state) - - expected_timestamp = event[nt.InstanceTransformer.SNAPSHOT_TIMESTAMP] - observed_timestamp = vertex.get( - cons.VertexProperties.UPDATE_TIMESTAMP) - self.assertEqual(expected_timestamp, observed_timestamp) - - expected_name = event[nt.InstanceTransformer.INSTANCE_NAME] - observed_name = vertex.get(cons.VertexProperties.NAME) - self.assertEqual(expected_name, observed_name) - - is_placeholder = vertex.get(cons.VertexProperties.IS_PLACEHOLDER) - self.assertEqual(False, is_placeholder) - def test_key_fields(self): LOG.debug('Test get key fields from nova instance transformer') transformer = get_nova_instance_transformer() @@ -120,6 +52,113 @@ class NovaInstanceTransformerTest(base.BaseTest): observed_key_fields = transformer.key_fields() self.assert_list_equal(expected_key_fields, observed_key_fields) + def test_transform(self): + LOG.debug('Test actual transform action') + + transformer = get_nova_instance_transformer() + + spec_list = mock_sync.simple_instance_generators( + host_num=1, + vm_num=1, + snapshot_events=10, + update_events=5 + ) + instance_events = mock_sync.generate_random_events_list(spec_list) + + for event in instance_events: + wrapper = transformer.transform(event) + self._validate_vertex_props(wrapper.vertex, event) + + # Validate the neighbor list: + # 1. Exactly one host neighbor and and + # 3. Make sure the host neighbor is correct + # 2. TODO(lhartal): validate volume neighbors + host_neighbor_exist = False + for neighbor in wrapper.neighbors: + + neighbor_subtype = neighbor.vertex[ + cons.VertexProperties.SUB_TYPE + ] + + if neighbor_subtype == nt.HOST_SUBTYPE: + self.assertEqual(False, + host_neighbor_exist, + 'Instance has only one host neighbor') + self._validate_host_neighbor(neighbor, event) + + def _validate_vertex_props(self, vertex, event): + + self.assertEqual(9, vertex.properties.__len__()) + + sync_mode = event['sync_mode'] + + it = nt.InstanceTransformer + expected_id = trans_base.extract_field_value( + event, + it.INSTANCE_ID[sync_mode] + ) + observed_id = vertex[cons.VertexProperties.ID] + self.assertEqual(expected_id, observed_id) + + self.assertEqual(cons.EntityTypes.RESOURCE, + vertex[cons.VertexProperties.TYPE]) + + self.assertEqual(nt.INSTANCE_SUBTYPE, + vertex[cons.VertexProperties.SUB_TYPE]) + + expected_project = trans_base.extract_field_value( + event, + it.PROJECT_ID[sync_mode] + ) + observed_project = vertex[cons.VertexProperties.PROJECT] + self.assertEqual(expected_project, observed_project) + + expected_state = trans_base.extract_field_value( + event, + it.INSTANCE_STATE[sync_mode] + ) + observed_state = vertex[cons.VertexProperties.STATE] + self.assertEqual(expected_state, observed_state) + + expected_timestamp = trans_base.extract_field_value( + event, + it.TIMESTAMP[sync_mode] + ) + observed_timestamp = vertex[cons.VertexProperties.UPDATE_TIMESTAMP] + self.assertEqual(expected_timestamp, observed_timestamp) + + expected_name = trans_base.extract_field_value( + event, + it.INSTANCE_NAME[sync_mode] + ) + observed_name = vertex[cons.VertexProperties.NAME] + self.assertEqual(expected_name, observed_name) + + is_placeholder = vertex[cons.VertexProperties.IS_PLACEHOLDER] + self.assertEqual(False, is_placeholder) + + is_deleted = vertex[cons.VertexProperties.IS_DELETED] + self.assertEqual(False, is_deleted) + + def _validate_host_neighbor(self, h_neighbor, event): + + it = nt.InstanceTransformer() + sync_mode = event['sync_mode'] + host_name = trans_base.extract_field_value( + event, + it.HOST_NAME[sync_mode] + ) + expected_neighbor = nt.HostTransformer.create_placeholder_vertex( + host_name + ) + self.assertEqual(expected_neighbor, h_neighbor.vertex) + + # Validate neighbor edge + edge = h_neighbor.edge + self.assertEqual(edge.source_id, h_neighbor.vertex.vertex_id) + self.assertEqual(edge.target_id, it.extract_key(event)) + self.assertEqual(edge.label, cons.EdgeLabels.CONTAINS) + def test_extract_key(self): LOG.debug('Test get key from nova instance transformer') @@ -141,10 +180,10 @@ class NovaInstanceTransformerTest(base.BaseTest): self.assertEqual(cons.EntityTypes.RESOURCE, observed_key_fields[0]) self.assertEqual(nt.INSTANCE_SUBTYPE, observed_key_fields[1]) - if cons.SyncMode.UPDATE == event['sync_mode']: - event_id = event[transformer.UPDATE_INSTANCE_ID] - else: - event_id = event[transformer.SNAPSHOT_INSTANCE_ID] + event_id = trans_base.extract_field_value( + event, + transformer.INSTANCE_ID[event['sync_mode']] + ) self.assertEqual(event_id, observed_key_fields[2]) @@ -183,8 +222,8 @@ class NovaInstanceTransformerTest(base.BaseTest): instance_id = '123456' vertex_id = nt.InstanceTransformer.build_instance_key(instance_id) - p_vertex = nt.InstanceTransformer.\ - create_placeholder_vertex(instance_id) + p_vertex = nt.InstanceTransformer.create_placeholder_vertex( + instance_id) self.assertEqual(vertex_id, p_vertex.vertex_id) self.assertEqual(5, p_vertex.properties.keys().__len__())