nova transformer - transform instance update events

Change-Id: I683ef276851a11704b576da5af3488881cc7d20e
This commit is contained in:
Liat Har-Tal 2015-12-30 13:56:53 +00:00
parent f70ca274b9
commit c7e0b55d35
4 changed files with 228 additions and 131 deletions

View File

@ -29,6 +29,15 @@ EntityWrapper = \
Neighbor = namedtuple('Neighbor', ['vertex', 'edge'])
def extract_field_value(entity_event, key_names):
value = entity_event
for key in key_names:
value = value[key]
return value
@six.add_metaclass(abc.ABCMeta)
class Transformer(object):
@ -36,20 +45,36 @@ class Transformer(object):
@abc.abstractmethod
def transform(self, entity_event):
"""Transforms an entity event into entity wrapper
"""Transforms an entity event into entity wrapper.
:return: An EntityWrapper. EntityWrapper is namedTuple that contains
an entity vertex and a list of vertex and an edge pair that describe
the entity's neighbors.
:return: An EntityWrapper. EntityWrapper - a namedTuple that contains
three fields:
1. Vertex - Entity vertex
2. Neighbors - vertex and an edge pairs
3. Action - CREATE/UPDATE/DELETE
:rtype: EntityWrapper
"""
pass
def key_fields(self):
@staticmethod
def key_fields():
"""Returns a field list that are must to create the entity key.
The field order is important to.
:return: field list
"""
return [cons.VertexProperties.TYPE,
cons.VertexProperties.SUB_TYPE,
cons.VertexProperties.ID]
@abc.abstractmethod
def extract_key(self, entity_event):
"""Extract entity key from given event
By given an entity event, return a entity key which
consisting key fields
:param entity_event: event that returns from the synchronizer
:return: key
"""
pass

View File

@ -15,6 +15,7 @@
from oslo_log import log as logging
import vitrage.common.constants as cons
from vitrage.common.exception import VitrageTransformerError
from vitrage.entity_graph.transformer import base
import vitrage.graph.utils as graph_utils
@ -26,27 +27,48 @@ HOST_SUBTYPE = 'nova.host'
class InstanceTransformer(base.Transformer):
# # Fields returned from Nova Instance snapshot
SNAPSHOT_INSTANCE_ID = 'id'
UPDATE_INSTANCE_ID = 'instance_id'
# Fields returned from Nova Instance snapshot
INSTANCE_ID = {
cons.SyncMode.SNAPSHOT: ('id',),
cons.SyncMode.INIT_SNAPSHOT: ('id',),
cons.SyncMode.UPDATE: ('payload', 'instance_id')
}
SNAPSHOT_INSTANCE_STATE = 'status'
UPDATE_INSTANCE_STATE = 'state'
INSTANCE_STATE = {
cons.SyncMode.SNAPSHOT: ('status',),
cons.SyncMode.INIT_SNAPSHOT: ('status',),
cons.SyncMode.UPDATE: ('payload', 'state')
}
SNAPSHOT_TIMESTAMP = 'updated'
UPDATE_TIMESTAMP = 'metadata;timestamp'
TIMESTAMP = {
cons.SyncMode.SNAPSHOT: ('updated',),
cons.SyncMode.INIT_SNAPSHOT: ('updated',),
cons.SyncMode.UPDATE: ('metadata', 'timestamp')
}
PROJECT_ID = 'tenant_id'
INSTANCE_NAME = 'name'
HOST_NAME = 'OS-EXT-SRV-ATTR:host'
HOST_NAME = {
cons.SyncMode.SNAPSHOT: ('OS-EXT-SRV-ATTR:host',),
cons.SyncMode.INIT_SNAPSHOT: ('OS-EXT-SRV-ATTR:host',),
cons.SyncMode.UPDATE: ('payload', 'host')
}
def __init__(self):
PROJECT_ID = {
cons.SyncMode.SNAPSHOT: ('tenant_id',),
cons.SyncMode.INIT_SNAPSHOT: ('tenant_id',),
cons.SyncMode.UPDATE: ('payload', 'tenant_id')
}
self.transform_methods = {
cons.SyncMode.SNAPSHOT: self._transform_snapshot_event,
cons.SyncMode.INIT_SNAPSHOT: self._transform_init_snapshot_event,
cons.SyncMode.UPDATE: self._transform_update_event
}
INSTANCE_NAME = {
cons.SyncMode.SNAPSHOT: ('name',),
cons.SyncMode.INIT_SNAPSHOT: ('name',),
cons.SyncMode.UPDATE: ('payload', 'hostname')
}
# Event types which need to refer them differently
INSTANCE_EVENT_TYPES = {
'compute.instance.delete.end': cons.EventAction.DELETE,
'compute.instance.create.start': cons.EventAction.CREATE
}
def transform(self, entity_event):
"""Transform an entity event into entity wrapper.
@ -62,64 +84,73 @@ class InstanceTransformer(base.Transformer):
:rtype:EntityWrapper
"""
sync_mode = entity_event['sync_mode']
return self.transform_methods[sync_mode](entity_event)
def _transform_snapshot_event(self, entity_event):
field_value = base.extract_field_value
metadata = {
cons.VertexProperties.NAME: field_value(
entity_event,
self.INSTANCE_NAME[sync_mode]
),
cons.VertexProperties.IS_PLACEHOLDER: False
}
entity_key = self.extract_key(entity_event)
metadata = {
cons.VertexProperties.NAME: entity_event[self.INSTANCE_NAME]
}
entity_id = field_value(entity_event, self.INSTANCE_ID[sync_mode])
project = field_value(entity_event, self.PROJECT_ID[sync_mode])
state = field_value(entity_event, self.INSTANCE_STATE[sync_mode])
update_timestamp = field_value(
entity_event,
self.TIMESTAMP[sync_mode]
)
entity_vertex = graph_utils.create_vertex(
entity_key,
entity_id=entity_event[self.SNAPSHOT_INSTANCE_ID],
entity_id=entity_id,
entity_type=cons.EntityTypes.RESOURCE,
entity_subtype=INSTANCE_SUBTYPE,
entity_project=entity_event[self.PROJECT_ID],
entity_state=entity_event[self.SNAPSHOT_INSTANCE_STATE],
update_timestamp=entity_event[self.SNAPSHOT_TIMESTAMP],
is_placeholder=False,
entity_project=project,
entity_state=state,
update_timestamp=update_timestamp,
metadata=metadata
)
host_neighbor = self.create_host_neighbor(
entity_vertex.vertex_id,
entity_event[self.HOST_NAME])
field_value(entity_event, self.HOST_NAME[sync_mode])
)
return base.EntityWrapper(
entity_vertex,
[host_neighbor],
cons.EventAction.UPDATE)
self._extract_action_type(entity_event))
def _transform_init_snapshot_event(self, entity_event):
entity_wrapper = self._transform_snapshot_event(entity_event)
# TODO(Liat): check why set is forbidden
# entity_wrapper.action = cons.EventAction.CREATE
entity_wrapper = base.EntityWrapper(entity_wrapper.vertex,
entity_wrapper.neighbors,
cons.EventAction.CREATE)
return entity_wrapper
def _transform_update_event(self, entity_event):
pass
# def key_fields(self):
# return [cons.VertexProperties.TYPE,
# cons.VertexProperties.SUB_TYPE,
# cons.VertexProperties.ID]
def extract_key(self, entity_event):
def _extract_action_type(self, entity_event):
sync_mode = entity_event['sync_mode']
if sync_mode == cons.SyncMode.UPDATE:
event_id = entity_event[self.UPDATE_INSTANCE_ID]
else:
event_id = entity_event[self.SNAPSHOT_INSTANCE_ID]
if cons.SyncMode.UPDATE == sync_mode:
return self.INSTANCE_EVENT_TYPES.get(
entity_event['sync_mode'],
cons.EventAction.UPDATE)
return self.build_instance_key(event_id)
if cons.SyncMode.SNAPSHOT == sync_mode:
return cons.EventAction.CREATE
if cons.SyncMode.INIT_SNAPSHOT == sync_mode:
return cons.EventAction.UPDATE
raise VitrageTransformerError(
'Invalid sync mode: (%s)' % sync_mode)
return None
def extract_key(self, entity_event):
instance_id = base.extract_field_value(
entity_event,
self.INSTANCE_ID[entity_event['sync_mode']])
return self.build_instance_key(instance_id)
@staticmethod
def build_instance_key(instance_id):

View File

@ -68,6 +68,8 @@ class TestProcessor(base.BaseTest):
self.NUM_EDGES_AFTER_CREATION)
# check update instance even
# TODO(Alexey): Create an event in update event structure
# (update snapshot fields won't work)
event['sync_mode'] = SyncMode.UPDATE
event['event_type'] = 'compute.instance.volume.attach'
event['hostname'] = 'new_host'

View File

@ -42,74 +42,6 @@ def get_instance_entity_spec_list(config_file_path, number_of_instances):
class NovaInstanceTransformerTest(base.BaseTest):
def test_transform_snapshot_event(self):
LOG.debug('Test transform snapshot event')
transformer = get_nova_instance_transformer()
spec_list = mock_sync.simple_instance_generators(1, 2, 10)
instance_events = mock_sync.generate_random_events_list(spec_list)
for event in instance_events:
entity_wrapper = transformer._transform_snapshot_event(event)
self.assertEqual(cons.EventAction.UPDATE,
entity_wrapper.action)
expected_key = transformer.extract_key(event)
self.assertEqual(expected_key, entity_wrapper.vertex.vertex_id)
self._validate_snapshot_vertex_props(entity_wrapper.vertex, event)
self.assertEqual(1, entity_wrapper.neighbors.__len__())
self._validate_host_neighbor(
entity_wrapper.neighbors[0],
event[transformer.HOST_NAME])
def _validate_host_neighbor(self, h_neighbor, host_name):
expected_neighbor = nt.HostTransformer.\
create_placeholder_vertex(host_name)
self.assertEqual(expected_neighbor, h_neighbor.vertex)
def _validate_snapshot_vertex_props(self, vertex, event):
# properties = vertex.properties
self.assertEqual(9, vertex.properties.__len__())
expected_id = event[nt.InstanceTransformer.SNAPSHOT_INSTANCE_ID]
observed_id = vertex.get(cons.VertexProperties.ID)
self.assertEqual(expected_id, observed_id)
self.assertEqual(cons.EntityTypes.RESOURCE,
vertex.get(cons.VertexProperties.TYPE))
self.assertEqual(nt.INSTANCE_SUBTYPE,
vertex.get(cons.VertexProperties.SUB_TYPE))
expected_subtype = event[nt.InstanceTransformer.SNAPSHOT_INSTANCE_ID]
observed_subtype = vertex.get(cons.VertexProperties.ID)
self.assertEqual(expected_subtype, observed_subtype)
expected_project = event[nt.InstanceTransformer.PROJECT_ID]
observed_project = vertex.get(cons.VertexProperties.PROJECT)
self.assertEqual(expected_project, observed_project)
expected_state = event[nt.InstanceTransformer.SNAPSHOT_INSTANCE_STATE]
observed_state = vertex.get(cons.VertexProperties.STATE)
self.assertEqual(expected_state, observed_state)
expected_timestamp = event[nt.InstanceTransformer.SNAPSHOT_TIMESTAMP]
observed_timestamp = vertex.get(
cons.VertexProperties.UPDATE_TIMESTAMP)
self.assertEqual(expected_timestamp, observed_timestamp)
expected_name = event[nt.InstanceTransformer.INSTANCE_NAME]
observed_name = vertex.get(cons.VertexProperties.NAME)
self.assertEqual(expected_name, observed_name)
is_placeholder = vertex.get(cons.VertexProperties.IS_PLACEHOLDER)
self.assertEqual(False, is_placeholder)
def test_key_fields(self):
LOG.debug('Test get key fields from nova instance transformer')
transformer = get_nova_instance_transformer()
@ -120,6 +52,113 @@ class NovaInstanceTransformerTest(base.BaseTest):
observed_key_fields = transformer.key_fields()
self.assert_list_equal(expected_key_fields, observed_key_fields)
def test_transform(self):
LOG.debug('Test actual transform action')
transformer = get_nova_instance_transformer()
spec_list = mock_sync.simple_instance_generators(
host_num=1,
vm_num=1,
snapshot_events=10,
update_events=5
)
instance_events = mock_sync.generate_random_events_list(spec_list)
for event in instance_events:
wrapper = transformer.transform(event)
self._validate_vertex_props(wrapper.vertex, event)
# Validate the neighbor list:
# 1. Exactly one host neighbor and and
# 3. Make sure the host neighbor is correct
# 2. TODO(lhartal): validate volume neighbors
host_neighbor_exist = False
for neighbor in wrapper.neighbors:
neighbor_subtype = neighbor.vertex[
cons.VertexProperties.SUB_TYPE
]
if neighbor_subtype == nt.HOST_SUBTYPE:
self.assertEqual(False,
host_neighbor_exist,
'Instance has only one host neighbor')
self._validate_host_neighbor(neighbor, event)
def _validate_vertex_props(self, vertex, event):
self.assertEqual(9, vertex.properties.__len__())
sync_mode = event['sync_mode']
it = nt.InstanceTransformer
expected_id = trans_base.extract_field_value(
event,
it.INSTANCE_ID[sync_mode]
)
observed_id = vertex[cons.VertexProperties.ID]
self.assertEqual(expected_id, observed_id)
self.assertEqual(cons.EntityTypes.RESOURCE,
vertex[cons.VertexProperties.TYPE])
self.assertEqual(nt.INSTANCE_SUBTYPE,
vertex[cons.VertexProperties.SUB_TYPE])
expected_project = trans_base.extract_field_value(
event,
it.PROJECT_ID[sync_mode]
)
observed_project = vertex[cons.VertexProperties.PROJECT]
self.assertEqual(expected_project, observed_project)
expected_state = trans_base.extract_field_value(
event,
it.INSTANCE_STATE[sync_mode]
)
observed_state = vertex[cons.VertexProperties.STATE]
self.assertEqual(expected_state, observed_state)
expected_timestamp = trans_base.extract_field_value(
event,
it.TIMESTAMP[sync_mode]
)
observed_timestamp = vertex[cons.VertexProperties.UPDATE_TIMESTAMP]
self.assertEqual(expected_timestamp, observed_timestamp)
expected_name = trans_base.extract_field_value(
event,
it.INSTANCE_NAME[sync_mode]
)
observed_name = vertex[cons.VertexProperties.NAME]
self.assertEqual(expected_name, observed_name)
is_placeholder = vertex[cons.VertexProperties.IS_PLACEHOLDER]
self.assertEqual(False, is_placeholder)
is_deleted = vertex[cons.VertexProperties.IS_DELETED]
self.assertEqual(False, is_deleted)
def _validate_host_neighbor(self, h_neighbor, event):
it = nt.InstanceTransformer()
sync_mode = event['sync_mode']
host_name = trans_base.extract_field_value(
event,
it.HOST_NAME[sync_mode]
)
expected_neighbor = nt.HostTransformer.create_placeholder_vertex(
host_name
)
self.assertEqual(expected_neighbor, h_neighbor.vertex)
# Validate neighbor edge
edge = h_neighbor.edge
self.assertEqual(edge.source_id, h_neighbor.vertex.vertex_id)
self.assertEqual(edge.target_id, it.extract_key(event))
self.assertEqual(edge.label, cons.EdgeLabels.CONTAINS)
def test_extract_key(self):
LOG.debug('Test get key from nova instance transformer')
@ -141,10 +180,10 @@ class NovaInstanceTransformerTest(base.BaseTest):
self.assertEqual(cons.EntityTypes.RESOURCE, observed_key_fields[0])
self.assertEqual(nt.INSTANCE_SUBTYPE, observed_key_fields[1])
if cons.SyncMode.UPDATE == event['sync_mode']:
event_id = event[transformer.UPDATE_INSTANCE_ID]
else:
event_id = event[transformer.SNAPSHOT_INSTANCE_ID]
event_id = trans_base.extract_field_value(
event,
transformer.INSTANCE_ID[event['sync_mode']]
)
self.assertEqual(event_id, observed_key_fields[2])
@ -183,8 +222,8 @@ class NovaInstanceTransformerTest(base.BaseTest):
instance_id = '123456'
vertex_id = nt.InstanceTransformer.build_instance_key(instance_id)
p_vertex = nt.InstanceTransformer.\
create_placeholder_vertex(instance_id)
p_vertex = nt.InstanceTransformer.create_placeholder_vertex(
instance_id)
self.assertEqual(vertex_id, p_vertex.vertex_id)
self.assertEqual(5, p_vertex.properties.keys().__len__())