openstack/vitrage
Code Issues Proposed changes

Neighbor validation for static datasource transformer

Browse Source 
Implement: blueprint static-datasource-config-format
Change-Id: Ic1a30e0048708b6a43f2248ffab979dd0b738367
This commit is contained in:
Yujun Zhang 2017-01-13 16:26:33 +08:00
parent 81a318d01f
commit c771ad1a1b
6 changed files with 97 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
vitrage/common/constants.py
View File

@ -122,6 +122,7 @@ class TopologyFields(object):
ENTITIES = 'entities'
ENTITY = 'entity'
TYPE = 'type'
ID = 'id'
RELATIONSHIPS = 'relationships'
RELATIONSHIP = 'relationship'

12
vitrage/tests/mocks/mock_driver.py
View File

@ -35,8 +35,10 @@ usage example:
import random
from vitrage.common.constants import DatasourceAction
from vitrage.common.constants import DatasourceProperties as DSProps
import vitrage.tests.mocks.trace_generator as tg
from vitrage.utils.datetime import utcnow
def generate_random_events_list(generator_spec_list):
@ -391,10 +393,12 @@ def simple_static_generators(switch_num=2, host_num=10,
if snapshot_events > 0:
if snap_vals is None:
snap_vals = {}
snap_vals[DSProps.DATASOURCE_ACTION] = 'update'
snap_vals.update({
DSProps.DATASOURCE_ACTION: DatasourceAction.SNAPSHOT,
DSProps.SAMPLE_DATE: utcnow()})
test_entity_spec_list.append(
{tg.DYNAMIC_INFO_FKEY: tg.DRIVER_STATIC_SNAPSHOT_D,
tg.STATIC_INFO_FKEY: None,
tg.STATIC_INFO_FKEY: tg.DRIVER_STATIC_SNAPSHOT_S,
tg.EXTERNAL_INFO_KEY: snap_vals,
tg.MAPPING_KEY: mapping,
tg.NAME_KEY: 'Static snapshot generator',
@ -404,7 +408,9 @@ def simple_static_generators(switch_num=2, host_num=10,
if update_events > 0:
if update_vals is None:
update_vals = {}
update_vals[DSProps.DATASOURCE_ACTION] = 'update'
update_vals.update({
DSProps.DATASOURCE_ACTION: DatasourceAction.UPDATE,
DSProps.SAMPLE_DATE: utcnow()})
test_entity_spec_list.append(
{tg.DYNAMIC_INFO_FKEY: tg.DRIVER_STATIC_SNAPSHOT_D,
tg.STATIC_INFO_FKEY: None,

97
vitrage/tests/mocks/trace_generator.py
View File

@ -26,7 +26,10 @@ from collections import defaultdict
from random import randint
# noinspection PyPep8Naming
from vitrage.common.constants import EdgeLabel
from vitrage.common.constants import TopologyFields
from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE
from vitrage.datasources.static import StaticFields
from vitrage.tests.mocks.entity_model import BasicEntityModel as Bem
import vitrage.tests.mocks.utils as utils
@ -57,6 +60,7 @@ DRIVER_NAGIOS_SNAPSHOT_S = 'driver_nagios_snapshot_static.json'
DRIVER_ZABBIX_SNAPSHOT_D = 'driver_zabbix_snapshot_dynamic.json'
DRIVER_SWITCH_SNAPSHOT_D = 'driver_switch_snapshot_dynamic.json'
DRIVER_STATIC_SNAPSHOT_D = 'driver_static_snapshot_dynamic.json'
DRIVER_STATIC_SNAPSHOT_S = 'driver_static_snapshot_static.json'
DRIVER_VOLUME_UPDATE_D = 'driver_volume_update_dynamic.json'
DRIVER_VOLUME_SNAPSHOT_D = 'driver_volume_snapshot_dynamic.json'
DRIVER_STACK_UPDATE_D = 'driver_stack_update_dynamic.json'
@ -547,81 +551,84 @@ def _get_static_snapshot_driver_values(spec):
"""
host_switch_mapping = spec[MAPPING_KEY]
static_info_spec = None
if spec[STATIC_INFO_FKEY] is not None:
static_info_spec = utils.load_specs(spec[STATIC_INFO_FKEY])
else:
static_info_spec = None
static_values = []
# use defaultdict to create placeholder
relationships = defaultdict(lambda: [])
entities = defaultdict(lambda: {})
touched = set({})
for host_index, switch_index in host_switch_mapping:
host_id = "h{}".format(host_index)
switch_id = "s{}".format(switch_index)
relationship = {
"source": switch_id,
"target": host_id,
"relationship_type": "attached"
TopologyFields.SOURCE: switch_id,
TopologyFields.TARGET: host_id,
TopologyFields.RELATIONSHIP_TYPE: EdgeLabel.ATTACHED
}
host_rel = relationship.copy()
host_rel['source'] = entities[switch_id]
relationships[host_id].append(host_rel)
switch_rel = relationship.copy()
switch_rel['target'] = entities[host_id]
relationships[switch_id].append(switch_rel)
rel = relationship.copy()
rel[TopologyFields.TARGET] = entities[host_id]
relationships[switch_id].append(rel)
for host_index, switch_index in host_switch_mapping:
mapping = {}
switch_id = "s{}".format(switch_index)
if switch_id not in entities:
if switch_id not in touched:
switch_name = "switch-{}".format(switch_index)
mapping = {
'name': switch_name,
'config_id': switch_id,
'id': str(randint(0, 100000)),
'type': 'switch',
'relationships': relationships[switch_id]
vals = {
StaticFields.CONFIG_ID: switch_id,
StaticFields.TYPE: 'switch',
StaticFields.ID: str(randint(0, 100000)),
StaticFields.NAME: switch_name,
StaticFields.RELATIONSHIPS: relationships[switch_id]
}
entities[switch_id].update(**mapping)
entities[switch_id].update(**vals)
touched.add(switch_id)
host_id = "h{}".format(host_index)
if host_id not in entities:
mapping = {
'config_id': host_id,
'type': 'nova.host',
'id': str(randint(0, 100000)),
'relationships': relationships[host_id]
if host_id not in touched:
vals = {
StaticFields.CONFIG_ID: host_id,
StaticFields.TYPE: NOVA_HOST_DATASOURCE,
StaticFields.ID: str(randint(0, 100000)),
StaticFields.RELATIONSHIPS: relationships[host_id]
}
entities[host_id].update(**mapping)
entities[host_id].update(**vals)
touched.add(host_id)
for vals in entities.values():
static_values.append(combine_data(static_info_spec,
mapping,
vals,
spec.get(EXTERNAL_INFO_KEY, None)))
for index in range(10):
custom_id = 'c{}'.format(index)
# self-pointing relationship
relationships = [
{
"source": custom_id,
"target": custom_id,
"relationship_type": "custom"
}
]
mapping = {
'config_id': custom_id,
'type': 'custom',
'id': str(randint(0, 100000)),
'relationships': relationships
custom_num = 10
for index in range(custom_num):
source_id = 'c{}'.format(index)
target_id = 'c{}'.format(custom_num - 1 - index)
source_name = 'custom-{}'.format(source_id)
vals = {
StaticFields.CONFIG_ID: source_id,
StaticFields.TYPE: 'custom',
StaticFields.ID: str(randint(0, 100000)),
StaticFields.NAME: source_name,
StaticFields.RELATIONSHIPS: [{
StaticFields.SOURCE: source_id,
StaticFields.TARGET: entities[target_id],
StaticFields.RELATIONSHIP_TYPE: 'custom'}]
}
entities[source_id].update(**vals)
static_values.append(combine_data(static_info_spec,
mapping,
vals,
spec.get(EXTERNAL_INFO_KEY, None)))
# TODO(yujunz) verify self-pointing relationship
return static_values

8
vitrage/tests/resources/mock_configurations/driver/driver_static_snapshot_dynamic.json
View File

@ -7,11 +7,5 @@
"vitrage_datasource_action": "snapshot",
"vitrage_sample_date": "2015-12-01T12:46:41Z",
"vitrage_event_type": "entity_update",
"relationships": [
{
"source": "[sc][1-9]",
"target": "[hc][1-9]",
"relationship_type": "attached|custom"
}
]
"relationships": []
}

6
vitrage/tests/resources/mock_configurations/driver/driver_static_snapshot_static.json Normal file
View File

@ -0,0 +1,6 @@
{
"state": "available",
"vitrage_datasource_action": "snapshot",
"vitrage_sample_date": "2015-12-01T12:46:41Z",
"vitrage_event_type": "entity_update"
}

48
vitrage/tests/unit/datasources/static/test_static_transformer.py
View File

@ -19,6 +19,7 @@ from oslo_log import log as logging
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import TopologyFields
from vitrage.common.constants import UpdateMethod
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE
@ -86,13 +87,13 @@ class TestStaticTransformer(base.BaseTest):
self.assertEqual(is_placeholder, True)
def test_snapshot_transform(self):
spec_list = mock_driver.simple_static_generators(snapshot_events=10)
events = mock_driver.generate_random_events_list(spec_list)
vals_list = mock_driver.simple_static_generators(snapshot_events=1)
events = mock_driver.generate_random_events_list(vals_list)
self._event_transform_test(events)
def test_update_transform(self):
spec_list = mock_driver.simple_static_generators(update_events=10)
events = mock_driver.generate_random_events_list(spec_list)
vals_list = mock_driver.simple_static_generators(update_events=1)
events = mock_driver.generate_random_events_list(vals_list)
self._event_transform_test(events)
def _event_transform_test(self, events):
@ -106,28 +107,35 @@ class TestStaticTransformer(base.BaseTest):
self._validate_neighbors(neighbors, vertex.vertex_id, event)
def _validate_vertex(self, vertex, event):
self._validate_common_props(vertex, event)
self.assertEqual(vertex[VProps.SAMPLE_TIMESTAMP],
event[DSProps.SAMPLE_DATE])
for k, v in event.get(TopologyFields.METADATA, {}):
self.assertEqual(vertex[k], v)
def _validate_common_props(self, vertex, event):
self.assertEqual(vertex[VProps.CATEGORY], EntityCategory.RESOURCE)
self.assertEqual(vertex[VProps.TYPE], event[VProps.TYPE])
self.assertEqual(vertex[VProps.ID], event[VProps.ID])
self.assertEqual(vertex[VProps.SAMPLE_TIMESTAMP],
event[DSProps.SAMPLE_DATE])
self.assertFalse(vertex[VProps.IS_DELETED])
def _validate_neighbors(self, neighbors, vertex_id, event):
for i in range(len(neighbors)):
if event['type'] == 'nova.host':
self._validate_switch_neighbor(neighbors[i],
event['relationships'][i],
vertex_id)
elif event['type'] == 'switch':
self._validate_host_neighbor(neighbors[i],
event['relationships'][i],
vertex_id)
self._validate_neighbor(neighbors[i],
event[TopologyFields.RELATIONSHIPS][i],
vertex_id)
def _validate_switch_neighbor(self, neighbor, rel, host_vertex_id):
# TODO(yujunz)
pass
def _validate_neighbor(self, neighbor, rel, vertex_id):
vertex = neighbor.vertex
self._validate_neighbor_vertex_props(vertex,
rel[TopologyFields.TARGET])
def _validate_host_neighbor(self, neighbor, rel, switch_vertex_id):
# TODO(yujunz)
pass
edge = neighbor.edge
self.assertEqual(edge.source_id, vertex_id)
self.assertEqual(edge.target_id, neighbor.vertex.vertex_id)
self.assertEqual(edge.label, rel[TopologyFields.RELATIONSHIP_TYPE])
def _validate_neighbor_vertex_props(self, vertex, event):
self._validate_common_props(vertex, event)
self.assertTrue(vertex[VProps.IS_PLACEHOLDER])