openstack/vitrage
Code Issues Proposed changes

few changes:

Browse Source 
1. bug fixs scenarioEvaluator, scenarioRepository, subgraph matching.
2. Tests for HA scenario

Change-Id: I7c2ede0b4f057d595fc38b8effca1840c26e3497
This commit is contained in:
Alexey 2017-04-12 11:29:44 +00:00
parent ffb1e4b0f7
commit 85b630020c
10 changed files with 402 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
vitrage/datasources/neutron/port/transformer.py
View File

@ -102,6 +102,7 @@ class PortTransformer(ResourceTransformerBase):
VProps.NAME: name,
VProps.PROJECT_ID: project_id,
'ip_addresses': tuple(ip_addresses),
'host_id': entity_event.get('binding:host_id'),
}
sample_timestamp = entity_event[DSProps.SAMPLE_DATE]

9
vitrage/datasources/nova/instance/transformer.py
View File

@ -41,22 +41,25 @@ class InstanceTransformer(ResourceTransformerBase):
name = extract_field_value(entity_event, 'name')
entity_id = extract_field_value(entity_event, 'id')
state = extract_field_value(entity_event, 'status')
host = extract_field_value(entity_event, 'OS-EXT-SRV-ATTR:host')
return self._create_vertex(entity_event, name, entity_id, state)
return self._create_vertex(entity_event, name, entity_id, state, host)
def _create_update_entity_vertex(self, entity_event):
name = extract_field_value(entity_event, 'hostname')
entity_id = extract_field_value(entity_event, 'instance_id')
state = extract_field_value(entity_event, 'state')
host = extract_field_value(entity_event, 'host')
return self._create_vertex(entity_event, name, entity_id, state)
return self._create_vertex(entity_event, name, entity_id, state, host)
def _create_vertex(self, entity_event, name, entity_id, state):
def _create_vertex(self, entity_event, name, entity_id, state, host):
metadata = {
VProps.NAME: name,
VProps.PROJECT_ID: entity_event.get('tenant_id', None),
'host_id': host
}
sample_timestamp = entity_event[DSProps.SAMPLE_DATE]

3
vitrage/evaluator/scenario_evaluator.py
View File

@ -198,7 +198,8 @@ class ScenarioEvaluator(object):
for match in matches:
spec, action_id = self._get_action_spec(action_spec, match)
match_hash = hash(tuple(sorted(match.items())))
items_ids = [match[1].vertex_id for match in match.items()]
match_hash = hash(tuple(sorted(items_ids)))
actions[action_id] = ActionInfo(spec, new_mode,
scenario.id, match_hash)

17
vitrage/evaluator/scenario_repository.py
View File

@ -130,12 +130,11 @@ class ScenarioRepository(object):
key = self._create_edge_scenario_key(edge_desc)
scenarios = self.relationship_scenarios[key]
if not self.contains(scenarios, scenario):
if not self._edge_contains(scenarios, scenario, edge_desc):
self.relationship_scenarios[key].append((edge_desc, scenario))
@staticmethod
def _create_edge_scenario_key(edge_desc):
return EdgeKeyScenario(edge_desc.edge.label,
frozenset(edge_desc.source.properties.items()),
frozenset(edge_desc.target.properties.items()))
@ -145,9 +144,17 @@ class ScenarioRepository(object):
key = frozenset(list(entity.properties.items()))
scenarios = self.entity_scenarios[key]
if not self.contains(scenarios, scenario):
if not self._entity_contains(scenarios, scenario, entity):
self.entity_scenarios[key].append((entity, scenario))
@staticmethod
def contains(scenarios, scenario):
return any(s[1].id == scenario.id for s in scenarios)
def _edge_contains(scenarios, scenario, edge):
return any(e.edge.source_id == edge.edge.source_id and
e.edge.target_id == edge.edge.target_id and
e.edge.label == edge.edge.label and s.id == scenario.id
for e, s in scenarios)
@staticmethod
def _entity_contains(scenarios, scenario, entity):
return any(e.vertex_id == entity.vertex_id and s.id == scenario.id
for e, s in scenarios)

1
vitrage/evaluator/template_data.py
View File

@ -187,6 +187,7 @@ class TemplateData(object):
edge_description.edge[EProps.IS_DELETED] = True
else:
edge_description.edge[EProps.IS_DELETED] = False
edge_description.edge[NEG_CONDITION] = False
edge_description.source[VProps.IS_DELETED] = False
edge_description.source[VProps.IS_PLACEHOLDER] = False

14
vitrage/graph/algo_driver/sub_graph_matching.py
View File

@ -119,6 +119,10 @@ def subgraph_matching(base_graph, subgraph, matches, validate=False):
v_id=v_with_unmapped_neighbors[MAPPED_V_ID],
vertex_attr_filter=subgraph_vertex_to_map)
graph_candidate_vertices = \
_remove_used_graph_candidates(graph_candidate_vertices,
curr_subgraph)
# STEP 5: STRUCTURE CHECK
edges = _get_edges_to_mapped_vertices(curr_subgraph,
subgraph_vertex_to_map.vertex_id)
@ -293,3 +297,13 @@ def _update_mapping_for_edge(known_match, mapping, graph, validate):
t_id = known_match.graph_element.target_id
sub_t_id = known_match.subgraph_element.target_id
return _update_mapping(mapping, graph, sub_t_id, t_id, validate)
def _remove_used_graph_candidates(graph_candidate_vertices, curr_subgraph):
ver_to_remove = []
for candidate in graph_candidate_vertices:
for sub_ver in curr_subgraph.get_vertices():
if sub_ver.get(GRAPH_VERTEX, False) and \
sub_ver[GRAPH_VERTEX].vertex_id == candidate.vertex_id:
ver_to_remove.append(candidate)
return [v for v in graph_candidate_vertices if v not in ver_to_remove]

295
vitrage/tests/functional/evaluator/test_scenario_evaluator.py
View File

@ -21,6 +21,8 @@ from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EdgeProperties as EProps
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.cinder.volume.transformer import \
CINDER_VOLUME_DATASOURCE
from vitrage.datasources.nagios.properties import NagiosProperties
from vitrage.datasources.nagios.properties import NagiosTestStatus
from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE
@ -794,6 +796,299 @@ class TestScenarioEvaluator(TestFunctionalBase):
VProps.IS_DELETED: False}
is_deleted = False
def test_ha(self):
event_queue, processor, evaluator = self._init_system()
entity_graph = processor.entity_graph
# find host
query = {
VProps.CATEGORY: 'RESOURCE',
VProps.TYPE: NOVA_HOST_DATASOURCE
}
hosts = entity_graph.get_vertices(vertex_attr_filter=query)
# find instances on host
query = {
VProps.CATEGORY: 'RESOURCE',
VProps.TYPE: NOVA_INSTANCE_DATASOURCE
}
instances = entity_graph.neighbors(hosts[0].vertex_id,
vertex_attr_filter=query)
entity_graph.remove_vertex(instances[2])
entity_graph.remove_vertex(instances[3])
# constants
num_orig_vertices = entity_graph.num_vertices()
num_orig_edges = entity_graph.num_edges()
# ################### STEP 1 ###################
# Add cinder volume 1
generator = mock_driver.simple_volume_generators(volume_num=1,
instance_num=1,
snapshot_events=1)
volume_event1 = mock_driver.generate_random_events_list(generator)[0]
volume_event1['display_name'] = 'volume-1'
volume_event1[VProps.ID] = 'volume-1'
volume_event1['attachments'][0]['server_id'] = instances[0][VProps.ID]
processor.process_event(volume_event1)
while not event_queue.empty():
processor.process_event(event_queue.get())
# test asserts
num_volumes = 1
num_deduced_alarms = 1
self.assertEqual(num_orig_vertices + num_volumes + num_deduced_alarms,
entity_graph.num_vertices())
self.assertEqual(num_orig_edges + num_volumes + num_deduced_alarms,
entity_graph.num_edges())
query = {VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.TYPE: CINDER_VOLUME_DATASOURCE}
instance_neighbors = entity_graph.neighbors(instances[0].vertex_id,
vertex_attr_filter=query)
self.assertEqual(1, len(instance_neighbors))
self.assertEqual(instance_neighbors[0][VProps.CATEGORY],
EntityCategory.RESOURCE)
self.assertEqual(instance_neighbors[0][VProps.TYPE],
CINDER_VOLUME_DATASOURCE)
self.assertEqual(instance_neighbors[0][VProps.NAME], 'volume-1')
self.assertEqual(instance_neighbors[0][VProps.IS_DELETED], False)
self.assertEqual(instance_neighbors[0][VProps.IS_PLACEHOLDER], False)
query = {VProps.CATEGORY: EntityCategory.ALARM, VProps.TYPE: 'vitrage'}
host_neighbors = entity_graph.neighbors(hosts[0].vertex_id,
vertex_attr_filter=query)
self.assertEqual(host_neighbors[0][VProps.CATEGORY],
EntityCategory.ALARM)
self.assertEqual(host_neighbors[0][VProps.TYPE], 'vitrage')
self.assertEqual(host_neighbors[0][VProps.NAME],
'ha_warning_deduced_alarm')
self.assertEqual(host_neighbors[0][VProps.IS_DELETED], False)
self.assertEqual(host_neighbors[0][VProps.IS_PLACEHOLDER], False)
# ################### STEP 2 ###################
# Add cinder volume 2
generator = mock_driver.simple_volume_generators(volume_num=1,
instance_num=1,
snapshot_events=1)
volume_event2 = mock_driver.generate_random_events_list(generator)[0]
volume_event2['display_name'] = 'volume-2'
volume_event2[VProps.ID] = 'volume-2'
volume_event2['attachments'][0]['server_id'] = instances[1][VProps.ID]
processor.process_event(volume_event2)
while not event_queue.empty():
processor.process_event(event_queue.get())
# test asserts
num_volumes = 2
num_deduced_alarms = 2
self.assertEqual(num_orig_vertices + num_volumes + num_deduced_alarms,
entity_graph.num_vertices())
self.assertEqual(num_orig_edges + num_volumes + num_deduced_alarms,
entity_graph.num_edges())
# check instance neighbors
query = {VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.TYPE: CINDER_VOLUME_DATASOURCE}
instance_neighbors = entity_graph.neighbors(instances[1].vertex_id,
vertex_attr_filter=query)
self.assertEqual(1, len(instance_neighbors))
self.assertEqual(instance_neighbors[0][VProps.CATEGORY],
EntityCategory.RESOURCE)
self.assertEqual(instance_neighbors[0][VProps.TYPE],
CINDER_VOLUME_DATASOURCE)
self.assertEqual(instance_neighbors[0][VProps.NAME], 'volume-2')
self.assertEqual(instance_neighbors[0][VProps.IS_DELETED], False)
self.assertEqual(instance_neighbors[0][VProps.IS_PLACEHOLDER], False)
# check ha_error_deduced_alarm
query = {VProps.CATEGORY: EntityCategory.ALARM,
VProps.TYPE: 'vitrage',
VProps.NAME: 'ha_error_deduced_alarm'}
host_neighbors = entity_graph.neighbors(hosts[0].vertex_id,
vertex_attr_filter=query)
self.assertEqual(1, len(host_neighbors))
self.assertEqual(host_neighbors[0][VProps.CATEGORY],
EntityCategory.ALARM)
self.assertEqual(host_neighbors[0][VProps.TYPE], 'vitrage')
self.assertEqual(host_neighbors[0][VProps.NAME],
'ha_error_deduced_alarm')
self.assertEqual(host_neighbors[0][VProps.IS_DELETED], False)
self.assertEqual(host_neighbors[0][VProps.IS_PLACEHOLDER], False)
# check ha_warning_deduced_alarm
query = {VProps.CATEGORY: EntityCategory.ALARM,
VProps.TYPE: 'vitrage',
VProps.NAME: 'ha_warning_deduced_alarm'}
host_neighbors = entity_graph.neighbors(hosts[0].vertex_id,
vertex_attr_filter=query)
self.assertEqual(1, len(host_neighbors))
self.assertEqual(host_neighbors[0][VProps.CATEGORY],
EntityCategory.ALARM)
self.assertEqual(host_neighbors[0][VProps.TYPE], 'vitrage')
self.assertEqual(host_neighbors[0][VProps.NAME],
'ha_warning_deduced_alarm')
self.assertEqual(host_neighbors[0][VProps.IS_DELETED], True)
self.assertEqual(host_neighbors[0][VProps.IS_PLACEHOLDER], False)
# ################### STEP 3 ###################
# Remove Cinder Volume 2
volume_event2[DSProps.DATASOURCE_ACTION] = DatasourceAction.UPDATE
volume_event2[DSProps.EVENT_TYPE] = 'volume.detach.start'
volume_event2['volume_id'] = volume_event2['id']
volume_event2['volume_attachment'] = volume_event2['attachments']
volume_event2['volume_attachment'][0]['instance_uuid'] = \
volume_event2['attachments'][0]['server_id']
processor.process_event(volume_event2)
while not event_queue.empty():
processor.process_event(event_queue.get())
# test asserts
self.assertEqual(num_orig_vertices + num_volumes + num_deduced_alarms +
# This is due to keeping alarm history :
# new alarm doesn't update same deleted alarm.
# Instead, it keeps the old one and creates a new one
1,
entity_graph.num_vertices())
self.assertEqual(num_orig_edges + num_volumes + num_deduced_alarms + 1,
entity_graph.num_edges())
query = {VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.TYPE: CINDER_VOLUME_DATASOURCE}
instance_neighbors = entity_graph.neighbors(instances[1].vertex_id,
vertex_attr_filter=query)
self.assertEqual(1, len(instance_neighbors))
self.assertEqual(instance_neighbors[0][VProps.CATEGORY],
EntityCategory.RESOURCE)
self.assertEqual(instance_neighbors[0][VProps.TYPE],
CINDER_VOLUME_DATASOURCE)
self.assertEqual(instance_neighbors[0][VProps.NAME], 'volume-2')
self.assertEqual(instance_neighbors[0][VProps.IS_DELETED], False)
self.assertEqual(instance_neighbors[0][VProps.IS_PLACEHOLDER], False)
# check ha_error_deduced_alarm
query = {VProps.CATEGORY: EntityCategory.ALARM,
VProps.TYPE: 'vitrage',
VProps.NAME: 'ha_error_deduced_alarm'}
host_neighbors = entity_graph.neighbors(hosts[0].vertex_id,
vertex_attr_filter=query)
self.assertEqual(1, len(host_neighbors))
self.assertEqual(host_neighbors[0][VProps.CATEGORY],
EntityCategory.ALARM)
self.assertEqual(host_neighbors[0][VProps.TYPE], 'vitrage')
self.assertEqual(host_neighbors[0][VProps.NAME],
'ha_error_deduced_alarm')
self.assertEqual(host_neighbors[0][VProps.IS_DELETED], True)
self.assertEqual(host_neighbors[0][VProps.IS_PLACEHOLDER], False)
# check new ha_warning_deduced_alarm
query = {VProps.CATEGORY: EntityCategory.ALARM,
VProps.TYPE: 'vitrage',
VProps.NAME: 'ha_warning_deduced_alarm',
VProps.IS_DELETED: False}
host_neighbors = entity_graph.neighbors(hosts[0].vertex_id,
vertex_attr_filter=query)
self.assertEqual(1, len(host_neighbors))
self.assertEqual(host_neighbors[0][VProps.CATEGORY],
EntityCategory.ALARM)
self.assertEqual(host_neighbors[0][VProps.TYPE], 'vitrage')
self.assertEqual(host_neighbors[0][VProps.NAME],
'ha_warning_deduced_alarm')
self.assertEqual(host_neighbors[0][VProps.IS_DELETED], False)
self.assertEqual(host_neighbors[0][VProps.IS_PLACEHOLDER], False)
# check old deleted ha_warning_deduced_alarm
query = {VProps.CATEGORY: EntityCategory.ALARM,
VProps.TYPE: 'vitrage',
VProps.NAME: 'ha_warning_deduced_alarm',
VProps.IS_DELETED: True}
host_neighbors = entity_graph.neighbors(hosts[0].vertex_id,
vertex_attr_filter=query)
self.assertEqual(1, len(host_neighbors))
self.assertEqual(host_neighbors[0][VProps.CATEGORY],
EntityCategory.ALARM)
self.assertEqual(host_neighbors[0][VProps.TYPE], 'vitrage')
self.assertEqual(host_neighbors[0][VProps.NAME],
'ha_warning_deduced_alarm')
self.assertEqual(host_neighbors[0][VProps.IS_DELETED], True)
self.assertEqual(host_neighbors[0][VProps.IS_PLACEHOLDER], False)
# ################### STEP 4 ###################
# Remove Cinder Volume 1
volume_event1[DSProps.DATASOURCE_ACTION] = DatasourceAction.UPDATE
volume_event1[DSProps.EVENT_TYPE] = 'volume.detach.start'
volume_event1['volume_id'] = volume_event1['id']
volume_event1['volume_attachment'] = volume_event1['attachments']
volume_event1['volume_attachment'][0]['instance_uuid'] = \
volume_event1['attachments'][0]['server_id']
processor.process_event(volume_event1)
while not event_queue.empty():
processor.process_event(event_queue.get())
# test asserts
self.assertEqual(num_orig_vertices + num_volumes + num_deduced_alarms +
# This is due to keeping alarm history :
# new alarm doesn't update same deleted alarm.
# Instead, it keeps the old one and creates a new one
1,
entity_graph.num_vertices())
self.assertEqual(num_orig_edges + num_volumes + num_deduced_alarms + 1,
entity_graph.num_edges())
query = {VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.TYPE: CINDER_VOLUME_DATASOURCE}
instance_neighbors = entity_graph.neighbors(instances[0].vertex_id,
vertex_attr_filter=query)
self.assertEqual(1, len(instance_neighbors))
self.assertEqual(instance_neighbors[0][VProps.CATEGORY],
EntityCategory.RESOURCE)
self.assertEqual(instance_neighbors[0][VProps.TYPE],
CINDER_VOLUME_DATASOURCE)
self.assertEqual(instance_neighbors[0][VProps.NAME], 'volume-1')
self.assertEqual(instance_neighbors[0][VProps.IS_DELETED], False)
self.assertEqual(instance_neighbors[0][VProps.IS_PLACEHOLDER], False)
# check ha_error_deduced_alarm
query = {VProps.CATEGORY: EntityCategory.ALARM,
VProps.TYPE: 'vitrage',
VProps.NAME: 'ha_error_deduced_alarm'}
host_neighbors = entity_graph.neighbors(hosts[0].vertex_id,
vertex_attr_filter=query)
self.assertEqual(1, len(host_neighbors))
self.assertEqual(host_neighbors[0][VProps.CATEGORY],
EntityCategory.ALARM)
self.assertEqual(host_neighbors[0][VProps.TYPE], 'vitrage')
self.assertEqual(host_neighbors[0][VProps.NAME],
'ha_error_deduced_alarm')
self.assertEqual(host_neighbors[0][VProps.IS_DELETED], True)
self.assertEqual(host_neighbors[0][VProps.IS_PLACEHOLDER], False)
# check old ha_warning_deduced_alarm
query = {VProps.CATEGORY: EntityCategory.ALARM,
VProps.TYPE: 'vitrage',
VProps.NAME: 'ha_warning_deduced_alarm'}
host_neighbors = entity_graph.neighbors(hosts[0].vertex_id,
vertex_attr_filter=query)
self.assertEqual(2, len(host_neighbors))
self.assertEqual(host_neighbors[0][VProps.CATEGORY],
EntityCategory.ALARM)
self.assertEqual(host_neighbors[0][VProps.TYPE], 'vitrage')
self.assertEqual(host_neighbors[0][VProps.NAME],
'ha_warning_deduced_alarm')
self.assertEqual(host_neighbors[0][VProps.IS_DELETED], True)
self.assertEqual(host_neighbors[0][VProps.IS_PLACEHOLDER], False)
self.assertEqual(host_neighbors[1][VProps.CATEGORY],
EntityCategory.ALARM)
self.assertEqual(host_neighbors[1][VProps.TYPE], 'vitrage')
self.assertEqual(host_neighbors[1][VProps.NAME],
'ha_warning_deduced_alarm')
self.assertEqual(host_neighbors[1][VProps.IS_DELETED], True)
self.assertEqual(host_neighbors[1][VProps.IS_PLACEHOLDER], False)
def get_host_after_event(self, event_queue, nagios_event,
processor, target_host):
processor.process_event(nagios_event)

67
vitrage/tests/resources/templates/evaluator/new.yaml Normal file
View File

@ -0,0 +1,67 @@
metadata:
name: HA_up_instance
description:
definitions:
entities:
- entity:
category: RESOURCE
type: nova.instance
template_id: instance1
- entity:
category: RESOURCE
type: nova.instance
template_id: instance2
- entity:
category: RESOURCE
type: nova.host
template_id: host
- entity:
category: RESOURCE
type: cinder.volume
template_id: volume1
- entity:
category: RESOURCE
type: cinder.volume
template_id: volume2
relationships:
- relationship:
source: host
relationship_type: contains
target: instance1
template_id : host_contains_instance1
- relationship:
source: volume1
relationship_type: attached
target: instance1
template_id : volume_attached_instance1
- relationship:
source: host
relationship_type: contains
target: instance2
template_id : host_contains_instance2
- relationship:
source: volume2
relationship_type: attached
target: instance2
template_id : volume_attached_instance2
scenarios:
- scenario:
condition: (host_contains_instance1 and volume_attached_instance1) and (host_contains_instance2 and not volume_attached_instance2)
actions:
- action:
action_type: raise_alarm
action_target:
target: host
properties:
alarm_name: ha_warning_deduced_alarm
severity: warning
- scenario:
condition: (host_contains_instance1 and volume_attached_instance1) and (host_contains_instance2 and volume_attached_instance2)
actions:
- action:
action_type: raise_alarm
action_target:
target: host
properties:
alarm_name: ha_error_deduced_alarm
severity: error

2
vitrage/tests/unit/datasources/nova/test_nova_instance_transformer.py
View File

@ -160,7 +160,7 @@ class NovaInstanceTransformerTest(base.BaseTest):
def _validate_vertex_props(self, vertex, event):
self.assertEqual(11, len(vertex.properties))
self.assertEqual(12, len(vertex.properties))
is_update_event = tbase.is_update_event(event)

4
vitrage/tests/unit/entity_graph/base.py
View File

@ -13,6 +13,7 @@
# under the License.
from oslo_config import cfg
from vitrage.datasources.cinder.volume.driver import CINDER_VOLUME_DATASOURCE
from vitrage.common.constants import DatasourceAction
from vitrage.common.constants import DatasourceProperties as DSProps
@ -46,7 +47,8 @@ class TestEntityGraphUnitBase(base.BaseTest):
NOVA_INSTANCE_DATASOURCE,
NOVA_ZONE_DATASOURCE,
NEUTRON_NETWORK_DATASOURCE,
NEUTRON_PORT_DATASOURCE],
NEUTRON_PORT_DATASOURCE,
CINDER_VOLUME_DATASOURCE],
help='Names of supported data sources'),
cfg.ListOpt('path',