Evaluator Engine / subgraph matching changes / small bug fixes

Change-Id: I2bbaae43795751c5f02d2873c137bee1831a96b7
This commit is contained in:
Elisha Rosensweig 2016-03-08 13:17:36 +02:00
parent 97caf84d0f
commit 8ab57009c0
23 changed files with 560 additions and 208 deletions

View File

@ -32,7 +32,7 @@ class VertexProperties(object):
class EdgeProperties(object):
RELATIONSHIP_NAME = 'relationship_name'
RELATIONSHIP_TYPE = 'relationship_type'
IS_DELETED = 'is_deleted'
UPDATE_TIMESTAMP = 'update_timestamp'

View File

@ -123,7 +123,7 @@ class EntityGraphApis(object):
try:
resources = self.entity_graph.neighbors(
v_id=alarm.vertex_id,
edge_attr_filter={EProps.RELATIONSHIP_NAME: EdgeLabels.ON},
edge_attr_filter={EProps.RELATIONSHIP_TYPE: EdgeLabels.ON},
direction=Direction.OUT)
resource = self._get_first(resources)

View File

@ -19,6 +19,8 @@ from oslo_log import log
from oslo_service import service as os_service
from vitrage.entity_graph.processor import processor as proc
from vitrage.evaluator.scenario_evaluator import ScenarioEvaluator
from vitrage.evaluator.scenario_repository import ScenarioRepository
LOG = log.getLogger(__name__)
@ -33,6 +35,11 @@ class VitrageGraphService(os_service.Service):
initialization_status,
e_graph=entity_graph)
self.scenario_repo = ScenarioRepository(cfg)
self.evaluator = ScenarioEvaluator(entity_graph,
self.scenario_repo,
event_queue)
def start(self):
LOG.info("Start VitrageGraphService")
@ -64,7 +71,6 @@ class VitrageGraphService(os_service.Service):
seconds and goes to sleep for 1 second. if there are more events in
the queue they are done when timer returns.
"""
start_time = datetime.datetime.now()
while not self.queue.empty():
time_delta = datetime.datetime.now() - start_time

View File

@ -46,5 +46,5 @@ class AddCausalRelationship(base.Recipe):
return {
TFields.SOURCE: params[TFields.SOURCE],
TFields.TARGET: params[TFields.TARGET],
EdgeProperties.RELATIONSHIP_NAME: EdgeLabels.CAUSES
EdgeProperties.RELATIONSHIP_TYPE: EdgeLabels.CAUSES
}

View File

@ -0,0 +1,167 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log
from vitrage.common.constants import EdgeProperties as EProps
from vitrage.common.constants import VertexProperties as VProps
from vitrage.evaluator.actions.action_executor import ActionExecutor
from vitrage.evaluator.actions.base import ActionMode
from vitrage.evaluator.template import ActionSpecs
from vitrage.evaluator.template import EdgeDescription
from vitrage.evaluator.template import ENTITY
from vitrage.graph.algo_driver.algorithm import Mapping
from vitrage.graph import create_algorithm
from vitrage.graph import create_graph
from vitrage.graph.driver import Vertex
LOG = log.getLogger(__name__)
class ScenarioEvaluator(object):
def __init__(self, entity_graph, scenario_repo, event_queue):
self._entity_graph = entity_graph
self._graph_algs = create_algorithm(entity_graph)
self._scenario_repo = scenario_repo
self._action_executor = ActionExecutor(event_queue)
self._entity_graph.subscribe(self.process_event)
self.enabled = True
def process_event(self, before, current, is_vertex):
"""Notification of a change in the entity graph.
:param before: The graph element (vertex or edge) prior to the
change that happened. None if the element was just created.
:param current: The graph element (vertex or edge) after the
change that happened. Deleted elements should arrive with the
is_deleted property set to True
"""
if not self.enabled:
return
# todo (erosensw): support for NOT conditions - reverse logic
before_scenarios = self._get_element_scenarios(before, is_vertex)
current_scenarios = self._get_element_scenarios(current, is_vertex)
before_scenarios, current_scenarios = \
self._remove_overlap_scenarios(before_scenarios, current_scenarios)
actions = self._get_actions(before,
before_scenarios,
ActionMode.UNDO)
actions.update(self._get_actions(current,
current_scenarios,
ActionMode.DO))
for action in actions.values():
# todo: named tuple?
self._action_executor.execute(action[0], action[1])
def _get_element_scenarios(self, element, is_vertex):
if not element \
or element.get(VProps.IS_DELETED) \
or element.get(EProps.IS_DELETED):
return []
elif is_vertex:
return self._scenario_repo.get_scenarios_by_vertex(element)
else: # is edge
source = self._entity_graph.get_vertex(element.source_id)
target = self._entity_graph.get_vertex(element.target_id)
edge_desc = EdgeDescription(element, source, target)
return self._scenario_repo.get_scenarios_by_edge(edge_desc)
@staticmethod
def _remove_overlap_scenarios(before, current):
intersection = filter(lambda x: x in before, current)
before = filter(lambda x: x not in intersection, before)
current = filter(lambda x: x not in intersection, current)
return before, current
def _get_actions(self, element, anchored_scenarios, mode):
actions = {}
for anchored_scenario in anchored_scenarios:
scenario_anchor = anchored_scenario[0]
scenario = anchored_scenario[1]
actions.update(self._process_scenario(element,
scenario,
scenario_anchor,
mode))
return actions
def _process_scenario(self, element, scenario, template_anchors, mode):
actions = {}
for action in scenario.actions:
if not isinstance(template_anchors, list):
template_anchors = [template_anchors]
for template_anchor in template_anchors:
matches = self._evaluate_full_condition(scenario.condition,
element,
template_anchor)
if matches:
for match in matches:
spec, action_id = self._get_action_spec(action, match)
actions[action_id] = (spec, mode)
return actions
@staticmethod
def _get_action_spec(action_spec, mappings):
targets = action_spec.targets
real_ids = {key: mappings[value] for key, value in targets.items()}
revised_spec = ActionSpecs(action_spec.type,
real_ids,
action_spec.properties)
action_id = ScenarioEvaluator._generate_action_id(revised_spec)
return revised_spec, action_id
@staticmethod
def _generate_action_id(action_spec):
return hash(
(action_spec.type,
tuple(sorted(action_spec.targets.items())),
tuple(sorted(action_spec.properties.items())))
)
def _evaluate_full_condition(self, condition, trigger, template_anchor):
condition_matches = []
for clause in condition:
# OR condition means aggregation of matches, without duplicates
simple_condition_matches = \
self._evaluate_and_condition(clause, trigger, template_anchor)
condition_matches += simple_condition_matches
return condition_matches
def _evaluate_and_condition(self, condition, trigger, template_anchor):
condition_g = create_graph("scenario condition")
for term in condition:
if not term.positive:
# todo(erosensw): add support for NOT clauses
LOG.error('Unsupported template with NOT operator')
return []
if term.type == ENTITY:
condition_g.add_vertex(term.variable)
else: # type = relationship
condition_g.add_vertex(term.variable.source)
condition_g.add_vertex(term.variable.target)
condition_g.add_edge(term.variable.edge)
if isinstance(trigger, Vertex):
anchor_map = Mapping(template_anchor, trigger, True)
else:
anchor_map = Mapping(template_anchor.edge, trigger, False)
return self._graph_algs.sub_graph_matching(condition_g, [anchor_map])

View File

@ -38,24 +38,27 @@ class ScenarioRepository(object):
def get_scenarios_by_vertex(self, vertex):
entity_key = frozenset(vertex.properties)
entity_key = frozenset(vertex.properties.items())
return [value for scenario_key, value in self.entity_scenarios
if scenario_key.issubset(entity_key)]
scenarios = []
for scenario_key, value in self.entity_scenarios.items():
if scenario_key.issubset(entity_key):
scenarios += value
return scenarios
def get_scenarios_by_edge(self, edge_description):
key = self._create_edge_scenario_key(edge_description)
scenarios = []
for scenario_key, value in self.relationship_scenarios:
for scenario_key, value in self.relationship_scenarios.items():
check_label = key.label == scenario_key.label
check_source_issubset = scenario_key.source.issubset(key.source)
check_target_issubset = scenario_key.target.issubset(key.target)
if check_label and check_source_issubset and check_target_issubset:
scenarios.append(value)
scenarios += value
return scenarios
@ -82,17 +85,22 @@ class ScenarioRepository(object):
self.add_template(template_def)
def _add_template_scenarios(self, template):
for scenario in template.scenarios:
for condition_var in scenario.condition:
self._handle_condition(scenario)
if condition_var.type == RELATIONSHIP:
edge_desc = condition_var.variable
self._add_relationship(scenario, edge_desc)
self._add_entity(scenario, edge_desc.source)
self._add_entity(scenario, edge_desc.target)
else: # Entity
self._add_entity(scenario, condition_var.variable)
def _handle_condition(self, scenario):
for clause in scenario.condition:
self._handle_clause(clause, scenario)
def _handle_clause(self, clause, scenario):
for condition_var in clause:
if condition_var.type == RELATIONSHIP:
edge_desc = condition_var.variable
self._add_relationship(scenario, edge_desc)
self._add_entity(scenario, edge_desc.source)
self._add_entity(scenario, edge_desc.target)
else: # Entity
self._add_entity(scenario, condition_var.variable)
@staticmethod
def _create_scenario_key(properties):
@ -109,8 +117,8 @@ class ScenarioRepository(object):
def _create_edge_scenario_key(self, edge_desc):
return EdgeKeyScenario(edge_desc.edge.label,
frozenset(edge_desc.source.properties),
frozenset(edge_desc.target.properties))
frozenset(edge_desc.source.properties.items()),
frozenset(edge_desc.target.properties.items()))
def _add_entity(self, scenario, entity):

View File

@ -126,8 +126,9 @@ class Template(object):
def _extract_properties(self, var_dict):
ignore_ids = [TFields.TEMPLATE_ID, TFields.SOURCE, TFields.TARGET]
return dict((key, var_dict[key]) for key in var_dict
if key != TFields.TEMPLATE_ID)
if key not in ignore_ids)
def _build_scenarios(self, scenarios_defs):
@ -178,13 +179,13 @@ class Template(object):
return self._extract_or_condition(condition_dnf)
if isinstance(condition_dnf, And):
return self._extract_and_condition(condition_dnf)
return [self._extract_and_condition(condition_dnf)]
if isinstance(condition_dnf, Not):
return [(self._extract_condition_var(condition_dnf, False))]
return [[(self._extract_condition_var(condition_dnf, False))]]
if isinstance(condition_dnf, Symbol):
return [(self._extract_condition_var(condition_dnf, True))]
return [[(self._extract_condition_var(condition_dnf, True))]]
def convert_to_dnf_format(self, condition_str):

View File

@ -16,7 +16,8 @@ import abc
from collections import namedtuple
import six
Mapping = namedtuple('Mapping', ['sub_graph_v_id', 'graph_v_id'])
Mapping = \
namedtuple('Mapping', ['subgraph_element', 'graph_element', 'is_vertex'])
@six.add_metaclass(abc.ABCMeta)
@ -42,7 +43,7 @@ class GraphAlgorithm(object):
pass
@abc.abstractmethod
def sub_graph_matching(self, sub_graph, known_mappings):
def sub_graph_matching(self, sub_graph, known_mappings, validate=False):
"""Search for occurrences of of a template graph in the graph
In sub-graph matching algorithms complexity is high in the general case
@ -51,6 +52,7 @@ class GraphAlgorithm(object):
:type known_mappings: list
:type sub_graph: driver.Graph
:type validate: bool
:rtype: list of dict
"""
pass

View File

@ -15,7 +15,7 @@
from oslo_log import log as logging
from algorithm import GraphAlgorithm
from sub_graph_matching import sub_graph_matching
from sub_graph_matching import subgraph_matching
from vitrage.graph.driver import NXGraph
from vitrage.graph.query import create_predicate
@ -66,5 +66,5 @@ class NXAlgorithm(GraphAlgorithm):
graph._g = self.graph._g.subgraph(n_result)
return graph
def sub_graph_matching(self, sub_graph, known_matches):
return sub_graph_matching(self.graph, sub_graph, known_matches)
def sub_graph_matching(self, subgraph, known_matches, validate=False):
return subgraph_matching(self.graph, subgraph, known_matches, validate)

View File

@ -22,71 +22,8 @@ MAPPED_V_ID = 'mapped_v_id'
NEIGHBORS_MAPPED = 'neighbors_mapped'
def get_edges_to_mapped_vertices(graph, vertex):
"""Get all edges (to/from) vertex where neighbor has a MAPPED_V_ID
:type graph: driver.Graph
:type vertex: driver.Vertex
:rtype: list of driver.Edge
"""
sub_graph_edges_to_mapped_vertices = []
for e in graph.get_edges(vertex.vertex_id):
t_neighbor = graph.get_vertex(e.other_vertex(vertex.vertex_id))
if not t_neighbor:
raise VitrageAlgorithmError('Cant get vertex for edge' + str(e))
if t_neighbor and t_neighbor.get(MAPPED_V_ID):
sub_graph_edges_to_mapped_vertices.append(e)
return sub_graph_edges_to_mapped_vertices
def graph_contains_sub_graph_edges(graph, sub_graph, sub_graph_edges):
"""Check if graph contains all the expected edges
For each (sub-graph) expected edge, check if a corresponding edge exists
in the graph with relevant properties check
:type graph: driver.Graph
:type sub_graph: driver.Graph
:type sub_graph_edges: list of driver.Edge
:rtype: bool
"""
for e in sub_graph_edges:
graph_v_id_source = sub_graph.get_vertex(e.source_id).get(MAPPED_V_ID)
graph_v_id_target = sub_graph.get_vertex(e.target_id).get(MAPPED_V_ID)
if not graph_v_id_source or not graph_v_id_target:
raise VitrageAlgorithmError('Cant get vertex for edge' + str(e))
found_graph_edge = graph.get_edge(graph_v_id_source,
graph_v_id_target,
e.label)
if not found_graph_edge or not check_filter(found_graph_edge, e):
return False
return True
def create_initial_sub_graph(graph, known_matches, sub_graph):
"""Create initial mapping graph from sub graph and known matches
copy the sub-graph to create the first candidate mapping graph.
In which known vertices mappings are added to vertices MAPPED_V_ID
"""
mapping = sub_graph.copy()
for known_match in known_matches:
sub_graph_vertex = sub_graph.get_vertex(known_match.sub_graph_v_id)
graph_vertex = graph.get_vertex(known_match.graph_v_id)
if check_filter(graph_vertex, sub_graph_vertex):
mv = sub_graph.get_vertex(sub_graph_vertex.vertex_id)
mv[MAPPED_V_ID] = known_match.graph_v_id
mapping.update_vertex(mv)
edges = get_edges_to_mapped_vertices(mapping, mv)
if not graph_contains_sub_graph_edges(graph, mapping, edges):
return None
else:
return None
return mapping
def sub_graph_matching(_graph_, sub_graph, known_matches):
"""Find all occurrences of sub_graph in the graph
def subgraph_matching(base_graph, subgraph, matches, validate=False):
"""Find all occurrences of subgraph in the graph
In the following, a partial mapping is a copy of the sub-graph.
As we go, vertices of curr_mapping graph will be updated with new
@ -124,24 +61,27 @@ def sub_graph_matching(_graph_, sub_graph, known_matches):
- Step 5: CHECK STRUCTURE
Filter candidate vertices according to edges
"""
final_sub_graphs = []
initial_sg = create_initial_sub_graph(_graph_, known_matches, sub_graph)
final_subgraphs = []
initial_sg = _create_initial_subgraph(matches,
base_graph,
subgraph,
validate)
if not initial_sg:
LOG.warning('sub_graph_matching: Initial sub-graph creation failed')
LOG.warning('sub_graph_matching: Known matches: %s',
str(known_matches))
return final_sub_graphs
_queue_ = [initial_sg]
LOG.warning('subgraph_matching: Initial sub-graph creation failed')
LOG.warning('subgraph_matching: Known matches: %s',
str(matches))
return final_subgraphs
queue = [initial_sg]
while _queue_:
curr_sub_graph = _queue_.pop(0)
while queue:
curr_subgraph = queue.pop(0)
# STEP 1: STOPPING CONDITION
mapped_vertices = filter(
lambda v: v.get(MAPPED_V_ID),
curr_sub_graph.get_vertices())
if len(mapped_vertices) == sub_graph.num_vertices():
final_sub_graphs.append(curr_sub_graph)
curr_subgraph.get_vertices())
if len(mapped_vertices) == subgraph.num_vertices():
final_subgraphs.append(curr_subgraph)
continue
# STEP 2: CAN WE THROW THIS SUB-GRAPH?
@ -155,34 +95,143 @@ def sub_graph_matching(_graph_, sub_graph, known_matches):
v_with_unmapped_neighbors = vertices_with_unmapped_neighbors.pop(0)
unmapped_neighbors = filter(
lambda v: not v.get(MAPPED_V_ID),
curr_sub_graph.neighbors(v_with_unmapped_neighbors.vertex_id))
curr_subgraph.neighbors(v_with_unmapped_neighbors.vertex_id))
if not unmapped_neighbors:
# Mark vertex as NEIGHBORS_MAPPED=True
v_with_unmapped_neighbors[NEIGHBORS_MAPPED] = True
curr_sub_graph.update_vertex(v_with_unmapped_neighbors)
_queue_.append(curr_sub_graph)
curr_subgraph.update_vertex(v_with_unmapped_neighbors)
queue.append(curr_subgraph)
continue
sub_graph_vertex_to_map = unmapped_neighbors.pop(0)
subgraph_vertex_to_map = unmapped_neighbors.pop(0)
# STEP 4: PROPERTIES CHECK
graph_candidate_vertices = _graph_.neighbors(
graph_candidate_vertices = base_graph.neighbors(
v_id=v_with_unmapped_neighbors[MAPPED_V_ID],
vertex_attr_filter=sub_graph_vertex_to_map)
vertex_attr_filter=subgraph_vertex_to_map)
# STEP 5: STRUCTURE CHECK
edges = get_edges_to_mapped_vertices(curr_sub_graph,
sub_graph_vertex_to_map)
edges = _get_edges_to_mapped_vertices(curr_subgraph,
subgraph_vertex_to_map.vertex_id)
for graph_vertex in graph_candidate_vertices:
sub_graph_vertex_to_map[MAPPED_V_ID] = graph_vertex.vertex_id
curr_sub_graph.update_vertex(sub_graph_vertex_to_map)
if graph_contains_sub_graph_edges(_graph_, curr_sub_graph, edges):
_queue_.append(curr_sub_graph.copy())
subgraph_vertex_to_map[MAPPED_V_ID] = graph_vertex.vertex_id
curr_subgraph.update_vertex(subgraph_vertex_to_map)
if _graph_contains_subgraph_edges(base_graph,
curr_subgraph,
edges):
queue.append(curr_subgraph.copy())
# Last thing: Convert results to the expected format!
result = []
for mapping in final_sub_graphs:
for mapping in final_subgraphs:
# TODO(ihefetz) If needed, Here we can easily extract the edge
# matches from the mapping graph
a = {v.vertex_id: v[MAPPED_V_ID] for v in mapping.get_vertices()}
result.append(a)
return result
def _get_edges_to_mapped_vertices(graph, vertex_id):
"""Get all edges (to/from) vertex where neighbor has a MAPPED_V_ID
:type graph: driver.Graph
:type vertex: driver.Vertex
:rtype: set of driver.Edge
"""
subgraph_edges_to_mapped_vertices = []
for e in graph.get_edges(vertex_id):
t_neighbor = graph.get_vertex(e.other_vertex(vertex_id))
if not t_neighbor:
raise VitrageAlgorithmError('Cant get vertex for edge' + str(e))
if t_neighbor and t_neighbor.get(MAPPED_V_ID):
subgraph_edges_to_mapped_vertices.append(e)
return set(subgraph_edges_to_mapped_vertices)
def _graph_contains_subgraph_edges(graph, subgraph, subgraph_edges):
"""Check if graph contains all the expected edges
For each (sub-graph) expected edge, check if a corresponding edge exists
in the graph with relevant properties check
:type graph: driver.Graph
:type subgraph: driver.Graph
:type subgraph_edges: set of driver.Edge
:rtype: bool
"""
for e in subgraph_edges:
graph_v_id_source = subgraph.get_vertex(e.source_id).get(MAPPED_V_ID)
graph_v_id_target = subgraph.get_vertex(e.target_id).get(MAPPED_V_ID)
if not graph_v_id_source or not graph_v_id_target:
raise VitrageAlgorithmError('Cant get vertex for edge' + str(e))
found_graph_edge = graph.get_edge(graph_v_id_source,
graph_v_id_target,
e.label)
if not found_graph_edge or not check_filter(found_graph_edge, e):
return False
return True
def _create_initial_subgraph(known_matches, graph, subgraph, validate=False):
"""Create initial mapping graph from sub graph and known matches
copy the sub-graph to create the first candidate mapping graph.
In which known vertices mappings are added to vertices MAPPED_V_ID
"""
mapping = subgraph.copy()
for match in known_matches:
if match.is_vertex:
if not _update_mapping_for_vertex(match, mapping, graph, validate):
return None
subgraph_id = match.subgraph_element.vertex_id
edges = _get_edges_to_mapped_vertices(mapping, subgraph_id)
else: # is edge
if not _update_mapping_for_edge(match, mapping, graph, validate):
return None
edges = _get_related_edges(mapping, match, subgraph, validate)
if not _graph_contains_subgraph_edges(graph, mapping, edges):
return None
return mapping
def _get_related_edges(mapping, match, subgraph, validate):
sub_target_id = match.subgraph_element.target_id
sub_source_id = match.subgraph_element.source_id
edges = _get_edges_to_mapped_vertices(mapping, sub_source_id)
edges.union(_get_edges_to_mapped_vertices(mapping, sub_target_id))
if not validate: # no need to check the mapped edge
known_edge = subgraph.get_edge(
sub_source_id,
sub_target_id,
match.subgraph_element.label
)
edges.remove(known_edge)
return edges
def _update_mapping(subgraph, graph, subgraph_id, graph_id, validate):
subgraph_vertex = subgraph.get_vertex(subgraph_id)
if validate:
graph_vertex = graph.get_vertex(graph_id)
if not check_filter(graph_vertex, subgraph_vertex, MAPPED_V_ID):
return False
subgraph_vertex[MAPPED_V_ID] = graph_id
subgraph.update_vertex(subgraph_vertex)
return True
def _update_mapping_for_vertex(known_match, mapping, graph, validate):
subgraph_id = known_match.subgraph_element.vertex_id
graph_id = known_match.graph_element.vertex_id
return _update_mapping(mapping, graph, subgraph_id, graph_id, validate)
def _update_mapping_for_edge(known_match, mapping, graph, validate):
s_id = known_match.graph_element.source_id
sub_s_id = known_match.subgraph_element.source_id
if not _update_mapping(mapping, graph, sub_s_id, s_id, validate):
return False
t_id = known_match.graph_element.target_id
sub_t_id = known_match.subgraph_element.target_id
return _update_mapping(mapping, graph, sub_t_id, t_id, validate)

View File

@ -13,29 +13,21 @@
# under the License.
import functools
from elements import Edge
from elements import Vertex
def _before_func(graph, item):
if not graph.is_subscribed():
return
if isinstance(item, Edge):
return graph.get_edge(item.source_id, item.target_id, item.label)
else:
return graph.get_vertex(item.vertex_id)
return graph.get_item(item)
def _after_func(graph, item, data_before=None):
if not graph.is_subscribed():
return
if isinstance(item, Edge):
edge = graph.get_edge(item.source_id, item.target_id, item.label)
edge_source_v = graph.get_vertex(item.source_id)
edge_target_v = graph.get_vertex(item.target_id)
data_after = edge, edge_source_v, edge_target_v
else:
data_after = graph.get_vertex(item.vertex_id),
graph.notifier.notify(data_before, *data_after)
element = graph.get_item(item)
is_vertex = isinstance(element, Vertex)
graph.notifier.notify(data_before, graph.get_item(item), is_vertex)
class Notifier(object):

View File

@ -13,18 +13,21 @@
# under the License.
def check_filter(data, attr_filter):
def check_filter(data, attr_filter, *args):
"""Check attr_filter against data
:param data: a dictionary of field_name: value
:param attr_filter: a dictionary of either
field_name : value (mandatory)
field_name : list of values - data[field_name] must match ANY of the values
:param ignore_keys: list of filter keys to ignore (if exist)
:rtype: bool
"""
if not attr_filter:
return True
for key, content in attr_filter.items():
if key in args:
continue
if not isinstance(content, list):
content = [content]
if not data.get(key) in content:

View File

@ -93,7 +93,7 @@ def create_edge(source_id,
properties = {
EConst.UPDATE_TIMESTAMP: update_timestamp,
EConst.IS_DELETED: is_deleted,
EConst.RELATIONSHIP_NAME: relationship_type,
EConst.RELATIONSHIP_TYPE: relationship_type,
}
if metadata:
properties.update(metadata)

View File

@ -0,0 +1,89 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import multiprocessing
from oslo_config import cfg
from oslo_log import log as logging
from vitrage.common.constants import EntityType
from vitrage.common.constants import VertexProperties as VProps
from vitrage.evaluator.scenario_evaluator import ScenarioEvaluator
from vitrage.evaluator.scenario_repository import ScenarioRepository
from vitrage.tests.functional.entity_graph.base import \
TestEntityGraphFunctionalBase
LOG = logging.getLogger(__name__)
class TestScenarioEvaluator(TestEntityGraphFunctionalBase):
@classmethod
def setUpClass(cls):
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.PROCESSOR_OPTS, group='entity_graph')
cls.conf.register_opts(cls.EVALUATOR_OPTS, group='evaluator')
cls.conf.register_opts(cls.PLUGINS_OPTS,
group='synchronizer_plugins')
TestScenarioEvaluator.load_plugins(cls.conf)
cls.scenario_repository = ScenarioRepository(cls.conf)
def test_deduced_state(self):
# Test Setup
processor = self._create_processor_with_graph(self.conf)
event_queue = multiprocessing.Queue()
ScenarioEvaluator(processor.entity_graph,
self.scenario_repository,
event_queue)
target_host = 'host-2'
host_v = self._get_host_from_graph(target_host, processor.entity_graph)
self.assertEqual('RUNNING', host_v[VProps.AGGREGATED_STATE],
'host should be RUNNING when starting')
nagios_event = {'last_check': '2016-02-07 15:26:04',
'resource_name': target_host,
'resource_type': EntityType.NOVA_HOST,
'service': 'Check_MK',
'status': 'CRITICAL',
'status_info': 'ok',
'sync_mode': 'snapshot',
'sync_type': 'nagios'}
processor.process_event(nagios_event)
# The set_state action should have added an event to the queue, so
processor.process_event(event_queue.get())
host_v = self._get_host_from_graph(target_host, processor.entity_graph)
self.assertEqual('SUBOPTIMAL', host_v[VProps.AGGREGATED_STATE],
'host should be SUBOPTIMAL after nagios alarm event')
# next disable the alarm
nagios_event['status'] = 'OK'
processor.process_event(nagios_event)
# The set_state action should have added an event to the queue, so
processor.process_event(event_queue.get())
host_v = self._get_host_from_graph(target_host, processor.entity_graph)
self.assertEqual('RUNNING', host_v[VProps.AGGREGATED_STATE],
'host should be RUNNING when starting')
@staticmethod
def _get_host_from_graph(host_name, entity_graph):
vertex_attrs = {VProps.TYPE: EntityType.NOVA_HOST,
VProps.NAME: host_name}
host_vertices = entity_graph.get_vertices(
vertex_attr_filter=vertex_attrs)
assert len(host_vertices) == 1, "incorrect number of vertices"
return host_vertices[0]

View File

@ -0,0 +1,28 @@
metadata:
id: deduced_state
definitions:
entities:
- entity:
category: ALARM
type: nagios
template_id: 1
- entity:
category: RESOURCE
type: nova.host
template_id: 2
relationships:
- relationship:
source: 1
target: 2
relationship_type: on
template_id : alarm_on_host
scenarios:
- scenario:
condition: alarm_on_host
actions:
- action:
action_type: set_state
properties:
state: SUBOPTIMAL
action_target:
target: 2

View File

@ -39,13 +39,13 @@ scenarios:
condition: alarm_on_host and host_contains_instance
actions:
- action:
action_type: RAISE_ALARM
action_type: raise_alarm
properties:
alarm_type: VM_CPU_SUBOPTIMAL_PERFORMANCE
action_target:
target: 4
- action:
action_type: SET_STATE
action_type: set_state
properties:
state: SUBOPTIMAL
action_target:
@ -54,7 +54,7 @@ scenarios:
condition: alarm_on_host and alarm_on_instance and host_contains_instance
actions:
- action:
action_type: ADD_CAUSAL_RELATIONSHIP
action_type: add_causal_relationship
action_target:
source: 1
target: 2

View File

@ -34,6 +34,11 @@ class TestEntityGraphUnitBase(base.BaseTest):
default=utils.get_resources_dir() + '/states_plugins'),
]
EVALUATOR_OPTS = [
cfg.StrOpt('templates_dir',
default=utils.get_resources_dir() + '/evaluator_templates',
)]
PLUGINS_OPTS = [
cfg.ListOpt('plugin_type',
default=['nagios',
@ -42,14 +47,16 @@ class TestEntityGraphUnitBase(base.BaseTest):
'nova.zone'],
help='Names of supported synchronizer plugins'),
]
NUM_NODES = 1
NUM_ZONES = 2
NUM_HOSTS = 4
NUM_INSTANCES = 16
def load_plugins(self, conf):
@staticmethod
def load_plugins(conf):
for plugin_name in conf.synchronizer_plugins.plugin_type:
load_plugin(self.conf, plugin_name)
load_plugin(conf, plugin_name)
def _create_processor_with_graph(self, conf, processor=None):
events = self._create_mock_events()

View File

@ -61,5 +61,5 @@ class AddCausalRelationshipTest(base.BaseTest):
target = add_edge_step_params.get(TField.TARGET)
self.assertEqual(target_vertex_id, target)
relation_name = add_edge_step_params[EdgeProperties.RELATIONSHIP_NAME]
relation_name = add_edge_step_params[EdgeProperties.RELATIONSHIP_TYPE]
self.assertEqual(EdgeLabels.CAUSES, relation_name)

View File

@ -89,6 +89,7 @@ class BasicTemplateTest(base.BaseTest):
self.assertEqual(len(relationships), len(relations_def))
exclude_keys = [TFields.TEMPLATE_ID, TFields.SOURCE, TFields.TARGET]
for relation_def in relations_def:
relation_def_dict = relation_def[TFields.RELATIONSHIP]
@ -98,9 +99,8 @@ class BasicTemplateTest(base.BaseTest):
relationship = relationships[template_id].edge
for key, value in relation_def_dict.iteritems():
if key == TFields.TEMPLATE_ID:
continue
self.assertEqual(value, relationship.properties[key])
if key not in exclude_keys:
self.assertEqual(value, relationship.properties[key])
def _validate_scenarios(self, scenarios, entities):
"""Validates scenario parsing
@ -119,7 +119,7 @@ class BasicTemplateTest(base.BaseTest):
condition = scenario.condition
self.assertEqual(len(condition), 1)
condition_var = condition[0]
condition_var = condition[0][0]
self.assertTrue(isinstance(condition_var, ConditionVar))
variable = condition_var.variable

View File

@ -32,8 +32,8 @@ from vitrage.tests import base
LOG = logging.getLogger(__name__)
ENTITY_GRAPH_HOSTS_PER_NODE = 32
ENTITY_GRAPH_VMS_PER_HOST = 32
ENTITY_GRAPH_HOSTS_PER_NODE = 8
ENTITY_GRAPH_VMS_PER_HOST = 8
ENTITY_GRAPH_ALARMS_PER_HOST = 8
ENTITY_GRAPH_TESTS_PER_HOST = 20
ENTITY_GRAPH_ALARMS_PER_VM = 8

View File

@ -126,10 +126,10 @@ class GraphTest(GraphTestBase):
g.add_vertex(v_host)
g.add_edge(e_node_to_host)
self.assertEqual(1, g.num_edges(), 'graph __len__ after add edge')
label = e_node_to_host[EProps.RELATIONSHIP_NAME]
label = e_node_to_host[EProps.RELATIONSHIP_TYPE]
e = g.get_edge(v_node.vertex_id, v_host.vertex_id, label)
self.assertEqual(e_node_to_host[EProps.RELATIONSHIP_NAME],
e[EProps.RELATIONSHIP_NAME],
self.assertEqual(e_node_to_host[EProps.RELATIONSHIP_TYPE],
e[EProps.RELATIONSHIP_TYPE],
'edge properties are saved')
self.assertEqual(e_node_to_host.source_id, e.source_id,
'edge vertex_id is saved')
@ -195,8 +195,8 @@ class GraphTest(GraphTestBase):
g.add_edge(another_edge)
self.assertEqual(2, g.num_edges(), 'graph __len__ after add edge')
e = g.get_edge(v_node.vertex_id, v_host.vertex_id, another_label)
self.assertEqual(another_edge[EProps.RELATIONSHIP_NAME],
e[EProps.RELATIONSHIP_NAME],
self.assertEqual(another_edge[EProps.RELATIONSHIP_TYPE],
e[EProps.RELATIONSHIP_TYPE],
'edge properties are saved')
self.assertEqual('DATA', e['some_meta'],
'edge properties are saved')
@ -278,7 +278,7 @@ class GraphTest(GraphTestBase):
v1_neighbors = g.neighbors(
v_id=v1.vertex_id,
edge_attr_filter={EProps.RELATIONSHIP_NAME: relationship_a})
edge_attr_filter={EProps.RELATIONSHIP_TYPE: relationship_a})
self._assert_set_equal({v2, v4}, v1_neighbors,
'Check V1 neighbors, edge property filter')
@ -300,7 +300,7 @@ class GraphTest(GraphTestBase):
v1_neighbors = g.neighbors(
v_id=v1.vertex_id,
direction=Direction.IN,
edge_attr_filter={EProps.RELATIONSHIP_NAME: relationship_c},
edge_attr_filter={EProps.RELATIONSHIP_TYPE: relationship_c},
vertex_attr_filter={VProps.TYPE: HOST})
self._assert_set_equal(
{v2}, v1_neighbors,
@ -327,7 +327,7 @@ class GraphTest(GraphTestBase):
v2_neighbors = g.neighbors(
v_id=v2.vertex_id,
edge_attr_filter={
EProps.RELATIONSHIP_NAME: [relationship_a, relationship_b]
EProps.RELATIONSHIP_TYPE: [relationship_a, relationship_b]
},
vertex_attr_filter={
VProps.CATEGORY: [RESOURCE, ALARM],
@ -382,8 +382,7 @@ class GraphTest(GraphTestBase):
self.assertEqual(OPENSTACK_NODE, found_vertex[VProps.TYPE],
'get_vertices check node vertex')
def _check_callback_result(self, result, msg, exp_prev, exp_curr,
exp_source_v=None, exp_target_v=None):
def _check_callback_result(self, result, msg, exp_prev, exp_curr):
def assert_none_or_equals(exp, act, msg):
if exp:
@ -396,10 +395,6 @@ class GraphTest(GraphTestBase):
msg + ' prev_item unexpected')
assert_none_or_equals(exp_curr, result[1],
msg + ' curr_item unexpected')
assert_none_or_equals(exp_source_v, result[2],
msg + ' edge_source_v unexpected')
assert_none_or_equals(exp_target_v, result[3],
msg + ' edge_target_v unexpected')
self.result = None
def _assert_none_or_equals(self, exp, act, msg):
@ -415,12 +410,11 @@ class GraphTest(GraphTestBase):
def callback(pre_item,
current_item,
edge_source_v=None,
edge_target_v=None):
is_vertex):
LOG.info('called with: pre_event_item ' + str(pre_item) +
' current_item ' + str(current_item))
self.assertIsNotNone(current_item)
self.result = pre_item, current_item, edge_source_v, edge_target_v
self.result = pre_item, current_item, is_vertex
# Check there is no notification without subscribing
g.add_vertex(v_alarm)
@ -440,7 +434,7 @@ class GraphTest(GraphTestBase):
g.add_edge(e_node_to_host)
self._check_callback_result(self.result, 'add edge', None,
e_node_to_host, v_node, v_host)
e_node_to_host)
updated_vertex = g.get_vertex(v_host.vertex_id)
updated_vertex[VProps.CATEGORY] = ALARM
@ -454,4 +448,4 @@ class GraphTest(GraphTestBase):
updated_edge['ZIG'] = 'ZAG'
g.update_edge(updated_edge)
self._check_callback_result(self.result, 'update edge', e_node_to_host,
updated_edge, v_node, updated_vertex)
updated_edge)

View File

@ -138,10 +138,10 @@ class GraphAlgorithmTest(GraphTestBase):
ga = create_algorithm(self.entity_graph)
# Get ids of some of the elements in the entity graph:
vm_alarm_id = self.entity_graph.get_vertex(
ALARM_ON_VM + str(self.vm_alarm_id - 1)).vertex_id
host_alarm_id = self.entity_graph.get_vertex(
ALARM_ON_HOST + str(self.host_alarm_id - 1)).vertex_id
vm_alarm = self.entity_graph.get_vertex(
ALARM_ON_VM + str(self.vm_alarm_id - 1))
host_alarm = self.entity_graph.get_vertex(
ALARM_ON_HOST + str(self.host_alarm_id - 1))
# Create a template for template matching
t = create_graph('template_graph')
@ -188,7 +188,7 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_vertex(t_v_alarm_fail)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host_alarm.vertex_id, host_alarm_id)])
Mapping(t_v_host_alarm, host_alarm, True)], validate=True)
self.assertEqual(
0,
len(mappings),
@ -198,7 +198,7 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_vertex(t_v_host_alarm)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host_alarm.vertex_id, host_alarm_id)])
Mapping(t_v_host_alarm, host_alarm, is_vertex=True)])
self.assertEqual(
1,
len(mappings),
@ -207,7 +207,7 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_vertex(t_v_host)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host_alarm.vertex_id, host_alarm_id)])
Mapping(t_v_host_alarm, host_alarm, is_vertex=True)])
self.assertEqual(
0,
len(mappings),
@ -216,15 +216,16 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_edge(e_alarm_on_host)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host_alarm.vertex_id, host_alarm_id)])
Mapping(t_v_host_alarm, host_alarm, is_vertex=True)])
self.assertEqual(
1, len(mappings),
'Template - Two connected vertices (host alarm -ON-> host)'
' template_root is a specific host alarm ' + str(mappings))
host_id = mappings[0][t_v_host.vertex_id]
host_vertex = self.entity_graph.get_vertex(host_id)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host.vertex_id, host_id)])
Mapping(t_v_host, host_vertex, is_vertex=True)])
self.assertEqual(
ENTITY_GRAPH_ALARMS_PER_HOST,
len(mappings),
@ -233,7 +234,7 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_vertex(t_v_vm)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host_alarm.vertex_id, host_alarm_id)])
Mapping(t_v_host_alarm, host_alarm, is_vertex=True)])
self.assertEqual(
0,
len(mappings),
@ -243,7 +244,7 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_vertex(t_v_vm_alarm)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)])
self.assertEqual(
0,
len(mappings),
@ -253,7 +254,7 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_edge(e_alarm_on_vm)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)])
self.assertEqual(
0,
len(mappings),
@ -263,7 +264,7 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_edge(e_host_contains_vm)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)])
self.assertEqual(
ENTITY_GRAPH_ALARMS_PER_HOST,
len(mappings),
@ -272,7 +273,7 @@ class GraphAlgorithmTest(GraphTestBase):
' template_root is a specific instance alarm ' + str(mappings))
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host_alarm.vertex_id, host_alarm_id)])
Mapping(t_v_host_alarm, host_alarm, is_vertex=True)])
self.assertEqual(
ENTITY_GRAPH_VMS_PER_HOST * ENTITY_GRAPH_ALARMS_PER_VM,
len(mappings),
@ -281,7 +282,7 @@ class GraphAlgorithmTest(GraphTestBase):
' template_root is a specific host alarm ' + str(mappings))
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host.vertex_id, host_id)])
Mapping(t_v_host, host_vertex, is_vertex=True)])
self.assertEqual(
ENTITY_GRAPH_VMS_PER_HOST * ENTITY_GRAPH_ALARMS_PER_VM
* ENTITY_GRAPH_ALARMS_PER_HOST,
@ -292,7 +293,7 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_vertex(t_v_switch)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)])
self.assertEqual(
0,
len(mappings),
@ -302,7 +303,7 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_edge(e_host_uses_switch)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)])
self.assertEqual(
ENTITY_GRAPH_ALARMS_PER_HOST,
len(mappings),
@ -312,7 +313,7 @@ class GraphAlgorithmTest(GraphTestBase):
+ str(mappings))
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host.vertex_id, host_id)])
Mapping(t_v_host, host_vertex, is_vertex=True)])
self.assertEqual(
ENTITY_GRAPH_VMS_PER_HOST * ENTITY_GRAPH_ALARMS_PER_VM
* ENTITY_GRAPH_ALARMS_PER_HOST,
@ -323,8 +324,8 @@ class GraphAlgorithmTest(GraphTestBase):
+ str(mappings))
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_switch.vertex_id, v_switch.vertex_id),
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
Mapping(t_v_switch, v_switch, is_vertex=True),
Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)])
self.assertEqual(
ENTITY_GRAPH_ALARMS_PER_HOST,
len(mappings),
@ -336,7 +337,7 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_vertex(t_v_node_not_in_graph)
t.add_edge(e_host_to_node_not_in_graph)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)])
self.assertEqual(
0,
len(mappings),
@ -351,7 +352,7 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_edge(e_node_contains_host)
t.add_edge(e_node_contains_switch)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)])
self.assertEqual(
1,
len(mappings),
@ -361,9 +362,8 @@ class GraphAlgorithmTest(GraphTestBase):
' template_root is a instance alarm ' + str(mappings))
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_node.vertex_id, v_node.vertex_id),
Mapping(t_v_switch.vertex_id, v_switch.vertex_id),
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
Mapping(e_node_contains_switch, e_node_to_switch, is_vertex=False),
Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)])
self.assertEqual(
1,
len(mappings),
@ -374,8 +374,21 @@ class GraphAlgorithmTest(GraphTestBase):
t.add_edge(e_node_contains_switch_fail)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_node.vertex_id, v_node.vertex_id),
Mapping(t_v_switch.vertex_id, v_switch.vertex_id)])
Mapping(t_v_node, v_node, is_vertex=True),
Mapping(t_v_switch, v_switch, is_vertex=True)], validate=True)
self.assertEqual(
0,
len(mappings),
'Template - FIVE connected vertices - 2 Known Mapping[node,switch]'
' Check that ALL edges between the 2 known mappings are checked'
' we now have node-CONTAINSfail->switch AND node-CONTAINS->switch'
' ')
mappings = ga.sub_graph_matching(t, [
Mapping(e_node_contains_switch,
e_node_to_switch, is_vertex=False)],
validate=True
)
self.assertEqual(
0,
len(mappings),
@ -386,8 +399,8 @@ class GraphAlgorithmTest(GraphTestBase):
t.remove_edge(e_node_contains_switch)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_node.vertex_id, v_node.vertex_id),
Mapping(t_v_switch.vertex_id, v_switch.vertex_id)])
Mapping(t_v_node, v_node, is_vertex=True),
Mapping(t_v_switch, v_switch, is_vertex=True)])
self.assertEqual(
0,
len(mappings),
@ -398,7 +411,7 @@ class GraphAlgorithmTest(GraphTestBase):
' ')
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)])
self.assertEqual(
0,
len(mappings),

View File

@ -28,24 +28,17 @@ LOG = logging.getLogger(__name__)
class NagiosSynchronizerTest(NagiosBaseTest):
OPTS = [
cfg.DictOpt('nagios',
default={
'synchronizer':
'vitrage.synchronizer.plugins.nagios.synchronizer'
'.NagiosSynchronizer',
'transformer': 'vitrage.synchronizer.plugins'
'.nagios.transformer.NagiosTransformer',
'user': '',
'password': '',
'url': '',
'config_file': utils.get_resources_dir() +
'/nagios/nagios_conf.yaml'},)
cfg.StrOpt(
'config_file',
default=utils.get_resources_dir() + '/nagios/nagios_conf.yaml',
help='Nagios configuation file'
),
]
@classmethod
def setUpClass(cls):
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.OPTS, group='synchronizer_plugins')
cls.conf.register_opts(cls.OPTS, group='nagios')
def test_get_all(self):
"""Check get_all functionality.