From 8f327e36620338b57d866a7661de41f9e79c5797 Mon Sep 17 00:00:00 2001 From: Alexey Weyl Date: Sun, 19 Mar 2017 16:51:23 +0000 Subject: [PATCH] support for not operator in templates condition Change-Id: I434b1967ee0cb91b54ab93e680a2a11bab3f8c3e --- doc/source/not_operator_support.rst | 310 +++++ vitrage/common/constants.py | 2 +- vitrage/datasources/static/__init__.py | 14 +- vitrage/datasources/static/driver.py | 25 +- vitrage/datasources/static/transformer.py | 17 +- vitrage/evaluator/scenario_evaluator.py | 203 +++- vitrage/evaluator/template_data.py | 60 +- vitrage/evaluator/template_fields.py | 4 +- .../graph/algo_driver/networkx_algorithm.py | 101 +- .../graph/algo_driver/sub_graph_matching.py | 78 +- vitrage/graph/driver/elements.py | 3 + vitrage/graph/driver/graph.py | 18 +- vitrage/graph/driver/networkx_graph.py | 12 +- .../evaluator/test_scenario_evaluator.py | 371 +++++- vitrage/tests/mocks/trace_generator.py | 9 +- .../complex_not_operator_deduced_alarm.yaml | 59 + .../simple_not_operator_deduced_alarm.yaml | 31 + .../datasources/static/test_static_driver.py | 6 +- .../static/test_static_transformer.py | 19 +- vitrage/tests/unit/entity_graph/base.py | 6 +- vitrage/tests/unit/graph/base.py | 3 +- vitrage/tests/unit/graph/test_graph_algo.py | 1045 +++++++++++++++-- 22 files changed, 2148 insertions(+), 248 deletions(-) create mode 100644 doc/source/not_operator_support.rst create mode 100644 vitrage/tests/resources/templates/evaluator/complex_not_operator_deduced_alarm.yaml create mode 100644 vitrage/tests/resources/templates/evaluator/simple_not_operator_deduced_alarm.yaml diff --git a/doc/source/not_operator_support.rst b/doc/source/not_operator_support.rst new file mode 100644 index 000000000..7aaea2533 --- /dev/null +++ b/doc/source/not_operator_support.rst @@ -0,0 +1,310 @@ +============================== +Templates Not Operator Support +============================== + +Overview +-------- + +The Templates language supports the "or" and "and" operators at the moment. +Many scenarios can't be described by using only those two operators and thus +we would like to add support for "not" operator as well. + + +Template Structure +================== +The template is written in YAML language, with the following structure. + :: + + metadata: + name: + description: + definitions: + entities: + - entity: ... + - entity: ... + relationships: + - relationship: ... + - relationship: ... + scenarios: + - scenario: + condition: + actions: + - action: ... + + +All the sections are in use as described in the "vitrage-template-format.rst" file. +But in the condition section it will be possible to write the "not" operator in addition to the "and" and "or" operators. +The "not" operator can be used only before a relationship expression. + + +Condition Format +---------------- +The condition which needs to be met will be phrased using the entities and +relationships previously defined. The condition details are described in the +"vitrage-template-format.rst" and the addition here is the new logical operator "not": + +- "not" - indicates that the expression must not be satisfied in order for the + condition to be met. + +The following are examples of valid expressions, where X, Y and Z are +relationships: + +- X +- X and Y +- X and Y and Z +- X and not Y +- X and not (Y or Z) +- X and not X + + +Supported Use Cases +=================== + +Use Case 1: +----------- +There exists an instance on Host but there is no Alarm on the instance. + + ++--------+ +--------+ Not +---------+ +| Host | ------> | Vm | < - - - - | Alarm | ++--------+ +--------+ +---------+ + + :: + + metadata: + name: no_alarm_on_instance_that_contained_in_host + description: when host contains vm that has no alarm on it, show implications on the host + definitions: + entities: + - entity: + category: ALARM + type: instance_mem_performance_problem + template_id: instance_alarm # some string + - entity: + category: RESOURCE + type: nova.host + template_id: host + - entity: + category: RESOURCE + type: nova.instance + template_id: instance + relationships: + - relationship: + source: instance_alarm + target: instance + relationship_type: on + template_id : alarm_on_instance + - relationship: + source: host + target: instance + relationship_type: contains + template_id : host_contains_instance + scenarios: + - scenario: + condition: host_contains_instance and not alarm_on_instance + actions: + - action: + action_type: set_state + properties: + state: available + action_target: + target: host + + +Use Case 2: +----------- + +There exists a host with no alarm. + ++--------+ Not +---------+ +| Host | < - - - - | Alarm | ++--------+ +---------+ + + :: + + metadata: + name: no_alarm_on_host + description: when there is no alarm on the host, show implications on the host + definitions: + entities: + - entity: + category: ALARM + type: host_high_mem_load + template_id: host_alarm # some string + - entity: + category: RESOURCE + type: nova.host + template_id: host + relationships: + - relationship: + source: host_alarm # source and target from entities section + target: host + relationship_type: on + template_id : alarm_on_host + scenarios: + - scenario: + condition: not alarm_on_host + actions: + - action: + action_type: set_state + properties: + state: available + action_target: + target: instance + + +Use Case 3: +----------- + +The Switch is attached to a Host that contains a Vm. +The Switch is also comprised to a Network which has a Port. +There is no edge between the Vm and the Port. + + +---------+ +---------+ + +----------- | Host | --------> | Vm | + | +---------+ +---------+ + | + v | ++----------+ N +| Switch | | o ++----------+ t + | + | v + | + | +---------+ +---------+ + +----------> | Network | <--------- | Port | + +---------+ +---------+ + + :: + + metadata: + name: no_connection_between_vm_and_port + description: when there is no edge between the port and the vm, show implications on the instances + definitions: + entities: + - entity: + category: RESOURCE + type: nova.host + template_id: host + - entity: + category: RESOURCE + type: nova.instance + template_id: instance + - entity: + category: RESOURCE + type: switch + template_id: switch + - entity: + category: RESOURCE + type: neutron.network + template_id: network + - entity: + category: RESOURCE + type: neutron.port + template_id: port + relationships: + - relationship: + source: host + target: instance + relationship_type: contains + template_id : host_contains_instance + - relationship: + source: switch + target: host + relationship_type: connected + template_id : host_connected_switch + - relationship: + source: switch + target: network + relationship_type: has + template_id : switch_has_network + - relationship: + source: port + target: network + relationship_type: attached + template_id : port_attached_network + - relationship: + source: vm + target: port + relationship_type: connected + template_id : vm_connected_port + scenarios: + - scenario: + condition: host_contains_instance and host_connected_switch and switch_has_network and port_attached_network and not vm_connected_port + actions: + - action: + action_type: raise_alarm + properties: + alarm_name: instance_mem_performance_problem + severity: warning + action_target: + target: instance + + + +Unsupported Use Cases +===================== + +Use Case 1: +----------- + +There is a Host contains Vm, which has no edge ("connection") to a stack that has an alarm on it. +Difference: The difference here from the graphs above, is that here there are +two connected component subgraphs (the first is host contains vm, the second is alarm on stack), +and the current mechanism doesn't support such a use case of not operator between many connected component subgraphs. +In the subgraphs above, we had only one vertex which was not connected to the main connected component subgraph. + + ++---------+ +---------+ Not +---------+ +---------+ +| Host | --------> | Vm | - - - - - -> | Stack | <--------- | Alarm | ++---------+ +---------+ +---------+ +---------+ + + :: + + metadata: + name: host_contains_vm_with_no_edge_to_stack_that_has_alarm_on_it + description: when host contains vm without and edge to a stack that has no alarms, show implications on the instances + definitions: + entities: + - entity: + category: RESOURCE + type: nova.host + template_id: host + - entity: + category: RESOURCE + type: nova.instance + template_id: instance + - entity: + category: RESOURCE + type: heat.stack + template_id: stack + - entity: + category: ALARM + type: stack_high_mem_load + template_id: stack_alarm + relationships: + - relationship: + source: host + target: instance + relationship_type: contains + template_id : host_contains_instance + - relationship: + source: stack_alarm + target: stack + relationship_type: on + template_id : alarm_on_stack + - relationship: + source: instance + target: stack + relationship_type: attached + template_id : instance_attached_stack + scenarios: + - scenario: + condition: host_contains_instance and alarm_on_stack and not instance_attached_stack + actions: + - action: + action_type: set_state + properties: + state: available + action_target: + target: instance diff --git a/vitrage/common/constants.py b/vitrage/common/constants.py index eed0c8778..029db3b72 100644 --- a/vitrage/common/constants.py +++ b/vitrage/common/constants.py @@ -111,7 +111,7 @@ class NotifierEventTypes(object): DEACTIVATE_MARK_DOWN_EVENT = 'vitrage.mark_down.deactivate' -class TopologyFields(object): +class TemplateTopologyFields(object): """yaml fields for topology definitions""" METADATA = 'metadata' DESCRIPTION = 'description' diff --git a/vitrage/datasources/static/__init__.py b/vitrage/datasources/static/__init__.py index 967949327..776c2778a 100644 --- a/vitrage/datasources/static/__init__.py +++ b/vitrage/datasources/static/__init__.py @@ -14,7 +14,6 @@ from oslo_config import cfg from vitrage.common.constants import DatasourceOpts as DSOpts -from vitrage.common.constants import TopologyFields from vitrage.common.constants import UpdateMethod STATIC_DATASOURCE = 'static' @@ -48,5 +47,16 @@ OPTS = [ help='static data sources configuration directory')] -class StaticFields(TopologyFields): +class StaticFields(object): + """yaml fields for static definitions""" + METADATA = 'metadata' + DEFINITIONS = 'definitions' + RELATIONSHIPS = 'relationships' + ENTITIES = 'entities' + RELATIONSHIP_TYPE = 'relationship_type' + SOURCE = 'source' + TARGET = 'target' STATIC_ID = 'static_id' + TYPE = 'type' + ID = 'id' + NAME = 'name' diff --git a/vitrage/datasources/static/driver.py b/vitrage/datasources/static/driver.py index 62960da87..211e61ae4 100644 --- a/vitrage/datasources/static/driver.py +++ b/vitrage/datasources/static/driver.py @@ -18,8 +18,6 @@ from six.moves import reduce from oslo_log import log -from vitrage.common.constants import TopologyFields -from vitrage.common.constants import VertexProperties as VProps from vitrage.datasources.driver_base import DriverBase from vitrage.datasources.static import STATIC_DATASOURCE from vitrage.datasources.static import StaticFields @@ -30,7 +28,7 @@ LOG = log.getLogger(__name__) class StaticDriver(DriverBase): # base fields are required for all entities, others are treated as metadata - BASE_FIELDS = {StaticFields.STATIC_ID, StaticFields.TYPE, VProps.ID} + BASE_FIELDS = {StaticFields.STATIC_ID, StaticFields.TYPE, StaticFields.ID} def __init__(self, conf): super(StaticDriver, self).__init__() @@ -40,7 +38,7 @@ class StaticDriver(DriverBase): def _is_valid_config(config): """check for validity of configuration""" # TODO(yujunz) check with yaml schema or reuse template validation - return TopologyFields.DEFINITIONS in config + return StaticFields.DEFINITIONS in config @staticmethod def get_event_types(): @@ -77,10 +75,10 @@ class StaticDriver(DriverBase): .format(path)) return [] - definitions = config[TopologyFields.DEFINITIONS] + definitions = config[StaticFields.DEFINITIONS] - entities = definitions[TopologyFields.ENTITIES] - relationships = definitions[TopologyFields.RELATIONSHIPS] + entities = definitions[StaticFields.ENTITIES] + relationships = definitions[StaticFields.RELATIONSHIPS] return cls._pack(entities, relationships) @@ -100,22 +98,23 @@ class StaticDriver(DriverBase): metadata = {key: value for key, value in iteritems(entity) if key not in cls.BASE_FIELDS} entities_dict[static_id] = entity - entity[TopologyFields.RELATIONSHIPS] = [] - entity[TopologyFields.METADATA] = metadata + entity[StaticFields.RELATIONSHIPS] = [] + entity[StaticFields.METADATA] = metadata else: LOG.warning("Skipped duplicated entity: {}".format(entity)) @classmethod def _pack_rel(cls, entities_dict, rel): - source_id = rel[TopologyFields.SOURCE] - target_id = rel[TopologyFields.TARGET] + source_id = rel[StaticFields.SOURCE] + target_id = rel[StaticFields.TARGET] if source_id == target_id: # self pointing relationship - entities_dict[source_id][TopologyFields.RELATIONSHIPS].append(rel) + entities_dict[source_id] + [StaticFields.RELATIONSHIPS].append(rel) else: source, target = entities_dict[source_id], entities_dict[target_id] - source[TopologyFields.RELATIONSHIPS].append( + source[StaticFields.RELATIONSHIPS].append( cls._expand_neighbor(rel, target)) @staticmethod diff --git a/vitrage/datasources/static/transformer.py b/vitrage/datasources/static/transformer.py index dde042bb6..8218b9826 100644 --- a/vitrage/datasources/static/transformer.py +++ b/vitrage/datasources/static/transformer.py @@ -18,7 +18,6 @@ from oslo_log import log as logging from vitrage.common.constants import DatasourceProperties as DSProps from vitrage.common.constants import EntityCategory -from vitrage.common.constants import TopologyFields from vitrage.common.constants import VertexProperties as VProps from vitrage.datasources.resource_transformer_base import \ ResourceTransformerBase @@ -58,7 +57,7 @@ class StaticTransformer(ResourceTransformerBase): return STATIC_DATASOURCE def _create_vertex(self, entity_event): - metadata = entity_event.get(TopologyFields.METADATA, {}) + metadata = entity_event.get(StaticFields.METADATA, {}) entity_type = entity_event[VProps.TYPE] entity_id = entity_event[VProps.ID] @@ -101,13 +100,15 @@ class StaticTransformer(ResourceTransformerBase): return None if rel[StaticFields.SOURCE] == rel[StaticFields.TARGET]: neighbor = entity_event - return self._create_neighbor(entity_event, - neighbor[VProps.ID], - neighbor[VProps.TYPE], - rel[TopologyFields.RELATIONSHIP_TYPE], - is_entity_source=is_entity_source) + return self._create_neighbor( + entity_event, + neighbor[VProps.ID], + neighbor[VProps.TYPE], + rel[StaticFields.RELATIONSHIP_TYPE], + is_entity_source=is_entity_source) def _create_static_neighbors(self, entity_event): - relationships = entity_event.get(TopologyFields.RELATIONSHIPS, []) + relationships = entity_event.get(StaticFields.RELATIONSHIPS, + []) return [self._create_static_neighbor(entity_event, rel) for rel in relationships] diff --git a/vitrage/evaluator/scenario_evaluator.py b/vitrage/evaluator/scenario_evaluator.py index 2dacf6b07..ccd774d21 100644 --- a/vitrage/evaluator/scenario_evaluator.py +++ b/vitrage/evaluator/scenario_evaluator.py @@ -26,9 +26,9 @@ from vitrage.evaluator.actions.base import ActionType import vitrage.evaluator.actions.priority_tools as pt from vitrage.evaluator.template_data import ActionSpecs from vitrage.evaluator.template_data import EdgeDescription -from vitrage.evaluator.template_data import ENTITY from vitrage.graph.algo_driver.algorithm import Mapping -from vitrage.graph.driver.networkx_graph import NXGraph +from vitrage.graph.algo_driver.sub_graph_matching import \ + NEG_CONDITION from vitrage.graph.driver import Vertex LOG = log.getLogger(__name__) @@ -87,7 +87,6 @@ class ScenarioEvaluator(object): str(before), str(current)) - # todo (erosensw): support for NOT conditions - reverse logic before_scenarios = self._get_element_scenarios(before, is_vertex) current_scenarios = self._get_element_scenarios(current, is_vertex) before_scenarios, current_scenarios = \ @@ -157,15 +156,52 @@ class ScenarioEvaluator(object): actions = {} for action in scenario.actions: for scenario_element in scenario_elements: - matches = self._evaluate_full_condition(scenario.condition, - element, - scenario_element) - if matches: - for match in matches: - spec, action_id = self._get_action_spec(action, match) - match_hash = hash(tuple(sorted(match.items()))) - actions[action_id] = \ - ActionInfo(spec, mode, scenario.id, match_hash) + matches = self._evaluate_subgraphs(scenario.subgraphs, + element, + scenario_element, + action.targets['target']) + + actions.update(self._get_actions_from_matches(matches, + mode, + action, + scenario)) + + return actions + + def _evaluate_subgraphs(self, + subgraphs, + element, + scenario_element, + action_target): + if isinstance(element, Vertex): + return self._find_vertex_subgraph_matching(subgraphs, + action_target, + element, + scenario_element) + else: + return self._find_edge_subgraph_matching(subgraphs, + action_target, + element, + scenario_element) + + def _get_actions_from_matches(self, + combined_matches, + mode, + action_spec, + scenario): + actions = {} + for is_switch_mode, matches in combined_matches: + new_mode = mode + if is_switch_mode: + new_mode = ActionMode.UNDO \ + if mode == ActionMode.DO else ActionMode.DO + + for match in matches: + spec, action_id = self._get_action_spec(action_spec, match) + match_hash = hash(tuple(sorted(match.items()))) + actions[action_id] = ActionInfo(spec, new_mode, + scenario.id, match_hash) + return actions @staticmethod @@ -190,53 +226,6 @@ class ScenarioEvaluator(object): tuple(sorted(action_spec.properties.items()))) ) - def _evaluate_full_condition(self, condition, element, scenario_element): - condition_matches = [] - for clause in condition: - # OR condition means aggregation of matches, without duplicates - and_condition_matches = \ - self._evaluate_and_condition(clause, element, scenario_element) - condition_matches += and_condition_matches - - return condition_matches - - def _evaluate_and_condition(self, condition, element, scenario_element): - - condition_g = NXGraph("scenario condition") - for term in condition: - if not term.positive: - # todo(erosensw): add support for NOT clauses - LOG.error('Template with NOT operator current not supported') - return [] - - if term.type == ENTITY: - term.variable[VProps.IS_DELETED] = False - condition_g.add_vertex(term.variable) - - else: # type = relationship - edge_desc = term.variable - self._set_relationship_not_deleted(edge_desc) - self._add_relationship(condition_g, edge_desc) - - if isinstance(element, Vertex): - initial_map = Mapping(scenario_element, element, True) - else: - initial_map = Mapping(scenario_element.edge, element, False) - return self._entity_graph.algo.sub_graph_matching(condition_g, - [initial_map]) - - @staticmethod - def _set_relationship_not_deleted(edge_description): - edge_description.source[VProps.IS_DELETED] = False - edge_description.target[VProps.IS_DELETED] = False - edge_description.edge[EProps.IS_DELETED] = False - - @staticmethod - def _add_relationship(condition_graph, edge_description): - condition_graph.add_vertex(edge_description.source) - condition_graph.add_vertex(edge_description.target) - condition_graph.add_edge(edge_description.edge) - def _analyze_and_filter_actions(self, actions): actions_to_perform = {} @@ -260,6 +249,102 @@ class ScenarioEvaluator(object): actions_to_perform[key] = new_dominant return actions_to_perform.values() + def _find_vertex_subgraph_matching(self, + subgraphs, + action_target, + vertex, + scenario_vertex): + """calculates subgraph matching for vertex + + iterates over all the subgraphs, and checks if the triggered vertex is + in the same connected component as the action then run subgraph + matching on the vertex and return its result, otherwise return an + empty list of matches. + """ + + matches = [] + for subgraph in subgraphs: + connected_component = subgraph.algo.graph_query_vertices( + root_id=action_target, + edge_query_dict={'!=': {NEG_CONDITION: True}}) + + is_switch_mode = \ + connected_component.get_vertex(scenario_vertex.vertex_id) + + if is_switch_mode: + initial_map = Mapping(scenario_vertex, vertex, True) + mat = self._entity_graph.algo.sub_graph_matching(subgraph, + initial_map) + matches.append((False, mat)) + else: + matches.append((True, [])) + return matches + + def _find_edge_subgraph_matching(self, + subgraphs, + action_target, + vertex, + scenario_edge): + """calculates subgraph matching for edge + + iterates over all the subgraphs, and checks if the triggered edge is a + negative edge then mark it as deleted=false and negative=false so that + subgraph matching on that edge will work correctly. after running + subgraph matching, we need to remove the negative vertices that were + added due to the change above. + """ + + matches = [] + for subgraph in subgraphs: + is_switch_mode = \ + subgraph.get_edge(scenario_edge.source.vertex_id, + scenario_edge.target.vertex_id, + scenario_edge.edge.label).get(NEG_CONDITION, + False) + + connected_component = subgraph.algo.graph_query_vertices( + root_id=action_target, + edge_query_dict={'!=': {NEG_CONDITION: True}}) + + # change the is_deleted and negative_condition props to false when + # is_switch_mode=true so that when we have an event on a + # negative_condition=true edge it will find the correct subgraph + self._switch_edge_negative_props(is_switch_mode, scenario_edge, + subgraph, False) + + initial_map = Mapping(scenario_edge.edge, vertex, False) + curr_matches = \ + self._entity_graph.algo.sub_graph_matching(subgraph, + initial_map) + + # switch back to the original values + self._switch_edge_negative_props(is_switch_mode, scenario_edge, + subgraph, True) + + self._remove_negative_vertices_from_matches(curr_matches, + connected_component) + + matches.append((is_switch_mode, curr_matches)) + return matches + + @staticmethod + def _switch_edge_negative_props(is_switch_mode, + scenario_edge, + subgraph, + status): + if is_switch_mode: + scenario_edge.edge[NEG_CONDITION] = status + scenario_edge.edge[EProps.IS_DELETED] = status + subgraph.update_edge(scenario_edge.edge) + + @staticmethod + def _remove_negative_vertices_from_matches(matches, connected_component): + for match in matches: + ver_ids = [v.vertex_id for v in connected_component.get_vertices()] + ver_to_remove = [id for id in match.keys() if id not in ver_ids] + for v_id in ver_to_remove: + del match[v_id] + class ActionTracker(object): """Keeps track of all active actions and relative dominance/priority. diff --git a/vitrage/evaluator/template_data.py b/vitrage/evaluator/template_data.py index cf0d3b36b..73de08a35 100644 --- a/vitrage/evaluator/template_data.py +++ b/vitrage/evaluator/template_data.py @@ -11,6 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + from collections import namedtuple from sympy.logic.boolalg import And from sympy.logic.boolalg import Not @@ -19,14 +20,17 @@ from sympy.logic.boolalg import to_dnf as sympy_to_dfn from sympy import Symbol +from vitrage.common.constants import EdgeProperties as EProps +from vitrage.common.constants import VertexProperties as VProps from vitrage.evaluator.template_fields import TemplateFields as TFields +from vitrage.graph.algo_driver.sub_graph_matching import NEG_CONDITION +from vitrage.graph.driver.networkx_graph import NXGraph from vitrage.graph import Edge from vitrage.graph import Vertex - ConditionVar = namedtuple('ConditionVar', ['variable', 'type', 'positive']) ActionSpecs = namedtuple('ActionSpecs', ['type', 'targets', 'properties']) -Scenario = namedtuple('Scenario', ['id', 'condition', 'actions']) +Scenario = namedtuple('Scenario', ['id', 'condition', 'actions', 'subgraphs']) EdgeDescription = namedtuple('EdgeDescription', ['edge', 'source', 'target']) @@ -132,12 +136,13 @@ class TemplateData(object): scenarios = [] for counter, scenarios_def in enumerate(scenarios_defs): - scenario_dict = scenarios_def[TFields.SCENARIO] condition = self._parse_condition(scenario_dict[TFields.CONDITION]) action_specs = self._build_actions(scenario_dict[TFields.ACTIONS]) + subgraphs = self._build_subgraphs(condition) scenario_id = "%s-scenario%s" % (self.name, str(counter)) - scenarios.append(Scenario(scenario_id, condition, action_specs)) + scenarios.append(Scenario(scenario_id, condition, + action_specs, subgraphs)) return scenarios @@ -156,6 +161,41 @@ class TemplateData(object): return actions + def _build_subgraphs(self, condition): + return [self._build_subgraph(clause) for clause in condition] + + def _build_subgraph(self, clause): + condition_g = NXGraph("scenario condition") + + for term in clause: + if term.type == ENTITY: + term.variable[VProps.IS_DELETED] = False + condition_g.add_vertex(term.variable) + + else: # type = relationship + edge_desc = term.variable + self._set_edge_relationship_info(edge_desc, term.positive) + self._add_edge_relationship(condition_g, edge_desc) + + return condition_g + + @staticmethod + def _set_edge_relationship_info(edge_description, is_positive_condition): + if not is_positive_condition: + edge_description.edge[NEG_CONDITION] = True + edge_description.edge[EProps.IS_DELETED] = True + else: + edge_description.edge[EProps.IS_DELETED] = False + + edge_description.source[VProps.IS_DELETED] = False + edge_description.target[VProps.IS_DELETED] = False + + @staticmethod + def _add_edge_relationship(condition_graph, edge_description): + condition_graph.add_vertex(edge_description.source) + condition_graph.add_vertex(edge_description.target) + condition_graph.add_edge(edge_description.edge) + def _parse_condition(self, condition_str): """Parse condition string into an object @@ -181,7 +221,7 @@ class TemplateData(object): return [self._extract_and_condition(condition_dnf)] if isinstance(condition_dnf, Not): - return [[(self._extract_condition_var(condition_dnf, False))]] + return [(self._extract_not_condition_var(condition_dnf))] if isinstance(condition_dnf, Symbol): return [[(self._extract_condition_var(condition_dnf, True))]] @@ -213,9 +253,15 @@ class TemplateData(object): return [self._extract_condition_var(arg, isinstance(arg, Symbol)) for arg in and_condition.args] - def _extract_condition_var(self, symbol, positive): + def _extract_not_condition_var(self, not_condition): + return [self._extract_condition_var(arg, False) + for arg in not_condition.args] - var, var_type = self._extract_var(str(symbol)) + def _extract_condition_var(self, symbol, positive): + if isinstance(symbol, Not): + return self._extract_not_condition_var(symbol)[0] + else: + var, var_type = self._extract_var(str(symbol)) return ConditionVar(var, var_type, positive) def _extract_var(self, template_id): diff --git a/vitrage/evaluator/template_fields.py b/vitrage/evaluator/template_fields.py index 3f8769e0a..920b794c6 100644 --- a/vitrage/evaluator/template_fields.py +++ b/vitrage/evaluator/template_fields.py @@ -12,10 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -from vitrage.common.constants import TopologyFields +from vitrage.common.constants import TemplateTopologyFields -class TemplateFields(TopologyFields): +class TemplateFields(TemplateTopologyFields): SCENARIOS = 'scenarios' diff --git a/vitrage/graph/algo_driver/networkx_algorithm.py b/vitrage/graph/algo_driver/networkx_algorithm.py index 136ee5659..77636dc53 100644 --- a/vitrage/graph/algo_driver/networkx_algorithm.py +++ b/vitrage/graph/algo_driver/networkx_algorithm.py @@ -17,7 +17,10 @@ from networkx.algorithms import simple_paths from oslo_log import log as logging +from vitrage.common.constants import EdgeProperties as EProps from vitrage.graph.algo_driver.algorithm import GraphAlgorithm +from vitrage.graph.algo_driver.algorithm import Mapping +from vitrage.graph.algo_driver.sub_graph_matching import NEG_CONDITION from vitrage.graph.algo_driver.sub_graph_matching import subgraph_matching from vitrage.graph.driver import Direction from vitrage.graph.driver import Edge @@ -97,23 +100,41 @@ class NXAlgorithm(GraphAlgorithm): str(self.graph._g.edges(data=True))) return graph - @staticmethod - def _edge_result_to_list(edge_result): - d = dict() - for source_id, target_id, label, data in edge_result: - d[(source_id, target_id, label)] = \ - Edge(source_id, target_id, label, properties=data) - return d.values() + def sub_graph_matching(self, + subgraph, + known_match, + validate=False): + """Finds all the matching subgraphs in the graph - @staticmethod - def _vertex_result_to_list(vertex_result): - d = dict() - for v_id, data in vertex_result: - d[v_id] = Vertex(vertex_id=v_id, properties=data) - return d.values() + In case the known_match has a subgraph edge with property + "negative_condition" then run subgraph matching on the edge vertices + and unite the results. + Otherwise just run subgraph matching and return its result. - def sub_graph_matching(self, subgraph, known_matches, validate=False): - return subgraph_matching(self.graph, subgraph, known_matches, validate) + :param subgraph: the subgraph to match + :param known_match: starting point at the subgraph and the graph + :param validate: + :return: all the matching subgraphs in the graph + """ + sge = known_match.subgraph_element + ge = known_match.graph_element + + if not known_match.is_vertex and sge.get(NEG_CONDITION): + source_matches = self._filtered_subgraph_matching(ge.source_id, + sge.source_id, + subgraph, + validate) + target_matches = self._filtered_subgraph_matching(ge.target_id, + sge.target_id, + subgraph, + validate) + + return self._list_union(source_matches, target_matches) + else: + return subgraph_matching(self.graph, + subgraph, + [known_match], + validate) def create_graph_from_matching_vertices(self, vertex_attr_filter=None, @@ -159,3 +180,53 @@ class NXAlgorithm(GraphAlgorithm): return simple_paths.all_simple_paths(self.graph._g, source=source, target=target) + + def _filtered_subgraph_matching(self, + ge_v_id, + sge_v_id, + subgraph, + validate): + """Runs subgraph_matching on edges vertices with filtering + + Runs subgraph_matching on edges vertices after checking if that vertex + has real neighbors in the entity graph. + """ + if self.graph.neighbors(ge_v_id, + edge_attr_filter={EProps.IS_DELETED: False}): + template_vertex = subgraph.get_vertex(sge_v_id) + graph_vertex = self.graph.get_vertex(ge_v_id) + match = Mapping(template_vertex, graph_vertex, True) + return subgraph_matching(self.graph, subgraph, [match], validate) + + return [] + + @staticmethod + def _edge_result_to_list(edge_result): + d = dict() + for source_id, target_id, label, data in edge_result: + d[(source_id, target_id, label)] = \ + Edge(source_id, target_id, label, properties=data) + return d.values() + + @staticmethod + def _vertex_result_to_list(vertex_result): + d = dict() + for v_id, data in vertex_result: + d[v_id] = Vertex(vertex_id=v_id, properties=data) + return d.values() + + @staticmethod + def _list_union(list_1, list_2): + """Union of list that aren't hashable + + Can't use here set union because the items in the lists are + dictionaries and they are not hashable for set. + + :return: list - union list + """ + + for target_item in list_2: + if target_item not in list_1: + list_1.append(target_item) + + return list_1 diff --git a/vitrage/graph/algo_driver/sub_graph_matching.py b/vitrage/graph/algo_driver/sub_graph_matching.py index 972c8226e..fdea5eec9 100644 --- a/vitrage/graph/algo_driver/sub_graph_matching.py +++ b/vitrage/graph/algo_driver/sub_graph_matching.py @@ -11,7 +11,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + from oslo_log import log as logging +import six from vitrage.common.exception import VitrageAlgorithmError from vitrage.graph.filter import check_filter @@ -22,6 +24,8 @@ LOG = logging.getLogger(__name__) MAPPED_V_ID = 'mapped_v_id' NEIGHBORS_MAPPED = 'neighbors_mapped' GRAPH_VERTEX = 'graph_vertex' +NEG_VERTEX = 'negative vertex' +NEG_CONDITION = 'negative_condition' def subgraph_matching(base_graph, subgraph, matches, validate=False): @@ -94,7 +98,10 @@ def subgraph_matching(base_graph, subgraph, matches, validate=False): continue # STEP 3: FIND A SUB-GRAPH VERTEX TO MAP - v_with_unmapped_neighbors = vertices_with_unmapped_neighbors.pop(0) + v_with_unmapped_neighbors = _choose_vertex( + vertices_with_unmapped_neighbors, + curr_subgraph) + unmapped_neighbors = list(filter( lambda v: not v.get(MAPPED_V_ID), curr_subgraph.neighbors(v_with_unmapped_neighbors.vertex_id))) @@ -104,7 +111,10 @@ def subgraph_matching(base_graph, subgraph, matches, validate=False): curr_subgraph.update_vertex(v_with_unmapped_neighbors) queue.append(curr_subgraph) continue - subgraph_vertex_to_map = unmapped_neighbors.pop(0) + subgraph_vertex_to_map = _choose_vertex( + unmapped_neighbors, + curr_subgraph, + curr_v=v_with_unmapped_neighbors) # STEP 4: PROPERTIES CHECK graph_candidate_vertices = base_graph.neighbors( @@ -114,25 +124,66 @@ def subgraph_matching(base_graph, subgraph, matches, validate=False): # STEP 5: STRUCTURE CHECK edges = _get_edges_to_mapped_vertices(curr_subgraph, subgraph_vertex_to_map.vertex_id) + neg_edges = set(e for e in edges if e.get(NEG_CONDITION)) + pos_edges = edges.difference(neg_edges) + + if not graph_candidate_vertices and neg_edges and not pos_edges: + subgraph_vertex_to_map[MAPPED_V_ID] = NEG_VERTEX + curr_subgraph.update_vertex(subgraph_vertex_to_map) + queue.append(curr_subgraph.copy()) + continue + + found_subgraphs = [] for graph_vertex in graph_candidate_vertices: subgraph_vertex_to_map[MAPPED_V_ID] = graph_vertex.vertex_id subgraph_vertex_to_map[GRAPH_VERTEX] = graph_vertex curr_subgraph.update_vertex(subgraph_vertex_to_map) - if _graph_contains_subgraph_edges(base_graph, - curr_subgraph, - edges): - queue.append(curr_subgraph.copy()) + if not _graph_contains_subgraph_edges(base_graph, + curr_subgraph, + pos_edges): + continue + if not _graph_contains_subgraph_edges(base_graph, + curr_subgraph, + neg_edges): + del found_subgraphs[:] + break + if neg_edges and not pos_edges: + subgraph_vertex_to_map[MAPPED_V_ID] = NEG_VERTEX + curr_subgraph.update_vertex(subgraph_vertex_to_map) + found_subgraphs.append(curr_subgraph.copy()) + + queue.extend(found_subgraphs) # Last thing: Convert results to the expected format! + return _generate_result(final_subgraphs) + + +def _generate_result(final_subgraphs): result = [] for mapping in final_subgraphs: - # TODO(ihefetz) If needed, Here we can easily extract the edge - # matches from the mapping graph - a = {v.vertex_id: v[GRAPH_VERTEX] for v in mapping.get_vertices()} - result.append(a) + subgraph_vertices = dict() + for v in mapping.get_vertices(): + v_id = v[MAPPED_V_ID] + if isinstance(v_id, six.string_types) and v_id is not NEG_VERTEX: + subgraph_vertices[v.vertex_id] = v[GRAPH_VERTEX] + + if subgraph_vertices not in result: + result.append(subgraph_vertices) return result +def _choose_vertex(vertices, subgraph, curr_v=None): + """Return a vertex with a positive edge if exists, otherwise the first one. + + """ + for v in vertices: + curr_vertex_id = curr_v.vertex_id if curr_v else None + if not subgraph.get_edges(v.vertex_id, curr_vertex_id, + attr_filter={NEG_CONDITION: True}): + return v + return vertices.pop(0) + + def _get_edges_to_mapped_vertices(graph, vertex_id): """Get all edges (to/from) vertex where neighbor has a MAPPED_V_ID @@ -169,7 +220,12 @@ def _graph_contains_subgraph_edges(graph, subgraph, subgraph_edges): found_graph_edge = graph.get_edge(graph_v_id_source, graph_v_id_target, e.label) - if not found_graph_edge or not check_filter(found_graph_edge, e): + + if not found_graph_edge and e.get(NEG_CONDITION): + continue + + if not found_graph_edge or not check_filter(found_graph_edge, e, + NEG_CONDITION): return False return True diff --git a/vitrage/graph/driver/elements.py b/vitrage/graph/driver/elements.py index 929274016..b9fb8ceb8 100644 --- a/vitrage/graph/driver/elements.py +++ b/vitrage/graph/driver/elements.py @@ -158,3 +158,6 @@ class Edge(PropertiesElement): :return: the other vertex id """ return self.source_id if self.target_id == v_id else self.target_id + + def has_vertex(self, v_id): + return self.source_id == v_id or self.target_id == v_id diff --git a/vitrage/graph/driver/graph.py b/vitrage/graph/driver/graph.py index 68b72edcd..43cf2cd61 100644 --- a/vitrage/graph/driver/graph.py +++ b/vitrage/graph/driver/graph.py @@ -202,11 +202,18 @@ class Graph(object): pass @abc.abstractmethod - def get_edges(self, v_id, direction=Direction.BOTH, + def get_edges(self, + v1_id, + v2_id=None, + direction=Direction.BOTH, attr_filter=None): """Fetch multiple edges from the graph, - Fetch an edge from the graph, according to its two vertices and label + Fetch all edges from the graph, according to its two vertices. + If only one vertex id is given it finds all the edges from this vertex + to all other vertices. + If two vertices ids are given it finds all the edges between those two + vertices. EXAMPLE ------- @@ -218,8 +225,11 @@ class Graph(object): v_id=v2.vertex_id, attr_filter={'LABEL': ['ON', 'WITH']}) - :param v_id: vertex id a vertex - :type v_id: str + :param v1_id: first vertex id of vertex + :type v1_id: str + + :param v2_id: second vertex id of vertex + :type v2_id: str :param direction: specify In/Out/Both for edge direction :type direction: int diff --git a/vitrage/graph/driver/networkx_graph.py b/vitrage/graph/driver/networkx_graph.py index 5d5a11dde..fad39f675 100644 --- a/vitrage/graph/driver/networkx_graph.py +++ b/vitrage/graph/driver/networkx_graph.py @@ -118,7 +118,10 @@ class NXGraph(Graph): return edge_copy(source_id, target_id, label, properties) return None - def get_edges(self, v_id, direction=Direction.BOTH, + def get_edges(self, + v1_id, + v2_id=None, + direction=Direction.BOTH, attr_filter=None): """Fetch multiple edges from the graph @@ -128,9 +131,14 @@ class NXGraph(Graph): return check_filter(edge_data, attr_filter) nodes, edges = self._neighboring_nodes_edges_query( - v_id, edge_predicate=check_edge, direction=direction) + v1_id, edge_predicate=check_edge, direction=direction) + edge_copies = set(edge_copy(u, v, label, data) for u, v, label, data in edges) + + if v2_id: + edge_copies = [e for e in edge_copies if e.has_vertex(v2_id)] + return edge_copies def _get_edges_by_direction(self, v_id, direction): diff --git a/vitrage/tests/functional/evaluator/test_scenario_evaluator.py b/vitrage/tests/functional/evaluator/test_scenario_evaluator.py index a8dfdf12a..f0b141d66 100644 --- a/vitrage/tests/functional/evaluator/test_scenario_evaluator.py +++ b/vitrage/tests/functional/evaluator/test_scenario_evaluator.py @@ -12,22 +12,32 @@ # License for the specific language governing permissions and limitations # under the License. +from six.moves import queue + from oslo_config import cfg -from six.moves import queue from vitrage.common.constants import DatasourceAction from vitrage.common.constants import DatasourceProperties as DSProps from vitrage.common.constants import EdgeProperties as EProps +from vitrage.common.constants import EntityCategory from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources.nagios.properties import NagiosProperties +from vitrage.datasources.nagios.properties import NagiosTestStatus +from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE +from vitrage.datasources.neutron.port import NEUTRON_PORT_DATASOURCE from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE +from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE +from vitrage.datasources.nova.zone import NOVA_ZONE_DATASOURCE from vitrage.evaluator.scenario_evaluator import ScenarioEvaluator from vitrage.evaluator.scenario_repository import ScenarioRepository +from vitrage.graph import create_edge from vitrage.tests.functional.base import \ TestFunctionalBase import vitrage.tests.mocks.mock_driver as mock_driver from vitrage.tests.mocks import utils _TARGET_HOST = 'host-2' +_TARGET_ZONE = 'zone-1' _NAGIOS_TEST_INFO = {'resource_name': _TARGET_HOST, DSProps.DATASOURCE_ACTION: DatasourceAction.SNAPSHOT} @@ -58,8 +68,9 @@ class TestScenarioEvaluator(TestFunctionalBase): event_queue, processor, evaluator = self._init_system() - host_v = self._get_host_from_graph(_TARGET_HOST, - processor.entity_graph) + host_v = self._get_entity_from_graph(NOVA_HOST_DATASOURCE, + _TARGET_HOST, + processor.entity_graph) self.assertEqual('AVAILABLE', host_v[VProps.AGGREGATED_STATE], 'host should be AVAILABLE when starting') @@ -85,8 +96,9 @@ class TestScenarioEvaluator(TestFunctionalBase): event_queue, processor, evaluator = self._init_system() - host_v = self._get_host_from_graph(_TARGET_HOST, - processor.entity_graph) + host_v = self._get_entity_from_graph(NOVA_HOST_DATASOURCE, + _TARGET_HOST, + processor.entity_graph) self.assertEqual('AVAILABLE', host_v[VProps.AGGREGATED_STATE], 'host should be AVAILABLE when starting') @@ -130,8 +142,9 @@ class TestScenarioEvaluator(TestFunctionalBase): event_queue, processor, evaluator = self._init_system() - host_v = self._get_host_from_graph(_TARGET_HOST, - processor.entity_graph) + host_v = self._get_entity_from_graph(NOVA_HOST_DATASOURCE, + _TARGET_HOST, + processor.entity_graph) self.assertEqual('AVAILABLE', host_v[VProps.AGGREGATED_STATE], 'host should be AVAILABLE when starting') @@ -168,8 +181,9 @@ class TestScenarioEvaluator(TestFunctionalBase): event_queue, processor, evaluator = self._init_system() - host_v = self._get_host_from_graph(_TARGET_HOST, - processor.entity_graph) + host_v = self._get_entity_from_graph(NOVA_HOST_DATASOURCE, + _TARGET_HOST, + processor.entity_graph) self.assertEqual('AVAILABLE', host_v[VProps.AGGREGATED_STATE], 'host should be AVAILABLE when starting') @@ -197,7 +211,6 @@ class TestScenarioEvaluator(TestFunctionalBase): self._get_deduced_alarms_on_host(host_v, processor.entity_graph) self.assertEqual(0, len(alarms)) - # todo: (erosensw) uncomment this test def test_overlapping_deduced_alarm_1(self): event_queue, processor, evaluator = self._init_system() @@ -293,13 +306,334 @@ class TestScenarioEvaluator(TestFunctionalBase): self.assertEqual(1, len(alarms)) self.assertEqual('WARNING', alarms[0]['severity']) + def test_simple_not_operator_deduced_alarm(self): + """Handles a simple not operator use case + + We have created the following template: if there is a neutron.port that + doesn't have a nagios alarm of type PORT_PROBLEM on it, then raise a + deduced alarm on the port called simple_port_deduced_alarm. + The test has 3 steps in it: + 1. create neutron.network and neutron.port and check that the + simple_port_deduced_alarm is raised on the neutron.port because it + doesn't have a nagios alarm on it. + 2. create a nagios alarm called PORT_PROBLEM on the port and check + that the alarm simple_port_deduced_alarm doesn't appear on the + neutron.port. + 3. delete the nagios alarm called PORT_PROBLEM from the port, and + check that the alarm alarm simple_port_deduced_alarm appear on the + neutron.port. + """ + + event_queue, processor, evaluator = self._init_system() + entity_graph = processor.entity_graph + + # constants + num_orig_vertices = entity_graph.num_vertices() + num_orig_edges = entity_graph.num_edges() + num_added_vertices = 2 + num_added_edges = 2 + num_deduced_vertices = 1 + num_deduced_edges = 1 + num_nagios_alarm_vertices = 1 + num_nagios_alarm_edges = 1 + + # find instances + query = { + VProps.CATEGORY: 'RESOURCE', + VProps.TYPE: NOVA_INSTANCE_DATASOURCE + } + instance_ver = entity_graph.get_vertices(vertex_attr_filter=query)[0] + + # update network + network_event = { + 'tenant_id': 'admin', + 'name': 'net-0', + 'updated_at': '2015-12-01T12:46:41Z', + 'status': 'active', + 'id': '12345', + 'vitrage_entity_type': 'neutron.network', + 'vitrage_datasource_action': 'snapshot', + 'vitrage_sample_date': '2015-12-01T12:46:41Z', + } + + # update port + port_event = { + 'tenant_id': 'admin', + 'name': 'port-0', + 'updated_at': '2015-12-01T12:46:41Z', + 'status': 'active', + 'id': '54321', + 'vitrage_entity_type': 'neutron.port', + 'vitrage_datasource_action': 'snapshot', + 'vitrage_sample_date': '2015-12-01T12:46:41Z', + 'network_id': '12345', + 'device_id': instance_ver.get(VProps.ID), + 'device_owner': 'compute:nova', + 'fixed_ips': {} + } + + processor.process_event(network_event) + processor.process_event(port_event) + port_vertex = entity_graph.get_vertices( + vertex_attr_filter={VProps.TYPE: NEUTRON_PORT_DATASOURCE})[0] + while not event_queue.empty(): + processor.process_event(event_queue.get()) + + # test asserts + query = {VProps.CATEGORY: EntityCategory.ALARM} + port_neighbors = entity_graph.neighbors(port_vertex.vertex_id, + vertex_attr_filter=query) + self.assertEqual(num_orig_vertices + num_added_vertices + + num_deduced_vertices, entity_graph.num_vertices()) + self.assertEqual(num_orig_edges + num_added_edges + num_deduced_edges, + entity_graph.num_edges()) + self.assertEqual(1, len(port_neighbors)) + self.assertEqual(port_neighbors[0][VProps.CATEGORY], + EntityCategory.ALARM) + self.assertEqual(port_neighbors[0][VProps.TYPE], 'vitrage') + self.assertEqual(port_neighbors[0][VProps.NAME], + 'simple_port_deduced_alarm') + + # Add PORT_PROBLEM alarm + test_vals = {'status': 'WARNING', + 'service': 'PORT_PROBLEM', + 'name': 'PORT_PROBLEM', + DSProps.DATASOURCE_ACTION: DatasourceAction.SNAPSHOT, + VProps.RESOURCE_ID: port_vertex.get(VProps.ID), + 'resource_name': port_vertex.get(VProps.ID), + 'resource_type': NEUTRON_PORT_DATASOURCE} + generator = mock_driver.simple_nagios_alarm_generators(1, 1, test_vals) + nagios_event = mock_driver.generate_random_events_list(generator)[0] + + processor.process_event(nagios_event) + while not event_queue.empty(): + processor.process_event(event_queue.get()) + + # test asserts + self.assertEqual(num_orig_vertices + num_added_vertices + + num_deduced_vertices + num_nagios_alarm_vertices, + entity_graph.num_vertices()) + self.assertEqual(num_orig_edges + num_added_edges + num_deduced_edges + + num_nagios_alarm_edges, entity_graph.num_edges()) + + query = {VProps.CATEGORY: EntityCategory.ALARM, VProps.TYPE: 'vitrage'} + port_neighbors = entity_graph.neighbors(port_vertex.vertex_id, + vertex_attr_filter=query) + self.assertEqual(1, len(port_neighbors)) + self.assertEqual(port_neighbors[0][VProps.CATEGORY], + EntityCategory.ALARM) + self.assertEqual(port_neighbors[0][VProps.TYPE], 'vitrage') + self.assertEqual(port_neighbors[0][VProps.NAME], + 'simple_port_deduced_alarm') + self.assertEqual(port_neighbors[0][VProps.IS_DELETED], True) + + query = {VProps.CATEGORY: EntityCategory.ALARM, VProps.TYPE: 'nagios'} + port_neighbors = entity_graph.neighbors(port_vertex.vertex_id, + vertex_attr_filter=query) + self.assertEqual(port_neighbors[0][VProps.CATEGORY], + EntityCategory.ALARM) + self.assertEqual(port_neighbors[0][VProps.TYPE], 'nagios') + self.assertEqual(port_neighbors[0][VProps.NAME], 'PORT_PROBLEM') + self.assertEqual(port_neighbors[0][VProps.IS_DELETED], False) + self.assertEqual(port_neighbors[0][VProps.IS_PLACEHOLDER], False) + + # disable PORT_PROBLEM alarm + nagios_event[NagiosProperties.STATUS] = NagiosTestStatus.OK + processor.process_event(nagios_event) + while not event_queue.empty(): + processor.process_event(event_queue.get()) + + # test asserts + self.assertEqual(num_orig_vertices + num_added_vertices + + num_deduced_vertices + num_nagios_alarm_vertices, + entity_graph.num_vertices()) + self.assertEqual(num_orig_edges + num_added_edges + num_deduced_edges + + num_nagios_alarm_edges, entity_graph.num_edges()) + + query = {VProps.CATEGORY: EntityCategory.ALARM, VProps.TYPE: 'vitrage'} + port_neighbors = entity_graph.neighbors(port_vertex.vertex_id, + vertex_attr_filter=query) + self.assertEqual(1, len(port_neighbors)) + self.assertEqual(port_neighbors[0][VProps.CATEGORY], + EntityCategory.ALARM) + self.assertEqual(port_neighbors[0][VProps.TYPE], 'vitrage') + self.assertEqual(port_neighbors[0][VProps.NAME], + 'simple_port_deduced_alarm') + self.assertEqual(port_neighbors[0][VProps.IS_DELETED], False) + + query = {VProps.CATEGORY: EntityCategory.ALARM, VProps.TYPE: 'nagios'} + port_neighbors = entity_graph.neighbors(port_vertex.vertex_id, + vertex_attr_filter=query) + self.assertEqual(port_neighbors[0][VProps.CATEGORY], + EntityCategory.ALARM) + self.assertEqual(port_neighbors[0][VProps.TYPE], 'nagios') + self.assertEqual(port_neighbors[0][VProps.NAME], 'PORT_PROBLEM') + self.assertEqual(port_neighbors[0][VProps.IS_DELETED], True) + + def test_complex_not_operator_deduced_alarm(self): + """Handles a complex not operator use case + + We have created the following template: if there is a openstack.cluster + that has a nova.zone which is connected to a neutron.network and also + there is no nagios alarm of type CLUSTER_PROBLEM on the cluster and no + nagios alarm of type NETWORK_PROBLEM on the neutron.network, then raise + a deduced alarm on the nova.zone called complex_zone_deduced_alarm. + The test has 3 steps in it: + 1. create a neutron.network and connect it to a zone, and check that + the complex_zone_deduced_alarm is raised on the nova.zone because it + doesn't have nagios alarms the openstack.cluster and on the + neutron.network. + 2. create a nagios alarm called NETWORK_PROBLEM on the network and + check that the alarm complex_zone_deduced_alarm doesn't appear on + the nova.zone. + 3. delete the nagios alarm called NETWORK_PROBLEM from the port, and + check that the alarm alarm complex_zone_deduced_alarm appear on the + nova.zone. + """ + + event_queue, processor, evaluator = self._init_system() + entity_graph = processor.entity_graph + + # constants + num_orig_vertices = entity_graph.num_vertices() + num_orig_edges = entity_graph.num_edges() + num_added_vertices = 2 + num_added_edges = 3 + num_deduced_vertices = 1 + num_deduced_edges = 1 + num_network_alarm_vertices = 1 + num_network_alarm_edges = 1 + + # update zone + generator = mock_driver.simple_zone_generators(1, 1, snapshot_events=1) + zone_event = mock_driver.generate_random_events_list(generator)[0] + zone_event['zoneName'] = 'zone-7' + + # update network + network_event = { + 'tenant_id': 'admin', + 'name': 'net-0', + 'updated_at': '2015-12-01T12:46:41Z', + 'status': 'active', + 'id': '12345', + 'vitrage_entity_type': 'neutron.network', + 'vitrage_datasource_action': 'snapshot', + 'vitrage_sample_date': '2015-12-01T12:46:41Z', + } + + # process events + processor.process_event(zone_event) + query = {VProps.TYPE: NOVA_ZONE_DATASOURCE, VProps.ID: 'zone-7'} + zone_vertex = entity_graph.get_vertices(vertex_attr_filter=query)[0] + processor.process_event(network_event) + query = {VProps.TYPE: NEUTRON_NETWORK_DATASOURCE} + network_vertex = entity_graph.get_vertices(vertex_attr_filter=query)[0] + + # add edge between network and zone + edge = create_edge(network_vertex.vertex_id, zone_vertex.vertex_id, + 'attached') + entity_graph.add_edge(edge) + + while not event_queue.empty(): + processor.process_event(event_queue.get()) + + # test asserts + query = {VProps.CATEGORY: EntityCategory.ALARM} + zone_neighbors = entity_graph.neighbors(zone_vertex.vertex_id, + vertex_attr_filter=query) + self.assertEqual(num_orig_vertices + num_added_vertices + + num_deduced_vertices, entity_graph.num_vertices()) + self.assertEqual(num_orig_edges + num_added_edges + num_deduced_edges, + entity_graph.num_edges()) + self.assertEqual(1, len(zone_neighbors)) + self.assertEqual(zone_neighbors[0][VProps.CATEGORY], + EntityCategory.ALARM) + self.assertEqual(zone_neighbors[0][VProps.TYPE], 'vitrage') + self.assertEqual(zone_neighbors[0][VProps.NAME], + 'complex_zone_deduced_alarm') + + # Add NETWORK_PROBLEM alarm + test_vals = {'status': 'WARNING', + 'service': 'NETWORK_PROBLEM', + 'name': 'NETWORK_PROBLEM', + DSProps.DATASOURCE_ACTION: DatasourceAction.SNAPSHOT, + VProps.RESOURCE_ID: network_vertex[VProps.ID], + 'resource_name': network_vertex[VProps.ID], + 'resource_type': NEUTRON_NETWORK_DATASOURCE} + generator = mock_driver.simple_nagios_alarm_generators(1, 1, test_vals) + nagios_event = mock_driver.generate_random_events_list(generator)[0] + + processor.process_event(nagios_event) + while not event_queue.empty(): + processor.process_event(event_queue.get()) + + self.assertEqual(num_orig_vertices + num_added_vertices + + num_deduced_vertices + num_network_alarm_vertices, + entity_graph.num_vertices()) + self.assertEqual(num_orig_edges + num_added_edges + num_deduced_edges + + num_network_alarm_edges, entity_graph.num_edges()) + + query = {VProps.CATEGORY: EntityCategory.ALARM} + network_neighbors = entity_graph.neighbors(network_vertex.vertex_id, + vertex_attr_filter=query) + self.assertEqual(1, len(network_neighbors)) + self.assertEqual(network_neighbors[0][VProps.CATEGORY], + EntityCategory.ALARM) + self.assertEqual(network_neighbors[0][VProps.TYPE], 'nagios') + self.assertEqual(network_neighbors[0][VProps.NAME], 'NETWORK_PROBLEM') + self.assertEqual(network_neighbors[0][VProps.IS_DELETED], False) + self.assertEqual(network_neighbors[0][VProps.IS_PLACEHOLDER], False) + + zone_neighbors = entity_graph.neighbors(zone_vertex.vertex_id, + vertex_attr_filter=query) + self.assertEqual(1, len(zone_neighbors)) + self.assertEqual(zone_neighbors[0][VProps.CATEGORY], + EntityCategory.ALARM) + self.assertEqual(zone_neighbors[0][VProps.TYPE], 'vitrage') + self.assertEqual(zone_neighbors[0][VProps.NAME], + 'complex_zone_deduced_alarm') + self.assertEqual(zone_neighbors[0][VProps.IS_DELETED], True) + + # delete NETWORK_PROBLEM alarm + nagios_event[NagiosProperties.STATUS] = NagiosTestStatus.OK + processor.process_event(nagios_event) + while not event_queue.empty(): + processor.process_event(event_queue.get()) + + self.assertEqual(num_orig_vertices + num_added_vertices + + num_deduced_vertices + num_network_alarm_vertices, + entity_graph.num_vertices()) + self.assertEqual(num_orig_edges + num_added_edges + num_deduced_edges + + num_network_alarm_edges, entity_graph.num_edges()) + + query = {VProps.CATEGORY: EntityCategory.ALARM} + network_neighbors = entity_graph.neighbors(network_vertex.vertex_id, + vertex_attr_filter=query) + self.assertEqual(1, len(network_neighbors)) + self.assertEqual(network_neighbors[0][VProps.CATEGORY], + EntityCategory.ALARM) + self.assertEqual(network_neighbors[0][VProps.TYPE], 'nagios') + self.assertEqual(network_neighbors[0][VProps.NAME], 'NETWORK_PROBLEM') + self.assertEqual(network_neighbors[0][VProps.IS_DELETED], True) + + zone_neighbors = entity_graph.neighbors(zone_vertex.vertex_id, + vertex_attr_filter=query) + self.assertEqual(1, len(zone_neighbors)) + self.assertEqual(zone_neighbors[0][VProps.CATEGORY], + EntityCategory.ALARM) + self.assertEqual(zone_neighbors[0][VProps.TYPE], 'vitrage') + self.assertEqual(zone_neighbors[0][VProps.NAME], + 'complex_zone_deduced_alarm') + self.assertEqual(zone_neighbors[0][VProps.IS_DELETED], False) + def get_host_after_event(self, event_queue, nagios_event, processor, target_host): processor.process_event(nagios_event) while not event_queue.empty(): processor.process_event(event_queue.get()) - host_v = self._get_host_from_graph(target_host, - processor.entity_graph) + host_v = self._get_entity_from_graph(NOVA_HOST_DATASOURCE, + target_host, + processor.entity_graph) return host_v def _init_system(self): @@ -311,13 +645,12 @@ class TestScenarioEvaluator(TestFunctionalBase): return event_queue, processor, evaluator @staticmethod - def _get_host_from_graph(host_name, entity_graph): - vertex_attrs = {VProps.TYPE: NOVA_HOST_DATASOURCE, - VProps.NAME: host_name} - host_vertices = entity_graph.get_vertices( - vertex_attr_filter=vertex_attrs) - assert len(host_vertices) == 1, "incorrect number of vertices" - return host_vertices[0] + def _get_entity_from_graph(entity_type, entity_name, entity_graph): + vertex_attrs = {VProps.TYPE: entity_type, + VProps.NAME: entity_name} + vertices = entity_graph.get_vertices(vertex_attr_filter=vertex_attrs) + # assert len(vertices) == 1, "incorrect number of vertices" + return vertices[0] @staticmethod def _get_deduced_alarms_on_host(host_v, entity_graph): diff --git a/vitrage/tests/mocks/trace_generator.py b/vitrage/tests/mocks/trace_generator.py index 4f562027a..3cce44cae 100644 --- a/vitrage/tests/mocks/trace_generator.py +++ b/vitrage/tests/mocks/trace_generator.py @@ -27,7 +27,6 @@ from random import randint # noinspection PyPep8Naming from vitrage.common.constants import EdgeLabel -from vitrage.common.constants import TopologyFields from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE from vitrage.datasources.static import StaticFields from vitrage.tests.mocks.entity_model import BasicEntityModel as Bem @@ -569,12 +568,12 @@ def _get_static_snapshot_driver_values(spec): switch_id = "s{}".format(switch_index) relationship = { - TopologyFields.SOURCE: switch_id, - TopologyFields.TARGET: host_id, - TopologyFields.RELATIONSHIP_TYPE: EdgeLabel.ATTACHED + StaticFields.SOURCE: switch_id, + StaticFields.TARGET: host_id, + StaticFields.RELATIONSHIP_TYPE: EdgeLabel.ATTACHED } rel = relationship.copy() - rel[TopologyFields.TARGET] = entities[host_id] + rel[StaticFields.TARGET] = entities[host_id] relationships[switch_id].append(rel) for host_index, switch_index in host_switch_mapping: diff --git a/vitrage/tests/resources/templates/evaluator/complex_not_operator_deduced_alarm.yaml b/vitrage/tests/resources/templates/evaluator/complex_not_operator_deduced_alarm.yaml new file mode 100644 index 000000000..6b2a70341 --- /dev/null +++ b/vitrage/tests/resources/templates/evaluator/complex_not_operator_deduced_alarm.yaml @@ -0,0 +1,59 @@ +metadata: + name: complex_correct_not_condition_template + description: complex not condition template for general tests +definitions: + entities: + - entity: + category: ALARM + type: nagios + name: NETWORK_PROBLEM + template_id: network_alarm + - entity: + category: ALARM + type: nagios + name: CLUSTER_PROBLEM + template_id: cluster_alarm + - entity: + category: RESOURCE + type: nova.zone + template_id: nova_zone + - entity: + category: RESOURCE + type: openstack.cluster + template_id: openstack_cluster + - entity: + category: RESOURCE + type: neutron.network + template_id: neutron_network + relationships: + - relationship: + source: cluster_alarm + target: openstack_cluster + relationship_type: on + template_id : alarm_on_cluster + - relationship: + source: openstack_cluster + target: nova_zone + relationship_type: contains + template_id : cluster_contains_zone + - relationship: + source: neutron_network + target: nova_zone + relationship_type: attached + template_id : network_attached_zone + - relationship: + source: network_alarm + target: neutron_network + relationship_type: on + template_id : alarm_on_network +scenarios: + - scenario: + condition: cluster_contains_zone and network_attached_zone and not alarm_on_cluster and not alarm_on_network + actions: + - action: + action_type: raise_alarm + properties: + alarm_name: complex_zone_deduced_alarm + severity: WARNING + action_target: + target: nova_zone diff --git a/vitrage/tests/resources/templates/evaluator/simple_not_operator_deduced_alarm.yaml b/vitrage/tests/resources/templates/evaluator/simple_not_operator_deduced_alarm.yaml new file mode 100644 index 000000000..4dc8a0f5d --- /dev/null +++ b/vitrage/tests/resources/templates/evaluator/simple_not_operator_deduced_alarm.yaml @@ -0,0 +1,31 @@ +metadata: + name: basic_correct_not_condition_template + description: basic not condition template for general tests +definitions: + entities: + - entity: + category: ALARM + type: nagios + name: PORT_PROBLEM + template_id: port_alarm + - entity: + category: RESOURCE + type: neutron.port + template_id: port + relationships: + - relationship: + source: port_alarm + target: port + relationship_type: on + template_id : alarm_on_port +scenarios: + - scenario: + condition: not alarm_on_port + actions: + - action: + action_type: raise_alarm + properties: + alarm_name: simple_port_deduced_alarm + severity: WARNING + action_target: + target: port diff --git a/vitrage/tests/unit/datasources/static/test_static_driver.py b/vitrage/tests/unit/datasources/static/test_static_driver.py index e9dee61f3..3c506f3d2 100644 --- a/vitrage/tests/unit/datasources/static/test_static_driver.py +++ b/vitrage/tests/unit/datasources/static/test_static_driver.py @@ -17,7 +17,6 @@ from oslo_config import cfg from vitrage.common.constants import DatasourceAction from vitrage.common.constants import DatasourceOpts as DSOpts from vitrage.common.constants import GraphAction -from vitrage.common.constants import TopologyFields from vitrage.datasources.static import driver from vitrage.datasources.static import STATIC_DATASOURCE from vitrage.datasources.static import StaticFields @@ -99,8 +98,9 @@ class TestStaticDriver(base.BaseTest): self._validate_static_entity(entity) def _validate_static_entity(self, entity): - self.assertTrue(isinstance(entity[TopologyFields.METADATA], dict)) - for rel in entity[TopologyFields.RELATIONSHIPS]: + self.assertTrue(isinstance(entity[StaticFields.METADATA], + dict)) + for rel in entity[StaticFields.RELATIONSHIPS]: self._validate_static_rel(entity, rel) def _validate_static_rel(self, entity, rel): diff --git a/vitrage/tests/unit/datasources/static/test_static_transformer.py b/vitrage/tests/unit/datasources/static/test_static_transformer.py index 62b763298..c598fc879 100644 --- a/vitrage/tests/unit/datasources/static/test_static_transformer.py +++ b/vitrage/tests/unit/datasources/static/test_static_transformer.py @@ -19,12 +19,12 @@ from oslo_config import cfg from vitrage.common.constants import DatasourceOpts as DSOpts from vitrage.common.constants import DatasourceProperties as DSProps from vitrage.common.constants import EntityCategory -from vitrage.common.constants import TopologyFields from vitrage.common.constants import UpdateMethod from vitrage.common.constants import VertexProperties as VProps from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE from vitrage.datasources.nova.host.transformer import HostTransformer from vitrage.datasources.static import STATIC_DATASOURCE +from vitrage.datasources.static import StaticFields from vitrage.datasources.static.transformer import StaticTransformer from vitrage.tests import base from vitrage.tests.mocks import mock_driver @@ -109,7 +109,7 @@ class TestStaticTransformer(base.BaseTest): self.assertEqual(vertex[VProps.SAMPLE_TIMESTAMP], event[DSProps.SAMPLE_DATE]) - for k, v in event.get(TopologyFields.METADATA, {}): + for k, v in event.get(StaticFields.METADATA, {}): self.assertEqual(vertex[k], v) def _validate_common_props(self, vertex, event): @@ -120,19 +120,22 @@ class TestStaticTransformer(base.BaseTest): def _validate_neighbors(self, neighbors, vertex_id, event): for i in range(len(neighbors)): - self._validate_neighbor(neighbors[i], - event[TopologyFields.RELATIONSHIPS][i], - vertex_id) + self._validate_neighbor( + neighbors[i], + event[StaticFields.RELATIONSHIPS][i], + vertex_id) def _validate_neighbor(self, neighbor, rel, vertex_id): vertex = neighbor.vertex - self._validate_neighbor_vertex_props(vertex, - rel[TopologyFields.TARGET]) + self._validate_neighbor_vertex_props( + vertex, + rel[StaticFields.TARGET]) edge = neighbor.edge self.assertEqual(edge.source_id, vertex_id) self.assertEqual(edge.target_id, neighbor.vertex.vertex_id) - self.assertEqual(edge.label, rel[TopologyFields.RELATIONSHIP_TYPE]) + self.assertEqual(edge.label, + rel[StaticFields.RELATIONSHIP_TYPE]) def _validate_neighbor_vertex_props(self, vertex, event): self._validate_common_props(vertex, event) diff --git a/vitrage/tests/unit/entity_graph/base.py b/vitrage/tests/unit/entity_graph/base.py index 5ae582daa..8e29a6951 100644 --- a/vitrage/tests/unit/entity_graph/base.py +++ b/vitrage/tests/unit/entity_graph/base.py @@ -18,6 +18,8 @@ from vitrage.common.constants import DatasourceAction from vitrage.common.constants import DatasourceProperties as DSProps from vitrage.common.constants import EntityCategory from vitrage.datasources.nagios import NAGIOS_DATASOURCE +from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE +from vitrage.datasources.neutron.port import NEUTRON_PORT_DATASOURCE from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE from vitrage.datasources.nova.zone import NOVA_ZONE_DATASOURCE @@ -42,7 +44,9 @@ class TestEntityGraphUnitBase(base.BaseTest): default=[NAGIOS_DATASOURCE, NOVA_HOST_DATASOURCE, NOVA_INSTANCE_DATASOURCE, - NOVA_ZONE_DATASOURCE], + NOVA_ZONE_DATASOURCE, + NEUTRON_NETWORK_DATASOURCE, + NEUTRON_PORT_DATASOURCE], help='Names of supported data sources'), cfg.ListOpt('path', diff --git a/vitrage/tests/unit/graph/base.py b/vitrage/tests/unit/graph/base.py index d348b9cfe..ce0e25147 100644 --- a/vitrage/tests/unit/graph/base.py +++ b/vitrage/tests/unit/graph/base.py @@ -38,10 +38,11 @@ from vitrage.tests import base LOG = logging.getLogger(__name__) +# number of vms and alarms on vms needs 7 or more ENTITY_GRAPH_HOSTS_PER_CLUSTER = 8 ENTITY_GRAPH_VMS_PER_HOST = 8 ENTITY_GRAPH_ALARMS_PER_HOST = 8 -ENTITY_GRAPH_TESTS_PER_HOST = 20 +ENTITY_GRAPH_TESTS_PER_HOST = 8 ENTITY_GRAPH_ALARMS_PER_VM = 8 RESOURCE = EntityCategory.RESOURCE diff --git a/vitrage/tests/unit/graph/test_graph_algo.py b/vitrage/tests/unit/graph/test_graph_algo.py index b0d4360b1..6a2511304 100644 --- a/vitrage/tests/unit/graph/test_graph_algo.py +++ b/vitrage/tests/unit/graph/test_graph_algo.py @@ -19,11 +19,15 @@ test_vitrage graph algorithms Tests for `vitrage` graph driver algorithms """ - from vitrage.common.constants import EdgeLabel from vitrage.common.constants import EdgeProperties as EProps from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources.heat.stack import HEAT_STACK_DATASOURCE +from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE from vitrage.graph.algo_driver.algorithm import Mapping +from vitrage.graph.algo_driver.sub_graph_matching import \ + NEG_CONDITION +from vitrage.graph.algo_driver.sub_graph_matching import subgraph_matching from vitrage.graph.driver.elements import Edge from vitrage.graph.driver.graph import Direction from vitrage.tests.unit.graph.base import * # noqa @@ -230,7 +234,7 @@ class GraphAlgorithmTest(GraphTestBase): ALARM_ON_HOST + str(self.host_alarm_id - 1)) # Create a template for template matching - t = NXGraph('template_graph') + template_graph = NXGraph('template_graph') t_v_host_alarm = graph_utils.create_vertex( vitrage_id='1', entity_category=ALARM, entity_type=ALARM_ON_HOST) t_v_alarm_fail = graph_utils.create_vertex( @@ -276,182 +280,213 @@ class GraphAlgorithmTest(GraphTestBase): t_v_switch, t_v_switch, t_v_node]: del(v[VProps.VITRAGE_ID]) - t.add_vertex(t_v_alarm_fail) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_host_alarm, host_alarm, True)], validate=True) + template_graph.add_vertex(t_v_alarm_fail) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_host_alarm, + host_alarm, + is_vertex=True), + validate=True) self.assertEqual( 0, len(mappings), 'Template - Single vertex alarm not in graph ' - 'Template_root is a specific host alarm ' + str(mappings)) - t.remove_vertex(t_v_alarm_fail) + 'Template_root is a specific host alarm ') + template_graph.remove_vertex(t_v_alarm_fail) - t.add_vertex(t_v_host_alarm) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_host_alarm, host_alarm, is_vertex=True)]) + template_graph.add_vertex(t_v_host_alarm) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_host_alarm, + host_alarm, + is_vertex=True)) self.assertEqual( 1, len(mappings), 'Template - Single vertex (host alarm) ' - 'Template_root is a specific host alarm ' + str(mappings)) + 'Template_root is a specific host alarm ') - t.add_vertex(t_v_host) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_host_alarm, host_alarm, is_vertex=True)]) + template_graph.add_vertex(t_v_host) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_host_alarm, + host_alarm, + is_vertex=True)) self.assertEqual( 0, len(mappings), 'Template - Two disconnected vertices (host alarm , host)' - 'Template_root is a specific host alarm ' + str(mappings)) + 'Template_root is a specific host alarm ') - t.add_edge(e_alarm_on_host) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_host_alarm, host_alarm, is_vertex=True)]) + template_graph.add_edge(e_alarm_on_host) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_host_alarm, + host_alarm, + is_vertex=True)) self.assertEqual( 1, len(mappings), 'Template - Two connected vertices (host alarm -ON-> host)' - ' template_root is a specific host alarm ' + str(mappings)) + ' template_root is a specific host alarm ') host = mappings[0][t_v_host.vertex_id] host_vertex = self.entity_graph.get_vertex(host.vertex_id) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_host, host_vertex, is_vertex=True)]) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_host, + host_vertex, + is_vertex=True)) self.assertEqual( ENTITY_GRAPH_ALARMS_PER_HOST, len(mappings), 'Template - Two connected vertices (host alarm -ON-> host)' - ' template_root is a specific host ' + str(mappings)) + ' template_root is a specific host ') - t.add_vertex(t_v_vm) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_host_alarm, host_alarm, is_vertex=True)]) + template_graph.add_vertex(t_v_vm) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_host_alarm, + host_alarm, + is_vertex=True)) self.assertEqual( 0, len(mappings), 'Template - Two connected vertices and a disconnected vertex' '(host alarm -ON-> host, instance)' - ' template_root is a specific host alarm ' + str(mappings)) + ' template_root is a specific host alarm ') - t.add_vertex(t_v_vm_alarm) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)]) + template_graph.add_vertex(t_v_vm_alarm) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_vm_alarm, + vm_alarm, + is_vertex=True)) self.assertEqual( 0, len(mappings), 'Template - Two connected vertices and two disconnected vertices' '(host alarm -ON-> host, instance, instance alarm)' - ' template_root is a specific instance alarm ' + str(mappings)) + ' template_root is a specific instance alarm ') - t.add_edge(e_alarm_on_vm) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)]) + template_graph.add_edge(e_alarm_on_vm) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_vm_alarm, + vm_alarm, + is_vertex=True)) self.assertEqual( 0, len(mappings), 'Template - Two connected vertices and two more connected vertices' '(host alarm -ON-> host, instance alarm -ON-> instance)' - ' template_root is a specific instance alarm ' + str(mappings)) + ' template_root is a specific instance alarm ') - t.add_edge(e_host_contains_vm) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)]) + template_graph.add_edge(e_host_contains_vm) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_vm_alarm, + vm_alarm, + is_vertex=True)) self.assertEqual( ENTITY_GRAPH_ALARMS_PER_HOST, len(mappings), 'Template - Four connected vertices' '(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm)' - ' template_root is a specific instance alarm ' + str(mappings)) + ' template_root is a specific instance alarm ') - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_host_alarm, host_alarm, is_vertex=True)]) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_host_alarm, + host_alarm, + is_vertex=True)) self.assertEqual( ENTITY_GRAPH_VMS_PER_HOST * ENTITY_GRAPH_ALARMS_PER_VM, len(mappings), 'Template - Four connected vertices' '(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm)' - ' template_root is a specific host alarm ' + str(mappings)) + ' template_root is a specific host alarm ') - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_host, host_vertex, is_vertex=True)]) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_host, + host_vertex, + is_vertex=True)) self.assertEqual( ENTITY_GRAPH_VMS_PER_HOST * ENTITY_GRAPH_ALARMS_PER_VM * ENTITY_GRAPH_ALARMS_PER_HOST, len(mappings), 'Template - Four connected vertices' '(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm)' - ' template_root is a specific host ' + str(mappings)) + ' template_root is a specific host ') - t.add_vertex(t_v_switch) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)]) + template_graph.add_vertex(t_v_switch) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_vm_alarm, + vm_alarm, + is_vertex=True)) self.assertEqual( 0, len(mappings), 'Template - Four connected vertices and a disconnected vertex' '(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm' - ',switch) template_root is a instance alarm ' + str(mappings)) + ',switch) template_root is a instance alarm ') - t.add_edge(e_host_uses_switch) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)]) + template_graph.add_edge(e_host_uses_switch) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_vm_alarm, + vm_alarm, + is_vertex=True)) self.assertEqual( ENTITY_GRAPH_ALARMS_PER_HOST, len(mappings), 'Template - Five connected vertices' '(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm' ',host -USES-> switch) template_root ' - 'is a specific instance alarm ' + str(mappings)) + 'is a specific instance alarm ') - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_host, host_vertex, is_vertex=True)]) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_host, + host_vertex, + is_vertex=True)) self.assertEqual( ENTITY_GRAPH_VMS_PER_HOST * ENTITY_GRAPH_ALARMS_PER_VM * ENTITY_GRAPH_ALARMS_PER_HOST, len(mappings), 'Template - Five connected vertices' '(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm' - ',host -USES-> switch) template_root is a specific host ' + - str(mappings)) + ',host -USES-> switch) template_root is a specific host ') - mappings = ga.sub_graph_matching(t, [ + mappings = subgraph_matching(self.entity_graph, template_graph, [ Mapping(t_v_switch, v_switch, is_vertex=True), - Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)]) + Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)], + validate=False) self.assertEqual( ENTITY_GRAPH_ALARMS_PER_HOST, len(mappings), 'Template - Five connected vertices, two mappings given' '(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm' - ',host -USES-> switch) template_root is a specific host ' + - str(mappings)) + ',host -USES-> switch) 7template_root is a specific host ') - t.add_vertex(t_v_node_not_in_graph) - t.add_edge(e_host_to_node_not_in_graph) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)]) + template_graph.add_vertex(t_v_node_not_in_graph) + template_graph.add_edge(e_host_to_node_not_in_graph) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_vm_alarm, + vm_alarm, + is_vertex=True)) self.assertEqual( 0, len(mappings), 'Template - Five connected vertices and a invalid edge' '(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm' - ',host -USES-> switch) template_root is a instance alarm ' + - str(mappings)) - t.remove_vertex(t_v_node_not_in_graph) + ',host -USES-> switch) template_root is a instance alarm ') + template_graph.remove_vertex(t_v_node_not_in_graph) - t.remove_vertex(t_v_host_alarm) - t.add_vertex(t_v_node) - t.add_edge(e_node_contains_host) - t.add_edge(e_node_contains_switch) - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)]) + template_graph.remove_vertex(t_v_host_alarm) + template_graph.add_vertex(t_v_node) + template_graph.add_edge(e_node_contains_host) + template_graph.add_edge(e_node_contains_switch) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_vm_alarm, + vm_alarm, + is_vertex=True)) self.assertEqual( 1, len(mappings), 'Template - FIVE connected vertices' '(host -CONTAINS-> instance <-ON- instance alarm' ',node -CONTAINS-> host -USES-> switch, node-CONTAINS->switch)' - ' template_root is a instance alarm ' + str(mappings)) + ' template_root is a instance alarm ') - mappings = ga.sub_graph_matching(t, [ + mappings = subgraph_matching(self.entity_graph, template_graph, [ Mapping(e_node_contains_switch, e_node_to_switch, is_vertex=False), Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)]) self.assertEqual( @@ -460,10 +495,10 @@ class GraphAlgorithmTest(GraphTestBase): 'Template - FIVE connected vertices' '(host -CONTAINS-> instance <-ON- instance alarm' ',node -CONTAINS-> host -USES-> switch, node-CONTAINS->switch)' - ' 3 Known Mappings[switch, node, vm alarm] ' + str(mappings)) + ' 3 Known Mappings[switch, node, vm alarm] ') - t.add_edge(e_node_contains_switch_fail) - mappings = ga.sub_graph_matching(t, [ + template_graph.add_edge(e_node_contains_switch_fail) + mappings = subgraph_matching(self.entity_graph, template_graph, [ Mapping(t_v_node, v_node, is_vertex=True), Mapping(t_v_switch, v_switch, is_vertex=True)], validate=True) self.assertEqual( @@ -474,11 +509,11 @@ class GraphAlgorithmTest(GraphTestBase): ' we now have node-CONTAINS fail->switch AND node-CONTAINS->switch' ' ') - mappings = ga.sub_graph_matching(t, [ - Mapping(e_node_contains_switch, - e_node_to_switch, is_vertex=False)], - validate=True - ) + mappings = ga.sub_graph_matching(template_graph, + Mapping(e_node_contains_switch, + e_node_to_switch, + is_vertex=False), + validate=True) self.assertEqual( 0, len(mappings), @@ -487,8 +522,8 @@ class GraphAlgorithmTest(GraphTestBase): ' we now have node-CONTAINS fail->switch AND node-CONTAINS->switch' ' ') - t.remove_edge(e_node_contains_switch) - mappings = ga.sub_graph_matching(t, [ + template_graph.remove_edge(e_node_contains_switch) + mappings = subgraph_matching(self.entity_graph, template_graph, [ Mapping(t_v_node, v_node, is_vertex=True), Mapping(t_v_switch, v_switch, is_vertex=True)]) self.assertEqual( @@ -501,8 +536,10 @@ class GraphAlgorithmTest(GraphTestBase): 'fail->switch)' ' ') - mappings = ga.sub_graph_matching(t, [ - Mapping(t_v_vm_alarm, vm_alarm, is_vertex=True)]) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_vm_alarm, + vm_alarm, + is_vertex=True)) self.assertEqual( 0, len(mappings), @@ -511,3 +548,837 @@ class GraphAlgorithmTest(GraphTestBase): ',node -CONTAINS-> host -USES-> switch, node-CONTAINS ' 'fail->switch)' ' template_root is a instance alarm') + + def test_template_matching_with_not_operator_of_complicated_subgraph(self): + """Test the template matching algorithm with 'not' operator + + Using the entity graph (created above) as a big graph we search + for a sub graph matches that has simple 'not' operator in the template + """ + ga = self.entity_graph.algo + + # Get ids of some of the elements in the entity graph: + host = self.entity_graph.get_vertex( + NOVA_HOST_DATASOURCE + str(ENTITY_GRAPH_HOSTS_PER_CLUSTER - 1)) + + # Create a template for template matching + template_graph = NXGraph('template_graph') + t_v_alarm_fail = graph_utils.create_vertex( + vitrage_id='1', entity_category=ALARM, entity_type='fail') + t_v_host = graph_utils.create_vertex( + vitrage_id='2', + entity_category=RESOURCE, + entity_type=NOVA_HOST_DATASOURCE) + t_v_vm = graph_utils.create_vertex( + vitrage_id='3', + entity_category=RESOURCE, + entity_type=NOVA_INSTANCE_DATASOURCE) + t_v_vm_alarm = graph_utils.create_vertex( + vitrage_id='4', entity_category=ALARM, entity_type=ALARM_ON_VM) + + e_host_contains_vm = graph_utils.create_edge( + t_v_host.vertex_id, t_v_vm.vertex_id, ELabel.CONTAINS) + e_alarm_not_on_vm = graph_utils.create_edge( + t_v_vm_alarm.vertex_id, t_v_vm.vertex_id, ELabel.ON) + e_alarm_not_on_vm[NEG_CONDITION] = True + e_alarm_not_on_vm[EProps.IS_DELETED] = True + e_alarm_not_on_host = graph_utils.create_edge( + t_v_alarm_fail.vertex_id, t_v_host.vertex_id, ELabel.ON) + e_alarm_not_on_host[NEG_CONDITION] = True + e_alarm_not_on_host[EProps.IS_DELETED] = True + + for v in [t_v_alarm_fail, t_v_host, t_v_vm, t_v_vm_alarm]: + del(v[VProps.VITRAGE_ID]) + + # add host vertex to subgraph + template_graph.add_vertex(t_v_host) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_host, host, True), + validate=True) + self.assertEqual( + 1, + len(mappings), + 'Template - Single vertex alarm not in graph ' + 'Template_root is a specific host ' + str(mappings)) + + # add vm vertex to subgraph + template_graph.add_vertex(t_v_vm) + template_graph.add_edge(e_host_contains_vm) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_host, host, True)) + self.assertEqual( + ENTITY_GRAPH_VMS_PER_HOST, len(mappings), + 'Template - Two connected vertices (host -> vm)' + ' template_root is a specific host ' + str(mappings)) + + # add not alarm to subgraph + template_graph.add_vertex(t_v_vm_alarm) + template_graph.add_edge(e_alarm_not_on_vm) + mappings = ga.sub_graph_matching(template_graph, + Mapping(t_v_host, host, True)) + self.assertEqual( + 0, + len(mappings), + 'Template - Three connected vertices (host -> vm <- NOT alarm)' + ' template_root is a specific host ' + str(mappings)) + + # create temporary entity graph + temp_entity_graph = self.entity_graph.copy() + temp_ga = temp_entity_graph.algo + vms = temp_entity_graph.neighbors( + host.vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.RESOURCE, + VProps.TYPE: NOVA_INSTANCE_DATASOURCE}) + + ################################################################### + # Use case 1: remove alarms of specific vm + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[0].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + for alarm in alarms: + temp_entity_graph.remove_vertex(alarm) + + mappings = temp_ga.sub_graph_matching(template_graph, + Mapping(t_v_host, host, True)) + self.assertEqual( + 1, + len(mappings), + 'Template - Three connected vertices (host -> vm <- NOT alarm)' + 'Template_root is a specific host ' + str(mappings)) + + # add another not alarm to subgraph + template_graph.add_vertex(t_v_alarm_fail) + template_graph.add_edge(e_alarm_not_on_host) + mappings = temp_ga.sub_graph_matching(template_graph, + Mapping(t_v_host, host, True)) + self.assertEqual( + 1, + len(mappings), + 'Template - Four connected vertices ' + '(NOT alarm -> host -> vm <- NOT alarm)' + ' template_root is a specific host alarm ' + str(mappings)) + + ################################################################### + # Use case 2: mark alarms and their edges as deleted + ################################################################### + vms = temp_entity_graph.neighbors( + host.vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.RESOURCE, + VProps.TYPE: NOVA_INSTANCE_DATASOURCE}) + alarms = temp_entity_graph.neighbors( + vms[1].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + for alarm in alarms: + alarm[VProps.IS_DELETED] = True + temp_entity_graph.update_vertex(alarm) + edges = temp_entity_graph.get_edges(alarm.vertex_id) + for edge in edges: + edge[EProps.IS_DELETED] = True + temp_entity_graph.update_edge(edge) + + mappings = temp_ga.sub_graph_matching(template_graph, + Mapping(t_v_host, host, True)) + self.assertEqual( + 2, + len(mappings), + 'Template - Three connected vertices (host -> vm <- NOT alarm)' + 'Template_root is a specific host ' + str(mappings)) + + ################################################################### + # Use case 3: mark alarm edges as deleted with event on the host + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[2].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + for alarm in alarms: + edges = temp_entity_graph.get_edges(alarm.vertex_id) + for edge in edges: + edge[EProps.IS_DELETED] = True + temp_entity_graph.update_edge(edge) + + mappings = temp_ga.sub_graph_matching(template_graph, + Mapping(t_v_host, host, True)) + self.assertEqual( + 3, + len(mappings), + 'Template - Three connected vertices (host -> vm <- NOT alarm)' + 'Template_root is a specific host ' + str(mappings)) + + ################################################################### + # Use case 4: event arrived on deleted alarm vertex on vm, that + # has other alarms on it + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[3].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + deleted_vertex = alarms[0] + deleted_vertex[VProps.IS_DELETED] = True + temp_entity_graph.update_vertex(deleted_vertex) + mappings = temp_ga.sub_graph_matching(template_graph, + Mapping(t_v_vm_alarm, + deleted_vertex, + True)) + self.assertEqual( + 0, + len(mappings), + 'Template - Four connected vertices ' + '(NOT alarm -> host -> vm <- NOT alarm)' + ' template_root is a specific host ' + str(mappings)) + + ################################################################### + # Use case 5: event arrived on deleted alarm edge on vm, that has + # other alarms on it + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[4].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + edges = list(temp_entity_graph.get_edges(alarms[0].vertex_id)) + deleted_edge = edges[0] + deleted_edge[EProps.IS_DELETED] = True + temp_entity_graph.update_edge(deleted_edge) + mappings = temp_ga.sub_graph_matching(template_graph, + Mapping(e_alarm_not_on_vm, + deleted_edge, + False)) + self.assertEqual( + 0, + len(mappings), + 'Template - Four connected vertices ' + '(NOT alarm -> host -> vm <- NOT alarm)' + ' template_root is a specific host ' + str(mappings)) + + ################################################################### + # Use case 6: event arrived on deleted alarm edge on vm, that has + # other is_deleted alarms on it + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[5].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + for alarm in alarms: + edges = temp_entity_graph.get_edges(alarm.vertex_id) + for edge in edges: + edge[EProps.IS_DELETED] = True + temp_entity_graph.update_edge(edge) + deleted_edge = \ + list(temp_entity_graph.get_edges(alarms[0].vertex_id))[0] + mappings = temp_ga.sub_graph_matching(template_graph, + Mapping(e_alarm_not_on_vm, + deleted_edge, + False)) + self.assertEqual( + 1, + len(mappings), + 'Template - Four connected vertices ' + '(NOT alarm -> host -> vm <- NOT alarm)' + ' template_root is a specific host ' + str(mappings)) + + def test_template_matching_with_not_operator_of_simple_subgraph(self): + """Test the template matching algorithm with 'not' operator + + Using the entity graph (created above) as a big graph we search + for a sub graph matches that has simple 'not' operator in the template + """ + # Get ids of some of the elements in the entity graph: + graph_host = self.entity_graph.get_vertex( + NOVA_HOST_DATASOURCE + str(ENTITY_GRAPH_HOSTS_PER_CLUSTER - 1)) + + # Create a template for template matching + template_graph = NXGraph('template_graph') + t_v_vm = graph_utils.create_vertex( + vitrage_id='1', + entity_category=RESOURCE, + entity_type=NOVA_INSTANCE_DATASOURCE) + t_v_vm_alarm = graph_utils.create_vertex( + vitrage_id='2', entity_category=ALARM, entity_type=ALARM_ON_VM) + + e_alarm_not_on_vm = graph_utils.create_edge( + t_v_vm_alarm.vertex_id, t_v_vm.vertex_id, ELabel.ON) + e_alarm_not_on_vm[NEG_CONDITION] = True + e_alarm_not_on_vm[EProps.IS_DELETED] = True + + for v in [t_v_vm, t_v_vm_alarm]: + del(v[VProps.VITRAGE_ID]) + + # add instance vertex to subgraph + template_graph.add_vertex(t_v_vm) + + # add not alarm on vm vertex to subgraph + template_graph.add_vertex(t_v_vm_alarm) + template_graph.add_edge(e_alarm_not_on_vm) + + # create copy of the entity graph + temp_entity_graph = self.entity_graph.copy() + temp_ga = temp_entity_graph.algo + vms = temp_entity_graph.neighbors( + graph_host.vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.RESOURCE, + VProps.TYPE: NOVA_INSTANCE_DATASOURCE}) + + ################################################################### + # Use case 1: find subgraphs (when edges are deleted) with event on + # the edge + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[0].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + for alarm in alarms: + edges = temp_entity_graph.get_edges(alarm.vertex_id) + for edge in edges: + edge[EProps.IS_DELETED] = True + temp_entity_graph.update_edge(edge) + + graph_alarm_edge = \ + list(temp_entity_graph.get_edges(alarms[0].vertex_id))[0] + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + self.assertEqual( + 1, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # find subgraphs (when edges are deleted) with event on the alarm + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(t_v_vm_alarm, alarms[0], True)) + self.assertEqual( + 1, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + ################################################################### + # Use case 2: find subgraphs (when vertices are deleted) with event + # on the edge + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[1].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + for alarm in alarms: + alarm[VProps.IS_DELETED] = True + temp_entity_graph.update_vertex(alarm) + + graph_alarm_edge = \ + list(temp_entity_graph.get_edges(alarms[0].vertex_id))[0] + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + self.assertEqual( + 1, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # find subgraphs (when vertices are deleted) with event on the alarm + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(t_v_vm_alarm, alarms[0], True)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + ################################################################### + # Use case 3: find subgraphs (when vertices and edges are deleted) + # with event on the edge + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[2].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + for alarm in alarms: + alarm[VProps.IS_DELETED] = True + temp_entity_graph.update_vertex(alarm) + edges = temp_entity_graph.get_edges(alarm.vertex_id) + for edge in edges: + edge[EProps.IS_DELETED] = True + temp_entity_graph.update_edge(edge) + + graph_alarm_edge = \ + list(temp_entity_graph.get_edges(alarms[0].vertex_id))[0] + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + + self.assertEqual( + 1, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # find subgraphs (when vertices and edges are deleted) with event + # on the alarm + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(t_v_vm_alarm, alarms[0], True)) + + self.assertEqual( + 1, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + ################################################################### + # Use case 4: find subgraphs (when one alarm of many is deleted) + # with event on the edge + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[3].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + graph_alarm = alarms[0] + graph_alarm[VProps.IS_DELETED] = True + + graph_alarm_edge = \ + list(temp_entity_graph.get_edges(alarms[0].vertex_id))[0] + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # find subgraphs (when one alarm of many is deleted) with event + # on the alarm + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(t_v_vm_alarm, alarms[0], True)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + ################################################################### + # Use case 5: find subgraphs (when one edge of alarm of many is + # deleted) with event on the edge + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[4].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + + graph_alarm_edge = \ + list(temp_entity_graph.get_edges(alarms[0].vertex_id))[0] + graph_alarm_edge[EProps.IS_DELETED] = True + temp_entity_graph.update_edge(graph_alarm_edge) + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + ################################################################### + # Use case 6: find subgraphs (when one alarm its edge are deleted + # from many alarms) with event on the edge + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[5].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + + alarms[0][VProps.IS_DELETED] = True + temp_entity_graph.update_vertex(alarms[0]) + graph_alarm_edge = \ + list(temp_entity_graph.get_edges(alarms[0].vertex_id))[0] + graph_alarm_edge[EProps.IS_DELETED] = True + temp_entity_graph.update_edge(graph_alarm_edge) + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + def test_template_matching_with_not_operator_of_problematic_subgraph(self): + """Test the template matching algorithm with 'not' operator + + Checking the following use case: + network -> vm <--- alarm -> stack -> network + """ + + # Get ids of some of the elements in the entity graph: + graph_host = self.entity_graph.get_vertex( + NOVA_HOST_DATASOURCE + str(ENTITY_GRAPH_HOSTS_PER_CLUSTER - 1)) + + # Create a template for template matching + template_graph = NXGraph('template_graph') + t_v_network = graph_utils.create_vertex( + vitrage_id='1', + entity_category=RESOURCE, + entity_type=NEUTRON_NETWORK_DATASOURCE) + t_v_vm = graph_utils.create_vertex( + vitrage_id='2', + entity_category=RESOURCE, + entity_type=NOVA_INSTANCE_DATASOURCE) + t_v_alarm = graph_utils.create_vertex( + vitrage_id='3', entity_category=ALARM, entity_type=ALARM_ON_VM) + t_v_stack = graph_utils.create_vertex( + vitrage_id='4', + entity_category=RESOURCE, + entity_type=HEAT_STACK_DATASOURCE) + + e_network_connect_vm = graph_utils.create_edge( + t_v_network.vertex_id, t_v_vm.vertex_id, ELabel.CONNECT) + e_alarm_not_on_vm = graph_utils.create_edge( + t_v_alarm.vertex_id, t_v_vm.vertex_id, ELabel.ON) + e_alarm_not_on_vm[NEG_CONDITION] = True + e_alarm_not_on_vm[EProps.IS_DELETED] = True + e_alarm_on_stack = graph_utils.create_edge( + t_v_alarm.vertex_id, t_v_stack.vertex_id, ELabel.ON) + e_stack_connect_network = graph_utils.create_edge( + t_v_network.vertex_id, t_v_stack.vertex_id, ELabel.CONNECT) + + for v in [t_v_vm, t_v_alarm, t_v_network, t_v_stack]: + del(v[VProps.VITRAGE_ID]) + + # add network vertex to subgraph + template_graph.add_vertex(t_v_network) + + # add vm vertex and connect it to host + template_graph.add_vertex(t_v_vm) + template_graph.add_edge(e_network_connect_vm) + + # add not alarm and connect it to vm + template_graph.add_vertex(t_v_alarm) + template_graph.add_edge(e_alarm_not_on_vm) + + # add stack vertex and connect it to alarm + template_graph.add_vertex(t_v_stack) + template_graph.add_edge(e_alarm_on_stack) + + # connect stack to network + template_graph.add_edge(e_stack_connect_network) + + # create copy of the entity graph + temp_entity_graph = self.entity_graph.copy() + temp_ga = temp_entity_graph.algo + vms = temp_entity_graph.neighbors( + graph_host.vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.RESOURCE, + VProps.TYPE: NOVA_INSTANCE_DATASOURCE}) + + ################################################################### + # Use case 1: alarm connected to vm + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[0].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + + # Action + for alarm in alarms[1:len(alarms)]: + alarm[VProps.IS_DELETED] = True + temp_entity_graph.update_vertex(alarm) + edges = temp_entity_graph.get_edges(alarm.vertex_id) + for edge in edges: + edge[EProps.IS_DELETED] = True + temp_entity_graph.update_edge(edge) + + # build problematic subgraph in entity graph + specific_alarm = alarms[0] + self._build_problematic_subgraph_in_entity_graph(specific_alarm, + vms[0], + temp_entity_graph, + 0) + + # trigger on edge between vm and alarm + graph_alarm_edges = \ + temp_entity_graph.get_edges(specific_alarm.vertex_id) + for edge in graph_alarm_edges: + if 'instance' in edge.target_id: + graph_alarm_edge = edge + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # trigger on alarm + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(t_v_alarm, specific_alarm, True)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + ################################################################### + # Use case 2: alarm not connected to vm (with edge is_deleted=True) + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[1].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + + # Action + for alarm in alarms: + edges = temp_entity_graph.get_edges(alarm.vertex_id) + for edge in edges: + edge[EProps.IS_DELETED] = True + temp_entity_graph.update_edge(edge) + + # build problematic subgraph in entity graph + specific_alarm = alarms[0] + self._build_problematic_subgraph_in_entity_graph(specific_alarm, + vms[1], + temp_entity_graph, + 1) + + # trigger on edge between vm and alarm + graph_alarm_edges = \ + temp_entity_graph.get_edges(specific_alarm.vertex_id) + for edge in graph_alarm_edges: + if 'instance' in edge.target_id: + graph_alarm_edge = edge + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + self.assertEqual( + 1, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # trigger on alarm + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(t_v_alarm, specific_alarm, True)) + self.assertEqual( + 1, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + ################################################################### + # Use case 3: alarm not connected to vm (without any edge) + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[2].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + + # Action + for alarm in alarms: + edges = temp_entity_graph.get_edges(alarm.vertex_id) + for edge in edges: + temp_entity_graph.remove_edge(edge) + + # build problematic subgraph in entity graph + specific_alarm = alarms[0] + self._build_problematic_subgraph_in_entity_graph(specific_alarm, + vms[2], + temp_entity_graph, + 2) + + # trigger on edge between vm and alarm + graph_alarm_edges = \ + temp_entity_graph.get_edges(specific_alarm.vertex_id) + for edge in graph_alarm_edges: + if 'instance' in edge.target_id: + graph_alarm_edge = edge + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + self.assertEqual( + 1, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # trigger on alarm + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(t_v_alarm, specific_alarm, True)) + self.assertEqual( + 1, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + ################################################################### + # Use case 4: alarm not connected to vm (with edge is_deleted=True) + # and other connected alarms exist + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[3].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + + # Action + edges = [e for e in temp_entity_graph.get_edges(alarms[0].vertex_id)] + edges[0][EProps.IS_DELETED] = True + temp_entity_graph.update_edge(edge) + + # build problematic subgraph in entity graph + for alarm in alarms: + self._build_problematic_subgraph_in_entity_graph(alarm, + vms[3], + temp_entity_graph, + 3) + + # trigger on edge (that was deleted) between vm and alarm + specific_alarm = alarms[0] + graph_alarm_edges = \ + temp_entity_graph.get_edges(specific_alarm.vertex_id) + for edge in graph_alarm_edges: + if 'instance' in edge.target_id: + graph_alarm_edge = edge + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # trigger on edge (that wasn't deleted) between vm and alarm + specific_alarm = alarms[1] + graph_alarm_edges = \ + temp_entity_graph.get_edges(specific_alarm.vertex_id) + for edge in graph_alarm_edges: + if 'instance' in edge.target_id: + graph_alarm_edge = edge + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # trigger on alarm + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(t_v_alarm, specific_alarm, True)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # trigger on instance + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(t_v_vm, vms[3], True)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + ################################################################### + # Use case 5: alarm not connected to vm (without any edge) and + # other connected alarms exist + ################################################################### + alarms = temp_entity_graph.neighbors( + vms[4].vertex_id, + vertex_attr_filter={VProps.CATEGORY: EntityCategory.ALARM, + VProps.TYPE: ALARM_ON_VM}) + + # Action + edges = [e for e in temp_entity_graph.get_edges(alarms[0].vertex_id)] + temp_entity_graph.remove_edge(edges[0]) + + # build problematic subgraph in entity graph + for alarm in alarms: + self._build_problematic_subgraph_in_entity_graph(alarm, + vms[4], + temp_entity_graph, + 4) + + # trigger on edge (that was deleted) between vm and alarm + specific_alarm = alarms[0] + graph_alarm_edges = \ + temp_entity_graph.get_edges(specific_alarm.vertex_id) + for edge in graph_alarm_edges: + if 'instance' in edge.target_id: + graph_alarm_edge = edge + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # trigger on edge (that wasn't deleted) between vm and alarm + specific_alarm = alarms[1] + graph_alarm_edges = \ + temp_entity_graph.get_edges(specific_alarm.vertex_id) + for edge in graph_alarm_edges: + if 'instance' in edge.target_id: + graph_alarm_edge = edge + + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(e_alarm_not_on_vm, graph_alarm_edge, False)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # trigger on alarm + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(t_v_alarm, specific_alarm, True)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + # trigger on instance + mappings = temp_ga.sub_graph_matching( + template_graph, + Mapping(t_v_vm, vms[4], True)) + self.assertEqual( + 0, + len(mappings), + 'Template - Two not connected vertices (vm <- alarm)') + + @staticmethod + def _build_problematic_subgraph_in_entity_graph(alarm, + vm, + temp_entity_graph, + num): + stack_vertex = graph_utils.create_vertex( + vitrage_id='stack' + str(num), + entity_category=RESOURCE, + entity_type=HEAT_STACK_DATASOURCE) + temp_entity_graph.update_vertex(stack_vertex) + + alarm_stack_edge = graph_utils.create_edge( + alarm.vertex_id, stack_vertex.vertex_id, ELabel.ON) + temp_entity_graph.update_edge(alarm_stack_edge) + + network_vertex = graph_utils.create_vertex( + vitrage_id='network' + str(num), + entity_category=RESOURCE, + entity_type=NEUTRON_NETWORK_DATASOURCE) + temp_entity_graph.update_vertex(network_vertex) + + network_stack_edge = graph_utils.create_edge( + network_vertex.vertex_id, stack_vertex.vertex_id, ELabel.CONNECT) + temp_entity_graph.update_edge(network_stack_edge) + + network_vm_edge = graph_utils.create_edge( + network_vertex.vertex_id, vm.vertex_id, ELabel.CONNECT) + temp_entity_graph.update_edge(network_vm_edge)