Merge "Add sub graph matching to graph algorithm driver"

This commit is contained in:
Jenkins 2016-02-21 09:20:11 +00:00 committed by Gerrit Code Review
commit 4585c1f782
7 changed files with 503 additions and 13 deletions

View File

@ -13,9 +13,10 @@
# under the License.
import abc
from collections import namedtuple
import six
from driver import Graph # noqa
Mapping = namedtuple('Mapping', ['sub_graph_v_id', 'graph_v_id'])
@six.add_metaclass(abc.ABCMeta)
@ -25,7 +26,7 @@ class GraphAlgorithm(object):
"""Create a new GraphAlgorithm
:param graph: graph instance
:type graph: Graph
:type graph: driver.Graph
"""
self.graph = graph
@ -36,6 +37,20 @@ class GraphAlgorithm(object):
BFS traversal over the graph starting from root, each vertex is
checked according to the query. A matching vertex will be added to the
resulting sub graph and traversal will continue to its neighbors
:rtype: Graph
:rtype: driver.Graph
"""
pass
@abc.abstractmethod
def sub_graph_matching(self, sub_graph, known_mappings):
"""Search for occurrences of of a template graph in the graph
In sub-graph matching algorithms complexity is high in the general case
Here it is considerably mitigated as we have an anchor in the graph.
TODO(ihefetz) document this
:type known_mappings: list
:type sub_graph: driver.Graph
:rtype: list of dict
"""
pass

View File

@ -118,6 +118,9 @@ class Vertex(object):
def get(self, k, d=None):
return self.properties.get(k, d)
def items(self):
return self.properties.items()
class Edge(object):
"""Class Edge represents a directional edge between two vertices
@ -227,6 +230,9 @@ class Edge(object):
"""
return self.source_id if self.target_id == v_id else self.target_id
def items(self):
return self.properties.items()
@six.add_metaclass(abc.ABCMeta)
class Graph(object):

View File

@ -17,6 +17,7 @@ from oslo_log import log as logging
from algorithm_driver import GraphAlgorithm
from query import create_predicate
from vitrage.graph import NXGraph
from vitrage.graph.sub_graph_matching import sub_graph_matching
LOG = logging.getLogger(__name__)
@ -27,7 +28,7 @@ class NXAlgorithm(GraphAlgorithm):
"""Create a new GraphAlgorithm
:param graph: graph instance
:type graph: NXGraph
:type graph: driver.Graph
"""
self.graph = graph
@ -64,3 +65,6 @@ class NXAlgorithm(GraphAlgorithm):
graph = NXGraph('graph')
graph._g = self.graph._g.subgraph(n_result)
return graph
def sub_graph_matching(self, sub_graph, known_matches):
return sub_graph_matching(self.graph, sub_graph, known_matches)

View File

@ -82,7 +82,7 @@ class NXGraph(Graph):
:rtype: Vertex
"""
properties = self._g.node.get(v_id, None)
if properties:
if properties is not None:
return vertex_copy(v_id, properties)
LOG.debug("get_vertex item not found. v_id=" + str(v_id))
return None
@ -95,7 +95,7 @@ class NXGraph(Graph):
", target_id=" + str(target_id) +
", label=" + str(label))
return None
if properties:
if properties is not None:
return edge_copy(source_id, target_id, label, properties)
return None

View File

@ -0,0 +1,187 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
from vitrage.common.exception import VitrageAlgorithmError
from vitrage.graph import check_filter
LOG = logging.getLogger(__name__)
MAPPED_V_ID = 'mapped_v_id'
NEIGHBORS_MAPPED = 'neighbors_mapped'
def get_edges_to_mapped_vertices(graph, vertex):
"""Get all edges (to/from) vertex where neighbor has a MAPPED_V_ID
:type graph: driver.Graph
:type vertex: driver.Vertex
:rtype: list of driver.Edge
"""
sub_graph_edges_to_mapped_vertices = []
for e in graph.get_edges(vertex.vertex_id):
t_neighbor = graph.get_vertex(e.other_vertex(vertex.vertex_id))
if not t_neighbor:
raise VitrageAlgorithmError('Cant get vertex for edge' + str(e))
if t_neighbor and t_neighbor.get(MAPPED_V_ID):
sub_graph_edges_to_mapped_vertices.append(e)
return sub_graph_edges_to_mapped_vertices
def graph_contains_sub_graph_edges(graph, sub_graph, sub_graph_edges):
"""Check if graph contains all the expected edges
For each (sub-graph) expected edge, check if a corresponding edge exists
in the graph with relevant properties check
:type graph: driver.Graph
:type sub_graph: driver.Graph
:type sub_graph_edges: list of driver.Edge
:rtype: bool
"""
for e in sub_graph_edges:
graph_v_id_source = sub_graph.get_vertex(e.source_id).get(MAPPED_V_ID)
graph_v_id_target = sub_graph.get_vertex(e.target_id).get(MAPPED_V_ID)
if not graph_v_id_source or not graph_v_id_target:
raise VitrageAlgorithmError('Cant get vertex for edge' + str(e))
found_graph_edge = graph.get_edge(graph_v_id_source,
graph_v_id_target,
e.label)
if not found_graph_edge or not check_filter(found_graph_edge, e):
return False
return True
def create_initial_sub_graph(graph, known_matches, sub_graph):
"""Create initial mapping graph from sub graph and known matches
copy the sub-graph to create the first candidate mapping graph.
In which known vertices mappings are added to vertices MAPPED_V_ID
"""
mapping = sub_graph.copy()
for known_match in known_matches:
sub_graph_vertex = sub_graph.get_vertex(known_match.sub_graph_v_id)
graph_vertex = graph.get_vertex(known_match.graph_v_id)
if check_filter(graph_vertex, sub_graph_vertex):
mv = sub_graph.get_vertex(sub_graph_vertex.vertex_id)
mv[MAPPED_V_ID] = known_match.graph_v_id
mapping.update_vertex(mv)
edges = get_edges_to_mapped_vertices(mapping, mv)
if not graph_contains_sub_graph_edges(graph, mapping, edges):
return None
else:
return None
return mapping
def sub_graph_matching(_graph_, sub_graph, known_matches):
"""Find all occurrences of sub_graph in the graph
In the following, a partial mapping is a copy of the sub-graph.
As we go, vertices of curr_mapping graph will be updated with new
fields used only for the traversal:
- MAPPED_V_ID:
The vertex_id of the corresponding vertex in the graph.
If it is not empty, than this vertex is already mapped
- NEIGHBORS_MAPPED:
True or None. When set True it means all the
neighbors of this vertex have already been mapped
Implementation Details:
----------------------
- Init Step:
copy the sub-graph to create the first candidate mapping graph. In which
known vertices mappings are added to vertices MAPPED_V_ID. So, we now
have a sub-graph copy where some of the vertices already have a mapping
Main loop steps:
- Steps 1:
Pop a partially mapped sub-graph from the queue.
If all its vertices have a MAPPED_V_ID, add it to final mappings
- Steps 2 & 3:
Find one template vertex that is not mapped but has a mapped neighbor
- Step 4: CHECK PROPERTIES
In the graph find candidate vertices that are linked to that neighbor
and match the template vertex properties
- Step 5: CHECK STRUCTURE
Filter candidate vertices according to edges
"""
final_sub_graphs = []
initial_sg = create_initial_sub_graph(_graph_, known_matches, sub_graph)
if not initial_sg:
LOG.warning('sub_graph_matching: Initial sub-graph creation failed')
LOG.warning('sub_graph_matching: Known matches: ' + str(known_matches))
return final_sub_graphs
_queue_ = [initial_sg]
while _queue_:
curr_sub_graph = _queue_.pop(0)
# STEP 1: STOPPING CONDITION
mapped_vertices = filter(
lambda v: v.get(MAPPED_V_ID),
curr_sub_graph.get_vertices())
if len(mapped_vertices) == sub_graph.num_vertices():
final_sub_graphs.append(curr_sub_graph)
continue
# STEP 2: CAN WE THROW THIS SUB-GRAPH?
vertices_with_unmapped_neighbors = filter(
lambda v: not v.get(NEIGHBORS_MAPPED),
mapped_vertices)
if not vertices_with_unmapped_neighbors:
continue
# STEP 3: FIND A SUB-GRAPH VERTEX TO MAP
v_with_unmapped_neighbors = vertices_with_unmapped_neighbors.pop(0)
unmapped_neighbors = filter(
lambda v: not v.get(MAPPED_V_ID),
curr_sub_graph.neighbors(v_with_unmapped_neighbors.vertex_id))
if not unmapped_neighbors:
# Mark vertex as NEIGHBORS_MAPPED=True
v_with_unmapped_neighbors[NEIGHBORS_MAPPED] = True
curr_sub_graph.update_vertex(v_with_unmapped_neighbors)
_queue_.append(curr_sub_graph)
continue
sub_graph_vertex_to_map = unmapped_neighbors.pop(0)
# STEP 4: PROPERTIES CHECK
graph_candidate_vertices = _graph_.neighbors(
v_id=v_with_unmapped_neighbors[MAPPED_V_ID],
vertex_attr_filter=sub_graph_vertex_to_map)
# STEP 5: STRUCTURE CHECK
edges = get_edges_to_mapped_vertices(curr_sub_graph,
sub_graph_vertex_to_map)
for graph_vertex in graph_candidate_vertices:
sub_graph_vertex_to_map[MAPPED_V_ID] = graph_vertex.vertex_id
curr_sub_graph.update_vertex(sub_graph_vertex_to_map)
if graph_contains_sub_graph_edges(_graph_, curr_sub_graph, edges):
_queue_.append(curr_sub_graph.copy())
# Last thing: Convert results to the expected format!
result = []
for mapping in final_sub_graphs:
# TODO(ihefetz) If needed, Here we can easily extract the edge
# matches from the mapping graph
a = {v.vertex_id: v[MAPPED_V_ID] for v in mapping.get_vertices()}
result.append(a)
return result

View File

@ -24,6 +24,8 @@ import time
from oslo_log import log as logging
from vitrage.common.constants import EdgeLabels as ELabel
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import EntityType
from vitrage.graph import create_graph
from vitrage.graph import utils as graph_utils
from vitrage.tests import base
@ -36,14 +38,14 @@ ENTITY_GRAPH_ALARMS_PER_HOST = 8
ENTITY_GRAPH_TESTS_PER_HOST = 20
ENTITY_GRAPH_ALARMS_PER_VM = 8
RESOURCE = 'RESOURCE'
ALARM = 'ALARM'
RESOURCE = EntityCategory.RESOURCE
ALARM = EntityCategory.ALARM
HOST = 'HOST'
INSTANCE = 'INSTANCE'
NODE = 'NODE'
HOST = EntityType.NOVA_HOST
INSTANCE = EntityType.NOVA_INSTANCE
NODE = EntityType.NODE
TEST = 'TEST'
SWITCH = 'SWITCH'
SWITCH = EntityType.SWITCH
ALARM_ON_VM = 'ALARM_ON_VM'
ALARM_ON_HOST = 'ALARM_ON_HOST'
TEST_ON_HOST = 'TEST_ON_HOST'

View File

@ -19,7 +19,7 @@ test_vitrage graph algorithms
Tests for `vitrage` graph driver algorithms
"""
from vitrage.common.constants import VertexProperties as VProps
from vitrage.graph import create_algorithm
from vitrage.graph import create_algorithm, Mapping # noqa
from vitrage.tests.unit.graph.base import * # noqa
@ -128,3 +128,279 @@ class GraphAlgorithmTest(GraphTestBase):
'num of BOTH edges Node (depth 3)')
self.assertEqual(1, subgraph.num_vertices(),
'num of BOTH vertices Node (depth 3)')
def test_template_matching(self):
"""Test the template matching algorithm
Using the entity graph (created above) as a big graph we search
for a sub graph match
"""
ga = create_algorithm(self.entity_graph)
# Get ids of some of the elements in the entity graph:
vm_alarm_id = self.entity_graph.get_vertex(
ALARM_ON_VM + str(self.vm_alarm_id - 1)).vertex_id
host_alarm_id = self.entity_graph.get_vertex(
ALARM_ON_HOST + str(self.host_alarm_id - 1)).vertex_id
# Create a template for template matching
t = create_graph('template_graph')
t_v_host_alarm = graph_utils.create_vertex(
vitrage_id='1', entity_category=ALARM, entity_type=ALARM_ON_HOST)
t_v_alarm_fail = graph_utils.create_vertex(
vitrage_id='1', entity_category=ALARM, entity_type='fail')
t_v_host = graph_utils.create_vertex(
vitrage_id='2', entity_category=RESOURCE, entity_type=HOST)
t_v_vm = graph_utils.create_vertex(
vitrage_id='3', entity_category=RESOURCE, entity_type=INSTANCE)
t_v_vm_alarm = graph_utils.create_vertex(
vitrage_id='4', entity_category=ALARM, entity_type=ALARM_ON_VM)
t_v_switch = graph_utils.create_vertex(
vitrage_id='5', entity_category=RESOURCE, entity_type=SWITCH)
t_v_node = graph_utils.create_vertex(
vitrage_id='6', entity_category=RESOURCE, entity_type=NODE)
t_v_node_not_in_graph = graph_utils.create_vertex(
vitrage_id='7', entity_category=RESOURCE,
entity_type=NODE + ' not in graph')
e_alarm_on_host = graph_utils.create_edge(
t_v_host_alarm.vertex_id, t_v_host.vertex_id, ELabel.ON)
e_host_contains_vm = graph_utils.create_edge(
t_v_host.vertex_id, t_v_vm.vertex_id, ELabel.CONTAINS)
e_alarm_on_vm = graph_utils.create_edge(
t_v_vm_alarm.vertex_id, t_v_vm.vertex_id, ELabel.ON)
e_host_uses_switch = graph_utils.create_edge(
t_v_host.vertex_id, t_v_switch.vertex_id, 'USES')
e_node_contains_host = graph_utils.create_edge(
t_v_node.vertex_id, t_v_host.vertex_id, ELabel.CONTAINS)
e_node_contains_switch = graph_utils.create_edge(
t_v_node.vertex_id, t_v_switch.vertex_id, ELabel.CONTAINS)
e_node_contains_switch_fail = graph_utils.create_edge(
t_v_node.vertex_id, t_v_switch.vertex_id, ELabel.CONTAINS + 'fail')
e_host_to_node_not_in_graph = graph_utils.create_edge(
t_v_node_not_in_graph.vertex_id, t_v_host.vertex_id, ELabel.ON)
for v in [t_v_host_alarm, t_v_host, t_v_vm, t_v_vm_alarm,
t_v_switch, t_v_switch, t_v_node]:
del(v[VProps.VITRAGE_ID])
t.add_vertex(t_v_alarm_fail)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host_alarm.vertex_id, host_alarm_id)])
self.assertEqual(
0,
len(mappings),
'Template - Single vertex alarm not in graph '
'Template_root is a specific host alarm ' + str(mappings))
t.remove_vertex(t_v_alarm_fail)
t.add_vertex(t_v_host_alarm)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host_alarm.vertex_id, host_alarm_id)])
self.assertEqual(
1,
len(mappings),
'Template - Single vertex (host alarm) '
'Template_root is a specific host alarm ' + str(mappings))
t.add_vertex(t_v_host)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host_alarm.vertex_id, host_alarm_id)])
self.assertEqual(
0,
len(mappings),
'Template - Two disconnected vertices (host alarm , host)'
'Template_root is a specific host alarm ' + str(mappings))
t.add_edge(e_alarm_on_host)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host_alarm.vertex_id, host_alarm_id)])
self.assertEqual(
1, len(mappings),
'Template - Two connected vertices (host alarm -ON-> host)'
' template_root is a specific host alarm ' + str(mappings))
host_id = mappings[0][t_v_host.vertex_id]
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host.vertex_id, host_id)])
self.assertEqual(
ENTITY_GRAPH_ALARMS_PER_HOST,
len(mappings),
'Template - Two connected vertices (host alarm -ON-> host)'
' template_root is a specific host ' + str(mappings))
t.add_vertex(t_v_vm)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host_alarm.vertex_id, host_alarm_id)])
self.assertEqual(
0,
len(mappings),
'Template - Two connected vertices and a disconnected vertex'
'(host alarm -ON-> host, instance)'
' template_root is a specific host alarm ' + str(mappings))
t.add_vertex(t_v_vm_alarm)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
self.assertEqual(
0,
len(mappings),
'Template - Two connected vertices and two disconnected vertices'
'(host alarm -ON-> host, instance, instance alarm)'
' template_root is a specific instance alarm ' + str(mappings))
t.add_edge(e_alarm_on_vm)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
self.assertEqual(
0,
len(mappings),
'Template - Two connected vertices and two more connected vertices'
'(host alarm -ON-> host, instance alarm -ON-> instance)'
' template_root is a specific instance alarm ' + str(mappings))
t.add_edge(e_host_contains_vm)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
self.assertEqual(
ENTITY_GRAPH_ALARMS_PER_HOST,
len(mappings),
'Template - Four connected vertices'
'(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm)'
' template_root is a specific instance alarm ' + str(mappings))
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host_alarm.vertex_id, host_alarm_id)])
self.assertEqual(
ENTITY_GRAPH_VMS_PER_HOST * ENTITY_GRAPH_ALARMS_PER_VM,
len(mappings),
'Template - Four connected vertices'
'(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm)'
' template_root is a specific host alarm ' + str(mappings))
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host.vertex_id, host_id)])
self.assertEqual(
ENTITY_GRAPH_VMS_PER_HOST * ENTITY_GRAPH_ALARMS_PER_VM
* ENTITY_GRAPH_ALARMS_PER_HOST,
len(mappings),
'Template - Four connected vertices'
'(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm)'
' template_root is a specific host ' + str(mappings))
t.add_vertex(t_v_switch)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
self.assertEqual(
0,
len(mappings),
'Template - Four connected vertices and a disconnected vertex'
'(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm'
',switch) template_root is a instance alarm ' + str(mappings))
t.add_edge(e_host_uses_switch)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
self.assertEqual(
ENTITY_GRAPH_ALARMS_PER_HOST,
len(mappings),
'Template - Five connected vertices'
'(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm'
',host -USES-> switch) template_root is a specific instance alarm '
+ str(mappings))
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_host.vertex_id, host_id)])
self.assertEqual(
ENTITY_GRAPH_VMS_PER_HOST * ENTITY_GRAPH_ALARMS_PER_VM
* ENTITY_GRAPH_ALARMS_PER_HOST,
len(mappings),
'Template - Five connected vertices'
'(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm'
',host -USES-> switch) template_root is a specific host '
+ str(mappings))
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_switch.vertex_id, v_switch.vertex_id),
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
self.assertEqual(
ENTITY_GRAPH_ALARMS_PER_HOST,
len(mappings),
'Template - Five connected vertices, two mappings given'
'(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm'
',host -USES-> switch) template_root is a specific host '
+ str(mappings))
t.add_vertex(t_v_node_not_in_graph)
t.add_edge(e_host_to_node_not_in_graph)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
self.assertEqual(
0,
len(mappings),
'Template - Five connected vertices and a invalid edge'
'(host alarm -ON-> host -CONTAINS-> instance <-ON- instance alarm'
',host -USES-> switch) template_root is a instance alarm '
+ str(mappings))
t.remove_vertex(t_v_node_not_in_graph)
t.remove_vertex(t_v_host_alarm)
t.add_vertex(t_v_node)
t.add_edge(e_node_contains_host)
t.add_edge(e_node_contains_switch)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
self.assertEqual(
1,
len(mappings),
'Template - FIVE connected vertices'
'(host -CONTAINS-> instance <-ON- instance alarm'
',node -CONTAINS-> host -USES-> switch, node-CONTAINS->switch)'
' template_root is a instance alarm ' + str(mappings))
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_node.vertex_id, v_node.vertex_id),
Mapping(t_v_switch.vertex_id, v_switch.vertex_id),
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
self.assertEqual(
1,
len(mappings),
'Template - FIVE connected vertices'
'(host -CONTAINS-> instance <-ON- instance alarm'
',node -CONTAINS-> host -USES-> switch, node-CONTAINS->switch)'
' 3 Known Mappings[switch, node, vm alarm] ' + str(mappings))
t.add_edge(e_node_contains_switch_fail)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_node.vertex_id, v_node.vertex_id),
Mapping(t_v_switch.vertex_id, v_switch.vertex_id)])
self.assertEqual(
0,
len(mappings),
'Template - FIVE connected vertices - 2 Known Mapping[node,switch]'
' Check that ALL edges between the 2 known mappings are checked'
' we now have node-CONTAINSfail->switch AND node-CONTAINS->switch'
' ')
t.remove_edge(e_node_contains_switch)
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_node.vertex_id, v_node.vertex_id),
Mapping(t_v_switch.vertex_id, v_switch.vertex_id)])
self.assertEqual(
0,
len(mappings),
'Template - FIVE connected vertices - 2 Known Mapping[node,switch]'
' But the edge between these 2 is not same as the graph '
'(host -CONTAINS-> instance <-ON- instance alarm'
',node -CONTAINS-> host -USES-> switch, node-CONTAINSfail->switch)'
' ')
mappings = ga.sub_graph_matching(t, [
Mapping(t_v_vm_alarm.vertex_id, vm_alarm_id)])
self.assertEqual(
0,
len(mappings),
'Template - FIVE connected vertices'
'(host -CONTAINS-> instance <-ON- instance alarm'
',node -CONTAINS-> host -USES-> switch, node-CONTAINSfail->switch)'
' template_root is a instance alarm')