openstack/vitrage
Code Issues Proposed changes

Merge "Code refactoring - EntityGraph class functionality moved to a processor utils"

Browse Source
This commit is contained in:
Jenkins 2017-01-25 09:15:41 +00:00 committed by Gerrit Code Review
commit 9d7457dddb
6 changed files with 105 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vitrage/cmd/graph.py
View File

@ -26,10 +26,10 @@ from vitrage.datasources import OPENSTACK_CLUSTER
from vitrage.datasources.transformer_base import CLUSTER_ID
from vitrage.entity_graph.consistency import service as consistency_svc
from vitrage.entity_graph.initialization_status import InitializationStatus
from vitrage.entity_graph.processor import entity_graph
from vitrage.entity_graph import service as entity_graph_svc
from vitrage.evaluator.scenario_evaluator import ScenarioEvaluator
from vitrage.evaluator.scenario_repository import ScenarioRepository
from vitrage.graph import create_graph
from vitrage import service
@ -67,7 +67,7 @@ def main():
def init(conf):
mp_queue = multiprocessing.Queue()
evaluator_q = queue.Queue()
e_graph = entity_graph.EntityGraph(
e_graph = create_graph(
'Entity Graph',
'%s:%s:%s' % (EntityCategory.RESOURCE, OPENSTACK_CLUSTER, CLUSTER_ID))
scenario_repo = ScenarioRepository(conf)

99
vitrage/entity_graph/processor/entity_graph.py
View File

@ -1,99 +0,0 @@
# Copyright 2015 - Alcatel-Lucent
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log
from vitrage.common.constants import EdgeProperties as EProps
from vitrage.common.constants import VertexProperties as VProps
from vitrage.graph import NXGraph
from vitrage.utils.datetime import utcnow
LOG = log.getLogger(__name__)
class EntityGraph(NXGraph):
def __init__(self, name, root_id=None):
super(EntityGraph, self).__init__(name, root_id)
def can_vertex_be_deleted(self, vertex):
"""Check if the vertex can be deleted
Vertex can be deleted if it's IS_PLACEHOLDER property is
True and if it has no neighbors that aren't marked deleted
"""
if not vertex[VProps.IS_PLACEHOLDER]:
return False
# check that vertex has no neighbors
neighbor_edges = self.get_edges(vertex.vertex_id)
return not any(True for neighbor_edge in neighbor_edges
if not self.is_edge_deleted(neighbor_edge))
def delete_placeholder_vertex(self, suspected_vertex):
"""Checks if it is a placeholder vertex, and if so deletes it """
if self.can_vertex_be_deleted(suspected_vertex):
LOG.debug("Delete placeholder vertex: %s", suspected_vertex)
self.remove_vertex(suspected_vertex)
@staticmethod
def is_vertex_deleted(vertex):
return vertex.get(VProps.IS_DELETED, False)
@staticmethod
def is_edge_deleted(edge):
return edge.get(EProps.IS_DELETED, False)
def mark_vertex_as_deleted(self, vertex):
"""Marks the vertex as is deleted, and updates deletion timestamp"""
vertex[VProps.IS_DELETED] = True
vertex[VProps.SAMPLE_TIMESTAMP] = str(utcnow())
self.update_vertex(vertex)
def mark_edge_as_deleted(self, edge):
"""Marks the edge as is deleted, and updates delete timestamp"""
edge[EProps.IS_DELETED] = True
edge[EProps.UPDATE_TIMESTAMP] = str(utcnow())
self.update_edge(edge)
def find_neighbor_types(self, neighbors):
"""Finds all the types (TYPE, SUB_TYPE) of the neighbors """
neighbor_types = set()
for (vertex, edge) in neighbors:
neighbor_types.add(self.get_vertex_category(vertex))
return neighbor_types
@staticmethod
def get_vertex_category(vertex):
category = vertex[VProps.CATEGORY]
type_ = vertex[VProps.TYPE]
return category, type_
@staticmethod
def can_update_vertex(graph_vertex, new_vertex):
return (not graph_vertex) or (not new_vertex[VProps.IS_PLACEHOLDER])
def update_entity_graph_vertex(self, graph_vertex, updated_vertex):
if updated_vertex[VProps.IS_PLACEHOLDER] and \
graph_vertex and not graph_vertex[VProps.IS_PLACEHOLDER]:
updated_vertex[VProps.IS_PLACEHOLDER] = False
updated_vertex[VProps.IS_DELETED] = graph_vertex[VProps.IS_DELETED]
self.update_vertex(updated_vertex)

32
vitrage/entity_graph/processor/processor.py
View File

@ -21,10 +21,10 @@ from vitrage.datasources.transformer_base import TransformerBase
from vitrage.entity_graph.mappings.datasource_info_mapper import \
DatasourceInfoMapper
from vitrage.entity_graph.processor import base as processor
from vitrage.entity_graph.processor import entity_graph
from vitrage.entity_graph.processor.notifier import GraphNotifier
from vitrage.entity_graph.processor import processor_utils as PUtils
from vitrage.entity_graph.transformer_manager import TransformerManager
from vitrage.graph import create_graph
from vitrage.graph import Direction
LOG = log.getLogger(__name__)
@ -39,8 +39,8 @@ class Processor(processor.ProcessorBase):
self.state_manager = DatasourceInfoMapper(self.conf)
self._initialize_events_actions()
self.initialization_status = initialization_status
self.entity_graph = entity_graph.EntityGraph("Entity Graph") if \
e_graph is None else e_graph
self.entity_graph = e_graph if e_graph is not None\
else create_graph("Entity Graph")
self._notifier = GraphNotifier(conf)
def process_event(self, event):
@ -96,8 +96,9 @@ class Processor(processor.ProcessorBase):
if (not graph_vertex) or \
PUtils.is_newer_vertex(graph_vertex, updated_vertex):
self.entity_graph.update_entity_graph_vertex(graph_vertex,
updated_vertex)
PUtils.update_entity_graph_vertex(self.entity_graph,
graph_vertex,
updated_vertex)
self._update_neighbors(updated_vertex, neighbors)
else:
LOG.warning("Update event arrived on invalid resource: %s",
@ -127,12 +128,12 @@ class Processor(processor.ProcessorBase):
deleted_vertex.vertex_id)
for edge in neighbor_edges:
self.entity_graph.mark_edge_as_deleted(edge)
PUtils.mark_deleted(self.entity_graph, edge)
for vertex in neighbor_vertices:
self.entity_graph.delete_placeholder_vertex(vertex)
PUtils.delete_placeholder_vertex(self.entity_graph, vertex)
self.entity_graph.mark_vertex_as_deleted(deleted_vertex)
PUtils.mark_deleted(self.entity_graph, deleted_vertex)
else:
LOG.warning("Delete event arrived on invalid resource: %s",
deleted_vertex)
@ -219,11 +220,12 @@ class Processor(processor.ProcessorBase):
for (vertex, edge) in neighbors:
graph_vertex = self.entity_graph.get_vertex(vertex.vertex_id)
if not graph_vertex or not PUtils.is_deleted(graph_vertex):
if self.entity_graph.can_update_vertex(graph_vertex, vertex):
if PUtils.can_update_vertex(graph_vertex, vertex):
LOG.debug("Updates vertex: %s", vertex)
self._calculate_aggregated_state(vertex, action)
self.entity_graph.update_entity_graph_vertex(graph_vertex,
vertex)
PUtils.update_entity_graph_vertex(self.entity_graph,
graph_vertex,
vertex)
if edge not in valid_edges:
LOG.debug("Updates edge: %s", edge)
@ -245,10 +247,10 @@ class Processor(processor.ProcessorBase):
# remove old edges and placeholder vertices if exist
for edge in obsolete_edges:
LOG.debug("Delete obsolete edge:\n%s", edge)
self.entity_graph.mark_edge_as_deleted(edge)
PUtils.mark_deleted(self.entity_graph, edge)
graph_ver = self.entity_graph.get_vertex(
edge.other_vertex(vertex.vertex_id))
self.entity_graph.delete_placeholder_vertex(graph_ver)
PUtils.delete_placeholder_vertex(self.entity_graph, graph_ver)
def _find_edges_status(self, vertex, neighbors):
"""Finds "vertex" valid and old connections
@ -262,7 +264,7 @@ class Processor(processor.ProcessorBase):
obsolete_edges = []
graph_neighbor_types = \
self.entity_graph.find_neighbor_types(neighbors)
PUtils.find_neighbor_types(neighbors)
for curr_edge in self.entity_graph.get_edges(
vertex.vertex_id,
@ -272,7 +274,7 @@ class Processor(processor.ProcessorBase):
neighbor_vertex = self.entity_graph.get_vertex(
curr_edge.other_vertex(vertex.vertex_id))
is_connection_type_exist = self.entity_graph.get_vertex_category(
is_connection_type_exist = PUtils.get_vertex_types(
neighbor_vertex) in graph_neighbor_types
if not is_connection_type_exist:

58
vitrage/entity_graph/processor/processor_utils.py
View File

@ -13,10 +13,17 @@
# under the License.
from dateutil import parser
from oslo_log import log
from vitrage.common.constants import EdgeProperties as EProps
from vitrage.common.constants import VertexProperties as VProps
from vitrage.graph import Edge
from vitrage.graph import Vertex
from vitrage.utils.datetime import utcnow
LOG = log.getLogger(__name__)
def is_newer_vertex(prev_vertex, new_vertex):
@ -37,3 +44,54 @@ def is_deleted(item):
return item and \
(isinstance(item, Vertex) and item.get(VProps.IS_DELETED, False)) or\
(isinstance(item, Edge) and item.get(EProps.IS_DELETED, False))
def mark_deleted(g, item):
if isinstance(item, Vertex):
item[VProps.IS_DELETED] = True
item[VProps.SAMPLE_TIMESTAMP] = str(utcnow())
g.update_vertex(item)
elif isinstance(item, Edge):
item[EProps.IS_DELETED] = True
item[EProps.UPDATE_TIMESTAMP] = str(utcnow())
g.update_edge(item)
def delete_placeholder_vertex(g, vertex):
"""Checks if it is a placeholder vertex, and if so deletes it """
if not vertex[VProps.IS_PLACEHOLDER]:
return
if not any(True for neighbor_edge in g.get_edges(vertex.vertex_id)
if not is_deleted(neighbor_edge)):
LOG.debug("Delete placeholder vertex: %s", vertex)
g.remove_vertex(vertex)
def find_neighbor_types(neighbors):
"""Finds all the types (TYPE, SUB_TYPE) of the neighbors """
neighbor_types = set()
for (vertex, edge) in neighbors:
neighbor_types.add(get_vertex_types(vertex))
return neighbor_types
def get_vertex_types(vertex):
category = vertex[VProps.CATEGORY]
type_ = vertex[VProps.TYPE]
return category, type_
def can_update_vertex(graph_vertex, new_vertex):
return (not graph_vertex) or (not new_vertex[VProps.IS_PLACEHOLDER])
def update_entity_graph_vertex(g, graph_vertex, updated_vertex):
if updated_vertex[VProps.IS_PLACEHOLDER] and \
graph_vertex and not graph_vertex[VProps.IS_PLACEHOLDER]:
updated_vertex[VProps.IS_PLACEHOLDER] = False
updated_vertex[VProps.IS_DELETED] = graph_vertex[VProps.IS_DELETED]
g.update_vertex(updated_vertex)

94
vitrage/tests/unit/entity_graph/processor/test_entity_graph.py
View File

@ -12,8 +12,8 @@
# License for the specific language governing permissions and limitations
# under the License.
from vitrage.common.constants import VertexProperties
from vitrage.entity_graph.processor import entity_graph as entity_g
from vitrage.entity_graph.processor import processor_utils as PUtils
from vitrage.graph import NXGraph
from vitrage.tests.unit.entity_graph.processor import base
@ -23,73 +23,31 @@ class TestEntityGraphManager(base.TestBaseProcessor):
def setUpClass(cls):
super(TestEntityGraphManager, cls).setUpClass()
def test_can_vertex_be_deleted(self):
entity_graph = entity_g.EntityGraph("Entity Graph")
# create vertex properties
instance_vertex = self._update_vertex_to_graph(entity_graph,
'RESOURCE', 'INSTANCE',
'123', False, True, {})
# check is placeholder vertex
is_placeholder_vertex = \
entity_graph.can_vertex_be_deleted(instance_vertex)
self.assertTrue(is_placeholder_vertex)
# add host vertex
host_vertex = self._update_vertex_to_graph(entity_graph, 'RESOURCE',
'HOST', '321',
False, True, {})
edge = self._update_edge_to_graph(entity_graph, host_vertex.vertex_id,
instance_vertex.vertex_id,
'contains')
# check is placeholder vertex
is_placeholder_vertex = \
entity_graph.can_vertex_be_deleted(instance_vertex)
self.assertFalse(is_placeholder_vertex)
# change host to is_deleted
entity_graph.mark_vertex_as_deleted(host_vertex)
entity_graph.mark_edge_as_deleted(edge)
# check is placeholder vertex
is_placeholder_vertex = \
entity_graph.can_vertex_be_deleted(instance_vertex)
self.assertTrue(is_placeholder_vertex)
def test_is_not_can_vertex_be_deleted(self):
entity_graph = entity_g.EntityGraph("Entity Graph")
# create vertex properties
prop = {VertexProperties.STATE: 'ACTIVE'}
vertex = self._update_vertex_to_graph(entity_graph, 'RESOURCE',
'INSTANCE', '12345',
False, False, prop)
# check is not placeholder vertex
is_placeholder_vertex = entity_graph.can_vertex_be_deleted(vertex)
self.assertFalse(is_placeholder_vertex)
def test_delete_placeholder_vertex(self):
entity_graph = entity_g.EntityGraph("Entity Graph")
entity_graph = NXGraph("Entity Graph")
# create vertex properties
vertex = self._update_vertex_to_graph(entity_graph, 'RESOURCE',
'INSTANCE', '12345',
False, True, {})
# check is placeholder vertex
is_placeholder_vertex = entity_graph.can_vertex_be_deleted(vertex)
self.assertTrue(is_placeholder_vertex)
# deal with placeholder vertex - mark it as deleted
entity_graph.delete_placeholder_vertex(vertex)
# deal with placeholder vertex
PUtils.delete_placeholder_vertex(entity_graph, vertex)
vertex = entity_graph.get_vertex(vertex.vertex_id)
self.assertTrue(not vertex)
self.assertTrue(vertex is None)
# create vertex properties
vertex = self._update_vertex_to_graph(entity_graph, 'RESOURCE',
'INSTANCE', '12345',
False, False, {})
# deal with non placeholder vertex
PUtils.delete_placeholder_vertex(entity_graph, vertex)
vertex = entity_graph.get_vertex(vertex.vertex_id)
self.assertTrue(vertex is not None)
def test_mark_vertex_as_deleted(self):
entity_graph = entity_g.EntityGraph("Entity Graph")
entity_graph = NXGraph("Entity Graph")
# create vertex properties
vertex = self._update_vertex_to_graph(entity_graph, 'RESOURCE',
@ -97,12 +55,12 @@ class TestEntityGraphManager(base.TestBaseProcessor):
False, True, {})
# check vitrage deleted
self.assertFalse(entity_graph.is_vertex_deleted(vertex))
entity_graph.mark_vertex_as_deleted(vertex)
self.assertTrue(entity_graph.is_vertex_deleted(vertex))
self.assertFalse(PUtils.is_deleted(vertex))
PUtils.mark_deleted(entity_graph, vertex)
self.assertTrue(PUtils.is_deleted(vertex))
def test_mark_edge_as_deleted(self):
entity_graph = entity_g.EntityGraph("Entity Graph")
entity_graph = NXGraph("Entity Graph")
# create vertex properties
vertex1 = self._update_vertex_to_graph(entity_graph, 'RESOURCE',
@ -115,13 +73,13 @@ class TestEntityGraphManager(base.TestBaseProcessor):
vertex2.vertex_id, 'contains')
# check vitrage deleted
self.assertFalse(entity_graph.is_edge_deleted(edge))
entity_graph.mark_edge_as_deleted(edge)
self.assertTrue(entity_graph.is_edge_deleted(edge))
self.assertFalse(PUtils.is_deleted(edge))
PUtils.mark_deleted(entity_graph, edge)
self.assertTrue(PUtils.is_deleted(edge))
def test_find_neighbor_types(self):
neighbors = []
entity_graph = entity_g.EntityGraph("Entity Graph")
entity_graph = NXGraph("Entity Graph")
entities_details = [('RESOURCE', 'HOST', '1', False, True),
('RESOURCE', 'STORAGE', '2', False, True),
('RESOURCE', 'APPLICATION', '3', False, True),
@ -137,5 +95,5 @@ class TestEntityGraphManager(base.TestBaseProcessor):
neighbors.append((vertex, None))
# get neighbors types
types = entity_graph.find_neighbor_types(neighbors)
types = PUtils.find_neighbor_types(neighbors)
self.assertEqual(4, len(types))

3
vitrage/tests/unit/entity_graph/processor/test_processor.py
View File

@ -24,6 +24,7 @@ from vitrage.entity_graph.initialization_status import InitializationStatus
from vitrage.entity_graph.mappings.operational_resource_state import \
OperationalResourceState
from vitrage.entity_graph.processor import processor as proc
from vitrage.entity_graph.processor import processor_utils as PUtils
import vitrage.graph.utils as graph_utils
from vitrage.tests.unit.entity_graph.base import TestEntityGraphUnitBase
from vitrage.utils.datetime import utcnow
@ -128,7 +129,7 @@ class TestProcessor(TestEntityGraphUnitBase):
# check deleted entity
self._check_graph(processor, self.NUM_VERTICES_AFTER_DELETION,
self.NUM_EDGES_AFTER_DELETION)
self.assertTrue(processor.entity_graph.is_vertex_deleted(vertex))
self.assertTrue(PUtils.is_deleted(vertex))
def test_update_relationship(self):
# setup