From a05242312486785b621d2fbb942b099a35a98a04 Mon Sep 17 00:00:00 2001 From: Idan Hefetz Date: Sun, 22 Jan 2017 17:20:23 +0000 Subject: [PATCH] Code refactoring - EntityGraph class functionality moved to a processor utils Change-Id: Ifec535e4eafbc6062fcadbd99e0a012b8eb5f214 --- vitrage/cmd/graph.py | 4 +- .../entity_graph/processor/entity_graph.py | 99 ------------------- vitrage/entity_graph/processor/processor.py | 32 +++--- .../entity_graph/processor/processor_utils.py | 58 +++++++++++ .../processor/test_entity_graph.py | 94 +++++------------- .../entity_graph/processor/test_processor.py | 3 +- 6 files changed, 105 insertions(+), 185 deletions(-) delete mode 100644 vitrage/entity_graph/processor/entity_graph.py diff --git a/vitrage/cmd/graph.py b/vitrage/cmd/graph.py index 42b19dd27..b9d5bd060 100644 --- a/vitrage/cmd/graph.py +++ b/vitrage/cmd/graph.py @@ -26,10 +26,10 @@ from vitrage.datasources import OPENSTACK_CLUSTER from vitrage.datasources.transformer_base import CLUSTER_ID from vitrage.entity_graph.consistency import service as consistency_svc from vitrage.entity_graph.initialization_status import InitializationStatus -from vitrage.entity_graph.processor import entity_graph from vitrage.entity_graph import service as entity_graph_svc from vitrage.evaluator.scenario_evaluator import ScenarioEvaluator from vitrage.evaluator.scenario_repository import ScenarioRepository +from vitrage.graph import create_graph from vitrage import service @@ -67,7 +67,7 @@ def main(): def init(conf): mp_queue = multiprocessing.Queue() evaluator_q = queue.Queue() - e_graph = entity_graph.EntityGraph( + e_graph = create_graph( 'Entity Graph', '%s:%s:%s' % (EntityCategory.RESOURCE, OPENSTACK_CLUSTER, CLUSTER_ID)) scenario_repo = ScenarioRepository(conf) diff --git a/vitrage/entity_graph/processor/entity_graph.py b/vitrage/entity_graph/processor/entity_graph.py deleted file mode 100644 index a52d62359..000000000 --- a/vitrage/entity_graph/processor/entity_graph.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2015 - Alcatel-Lucent -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_log import log - -from vitrage.common.constants import EdgeProperties as EProps -from vitrage.common.constants import VertexProperties as VProps -from vitrage.graph import NXGraph -from vitrage.utils.datetime import utcnow - - -LOG = log.getLogger(__name__) - - -class EntityGraph(NXGraph): - - def __init__(self, name, root_id=None): - super(EntityGraph, self).__init__(name, root_id) - - def can_vertex_be_deleted(self, vertex): - """Check if the vertex can be deleted - - Vertex can be deleted if it's IS_PLACEHOLDER property is - True and if it has no neighbors that aren't marked deleted - """ - - if not vertex[VProps.IS_PLACEHOLDER]: - return False - - # check that vertex has no neighbors - neighbor_edges = self.get_edges(vertex.vertex_id) - - return not any(True for neighbor_edge in neighbor_edges - if not self.is_edge_deleted(neighbor_edge)) - - def delete_placeholder_vertex(self, suspected_vertex): - """Checks if it is a placeholder vertex, and if so deletes it """ - - if self.can_vertex_be_deleted(suspected_vertex): - LOG.debug("Delete placeholder vertex: %s", suspected_vertex) - self.remove_vertex(suspected_vertex) - - @staticmethod - def is_vertex_deleted(vertex): - return vertex.get(VProps.IS_DELETED, False) - - @staticmethod - def is_edge_deleted(edge): - return edge.get(EProps.IS_DELETED, False) - - def mark_vertex_as_deleted(self, vertex): - """Marks the vertex as is deleted, and updates deletion timestamp""" - vertex[VProps.IS_DELETED] = True - vertex[VProps.SAMPLE_TIMESTAMP] = str(utcnow()) - self.update_vertex(vertex) - - def mark_edge_as_deleted(self, edge): - """Marks the edge as is deleted, and updates delete timestamp""" - edge[EProps.IS_DELETED] = True - edge[EProps.UPDATE_TIMESTAMP] = str(utcnow()) - self.update_edge(edge) - - def find_neighbor_types(self, neighbors): - """Finds all the types (TYPE, SUB_TYPE) of the neighbors """ - - neighbor_types = set() - for (vertex, edge) in neighbors: - neighbor_types.add(self.get_vertex_category(vertex)) - return neighbor_types - - @staticmethod - def get_vertex_category(vertex): - category = vertex[VProps.CATEGORY] - type_ = vertex[VProps.TYPE] - return category, type_ - - @staticmethod - def can_update_vertex(graph_vertex, new_vertex): - return (not graph_vertex) or (not new_vertex[VProps.IS_PLACEHOLDER]) - - def update_entity_graph_vertex(self, graph_vertex, updated_vertex): - if updated_vertex[VProps.IS_PLACEHOLDER] and \ - graph_vertex and not graph_vertex[VProps.IS_PLACEHOLDER]: - - updated_vertex[VProps.IS_PLACEHOLDER] = False - updated_vertex[VProps.IS_DELETED] = graph_vertex[VProps.IS_DELETED] - - self.update_vertex(updated_vertex) diff --git a/vitrage/entity_graph/processor/processor.py b/vitrage/entity_graph/processor/processor.py index e053e97ee..8cfcf30da 100644 --- a/vitrage/entity_graph/processor/processor.py +++ b/vitrage/entity_graph/processor/processor.py @@ -21,10 +21,10 @@ from vitrage.datasources.transformer_base import TransformerBase from vitrage.entity_graph.mappings.datasource_info_mapper import \ DatasourceInfoMapper from vitrage.entity_graph.processor import base as processor -from vitrage.entity_graph.processor import entity_graph from vitrage.entity_graph.processor.notifier import GraphNotifier from vitrage.entity_graph.processor import processor_utils as PUtils from vitrage.entity_graph.transformer_manager import TransformerManager +from vitrage.graph import create_graph from vitrage.graph import Direction LOG = log.getLogger(__name__) @@ -39,8 +39,8 @@ class Processor(processor.ProcessorBase): self.state_manager = DatasourceInfoMapper(self.conf) self._initialize_events_actions() self.initialization_status = initialization_status - self.entity_graph = entity_graph.EntityGraph("Entity Graph") if \ - e_graph is None else e_graph + self.entity_graph = e_graph if e_graph is not None\ + else create_graph("Entity Graph") self._notifier = GraphNotifier(conf) def process_event(self, event): @@ -96,8 +96,9 @@ class Processor(processor.ProcessorBase): if (not graph_vertex) or \ PUtils.is_newer_vertex(graph_vertex, updated_vertex): - self.entity_graph.update_entity_graph_vertex(graph_vertex, - updated_vertex) + PUtils.update_entity_graph_vertex(self.entity_graph, + graph_vertex, + updated_vertex) self._update_neighbors(updated_vertex, neighbors) else: LOG.warning("Update event arrived on invalid resource: %s", @@ -127,12 +128,12 @@ class Processor(processor.ProcessorBase): deleted_vertex.vertex_id) for edge in neighbor_edges: - self.entity_graph.mark_edge_as_deleted(edge) + PUtils.mark_deleted(self.entity_graph, edge) for vertex in neighbor_vertices: - self.entity_graph.delete_placeholder_vertex(vertex) + PUtils.delete_placeholder_vertex(self.entity_graph, vertex) - self.entity_graph.mark_vertex_as_deleted(deleted_vertex) + PUtils.mark_deleted(self.entity_graph, deleted_vertex) else: LOG.warning("Delete event arrived on invalid resource: %s", deleted_vertex) @@ -219,11 +220,12 @@ class Processor(processor.ProcessorBase): for (vertex, edge) in neighbors: graph_vertex = self.entity_graph.get_vertex(vertex.vertex_id) if not graph_vertex or not PUtils.is_deleted(graph_vertex): - if self.entity_graph.can_update_vertex(graph_vertex, vertex): + if PUtils.can_update_vertex(graph_vertex, vertex): LOG.debug("Updates vertex: %s", vertex) self._calculate_aggregated_state(vertex, action) - self.entity_graph.update_entity_graph_vertex(graph_vertex, - vertex) + PUtils.update_entity_graph_vertex(self.entity_graph, + graph_vertex, + vertex) if edge not in valid_edges: LOG.debug("Updates edge: %s", edge) @@ -245,10 +247,10 @@ class Processor(processor.ProcessorBase): # remove old edges and placeholder vertices if exist for edge in obsolete_edges: LOG.debug("Delete obsolete edge:\n%s", edge) - self.entity_graph.mark_edge_as_deleted(edge) + PUtils.mark_deleted(self.entity_graph, edge) graph_ver = self.entity_graph.get_vertex( edge.other_vertex(vertex.vertex_id)) - self.entity_graph.delete_placeholder_vertex(graph_ver) + PUtils.delete_placeholder_vertex(self.entity_graph, graph_ver) def _find_edges_status(self, vertex, neighbors): """Finds "vertex" valid and old connections @@ -262,7 +264,7 @@ class Processor(processor.ProcessorBase): obsolete_edges = [] graph_neighbor_types = \ - self.entity_graph.find_neighbor_types(neighbors) + PUtils.find_neighbor_types(neighbors) for curr_edge in self.entity_graph.get_edges( vertex.vertex_id, @@ -272,7 +274,7 @@ class Processor(processor.ProcessorBase): neighbor_vertex = self.entity_graph.get_vertex( curr_edge.other_vertex(vertex.vertex_id)) - is_connection_type_exist = self.entity_graph.get_vertex_category( + is_connection_type_exist = PUtils.get_vertex_types( neighbor_vertex) in graph_neighbor_types if not is_connection_type_exist: diff --git a/vitrage/entity_graph/processor/processor_utils.py b/vitrage/entity_graph/processor/processor_utils.py index b16173b98..55a0ff5fb 100644 --- a/vitrage/entity_graph/processor/processor_utils.py +++ b/vitrage/entity_graph/processor/processor_utils.py @@ -13,10 +13,17 @@ # under the License. from dateutil import parser + +from oslo_log import log + from vitrage.common.constants import EdgeProperties as EProps from vitrage.common.constants import VertexProperties as VProps from vitrage.graph import Edge from vitrage.graph import Vertex +from vitrage.utils.datetime import utcnow + + +LOG = log.getLogger(__name__) def is_newer_vertex(prev_vertex, new_vertex): @@ -37,3 +44,54 @@ def is_deleted(item): return item and \ (isinstance(item, Vertex) and item.get(VProps.IS_DELETED, False)) or\ (isinstance(item, Edge) and item.get(EProps.IS_DELETED, False)) + + +def mark_deleted(g, item): + if isinstance(item, Vertex): + item[VProps.IS_DELETED] = True + item[VProps.SAMPLE_TIMESTAMP] = str(utcnow()) + g.update_vertex(item) + elif isinstance(item, Edge): + item[EProps.IS_DELETED] = True + item[EProps.UPDATE_TIMESTAMP] = str(utcnow()) + g.update_edge(item) + + +def delete_placeholder_vertex(g, vertex): + """Checks if it is a placeholder vertex, and if so deletes it """ + + if not vertex[VProps.IS_PLACEHOLDER]: + return + if not any(True for neighbor_edge in g.get_edges(vertex.vertex_id) + if not is_deleted(neighbor_edge)): + LOG.debug("Delete placeholder vertex: %s", vertex) + g.remove_vertex(vertex) + + +def find_neighbor_types(neighbors): + """Finds all the types (TYPE, SUB_TYPE) of the neighbors """ + + neighbor_types = set() + for (vertex, edge) in neighbors: + neighbor_types.add(get_vertex_types(vertex)) + return neighbor_types + + +def get_vertex_types(vertex): + category = vertex[VProps.CATEGORY] + type_ = vertex[VProps.TYPE] + return category, type_ + + +def can_update_vertex(graph_vertex, new_vertex): + return (not graph_vertex) or (not new_vertex[VProps.IS_PLACEHOLDER]) + + +def update_entity_graph_vertex(g, graph_vertex, updated_vertex): + if updated_vertex[VProps.IS_PLACEHOLDER] and \ + graph_vertex and not graph_vertex[VProps.IS_PLACEHOLDER]: + + updated_vertex[VProps.IS_PLACEHOLDER] = False + updated_vertex[VProps.IS_DELETED] = graph_vertex[VProps.IS_DELETED] + + g.update_vertex(updated_vertex) diff --git a/vitrage/tests/unit/entity_graph/processor/test_entity_graph.py b/vitrage/tests/unit/entity_graph/processor/test_entity_graph.py index 94901bd8b..c76c4bbd4 100644 --- a/vitrage/tests/unit/entity_graph/processor/test_entity_graph.py +++ b/vitrage/tests/unit/entity_graph/processor/test_entity_graph.py @@ -12,8 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. -from vitrage.common.constants import VertexProperties -from vitrage.entity_graph.processor import entity_graph as entity_g +from vitrage.entity_graph.processor import processor_utils as PUtils +from vitrage.graph import NXGraph from vitrage.tests.unit.entity_graph.processor import base @@ -23,73 +23,31 @@ class TestEntityGraphManager(base.TestBaseProcessor): def setUpClass(cls): super(TestEntityGraphManager, cls).setUpClass() - def test_can_vertex_be_deleted(self): - entity_graph = entity_g.EntityGraph("Entity Graph") - - # create vertex properties - instance_vertex = self._update_vertex_to_graph(entity_graph, - 'RESOURCE', 'INSTANCE', - '123', False, True, {}) - - # check is placeholder vertex - is_placeholder_vertex = \ - entity_graph.can_vertex_be_deleted(instance_vertex) - self.assertTrue(is_placeholder_vertex) - - # add host vertex - host_vertex = self._update_vertex_to_graph(entity_graph, 'RESOURCE', - 'HOST', '321', - False, True, {}) - edge = self._update_edge_to_graph(entity_graph, host_vertex.vertex_id, - instance_vertex.vertex_id, - 'contains') - - # check is placeholder vertex - is_placeholder_vertex = \ - entity_graph.can_vertex_be_deleted(instance_vertex) - self.assertFalse(is_placeholder_vertex) - - # change host to is_deleted - entity_graph.mark_vertex_as_deleted(host_vertex) - entity_graph.mark_edge_as_deleted(edge) - - # check is placeholder vertex - is_placeholder_vertex = \ - entity_graph.can_vertex_be_deleted(instance_vertex) - self.assertTrue(is_placeholder_vertex) - - def test_is_not_can_vertex_be_deleted(self): - entity_graph = entity_g.EntityGraph("Entity Graph") - - # create vertex properties - prop = {VertexProperties.STATE: 'ACTIVE'} - vertex = self._update_vertex_to_graph(entity_graph, 'RESOURCE', - 'INSTANCE', '12345', - False, False, prop) - - # check is not placeholder vertex - is_placeholder_vertex = entity_graph.can_vertex_be_deleted(vertex) - self.assertFalse(is_placeholder_vertex) - def test_delete_placeholder_vertex(self): - entity_graph = entity_g.EntityGraph("Entity Graph") + entity_graph = NXGraph("Entity Graph") # create vertex properties vertex = self._update_vertex_to_graph(entity_graph, 'RESOURCE', 'INSTANCE', '12345', False, True, {}) - # check is placeholder vertex - is_placeholder_vertex = entity_graph.can_vertex_be_deleted(vertex) - self.assertTrue(is_placeholder_vertex) - - # deal with placeholder vertex - mark it as deleted - entity_graph.delete_placeholder_vertex(vertex) + # deal with placeholder vertex + PUtils.delete_placeholder_vertex(entity_graph, vertex) vertex = entity_graph.get_vertex(vertex.vertex_id) - self.assertTrue(not vertex) + self.assertTrue(vertex is None) + + # create vertex properties + vertex = self._update_vertex_to_graph(entity_graph, 'RESOURCE', + 'INSTANCE', '12345', + False, False, {}) + + # deal with non placeholder vertex + PUtils.delete_placeholder_vertex(entity_graph, vertex) + vertex = entity_graph.get_vertex(vertex.vertex_id) + self.assertTrue(vertex is not None) def test_mark_vertex_as_deleted(self): - entity_graph = entity_g.EntityGraph("Entity Graph") + entity_graph = NXGraph("Entity Graph") # create vertex properties vertex = self._update_vertex_to_graph(entity_graph, 'RESOURCE', @@ -97,12 +55,12 @@ class TestEntityGraphManager(base.TestBaseProcessor): False, True, {}) # check vitrage deleted - self.assertFalse(entity_graph.is_vertex_deleted(vertex)) - entity_graph.mark_vertex_as_deleted(vertex) - self.assertTrue(entity_graph.is_vertex_deleted(vertex)) + self.assertFalse(PUtils.is_deleted(vertex)) + PUtils.mark_deleted(entity_graph, vertex) + self.assertTrue(PUtils.is_deleted(vertex)) def test_mark_edge_as_deleted(self): - entity_graph = entity_g.EntityGraph("Entity Graph") + entity_graph = NXGraph("Entity Graph") # create vertex properties vertex1 = self._update_vertex_to_graph(entity_graph, 'RESOURCE', @@ -115,13 +73,13 @@ class TestEntityGraphManager(base.TestBaseProcessor): vertex2.vertex_id, 'contains') # check vitrage deleted - self.assertFalse(entity_graph.is_edge_deleted(edge)) - entity_graph.mark_edge_as_deleted(edge) - self.assertTrue(entity_graph.is_edge_deleted(edge)) + self.assertFalse(PUtils.is_deleted(edge)) + PUtils.mark_deleted(entity_graph, edge) + self.assertTrue(PUtils.is_deleted(edge)) def test_find_neighbor_types(self): neighbors = [] - entity_graph = entity_g.EntityGraph("Entity Graph") + entity_graph = NXGraph("Entity Graph") entities_details = [('RESOURCE', 'HOST', '1', False, True), ('RESOURCE', 'STORAGE', '2', False, True), ('RESOURCE', 'APPLICATION', '3', False, True), @@ -137,5 +95,5 @@ class TestEntityGraphManager(base.TestBaseProcessor): neighbors.append((vertex, None)) # get neighbors types - types = entity_graph.find_neighbor_types(neighbors) + types = PUtils.find_neighbor_types(neighbors) self.assertEqual(4, len(types)) diff --git a/vitrage/tests/unit/entity_graph/processor/test_processor.py b/vitrage/tests/unit/entity_graph/processor/test_processor.py index 8e2790465..68c527319 100644 --- a/vitrage/tests/unit/entity_graph/processor/test_processor.py +++ b/vitrage/tests/unit/entity_graph/processor/test_processor.py @@ -24,6 +24,7 @@ from vitrage.entity_graph.initialization_status import InitializationStatus from vitrage.entity_graph.mappings.operational_resource_state import \ OperationalResourceState from vitrage.entity_graph.processor import processor as proc +from vitrage.entity_graph.processor import processor_utils as PUtils import vitrage.graph.utils as graph_utils from vitrage.tests.unit.entity_graph.base import TestEntityGraphUnitBase from vitrage.utils.datetime import utcnow @@ -128,7 +129,7 @@ class TestProcessor(TestEntityGraphUnitBase): # check deleted entity self._check_graph(processor, self.NUM_VERTICES_AFTER_DELETION, self.NUM_EDGES_AFTER_DELETION) - self.assertTrue(processor.entity_graph.is_vertex_deleted(vertex)) + self.assertTrue(PUtils.is_deleted(vertex)) def test_update_relationship(self): # setup