From 0e93ae638ea0c1dc0175e7436564b907f6091e49 Mon Sep 17 00:00:00 2001 From: Alexey Date: Wed, 17 May 2017 11:29:18 +0000 Subject: [PATCH] performance improvement in the processor. Use set instead of using list. Change-Id: I61fd045b8e228d06aeb3221dc753079a123ec11c --- vitrage/datasources/aodh/__init__.py | 4 ++-- vitrage/datasources/nagios/__init__.py | 4 ++-- vitrage/datasources/services.py | 2 +- vitrage/datasources/static/__init__.py | 1 + vitrage/datasources/zabbix/__init__.py | 2 +- vitrage/entity_graph/processor/processor.py | 26 ++++++++++----------- vitrage_tempest_tests/tests/api/base.py | 3 --- 7 files changed, 19 insertions(+), 23 deletions(-) diff --git a/vitrage/datasources/aodh/__init__.py b/vitrage/datasources/aodh/__init__.py index eb85d1af5..9ae580a91 100644 --- a/vitrage/datasources/aodh/__init__.py +++ b/vitrage/datasources/aodh/__init__.py @@ -36,7 +36,7 @@ OPTS = [ ' datasource itself.', required=True), cfg.IntOpt(DSOpts.CHANGES_INTERVAL, - default=20, - min=20, + default=10, + min=10, help='interval between checking changes in aodh data source'), ] diff --git a/vitrage/datasources/nagios/__init__.py b/vitrage/datasources/nagios/__init__.py index b588ffce2..39e1b0ace 100644 --- a/vitrage/datasources/nagios/__init__.py +++ b/vitrage/datasources/nagios/__init__.py @@ -36,8 +36,8 @@ OPTS = [ ' datasource itself.', required=True), cfg.IntOpt(DSOpts.CHANGES_INTERVAL, - default=30, - min=30, + default=20, + min=10, help='interval between checking changes in nagios data source'), cfg.StrOpt('user', default='nagiosadmin', help='Nagios user name'), diff --git a/vitrage/datasources/services.py b/vitrage/datasources/services.py index 956e5e4b6..775989d88 100644 --- a/vitrage/datasources/services.py +++ b/vitrage/datasources/services.py @@ -120,5 +120,5 @@ class ChangesService(DatasourceService): for entity in datasource.get_changes(DatasourceAction.UPDATE): self.send_to_queue(entity) except Exception as e: - LOG.error("Get changes Failed - %s", str(e)) + LOG.exception("Get changes Failed - %s", e) LOG.debug("end get changes") diff --git a/vitrage/datasources/static/__init__.py b/vitrage/datasources/static/__init__.py index 776c2778a..aa02fb576 100644 --- a/vitrage/datasources/static/__init__.py +++ b/vitrage/datasources/static/__init__.py @@ -38,6 +38,7 @@ OPTS = [ required=True), cfg.IntOpt(DSOpts.CHANGES_INTERVAL, default=30, + min=10, help='interval in seconds between checking changes in the' 'static configuration files'), # NOTE: This folder is already used by static_physical datasource. Legacy diff --git a/vitrage/datasources/zabbix/__init__.py b/vitrage/datasources/zabbix/__init__.py index 3393f47c1..191f809f4 100644 --- a/vitrage/datasources/zabbix/__init__.py +++ b/vitrage/datasources/zabbix/__init__.py @@ -36,7 +36,7 @@ OPTS = [ ' datasource itself.', required=True), cfg.IntOpt(DSOpts.CHANGES_INTERVAL, - default=30, + default=20, min=10, help='interval between checking changes in zabbix data source'), cfg.StrOpt('user', default='admin', diff --git a/vitrage/entity_graph/processor/processor.py b/vitrage/entity_graph/processor/processor.py index 3dfed8e67..bec6d7de5 100644 --- a/vitrage/entity_graph/processor/processor.py +++ b/vitrage/entity_graph/processor/processor.py @@ -84,7 +84,7 @@ class Processor(processor.ProcessorBase): # so it will be done in one central place self._find_and_fix_graph_vertex(new_vertex, neighbors) self.entity_graph.add_vertex(new_vertex) - self._connect_neighbors(neighbors, [], GraphAction.CREATE_ENTITY) + self._connect_neighbors(neighbors, set(), GraphAction.CREATE_ENTITY) def update_entity(self, updated_vertex, neighbors): """Updates the vertex in the entity graph @@ -235,8 +235,8 @@ class Processor(processor.ProcessorBase): 2. connects the new neighbors. """ - (valid_edges, obsolete_edges) = self._find_edges_status( - vertex, neighbors) + (valid_edges, obsolete_edges) = self._find_edges_status(vertex, + neighbors) self._delete_old_connections(vertex, obsolete_edges) self._connect_neighbors(neighbors, valid_edges, @@ -293,15 +293,15 @@ class Processor(processor.ProcessorBase): longer connected to those entities), and which are valid connections. """ - valid_edges = [] - obsolete_edges = [] + valid_edges = set() + obsolete_edges = set() graph_neighbor_types = \ PUtils.find_neighbor_types(neighbors) - for curr_edge in self.entity_graph.get_edges( - vertex.vertex_id, - direction=Direction.BOTH): + neighbor_edges = set(e for v, e in neighbors) + for curr_edge in self.entity_graph.get_edges(vertex.vertex_id, + direction=Direction.BOTH): # check if the edge in the graph has a a connection to the # same type of resources in the new neighbors list neighbor_vertex = self.entity_graph.get_vertex( @@ -311,14 +311,13 @@ class Processor(processor.ProcessorBase): neighbor_vertex) in graph_neighbor_types if not is_connection_type_exist: - valid_edges.append(curr_edge) + valid_edges.add(curr_edge) continue - neighbor_edges = [e for v, e in neighbors] if curr_edge in neighbor_edges: - valid_edges.append(curr_edge) + valid_edges.add(curr_edge) else: - obsolete_edges.append(curr_edge) + obsolete_edges.add(curr_edge) return valid_edges, obsolete_edges @@ -457,7 +456,6 @@ class Processor(processor.ProcessorBase): raise VitrageError( 'found too many vertices with same properties: %s ', vertex) - graph_vertex = None if not graph_vertices \ - else graph_vertices[0] + graph_vertex = None if not graph_vertices else graph_vertices[0] return graph_vertex diff --git a/vitrage_tempest_tests/tests/api/base.py b/vitrage_tempest_tests/tests/api/base.py index 1fdc1b01c..68b499e97 100644 --- a/vitrage_tempest_tests/tests/api/base.py +++ b/vitrage_tempest_tests/tests/api/base.py @@ -134,14 +134,11 @@ class BaseApiTest(base.BaseTestCase): if public_net: kwargs.update({"networks": [{'uuid': public_net['id']}]}) - # public_net = self._get_public_network() - # nics = [{'net-id': public_net['id']}] img = images_list.next() resources = [self.nova_client.servers.create( name='%s-%s' % ('vm', index), flavor=flavors_list[0], image=img, - # nics=nics, **kwargs) for index in range(num_instances)] self._wait_for_status(30,