diff --git a/releasenotes/notes/remove_end_messages-e92e7b41a52c61cf.yaml b/releasenotes/notes/remove_end_messages-e92e7b41a52c61cf.yaml new file mode 100644 index 000000000..3f820dc20 --- /dev/null +++ b/releasenotes/notes/remove_end_messages-e92e7b41a52c61cf.yaml @@ -0,0 +1,9 @@ +--- +features: + - Datasource ``end messages`` previously used to notify the processor + that get_all finished successfully. Are no longer used and are removed. +deprecations: + - Config option ``initialization_interval`` is deprecated and no longer + used, due to the removal of datasource ``end messages``. + - Config option ``initialization_max_retries`` is deprecated and no longer + used, due to the removal of datasource ``end messages``. diff --git a/vitrage/common/constants.py b/vitrage/common/constants.py index 61e6728cc..c9e928928 100644 --- a/vitrage/common/constants.py +++ b/vitrage/common/constants.py @@ -114,7 +114,6 @@ class GraphAction(object): DELETE_RELATIONSHIP = 'delete_relationship' UPDATE_RELATIONSHIP = 'update_relationship' REMOVE_DELETED_ENTITY = 'remove_deleted_entity' - END_MESSAGE = 'end_message' class NotifierEventTypes(object): diff --git a/vitrage/datasources/driver_base.py b/vitrage/datasources/driver_base.py index cccd70962..50f009ee2 100644 --- a/vitrage/datasources/driver_base.py +++ b/vitrage/datasources/driver_base.py @@ -17,9 +17,7 @@ import six from oslo_log import log -from vitrage.common.constants import DatasourceAction from vitrage.common.constants import DatasourceProperties as DSProps -from vitrage.common.constants import GraphAction from vitrage.common.constants import VertexProperties as VProps from vitrage.utils import datetime as datetime_utils @@ -41,28 +39,14 @@ class DriverBase(object): def callback_on_fault(self, exception): pass - @staticmethod - def _get_end_message(entity_type): - end_message = { - DSProps.ENTITY_TYPE: entity_type, - DSProps.DATASOURCE_ACTION: DatasourceAction.INIT_SNAPSHOT, - DSProps.EVENT_TYPE: GraphAction.END_MESSAGE - } - return end_message - def get_changes(self, datasource_action): pass @classmethod def make_pickleable(cls, entities, entity_type, datasource_action, *args): - pickleable_entities = cls.make_pickleable_without_end_msg( + return cls.make_pickleable_without_end_msg( entities, entity_type, datasource_action, *args) - if datasource_action == DatasourceAction.INIT_SNAPSHOT: - pickleable_entities.append(cls._get_end_message(entity_type)) - - return pickleable_entities - @classmethod def make_pickleable_without_end_msg(cls, entities, entity_type, datasource_action, *args): diff --git a/vitrage/datasources/transformer_base.py b/vitrage/datasources/transformer_base.py index 127b47e5d..2ca393275 100644 --- a/vitrage/datasources/transformer_base.py +++ b/vitrage/datasources/transformer_base.py @@ -127,19 +127,14 @@ class TransformerBase(object): :rtype:EntityWrapper """ - if not self._is_end_message(entity_event): - entity_vertex = self._create_entity_vertex(entity_event) - neighbors = self._create_neighbors(entity_event) - action = self._extract_graph_action(entity_event) + entity_vertex = self._create_entity_vertex(entity_event) + neighbors = self._create_neighbors(entity_event) + action = self._extract_graph_action(entity_event) - if action == GraphAction.DELETE_ENTITY: - self._delete_id_from_cache(entity_vertex.vertex_id) + if action == GraphAction.DELETE_ENTITY: + self._delete_id_from_cache(entity_vertex.vertex_id) - return EntityWrapper(entity_vertex, neighbors, action) - else: - return EntityWrapper(self._create_end_vertex(entity_event), - None, - GraphAction.END_MESSAGE) + return EntityWrapper(entity_vertex, neighbors, action) def _create_entity_vertex(self, entity_event): if is_update_event(entity_event) and \ @@ -343,20 +338,6 @@ class TransformerBase(object): raise VitrageTransformerError( 'Invalid action type: (%s)' % datasource_action) - @staticmethod - def _create_end_vertex(entity_event): - entity_type = entity_event[DSProps.ENTITY_TYPE] - return graph_utils.create_vertex('END_MESSAGE:' + entity_type, - vitrage_type=entity_type) - - @staticmethod - def _is_end_message(entity_event): - - ds_action = entity_event[DSProps.DATASOURCE_ACTION] - is_snapshot_event = ds_action == DatasourceAction.INIT_SNAPSHOT - event_type = entity_event.get(DSProps.EVENT_TYPE, None) - return is_snapshot_event and event_type == GraphAction.END_MESSAGE - @staticmethod def _format_update_timestamp(update_timestamp, sample_timestamp): update_timestamp = update_timestamp if update_timestamp \ diff --git a/vitrage/entity_graph/consistency/__init__.py b/vitrage/entity_graph/consistency/__init__.py index 0fb74d63f..9b4ab01c0 100644 --- a/vitrage/entity_graph/consistency/__init__.py +++ b/vitrage/entity_graph/consistency/__init__.py @@ -23,12 +23,18 @@ OPTS = [ cfg.IntOpt('initialization_interval', default=1, min=1, + deprecated_for_removal=True, + deprecated_since='Stein', + deprecated_reason='This config option is no longer used.', help='interval between consistency initialization checks for ' 'finding if all end messages from datasources were ' 'received (in seconds)'), cfg.IntOpt('initialization_max_retries', default=30, min=1, + deprecated_for_removal=True, + deprecated_since='Stein', + deprecated_reason='This config option is no longer used.', help='maximum retries for consistency initialization ' 'for finding if all end messages from datasources were ' 'received (in seconds)'), diff --git a/vitrage/entity_graph/consistency/consistency_enforcer.py b/vitrage/entity_graph/consistency/consistency_enforcer.py index 3831d6f6c..ffad08144 100644 --- a/vitrage/entity_graph/consistency/consistency_enforcer.py +++ b/vitrage/entity_graph/consistency/consistency_enforcer.py @@ -13,7 +13,6 @@ # under the License. from datetime import timedelta -import time from oslo_log import log @@ -136,19 +135,6 @@ class ConsistencyEnforcer(object): (vertex.get(VProps.VITRAGE_DATASOURCE_NAME) in self.datasources_to_mark_deleted) - def _wait_for_action(self, function): - count_retries = 0 - while True: - if count_retries >= \ - self.conf.consistency.initialization_max_retries: - return False - - if function(): - return True - - count_retries += 1 - time.sleep(self.conf.consistency.initialization_interval) - def _init_datasources_to_mark_deleted(self): self.datasources_to_mark_deleted = [] diff --git a/vitrage/entity_graph/processor/processor.py b/vitrage/entity_graph/processor/processor.py index 9081f6e5b..41ac7d110 100644 --- a/vitrage/entity_graph/processor/processor.py +++ b/vitrage/entity_graph/processor/processor.py @@ -55,8 +55,8 @@ class Processor(processor.ProcessorBase): self._enrich_event(event) entity = self.transformer_manager.transform(event) - if entity.action not in self.actions.keys(): - LOG.debug('deprecated or unknown entity %s ignored', str(entity)) + if entity.action not in self.actions: + LOG.warning('Deprecated or unknown entity %s ignored', str(entity)) return self._calculate_vitrage_aggregated_values(entity.vertex, entity.action) diff --git a/vitrage/tests/functional/entity_graph/consistency/test_consistency.py b/vitrage/tests/functional/entity_graph/consistency/test_consistency.py index d52fb704c..b9c765957 100644 --- a/vitrage/tests/functional/entity_graph/consistency/test_consistency.py +++ b/vitrage/tests/functional/entity_graph/consistency/test_consistency.py @@ -21,10 +21,7 @@ from testtools import matchers from vitrage.common.constants import EntityCategory from vitrage.common.constants import VertexProperties as VProps -from vitrage.datasources.nagios import NAGIOS_DATASOURCE -from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE -from vitrage.datasources.nova.zone import NOVA_ZONE_DATASOURCE from vitrage.entity_graph.consistency.consistency_enforcer \ import ConsistencyEnforcer from vitrage.entity_graph.processor.processor import Processor @@ -44,11 +41,6 @@ class TestConsistencyFunctional(TestFunctionalBase, TestConfiguration): cfg.IntOpt('min_time_to_delete', default=1, min=1), - cfg.IntOpt('initialization_interval', - default=1, - min=1), - cfg.IntOpt('initialization_max_retries', - default=10), ] EVALUATOR_OPTS = [ @@ -252,13 +244,6 @@ class TestConsistencyFunctional(TestFunctionalBase, TestConfiguration): datasource_name='cinder.volume', resource_type='cinder.volume') - def _set_end_messages(self): - self.initialization_status.end_messages[NOVA_ZONE_DATASOURCE] = True - self.initialization_status.end_messages[NOVA_HOST_DATASOURCE] = True - self.initialization_status.end_messages[NOVA_INSTANCE_DATASOURCE] = \ - True - self.initialization_status.end_messages[NAGIOS_DATASOURCE] = True - def _update_timestamp(self, lst, timestamp): for vertex in lst: vertex[VProps.VITRAGE_SAMPLE_TIMESTAMP] = str(timestamp) diff --git a/vitrage/tests/unit/datasources/static/test_static_driver.py b/vitrage/tests/unit/datasources/static/test_static_driver.py index 04864905b..75e9cc53e 100644 --- a/vitrage/tests/unit/datasources/static/test_static_driver.py +++ b/vitrage/tests/unit/datasources/static/test_static_driver.py @@ -43,9 +43,9 @@ class TestStaticDriver(base.BaseTest): DatasourceAction.INIT_SNAPSHOT) # Test assertions - self.assertThat(static_entities, matchers.HasLength(9)) + self.assertThat(static_entities, matchers.HasLength(8)) - for entity in static_entities[:-1]: # exclude end message + for entity in static_entities: self._validate_static_entity(entity) # noinspection PyAttributeOutsideInit