From b56311901ab5675890e9e078480ca396ea0ca85a Mon Sep 17 00:00:00 2001 From: Alexey Weyl Date: Mon, 29 Feb 2016 13:26:19 +0200 Subject: [PATCH] state normalization support implementation implements: blueprint state-normalization-support Change-Id: I6272d01f12ae4a0a901e0aa61b7defbb8e114c39 --- vitrage/cmd/graph.py | 2 +- vitrage/common/constants.py | 3 + vitrage/common/file_utils.py | 3 +- vitrage/entity_graph/__init__.py | 11 +- vitrage/entity_graph/processor/processor.py | 56 ++++++- vitrage/entity_graph/service.py | 6 +- vitrage/entity_graph/states/__init__.py | 15 ++ vitrage/entity_graph/states/alarm_state.py | 21 +++ vitrage/entity_graph/states/resource_state.py | 32 ++++ vitrage/entity_graph/states/state_manager.py | 146 ++++++++++++++++++ vitrage/entity_graph/transformer_manager.py | 5 +- vitrage/graph/utils.py | 4 - vitrage/opts.py | 3 +- .../plugins/nova/instance/transformer.py | 11 +- .../plugins/static_physical/synchronizer.py | 49 +++--- vitrage/tests/functional/__init__.py | 2 +- vitrage/tests/functional/entity_graph/base.py | 52 +++++++ .../consistency/test_consistency.py | 18 ++- .../entity_graph/processor/__init__.py | 15 ++ .../entity_graph/processor/test_processor.py | 48 ++++++ .../entity_graph/test_state_manager.py | 69 +++++++++ .../resources/states_plugins/nagios.yaml | 28 ++++ .../resources/states_plugins/nova.host.yaml | 26 ++++ .../states_plugins/nova.instance.yaml | 46 ++++++ .../resources/states_plugins/nova.zone.yaml | 26 ++++ .../states_plugins/openstack.node.yaml | 26 ++++ vitrage/tests/unit/entity_graph/__init__.py | 85 +--------- vitrage/tests/unit/entity_graph/base.py | 123 +++++++++++++++ .../tests/unit/entity_graph/processor/base.py | 7 +- .../processor/test_entity_graph.py | 2 +- .../entity_graph/processor/test_processor.py | 119 +++++++++----- .../unit/entity_graph/test_state_manager.py | 122 +++++++++++++++ vitrage_tempest_tests/tests/base_mock.py | 12 +- 33 files changed, 1002 insertions(+), 191 deletions(-) create mode 100644 vitrage/entity_graph/states/__init__.py create mode 100644 vitrage/entity_graph/states/alarm_state.py create mode 100644 vitrage/entity_graph/states/resource_state.py create mode 100644 vitrage/entity_graph/states/state_manager.py create mode 100644 vitrage/tests/functional/entity_graph/base.py create mode 100644 vitrage/tests/functional/entity_graph/processor/__init__.py create mode 100644 vitrage/tests/functional/entity_graph/processor/test_processor.py create mode 100644 vitrage/tests/functional/entity_graph/test_state_manager.py create mode 100644 vitrage/tests/resources/states_plugins/nagios.yaml create mode 100644 vitrage/tests/resources/states_plugins/nova.host.yaml create mode 100644 vitrage/tests/resources/states_plugins/nova.instance.yaml create mode 100644 vitrage/tests/resources/states_plugins/nova.zone.yaml create mode 100644 vitrage/tests/resources/states_plugins/openstack.node.yaml create mode 100644 vitrage/tests/unit/entity_graph/base.py create mode 100644 vitrage/tests/unit/entity_graph/test_state_manager.py diff --git a/vitrage/cmd/graph.py b/vitrage/cmd/graph.py index 49f247bbf..e44afebad 100644 --- a/vitrage/cmd/graph.py +++ b/vitrage/cmd/graph.py @@ -45,7 +45,7 @@ def main(): conf, synchronizer_launcher.create_send_to_queue_callback(event_queue)) launcher.launch_service(entity_graph_svc.VitrageGraphService( - event_queue, e_graph, initialization_status)) + conf, event_queue, e_graph, initialization_status)) launcher.launch_service(api_handler_svc.VitrageApiHandlerService( e_graph)) diff --git a/vitrage/common/constants.py b/vitrage/common/constants.py index daec5abe9..4a16a82a8 100644 --- a/vitrage/common/constants.py +++ b/vitrage/common/constants.py @@ -1,4 +1,5 @@ # Copyright 2015 - Alcatel-Lucent +# Copyright 2016 - Nokia # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -19,6 +20,8 @@ class VertexProperties(object): ID = 'id' IS_DELETED = 'is_deleted' STATE = 'state' + VITRAGE_STATE = 'vitrage_state' + AGGREGATED_STATE = 'aggregated_state' PROJECT_ID = 'project_id' UPDATE_TIMESTAMP = 'update_timestamp' NAME = 'name' diff --git a/vitrage/common/file_utils.py b/vitrage/common/file_utils.py index 03336a745..f40eb940a 100644 --- a/vitrage/common/file_utils.py +++ b/vitrage/common/file_utils.py @@ -20,7 +20,8 @@ import yaml LOG = log.getLogger(__name__) -def load_files(dir_path, suffix=None, +def load_files(dir_path, + suffix=None, with_pathname=False, with_exception=False): try: diff --git a/vitrage/entity_graph/__init__.py b/vitrage/entity_graph/__init__.py index 7a05fb1f4..2b8bc78de 100644 --- a/vitrage/entity_graph/__init__.py +++ b/vitrage/entity_graph/__init__.py @@ -1,4 +1,5 @@ # Copyright 2015 - Alcatel-Lucent +# Copyright 2016 - Nokia # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -12,4 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. -__author__ = 'stack' +from oslo_config import cfg + + +OPTS = [ + cfg.StrOpt('states_plugins_dir', + default='/etc/vitrage/states_plugins', + help='A path for the configuration files of the plugins states' + ), +] diff --git a/vitrage/entity_graph/processor/processor.py b/vitrage/entity_graph/processor/processor.py index 39a235ba8..5f3ea9012 100644 --- a/vitrage/entity_graph/processor/processor.py +++ b/vitrage/entity_graph/processor/processor.py @@ -1,4 +1,5 @@ # Copyright 2015 - Alcatel-Lucent +# Copyright 2016 - Nokia # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -18,7 +19,8 @@ from vitrage.common.constants import EventAction from vitrage.common.constants import VertexProperties as VProps from vitrage.entity_graph.processor import base as processor from vitrage.entity_graph.processor import entity_graph -from vitrage.entity_graph import transformer_manager +from vitrage.entity_graph.states.state_manager import StateManager +from vitrage.entity_graph.transformer_manager import TransformerManager from vitrage.graph import Direction LOG = log.getLogger(__name__) @@ -28,8 +30,10 @@ class Processor(processor.ProcessorBase): NUMBER_OF_PLUGINS = 5 - def __init__(self, initialization_status, e_graph=None): - self.transformer = transformer_manager.TransformerManager() + def __init__(self, cfg, initialization_status, e_graph=None): + self.cfg = cfg + self.transformer_manager = TransformerManager() + self.state_manager = StateManager(self.cfg) self._initialize_events_actions() self.initialization_status = initialization_status self.entity_graph = entity_graph.EntityGraph("Entity Graph") if \ @@ -47,7 +51,7 @@ class Processor(processor.ProcessorBase): """ entity = self.transform_entity(event) - # TODO(Alexey): need to check here the NOT_RELEVANT action as well + self._calculate_aggregated_state(entity.vertex, entity.action) return self.actions[entity.action](entity.vertex, entity.neighbors) def create_entity(self, new_vertex, neighbors): @@ -64,7 +68,7 @@ class Processor(processor.ProcessorBase): LOG.debug("Add entity to entity graph: %s", new_vertex) self.entity_graph.add_vertex(new_vertex) - self._connect_neighbors(neighbors, []) + self._connect_neighbors(neighbors, [], EventAction.CREATE) def update_entity(self, updated_vertex, neighbors): """Updates the vertex in the entity graph @@ -135,7 +139,9 @@ class Processor(processor.ProcessorBase): self.initialization_status.RECEIVED_ALL_END_MESSAGES def transform_entity(self, event): - return self.transformer.transform(event) + entity = self.transformer_manager.transform(event) + LOG.debug('Transformed entity: %s', entity) + return entity def _update_neighbors(self, vertex, neighbors): """Updates vertices neighbor connections @@ -147,9 +153,9 @@ class Processor(processor.ProcessorBase): (valid_edges, obsolete_edges) = self._find_edges_status( vertex, neighbors) self._delete_old_connections(vertex, obsolete_edges) - self._connect_neighbors(neighbors, valid_edges) + self._connect_neighbors(neighbors, valid_edges, EventAction.UPDATE) - def _connect_neighbors(self, neighbors, valid_edges): + def _connect_neighbors(self, neighbors, valid_edges, action): """Updates the neighbor vertex and adds the connection edges """ LOG.debug("Connect neighbors. Neighbors: %s, valid_edges: %s", neighbors, valid_edges) @@ -159,6 +165,7 @@ class Processor(processor.ProcessorBase): not self.entity_graph.is_vertex_deleted(graph_vertex): if self.entity_graph.can_update_vertex(graph_vertex, vertex): LOG.debug("Updates vertex: %s", vertex) + self._calculate_aggregated_state(vertex, action) self.entity_graph.update_vertex(vertex) if edge not in valid_edges: @@ -227,3 +234,36 @@ class Processor(processor.ProcessorBase): EventAction.DELETE: self.delete_entity, EventAction.END_MESSAGE: self.handle_end_message } + + def _calculate_aggregated_state(self, vertex, action): + LOG.debug("calculate event state") + + if action == EventAction.UPDATE or action == EventAction.DELETE: + graph_vertex = self.entity_graph.get_vertex(vertex.vertex_id) + elif action == EventAction.CREATE: + graph_vertex = None + elif action == EventAction.END_MESSAGE: + return None + else: + LOG.info('not recognized action: %s for vertex: %s', + action, vertex) + + state = self._get_updated_property(vertex, + graph_vertex, + VProps.STATE) + vitrage_state = self._get_updated_property(vertex, + graph_vertex, + VProps.VITRAGE_STATE) + + vertex[VProps.AGGREGATED_STATE] = self.state_manager.aggregated_state( + state, vitrage_state, vertex[VProps.TYPE]) + + @staticmethod + def _get_updated_property(new_vertex, graph_vertex, prop): + if new_vertex and prop in new_vertex.properties and new_vertex[prop]: + return new_vertex[prop] + elif graph_vertex and prop in graph_vertex.properties \ + and graph_vertex[prop]: + return graph_vertex[prop] + + return None diff --git a/vitrage/entity_graph/service.py b/vitrage/entity_graph/service.py index 06b9ede59..356a06099 100644 --- a/vitrage/entity_graph/service.py +++ b/vitrage/entity_graph/service.py @@ -25,10 +25,12 @@ LOG = log.getLogger(__name__) class VitrageGraphService(os_service.Service): - def __init__(self, event_queue, entity_graph, initialization_status): + def __init__(self, cfg, event_queue, entity_graph, initialization_status): super(VitrageGraphService, self).__init__() self.queue = event_queue - self.processor = proc.Processor(initialization_status, + self.cfg = cfg + self.processor = proc.Processor(self.cfg, + initialization_status, e_graph=entity_graph) def start(self): diff --git a/vitrage/entity_graph/states/__init__.py b/vitrage/entity_graph/states/__init__.py new file mode 100644 index 000000000..dd32b852f --- /dev/null +++ b/vitrage/entity_graph/states/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__author__ = 'stack' diff --git a/vitrage/entity_graph/states/alarm_state.py b/vitrage/entity_graph/states/alarm_state.py new file mode 100644 index 000000000..34fcba7d2 --- /dev/null +++ b/vitrage/entity_graph/states/alarm_state.py @@ -0,0 +1,21 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class AlarmState(object): + HIGH = 'HIGH' + MEDIUM = 'MEDIUM' + LOW = 'LOW' + OK = 'OK' + UNKNOWN = 'UNKNOWN' diff --git a/vitrage/entity_graph/states/resource_state.py b/vitrage/entity_graph/states/resource_state.py new file mode 100644 index 000000000..134d66975 --- /dev/null +++ b/vitrage/entity_graph/states/resource_state.py @@ -0,0 +1,32 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class ResourceState(object): + TERMINATED = 'TERMINATED' + ERROR = 'ERROR' + UNRECOGNIZED = 'UNRECOGNIZED' + UNAVAILABLE = 'UNAVAILABLE' + SUSPENDED = 'SUSPENDED' + HIBERNATE = 'HIBERNATE' + PAUSED = 'PAUSED' + TERMINATING = 'TERMINATING' + SUSPENDING = 'SUSPENDING' + REBUILDING = 'REBUILDING' + STARTING = 'STARTING' + SUBOPTIMAL = 'SUBOPTIMAL' + AVAILABLE = 'AVAILABLE' + RUNNING = 'RUNNING' + CREATING = 'CREATING' + UNDEFINED = 'UNDEFINED' diff --git a/vitrage/entity_graph/states/state_manager.py b/vitrage/entity_graph/states/state_manager.py new file mode 100644 index 000000000..117367ba3 --- /dev/null +++ b/vitrage/entity_graph/states/state_manager.py @@ -0,0 +1,146 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from oslo_log import log + +from vitrage.common.constants import EntityCategory +from vitrage.common import file_utils +from vitrage.entity_graph.states.alarm_state import AlarmState +from vitrage.entity_graph.states.resource_state import ResourceState + +LOG = log.getLogger(__name__) + + +class StateManager(object): + + STATES = 'states' + PRIORITIES = 'priorities' + UNKNOWN_TYPE = 'unknown_type' + + def __init__(self, cfg): + self.cfg = cfg + self.category_unknown_type = self._init_category_unknown_type() + self.category_additional_data = self._init_category_additional_data() + self.states_plugins = self._load_state_configurations() + + def normalize_state(self, plugin_name, state): + upper_state = state if not state else state.upper() + return self.states_plugins[plugin_name][self.STATES][upper_state] \ + if upper_state in self.states_plugins[plugin_name][self.STATES] else \ + self.states_plugins[plugin_name][self.UNKNOWN_TYPE] + + def state_priority(self, plugin_name, normalized_state): + # no need to check if normalized_state exists, cause it exists for sure + upper_state = normalized_state if not normalized_state else \ + normalized_state.upper() + return self.states_plugins[plugin_name][self.PRIORITIES][upper_state] + + def aggregated_state(self, state1, state2, plugin_name, + is_normalized=False): + upper_state1 = state1 if not state1 else state1.upper() + upper_state2 = state2 if not state2 else state2.upper() + + normalized_state1 = upper_state1.upper() if is_normalized else \ + self.normalize_state(plugin_name, upper_state1) + normalized_state2 = upper_state2.upper() if is_normalized else \ + self.normalize_state(plugin_name, upper_state2) + + priority_state1 = self.state_priority(plugin_name, + normalized_state1) + priority_state2 = self.state_priority(plugin_name, + normalized_state2) + + return normalized_state1 if priority_state1 > priority_state2 \ + else normalized_state2 + + def _load_state_configurations(self): + states_plugins = {} + + files = file_utils.load_files( + self.cfg.entity_graph.states_plugins_dir, '.yaml') + + for file_name in files: + full_path = self.cfg.entity_graph.states_plugins_dir + '/' \ + + file_name + states, priorities, unknown_type = \ + self._retrieve_states_and_priorities_from_file(full_path) + states_plugins[os.path.splitext(file_name)[0]] = { + self.STATES: states, + self.PRIORITIES: priorities, + self.UNKNOWN_TYPE: unknown_type + } + + # TODO(Alexey): implement this after finishing implement load + # specific plugins from configuration + # self._is_all_plugins_states_exists() + + return states_plugins + + def _retrieve_states_and_priorities_from_file(self, full_path): + states = {} + priorities = {} + config = file_utils.load_yaml_file(full_path, with_exception=True) + + for item in config['states']: + normalized_state = item['normalized state'] + + # original to normalized state + normalized_state_name = normalized_state['name'] + for original_state in normalized_state['original states']: + states[original_state['name'].upper()] = normalized_state_name + + self._add_default_states(states, priorities) + + # normalized state priority + priorities[normalized_state_name] = \ + int(normalized_state['priority']) + + self.category_additional_data[config['category']](states, + priorities, + full_path) + + category_unknown_type = self.category_unknown_type[config['category']] + return states, priorities, category_unknown_type + + @staticmethod + def _add_default_states(states, priorities): + states[None] = ResourceState.UNDEFINED + priorities[ResourceState.UNDEFINED] = 0 + + @staticmethod + def _init_category_unknown_type(): + return { + EntityCategory.RESOURCE: ResourceState.UNRECOGNIZED, + EntityCategory.ALARM: AlarmState.UNKNOWN + } + + def _init_category_additional_data(self): + return { + EntityCategory.RESOURCE: self._resource_additional_states, + EntityCategory.ALARM: self._alarm_additional_states + } + + @staticmethod + def _resource_additional_states(states, priorities, full_path): + if ResourceState.UNRECOGNIZED not in priorities: + raise ValueError('%s state is not defined in %s', + ResourceState.UNRECOGNIZED, full_path) + + @staticmethod + def _alarm_additional_states(states, priorities, full_path): + if AlarmState.UNKNOWN not in priorities: + raise ValueError('%s state is not defined in %s', + AlarmState.UNKNOWN, full_path) diff --git a/vitrage/entity_graph/transformer_manager.py b/vitrage/entity_graph/transformer_manager.py index 5bf41bd64..b6b9a2ed8 100644 --- a/vitrage/entity_graph/transformer_manager.py +++ b/vitrage/entity_graph/transformer_manager.py @@ -1,4 +1,5 @@ # Copyright 2015 - Alcatel-Lucent +# Copyright 2016 - Nokia # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -32,10 +33,10 @@ LOG = logging.getLogger(__name__) class TransformerManager(object): def __init__(self): - self.transformers = self.register_transformer_classes() + self.transformers = self._register_transformer_classes() @staticmethod - def register_transformer_classes(): + def _register_transformer_classes(): transformers = {} diff --git a/vitrage/graph/utils.py b/vitrage/graph/utils.py index 9103db3a2..3683b95d3 100644 --- a/vitrage/graph/utils.py +++ b/vitrage/graph/utils.py @@ -22,7 +22,6 @@ def create_vertex(vitrage_id, entity_id=None, entity_category=None, entity_type=None, - entity_project=None, entity_state=None, is_deleted=False, update_timestamp=None, @@ -38,8 +37,6 @@ def create_vertex(vitrage_id, :type entity_category: str :param entity_type: :type entity_type: str - :param entity_project: - :type entity_project: str :param entity_state: :type entity_state: str :param is_deleted: @@ -56,7 +53,6 @@ def create_vertex(vitrage_id, properties = { VConst.ID: entity_id, - VConst.PROJECT_ID: entity_project, VConst.STATE: entity_state, VConst.TYPE: entity_type, VConst.CATEGORY: entity_category, diff --git a/vitrage/opts.py b/vitrage/opts.py index dceb0a999..706abd5b8 100644 --- a/vitrage/opts.py +++ b/vitrage/opts.py @@ -26,5 +26,6 @@ def list_opts(): ('synchronizer', vitrage.synchronizer.OPTS), ('evaluator', vitrage.evaluator.OPTS), ('synchronizer_plugins', vitrage.synchronizer.plugins.OPTS), - ('consistency', vitrage.entity_graph.consistency.OPTS) + ('consistency', vitrage.entity_graph.consistency.OPTS), + ('entity_graph', vitrage.entity_graph.OPTS) ] diff --git a/vitrage/synchronizer/plugins/nova/instance/transformer.py b/vitrage/synchronizer/plugins/nova/instance/transformer.py index 23f278eba..e32dde075 100644 --- a/vitrage/synchronizer/plugins/nova/instance/transformer.py +++ b/vitrage/synchronizer/plugins/nova/instance/transformer.py @@ -85,12 +85,13 @@ class InstanceTransformer(transformer_base.TransformerBase): def _create_entity_vertex(self, entity_event): sync_mode = entity_event[SyncProps.SYNC_MODE] + project = extract_field_value(entity_event, self.PROJECT_ID[sync_mode]) metadata = { - VProps.NAME: extract_field_value( - entity_event, - self.INSTANCE_NAME[sync_mode]), - VProps.IS_PLACEHOLDER: False + VProps.NAME: extract_field_value(entity_event, + self.INSTANCE_NAME[sync_mode]), + VProps.IS_PLACEHOLDER: False, + VProps.PROJECT_ID: project } entity_key = self.extract_key(entity_event) @@ -98,7 +99,6 @@ class InstanceTransformer(transformer_base.TransformerBase): entity_id = extract_field_value( entity_event, self.INSTANCE_ID[sync_mode]) - project = extract_field_value(entity_event, self.PROJECT_ID[sync_mode]) state = extract_field_value( entity_event, self.INSTANCE_STATE[sync_mode]) @@ -111,7 +111,6 @@ class InstanceTransformer(transformer_base.TransformerBase): entity_id=entity_id, entity_category=EntityCategory.RESOURCE, entity_type=self.INSTANCE_TYPE, - entity_project=project, entity_state=state, update_timestamp=update_timestamp, metadata=metadata) diff --git a/vitrage/synchronizer/plugins/static_physical/synchronizer.py b/vitrage/synchronizer/plugins/static_physical/synchronizer.py index 4d8da6b6e..6ac47375f 100644 --- a/vitrage/synchronizer/plugins/static_physical/synchronizer.py +++ b/vitrage/synchronizer/plugins/static_physical/synchronizer.py @@ -69,36 +69,35 @@ class StaticPhysicalSynchronizer(SynchronizerBase): def _get_changes_entities(self): entities_updates = [] - if os.path.isdir(self.cfg.synchronizer_plugins.static_plugins_dir): - entities_updates = [] - files = file_utils.load_files( - self.cfg.synchronizer_plugins.static_plugins_dir, '.yaml') + entities_updates = [] + files = file_utils.load_files( + self.cfg.synchronizer_plugins.static_plugins_dir, '.yaml') - for file in files: - full_path = self.cfg.synchronizer_plugins.static_plugins_dir +\ - '/' + file - config = file_utils.load_yaml_file(full_path) - if config: - if file in self.cache: - if str(config) != str(self.cache[file]): - # TODO(alexey_weyl): need also to remove deleted - # files from cache + for file in files: + full_path = self.cfg.synchronizer_plugins.static_plugins_dir +\ + '/' + file + config = file_utils.load_yaml_file(full_path) + if config: + if file in self.cache: + if str(config) != str(self.cache[file]): + # TODO(alexey_weyl): need also to remove deleted + # files from cache - self._update_on_existing_entities( - self.cache[file][self.ENTITIES_SECTION], - config[self.ENTITIES_SECTION], - entities_updates) + self._update_on_existing_entities( + self.cache[file][self.ENTITIES_SECTION], + config[self.ENTITIES_SECTION], + entities_updates) - self._update_on_new_entities( - config[self.ENTITIES_SECTION], - self.cache[file][self.ENTITIES_SECTION], - entities_updates) + self._update_on_new_entities( + config[self.ENTITIES_SECTION], + self.cache[file][self.ENTITIES_SECTION], + entities_updates) - self.cache[file] = config - else: self.cache[file] = config - entities_updates += \ - self._get_entities_from_file(file, full_path) + else: + self.cache[file] = config + entities_updates += \ + self._get_entities_from_file(file, full_path) return entities_updates diff --git a/vitrage/tests/functional/__init__.py b/vitrage/tests/functional/__init__.py index 7a05fb1f4..dd32b852f 100644 --- a/vitrage/tests/functional/__init__.py +++ b/vitrage/tests/functional/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2015 - Alcatel-Lucent +# Copyright 2016 - Nokia # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain diff --git a/vitrage/tests/functional/entity_graph/base.py b/vitrage/tests/functional/entity_graph/base.py new file mode 100644 index 000000000..3a8e50388 --- /dev/null +++ b/vitrage/tests/functional/entity_graph/base.py @@ -0,0 +1,52 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from vitrage.common.constants import SynchronizerProperties as SyncProps +from vitrage.common.constants import SyncMode +from vitrage.entity_graph.initialization_status import InitializationStatus +from vitrage.entity_graph.processor import processor as proc +from vitrage.tests.mocks import mock_syncronizer as mock_sync +from vitrage.tests.unit.entity_graph.base import TestEntityGraphUnitBase + + +class TestEntityGraphFunctionalBase(TestEntityGraphUnitBase): + + def _create_processor_with_graph(self, conf, processor=None): + events = self._create_mock_events() + + if not processor: + processor = proc.Processor(conf, InitializationStatus()) + + for event in events: + processor.process_event(event) + + return processor + + def _create_mock_events(self): + gen_list = mock_sync.simple_zone_generators( + self.NUM_ZONES, + self.NUM_HOSTS, + snapshot_events=self.NUM_ZONES, + snap_vals={SyncProps.SYNC_MODE: SyncMode.INIT_SNAPSHOT}) + gen_list += mock_sync.simple_host_generators( + self.NUM_ZONES, + self.NUM_HOSTS, + self.NUM_HOSTS, + snap_vals={SyncProps.SYNC_MODE: SyncMode.INIT_SNAPSHOT}) + gen_list += mock_sync.simple_instance_generators( + self.NUM_HOSTS, + self.NUM_INSTANCES, + self.NUM_INSTANCES, + snap_vals={SyncProps.SYNC_MODE: SyncMode.INIT_SNAPSHOT}) + return mock_sync.generate_sequential_events_list(gen_list) diff --git a/vitrage/tests/functional/entity_graph/consistency/test_consistency.py b/vitrage/tests/functional/entity_graph/consistency/test_consistency.py index c5b61229e..b2cbad3f5 100644 --- a/vitrage/tests/functional/entity_graph/consistency/test_consistency.py +++ b/vitrage/tests/functional/entity_graph/consistency/test_consistency.py @@ -28,12 +28,13 @@ from vitrage.entity_graph.consistency.consistency_enforcer \ from vitrage.entity_graph.initialization_status import InitializationStatus from vitrage.entity_graph.processor.processor import Processor import vitrage.graph.utils as graph_utils -from vitrage.tests.unit.entity_graph import TestEntityGraphBase +from vitrage.tests.functional.entity_graph.base import \ + TestEntityGraphFunctionalBase -class TestConsistency(TestEntityGraphBase): +class TestConsistencyFunctional(TestEntityGraphFunctionalBase): - OPTS = [ + CONSISTENCY_OPTS = [ cfg.IntOpt('consistency_interval', default=1, min=1), @@ -43,11 +44,12 @@ class TestConsistency(TestEntityGraphBase): ] def setUp(self): - super(TestConsistency, self).setUp() + super(TestConsistencyFunctional, self).setUp() self.initialization_status = InitializationStatus() - self.processor = Processor(self.initialization_status) self.conf = cfg.ConfigOpts() - self.conf.register_opts(self.OPTS, group='consistency') + self.conf.register_opts(self.CONSISTENCY_OPTS, group='consistency') + self.conf.register_opts(self.PROCESSOR_OPTS, group='entity_graph') + self.processor = Processor(self.conf, self.initialization_status) self.consistency_enforcer = ConsistencyEnforcer( self.conf, self.processor.entity_graph, self.initialization_status) @@ -57,7 +59,7 @@ class TestConsistency(TestEntityGraphBase): # Setup num_external_alarms = self.NUM_HOSTS - 2 num_instances_per_host = 4 - self._create_processor_with_graph(processor=self.processor) + self._create_processor_with_graph(self.conf, processor=self.processor) self._add_alarms() self._set_end_messages() self.assertEqual(self._num_total_expected_vertices() + @@ -105,7 +107,7 @@ class TestConsistency(TestEntityGraphBase): len(self.processor.entity_graph.get_vertices())) def _periodic_process_setup_stage(self, consistency_interval): - self._create_processor_with_graph(processor=self.processor) + self._create_processor_with_graph(self.conf, processor=self.processor) current_time = utcnow() # set all vertices to be have timestamp that consistency won't get diff --git a/vitrage/tests/functional/entity_graph/processor/__init__.py b/vitrage/tests/functional/entity_graph/processor/__init__.py new file mode 100644 index 000000000..dd32b852f --- /dev/null +++ b/vitrage/tests/functional/entity_graph/processor/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__author__ = 'stack' diff --git a/vitrage/tests/functional/entity_graph/processor/test_processor.py b/vitrage/tests/functional/entity_graph/processor/test_processor.py new file mode 100644 index 000000000..c1f100aad --- /dev/null +++ b/vitrage/tests/functional/entity_graph/processor/test_processor.py @@ -0,0 +1,48 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from vitrage.tests.functional.entity_graph.base import \ + TestEntityGraphFunctionalBase + + +class TestProcessorFunctional(TestEntityGraphFunctionalBase): + + ZONE_SPEC = 'ZONE_SPEC' + HOST_SPEC = 'HOST_SPEC' + INSTANCE_SPEC = 'INSTANCE_SPEC' + NUM_VERTICES_AFTER_CREATION = 2 + NUM_EDGES_AFTER_CREATION = 1 + NUM_VERTICES_AFTER_DELETION = 1 + NUM_EDGES_AFTER_DELETION = 0 + + def setUp(self): + super(TestProcessorFunctional, self).setUp() + self.conf = cfg.ConfigOpts() + self.conf.register_opts(self.PROCESSOR_OPTS, group='entity_graph') + + def test_create_entity_graph(self): + processor = self._create_processor_with_graph(self.conf) + + # check number of entities + num_vertices = len(processor.entity_graph) + self.assertEqual(self._num_total_expected_vertices(), num_vertices) + + # TODO(Alexey): add this check and to check also the number of edges + # check all entities create a tree and no free floating vertices exists + # it will be done only after we will have zone plugin + # vertex = graph.find_vertex_in_graph() + # bfs_list = graph.algo.bfs(graph) + # self.assertEqual(num_vertices, len(bfs_list)) diff --git a/vitrage/tests/functional/entity_graph/test_state_manager.py b/vitrage/tests/functional/entity_graph/test_state_manager.py new file mode 100644 index 000000000..698de128e --- /dev/null +++ b/vitrage/tests/functional/entity_graph/test_state_manager.py @@ -0,0 +1,69 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from vitrage.common.constants import EventAction +from vitrage.common.constants import SyncMode +from vitrage.common.constants import VertexProperties as VProps +from vitrage.entity_graph.initialization_status import InitializationStatus +from vitrage.entity_graph.processor import processor as proc +from vitrage.entity_graph.states.resource_state import ResourceState +from vitrage.synchronizer.plugins.nova.instance.transformer import \ + InstanceTransformer +from vitrage.tests.functional.entity_graph.base import \ + TestEntityGraphFunctionalBase + + +class TestStateManagerFunctional(TestEntityGraphFunctionalBase): + + def setUp(self): + super(TestStateManagerFunctional, self).setUp() + self.conf = cfg.ConfigOpts() + self.conf.register_opts(self.PROCESSOR_OPTS, group='entity_graph') + + def test_state_on_update(self): + # setup + processor = proc.Processor(self.conf, InitializationStatus()) + event = self._create_event(spec_type='INSTANCE_SPEC', + sync_mode=SyncMode.INIT_SNAPSHOT) + + # action + processor.process_event(event) + + # test assertions + instance_transformer = InstanceTransformer({}) + vitrage_id = instance_transformer.extract_key(event) + vertex = processor.entity_graph.get_vertex(vitrage_id) + self.assertEqual(ResourceState.RUNNING, + vertex[VProps.AGGREGATED_STATE]) + + def test_state_on_neighbor_update(self): + # setup + vertex, neighbors, processor = self._create_entity( + spec_type='INSTANCE_SPEC', + sync_mode=SyncMode.INIT_SNAPSHOT) + self.assertEqual(2, processor.entity_graph.num_vertices()) + + neighbors[0].vertex[VProps.STATE] = 'available' + neighbors[0].vertex[VProps.IS_PLACEHOLDER] = False + + # action + processor._connect_neighbors(neighbors, [], EventAction.UPDATE) + + # test assertions + neighbor_vertex = processor.entity_graph.get_vertex( + neighbors[0].vertex.vertex_id) + self.assertEqual(ResourceState.AVAILABLE, + neighbor_vertex[VProps.AGGREGATED_STATE]) diff --git a/vitrage/tests/resources/states_plugins/nagios.yaml b/vitrage/tests/resources/states_plugins/nagios.yaml new file mode 100644 index 000000000..fda6b7075 --- /dev/null +++ b/vitrage/tests/resources/states_plugins/nagios.yaml @@ -0,0 +1,28 @@ +category: ALARM +states: + - normalized state: + name: UNKNOWN + priority: 50 + original states: + - name: UNKNOWN + - normalized state: + name: HIGH + priority: 40 + original states: + - name: CRITITCAL + - name: DOWN + - normalized state: + name: MEDIUM + priority: 30 + original states: + - name: WARNING + - normalized state: + name: LOW + priority: 20 + original states: + - normalized state: + name: OK + priority: 10 + original states: + - name: OK + - name: UP diff --git a/vitrage/tests/resources/states_plugins/nova.host.yaml b/vitrage/tests/resources/states_plugins/nova.host.yaml new file mode 100644 index 000000000..d35d056f1 --- /dev/null +++ b/vitrage/tests/resources/states_plugins/nova.host.yaml @@ -0,0 +1,26 @@ +category: RESOURCE +states: + - normalized state: + name: TERMINATED + priority: 150 + original states: + - name: DELETED + - normalized state: + name: ERROR + priority: 140 + original states: + - name: ERROR + - normalized state: + name: UNRECOGNIZED + priority: 130 + original states: + - normalized state: + name: SUBOPTIMAL + priority: 40 + original states: + - name: SUBOPTIMAL + - normalized state: + name: AVAILABLE + priority: 30 + original states: + - name: available diff --git a/vitrage/tests/resources/states_plugins/nova.instance.yaml b/vitrage/tests/resources/states_plugins/nova.instance.yaml new file mode 100644 index 000000000..ad7668038 --- /dev/null +++ b/vitrage/tests/resources/states_plugins/nova.instance.yaml @@ -0,0 +1,46 @@ +category: RESOURCE +states: + - normalized state: + name: TERMINATED + priority: 150 + original states: + - name: DELETED + - normalized state: + name: ERROR + priority: 140 + original states: + - name: ERROR + - normalized state: + name: UNRECOGNIZED + priority: 130 + original states: + - normalized state: + name: SUSPENDED + priority: 110 + original states: + - name: SUSPENDED + - normalized state: + name: REBUILDING + priority: 60 + original states: + - name: REBUILD + - normalized state: + name: STARTING + priority: 50 + original states: + - name: VERIFY_RESIZE + - name: REVERT_RESIZE + - name: PASSWORD + - name: REBOOT + - name: BUILD + - name: HARD_REBOOT + - normalized state: + name: SUBOPTIMAL + priority: 40 + original states: + - name: SUBOPTIMAL + - normalized state: + name: RUNNING + priority: 20 + original states: + - name: ACTIVE diff --git a/vitrage/tests/resources/states_plugins/nova.zone.yaml b/vitrage/tests/resources/states_plugins/nova.zone.yaml new file mode 100644 index 000000000..d35d056f1 --- /dev/null +++ b/vitrage/tests/resources/states_plugins/nova.zone.yaml @@ -0,0 +1,26 @@ +category: RESOURCE +states: + - normalized state: + name: TERMINATED + priority: 150 + original states: + - name: DELETED + - normalized state: + name: ERROR + priority: 140 + original states: + - name: ERROR + - normalized state: + name: UNRECOGNIZED + priority: 130 + original states: + - normalized state: + name: SUBOPTIMAL + priority: 40 + original states: + - name: SUBOPTIMAL + - normalized state: + name: AVAILABLE + priority: 30 + original states: + - name: available diff --git a/vitrage/tests/resources/states_plugins/openstack.node.yaml b/vitrage/tests/resources/states_plugins/openstack.node.yaml new file mode 100644 index 000000000..d35d056f1 --- /dev/null +++ b/vitrage/tests/resources/states_plugins/openstack.node.yaml @@ -0,0 +1,26 @@ +category: RESOURCE +states: + - normalized state: + name: TERMINATED + priority: 150 + original states: + - name: DELETED + - normalized state: + name: ERROR + priority: 140 + original states: + - name: ERROR + - normalized state: + name: UNRECOGNIZED + priority: 130 + original states: + - normalized state: + name: SUBOPTIMAL + priority: 40 + original states: + - name: SUBOPTIMAL + - normalized state: + name: AVAILABLE + priority: 30 + original states: + - name: available diff --git a/vitrage/tests/unit/entity_graph/__init__.py b/vitrage/tests/unit/entity_graph/__init__.py index 898913fc5..dd32b852f 100644 --- a/vitrage/tests/unit/entity_graph/__init__.py +++ b/vitrage/tests/unit/entity_graph/__init__.py @@ -12,87 +12,4 @@ # License for the specific language governing permissions and limitations # under the License. -from vitrage.common.constants import EntityCategory -from vitrage.common.constants import SynchronizerProperties as SyncProps -from vitrage.common.constants import SyncMode -from vitrage.common.datetime_utils import utcnow -from vitrage.entity_graph.initialization_status import InitializationStatus -from vitrage.entity_graph.processor import processor as proc -import vitrage.graph.utils as graph_utils -from vitrage.tests import base -from vitrage.tests.mocks import mock_syncronizer as mock_sync - - -class TestEntityGraphBase(base.BaseTest): - - NUM_NODES = 1 - NUM_ZONES = 2 - NUM_HOSTS = 4 - NUM_INSTANCES = 16 - - def _create_processor_with_graph(self, processor=None): - events = self._create_mock_events() - - if not processor: - processor = proc.Processor(InitializationStatus()) - - for event in events: - processor.process_event(event) - - return processor - - def _create_mock_events(self): - gen_list = mock_sync.simple_zone_generators( - self.NUM_ZONES, - self.NUM_HOSTS, - snapshot_events=self.NUM_ZONES, - snap_vals={SyncProps.SYNC_MODE: SyncMode.INIT_SNAPSHOT}) - gen_list += mock_sync.simple_host_generators( - self.NUM_ZONES, - self.NUM_HOSTS, - self.NUM_HOSTS, - snap_vals={SyncProps.SYNC_MODE: SyncMode.INIT_SNAPSHOT}) - gen_list += mock_sync.simple_instance_generators( - self.NUM_HOSTS, - self.NUM_INSTANCES, - self.NUM_INSTANCES, - snap_vals={SyncProps.SYNC_MODE: SyncMode.INIT_SNAPSHOT}) - return mock_sync.generate_sequential_events_list(gen_list) - - @staticmethod - def _create_event(spec_type=None, sync_mode=None, - event_type=None, properties=None): - # generate event - spec_list = mock_sync.simple_instance_generators(1, 1, 1) - events_list = mock_sync.generate_random_events_list( - spec_list) - - # update properties - if sync_mode is not None: - events_list[0][SyncProps.SYNC_MODE] = sync_mode - - if event_type is not None: - events_list[0][SyncProps.EVENT_TYPE] = event_type - - if properties is not None: - for key, value in properties.iteritems(): - events_list[0][key] = value - - return events_list[0] - - @staticmethod - def _create_alarm(vitrage_id, alarm_type): - return graph_utils.create_vertex( - vitrage_id, - entity_id=vitrage_id, - entity_category=EntityCategory.ALARM, - entity_type=alarm_type, - entity_state='Running', - is_deleted=False, - update_timestamp=utcnow(), - is_placeholder=False, - ) - - def _num_total_expected_vertices(self): - return self.NUM_NODES + self.NUM_ZONES + self.NUM_HOSTS + \ - self.NUM_INSTANCES +__author__ = 'stack' diff --git a/vitrage/tests/unit/entity_graph/base.py b/vitrage/tests/unit/entity_graph/base.py new file mode 100644 index 000000000..55c9d7dd4 --- /dev/null +++ b/vitrage/tests/unit/entity_graph/base.py @@ -0,0 +1,123 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import SynchronizerProperties as SyncProps +from vitrage.common.constants import SyncMode +from vitrage.common.datetime_utils import utcnow +from vitrage.entity_graph.initialization_status import InitializationStatus +from vitrage.entity_graph.processor import processor as proc +import vitrage.graph.utils as graph_utils +from vitrage.tests import base +from vitrage.tests.mocks import mock_syncronizer as mock_sync +from vitrage.tests.mocks import utils + + +class TestEntityGraphUnitBase(base.BaseTest): + + PROCESSOR_OPTS = [ + cfg.StrOpt('states_plugins_dir', + default=utils.get_resources_dir() + '/states_plugins'), + ] + + NUM_NODES = 1 + NUM_ZONES = 2 + NUM_HOSTS = 4 + NUM_INSTANCES = 16 + + def _create_processor_with_graph(self, conf, processor=None): + events = self._create_mock_events() + + if not processor: + processor = proc.Processor(conf, InitializationStatus()) + + for event in events: + processor.process_event(event) + + return processor + + def _create_mock_events(self): + gen_list = mock_sync.simple_zone_generators( + self.NUM_ZONES, + self.NUM_HOSTS, + snapshot_events=self.NUM_ZONES, + snap_vals={SyncProps.SYNC_MODE: SyncMode.INIT_SNAPSHOT}) + gen_list += mock_sync.simple_host_generators( + self.NUM_ZONES, + self.NUM_HOSTS, + self.NUM_HOSTS, + snap_vals={SyncProps.SYNC_MODE: SyncMode.INIT_SNAPSHOT}) + gen_list += mock_sync.simple_instance_generators( + self.NUM_HOSTS, + self.NUM_INSTANCES, + self.NUM_INSTANCES, + snap_vals={SyncProps.SYNC_MODE: SyncMode.INIT_SNAPSHOT}) + return mock_sync.generate_sequential_events_list(gen_list) + + def _create_entity(self, processor=None, spec_type=None, sync_mode=None, + event_type=None, properties=None): + # create instance event with host neighbor + event = self._create_event(spec_type=spec_type, + sync_mode=sync_mode, + event_type=event_type, + properties=properties) + + # add instance entity with host + if processor is None: + processor = proc.Processor(self.conf, InitializationStatus()) + + vertex, neighbors, event_type = processor.transform_entity(event) + processor.create_entity(vertex, neighbors) + + return vertex, neighbors, processor + + @staticmethod + def _create_event(spec_type=None, sync_mode=None, + event_type=None, properties=None): + # generate event + spec_list = mock_sync.simple_instance_generators(1, 1, 1) + events_list = mock_sync.generate_random_events_list( + spec_list) + + # update properties + if sync_mode is not None: + events_list[0][SyncProps.SYNC_MODE] = sync_mode + + if event_type is not None: + events_list[0][SyncProps.EVENT_TYPE] = event_type + + if properties is not None: + for key, value in properties.iteritems(): + events_list[0][key] = value + + return events_list[0] + + @staticmethod + def _create_alarm(vitrage_id, alarm_type): + return graph_utils.create_vertex( + vitrage_id, + entity_id=vitrage_id, + entity_category=EntityCategory.ALARM, + entity_type=alarm_type, + entity_state='Running', + is_deleted=False, + update_timestamp=utcnow(), + is_placeholder=False, + ) + + def _num_total_expected_vertices(self): + return self.NUM_NODES + self.NUM_ZONES + self.NUM_HOSTS + \ + self.NUM_INSTANCES diff --git a/vitrage/tests/unit/entity_graph/processor/base.py b/vitrage/tests/unit/entity_graph/processor/base.py index 73e3db1f0..cf89ac3b8 100644 --- a/vitrage/tests/unit/entity_graph/processor/base.py +++ b/vitrage/tests/unit/entity_graph/processor/base.py @@ -18,13 +18,14 @@ from vitrage.graph import driver as graph from vitrage.tests import base -class BaseProcessor(base.BaseTest): +class TestBaseProcessor(base.BaseTest): def setUp(self): - super(BaseProcessor, self).setUp() + super(TestBaseProcessor, self).setUp() self.transform = transformer_manager.TransformerManager() - def _update_vertex_to_graph(self, entity_graph, category, type, id, + @staticmethod + def _update_vertex_to_graph(entity_graph, category, type, id, is_deleted, is_placeholder_data, additional_prop): # create vertex properties diff --git a/vitrage/tests/unit/entity_graph/processor/test_entity_graph.py b/vitrage/tests/unit/entity_graph/processor/test_entity_graph.py index e0f4e4dc3..f745f3402 100644 --- a/vitrage/tests/unit/entity_graph/processor/test_entity_graph.py +++ b/vitrage/tests/unit/entity_graph/processor/test_entity_graph.py @@ -17,7 +17,7 @@ from vitrage.entity_graph.processor import entity_graph as entity_g from vitrage.tests.unit.entity_graph.processor import base -class TestEntityGraphManager(base.BaseProcessor): +class TestEntityGraphManager(base.TestBaseProcessor): def setUp(self): super(TestEntityGraphManager, self).setUp() diff --git a/vitrage/tests/unit/entity_graph/processor/test_processor.py b/vitrage/tests/unit/entity_graph/processor/test_processor.py index adf2ed097..6db19ffc8 100644 --- a/vitrage/tests/unit/entity_graph/processor/test_processor.py +++ b/vitrage/tests/unit/entity_graph/processor/test_processor.py @@ -14,16 +14,20 @@ import unittest +from oslo_config import cfg + +from vitrage.common.constants import EventAction from vitrage.common.constants import SynchronizerProperties as SyncProps from vitrage.common.constants import SyncMode -from vitrage.common.constants import VertexProperties +from vitrage.common.constants import VertexProperties as VProps from vitrage.common.datetime_utils import utcnow from vitrage.entity_graph.initialization_status import InitializationStatus from vitrage.entity_graph.processor import processor as proc -from vitrage.tests.unit.entity_graph import TestEntityGraphBase +from vitrage.entity_graph.states.resource_state import ResourceState +from vitrage.tests.unit.entity_graph.base import TestEntityGraphUnitBase -class TestProcessorBase(TestEntityGraphBase): +class TestProcessor(TestEntityGraphUnitBase): ZONE_SPEC = 'ZONE_SPEC' HOST_SPEC = 'HOST_SPEC' @@ -34,27 +38,15 @@ class TestProcessorBase(TestEntityGraphBase): NUM_EDGES_AFTER_DELETION = 0 def setUp(self): - super(TestProcessorBase, self).setUp() - - def test_create_entity_graph(self): - processor = self._create_processor_with_graph() - - # check number of entities - num_vertices = len(processor.entity_graph) - self.assertEqual(self._num_total_expected_vertices(), num_vertices) - - # TODO(Alexey): add this check and to check also the number of edges - # check all entities create a tree and no free floating vertices exists - # it will be done only after we will have zone plugin - # vertex = graph.find_vertex_in_graph() - # bfs_list = graph.algo.bfs(graph) - # self.assertEqual(num_vertices, len(bfs_list)) + super(TestProcessor, self).setUp() + self.conf = cfg.ConfigOpts() + self.conf.register_opts(self.PROCESSOR_OPTS, group='entity_graph') # TODO(Alexey): un skip this test when instance transformer update is ready @unittest.skip('Not ready yet') def test_process_event(self): # check create instance event - processor = proc.Processor(InitializationStatus()) + processor = proc.Processor(self.conf, InitializationStatus()) event = self._create_event(spec_type=self.INSTANCE_SPEC, sync_mode=SyncMode.INIT_SNAPSHOT) processor.process_event(event) @@ -90,18 +82,18 @@ class TestProcessorBase(TestEntityGraphBase): # check added entity vertex = processor.entity_graph.get_vertex(vertex.vertex_id) - self.assertEqual('STARTING', vertex.properties[VertexProperties.STATE]) + self.assertEqual('STARTING', vertex.properties[VProps.STATE]) # update instance event with state running - vertex.properties[VertexProperties.STATE] = 'RUNNING' - vertex.properties[VertexProperties.UPDATE_TIMESTAMP] = str(utcnow()) + vertex.properties[VProps.STATE] = 'RUNNING' + vertex.properties[VProps.UPDATE_TIMESTAMP] = str(utcnow()) processor.update_entity(vertex, neighbors) # check state self._check_graph(processor, self.NUM_VERTICES_AFTER_CREATION, self.NUM_EDGES_AFTER_CREATION) vertex = processor.entity_graph.get_vertex(vertex.vertex_id) - self.assertEqual('RUNNING', vertex.properties[VertexProperties.STATE]) + self.assertEqual('RUNNING', vertex.properties[VProps.STATE]) def test_change_parent(self): # create instance event with host neighbor and check validity @@ -110,7 +102,7 @@ class TestProcessorBase(TestEntityGraphBase): # update instance event with state running (neighbor_vertex, neighbor_edge) = neighbors[0] old_neighbor_id = neighbor_vertex.vertex_id - neighbor_vertex.properties[VertexProperties.ID] = 'newhost-2' + neighbor_vertex.properties[VProps.ID] = 'newhost-2' neighbor_vertex.vertex_id = 'RESOURCE_HOST_newhost-2' neighbor_edge.source_id = 'RESOURCE_HOST_newhost-2' processor.update_entity(vertex, neighbors) @@ -141,7 +133,7 @@ class TestProcessorBase(TestEntityGraphBase): # update instance event with state running (neighbor_vertex, neighbor_edge) = neighbors[0] old_neighbor_id = neighbor_vertex.vertex_id - neighbor_vertex.properties[VertexProperties.ID] = 'newhost-2' + neighbor_vertex.properties[VProps.ID] = 'newhost-2' neighbor_vertex.vertex_id = 'RESOURCE_HOST_newhost-2' neighbor_edge.source_id = 'RESOURCE_HOST_newhost-2' processor._update_neighbors(vertex, neighbors) @@ -171,6 +163,66 @@ class TestProcessorBase(TestEntityGraphBase): self.NUM_VERTICES_AFTER_DELETION, self.NUM_EDGES_AFTER_DELETION) + def test_calculate_aggregated_state(self): + # setup + instances = [] + for i in range(6): + (vertex, neighbors, processor) = self._create_and_check_entity() + instances.append((vertex, processor)) + + # action + # state already exists and its updated + instances[0][0][VProps.STATE] = 'SUSPENDED' + instances[0][1]._calculate_aggregated_state(instances[0][0], + EventAction.UPDATE) + + # vitrage state doesn't exist and its updated + del instances[1][0][VProps.STATE] + instances[1][1].entity_graph.update_vertex(instances[1][0]) + instances[1][0][VProps.VITRAGE_STATE] = 'SUBOPTIMAL' + instances[1][1]._calculate_aggregated_state(instances[1][0], + EventAction.UPDATE) + + # state exists and vitrage state changes + instances[2][0][VProps.VITRAGE_STATE] = 'SUBOPTIMAL' + instances[2][1]._calculate_aggregated_state(instances[2][0], + EventAction.UPDATE) + + # vitrage state exists and state changes + del instances[3][0][VProps.STATE] + instances[3][0][VProps.VITRAGE_STATE] = 'SUBOPTIMAL' + instances[3][1].entity_graph.update_vertex(instances[3][0]) + instances[3][0][VProps.STATE] = 'SUSPENDED' + instances[3][1]._calculate_aggregated_state(instances[3][0], + EventAction.UPDATE) + + # state and vitrage state exists and state changes + instances[4][0][VProps.VITRAGE_STATE] = 'SUBOPTIMAL' + instances[4][1].entity_graph.update_vertex(instances[4][0]) + instances[4][0][VProps.STATE] = 'SUSPENDED' + instances[4][1]._calculate_aggregated_state(instances[4][0], + EventAction.UPDATE) + + # state and vitrage state exists and vitrage state changes + instances[5][0][VProps.VITRAGE_STATE] = 'SUBOPTIMAL' + instances[5][1].entity_graph.update_vertex(instances[5][0]) + instances[5][1]._calculate_aggregated_state(instances[5][0], + EventAction.UPDATE) + + # test assertions + self.assertEqual(ResourceState.SUSPENDED, + instances[0][0][VProps.AGGREGATED_STATE]) + self.assertEqual(ResourceState.SUBOPTIMAL, + instances[1][0][VProps.AGGREGATED_STATE]) + self.assertEqual(ResourceState.SUBOPTIMAL, + instances[2][0][VProps.AGGREGATED_STATE]) + self.assertEqual(ResourceState.SUSPENDED, + instances[3][0][VProps.AGGREGATED_STATE]) + self.assertEqual(ResourceState.SUSPENDED, + instances[4][0][VProps.AGGREGATED_STATE]) + self.assertEqual(ResourceState.SUBOPTIMAL, + instances[5][0][VProps.AGGREGATED_STATE]) + def _create_and_check_entity(self, properties={}): # create instance event with host neighbor (vertex, neighbors, processor) = self._create_entity( @@ -185,23 +237,6 @@ class TestProcessorBase(TestEntityGraphBase): return vertex, neighbors, processor - def _create_entity(self, processor=None, spec_type=None, sync_mode=None, - event_type=None, properties=None): - # create instance event with host neighbor - event = self._create_event(spec_type=spec_type, - sync_mode=sync_mode, - event_type=event_type, - properties=properties) - - # add instance entity with host - if processor is None: - processor = proc.Processor(InitializationStatus()) - - (vertex, neighbors, event_type) = processor.transform_entity(event) - processor.create_entity(vertex, neighbors) - - return vertex, neighbors, processor - def _check_graph(self, processor, num_vertices, num_edges): self.assertEqual(num_vertices, len(processor.entity_graph)) self.assertEqual(num_edges, processor.entity_graph.num_edges()) diff --git a/vitrage/tests/unit/entity_graph/test_state_manager.py b/vitrage/tests/unit/entity_graph/test_state_manager.py new file mode 100644 index 000000000..b48adc7ba --- /dev/null +++ b/vitrage/tests/unit/entity_graph/test_state_manager.py @@ -0,0 +1,122 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from vitrage.entity_graph.states.resource_state import ResourceState +from vitrage.entity_graph.states.state_manager import StateManager +from vitrage.tests import base +from vitrage.tests.mocks import utils + + +class TestStateManager(base.BaseTest): + + OPTS = [ + cfg.StrOpt('states_plugins_dir', + default=utils.get_resources_dir() + '/states_plugins'), + ] + + def setUp(self): + super(TestStateManager, self).setUp() + self.conf = cfg.ConfigOpts() + self.conf.register_opts(self.OPTS, group='entity_graph') + + def test_load_state_plugins(self): + # action + StateManager(self.conf) + + # test assertions + # TODO(Alexey): check that UNRECOGNIZED exists + # TODO(Alexey): check that if UNRECOGNIZED is missing then it throws + # exception + # TODO(Alexey): check that all of the state plugins configured exists + # TODO(Alexey): check that if one of the state plugins is missing then + # it throws an exception + + def test_normalize_state(self): + # setup + state_manager = StateManager(self.conf) + + # action + normalized_state = \ + state_manager.normalize_state('nova.instance', 'REBUILD') + + # test assertions + self.assertEqual(ResourceState.REBUILDING, normalized_state) + + def test_state_priority(self): + # setup + state_manager = StateManager(self.conf) + + # action + state_priority = \ + state_manager.state_priority('nova.instance', + ResourceState.REBUILDING) + + # test assertions + self.assertEqual(60, state_priority) + + def test_aggregated_state_normalized(self): + # setup + state_manager = StateManager(self.conf) + + # action + aggregated_state_nova_instance_1 = state_manager.aggregated_state( + ResourceState.REBUILDING, ResourceState.SUBOPTIMAL, + 'nova.instance', True) + aggregated_state_nova_instance_2 = state_manager.aggregated_state( + ResourceState.SUBOPTIMAL, ResourceState.REBUILDING, + 'nova.instance', True) + + # test assertions + self.assertEqual(ResourceState.REBUILDING, + aggregated_state_nova_instance_1) + self.assertEqual(ResourceState.REBUILDING, + aggregated_state_nova_instance_2) + + def test_aggregated_state_not_normalized(self): + # setup + state_manager = StateManager(self.conf) + + # action + aggregated_state_nova_instance_1 = state_manager.aggregated_state( + 'REBOOT', 'REBUILD', 'nova.instance') + aggregated_state_nova_instance_2 = state_manager.aggregated_state( + 'REBUILD', 'REBOOT', 'nova.instance') + + # test assertions + self.assertEqual(ResourceState.REBUILDING, + aggregated_state_nova_instance_1) + self.assertEqual(ResourceState.REBUILDING, + aggregated_state_nova_instance_2) + + def test_aggregated_state_functionalities(self): + # setup + state_manager = StateManager(self.conf) + + # action + aggregated_state_nova_instance_1 = state_manager.aggregated_state( + 'ACTIVE', None, 'nova.instance') + aggregated_state_nova_instance_2 = state_manager.aggregated_state( + None, 'ACTIVE', 'nova.instance') + aggregated_state_nova_instance_3 = state_manager.aggregated_state( + None, None, 'nova.instance') + + # test assertions + self.assertEqual(ResourceState.RUNNING, + aggregated_state_nova_instance_1) + self.assertEqual(ResourceState.RUNNING, + aggregated_state_nova_instance_2) + self.assertEqual(ResourceState.UNDEFINED, + aggregated_state_nova_instance_3) diff --git a/vitrage_tempest_tests/tests/base_mock.py b/vitrage_tempest_tests/tests/base_mock.py index 09946c0cf..1dcd16f59 100644 --- a/vitrage_tempest_tests/tests/base_mock.py +++ b/vitrage_tempest_tests/tests/base_mock.py @@ -14,17 +14,27 @@ import testtools +from oslo_config import cfg + from vitrage.entity_graph.initialization_status import InitializationStatus from vitrage.entity_graph.processor import processor as proc from vitrage.tests.mocks import mock_syncronizer as mock_sync +from vitrage.tests.mocks import utils class BaseMock(testtools.TestCase): """Base test class for Vitrage API tests.""" + PROCESSOR_OPTS = [ + cfg.StrOpt('states_plugins_dir', + default=utils.get_resources_dir() + '/states_plugins'), + ] + def create_processor_with_graph(self): + self.conf = cfg.ConfigOpts() + self.conf.register_opts(self.PROCESSOR_OPTS, group='entity_graph') events = self._create_mock_events() - processor = proc.Processor(InitializationStatus()) + processor = proc.Processor(self.conf, InitializationStatus()) for event in events: processor.process_event(event)