From 797240c43638d74c77110ca837ad182e8d9c0bcb Mon Sep 17 00:00:00 2001 From: Liat Har-Tal Date: Tue, 8 Dec 2015 13:29:02 +0000 Subject: [PATCH] nova instance transformer and tests Change-Id: Id3eb11592c36ac196680f6b6d21bf40f1a67aebf --- requirements.txt | 3 +- setup.cfg | 7 +- test-requirements.txt | 2 + vitrage/common/__init__.py | 1 + vitrage/common/constants.py | 41 +++++++++ vitrage/entity_graph/properties.py | 30 ------- vitrage/entity_graph/transformer/base.py | 12 ++- .../transformer/nova_transformer.py | 47 +++++++++++ .../transformer/transformer_manager.py | 29 +++++++ vitrage/graph/driver.py | 2 +- vitrage/tests/mocks/mock_generators.py | 10 +-- vitrage/tests/mocks/mock_syncronizer.py | 2 +- ...nfig_1.txt => mock_nova_inst_snapshot.txt} | 2 +- ...config_2.txt => mock_nova_inst_update.txt} | 4 +- vitrage/tests/unit/base.py | 8 ++ .../test_nova_instance_transformer.py | 26 ------ .../transformers/test_nova_transformers.py | 83 +++++++++++++++++++ vitrage/utils/__init__.py | 1 + 18 files changed, 239 insertions(+), 71 deletions(-) create mode 100644 vitrage/common/__init__.py create mode 100644 vitrage/common/constants.py delete mode 100644 vitrage/entity_graph/properties.py create mode 100644 vitrage/entity_graph/transformer/nova_transformer.py create mode 100644 vitrage/entity_graph/transformer/transformer_manager.py rename vitrage/tests/resources/{mock_nova_instance_snapshot_config_1.txt => mock_nova_inst_snapshot.txt} (97%) rename vitrage/tests/resources/{mock_nova_instance_update_config_2.txt => mock_nova_inst_update.txt} (94%) delete mode 100644 vitrage/tests/unit/transformers/test_nova_instance_transformer.py create mode 100644 vitrage/tests/unit/transformers/test_nova_transformers.py create mode 100644 vitrage/utils/__init__.py diff --git a/requirements.txt b/requirements.txt index f9b69f2b6..2ec8da1aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,9 +6,10 @@ pbr>=1.6 Babel>=1.3 networkx>=1.10 oslo.log>=1.12.0 # Apache-2.0 +oslo.policy>=0.3.0 pecan>=0.8.0 PasteDeploy>=1.5.0 Werkzeug>=0.7 -oslo.policy>=0.3.0 keystonemiddleware>=2.3.0 +stevedore>=1.5.0 # Apache-2.0 exrex>=0.9.4 diff --git a/setup.cfg b/setup.cfg index e791c813e..dd47a01db 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,4 +38,9 @@ build-dir = doc/build all_files = 1 [upload_sphinx] -upload-dir = doc/build/html \ No newline at end of file +upload-dir = doc/build/html + +[entry_points] +vitrage.transformers = + nova.instance = vitrage.entity_graph.transformer.nova_transformer.InstanceTransformer + nova.host = vitrage.entity_graph.transformer.nova_transformer.HostTransformer diff --git a/test-requirements.txt b/test-requirements.txt index a5e24a9a6..ff3a29b53 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -15,3 +15,5 @@ testrepository>=0.0.18 testscenarios>=0.4 testtools>=1.4.0 exrex>=0.9.4 +stevedore>=1.5.0 # Apache-2.0 + diff --git a/vitrage/common/__init__.py b/vitrage/common/__init__.py new file mode 100644 index 000000000..6a20380a1 --- /dev/null +++ b/vitrage/common/__init__.py @@ -0,0 +1 @@ +__author__ = 'stack' diff --git a/vitrage/common/constants.py b/vitrage/common/constants.py new file mode 100644 index 000000000..bf4873f21 --- /dev/null +++ b/vitrage/common/constants.py @@ -0,0 +1,41 @@ +# Copyright 2015 - Alcatel-Lucent +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class VertexConstants(object): + TYPE = 'TYPE' + SUB_TYPE = 'SUB_TYPE' + ID = 'ID' + IS_VERTEX_DELETED = 'IS_VERTEX_DELETED' + VERTEX_DELETION_TIMESTAMP = 'VERTEX_DELETION_TIMESTAMP' + STATE = 'STATE' + PROJECT = 'PROJECT' + UPDATE_TIMESTAMP = 'UPDATE_TIMESTAMP' + + +class EdgeConstants(object): + RELATION_NAME = 'RELATION_NAME' + IS_EDGE_DELETED = 'IS_EDGE_DELETED' + EDGE_DELETION_TIMESTAMP = 'EDGE_DELETION_TIMESTAMP' + + +class EdgeLabels(object): + ON = 'on' + CONTAINS = 'contains' + + +class SynchronizerMessageMode(object): + SNAPSHOT = 'snapshot' + INIT_SNAPSHOT = 'init_snapshot' + UPDATE = 'update' diff --git a/vitrage/entity_graph/properties.py b/vitrage/entity_graph/properties.py deleted file mode 100644 index 1d5e12336..000000000 --- a/vitrage/entity_graph/properties.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2015 - Alcatel-Lucent -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Vertex properties -TYPE = 'TYPE' -SUB_TYPE = 'SUB_TYPE' -ID = 'ID' -VERTEX_ID = 'VERTEX_ID' -EVENT_TYPE = 'EVENT_TYPE' -IS_VERTEX_DELETED = 'IS_VERTEX_DELETED' -VERTEX_DELETION_TIMESTAMP = 'VERTEX_DELETION_TIMESTAMP' -STATE = 'STATE' -PROJECT = 'PROJECT' -TIMESTAMP = 'TIMESTAMP' - -# Edge properties -RELATION_NAME = 'RELATION_NAME' -IS_EDGE_DELETED = 'IS_EDGE_DELETED' -EDGE_DELETION_TIMESTAMP = 'EDGE_DELETION_TIMESTAMP' diff --git a/vitrage/entity_graph/transformer/base.py b/vitrage/entity_graph/transformer/base.py index 3bb6daf01..b0636d01c 100644 --- a/vitrage/entity_graph/transformer/base.py +++ b/vitrage/entity_graph/transformer/base.py @@ -13,12 +13,19 @@ # under the License. import abc +import six +from collections import namedtuple from oslo_log import log as logging LOG = logging.getLogger(__name__) +EntityWrapper = \ + namedtuple('EntityWrapper', ['entity_vertex', 'neighbors'], 'action') + + +@six.add_metaclass(abc.ABCMeta) class Transformer(object): @abc.abstractmethod @@ -28,13 +35,14 @@ class Transformer(object): :return: An EntityWrapper. EntityWrapper is namedTuple that contains an entity vertex and a list of vertex and an edge pair that describe the entity's neighbors. + :rtype: EntityWrapper """ pass @abc.abstractmethod - def get_key_fields(self): + def key_fields(self): pass @abc.abstractmethod - def get_key(self, entity_event): + def extract_key(self, entity_event): pass diff --git a/vitrage/entity_graph/transformer/nova_transformer.py b/vitrage/entity_graph/transformer/nova_transformer.py new file mode 100644 index 000000000..d07ccaef2 --- /dev/null +++ b/vitrage/entity_graph/transformer/nova_transformer.py @@ -0,0 +1,47 @@ +# Copyright 2015 - Alcatel-Lucent +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +from vitrage.common.constants import VertexConstants as vertex_cons +from vitrage.entity_graph.transformer import base + +LOG = logging.getLogger(__name__) + +KEY_SEPARATOR = ':' + +ENTITY_TYPE = 'RESOURCE' +INSTANCE_SUB_TYPE = 'nova.instance' +HOST_SUB_TYPE = 'nova.host' + + +class InstanceTransformer(base.Transformer): + + # # Fields returned from Nova Instance snapshot + ENTITY_ID_DICT = {'snapshot': 'id', + 'init_snapshot': 'id', + 'update': 'instance_id'} + + def transform(self, entity_event): + pass + + def key_fields(self): + return [vertex_cons.TYPE, vertex_cons.SUB_TYPE, vertex_cons.ID] + + def extract_key(self, entity_event): + + sync_mode = entity_event['sync_mode'] + return KEY_SEPARATOR.join( + [ENTITY_TYPE, + INSTANCE_SUB_TYPE, + entity_event[self.ENTITY_ID_DICT[sync_mode]]]) diff --git a/vitrage/entity_graph/transformer/transformer_manager.py b/vitrage/entity_graph/transformer/transformer_manager.py new file mode 100644 index 000000000..0a45de799 --- /dev/null +++ b/vitrage/entity_graph/transformer/transformer_manager.py @@ -0,0 +1,29 @@ +# Copyright 2014 - Mirantis, Inc. +# Copyright 2014 - StackStorm, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_log import log as logging + + +LOG = logging.getLogger(__name__) + + +class TransformerManager(object): + + def __init__(self): + self.transformers = self.register_transformer_classes() + + @staticmethod + def register_transformer_classes(): + pass diff --git a/vitrage/graph/driver.py b/vitrage/graph/driver.py index c7c0b4c9d..3360a2480 100644 --- a/vitrage/graph/driver.py +++ b/vitrage/graph/driver.py @@ -67,7 +67,7 @@ class Edge(object): :param source_id: source vertex id :type source_id: str - :param target_id: target vertex id + :param target_id: target vertex id` :type target_id: str :param label: diff --git a/vitrage/tests/mocks/mock_generators.py b/vitrage/tests/mocks/mock_generators.py index 319f5b773..11ae9d51c 100644 --- a/vitrage/tests/mocks/mock_generators.py +++ b/vitrage/tests/mocks/mock_generators.py @@ -25,14 +25,14 @@ multiple instances of the same entity type. from os.path import curdir from os import walk -from random import randint +import random from entity_model import CommonEntityModel as cem def _get_filename_path(filename): base_dir = None - for i in walk("../%s" % curdir): + for i in walk("../../%s" % curdir): if i[0].find('resources') != -1 and filename in i[2]: base_dir = i[0] break @@ -46,8 +46,6 @@ class MockEventGenerator(object): """Represents a single generator. A generator can generate events for several instances of the same type - - file is expected to be in the ../resources folder """ def __init__(self, filename, instance_num, generator_name='generator'): @@ -75,7 +73,7 @@ class MockEventGenerator(object): param_type = line_params[1].lower() params_dict[param_type][line_params[0]] = line_params[2] except KeyError as ke: - print("Syntax error: {0}".format(ke.message)) + print("Syntax error ({0}): {1}".format(ke.errno, ke.strerror)) def prepare_instance_models(self): """Create the models for all the instances """ @@ -97,7 +95,7 @@ class MockEventGenerator(object): data_stream = [] for _ in xrange(event_num): - model = self.models[randint(0, self.instance_num - 1)] + model = self.models[random.randint(0, self.instance_num - 1)] model.generate_dynamic_params() data_stream.append(model.params) return data_stream diff --git a/vitrage/tests/mocks/mock_syncronizer.py b/vitrage/tests/mocks/mock_syncronizer.py index 406c5aed0..a8a7ce649 100644 --- a/vitrage/tests/mocks/mock_syncronizer.py +++ b/vitrage/tests/mocks/mock_syncronizer.py @@ -20,7 +20,7 @@ of what can be returned usage example: test_entity_spec_list = [ - {'filename': '../resources/mock_nova_instance_config_1.txt', + {'filename': '../resources/mock_nova_inst_snapshot.txt', '#instances': 10, 'name': 'Instance (vm) generator' } diff --git a/vitrage/tests/resources/mock_nova_instance_snapshot_config_1.txt b/vitrage/tests/resources/mock_nova_inst_snapshot.txt similarity index 97% rename from vitrage/tests/resources/mock_nova_instance_snapshot_config_1.txt rename to vitrage/tests/resources/mock_nova_inst_snapshot.txt index 4686a6b77..e0082da7a 100644 --- a/vitrage/tests/resources/mock_nova_instance_snapshot_config_1.txt +++ b/vitrage/tests/resources/mock_nova_inst_snapshot.txt @@ -1,4 +1,4 @@ -event_type S snapshot +sync_mode D snapshot|init_snapshot OS-DCF:diskConfig S AUTO OS-EXT-AZ:availability_zone S nova OS-EXT-SRV-ATTR:host D [a-z]{4}-devstack diff --git a/vitrage/tests/resources/mock_nova_instance_update_config_2.txt b/vitrage/tests/resources/mock_nova_inst_update.txt similarity index 94% rename from vitrage/tests/resources/mock_nova_instance_update_config_2.txt rename to vitrage/tests/resources/mock_nova_inst_update.txt index a9a29eed7..af1045180 100644 --- a/vitrage/tests/resources/mock_nova_instance_update_config_2.txt +++ b/vitrage/tests/resources/mock_nova_inst_update.txt @@ -1,4 +1,4 @@ -event_type S update +sync_mode S update u-OS-DCF:diskConfig S AUTO u-OS-EXT-AZ:availability_zone S nova u-OS-EXT-SRV-ATTR:host D [a-z]{4}-devstack @@ -22,7 +22,7 @@ u-config_drive D True|False u-created D \d{2}:\d{2}:\d{2} \d{2}:\d{2}:\d{2} u-flavor D m1.nano u-hostId D [0-9a-f]{56} -u-id S [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} +instance_id S [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} u-image D cirros-[a-z]+ ([a-z0-9]+) u-key_name D - u-metadata D \{\} diff --git a/vitrage/tests/unit/base.py b/vitrage/tests/unit/base.py index 01b817cd4..73e6acffd 100644 --- a/vitrage/tests/unit/base.py +++ b/vitrage/tests/unit/base.py @@ -13,8 +13,16 @@ # under the License. from oslotest import base +import sys class BaseTest(base.BaseTestCase): """Test case base class for all unit tests.""" + + def assert_list_equal(self, l1, l2): + if tuple(sys.version_info)[0:2] < (2, 7): + # for python 2.6 compatibility + self.assertEqual(l1, l2) + else: + super(BaseTest, self).assertListEqual(l1, l2) diff --git a/vitrage/tests/unit/transformers/test_nova_instance_transformer.py b/vitrage/tests/unit/transformers/test_nova_instance_transformer.py deleted file mode 100644 index 1d2cd46bd..000000000 --- a/vitrage/tests/unit/transformers/test_nova_instance_transformer.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2015 - Alcatel-Lucent -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_log import log as logging -from vitrage.tests.unit import base - -LOG = logging.getLogger(__name__) - - -class TransformNovaInstanceTest(base.BaseTest): - - def test_transform_instance(self): - - LOG.debug('Test transform Nova instance into entity wrapper') - self.assertTrue(True) diff --git a/vitrage/tests/unit/transformers/test_nova_transformers.py b/vitrage/tests/unit/transformers/test_nova_transformers.py new file mode 100644 index 000000000..ffcae9e39 --- /dev/null +++ b/vitrage/tests/unit/transformers/test_nova_transformers.py @@ -0,0 +1,83 @@ +# Copyright 2015 - Alcatel-Lucent +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging + +from vitrage.common.constants import VertexConstants as vertexCons +from vitrage.entity_graph.transformer import nova_transformer as nt +from vitrage.tests.mocks import mock_syncronizer as mock_sync +from vitrage.tests.unit import base + +LOG = logging.getLogger(__name__) + + +def get_nova_instance_transformer(): + return nt.InstanceTransformer() + + +def get_instance_entity_spec_list(config_file_path, number_of_instances): + + """Returns a list of nova instance specifications by + + given specific configuration file. + + :rtype : list + """ + return { + 'filename': config_file_path, + '#instances': number_of_instances, + 'name': 'Instance generator' + } + + +class NovaInstanceTransformerTest(base.BaseTest): + + def test_key_fields(self): + LOG.debug('Test get key fields from nova instance transformer') + transformer = get_nova_instance_transformer() + + expected_key_fields = [vertexCons.TYPE, + vertexCons.SUB_TYPE, + vertexCons.ID] + observed_key_fields = transformer.key_fields() + self.assert_list_equal(expected_key_fields, observed_key_fields) + + def test_extract_key(self): + LOG.debug('Test get key from nova instance transformer') + + transformer = get_nova_instance_transformer() + + instance_specifications = [ + get_instance_entity_spec_list('mock_nova_inst_snapshot.txt', 1), + get_instance_entity_spec_list('mock_nova_inst_update.txt', 1) + ] + + spec_list = mock_sync.get_mock_generators(instance_specifications) + instance_events = mock_sync.generate_random_events_list(spec_list) + + for event in instance_events: + observed_key = transformer.extract_key(event) + observed_key_fields = observed_key.split(nt.KEY_SEPARATOR) + + self.assertEqual(nt.ENTITY_TYPE, observed_key_fields[0]) + self.assertEqual(nt.INSTANCE_SUB_TYPE, observed_key_fields[1]) + + event_id = event[transformer.ENTITY_ID_DICT[event['sync_mode']]] + self.assertEqual(event_id, observed_key_fields[2]) + + expected_key = nt.KEY_SEPARATOR.join( + [nt.ENTITY_TYPE, + nt.INSTANCE_SUB_TYPE, + event_id]) + self.assertEqual(expected_key, observed_key) diff --git a/vitrage/utils/__init__.py b/vitrage/utils/__init__.py new file mode 100644 index 000000000..6a20380a1 --- /dev/null +++ b/vitrage/utils/__init__.py @@ -0,0 +1 @@ +__author__ = 'stack'