diff --git a/distil/collector/base.py b/distil/collector/base.py index 6767e02..3a6d1d8 100644 --- a/distil/collector/base.py +++ b/distil/collector/base.py @@ -205,7 +205,9 @@ class BaseCollector(object): service = (mapping['service'] if 'service' in mapping else mapping['meter']) - transformer = d_transformer.get_transformer(mapping['transformer']) + transformer = d_transformer.get_transformer( + mapping['transformer'], + override_config=mapping.get('transformer_config', {})) for res_id, entries in usage_by_resource.items(): transformed = transformer.transform_usage( diff --git a/distil/common/general.py b/distil/common/general.py index e4e4a68..7689276 100644 --- a/distil/common/general.py +++ b/distil/common/general.py @@ -33,7 +33,7 @@ LOG = logging.getLogger(__name__) _TRANS_CONFIG = None -def get_transformer_config(): +def get_transformer_config(name): global _TRANS_CONFIG if not _TRANS_CONFIG: @@ -43,7 +43,7 @@ def get_transformer_config(): except IOError as e: raise e - return _TRANS_CONFIG + return _TRANS_CONFIG.get(name, {}) def get_windows(start, end): diff --git a/distil/common/openstack.py b/distil/common/openstack.py index 0c8dc24..87bc4f4 100644 --- a/distil/common/openstack.py +++ b/distil/common/openstack.py @@ -17,12 +17,14 @@ import re from ceilometerclient import client as ceilometerclient from cinderclient.v2 import client as cinderclient +from cinderclient.exceptions import NotFound as CinderNotFound from glanceclient import client as glanceclient from keystoneauth1.identity import v3 from keystoneauth1.exceptions import NotFound from keystoneauth1 import session from keystoneclient.v3 import client as ks_client from novaclient import client as novaclient +from novaclient.exceptions import NotFound as NovaNotFound from oslo_config import cfg from distil.common import cache as distil_cache @@ -30,7 +32,7 @@ from distil.common import general CONF = cfg.CONF KS_SESSION = None -cache = defaultdict(list) +cache = defaultdict(dict) ROOT_DEVICE_PATTERN = re.compile('^/dev/(x?v|s|h)da1?$') @@ -163,20 +165,43 @@ def get_root_volume(instance_id): @general.disable_ssl_warnings -def get_volume_type(volume_type): - if not cache.get("volume_types"): +def get_flavor_name(flavor_id): + if flavor_id not in cache["flavors"]: + nova = get_nova_client() + try: + flavor_name = nova.flavors.get(flavor_id).name + except NovaNotFound: + return None + cache["flavors"][flavor_id] = flavor_name + return cache["flavors"][flavor_id] + + +@general.disable_ssl_warnings +def get_volume_type_for_volume(volume_id): + if volume_id not in cache["volume_id_to_type"]: cinder = get_cinder_client() - for vtype in cinder.volume_types.list(): - cache['volume_types'].append({'id': vtype.id, 'name': vtype.name}) + try: + vol = cinder.volumes.get(volume_id) + except CinderNotFound: + return None + cache["volume_id_to_type"][volume_id] = vol.volume_type + return cache["volume_id_to_type"][volume_id] - for vtype in cache['volume_types']: - # check name first, as that will be more common - if vtype['name'] == volume_type: - return volume_type - elif vtype['id'] == volume_type: - return vtype['name'] - return None +@general.disable_ssl_warnings +def get_volume_type_name(volume_type): + if volume_type not in cache["volume_types"]: + cinder = get_cinder_client() + try: + vtype = cinder.volume_types.get(volume_type) + except CinderNotFound: + try: + vtype = cinder.volume_types.find(name=volume_type) + except CinderNotFound: + return None + cache["volume_types"][vtype.id] = vtype.name + cache["volume_types"][vtype.name] = vtype.name + return cache["volume_types"][volume_type] @general.disable_ssl_warnings diff --git a/distil/tests/unit/transformer/test_arithmetic.py b/distil/tests/unit/transformer/test_arithmetic.py index 71dcbc5..471012a 100644 --- a/distil/tests/unit/transformer/test_arithmetic.py +++ b/distil/tests/unit/transformer/test_arithmetic.py @@ -20,7 +20,7 @@ from distil.common.constants import date_format from distil.common import general from distil.common import openstack from distil.tests.unit import base -from distil.transformer import arithmetic +from distil.transformer import get_transformer p = lambda t: datetime.datetime.strptime(t, date_format) @@ -53,7 +53,7 @@ class TestMaxTransformer(base.DistilTestCase): {'timestamp': FAKE_DATA.t1.isoformat(), 'volume': 6}, ] - xform = arithmetic.MaxTransformer() + xform = get_transformer('max') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -71,7 +71,7 @@ class TestMaxTransformer(base.DistilTestCase): {'timestamp': FAKE_DATA.t1, 'volume': 25}, ] - xform = arithmetic.MaxTransformer() + xform = get_transformer('max') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -86,7 +86,7 @@ class TestMaxTransformer(base.DistilTestCase): {'timestamp': FAKE_DATA.t0, 'volume': None}, ] - xform = arithmetic.MaxTransformer() + xform = get_transformer('max') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -103,7 +103,7 @@ class TestMaxTransformer(base.DistilTestCase): {'timestamp': FAKE_DATA.t1, 'volume': 27}, ] - xform = arithmetic.MaxTransformer() + xform = get_transformer('max') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -135,7 +135,7 @@ class TestBlockStorageMaxTransformer(base.DistilTestCase): 'metadata': {}}, ] - xform = arithmetic.BlockStorageMaxTransformer() + xform = get_transformer('storagemax') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -156,7 +156,7 @@ class TestBlockStorageMaxTransformer(base.DistilTestCase): 'metadata': {}}, ] - xform = arithmetic.BlockStorageMaxTransformer() + xform = get_transformer('storagemax') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -172,7 +172,7 @@ class TestBlockStorageMaxTransformer(base.DistilTestCase): 'metadata': {}}, ] - xform = arithmetic.BlockStorageMaxTransformer() + xform = get_transformer('storagemax') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -192,7 +192,7 @@ class TestBlockStorageMaxTransformer(base.DistilTestCase): 'metadata': {}}, ] - xform = arithmetic.BlockStorageMaxTransformer() + xform = get_transformer('storagemax') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -241,7 +241,7 @@ class TestObjectStorageMaxTransformer(base.DistilTestCase): 'metadata': {}}, ] - xform = arithmetic.ObjectStorageMaxTransformer() + xform = get_transformer('objectstoragemax') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -270,7 +270,7 @@ class TestObjectStorageMaxTransformer(base.DistilTestCase): 'metadata': {}}, ] - xform = arithmetic.ObjectStorageMaxTransformer() + xform = get_transformer('objectstoragemax') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -290,7 +290,7 @@ class TestObjectStorageMaxTransformer(base.DistilTestCase): 'metadata': {}}, ] - xform = arithmetic.ObjectStorageMaxTransformer() + xform = get_transformer('objectstoragemax') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -318,7 +318,7 @@ class TestObjectStorageMaxTransformer(base.DistilTestCase): 'metadata': {}}, ] - xform = arithmetic.ObjectStorageMaxTransformer() + xform = get_transformer('objectstoragemax') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -339,7 +339,7 @@ class TestSumTransformer(base.DistilTestCase): {'timestamp': '2014-01-01T01:00:00', 'volume': 1}, ] - xform = arithmetic.SumTransformer() + xform = get_transformer('sum') usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -354,7 +354,7 @@ class TestSumTransformer(base.DistilTestCase): {'timestamp': FAKE_DATA.t0.isoformat(), 'volume': None}, ] - xform = arithmetic.SumTransformer() + xform = get_transformer('sum') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -371,8 +371,58 @@ class TestSumTransformer(base.DistilTestCase): {'timestamp': FAKE_DATA.t0_50.isoformat(), 'volume': 25}, ] - xform = arithmetic.SumTransformer() + xform = get_transformer('sum') usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) self.assertEqual({'some_meter': 50}, usage) + + +@mock.patch.object(general, 'get_transformer_config', mock.Mock()) +class TestDatabaseVolumeMaxTransformer(base.DistilTestCase): + + @mock.patch.object( + openstack, 'get_volume_type_for_volume', + mock.Mock(return_value='b1.nvme1000')) + def test_all_different_values(self): + """ + Tests that the transformer correctly grabs the highest value, + when all values are different. + """ + + data = [ + {'timestamp': FAKE_DATA.t0, 'volume': 1, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {'volume.size': '24', 'volume_id': 'vol_id'}}, + {'timestamp': FAKE_DATA.t0_10, 'volume': 1, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {'volume.size': '13', 'volume_id': 'vol_id'}}, + {'timestamp': FAKE_DATA.t0_20, 'volume': 1, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {'volume.size': '7', 'volume_id': 'vol_id'}}, + {'timestamp': FAKE_DATA.t0_30, 'volume': 1, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {'volume.size': '13', 'volume_id': 'vol_id'}}, + {'timestamp': FAKE_DATA.t0_40, 'volume': 1, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {'volume.size': '3', 'volume_id': 'vol_id'}}, + {'timestamp': FAKE_DATA.t0_50, 'volume': 1, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {'volume.size': '25', 'volume_id': 'vol_id'}}, + {'timestamp': FAKE_DATA.t1, 'volume': 1, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {'volume.size': '13', 'volume_id': 'vol_id'}}, + ] + + xform = get_transformer('databasevolumemax') + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'b1.nvme1000': 25}, usage) diff --git a/distil/tests/unit/transformer/test_conversion.py b/distil/tests/unit/transformer/test_conversion.py index 798665d..76a3f66 100644 --- a/distil/tests/unit/transformer/test_conversion.py +++ b/distil/tests/unit/transformer/test_conversion.py @@ -17,8 +17,9 @@ import mock from distil.common.constants import date_format from distil.common import general +from distil.common import openstack from distil.tests.unit import base -from distil.transformer import conversion +from distil.transformer import get_transformer p = lambda t: datetime.datetime.strptime(t, date_format) @@ -41,17 +42,28 @@ FAKE_CONFIG = { "tracked_states": ["active", "building", "paused", "rescued", "resized"] }, - "from_image": { + "fromimage": { "service": "volume.size", "md_keys": ["image_ref", "image_meta.base_image_ref"], "none_values": ["None", ""], "size_keys": ["root_gb"] + }, + "databaseuptime": { + "tracked_states": ["ACTIVE", "BUILD"] + }, + "databasemanagementuptime": { + "tracked_states": ["ACTIVE", "BUILD"], + "service_name": "d1.managment" } } +def fake_get_transformer_config(name): + return FAKE_CONFIG.get(name, {}) + + @mock.patch.object(general, 'get_transformer_config', - mock.Mock(return_value=FAKE_CONFIG)) + fake_get_transformer_config) class TestUpTimeTransformer(base.DistilTestCase): def test_trivial_run(self): """ @@ -59,7 +71,7 @@ class TestUpTimeTransformer(base.DistilTestCase): """ state = [] - xform = conversion.UpTimeTransformer() + xform = get_transformer('uptime') result = xform.transform_usage('state', state, FAKE_DATA.t0, FAKE_DATA.t1) @@ -79,7 +91,7 @@ class TestUpTimeTransformer(base.DistilTestCase): 'status': 'active'}} ] - xform = conversion.UpTimeTransformer() + xform = get_transformer('uptime') result = xform.transform_usage('state', state, FAKE_DATA.t0, FAKE_DATA.t1) @@ -100,7 +112,7 @@ class TestUpTimeTransformer(base.DistilTestCase): 'status': 'stopped'}} ] - xform = conversion.UpTimeTransformer() + xform = get_transformer('uptime') result = xform.transform_usage('state', state, FAKE_DATA.t0, FAKE_DATA.t1) @@ -122,7 +134,7 @@ class TestUpTimeTransformer(base.DistilTestCase): 'status': 'stopped'}} ] - xform = conversion.UpTimeTransformer() + xform = get_transformer('uptime') result = xform.transform_usage('state', state, FAKE_DATA.t0, FAKE_DATA.t1) @@ -145,7 +157,7 @@ class TestUpTimeTransformer(base.DistilTestCase): 'status': 'active'}} ] - xform = conversion.UpTimeTransformer() + xform = get_transformer('uptime') result = xform.transform_usage('state', state, FAKE_DATA.t0, FAKE_DATA.t1) @@ -168,7 +180,7 @@ class TestUpTimeTransformer(base.DistilTestCase): 'status': 'active'}} ] - xform = conversion.UpTimeTransformer() + xform = get_transformer('uptime') result = xform.transform_usage('state', state, FAKE_DATA.t0, FAKE_DATA.t1) @@ -192,7 +204,7 @@ class TestUpTimeTransformer(base.DistilTestCase): 'status': 'active'}} ] - xform = conversion.UpTimeTransformer() + xform = get_transformer('uptime') result = xform.transform_usage('state', state, FAKE_DATA.t0, FAKE_DATA.t1) @@ -212,7 +224,7 @@ class TestUpTimeTransformer(base.DistilTestCase): 'state': 'active'}} ] - xform = conversion.UpTimeTransformer() + xform = get_transformer('uptime') result = xform.transform_usage('state', state, FAKE_DATA.t0, FAKE_DATA.t1) @@ -230,7 +242,7 @@ class TestUpTimeTransformer(base.DistilTestCase): 'metadata': {'instance_type': FAKE_DATA.flavor}} ] - xform = conversion.UpTimeTransformer() + xform = get_transformer('uptime') result = xform.transform_usage('state', state, FAKE_DATA.t0, FAKE_DATA.t1) @@ -252,7 +264,7 @@ class TestUpTimeTransformer(base.DistilTestCase): 'status': 'active'}} ] - xform = conversion.UpTimeTransformer() + xform = get_transformer('uptime') result = xform.transform_usage( 'instance', entries, @@ -265,7 +277,7 @@ class TestUpTimeTransformer(base.DistilTestCase): @mock.patch.object(general, 'get_transformer_config', - mock.Mock(return_value=FAKE_CONFIG)) + fake_get_transformer_config) class TestFromImageTransformer(base.DistilTestCase): """ These tests rely on config settings for from_image, @@ -290,7 +302,7 @@ class TestFromImageTransformer(base.DistilTestCase): 'metadata': {'image_ref': "None"}} ] - xform = conversion.FromImageTransformer() + xform = get_transformer('fromimage') usage = xform.transform_usage('instance', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -313,7 +325,7 @@ class TestFromImageTransformer(base.DistilTestCase): 'metadata': {'image_ref': "None"}} ] - xform = conversion.FromImageTransformer() + xform = get_transformer('fromimage') usage = xform.transform_usage('instance', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -335,7 +347,7 @@ class TestFromImageTransformer(base.DistilTestCase): 'root_gb': "20"}} ] - xform = conversion.FromImageTransformer() + xform = get_transformer('fromimage') usage = xform.transform_usage('instance', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -358,7 +370,7 @@ class TestFromImageTransformer(base.DistilTestCase): 'root_gb': "20"}} ] - xform = conversion.FromImageTransformer() + xform = get_transformer('fromimage') usage = xform.transform_usage('instance', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -366,7 +378,7 @@ class TestFromImageTransformer(base.DistilTestCase): @mock.patch.object(general, 'get_transformer_config', - mock.Mock(return_value=FAKE_CONFIG)) + fake_get_transformer_config) class TestNetworkServiceTransformer(base.DistilTestCase): def test_basic_sum(self): """Tests that the transformer correctly calculate the sum value. @@ -378,7 +390,7 @@ class TestNetworkServiceTransformer(base.DistilTestCase): {'timestamp': '2014-01-01T01:00:00', 'volume': 2}, ] - xform = conversion.NetworkServiceTransformer() + xform = get_transformer('networkservice') usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -394,7 +406,7 @@ class TestNetworkServiceTransformer(base.DistilTestCase): {'timestamp': '2014-01-01T01:00:00', 'volume': 2}, ] - xform = conversion.NetworkServiceTransformer() + xform = get_transformer('networkservice') usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -413,7 +425,7 @@ class TestMagnumTransformer(base.DistilTestCase): {'timestamp': '2014-01-01T01:00:00', 'volume': 2}, ] - xform = conversion.MagnumTransformer() + xform = get_transformer('magnum') usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -429,8 +441,213 @@ class TestMagnumTransformer(base.DistilTestCase): {'timestamp': '2014-01-01T01:00:00', 'volume': 18}, ] - xform = conversion.MagnumTransformer() + xform = get_transformer('magnum') usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) self.assertEqual({'fake_meter': 0}, usage) + + +@mock.patch.object(general, 'get_transformer_config', + fake_get_transformer_config) +class TestDatabaseUpTimeTransformer(base.DistilTestCase): + + @mock.patch.object( + openstack, 'get_flavor_name', + mock.Mock(return_value=FAKE_DATA.flavor)) + def test_online_constant_flavor(self): + """ + Test that a machine online for a 1h period with constant + flavor works and gives 1h of uptime. + """ + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'ACTIVE'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'ACTIVE'}} + ] + + xform = get_transformer('databaseuptime') + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({FAKE_DATA.flavor: 3600}, result) + + @mock.patch.object( + openstack, 'get_flavor_name', + mock.Mock(return_value=FAKE_DATA.flavor)) + def test_offline_constant_flavor(self): + """ + Test that a machine offline for a 1h period with constant flavor + works and gives zero uptime. + """ + + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'stopped'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'stopped'}} + ] + + xform = get_transformer('databaseuptime') + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({}, result) + + @mock.patch.object( + openstack, 'get_flavor_name', + mock.Mock(return_value=FAKE_DATA.flavor)) + def test_shutdown_during_period(self): + """ + Test that a machine run for 0.5 then shutdown gives 0.5h uptime. + """ + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'ACTIVE'}}, + {'timestamp': FAKE_DATA.t0_30.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'stopped'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'stopped'}} + ] + + xform = get_transformer('databaseuptime') + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({FAKE_DATA.flavor: 1800}, result) + + def test_online_flavor_change(self): + """ + Test that a machine run for 0.5h as m1.tiny, resized to m1.large, + and run for a further 0.5 yields 0.5h of uptime in each class. + """ + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'ACTIVE'}}, + {'timestamp': FAKE_DATA.t0_30.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor2, + 'status': 'ACTIVE'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor2, + 'status': 'ACTIVE'}} + ] + + xform = get_transformer('databaseuptime') + + def fake_get_flavor(name): + return name + + with mock.patch.object( + openstack, 'get_flavor_name', fake_get_flavor): + result = xform.transform_usage( + 'state', state, FAKE_DATA.t0, FAKE_DATA.t1) + + self.assertDictEqual( + {FAKE_DATA.flavor: 1800, FAKE_DATA.flavor2: 1800}, + result + ) + + @mock.patch.object( + openstack, 'get_flavor_name', + mock.Mock(return_value=FAKE_DATA.flavor)) + def test_no_state_in_metedata(self): + """ + Test that the transformer doesn't fall over if there isn't one of + the two state/status key options in the metadata. + """ + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor}} + ] + + xform = get_transformer('databaseuptime') + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({}, result) + + +@mock.patch.object(general, 'get_transformer_config', + fake_get_transformer_config) +class TestDatabaseManagementUpTimeTransformer(base.DistilTestCase): + + @mock.patch.object( + openstack, 'get_flavor_name', + mock.Mock(return_value=FAKE_DATA.flavor)) + def test_online_constant_flavor(self): + """ + Test that a machine online for a 1h period with constant + flavor works and gives 1h of uptime. + """ + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'status': 'ACTIVE'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'status': 'ACTIVE'}} + ] + + xform = get_transformer('databasemanagementuptime') + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({"d1.managment": 3600}, result) + + @mock.patch.object( + openstack, 'get_flavor_name', + mock.Mock(return_value=FAKE_DATA.flavor)) + def test_offline_constant_flavor(self): + """ + Test that a machine offline for a 1h period with constant flavor + works and gives zero uptime. + """ + + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'stopped'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'stopped'}} + ] + + xform = get_transformer('databasemanagementuptime') + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({}, result) + + @mock.patch.object( + openstack, 'get_flavor_name', + mock.Mock(return_value=FAKE_DATA.flavor)) + def test_shutdown_during_period(self): + """ + Test that a machine run for 0.5 then shutdown gives 0.5h uptime. + """ + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'ACTIVE'}}, + {'timestamp': FAKE_DATA.t0_30.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'stopped'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'flavor.id': FAKE_DATA.flavor, + 'status': 'stopped'}} + ] + + xform = get_transformer('databasemanagementuptime') + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({"d1.managment": 1800}, result) diff --git a/distil/transformer/__init__.py b/distil/transformer/__init__.py index 9dfe134..4686166 100644 --- a/distil/transformer/__init__.py +++ b/distil/transformer/__init__.py @@ -20,8 +20,10 @@ from distil.common import general class BaseTransformer(object): - def __init__(self, *args, **kwargs): - self.config = general.get_transformer_config() + def __init__(self, name, override_config=None): + self.config = general.get_transformer_config(name) + if override_config: + self.config.update(override_config) def transform_usage(self, meter_name, raw_data, start_at, end_at): return self._transform_usage(meter_name, raw_data, start_at, end_at) @@ -35,5 +37,6 @@ def get_transformer(name, **kwargs): 'distil.transformer', name, invoke_on_load=True, + invoke_args=(name,), invoke_kwds=kwargs ).driver diff --git a/distil/transformer/arithmetic.py b/distil/transformer/arithmetic.py index 11c40bf..f210ef9 100644 --- a/distil/transformer/arithmetic.py +++ b/distil/transformer/arithmetic.py @@ -59,7 +59,7 @@ class BlockStorageMaxTransformer(MaxTransformer): if "volume_type" in data[-1]['metadata']: vtype = data[-1]['metadata']['volume_type'] - service = openstack.get_volume_type(vtype) + service = openstack.get_volume_type_name(vtype) if not service: service = name else: @@ -69,6 +69,30 @@ class BlockStorageMaxTransformer(MaxTransformer): return {service: max_vol * hours} +class DatabaseVolumeMaxTransformer(BaseTransformer): + """ + Variantion on the GaugeMax Transformer that checks for + volume_type and uses that as the service, or uses the + default service name. + + It also gets the actual volume size from metadata. + """ + + def _transform_usage(self, name, data, start, end): + if not data: + return None + + max_vol = max([int(v["metadata"]["volume.size"]) for v in data]) + + volume_type = openstack.get_volume_type_for_volume( + data[-1]['metadata']['volume_id']) + if not volume_type: + return None + + hours = (end - start).total_seconds() / 3600.0 + return {volume_type: max_vol * hours} + + class ObjectStorageMaxTransformer(MaxTransformer): """ Variantion on the GaugeMax Transformer that checks for diff --git a/distil/transformer/conversion.py b/distil/transformer/conversion.py index 866a034..9863d9b 100644 --- a/distil/transformer/conversion.py +++ b/distil/transformer/conversion.py @@ -40,7 +40,7 @@ class UpTimeTransformer(BaseTransformer): def _transform_usage(self, name, data, start, end): # get tracked states from config - tracked = self.config['uptime']['tracked_states'] + tracked = self.config['tracked_states'] usage_dict = {} @@ -118,10 +118,10 @@ class FromImageTransformer(BaseTransformer): """ def _transform_usage(self, name, data, start, end): - checks = self.config['from_image']['md_keys'] - none_values = self.config['from_image']['none_values'] - service = self.config['from_image']['service'] - size_sources = self.config['from_image']['size_keys'] + checks = self.config['md_keys'] + none_values = self.config['none_values'] + service = self.config['service'] + size_sources = self.config['size_keys'] size = 0 for entry in data: @@ -183,3 +183,54 @@ class MagnumTransformer(BaseTransformer): hours = (end - start).total_seconds() / 3600.0 return {name: max_vol * hours} + +class DatabaseUpTimeTransformer(UpTimeTransformer): + """ + Transformer to calculate uptime based on states, + which is broken apart into flavor at point in time. + """ + + def _clean_entry(self, entry): + try: + timestamp = datetime.strptime( + entry['timestamp'], constants.date_format) + except ValueError: + timestamp = datetime.strptime( + entry['timestamp'], constants.date_format_f) + + flavor = openstack.get_flavor_name( + entry['metadata'].get('flavor.id')) + + result = { + 'status': entry['metadata'].get('status'), + 'flavor': flavor, + 'timestamp': timestamp + } + + return result + + +class DatabaseManagementUpTimeTransformer(UpTimeTransformer): + """ + Transformer to calculate uptime based on states, + which is broken apart into flavor at point in time. + """ + + def _clean_entry(self, entry): + management_service_name = self.config.get( + 'service_name', 'd1.management') + + try: + timestamp = datetime.strptime( + entry['timestamp'], constants.date_format) + except ValueError: + timestamp = datetime.strptime( + entry['timestamp'], constants.date_format_f) + + result = { + 'status': entry['metadata'].get('status'), + 'flavor': management_service_name, + 'timestamp': timestamp + } + + return result diff --git a/etc/meter_mappings.yml.sample b/etc/meter_mappings.yml.sample index be646da..04addea 100644 --- a/etc/meter_mappings.yml.sample +++ b/etc/meter_mappings.yml.sample @@ -253,3 +253,48 @@ sources: - name +- + meter: database.instance + type: Database Instance + transformer: databaseuptime + unit: second + metadata: + name: + sources: + - name + datastore: + sources: + - datastore + +- + meter: database.instance + service: b1.standard + type: Database Volume + transformer: databasevolumemax + unit: gigabyte + res_id_template: "%s-volume" + metadata: + name: + sources: + - name + template: "%s - volume" + datastore: + sources: + - datastore + +- + meter: database.instance + type: Database Management + transformer: databasemanagementuptime + transformer_config: + service_name: d1.management + unit: second + res_id_template: "%s-management" + metadata: + name: + sources: + - name + template: "%s - Management Fee" + datastore: + sources: + - datastore \ No newline at end of file diff --git a/etc/transformer.yaml.sample b/etc/transformer.yaml.sample index 7b3ed82..4eecf77 100644 --- a/etc/transformer.yaml.sample +++ b/etc/transformer.yaml.sample @@ -11,7 +11,7 @@ uptime: - suspended - shutoff - stopped -from_image: +fromimage: service: b1.standard # What metadata values to check md_keys: @@ -23,3 +23,30 @@ from_image: # where to get volume size from size_keys: - root_gb +databaseuptime: + tracked_states: + - HEALTHY + - ACTIVE + - BLOCKED + - REBOOT + - RESIZE + - BACKUP + - SHUTDOWN + - RESTART_REQUIRED + - PROMOTE + - EJECT + - UPGRADE + - DETACH +databasemanagementuptime: + - HEALTHY + - ACTIVE + - BLOCKED + - REBOOT + - RESIZE + - BACKUP + - SHUTDOWN + - RESTART_REQUIRED + - PROMOTE + - EJECT + - UPGRADE + - DETACH diff --git a/setup.cfg b/setup.cfg index a938af9..145c32b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,6 +46,10 @@ distil.transformer = uptime = distil.transformer.conversion:UpTimeTransformer fromimage = distil.transformer.conversion:FromImageTransformer networkservice = distil.transformer.conversion:NetworkServiceTransformer + magnum = distil.transformer.conversion:MagnumTransformer + databaseuptime = distil.transformer.conversion:DatabaseUpTimeTransformer + databasevolumemax = distil.transformer.arithmetic:DatabaseVolumeMaxTransformer + databasemanagementuptime = distil.transformer.conversion:DatabaseManagementUpTimeTransformer distil.erp = odoo = distil.erp.drivers.odoo:OdooDriver diff --git a/tox.ini b/tox.ini index ed33a85..11b494f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,11 @@ [tox] -envlist = py35,py27,pep8 +envlist = py3,py27,pep8 minversion = 1.6 skipsdist = True [testenv] usedevelop = True -install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt} {opts} {packages} +install_command = pip install -c https://opendev.org/openstack/requirements/raw/branch/stable/ussuri/upper-constraints.txt {opts} {packages} setenv = VIRTUAL_ENV={envdir} DISTIL_TESTS_CONFIGS_DIR={toxinidir}/distil/tests/etc/