diff --git a/distil/collector/base.py b/distil/collector/base.py index 0a3cccc..6767e02 100644 --- a/distil/collector/base.py +++ b/distil/collector/base.py @@ -223,7 +223,7 @@ class BaseCollector(object): # hashing the name only for swift to get a consistent # id for swift billing. Another change will be proposed to # openstack-billing to handle this case as well. - if 'o1.standard' in transformed: + if mapping['type'] == "Object Storage Container": res_id = hashlib.md5(res_id.encode('utf-8')).hexdigest() LOG.debug( diff --git a/distil/common/openstack.py b/distil/common/openstack.py index f72ca95..16102b1 100644 --- a/distil/common/openstack.py +++ b/distil/common/openstack.py @@ -150,3 +150,27 @@ def get_volume_type(volume_type): return vtype['name'] return None + + +@general.disable_ssl_warnings +def get_object_storage_url(project_id): + ks = get_keystone_client() + try: + endpoint = ks.endpoints.list( + service=ks.services.list(type="object-store")[0], + interface="public", + region=CONF.keystone_authtoken.region_name)[0] + return endpoint.url % {'tenant_id': project_id} + except KeyError: + return None + + +@general.disable_ssl_warnings +def get_container_policy(project_id, container_name): + sess = _get_keystone_session() + url = get_object_storage_url(project_id) + if url: + resp = sess.head("%s/%s" % (url, container_name)) + if resp: + return resp.headers.get('X-Storage-Policy') + return None diff --git a/distil/common/wsgi.py b/distil/common/wsgi.py index 03244e3..9c02509 100644 --- a/distil/common/wsgi.py +++ b/distil/common/wsgi.py @@ -19,6 +19,8 @@ """Utility methods for working with WSGI servers.""" import datetime +import six + from xml.dom import minidom from xml.parsers import expat @@ -61,7 +63,7 @@ class JSONDictSerializer(DictSerializer): if isinstance(obj, datetime.datetime): _dtime = obj - datetime.timedelta(microseconds=obj.microsecond) return _dtime.isoformat() - return unicode(obj) + return six.text_type(obj) return jsonutils.dumps(data, default=sanitizer) diff --git a/distil/tests/unit/transformer/test_arithmetic.py b/distil/tests/unit/transformer/test_arithmetic.py index 18ead90..71dcbc5 100644 --- a/distil/tests/unit/transformer/test_arithmetic.py +++ b/distil/tests/unit/transformer/test_arithmetic.py @@ -18,6 +18,7 @@ import mock from distil.common.constants import date_format from distil.common import general +from distil.common import openstack from distil.tests.unit import base from distil.transformer import arithmetic @@ -110,7 +111,7 @@ class TestMaxTransformer(base.DistilTestCase): @mock.patch.object(general, 'get_transformer_config', mock.Mock()) -class TestStorageMaxTransformer(base.DistilTestCase): +class TestBlockStorageMaxTransformer(base.DistilTestCase): def test_all_different_values(self): """ Tests that the transformer correctly grabs the highest value, @@ -134,7 +135,7 @@ class TestStorageMaxTransformer(base.DistilTestCase): 'metadata': {}}, ] - xform = arithmetic.StorageMaxTransformer() + xform = arithmetic.BlockStorageMaxTransformer() usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -155,7 +156,7 @@ class TestStorageMaxTransformer(base.DistilTestCase): 'metadata': {}}, ] - xform = arithmetic.StorageMaxTransformer() + xform = arithmetic.BlockStorageMaxTransformer() usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -171,7 +172,7 @@ class TestStorageMaxTransformer(base.DistilTestCase): 'metadata': {}}, ] - xform = arithmetic.StorageMaxTransformer() + xform = arithmetic.BlockStorageMaxTransformer() usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) @@ -191,13 +192,140 @@ class TestStorageMaxTransformer(base.DistilTestCase): 'metadata': {}}, ] - xform = arithmetic.StorageMaxTransformer() + xform = arithmetic.BlockStorageMaxTransformer() usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, FAKE_DATA.t1) self.assertEqual({'some_meter': 27}, usage) +@mock.patch.object(general, 'get_transformer_config', mock.Mock()) +class TestObjectStorageMaxTransformer(base.DistilTestCase): + + @mock.patch.object( + openstack, 'get_container_policy', mock.Mock(return_value='test-policy')) + def test_all_different_values(self): + """ + Tests that the transformer correctly grabs the highest value, + when all values are different. + """ + + data = [ + {'timestamp': FAKE_DATA.t0, 'volume': 12, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_10, 'volume': 3, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_20, 'volume': 7, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_30, 'volume': 3, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_40, 'volume': 25, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_50, 'volume': 2, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + {'timestamp': FAKE_DATA.t1, 'volume': 6, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + ] + + xform = arithmetic.ObjectStorageMaxTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'test-policy': 25}, usage) + + @mock.patch.object( + openstack, 'get_container_policy', mock.Mock(return_value='test-policy')) + def test_all_same_values(self): + """ + Tests that that transformer correctly grabs any value, + when all values are the same. + """ + + data = [ + {'timestamp': FAKE_DATA.t0, 'volume': 25, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_30, 'volume': 25, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + {'timestamp': FAKE_DATA.t1, 'volume': 25, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + ] + + xform = arithmetic.ObjectStorageMaxTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'test-policy': 25}, usage) + + @mock.patch.object( + openstack, 'get_container_policy', mock.Mock(return_value='test-policy')) + def test_none_value(self): + """ + Tests that that transformer correctly handles a None value. + """ + + data = [ + {'timestamp': FAKE_DATA.t0, 'volume': None, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + ] + + xform = arithmetic.ObjectStorageMaxTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'test-policy': 0}, usage) + + @mock.patch.object( + openstack, 'get_container_policy', mock.Mock(return_value=None)) + def test_none_and_other_values(self): + """ + Tests that that transformer correctly handles a None value. + """ + + data = [ + {'timestamp': FAKE_DATA.t0, 'volume': None, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_30, 'volume': 25, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + {'timestamp': FAKE_DATA.t1, 'volume': 27, + 'resource_id': '55d37509be3142de963caf82a9c7c447/stuff', + 'project_id': '55d37509be3142de963caf82a9c7c447', + 'metadata': {}}, + ] + + xform = arithmetic.ObjectStorageMaxTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'some_meter': 27}, usage) + + +@mock.patch.object(general, 'get_transformer_config', mock.Mock()) @mock.patch.object(general, 'get_transformer_config', mock.Mock()) class TestSumTransformer(base.DistilTestCase): def test_basic_sum(self): diff --git a/distil/transformer/arithmetic.py b/distil/transformer/arithmetic.py index f9167eb..11c40bf 100644 --- a/distil/transformer/arithmetic.py +++ b/distil/transformer/arithmetic.py @@ -29,17 +29,22 @@ class MaxTransformer(BaseTransformer): 'gigabyte-hours'. """ + def _get_max_vol(self, data): + if len(data): + max_vol = max([(v["volume"] if v["volume"] else 0) for v in data]) + if max_vol: + return max_vol + return 0 + def _transform_usage(self, meter_name, raw_data, start_at, end_at): - max_vol = max([(v["volume"] if v["volume"] else 0) - for v in raw_data]) if len(raw_data) else 0 - max_vol = max_vol if max_vol else 0 + max_vol = self._get_max_vol(raw_data) hours = (end_at - start_at).total_seconds() / 3600.0 return {meter_name: max_vol * hours} -class StorageMaxTransformer(BaseTransformer): +class BlockStorageMaxTransformer(MaxTransformer): """ Variantion on the GaugeMax Transformer that checks for volume_type and uses that as the service, or uses the @@ -50,8 +55,7 @@ class StorageMaxTransformer(BaseTransformer): if not data: return None - max_vol = max([(v["volume"] if v["volume"] else 0) - for v in data]) or 0 + max_vol = self._get_max_vol(data) if "volume_type" in data[-1]['metadata']: vtype = data[-1]['metadata']['volume_type'] @@ -65,6 +69,30 @@ class StorageMaxTransformer(BaseTransformer): return {service: max_vol * hours} +class ObjectStorageMaxTransformer(MaxTransformer): + """ + Variantion on the GaugeMax Transformer that checks for + object storage container policy and uses that as the service, + or uses the default service name. + """ + + def _transform_usage(self, name, data, start, end): + if not data: + return None + + container_name = data[-1]['resource_id'].split('/')[1] + project_id = data[-1]['project_id'] + + service = openstack.get_container_policy(project_id, container_name) + if not service: + service = name + + max_vol = self._get_max_vol(data) + + hours = (end - start).total_seconds() / 3600.0 + return {service: max_vol * hours} + + class SumTransformer(BaseTransformer): """Transformer for sum-integration of a gauge value for given period. """ diff --git a/etc/meter_mappings.yml.sample b/etc/meter_mappings.yml.sample index 32ab119..be646da 100644 --- a/etc/meter_mappings.yml.sample +++ b/etc/meter_mappings.yml.sample @@ -35,7 +35,7 @@ meter: volume.size service: b1.standard type: Volume - transformer: max + transformer: blockstoragemax unit: gigabyte metadata: name: @@ -152,7 +152,7 @@ meter: storage.containers.objects.size service: o1.standard type: Object Storage Container - transformer: max + transformer: objectstoragemax unit: byte # NOTE(flwang): Nothing in resource_metadata from ceilometer actually. # But to avoid any unnecessary issue and keeping consistency. Just keep diff --git a/setup.cfg b/setup.cfg index 5ed374f..a938af9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,7 +39,9 @@ distil.collector = distil.transformer = max = distil.transformer.arithmetic:MaxTransformer - storagemax = distil.transformer.arithmetic:StorageMaxTransformer + storagemax = distil.transformer.arithmetic:BlockStorageMaxTransformer + blockstoragemax = distil.transformer.arithmetic:BlockStorageMaxTransformer + objectstoragemax = distil.transformer.arithmetic:ObjectStorageMaxTransformer sum = distil.transformer.arithmetic:SumTransformer uptime = distil.transformer.conversion:UpTimeTransformer fromimage = distil.transformer.conversion:FromImageTransformer