x/distil
Code Issues Proposed changes

Add support for object storage policies

Browse Source 
Introduce a new ObjectStorageMax transformer which
takes a container and bases the service for it on the
policy used for it in swift so that we can charge differently
for each policy.

Change-Id: Id1a0d7783ae72de0a2da36245badbd770508be81
This commit is contained in:
Adrian Turjak 2019-11-06 17:57:12 +13:00
parent 33c5f067ba
commit 226ceacf90
7 changed files with 200 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
distil/collector/base.py
View File

@ -223,7 +223,7 @@ class BaseCollector(object):
# hashing the name only for swift to get a consistent
# id for swift billing. Another change will be proposed to
# openstack-billing to handle this case as well.
if 'o1.standard' in transformed:
if mapping['type'] == "Object Storage Container":
res_id = hashlib.md5(res_id.encode('utf-8')).hexdigest()
LOG.debug(

24
distil/common/openstack.py
View File

@ -150,3 +150,27 @@ def get_volume_type(volume_type):
return vtype['name']
return None
@general.disable_ssl_warnings
def get_object_storage_url(project_id):
ks = get_keystone_client()
try:
endpoint = ks.endpoints.list(
service=ks.services.list(type="object-store")[0],
interface="public",
region=CONF.keystone_authtoken.region_name)[0]
return endpoint.url % {'tenant_id': project_id}
except KeyError:
return None
@general.disable_ssl_warnings
def get_container_policy(project_id, container_name):
sess = _get_keystone_session()
url = get_object_storage_url(project_id)
if url:
resp = sess.head("%s/%s" % (url, container_name))
if resp:
return resp.headers.get('X-Storage-Policy')
return None

4
distil/common/wsgi.py
View File

@ -19,6 +19,8 @@
"""Utility methods for working with WSGI servers."""
import datetime
import six
from xml.dom import minidom
from xml.parsers import expat
@ -61,7 +63,7 @@ class JSONDictSerializer(DictSerializer):
if isinstance(obj, datetime.datetime):
_dtime = obj - datetime.timedelta(microseconds=obj.microsecond)
return _dtime.isoformat()
return unicode(obj)
return six.text_type(obj)
return jsonutils.dumps(data, default=sanitizer)

138
distil/tests/unit/transformer/test_arithmetic.py
View File

@ -18,6 +18,7 @@ import mock
from distil.common.constants import date_format
from distil.common import general
from distil.common import openstack
from distil.tests.unit import base
from distil.transformer import arithmetic
@ -110,7 +111,7 @@ class TestMaxTransformer(base.DistilTestCase):
@mock.patch.object(general, 'get_transformer_config', mock.Mock())
class TestStorageMaxTransformer(base.DistilTestCase):
class TestBlockStorageMaxTransformer(base.DistilTestCase):
def test_all_different_values(self):
"""
Tests that the transformer correctly grabs the highest value,
@ -134,7 +135,7 @@ class TestStorageMaxTransformer(base.DistilTestCase):
'metadata': {}},
]
xform = arithmetic.StorageMaxTransformer()
xform = arithmetic.BlockStorageMaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -155,7 +156,7 @@ class TestStorageMaxTransformer(base.DistilTestCase):
'metadata': {}},
]
xform = arithmetic.StorageMaxTransformer()
xform = arithmetic.BlockStorageMaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -171,7 +172,7 @@ class TestStorageMaxTransformer(base.DistilTestCase):
'metadata': {}},
]
xform = arithmetic.StorageMaxTransformer()
xform = arithmetic.BlockStorageMaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -191,13 +192,140 @@ class TestStorageMaxTransformer(base.DistilTestCase):
'metadata': {}},
]
xform = arithmetic.StorageMaxTransformer()
xform = arithmetic.BlockStorageMaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 27}, usage)
@mock.patch.object(general, 'get_transformer_config', mock.Mock())
class TestObjectStorageMaxTransformer(base.DistilTestCase):
@mock.patch.object(
openstack, 'get_container_policy', mock.Mock(return_value='test-policy'))
def test_all_different_values(self):
"""
Tests that the transformer correctly grabs the highest value,
when all values are different.
"""
data = [
{'timestamp': FAKE_DATA.t0, 'volume': 12,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
{'timestamp': FAKE_DATA.t0_10, 'volume': 3,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
{'timestamp': FAKE_DATA.t0_20, 'volume': 7,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
{'timestamp': FAKE_DATA.t0_30, 'volume': 3,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
{'timestamp': FAKE_DATA.t0_40, 'volume': 25,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
{'timestamp': FAKE_DATA.t0_50, 'volume': 2,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
{'timestamp': FAKE_DATA.t1, 'volume': 6,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
]
xform = arithmetic.ObjectStorageMaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'test-policy': 25}, usage)
@mock.patch.object(
openstack, 'get_container_policy', mock.Mock(return_value='test-policy'))
def test_all_same_values(self):
"""
Tests that that transformer correctly grabs any value,
when all values are the same.
"""
data = [
{'timestamp': FAKE_DATA.t0, 'volume': 25,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
{'timestamp': FAKE_DATA.t0_30, 'volume': 25,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
{'timestamp': FAKE_DATA.t1, 'volume': 25,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
]
xform = arithmetic.ObjectStorageMaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'test-policy': 25}, usage)
@mock.patch.object(
openstack, 'get_container_policy', mock.Mock(return_value='test-policy'))
def test_none_value(self):
"""
Tests that that transformer correctly handles a None value.
"""
data = [
{'timestamp': FAKE_DATA.t0, 'volume': None,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
]
xform = arithmetic.ObjectStorageMaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'test-policy': 0}, usage)
@mock.patch.object(
openstack, 'get_container_policy', mock.Mock(return_value=None))
def test_none_and_other_values(self):
"""
Tests that that transformer correctly handles a None value.
"""
data = [
{'timestamp': FAKE_DATA.t0, 'volume': None,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
{'timestamp': FAKE_DATA.t0_30, 'volume': 25,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
{'timestamp': FAKE_DATA.t1, 'volume': 27,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {}},
]
xform = arithmetic.ObjectStorageMaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 27}, usage)
@mock.patch.object(general, 'get_transformer_config', mock.Mock())
@mock.patch.object(general, 'get_transformer_config', mock.Mock())
class TestSumTransformer(base.DistilTestCase):
def test_basic_sum(self):

40
distil/transformer/arithmetic.py
View File

@ -29,17 +29,22 @@ class MaxTransformer(BaseTransformer):
'gigabyte-hours'.
"""
def _get_max_vol(self, data):
if len(data):
max_vol = max([(v["volume"] if v["volume"] else 0) for v in data])
if max_vol:
return max_vol
return 0
def _transform_usage(self, meter_name, raw_data, start_at, end_at):
max_vol = max([(v["volume"] if v["volume"] else 0)
for v in raw_data]) if len(raw_data) else 0
max_vol = max_vol if max_vol else 0
max_vol = self._get_max_vol(raw_data)
hours = (end_at - start_at).total_seconds() / 3600.0
return {meter_name: max_vol * hours}
class StorageMaxTransformer(BaseTransformer):
class BlockStorageMaxTransformer(MaxTransformer):
"""
Variantion on the GaugeMax Transformer that checks for
volume_type and uses that as the service, or uses the
@ -50,8 +55,7 @@ class StorageMaxTransformer(BaseTransformer):
if not data:
return None
max_vol = max([(v["volume"] if v["volume"] else 0)
for v in data]) or 0
max_vol = self._get_max_vol(data)
if "volume_type" in data[-1]['metadata']:
vtype = data[-1]['metadata']['volume_type']
@ -65,6 +69,30 @@ class StorageMaxTransformer(BaseTransformer):
return {service: max_vol * hours}
class ObjectStorageMaxTransformer(MaxTransformer):
"""
Variantion on the GaugeMax Transformer that checks for
object storage container policy and uses that as the service,
or uses the default service name.
"""
def _transform_usage(self, name, data, start, end):
if not data:
return None
container_name = data[-1]['resource_id'].split('/')[1]
project_id = data[-1]['project_id']
service = openstack.get_container_policy(project_id, container_name)
if not service:
service = name
max_vol = self._get_max_vol(data)
hours = (end - start).total_seconds() / 3600.0
return {service: max_vol * hours}
class SumTransformer(BaseTransformer):
"""Transformer for sum-integration of a gauge value for given period.
"""

4
etc/meter_mappings.yml.sample
View File

@ -35,7 +35,7 @@
meter: volume.size
service: b1.standard
type: Volume
transformer: max
transformer: blockstoragemax
unit: gigabyte
metadata:
name:
@ -152,7 +152,7 @@
meter: storage.containers.objects.size
service: o1.standard
type: Object Storage Container
transformer: max
transformer: objectstoragemax
unit: byte
# NOTE(flwang): Nothing in resource_metadata from ceilometer actually.
# But to avoid any unnecessary issue and keeping consistency. Just keep

4
setup.cfg
View File

@ -39,7 +39,9 @@ distil.collector =
distil.transformer =
max = distil.transformer.arithmetic:MaxTransformer
storagemax = distil.transformer.arithmetic:StorageMaxTransformer
storagemax = distil.transformer.arithmetic:BlockStorageMaxTransformer
blockstoragemax = distil.transformer.arithmetic:BlockStorageMaxTransformer
objectstoragemax = distil.transformer.arithmetic:ObjectStorageMaxTransformer
sum = distil.transformer.arithmetic:SumTransformer
uptime = distil.transformer.conversion:UpTimeTransformer
fromimage = distil.transformer.conversion:FromImageTransformer