Add support for object storage policies

Introduce a new ObjectStorageMax transformer which
takes a container and bases the service for it on the
policy used for it in swift so that we can charge differently
for each policy.

Change-Id: Id1a0d7783ae72de0a2da36245badbd770508be81

distil/collector/base.py

@@ -223,7 +223,7 @@ class BaseCollector(object):
                 # hashing the name only for swift to get a consistent
                 # id for swift billing. Another change will be proposed to
                 # openstack-billing to handle this case as well.
-                if 'o1.standard' in transformed:
+                if mapping['type'] == "Object Storage Container":
                     res_id = hashlib.md5(res_id.encode('utf-8')).hexdigest()
 
                 LOG.debug(

distil/common/openstack.py

@@ -150,3 +150,27 @@ def get_volume_type(volume_type):
             return vtype['name']
 
     return None
+
+
+@general.disable_ssl_warnings
+def get_object_storage_url(project_id):
+    ks = get_keystone_client()
+    try:
+        endpoint = ks.endpoints.list(
+            service=ks.services.list(type="object-store")[0],
+            interface="public",
+            region=CONF.keystone_authtoken.region_name)[0]
+        return endpoint.url % {'tenant_id': project_id}
+    except KeyError:
+        return None
+
+
+@general.disable_ssl_warnings
+def get_container_policy(project_id, container_name):
+    sess = _get_keystone_session()
+    url = get_object_storage_url(project_id)
+    if url:
+        resp = sess.head("%s/%s" % (url, container_name))
+        if resp:
+            return resp.headers.get('X-Storage-Policy')
+    return None

distil/common/wsgi.py

@@ -19,6 +19,8 @@
 """Utility methods for working with WSGI servers."""
 
 import datetime
+import six
+
 from xml.dom import minidom
 from xml.parsers import expat
 
@@ -61,7 +63,7 @@ class JSONDictSerializer(DictSerializer):
             if isinstance(obj, datetime.datetime):
                 _dtime = obj - datetime.timedelta(microseconds=obj.microsecond)
                 return _dtime.isoformat()
-            return unicode(obj)
+            return six.text_type(obj)
         return jsonutils.dumps(data, default=sanitizer)
 
 

distil/tests/unit/transformer/test_arithmetic.py

@@ -18,6 +18,7 @@ import mock
 
 from distil.common.constants import date_format
 from distil.common import general
+from distil.common import openstack
 from distil.tests.unit import base
 from distil.transformer import arithmetic
 
@@ -110,7 +111,7 @@ class TestMaxTransformer(base.DistilTestCase):
 
 
 @mock.patch.object(general, 'get_transformer_config', mock.Mock())
-class TestStorageMaxTransformer(base.DistilTestCase):
+class TestBlockStorageMaxTransformer(base.DistilTestCase):
     def test_all_different_values(self):
         """
         Tests that the transformer correctly grabs the highest value,
@@ -134,7 +135,7 @@ class TestStorageMaxTransformer(base.DistilTestCase):
              'metadata': {}},
         ]
 
-        xform = arithmetic.StorageMaxTransformer()
+        xform = arithmetic.BlockStorageMaxTransformer()
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -155,7 +156,7 @@ class TestStorageMaxTransformer(base.DistilTestCase):
              'metadata': {}},
         ]
 
-        xform = arithmetic.StorageMaxTransformer()
+        xform = arithmetic.BlockStorageMaxTransformer()
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -171,7 +172,7 @@ class TestStorageMaxTransformer(base.DistilTestCase):
              'metadata': {}},
         ]
 
-        xform = arithmetic.StorageMaxTransformer()
+        xform = arithmetic.BlockStorageMaxTransformer()
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -191,13 +192,140 @@ class TestStorageMaxTransformer(base.DistilTestCase):
              'metadata': {}},
         ]
 
-        xform = arithmetic.StorageMaxTransformer()
+        xform = arithmetic.BlockStorageMaxTransformer()
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
         self.assertEqual({'some_meter': 27}, usage)
 
 
+@mock.patch.object(general, 'get_transformer_config', mock.Mock())
+class TestObjectStorageMaxTransformer(base.DistilTestCase):
+
+    @mock.patch.object(
+        openstack, 'get_container_policy', mock.Mock(return_value='test-policy'))
+    def test_all_different_values(self):
+        """
+        Tests that the transformer correctly grabs the highest value,
+        when all values are different.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0, 'volume': 12,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_10, 'volume': 3,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_20, 'volume': 7,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_30, 'volume': 3,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_40, 'volume': 25,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_50, 'volume': 2,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t1, 'volume': 6,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+        ]
+
+        xform = arithmetic.ObjectStorageMaxTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'test-policy': 25}, usage)
+
+    @mock.patch.object(
+        openstack, 'get_container_policy', mock.Mock(return_value='test-policy'))
+    def test_all_same_values(self):
+        """
+        Tests that that transformer correctly grabs any value,
+        when all values are the same.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0, 'volume': 25,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_30, 'volume': 25,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t1, 'volume': 25,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+        ]
+
+        xform = arithmetic.ObjectStorageMaxTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'test-policy': 25}, usage)
+
+    @mock.patch.object(
+        openstack, 'get_container_policy', mock.Mock(return_value='test-policy'))
+    def test_none_value(self):
+        """
+        Tests that that transformer correctly handles a None value.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0, 'volume': None,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+        ]
+
+        xform = arithmetic.ObjectStorageMaxTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'test-policy': 0}, usage)
+
+    @mock.patch.object(
+        openstack, 'get_container_policy', mock.Mock(return_value=None))
+    def test_none_and_other_values(self):
+        """
+        Tests that that transformer correctly handles a None value.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0, 'volume': None,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_30, 'volume': 25,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t1, 'volume': 27,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {}},
+        ]
+
+        xform = arithmetic.ObjectStorageMaxTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'some_meter': 27}, usage)
+
+
+@mock.patch.object(general, 'get_transformer_config', mock.Mock())
 @mock.patch.object(general, 'get_transformer_config', mock.Mock())
 class TestSumTransformer(base.DistilTestCase):
     def test_basic_sum(self):

distil/transformer/arithmetic.py

@@ -29,17 +29,22 @@ class MaxTransformer(BaseTransformer):
     'gigabyte-hours'.
     """
 
+    def _get_max_vol(self, data):
+        if len(data):
+            max_vol = max([(v["volume"] if v["volume"] else 0) for v in data])
+            if max_vol:
+                return max_vol
+        return 0
+
     def _transform_usage(self, meter_name, raw_data, start_at, end_at):
-        max_vol = max([(v["volume"] if v["volume"] else 0)
+        max_vol = self._get_max_vol(raw_data)
-                       for v in raw_data]) if len(raw_data) else 0
-        max_vol = max_vol if max_vol else 0
 
         hours = (end_at - start_at).total_seconds() / 3600.0
 
         return {meter_name: max_vol * hours}
 
 
-class StorageMaxTransformer(BaseTransformer):
+class BlockStorageMaxTransformer(MaxTransformer):
     """
     Variantion on the GaugeMax Transformer that checks for
     volume_type and uses that as the service, or uses the
@@ -50,8 +55,7 @@ class StorageMaxTransformer(BaseTransformer):
         if not data:
             return None
 
-        max_vol = max([(v["volume"] if v["volume"] else 0)
+        max_vol = self._get_max_vol(data)
-                       for v in data]) or 0
 
         if "volume_type" in data[-1]['metadata']:
             vtype = data[-1]['metadata']['volume_type']
@@ -65,6 +69,30 @@ class StorageMaxTransformer(BaseTransformer):
         return {service: max_vol * hours}
 
 
+class ObjectStorageMaxTransformer(MaxTransformer):
+    """
+    Variantion on the GaugeMax Transformer that checks for
+    object storage container policy and uses that as the service,
+    or uses the default service name.
+    """
+
+    def _transform_usage(self, name, data, start, end):
+        if not data:
+            return None
+
+        container_name = data[-1]['resource_id'].split('/')[1]
+        project_id = data[-1]['project_id']
+
+        service = openstack.get_container_policy(project_id, container_name)
+        if not service:
+            service = name
+
+        max_vol = self._get_max_vol(data)
+
+        hours = (end - start).total_seconds() / 3600.0
+        return {service: max_vol * hours}
+
+
 class SumTransformer(BaseTransformer):
     """Transformer for sum-integration of a gauge value for given period.
     """

etc/meter_mappings.yml.sample

@@ -35,7 +35,7 @@
   meter: volume.size
   service: b1.standard
   type: Volume
-  transformer: max
+  transformer: blockstoragemax
   unit: gigabyte
   metadata:
     name:
@@ -152,7 +152,7 @@
   meter: storage.containers.objects.size
   service: o1.standard
   type: Object Storage Container
-  transformer: max
+  transformer: objectstoragemax
   unit: byte
   # NOTE(flwang): Nothing in resource_metadata from ceilometer actually.
   # But to avoid any unnecessary issue and keeping consistency. Just keep

setup.cfg

@@ -39,7 +39,9 @@ distil.collector =
 
 distil.transformer =
     max = distil.transformer.arithmetic:MaxTransformer
-    storagemax = distil.transformer.arithmetic:StorageMaxTransformer
+    storagemax = distil.transformer.arithmetic:BlockStorageMaxTransformer
+    blockstoragemax = distil.transformer.arithmetic:BlockStorageMaxTransformer
+    objectstoragemax = distil.transformer.arithmetic:ObjectStorageMaxTransformer
     sum = distil.transformer.arithmetic:SumTransformer
     uptime = distil.transformer.conversion:UpTimeTransformer
     fromimage = distil.transformer.conversion:FromImageTransformer