Merge "Transformer improvement"

distil/collector/ceilometer.py

@@ -66,4 +66,8 @@ class CeilometerCollector(base.BaseCollector):
 
         sample_objs = self.cclient.new_samples.list(q=query)
 
+        # The samples are in descending order by default, should change it to
+        # be ascending, making the logic consistent with deprecated code.
+        sample_objs.reverse()
+
         return [obj.to_dict() for obj in sample_objs]

distil/common/openstack.py

@@ -147,12 +147,3 @@ def get_volume_type(volume_type):
             return vtype['name']
 
     return None
-
-
-@general.disable_ssl_warnings
-def get_flavor_name(flavor_id):
-    """Grabs the correct flavor name from Nova given the correct ID."""
-    if flavor_id not in cache:
-        nova = get_nova_client()
-        cache[flavor_id] = nova.flavors.get(flavor_id).name
-    return cache[flavor_id]

distil/tests/unit/transformer/test_arithmetic.py

@@ -0,0 +1,250 @@
+# Copyright (C) 2017 Catalyst IT Ltd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+
+import mock
+
+from distil.common.constants import date_format
+from distil.common import general
+from distil.tests.unit import base
+from distil.transformer import arithmetic
+
+p = lambda t: datetime.datetime.strptime(t, date_format)
+
+
+class FAKE_DATA:
+    t0 = p('2014-01-01T00:00:00')
+    t0_10 = p('2014-01-01T00:10:00')
+    t0_20 = p('2014-01-01T00:30:00')
+    t0_30 = p('2014-01-01T00:30:00')
+    t0_40 = p('2014-01-01T00:40:00')
+    t0_50 = p('2014-01-01T00:50:00')
+    t1 = p('2014-01-01T01:00:00')
+
+
+@mock.patch.object(general, 'get_transformer_config', mock.Mock())
+class TestMaxTransformer(base.DistilTestCase):
+    def test_all_different_values(self):
+        """
+        Tests that the transformer correctly grabs the highest value,
+        when all values are different.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0.isoformat(), 'volume': 12},
+            {'timestamp': FAKE_DATA.t0_10.isoformat(), 'volume': 3},
+            {'timestamp': FAKE_DATA.t0_20.isoformat(), 'volume': 7},
+            {'timestamp': FAKE_DATA.t0_30.isoformat(), 'volume': 3},
+            {'timestamp': FAKE_DATA.t0_40.isoformat(), 'volume': 25},
+            {'timestamp': FAKE_DATA.t0_50.isoformat(), 'volume': 2},
+            {'timestamp': FAKE_DATA.t1.isoformat(), 'volume': 6},
+        ]
+
+        xform = arithmetic.MaxTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'some_meter': 25}, usage)
+
+    def test_all_same_values(self):
+        """
+        Tests that that transformer correctly grabs any value,
+        when all values are the same.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0, 'volume': 25},
+            {'timestamp': FAKE_DATA.t0_30, 'volume': 25},
+            {'timestamp': FAKE_DATA.t1, 'volume': 25},
+        ]
+
+        xform = arithmetic.MaxTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'some_meter': 25}, usage)
+
+    def test_none_value(self):
+        """
+        Tests that that transformer correctly handles a None value.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0, 'volume': None},
+        ]
+
+        xform = arithmetic.MaxTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'some_meter': 0}, usage)
+
+    def test_none_and_other_values(self):
+        """
+        Tests that that transformer correctly handles a None value.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0, 'volume': None},
+            {'timestamp': FAKE_DATA.t0_30, 'volume': 25},
+            {'timestamp': FAKE_DATA.t1, 'volume': 27},
+        ]
+
+        xform = arithmetic.MaxTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'some_meter': 27}, usage)
+
+
+@mock.patch.object(general, 'get_transformer_config', mock.Mock())
+class TestStorageMaxTransformer(base.DistilTestCase):
+    def test_all_different_values(self):
+        """
+        Tests that the transformer correctly grabs the highest value,
+        when all values are different.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0, 'volume': 12,
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_10, 'volume': 3,
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_20, 'volume': 7,
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_30, 'volume': 3,
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_40, 'volume': 25,
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_50, 'volume': 2,
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t1, 'volume': 6,
+             'metadata': {}},
+        ]
+
+        xform = arithmetic.StorageMaxTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'some_meter': 25}, usage)
+
+    def test_all_same_values(self):
+        """
+        Tests that that transformer correctly grabs any value,
+        when all values are the same.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0, 'volume': 25,
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_30, 'volume': 25,
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t1, 'volume': 25,
+             'metadata': {}},
+        ]
+
+        xform = arithmetic.StorageMaxTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'some_meter': 25}, usage)
+
+    def test_none_value(self):
+        """
+        Tests that that transformer correctly handles a None value.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0, 'volume': None,
+             'metadata': {}},
+        ]
+
+        xform = arithmetic.StorageMaxTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'some_meter': 0}, usage)
+
+    def test_none_and_other_values(self):
+        """
+        Tests that that transformer correctly handles a None value.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0, 'volume': None,
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t0_30, 'volume': 25,
+             'metadata': {}},
+            {'timestamp': FAKE_DATA.t1, 'volume': 27,
+             'metadata': {}},
+        ]
+
+        xform = arithmetic.StorageMaxTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'some_meter': 27}, usage)
+
+
+@mock.patch.object(general, 'get_transformer_config', mock.Mock())
+class TestSumTransformer(base.DistilTestCase):
+    def test_basic_sum(self):
+        """
+        Tests that the transformer correctly calculate the sum value.
+        """
+
+        data = [
+            {'timestamp': '2014-01-01T00:00:00', 'volume': 1},
+            {'timestamp': '2014-01-01T00:10:00', 'volume': 1},
+            {'timestamp': '2014-01-01T01:00:00', 'volume': 1},
+        ]
+
+        xform = arithmetic.SumTransformer()
+        usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'fake_meter': 2}, usage)
+
+    def test_none_value(self):
+        """
+        Tests that that transformer correctly handles a None value.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0.isoformat(), 'volume': None},
+        ]
+
+        xform = arithmetic.SumTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'some_meter': 0}, usage)
+
+    def test_none_and_other_values(self):
+        """
+        Tests that that transformer correctly handles a None value.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0.isoformat(), 'volume': None},
+            {'timestamp': FAKE_DATA.t0_30.isoformat(), 'volume': 25},
+            {'timestamp': FAKE_DATA.t0_50.isoformat(), 'volume': 25},
+        ]
+
+        xform = arithmetic.SumTransformer()
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'some_meter': 50}, usage)

distil/tests/unit/transformer/test_conversion.py

@@ -0,0 +1,374 @@
+# Copyright (C) 2017 Catalyst IT Ltd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import datetime
+
+import mock
+
+from distil.common.constants import date_format
+from distil.common import general
+from distil.tests.unit import base
+from distil.transformer import conversion
+
+p = lambda t: datetime.datetime.strptime(t, date_format)
+
+
+class FAKE_DATA:
+    t0 = p('2014-01-01T00:00:00')
+    t0_10 = p('2014-01-01T00:10:00')
+    t0_30 = p('2014-01-01T00:30:00')
+    t1 = p('2014-01-01T01:00:00')
+
+    # and one outside the window
+    tpre = p('2013-12-31T23:50:00')
+
+    flavor = '1'
+    flavor2 = '2'
+
+
+FAKE_CONFIG = {
+    "uptime": {
+        "tracked_states": ["active", "building", "paused", "rescued",
+                           "resized"]
+    },
+    "from_image": {
+        "service": "volume.size",
+        "md_keys": ["image_ref", "image_meta.base_image_ref"],
+        "none_values": ["None", ""],
+        "size_keys": ["root_gb"]
+    }
+}
+
+
+@mock.patch.object(general, 'get_transformer_config',
+                   mock.Mock(return_value=FAKE_CONFIG))
+class TestUpTimeTransformer(base.DistilTestCase):
+    def test_trivial_run(self):
+        """
+        Test that an no input data produces empty uptime.
+        """
+        state = []
+
+        xform = conversion.UpTimeTransformer()
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({}, result)
+
+    def test_online_constant_flavor(self):
+        """
+        Test that a machine online for a 1h period with constant
+        flavor works and gives 1h of uptime.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'active'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'active'}}
+        ]
+
+        xform = conversion.UpTimeTransformer()
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({FAKE_DATA.flavor: 3600}, result)
+
+    def test_offline_constant_flavor(self):
+        """
+        Test that a machine offline for a 1h period with constant flavor
+        works and gives zero uptime.
+        """
+
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'stopped'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'stopped'}}
+        ]
+
+        xform = conversion.UpTimeTransformer()
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({}, result)
+
+    def test_shutdown_during_period(self):
+        """
+        Test that a machine run for 0.5 then shutdown gives 0.5h uptime.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'active'}},
+            {'timestamp': FAKE_DATA.t0_30.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'stopped'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'stopped'}}
+        ]
+
+        xform = conversion.UpTimeTransformer()
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({FAKE_DATA.flavor: 1800}, result)
+
+    def test_online_flavor_change(self):
+        """
+        Test that a machine run for 0.5h as m1.tiny, resized to m1.large,
+        and run for a further 0.5 yields 0.5h of uptime in each class.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'active'}},
+            {'timestamp': FAKE_DATA.t0_30.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor2,
+                          'status': 'active'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor2,
+                          'status': 'active'}}
+        ]
+
+        xform = conversion.UpTimeTransformer()
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertDictEqual(
+            {FAKE_DATA.flavor: 1800, FAKE_DATA.flavor2: 1800},
+            result
+        )
+
+    def test_period_leadin_none_available(self):
+        """
+        Test that if the first data point is well into the window, and we had
+        no lead-in data, we assume no usage until our first real data point.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.t0_10.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'active'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'active'}}
+        ]
+
+        xform = conversion.UpTimeTransformer()
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({FAKE_DATA.flavor: 3000}, result)
+
+    def test_period_leadin_available(self):
+        """
+        Test that if the first data point is well into the window, but we *do*
+        have lead-in data, then we use the lead-in clipped to the start of the
+        window.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.tpre.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'active'}},
+            {'timestamp': FAKE_DATA.t0_10.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'active'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'status': 'active'}}
+        ]
+
+        xform = conversion.UpTimeTransformer()
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({FAKE_DATA.flavor: 3600}, result)
+
+    def test_notification_case(self):
+        """
+        Test that the transformer handles the notification metedata key,
+        if/when it can't find the status key.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'state': 'active'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor,
+                          'state': 'active'}}
+        ]
+
+        xform = conversion.UpTimeTransformer()
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({FAKE_DATA.flavor: 3600}, result)
+
+    def test_no_state_in_metedata(self):
+        """
+        Test that the transformer doesn't fall over if there isn't one of
+        the two state/status key options in the metadata.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'instance_type': FAKE_DATA.flavor}}
+        ]
+
+        xform = conversion.UpTimeTransformer()
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({}, result)
+
+
+@mock.patch.object(general, 'get_transformer_config',
+                   mock.Mock(return_value=FAKE_CONFIG))
+class TestFromImageTransformer(base.DistilTestCase):
+    """
+    These tests rely on config settings for from_image,
+    as defined in test constants, or in conf.yaml
+    """
+
+    def test_from_volume_case(self):
+        """
+        If instance is booted from volume transformer should return none.
+        """
+        data = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'image_ref': ""}},
+            {'timestamp': FAKE_DATA.t0_30.isoformat(),
+             'metadata': {'image_ref': "None"}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'image_ref': "None"}}
+        ]
+
+        data2 = [
+            {'timestamp': FAKE_DATA.t0_30.isoformat(),
+             'metadata': {'image_ref': "None"}}
+        ]
+
+        xform = conversion.FromImageTransformer()
+
+        usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+        usage2 = xform.transform_usage('instance', data2, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertIsNone(usage)
+        self.assertIsNone(usage2)
+
+    def test_default_to_from_volume_case(self):
+        """
+        Unless all image refs contain something, assume booted from volume.
+        """
+        data = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'image_ref': ""}},
+            {'timestamp': FAKE_DATA.t0_30.isoformat(),
+             'metadata': {'image_ref': "d5a4f118023928195f4ef"}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'image_ref': "None"}}
+        ]
+
+        xform = conversion.FromImageTransformer()
+        usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertIsNone(usage)
+
+    def test_from_image_case(self):
+        """
+        If all image refs contain something, should return entry.
+        """
+        data = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'image_ref': "d5a4f118023928195f4ef",
+                          'root_gb': "20"}},
+            {'timestamp': FAKE_DATA.t0_30.isoformat(),
+             'metadata': {'image_ref': "d5a4f118023928195f4ef",
+                          'root_gb': "20"}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'image_ref': "d5a4f118023928195f4ef",
+                          'root_gb': "20"}}
+        ]
+
+        xform = conversion.FromImageTransformer()
+        usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'volume.size': 20}, usage)
+
+    def test_from_image_case_highest_size(self):
+        """
+        If all image refs contain something,
+        should return entry with highest size from data.
+        """
+        data = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'image_ref': "d5a4f118023928195f4ef",
+                          'root_gb': "20"}},
+            {'timestamp': FAKE_DATA.t0_30.isoformat(),
+             'metadata': {'image_ref': "d5a4f118023928195f4ef",
+                          'root_gb': "60"}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'image_ref': "d5a4f118023928195f4ef",
+                          'root_gb': "20"}}
+        ]
+
+        xform = conversion.FromImageTransformer()
+        usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'volume.size': 60}, usage)
+
+
+@mock.patch.object(general, 'get_transformer_config',
+                   mock.Mock(return_value=FAKE_CONFIG))
+class TestNetworkServiceTransformer(base.DistilTestCase):
+    def test_basic_sum(self):
+        """Tests that the transformer correctly calculate the sum value.
+        """
+
+        data = [
+            {'timestamp': '2014-01-01T00:00:00', 'volume': 1},
+            {'timestamp': '2014-01-01T00:10:00', 'volume': 0},
+            {'timestamp': '2014-01-01T01:00:00', 'volume': 2},
+        ]
+
+        xform = conversion.NetworkServiceTransformer()
+        usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'fake_meter': 1}, usage)
+
+    def test_only_pending_service(self):
+        """Tests that the transformer correctly calculate the sum value.
+        """
+
+        data = [
+            {'timestamp': '2014-01-01T00:00:00', 'volume': 2},
+            {'timestamp': '2014-01-01T00:10:00', 'volume': 2},
+            {'timestamp': '2014-01-01T01:00:00', 'volume': 2},
+        ]
+
+        xform = conversion.NetworkServiceTransformer()
+        usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'fake_meter': 0}, usage)

distil/transformer/arithmetic.py

@@ -30,9 +30,11 @@ class MaxTransformer(BaseTransformer):
     """
 
     def _transform_usage(self, meter_name, raw_data, start_at, end_at):
-        max_vol = max([v["volume"]
+        max_vol = max([v["volume"] for v in raw_data]) if len(raw_data) else 0
-                       for v in raw_data]) if len(raw_data) else 0
+        max_vol = max_vol if max_vol else 0
+
         hours = (end_at - start_at).total_seconds() / 3600.0
+
         return {meter_name: max_vol * hours}
 
 
@@ -47,13 +49,7 @@ class StorageMaxTransformer(BaseTransformer):
         if not data:
             return None
 
-        max_vol = max([v["volume"] for v in data])
+        max_vol = max([v["volume"] for v in data]) or 0
-
-        if max_vol is None:
-            max_vol = 0
-            LOG.warning("None max_vol value for %s in window: %s - %s " %
-                        (name, start.strftime(constants.iso_time),
-                         end.strftime(constants.iso_time)))
 
         if "volume_type" in data[-1]['metadata']:
             vtype = data[-1]['metadata']['volume_type']
@@ -83,5 +79,6 @@ class SumTransformer(BaseTransformer):
                                                '%Y-%m-%dT%H:%M:%S')
 
             if t >= start_at and t < end_at:
-                sum_vol += sample["volume"]
+                sum_vol += sample["volume"] or 0
+
         return {meter_name: sum_vol}

distil/transformer/conversion.py

@@ -12,7 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import datetime
+from datetime import datetime
 
 from distil.transformer import BaseTransformer
 from distil.common import constants
@@ -29,14 +29,12 @@ class UpTimeTransformer(BaseTransformer):
         # get tracked states from config
         tracked = self.config['uptime']['tracked_states']
 
-        tracked_states = {constants.states[i] for i in tracked}
-
         usage_dict = {}
 
         def sort_and_clip_end(usage):
             cleaned = (self._clean_entry(s) for s in usage)
-            clipped = (s for s in cleaned if s['timestamp'] < end)
+            clipped = [s for s in cleaned if s['timestamp'] < end]
-            return sorted(clipped, key=lambda x: x['timestamp'])
+            return clipped
 
         state = sort_and_clip_end(data)
 
@@ -57,7 +55,7 @@ class UpTimeTransformer(BaseTransformer):
             usage_dict[flav] = usage_dict.get(flav, 0) + diff.total_seconds()
 
         for val in state[1:]:
-            if last_state["volume"] in tracked_states:
+            if last_state["status"] in tracked:
                 diff = val["timestamp"] - last_timestamp
                 if val['timestamp'] > last_timestamp:
                     # if diff < 0 then we were looking back before the start
@@ -70,29 +68,25 @@ class UpTimeTransformer(BaseTransformer):
 
         # extend the last state we know about, to the end of the window,
         # if we saw any actual uptime.
-        if (end and last_state['volume'] in tracked_states
+        if (end and last_state['status'] in tracked and seen_sample_in_window):
-                and seen_sample_in_window):
             diff = end - last_timestamp
             _add_usage(diff)
 
-        # map the flavors to names on the way out
+        return usage_dict
-        return {openstack.get_flavor_name(f): v for f, v in usage_dict.items()}
 
     def _clean_entry(self, entry):
         result = {
-            'volume': entry['volume'],
+            'status': entry['metadata'].get(
-            'flavor': entry['metadata'].get(
+                'status', entry['metadata'].get(
-                'flavor.id', entry['metadata'].get(
+                    'state', ""
-                    'instance_flavor_id', 0
                 )
+            ),
+            'flavor': entry['metadata'].get('instance_type'),
+            'timestamp': datetime.strptime(
+                entry['timestamp'],
+                "%Y-%m-%dT%H:%M:%S"
             )
         }
-        try:
-            result['timestamp'] = datetime.datetime.strptime(
-                entry['timestamp'], constants.date_format)
-        except ValueError:
-            result['timestamp'] = datetime.datetime.strptime(
-                entry['timestamp'], constants.date_format_f)
         return result
 
 
@@ -126,7 +120,9 @@ class FromImageTransformer(BaseTransformer):
                         size = root_size
                 except KeyError:
                     pass
+
         hours = (end - start).total_seconds() / 3600.0
+
         return {service: size * hours}
 
 
@@ -141,7 +137,8 @@ class NetworkServiceTransformer(BaseTransformer):
         # blob/master/ceilometer/network/services/vpnaas.py#L55), so we have
         # to check the volume to make sure only the active service is
         # charged(0=inactive, 1=active).
-        max_vol = max([v["volume"] for v in data
+        volumes = [v["volume"] for v in data if
-                       if v["volume"] < 2]) if len(data) else 0
+                   v["volume"] < 2]
+        max_vol = max(volumes) if len(volumes) else 0
         hours = (end - start).total_seconds() / 3600.0
         return {name: max_vol * hours}

etc/distil.conf.sample

@@ -1,41 +0,0 @@
-[DEFAULT]
-debug = True
-ignore_tenants = demo
-timezone = Pacific/Auckland
-host = localhost
-port = 9999
-
-[collector]
-periodic_interval = 10
-dawn_of_time = 2016-05-18 01:00:00
-meter_mappings_file = /etc/distil/meter_mappings.yml
-
-[rater]
-rater_type = odoo
-rate_file_path = /etc/distil/rates.csv
-
-[odoo]
-version=8.0
-hostname=
-port=443
-protocol=jsonrpc+ssl
-database=
-user=
-password=
-
-[database]
-connection = mysql://root:passw0rd@127.0.0.1/distil?charset=utf8
-backend = sqlalchemy
-
-[keystone_authtoken]
-memcache_servers = 127.0.0.1:11211
-signing_dir = /var/cache/distil
-cafile = /opt/stack/data/ca-bundle.pem
-auth_uri = http://127.0.0.1:5000
-project_domain_id = default
-project_name = service
-user_domain_id = default
-password = passw0rd
-username = distil
-auth_url = http://127.0.0.1:35357
-auth_type = password

etc/transformer.yaml

@@ -1,116 +0,0 @@
-# configuration for defining usage collection
-collection:
-  # defines which meter is mapped to which transformer
-  meter_mappings:
-    # meter name as seen in ceilometer
-    state: 
-      # type of resource it maps to (seen on sales order)
-      type: Virtual Machine
-      # which transformer to use
-      transformer: uptime
-      # what unit type is coming in via the meter
-      unit: second 
-      metadata:
-        name:
-          sources:
-            # which keys to search for in the ceilometer entry metadata
-            # this can be more than one as metadata is inconsistent between
-            # source types
-            - display_name
-        availability zone:
-          sources:
-            - OS-EXT-AZ:availability_zone
-    ip.floating:
-      type: Floating IP
-      transformer: max
-      unit: hour
-      metadata:
-        ip address:
-          sources:
-            - floating_ip_address
-    volume.size:
-      type: Volume
-      transformer: max
-      unit: gigabyte
-      metadata:
-        name:
-          sources:
-            - display_name
-        availability zone:
-          sources:
-            - availability_zone
-    instance:
-      type: Volume
-      transformer: fromimage
-      unit: gigabyte
-      # if true allows id pattern, and metadata patterns
-      transform_info: True
-      # allows us to put the id into a pattern,
-      # only if transform_info is true,
-      # such as to append something to it
-      res_id_template: "%s-root_disk"
-      metadata:
-        name:
-          sources:
-            - display_name
-          template: "%s - root disk"
-        availability zone:
-          sources:
-            - availability_zone
-    image.size:
-      type: Image
-      transformer: max
-      unit: byte
-      metadata:
-        name:
-          sources:
-            - name
-            - properties.image_name
-    bandwidth:
-      type: Network Traffic
-      transformer: sum
-      unit: byte
-      metadata:
-        meter_label_id:
-          sources:
-            - label_id
-    network.services.vpn:
-      type: VPN
-      transformer: networkservice
-      unit: hour
-      metadata:
-        name:
-          sources:
-            - name
-        subnet:
-          sources:
-            - subnet_id
-    network:
-      type: Network
-      transformer: max
-      unit: hour
-      metadata:
-        name:
-          sources:
-            - name
-# transformer configs
-transformers:
-  uptime:
-    # states marked as "billable" for VMs. 
-    tracked_states:
-      - active
-      - paused
-      - rescued
-      - resized
-  from_image:
-    service: volume.size
-    # What metadata values to check
-    md_keys:
-      - image_ref
-      - image_meta.base_image_ref
-    none_values:
-      - None
-      - ""
-    # where to get volume size from
-    size_keys:
-      - root_gb

etc/transformer.yaml.sample

@@ -0,0 +1,26 @@
+uptime:
+  # states marked as "billable" for VMs.
+  tracked_states:
+    - active
+    - paused
+    - rescue
+    - rescued
+    - resize
+    - resized
+    - verify_resize
+    - suspended
+  # uncomment these when we want to bill shutdown:
+  #  - shutoff
+  #  - stopped
+from_image:
+  service: b1.standard
+  # What metadata values to check
+  md_keys:
+    - image_ref
+    - image_meta.base_image_ref
+  none_values:
+    - None
+    - ""
+  # where to get volume size from
+  size_keys:
+    - root_gb