x/distil
Code Issues Proposed changes

Merge "Transformer improvement"

Browse Source
This commit is contained in:
Jenkins 2017-01-25 19:32:51 +00:00 committed by Gerrit Code Review
commit 6d1dec6cd9
10 changed files with 680 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
distil/collector/ceilometer.py
View File

@ -66,4 +66,8 @@ class CeilometerCollector(base.BaseCollector):
sample_objs = self.cclient.new_samples.list(q=query)
# The samples are in descending order by default, should change it to
# be ascending, making the logic consistent with deprecated code.
sample_objs.reverse()
return [obj.to_dict() for obj in sample_objs]

9
distil/common/openstack.py
View File

@ -147,12 +147,3 @@ def get_volume_type(volume_type):
return vtype['name']
return None
@general.disable_ssl_warnings
def get_flavor_name(flavor_id):
"""Grabs the correct flavor name from Nova given the correct ID."""
if flavor_id not in cache:
nova = get_nova_client()
cache[flavor_id] = nova.flavors.get(flavor_id).name
return cache[flavor_id]

0
distil/tests/unit/transformer/__init__.py Normal file
View File

250
distil/tests/unit/transformer/test_arithmetic.py Normal file
View File

@ -0,0 +1,250 @@
# Copyright (C) 2017 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import mock
from distil.common.constants import date_format
from distil.common import general
from distil.tests.unit import base
from distil.transformer import arithmetic
p = lambda t: datetime.datetime.strptime(t, date_format)
class FAKE_DATA:
t0 = p('2014-01-01T00:00:00')
t0_10 = p('2014-01-01T00:10:00')
t0_20 = p('2014-01-01T00:30:00')
t0_30 = p('2014-01-01T00:30:00')
t0_40 = p('2014-01-01T00:40:00')
t0_50 = p('2014-01-01T00:50:00')
t1 = p('2014-01-01T01:00:00')
@mock.patch.object(general, 'get_transformer_config', mock.Mock())
class TestMaxTransformer(base.DistilTestCase):
def test_all_different_values(self):
"""
Tests that the transformer correctly grabs the highest value,
when all values are different.
"""
data = [
{'timestamp': FAKE_DATA.t0.isoformat(), 'volume': 12},
{'timestamp': FAKE_DATA.t0_10.isoformat(), 'volume': 3},
{'timestamp': FAKE_DATA.t0_20.isoformat(), 'volume': 7},
{'timestamp': FAKE_DATA.t0_30.isoformat(), 'volume': 3},
{'timestamp': FAKE_DATA.t0_40.isoformat(), 'volume': 25},
{'timestamp': FAKE_DATA.t0_50.isoformat(), 'volume': 2},
{'timestamp': FAKE_DATA.t1.isoformat(), 'volume': 6},
]
xform = arithmetic.MaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 25}, usage)
def test_all_same_values(self):
"""
Tests that that transformer correctly grabs any value,
when all values are the same.
"""
data = [
{'timestamp': FAKE_DATA.t0, 'volume': 25},
{'timestamp': FAKE_DATA.t0_30, 'volume': 25},
{'timestamp': FAKE_DATA.t1, 'volume': 25},
]
xform = arithmetic.MaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 25}, usage)
def test_none_value(self):
"""
Tests that that transformer correctly handles a None value.
"""
data = [
{'timestamp': FAKE_DATA.t0, 'volume': None},
]
xform = arithmetic.MaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 0}, usage)
def test_none_and_other_values(self):
"""
Tests that that transformer correctly handles a None value.
"""
data = [
{'timestamp': FAKE_DATA.t0, 'volume': None},
{'timestamp': FAKE_DATA.t0_30, 'volume': 25},
{'timestamp': FAKE_DATA.t1, 'volume': 27},
]
xform = arithmetic.MaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 27}, usage)
@mock.patch.object(general, 'get_transformer_config', mock.Mock())
class TestStorageMaxTransformer(base.DistilTestCase):
def test_all_different_values(self):
"""
Tests that the transformer correctly grabs the highest value,
when all values are different.
"""
data = [
{'timestamp': FAKE_DATA.t0, 'volume': 12,
'metadata': {}},
{'timestamp': FAKE_DATA.t0_10, 'volume': 3,
'metadata': {}},
{'timestamp': FAKE_DATA.t0_20, 'volume': 7,
'metadata': {}},
{'timestamp': FAKE_DATA.t0_30, 'volume': 3,
'metadata': {}},
{'timestamp': FAKE_DATA.t0_40, 'volume': 25,
'metadata': {}},
{'timestamp': FAKE_DATA.t0_50, 'volume': 2,
'metadata': {}},
{'timestamp': FAKE_DATA.t1, 'volume': 6,
'metadata': {}},
]
xform = arithmetic.StorageMaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 25}, usage)
def test_all_same_values(self):
"""
Tests that that transformer correctly grabs any value,
when all values are the same.
"""
data = [
{'timestamp': FAKE_DATA.t0, 'volume': 25,
'metadata': {}},
{'timestamp': FAKE_DATA.t0_30, 'volume': 25,
'metadata': {}},
{'timestamp': FAKE_DATA.t1, 'volume': 25,
'metadata': {}},
]
xform = arithmetic.StorageMaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 25}, usage)
def test_none_value(self):
"""
Tests that that transformer correctly handles a None value.
"""
data = [
{'timestamp': FAKE_DATA.t0, 'volume': None,
'metadata': {}},
]
xform = arithmetic.StorageMaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 0}, usage)
def test_none_and_other_values(self):
"""
Tests that that transformer correctly handles a None value.
"""
data = [
{'timestamp': FAKE_DATA.t0, 'volume': None,
'metadata': {}},
{'timestamp': FAKE_DATA.t0_30, 'volume': 25,
'metadata': {}},
{'timestamp': FAKE_DATA.t1, 'volume': 27,
'metadata': {}},
]
xform = arithmetic.StorageMaxTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 27}, usage)
@mock.patch.object(general, 'get_transformer_config', mock.Mock())
class TestSumTransformer(base.DistilTestCase):
def test_basic_sum(self):
"""
Tests that the transformer correctly calculate the sum value.
"""
data = [
{'timestamp': '2014-01-01T00:00:00', 'volume': 1},
{'timestamp': '2014-01-01T00:10:00', 'volume': 1},
{'timestamp': '2014-01-01T01:00:00', 'volume': 1},
]
xform = arithmetic.SumTransformer()
usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'fake_meter': 2}, usage)
def test_none_value(self):
"""
Tests that that transformer correctly handles a None value.
"""
data = [
{'timestamp': FAKE_DATA.t0.isoformat(), 'volume': None},
]
xform = arithmetic.SumTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 0}, usage)
def test_none_and_other_values(self):
"""
Tests that that transformer correctly handles a None value.
"""
data = [
{'timestamp': FAKE_DATA.t0.isoformat(), 'volume': None},
{'timestamp': FAKE_DATA.t0_30.isoformat(), 'volume': 25},
{'timestamp': FAKE_DATA.t0_50.isoformat(), 'volume': 25},
]
xform = arithmetic.SumTransformer()
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 50}, usage)

374
distil/tests/unit/transformer/test_conversion.py Normal file
View File

@ -0,0 +1,374 @@
# Copyright (C) 2017 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import mock
from distil.common.constants import date_format
from distil.common import general
from distil.tests.unit import base
from distil.transformer import conversion
p = lambda t: datetime.datetime.strptime(t, date_format)
class FAKE_DATA:
t0 = p('2014-01-01T00:00:00')
t0_10 = p('2014-01-01T00:10:00')
t0_30 = p('2014-01-01T00:30:00')
t1 = p('2014-01-01T01:00:00')
# and one outside the window
tpre = p('2013-12-31T23:50:00')
flavor = '1'
flavor2 = '2'
FAKE_CONFIG = {
"uptime": {
"tracked_states": ["active", "building", "paused", "rescued",
"resized"]
},
"from_image": {
"service": "volume.size",
"md_keys": ["image_ref", "image_meta.base_image_ref"],
"none_values": ["None", ""],
"size_keys": ["root_gb"]
}
}
@mock.patch.object(general, 'get_transformer_config',
mock.Mock(return_value=FAKE_CONFIG))
class TestUpTimeTransformer(base.DistilTestCase):
def test_trivial_run(self):
"""
Test that an no input data produces empty uptime.
"""
state = []
xform = conversion.UpTimeTransformer()
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({}, result)
def test_online_constant_flavor(self):
"""
Test that a machine online for a 1h period with constant
flavor works and gives 1h of uptime.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'active'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'active'}}
]
xform = conversion.UpTimeTransformer()
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({FAKE_DATA.flavor: 3600}, result)
def test_offline_constant_flavor(self):
"""
Test that a machine offline for a 1h period with constant flavor
works and gives zero uptime.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'stopped'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'stopped'}}
]
xform = conversion.UpTimeTransformer()
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({}, result)
def test_shutdown_during_period(self):
"""
Test that a machine run for 0.5 then shutdown gives 0.5h uptime.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'active'}},
{'timestamp': FAKE_DATA.t0_30.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'stopped'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'stopped'}}
]
xform = conversion.UpTimeTransformer()
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({FAKE_DATA.flavor: 1800}, result)
def test_online_flavor_change(self):
"""
Test that a machine run for 0.5h as m1.tiny, resized to m1.large,
and run for a further 0.5 yields 0.5h of uptime in each class.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'active'}},
{'timestamp': FAKE_DATA.t0_30.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor2,
'status': 'active'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor2,
'status': 'active'}}
]
xform = conversion.UpTimeTransformer()
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertDictEqual(
{FAKE_DATA.flavor: 1800, FAKE_DATA.flavor2: 1800},
result
)
def test_period_leadin_none_available(self):
"""
Test that if the first data point is well into the window, and we had
no lead-in data, we assume no usage until our first real data point.
"""
state = [
{'timestamp': FAKE_DATA.t0_10.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'active'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'active'}}
]
xform = conversion.UpTimeTransformer()
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({FAKE_DATA.flavor: 3000}, result)
def test_period_leadin_available(self):
"""
Test that if the first data point is well into the window, but we *do*
have lead-in data, then we use the lead-in clipped to the start of the
window.
"""
state = [
{'timestamp': FAKE_DATA.tpre.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'active'}},
{'timestamp': FAKE_DATA.t0_10.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'active'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'status': 'active'}}
]
xform = conversion.UpTimeTransformer()
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({FAKE_DATA.flavor: 3600}, result)
def test_notification_case(self):
"""
Test that the transformer handles the notification metedata key,
if/when it can't find the status key.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'state': 'active'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor,
'state': 'active'}}
]
xform = conversion.UpTimeTransformer()
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({FAKE_DATA.flavor: 3600}, result)
def test_no_state_in_metedata(self):
"""
Test that the transformer doesn't fall over if there isn't one of
the two state/status key options in the metadata.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'instance_type': FAKE_DATA.flavor}}
]
xform = conversion.UpTimeTransformer()
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({}, result)
@mock.patch.object(general, 'get_transformer_config',
mock.Mock(return_value=FAKE_CONFIG))
class TestFromImageTransformer(base.DistilTestCase):
"""
These tests rely on config settings for from_image,
as defined in test constants, or in conf.yaml
"""
def test_from_volume_case(self):
"""
If instance is booted from volume transformer should return none.
"""
data = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'image_ref': ""}},
{'timestamp': FAKE_DATA.t0_30.isoformat(),
'metadata': {'image_ref': "None"}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'image_ref': "None"}}
]
data2 = [
{'timestamp': FAKE_DATA.t0_30.isoformat(),
'metadata': {'image_ref': "None"}}
]
xform = conversion.FromImageTransformer()
usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
FAKE_DATA.t1)
usage2 = xform.transform_usage('instance', data2, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertIsNone(usage)
self.assertIsNone(usage2)
def test_default_to_from_volume_case(self):
"""
Unless all image refs contain something, assume booted from volume.
"""
data = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'image_ref': ""}},
{'timestamp': FAKE_DATA.t0_30.isoformat(),
'metadata': {'image_ref': "d5a4f118023928195f4ef"}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'image_ref': "None"}}
]
xform = conversion.FromImageTransformer()
usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertIsNone(usage)
def test_from_image_case(self):
"""
If all image refs contain something, should return entry.
"""
data = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'image_ref': "d5a4f118023928195f4ef",
'root_gb': "20"}},
{'timestamp': FAKE_DATA.t0_30.isoformat(),
'metadata': {'image_ref': "d5a4f118023928195f4ef",
'root_gb': "20"}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'image_ref': "d5a4f118023928195f4ef",
'root_gb': "20"}}
]
xform = conversion.FromImageTransformer()
usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'volume.size': 20}, usage)
def test_from_image_case_highest_size(self):
"""
If all image refs contain something,
should return entry with highest size from data.
"""
data = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'image_ref': "d5a4f118023928195f4ef",
'root_gb': "20"}},
{'timestamp': FAKE_DATA.t0_30.isoformat(),
'metadata': {'image_ref': "d5a4f118023928195f4ef",
'root_gb': "60"}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'image_ref': "d5a4f118023928195f4ef",
'root_gb': "20"}}
]
xform = conversion.FromImageTransformer()
usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'volume.size': 60}, usage)
@mock.patch.object(general, 'get_transformer_config',
mock.Mock(return_value=FAKE_CONFIG))
class TestNetworkServiceTransformer(base.DistilTestCase):
def test_basic_sum(self):
"""Tests that the transformer correctly calculate the sum value.
"""
data = [
{'timestamp': '2014-01-01T00:00:00', 'volume': 1},
{'timestamp': '2014-01-01T00:10:00', 'volume': 0},
{'timestamp': '2014-01-01T01:00:00', 'volume': 2},
]
xform = conversion.NetworkServiceTransformer()
usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'fake_meter': 1}, usage)
def test_only_pending_service(self):
"""Tests that the transformer correctly calculate the sum value.
"""
data = [
{'timestamp': '2014-01-01T00:00:00', 'volume': 2},
{'timestamp': '2014-01-01T00:10:00', 'volume': 2},
{'timestamp': '2014-01-01T01:00:00', 'volume': 2},
]
xform = conversion.NetworkServiceTransformer()
usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'fake_meter': 0}, usage)

17
distil/transformer/arithmetic.py
View File

@ -30,9 +30,11 @@ class MaxTransformer(BaseTransformer):
"""
def _transform_usage(self, meter_name, raw_data, start_at, end_at):
max_vol = max([v["volume"]
for v in raw_data]) if len(raw_data) else 0
max_vol = max([v["volume"] for v in raw_data]) if len(raw_data) else 0
max_vol = max_vol if max_vol else 0
hours = (end_at - start_at).total_seconds() / 3600.0
return {meter_name: max_vol * hours}
@ -47,13 +49,7 @@ class StorageMaxTransformer(BaseTransformer):
if not data:
return None
max_vol = max([v["volume"] for v in data])
if max_vol is None:
max_vol = 0
LOG.warning("None max_vol value for %s in window: %s - %s " %
(name, start.strftime(constants.iso_time),
end.strftime(constants.iso_time)))
max_vol = max([v["volume"] for v in data]) or 0
if "volume_type" in data[-1]['metadata']:
vtype = data[-1]['metadata']['volume_type']
@ -83,5 +79,6 @@ class SumTransformer(BaseTransformer):
'%Y-%m-%dT%H:%M:%S')
if t >= start_at and t < end_at:
sum_vol += sample["volume"]
sum_vol += sample["volume"] or 0
return {meter_name: sum_vol}

41
distil/transformer/conversion.py
View File

@ -12,7 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from datetime import datetime
from distil.transformer import BaseTransformer
from distil.common import constants
@ -29,14 +29,12 @@ class UpTimeTransformer(BaseTransformer):
# get tracked states from config
tracked = self.config['uptime']['tracked_states']
tracked_states = {constants.states[i] for i in tracked}
usage_dict = {}
def sort_and_clip_end(usage):
cleaned = (self._clean_entry(s) for s in usage)
clipped = (s for s in cleaned if s['timestamp'] < end)
return sorted(clipped, key=lambda x: x['timestamp'])
clipped = [s for s in cleaned if s['timestamp'] < end]
return clipped
state = sort_and_clip_end(data)
@ -57,7 +55,7 @@ class UpTimeTransformer(BaseTransformer):
usage_dict[flav] = usage_dict.get(flav, 0) + diff.total_seconds()
for val in state[1:]:
if last_state["volume"] in tracked_states:
if last_state["status"] in tracked:
diff = val["timestamp"] - last_timestamp
if val['timestamp'] > last_timestamp:
# if diff < 0 then we were looking back before the start
@ -70,29 +68,25 @@ class UpTimeTransformer(BaseTransformer):
# extend the last state we know about, to the end of the window,
# if we saw any actual uptime.
if (end and last_state['volume'] in tracked_states
and seen_sample_in_window):
if (end and last_state['status'] in tracked and seen_sample_in_window):
diff = end - last_timestamp
_add_usage(diff)
# map the flavors to names on the way out
return {openstack.get_flavor_name(f): v for f, v in usage_dict.items()}
return usage_dict
def _clean_entry(self, entry):
result = {
'volume': entry['volume'],
'flavor': entry['metadata'].get(
'flavor.id', entry['metadata'].get(
'instance_flavor_id', 0
'status': entry['metadata'].get(
'status', entry['metadata'].get(
'state', ""
)
),
'flavor': entry['metadata'].get('instance_type'),
'timestamp': datetime.strptime(
entry['timestamp'],
"%Y-%m-%dT%H:%M:%S"
)
}
try:
result['timestamp'] = datetime.datetime.strptime(
entry['timestamp'], constants.date_format)
except ValueError:
result['timestamp'] = datetime.datetime.strptime(
entry['timestamp'], constants.date_format_f)
return result
@ -126,7 +120,9 @@ class FromImageTransformer(BaseTransformer):
size = root_size
except KeyError:
pass
hours = (end - start).total_seconds() / 3600.0
return {service: size * hours}
@ -141,7 +137,8 @@ class NetworkServiceTransformer(BaseTransformer):
# blob/master/ceilometer/network/services/vpnaas.py#L55), so we have
# to check the volume to make sure only the active service is
# charged(0=inactive, 1=active).
max_vol = max([v["volume"] for v in data
if v["volume"] < 2]) if len(data) else 0
volumes = [v["volume"] for v in data if
v["volume"] < 2]
max_vol = max(volumes) if len(volumes) else 0
hours = (end - start).total_seconds() / 3600.0
return {name: max_vol * hours}

41
etc/distil.conf.sample
View File

@ -1,41 +0,0 @@
[DEFAULT]
debug = True
ignore_tenants = demo
timezone = Pacific/Auckland
host = localhost
port = 9999
[collector]
periodic_interval = 10
dawn_of_time = 2016-05-18 01:00:00
meter_mappings_file = /etc/distil/meter_mappings.yml
[rater]
rater_type = odoo
rate_file_path = /etc/distil/rates.csv
[odoo]
version=8.0
hostname=
port=443
protocol=jsonrpc+ssl
database=
user=
password=
[database]
connection = mysql://root:passw0rd@127.0.0.1/distil?charset=utf8
backend = sqlalchemy
[keystone_authtoken]
memcache_servers = 127.0.0.1:11211
signing_dir = /var/cache/distil
cafile = /opt/stack/data/ca-bundle.pem
auth_uri = http://127.0.0.1:5000
project_domain_id = default
project_name = service
user_domain_id = default
password = passw0rd
username = distil
auth_url = http://127.0.0.1:35357
auth_type = password

116
etc/transformer.yaml
View File

@ -1,116 +0,0 @@
# configuration for defining usage collection
collection:
# defines which meter is mapped to which transformer
meter_mappings:
# meter name as seen in ceilometer
state:
# type of resource it maps to (seen on sales order)
type: Virtual Machine
# which transformer to use
transformer: uptime
# what unit type is coming in via the meter
unit: second
metadata:
name:
sources:
# which keys to search for in the ceilometer entry metadata
# this can be more than one as metadata is inconsistent between
# source types
- display_name
availability zone:
sources:
- OS-EXT-AZ:availability_zone
ip.floating:
type: Floating IP
transformer: max
unit: hour
metadata:
ip address:
sources:
- floating_ip_address
volume.size:
type: Volume
transformer: max
unit: gigabyte
metadata:
name:
sources:
- display_name
availability zone:
sources:
- availability_zone
instance:
type: Volume
transformer: fromimage
unit: gigabyte
# if true allows id pattern, and metadata patterns
transform_info: True
# allows us to put the id into a pattern,
# only if transform_info is true,
# such as to append something to it
res_id_template: "%s-root_disk"
metadata:
name:
sources:
- display_name
template: "%s - root disk"
availability zone:
sources:
- availability_zone
image.size:
type: Image
transformer: max
unit: byte
metadata:
name:
sources:
- name
- properties.image_name
bandwidth:
type: Network Traffic
transformer: sum
unit: byte
metadata:
meter_label_id:
sources:
- label_id
network.services.vpn:
type: VPN
transformer: networkservice
unit: hour
metadata:
name:
sources:
- name
subnet:
sources:
- subnet_id
network:
type: Network
transformer: max
unit: hour
metadata:
name:
sources:
- name
# transformer configs
transformers:
uptime:
# states marked as "billable" for VMs.
tracked_states:
- active
- paused
- rescued
- resized
from_image:
service: volume.size
# What metadata values to check
md_keys:
- image_ref
- image_meta.base_image_ref
none_values:
- None
- ""
# where to get volume size from
size_keys:
- root_gb

26
etc/transformer.yaml.sample Normal file
View File

@ -0,0 +1,26 @@
uptime:
# states marked as "billable" for VMs.
tracked_states:
- active
- paused
- rescue
- rescued
- resize
- resized
- verify_resize
- suspended
# uncomment these when we want to bill shutdown:
# - shutoff
# - stopped
from_image:
service: b1.standard
# What metadata values to check
md_keys:
- image_ref
- image_meta.base_image_ref
none_values:
- None
- ""
# where to get volume size from
size_keys:
- root_gb