From 8fd8d4e40a03613c794bb8e86be7b11851a2e63f Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Wed, 6 Feb 2013 11:33:34 +0100 Subject: [PATCH] Use Mongo finalize to compute avg and duration This reduces the amount of things we do in Python and prepare for future usage of more aggregation. It also returns real duration; API is then free to override this with whatever it wants (and it does so). Change-Id: I9850a4ab146612fbf85587f5553881a22acac67b Signed-off-by: Julien Danjou --- ceilometer/storage/impl_mongodb.py | 47 ++++++++++++--------------- ceilometer/storage/impl_sqlalchemy.py | 2 +- tests/storage/base.py | 7 ++++ tools/test-requires | 3 +- tools/test-requires-folsom | 3 +- 5 files changed, 32 insertions(+), 30 deletions(-) diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py index 01b97da75..c201dae8f 100644 --- a/ceilometer/storage/impl_mongodb.py +++ b/ceilometer/storage/impl_mongodb.py @@ -180,7 +180,7 @@ class Connection(base.Connection): function () { emit('statistics', { min : this.counter_volume, max : this.counter_volume, - qty : this.counter_volume, + sum : this.counter_volume, count : 1, timestamp_min : this.timestamp, timestamp_max : this.timestamp } ) @@ -196,7 +196,7 @@ class Connection(base.Connection): if ( values[i].max > res.max ) res.max = values[i].max; res.count += values[i].count; - res.qty += values[i].qty; + res.sum += values[i].sum; if ( values[i].timestamp_min < res.timestamp_min ) res.timestamp_min = values[i].timestamp_min; if ( values[i].timestamp_max > res.timestamp_max ) @@ -206,6 +206,13 @@ class Connection(base.Connection): } """) + FINALIZE_STATS = bson.code.Code(""" + function (key, value) { + value.avg = value.sum / value.count; + value.duration = (value.timestamp_max - value.timestamp_min) / 1000; + return value; + }""") + def __init__(self, conf): opts = self._parse_connection_url(conf.database_connection) LOG.info('connecting to MongoDB on %s:%s', opts['host'], opts['port']) @@ -466,34 +473,20 @@ class Connection(base.Connection): results = self.db.meter.map_reduce(self.MAP_STATS, self.REDUCE_STATS, {'inline': 1}, + finalize=self.FINALIZE_STATS, query=q, ) if results['results']: - r = results['results'][0]['value'] - (start, end) = self._fix_interval_min_max(r['timestamp_min'], - r['timestamp_max']) - else: - start = None - end = None - r = {'count': 0, - 'min': None, - 'max': None, - 'avg': None, - 'qty': None, - 'duration': None, - 'duration_start': None, - 'duration_end': None, - } - count = int(r['count']) - return {'min': r['min'], - 'sum': r['qty'], - 'count': count, - 'avg': (r['qty'] / count) if count > 0 else None, - 'max': r['max'], - 'duration': 0, - 'duration_start': start, - 'duration_end': end, - } + return results['results'][0]['value'] + + return {'count': 0, + 'min': None, + 'max': None, + 'avg': None, + 'sum': None, + 'duration': None, + 'duration_start': None, + 'duration_end': None} def get_volume_sum(self, event_filter): """Return the sum of the volume field for the events diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py index cf758a8e8..14b5da9e6 100644 --- a/ceilometer/storage/impl_sqlalchemy.py +++ b/ceilometer/storage/impl_sqlalchemy.py @@ -407,7 +407,7 @@ class Connection(base.Connection): 'max': res[4], 'avg': (res[2] / count) if count > 0 else None, 'sum': res[2], - 'duration': None, + 'duration': (res[1] - res[0]).seconds, 'duration_start': res[0], 'duration_end': res[1], } diff --git a/tests/storage/base.py b/tests/storage/base.py index ad5bff2ca..769d35345 100644 --- a/tests/storage/base.py +++ b/tests/storage/base.py @@ -769,6 +769,9 @@ class StatisticsTest(DBTestBase): meter='volume.size', ) results = self.conn.get_meter_statistics(f) + self.assertEqual(results['duration'], + (datetime.datetime(2012, 9, 25, 12, 32) + - datetime.datetime(2012, 9, 25, 10, 30)).seconds) assert results['count'] == 3 assert results['min'] == 8 assert results['max'] == 10 @@ -783,6 +786,7 @@ class StatisticsTest(DBTestBase): end='2012-09-25T11:32:00', ) results = self.conn.get_meter_statistics(f) + self.assertEqual(results['duration'], 0) assert results['count'] == 1 assert results['min'] == 6 assert results['max'] == 6 @@ -795,6 +799,9 @@ class StatisticsTest(DBTestBase): meter='volume.size', ) results = self.conn.get_meter_statistics(f) + self.assertEqual(results['duration'], + (datetime.datetime(2012, 9, 25, 12, 32) + - datetime.datetime(2012, 9, 25, 10, 30)).seconds) assert results['count'] == 3 assert results['min'] == 5 assert results['max'] == 7 diff --git a/tools/test-requires b/tools/test-requires index 66b0be3e4..d768aba1d 100644 --- a/tools/test-requires +++ b/tools/test-requires @@ -5,7 +5,8 @@ mox Babel>=0.9.6 # NOTE(dhellmann): Ming is necessary to provide the Mongo-in-memory # implementation of MongoDB. -Ming>=0.3.2 +# NOTE(jd): the support for finalize in map reduce in MIM is not yet merged +git+http://git.naquadah.org/git/~jd/merciless.git#egg=Ming http://tarballs.openstack.org/nova/nova-master.tar.gz http://tarballs.openstack.org/glance/glance-master.tar.gz setuptools-git>=0.4 diff --git a/tools/test-requires-folsom b/tools/test-requires-folsom index 9ac3453cc..0c52ed0fb 100644 --- a/tools/test-requires-folsom +++ b/tools/test-requires-folsom @@ -5,7 +5,8 @@ mock mox # NOTE(dhellmann): Ming is necessary to provide the Mongo-in-memory # implementation of MongoDB. -Ming>=0.3.2 +# NOTE(jd): the support for finalize in map reduce in MIM is not yet merged +git+http://git.naquadah.org/git/~jd/merciless.git#egg=Ming http://tarballs.openstack.org/glance/glance-stable-folsom.tar.gz setuptools-git>=0.4 # 1.7.4 is the Folsom release