Merge "Use Mongo finalize to compute avg and duration"

ceilometer/storage/impl_mongodb.py

@@ -180,7 +180,7 @@ class Connection(base.Connection):
     function () {
         emit('statistics', { min : this.counter_volume,
                              max : this.counter_volume,
-                             qty : this.counter_volume,
+                             sum : this.counter_volume,
                              count : 1,
                              timestamp_min : this.timestamp,
                              timestamp_max : this.timestamp } )
@@ -196,7 +196,7 @@ class Connection(base.Connection):
             if ( values[i].max > res.max )
                res.max = values[i].max;
             res.count += values[i].count;
-            res.qty += values[i].qty;
+            res.sum += values[i].sum;
             if ( values[i].timestamp_min < res.timestamp_min )
                res.timestamp_min = values[i].timestamp_min;
             if ( values[i].timestamp_max > res.timestamp_max )
@@ -206,6 +206,13 @@ class Connection(base.Connection):
     }
     """)
 
+    FINALIZE_STATS = bson.code.Code("""
+    function (key, value) {
+        value.avg = value.sum / value.count;
+        value.duration = (value.timestamp_max - value.timestamp_min) / 1000;
+        return value;
+    }""")
+
     def __init__(self, conf):
         opts = self._parse_connection_url(conf.database_connection)
         LOG.info('connecting to MongoDB on %s:%s', opts['host'], opts['port'])
@@ -466,34 +473,20 @@ class Connection(base.Connection):
         results = self.db.meter.map_reduce(self.MAP_STATS,
                                            self.REDUCE_STATS,
                                            {'inline': 1},
+                                           finalize=self.FINALIZE_STATS,
                                            query=q,
                                            )
         if results['results']:
-            r = results['results'][0]['value']
-            (start, end) = self._fix_interval_min_max(r['timestamp_min'],
-                                                      r['timestamp_max'])
-        else:
-            start = None
-            end = None
-            r = {'count': 0,
-                 'min': None,
-                 'max': None,
-                 'avg': None,
-                 'qty': None,
-                 'duration': None,
-                 'duration_start': None,
-                 'duration_end': None,
-                 }
-        count = int(r['count'])
-        return {'min': r['min'],
-                'sum': r['qty'],
-                'count': count,
-                'avg': (r['qty'] / count) if count > 0 else None,
-                'max': r['max'],
-                'duration': 0,
-                'duration_start': start,
-                'duration_end': end,
-                }
+            return results['results'][0]['value']
+
+        return {'count': 0,
+                'min': None,
+                'max': None,
+                'avg': None,
+                'sum': None,
+                'duration': None,
+                'duration_start': None,
+                'duration_end': None}
 
     def get_volume_sum(self, event_filter):
         """Return the sum of the volume field for the events

ceilometer/storage/impl_sqlalchemy.py

@@ -407,7 +407,7 @@ class Connection(base.Connection):
                 'max': res[4],
                 'avg': (res[2] / count) if count > 0 else None,
                 'sum': res[2],
-                'duration': None,
+                'duration': (res[1] - res[0]).seconds,
                 'duration_start': res[0],
                 'duration_end': res[1],
                 }

tests/storage/base.py

@@ -769,6 +769,9 @@ class StatisticsTest(DBTestBase):
             meter='volume.size',
         )
         results = self.conn.get_meter_statistics(f)
+        self.assertEqual(results['duration'],
+                         (datetime.datetime(2012, 9, 25, 12, 32)
+                          - datetime.datetime(2012, 9, 25, 10, 30)).seconds)
         assert results['count'] == 3
         assert results['min'] == 8
         assert results['max'] == 10
@@ -783,6 +786,7 @@ class StatisticsTest(DBTestBase):
             end='2012-09-25T11:32:00',
         )
         results = self.conn.get_meter_statistics(f)
+        self.assertEqual(results['duration'], 0)
         assert results['count'] == 1
         assert results['min'] == 6
         assert results['max'] == 6
@@ -795,6 +799,9 @@ class StatisticsTest(DBTestBase):
             meter='volume.size',
         )
         results = self.conn.get_meter_statistics(f)
+        self.assertEqual(results['duration'],
+                         (datetime.datetime(2012, 9, 25, 12, 32)
+                          - datetime.datetime(2012, 9, 25, 10, 30)).seconds)
         assert results['count'] == 3
         assert results['min'] == 5
         assert results['max'] == 7

tools/test-requires

@@ -6,7 +6,8 @@ mox
 Babel>=0.9.6
 # NOTE(dhellmann): Ming is necessary to provide the Mongo-in-memory
 # implementation of MongoDB.
-Ming>=0.3.2
+# NOTE(jd): the support for finalize in map reduce in MIM is not yet merged
+git+http://git.naquadah.org/git/~jd/merciless.git#egg=Ming
 http://tarballs.openstack.org/nova/nova-master.tar.gz
 http://tarballs.openstack.org/glance/glance-master.tar.gz
 setuptools-git>=0.4

tools/test-requires-folsom

@@ -6,7 +6,8 @@ mock
 mox
 # NOTE(dhellmann): Ming is necessary to provide the Mongo-in-memory
 # implementation of MongoDB.
-Ming>=0.3.2
+# NOTE(jd): the support for finalize in map reduce in MIM is not yet merged
+git+http://git.naquadah.org/git/~jd/merciless.git#egg=Ming
 http://tarballs.openstack.org/glance/glance-stable-folsom.tar.gz
 setuptools-git>=0.4
 # 1.7.4 is the Folsom release