openstack/aodh
Code Issues Proposed changes

Enforce reverse time-order for sample return

Browse Source 
Implements: blueprint api-sample-sorted

Added a new test test_get_samples_in_default_order() for class RawSampleTest in
tests/storage/base.py to check that the samples are sorted by timestamp, from
newest to oldest, e.g. first sample is newest

Added more samples to the test database in tests/storage/base.py , to make
testing of time ordering more robust, and appropriately changed tests to match
the expected results from the new modified test database.

Added code to sort samples in reverse time order, in get_samples() of
ceilometer/storage/impl_mongodb.py and ceilometer/storage/impl_sqlalchemy.py

Change-Id: Ib822bbbfc8fd47cf6685926762295577b69fd417
This commit is contained in:
Terri Yu 2013-05-30 18:54:49 +00:00
parent 927eaf082b
commit 1961ec9c73
3 changed files with 53 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ceilometer/storage/impl_mongodb.py
View File

@ -448,7 +448,9 @@ class Connection(base.Connection):
if limit == 0: if limit == 0:
return return
q = make_query_from_filter(sample_filter, require_meter=False) q = make_query_from_filter(sample_filter, require_meter=False)
samples = self.db.meter.find(q).limit(limit or 0) samples = self.db.meter.find(q, limit=limit,
sort=[("timestamp", pymongo.DESCENDING)])
for s in samples: for s in samples:
# Remove the ObjectId generated by the database when # Remove the ObjectId generated by the database when
# the sample was inserted. It is an implementation # the sample was inserted. It is an implementation

3
ceilometer/storage/impl_sqlalchemy.py
View File

@ -23,6 +23,7 @@ import operator
import os import os
import uuid import uuid
from sqlalchemy import func from sqlalchemy import func
from sqlalchemy import desc
from ceilometer.openstack.common import log from ceilometer.openstack.common import log
from ceilometer.openstack.common import timeutils from ceilometer.openstack.common import timeutils
@ -337,7 +338,7 @@ class Connection(base.Connection):
require_meter=False) require_meter=False)
if limit: if limit:
query = query.limit(limit) query = query.limit(limit)
samples = query.all() samples = query.from_self().order_by(desc(Meter.timestamp)).all()
for s in samples: for s in samples:
# Remove the id generated by the database when # Remove the id generated by the database when

70
tests/storage/base.py
View File

@ -41,6 +41,17 @@ class DBTestBase(test_db.TestBase):
self.prepare_data() self.prepare_data()
def prepare_data(self): def prepare_data(self):
original_timestamps = [(2012, 7, 2, 10, 40), (2012, 7, 2, 10, 41),
(2012, 7, 2, 10, 41), (2012, 7, 2, 10, 42),
(2012, 7, 2, 10, 43)]
timestamps_for_test_samples_default_order = [(2012, 7, 2, 10, 44),
(2011, 5, 30, 18, 3),
(2012, 12, 1, 1, 25),
(2012, 2, 29, 6, 59),
(2013, 5, 31, 23, 7)]
timestamp_list = (original_timestamps +
timestamps_for_test_samples_default_order)
self.msgs = [] self.msgs = []
self.counter = counter.Counter( self.counter = counter.Counter(
'instance', 'instance',
@ -50,7 +61,7 @@ class DBTestBase(test_db.TestBase):
user_id='user-id', user_id='user-id',
project_id='project-id', project_id='project-id',
resource_id='resource-id', resource_id='resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40), timestamp=datetime.datetime(*timestamp_list[0]),
resource_metadata={'display_name': 'test-server', resource_metadata={'display_name': 'test-server',
'tag': 'self.counter', 'tag': 'self.counter',
} }
@ -71,7 +82,7 @@ class DBTestBase(test_db.TestBase):
user_id='user-id', user_id='user-id',
project_id='project-id', project_id='project-id',
resource_id='resource-id-alternate', resource_id='resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41), timestamp=datetime.datetime(*timestamp_list[1]),
resource_metadata={'display_name': 'test-server', resource_metadata={'display_name': 'test-server',
'tag': 'self.counter2', 'tag': 'self.counter2',
} }
@ -92,7 +103,7 @@ class DBTestBase(test_db.TestBase):
user_id='user-id-alternate', user_id='user-id-alternate',
project_id='project-id', project_id='project-id',
resource_id='resource-id-alternate', resource_id='resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41), timestamp=datetime.datetime(*timestamp_list[2]),
resource_metadata={'display_name': 'test-server', resource_metadata={'display_name': 'test-server',
'tag': 'self.counter3', 'tag': 'self.counter3',
} }
@ -105,7 +116,11 @@ class DBTestBase(test_db.TestBase):
self.conn.record_metering_data(self.msg3) self.conn.record_metering_data(self.msg3)
self.msgs.append(self.msg3) self.msgs.append(self.msg3)
for i in range(2, 4): start_idx = 3
end_idx = len(timestamp_list)
for i, ts in zip(range(start_idx - 1, end_idx - 1),
timestamp_list[start_idx:end_idx]):
c = counter.Counter( c = counter.Counter(
'instance', 'instance',
counter.TYPE_CUMULATIVE, counter.TYPE_CUMULATIVE,
@ -114,7 +129,7 @@ class DBTestBase(test_db.TestBase):
user_id='user-id-%s' % i, user_id='user-id-%s' % i,
project_id='project-id-%s' % i, project_id='project-id-%s' % i,
resource_id='resource-id-%s' % i, resource_id='resource-id-%s' % i,
timestamp=datetime.datetime(2012, 7, 2, 10, 40 + i), timestamp=datetime.datetime(*ts),
resource_metadata={'display_name': 'test-server', resource_metadata={'display_name': 'test-server',
'tag': 'counter-%s' % i}, 'tag': 'counter-%s' % i},
) )
@ -131,11 +146,10 @@ class UserTest(DBTestBase):
def test_get_users(self): def test_get_users(self):
users = self.conn.get_users() users = self.conn.get_users()
assert set(users) == set(['user-id', expected = set(['user-id', 'user-id-alternate', 'user-id-2',
'user-id-alternate', 'user-id-3', 'user-id-4', 'user-id-5', 'user-id-6',
'user-id-2', 'user-id-7', 'user-id-8'])
'user-id-3', self.assertEqual(set(users), expected)
])
def test_get_users_by_source(self): def test_get_users_by_source(self):
users = self.conn.get_users(source='test-1') users = self.conn.get_users(source='test-1')
@ -146,8 +160,10 @@ class ProjectTest(DBTestBase):
def test_get_projects(self): def test_get_projects(self):
projects = self.conn.get_projects() projects = self.conn.get_projects()
expected = set(['project-id', 'project-id-2', 'project-id-3']) expected = set(['project-id', 'project-id-2', 'project-id-3',
assert set(projects) == expected 'project-id-4', 'project-id-5', 'project-id-6',
'project-id-7', 'project-id-8'])
self.assertEqual(set(projects), expected)
def test_get_projects_by_source(self): def test_get_projects_by_source(self):
projects = self.conn.get_projects(source='test-1') projects = self.conn.get_projects(source='test-1')
@ -160,7 +176,7 @@ class ResourceTest(DBTestBase):
def test_get_resources(self): def test_get_resources(self):
msgs_sources = [msg['source'] for msg in self.msgs] msgs_sources = [msg['source'] for msg in self.msgs]
resources = list(self.conn.get_resources()) resources = list(self.conn.get_resources())
assert len(resources) == 4 self.assertEqual(len(resources), 9)
for resource in resources: for resource in resources:
if resource.resource_id != 'resource-id': if resource.resource_id != 'resource-id':
continue continue
@ -179,15 +195,17 @@ class ResourceTest(DBTestBase):
timestamp = datetime.datetime(2012, 7, 2, 10, 42) timestamp = datetime.datetime(2012, 7, 2, 10, 42)
resources = list(self.conn.get_resources(start_timestamp=timestamp)) resources = list(self.conn.get_resources(start_timestamp=timestamp))
resource_ids = [r.resource_id for r in resources] resource_ids = [r.resource_id for r in resources]
expected = set(['resource-id-2', 'resource-id-3']) expected = set(['resource-id-2', 'resource-id-3', 'resource-id-4',
assert set(resource_ids) == expected 'resource-id-6', 'resource-id-8'])
self.assertEqual(set(resource_ids), expected)
def test_get_resources_end_timestamp(self): def test_get_resources_end_timestamp(self):
timestamp = datetime.datetime(2012, 7, 2, 10, 42) timestamp = datetime.datetime(2012, 7, 2, 10, 42)
resources = list(self.conn.get_resources(end_timestamp=timestamp)) resources = list(self.conn.get_resources(end_timestamp=timestamp))
resource_ids = [r.resource_id for r in resources] resource_ids = [r.resource_id for r in resources]
expected = set(['resource-id', 'resource-id-alternate']) expected = set(['resource-id', 'resource-id-alternate',
assert set(resource_ids) == expected 'resource-id-5', 'resource-id-7'])
self.assertEqual(set(resource_ids), expected)
def test_get_resources_both_timestamps(self): def test_get_resources_both_timestamps(self):
start_ts = datetime.datetime(2012, 7, 2, 10, 42) start_ts = datetime.datetime(2012, 7, 2, 10, 42)
@ -220,7 +238,7 @@ class ResourceTest(DBTestBase):
got_not_imp = False got_not_imp = False
try: try:
resources = list(self.conn.get_resources(metaquery=q)) resources = list(self.conn.get_resources(metaquery=q))
assert len(resources) == 4 self.assertEqual(len(resources), 9)
except NotImplementedError: except NotImplementedError:
got_not_imp = True got_not_imp = True
self.assertTrue(got_not_imp) self.assertTrue(got_not_imp)
@ -233,7 +251,7 @@ class ResourceTest(DBTestBase):
def test_get_resources_by_empty_metaquery(self): def test_get_resources_by_empty_metaquery(self):
resources = list(self.conn.get_resources(metaquery={})) resources = list(self.conn.get_resources(metaquery={}))
self.assertTrue(len(resources) == 4) self.assertEqual(len(resources), 9)
class MeterTest(DBTestBase): class MeterTest(DBTestBase):
@ -241,7 +259,7 @@ class MeterTest(DBTestBase):
def test_get_meters(self): def test_get_meters(self):
msgs_sources = [msg['source'] for msg in self.msgs] msgs_sources = [msg['source'] for msg in self.msgs]
results = list(self.conn.get_meters()) results = list(self.conn.get_meters())
assert len(results) == 4 self.assertEqual(len(results), 9)
for meter in results: for meter in results:
self.assertIn(meter.source, msgs_sources) self.assertIn(meter.source, msgs_sources)
@ -259,14 +277,14 @@ class MeterTest(DBTestBase):
try: try:
results = list(self.conn.get_meters(metaquery=q)) results = list(self.conn.get_meters(metaquery=q))
assert results assert results
assert len(results) == 4 self.assertEqual(len(results), 9)
except NotImplementedError: except NotImplementedError:
got_not_imp = True got_not_imp = True
self.assertTrue(got_not_imp) self.assertTrue(got_not_imp)
def test_get_meters_by_empty_metaquery(self): def test_get_meters_by_empty_metaquery(self):
results = list(self.conn.get_meters(metaquery={})) results = list(self.conn.get_meters(metaquery={}))
self.assertTrue(len(results) == 4) self.assertEqual(len(results), 9)
class RawSampleTest(DBTestBase): class RawSampleTest(DBTestBase):
@ -281,6 +299,14 @@ class RawSampleTest(DBTestBase):
results = list(self.conn.get_samples(f, limit=3)) results = list(self.conn.get_samples(f, limit=3))
self.assertEqual(len(results), 3) self.assertEqual(len(results), 3)
def test_get_samples_in_default_order(self):
f = storage.SampleFilter()
prev_timestamp = None
for sample in self.conn.get_samples(f):
if prev_timestamp is not None:
self.assertTrue(prev_timestamp >= sample.timestamp)
prev_timestamp = sample.timestamp
def test_get_samples_by_user(self): def test_get_samples_by_user(self):
f = storage.SampleFilter(user='user-id') f = storage.SampleFilter(user='user-id')
results = list(self.conn.get_samples(f)) results = list(self.conn.get_samples(f))