From 7c4c0e8fc098fbadec2e30ea2bdf1068b5b4288a Mon Sep 17 00:00:00 2001 From: Eoghan Glynn Date: Thu, 9 Jan 2014 11:11:43 +0000 Subject: [PATCH] Remove redundant meter (name,type,unit) tuples from Resource model Fixes bug 1267162 Previously, a list of meter (name,type,unit) tuples with per-sample cardinality was added to the Resource model by the storage driver get_resource method. However this potential huge and highly repetitive list was then promptly discarded by the API layer, as the meter links in the API Resource representation are populated via subsequent calls into the storage driver get_meters method. Now, we avoid the scaling issue by suppressing the creation of this unused meter list. Change-Id: I18dee7552836c724798ea1340e2e933c09d8a294 --- ceilometer/storage/impl_db2.py | 10 +------ ceilometer/storage/impl_hbase.py | 17 +++--------- ceilometer/storage/impl_mongodb.py | 10 ------- ceilometer/storage/impl_sqlalchemy.py | 8 ------ ceilometer/storage/models.py | 26 +------------------ .../tests/storage/test_storage_scenarios.py | 2 -- 6 files changed, 5 insertions(+), 68 deletions(-) diff --git a/ceilometer/storage/impl_db2.py b/ceilometer/storage/impl_db2.py index d7677f365..5d3276fc5 100644 --- a/ceilometer/storage/impl_db2.py +++ b/ceilometer/storage/impl_db2.py @@ -425,17 +425,10 @@ class Connection(base.Connection): resource = lambda x: x['resource_id'] meters = self.db.meter.find(q, sort=sort_instructions) for resource_id, r_meters in itertools.groupby(meters, key=resource): - resource_meters = [] # Because we have to know first/last timestamp, and we need a full # list of references to the resource's meters, we need a tuple # here. r_meters = tuple(r_meters) - for meter in r_meters: - resource_meters.append(models.ResourceMeter( - counter_name=meter['counter_name'], - counter_type=meter['counter_type'], - counter_unit=meter.get('counter_unit', '')) - ) latest_meter = r_meters[0] last_ts = latest_meter['timestamp'] first_ts = r_meters[-1]['timestamp'] @@ -446,8 +439,7 @@ class Connection(base.Connection): last_sample_timestamp=last_ts, source=latest_meter['source'], user_id=latest_meter['user_id'], - metadata=latest_meter['resource_metadata'], - meter=resource_meters) + metadata=latest_meter['resource_metadata']) def get_meters(self, user=None, project=None, resource=None, source=None, metaquery={}, pagination=None): diff --git a/ceilometer/storage/impl_hbase.py b/ceilometer/storage/impl_hbase.py index 465c30240..349c375a4 100644 --- a/ceilometer/storage/impl_hbase.py +++ b/ceilometer/storage/impl_hbase.py @@ -300,7 +300,7 @@ class Connection(base.Connection): if pagination: raise NotImplementedError(_('Pagination not implemented')) - def make_resource(data, first_ts, last_ts, meter_refs): + def make_resource(data, first_ts, last_ts): """Transform HBase fields to Resource model.""" # convert HBase metadata e.g. f:r_display_name to display_name data['f:metadata'] = _metadata_from_document(data) @@ -313,10 +313,6 @@ class Connection(base.Connection): source=data['f:source'], user_id=data['f:user_id'], metadata=data['f:metadata'], - meter=[ - models.ResourceMeter(*(m.split("!"))) - for m in meter_refs - ], ) meter_table = self.conn.table(self.METER_TABLE) @@ -343,11 +339,6 @@ class Connection(base.Connection): meters, key=_resource_id_from_record_tuple): meter_rows = [data[1] for data in sorted( r_meters, key=_timestamp_from_record_tuple)] - meter_references = [ - _format_meter_reference(m['f:counter_name'], - m['f:counter_type'], - m['f:counter_unit']) - for m in meter_rows] latest_data = meter_rows[-1] min_ts = timeutils.parse_strtime(meter_rows[0]['f:timestamp']) @@ -358,15 +349,13 @@ class Connection(base.Connection): yield make_resource( latest_data, min_ts, - max_ts, - meter_references + max_ts ) else: yield make_resource( latest_data, min_ts, - max_ts, - meter_references + max_ts ) def get_meters(self, user=None, project=None, resource=None, source=None, diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py index 86bfccfad..38f65a8e6 100644 --- a/ceilometer/storage/impl_mongodb.py +++ b/ceilometer/storage/impl_mongodb.py @@ -661,16 +661,6 @@ class Connection(base.Connection): last_sample_timestamp=result['last_sample_timestamp'], source=result['source'], metadata=result['metadata'], - meter=[ - models.ResourceMeter( - counter_name=m_n, - counter_type=m_t, - counter_unit=m_u, - ) - for m_n, m_u, m_t in zip(result['meters_name'], - result['meters_unit'], - result['meters_type']) - ], ) def get_meters(self, user=None, project=None, resource=None, source=None, diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py index 8e31d18c3..8cd455d71 100644 --- a/ceilometer/storage/impl_sqlalchemy.py +++ b/ceilometer/storage/impl_sqlalchemy.py @@ -467,14 +467,6 @@ class Connection(base.Connection): source=meter.sources[0].id, user_id=meter.user_id, metadata=meter.resource_metadata, - meter=[ - api_models.ResourceMeter( - counter_name=m.counter_name, - counter_type=m.counter_type, - counter_unit=m.counter_unit, - ) - for m in meter.resource.meters - ], ) @staticmethod diff --git a/ceilometer/storage/models.py b/ceilometer/storage/models.py index 44c206a59..55b8e69e0 100644 --- a/ceilometer/storage/models.py +++ b/ceilometer/storage/models.py @@ -140,8 +140,7 @@ class Resource(Model): def __init__(self, resource_id, project_id, first_sample_timestamp, last_sample_timestamp, - source, user_id, metadata, - meter): + source, user_id, metadata): """Create a new resource. :param resource_id: UUID of the resource @@ -151,7 +150,6 @@ class Resource(Model): :param source: the identifier for the user/project id definition :param user_id: UUID of user owning the resource :param metadata: most current metadata for the resource (a dict) - :param meter: list of the meters reporting data for the resource, """ Model.__init__(self, resource_id=resource_id, @@ -161,28 +159,6 @@ class Resource(Model): source=source, user_id=user_id, metadata=metadata, - meter=meter, - ) - - -class ResourceMeter(Model): - """The definitions of the meters for which data has been collected - for a resource. - - See Resource.meter field. - """ - - def __init__(self, counter_name, counter_type, counter_unit): - """Create a new resource meter. - - :param counter_name: the name of the counter updating the resource - :param counter_type: one of gauge, delta, cumulative - :param counter_unit: official units name for the sample data - """ - Model.__init__(self, - counter_name=counter_name, - counter_type=counter_type, - counter_unit=counter_unit, ) diff --git a/ceilometer/tests/storage/test_storage_scenarios.py b/ceilometer/tests/storage/test_storage_scenarios.py index ceb16e1bb..8da2d2f6c 100644 --- a/ceilometer/tests/storage/test_storage_scenarios.py +++ b/ceilometer/tests/storage/test_storage_scenarios.py @@ -173,8 +173,6 @@ class ResourceTest(DBTestBase, self.assertIn(resource.source, msgs_sources) self.assertEqual(resource.user_id, 'user-id') self.assertEqual(resource.metadata['display_name'], 'test-server') - self.assertIn(models.ResourceMeter('instance', 'cumulative', ''), - resource.meter) break else: assert False, 'Never found resource-id'