From 13764537a465b05b4d7fa864d30c81f7192d2490 Mon Sep 17 00:00:00 2001 From: Gordon Chung Date: Mon, 9 Dec 2013 17:53:04 -0500 Subject: [PATCH] add newly added constraints to expire clear_expired_metering_data - cascade deletes to metadata tables on meter deletion - delete associations on object deletion - add check to not delete user/project entries associated with Alarm Change-Id: I24cbaea9ede6d142dc22eaf070b3f7d1f14b484a Closes-Bug: #1258071 --- ceilometer/storage/impl_sqlalchemy.py | 57 +++++++++++++------ ceilometer/storage/sqlalchemy/models.py | 10 +++- .../tests/storage/test_impl_sqlalchemy.py | 42 ++++++++++++++ .../tests/storage/test_storage_scenarios.py | 50 +++++++++++++++- 4 files changed, 139 insertions(+), 20 deletions(-) diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py index 1a041f549..e515afee3 100644 --- a/ceilometer/storage/impl_sqlalchemy.py +++ b/ceilometer/storage/impl_sqlalchemy.py @@ -279,28 +279,49 @@ class Connection(base.Connection): :param ttl: Number of seconds to keep records for. """ + session = sqlalchemy_session.get_session() - query = session.query(models.Meter.id) - end = timeutils.utcnow() - datetime.timedelta(seconds=ttl) - query = query.filter(models.Meter.timestamp < end) - query.delete() + with session.begin(): + end = timeutils.utcnow() - datetime.timedelta(seconds=ttl) + meter_query = session.query(models.Meter)\ + .filter(models.Meter.timestamp < end) + for meter_obj in meter_query.all(): + session.delete(meter_obj) - query = session.query(models.User.id).filter(~models.User.id.in_( - session.query(models.Meter.user_id).group_by(models.Meter.user_id) - )) - query.delete(synchronize_session='fetch') + query = session.query(models.User).filter( + ~models.User.id.in_(session.query(models.Meter.user_id) + .group_by(models.Meter.user_id)), + ~models.User.id.in_(session.query(models.Alarm.user_id) + .group_by(models.Alarm.user_id)), + ~models.User.id.in_(session.query(models.AlarmChange.user_id) + .group_by(models.AlarmChange.user_id)) + ) + for user_obj in query.all(): + session.delete(user_obj) - query = session.query(models.Project.id)\ - .filter(~models.Project.id.in_( - session.query(models.Meter.project_id).group_by( - models.Meter.project_id))) - query.delete(synchronize_session='fetch') + query = session.query(models.Project)\ + .filter(~models.Project.id.in_( + session.query(models.Meter.project_id) + .group_by(models.Meter.project_id)), + ~models.Project.id.in_( + session.query(models.Alarm.project_id) + .group_by(models.Alarm.project_id)), + ~models.Project.id.in_( + session.query(models.AlarmChange.project_id) + .group_by(models.AlarmChange.project_id)), + ~models.Project.id.in_( + session.query(models.AlarmChange.on_behalf_of) + .group_by(models.AlarmChange.on_behalf_of)) + ) + for project_obj in query.all(): + session.delete(project_obj) - query = session.query(models.Resource.id)\ - .filter(~models.Resource.id.in_( - session.query(models.Meter.resource_id).group_by( - models.Meter.resource_id))) - query.delete(synchronize_session='fetch') + query = session.query(models.Resource)\ + .filter(~models.Resource.id.in_( + session.query(models.Meter.resource_id).group_by( + models.Meter.resource_id))) + for res_obj in query.all(): + session.delete(res_obj) @staticmethod def get_users(source=None): diff --git a/ceilometer/storage/sqlalchemy/models.py b/ceilometer/storage/sqlalchemy/models.py index 9c3978540..ed7cd0066 100644 --- a/ceilometer/storage/sqlalchemy/models.py +++ b/ceilometer/storage/sqlalchemy/models.py @@ -201,7 +201,6 @@ class Meter(Base): ) id = Column(Integer, primary_key=True) counter_name = Column(String(255)) - sources = relationship("Source", secondary=lambda: sourceassoc) user_id = Column(String(255), ForeignKey('user.id')) project_id = Column(String(255), ForeignKey('project.id')) resource_id = Column(String(255), ForeignKey('resource.id')) @@ -212,6 +211,15 @@ class Meter(Base): timestamp = Column(PreciseTimestamp(), default=timeutils.utcnow) message_signature = Column(String(1000)) message_id = Column(String(1000)) + sources = relationship("Source", secondary=lambda: sourceassoc) + meta_text = relationship("MetaText", backref="meter", + cascade="all, delete-orphan") + meta_float = relationship("MetaFloat", backref="meter", + cascade="all, delete-orphan") + meta_int = relationship("MetaBigInt", backref="meter", + cascade="all, delete-orphan") + meta_bool = relationship("MetaBool", backref="meter", + cascade="all, delete-orphan") class User(Base): diff --git a/ceilometer/tests/storage/test_impl_sqlalchemy.py b/ceilometer/tests/storage/test_impl_sqlalchemy.py index 3b2a5c331..f8f0ec283 100644 --- a/ceilometer/tests/storage/test_impl_sqlalchemy.py +++ b/ceilometer/tests/storage/test_impl_sqlalchemy.py @@ -28,9 +28,12 @@ import repr from mock import patch +import ceilometer.openstack.common.db.sqlalchemy.session as sqlalchemy_session +from ceilometer.openstack.common import timeutils from ceilometer.storage import models from ceilometer.storage.sqlalchemy import models as sql_models from ceilometer.tests import db as tests_db +from ceilometer.tests.storage import test_storage_scenarios as scenarios from ceilometer import utils @@ -175,3 +178,42 @@ class ModelTest(tests_db.TestBase): def test_model_table_args(self): self.assertIsNotNone(sql_models.table_args()) + + +class RelationshipTest(scenarios.DBTestBase): + database_connection = 'mysql://localhost' + + def test_clear_metering_data_meta_tables(self): + timeutils.utcnow.override_time = datetime.datetime(2012, 7, 2, 10, 45) + self.conn.clear_expired_metering_data(3 * 60) + + session = sqlalchemy_session.get_session() + meta_tables = [sql_models.MetaText, sql_models.MetaFloat, + sql_models.MetaBigInt, sql_models.MetaBool] + for table in meta_tables: + self.assertEqual(session.query(table) + .filter(~table.id.in_( + session.query(sql_models.Meter.id) + .group_by(sql_models.Meter.id) + )).count(), 0) + + def test_clear_metering_data_associations(self): + timeutils.utcnow.override_time = datetime.datetime(2012, 7, 2, 10, 45) + self.conn.clear_expired_metering_data(3 * 60) + + session = sqlalchemy_session.get_session() + self.assertEqual(session.query(sql_models.sourceassoc) + .filter(~sql_models.sourceassoc.c.meter_id.in_( + session.query(sql_models.Meter.id) + .group_by(sql_models.Meter.id) + )).count(), 0) + self.assertEqual(session.query(sql_models.sourceassoc) + .filter(~sql_models.sourceassoc.c.project_id.in_( + session.query(sql_models.Project.id) + .group_by(sql_models.Project.id) + )).count(), 0) + self.assertEqual(session.query(sql_models.sourceassoc) + .filter(~sql_models.sourceassoc.c.user_id.in_( + session.query(sql_models.User.id) + .group_by(sql_models.User.id) + )).count(), 0) diff --git a/ceilometer/tests/storage/test_storage_scenarios.py b/ceilometer/tests/storage/test_storage_scenarios.py index 374cc653e..7e5c4d5f3 100644 --- a/ceilometer/tests/storage/test_storage_scenarios.py +++ b/ceilometer/tests/storage/test_storage_scenarios.py @@ -16,7 +16,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -""" Base classes for DB backend implemtation test +""" Base classes for DB backend implementation test """ import datetime @@ -662,6 +662,54 @@ class RawSampleTest(DBTestBase, results = list(self.conn.get_resources()) self.assertEqual(len(results), 9) + def test_clear_metering_data_with_alarms(self): + # NOTE(jd) Override this test in MongoDB because our code doesn't clear + # the collections, this is handled by MongoDB TTL feature. + if self.CONF.database.connection.startswith('mongodb://'): + return + + alarm = models.Alarm(alarm_id='r3d', + enabled=True, + type='threshold', + name='red-alert', + description='my red-alert', + timestamp=None, + user_id='user-id', + project_id='project-id', + state="insufficient data", + state_timestamp=None, + ok_actions=[], + alarm_actions=['http://nowhere/alarms'], + insufficient_data_actions=[], + repeat_actions=False, + rule=dict(comparison_operator='eq', + threshold=36, + statistic='count', + evaluation_periods=1, + period=60, + meter_name='test.one', + query=[{'field': 'key', + 'op': 'eq', + 'value': 'value', + 'type': 'string'}]), + ) + + self.conn.create_alarm(alarm) + timeutils.utcnow.override_time = datetime.datetime(2012, 7, 2, 10, 45) + self.conn.clear_expired_metering_data(5) + # user and project with Alarms associated with it aren't deleted. + f = storage.SampleFilter(meter='instance') + results = list(self.conn.get_samples(f)) + self.assertEqual(len(results), 2) + results = list(self.conn.get_users()) + self.assertEqual(len(results), 3) + self.assertIn('user-id', results) + results = list(self.conn.get_projects()) + self.assertEqual(len(results), 3) + self.assertIn('project-id', results) + results = list(self.conn.get_resources()) + self.assertEqual(len(results), 2) + class StatisticsTest(DBTestBase, tests_db.MixinTestsWithBackendScenarios):