From b41d8243df7b711df98d60fbc7f7e7ab3602de16 Mon Sep 17 00:00:00 2001 From: gordon chung Date: Tue, 21 Oct 2014 16:23:34 -0400 Subject: [PATCH] modify events sql schema to reduce empty columns in current design, three columns in Trait table are always empty. this adds size to each row. since we are already checking for type of value, we should just drop the items into their own tables. this patch: - moves trait table to four smaller tables similar to metadata - does not map traits to dict-like attribute in events model as query performance is horrible. - bulk inserts traits this patch aims to improve write performance. Closes-Bug: #1381736 Change-Id: I2bd4c16b57fe2c1ad67646ccd4417078f0f0f734 --- ceilometer/event/storage/impl_sqlalchemy.py | 358 +++++++++--------- ceilometer/event/storage/models.py | 1 + .../versions/041_expand_event_traits.py | 107 ++++++ ceilometer/storage/sqlalchemy/models.py | 112 ++---- ceilometer/storage/sqlalchemy/utils.py | 15 - .../tests/storage/test_impl_sqlalchemy.py | 87 +---- 6 files changed, 346 insertions(+), 334 deletions(-) create mode 100644 ceilometer/storage/sqlalchemy/migrate_repo/versions/041_expand_event_traits.py diff --git a/ceilometer/event/storage/impl_sqlalchemy.py b/ceilometer/event/storage/impl_sqlalchemy.py index 35f992ce6..30f97bc92 100644 --- a/ceilometer/event/storage/impl_sqlalchemy.py +++ b/ceilometer/event/storage/impl_sqlalchemy.py @@ -14,13 +14,11 @@ """SQLAlchemy storage backend.""" from __future__ import absolute_import -import operator import os from oslo.db import exception as dbexc from oslo.db.sqlalchemy import session as db_session from oslo_config import cfg -import six import sqlalchemy as sa from ceilometer.event.storage import base @@ -28,7 +26,6 @@ from ceilometer.event.storage import models as api_models from ceilometer.i18n import _ from ceilometer.openstack.common import log from ceilometer.storage.sqlalchemy import models -from ceilometer.storage.sqlalchemy import utils as sql_utils from ceilometer import utils LOG = log.getLogger(__name__) @@ -44,6 +41,36 @@ AVAILABLE_STORAGE_CAPABILITIES = { } +TRAIT_MAPLIST = [(api_models.Trait.NONE_TYPE, models.TraitText), + (api_models.Trait.TEXT_TYPE, models.TraitText), + (api_models.Trait.INT_TYPE, models.TraitInt), + (api_models.Trait.FLOAT_TYPE, models.TraitFloat), + (api_models.Trait.DATETIME_TYPE, models.TraitDatetime)] + + +TRAIT_ID_TO_MODEL = dict((x, y) for x, y in TRAIT_MAPLIST) +TRAIT_MODEL_TO_ID = dict((y, x) for x, y in TRAIT_MAPLIST) + + +trait_models_dict = {'string': models.TraitText, + 'integer': models.TraitInt, + 'datetime': models.TraitDatetime, + 'float': models.TraitFloat} + + +def _build_trait_query(session, trait_type, key, value, op='eq'): + trait_model = trait_models_dict[trait_type] + op_dict = {'eq': (trait_model.value == value), + 'lt': (trait_model.value < value), + 'le': (trait_model.value <= value), + 'gt': (trait_model.value > value), + 'ge': (trait_model.value >= value), + 'ne': (trait_model.value != value)} + conditions = [trait_model.key == key, op_dict[op]] + return (session.query(trait_model.event_id.label('ev_id')) + .filter(*conditions)) + + class Connection(base.Connection): """Put the event data into a SQLAlchemy database. @@ -61,21 +88,31 @@ class Connection(base.Connection): generated = timestamp of event event_type_id = event type -> eventtype.id } - - Trait - - trait value + - TraitInt + - int trait value - { event_id: event -> event.id - trait_type_id: trait type -> traittype.id - t_string: string value - t_float: float value - t_int: integer value - t_datetime: timestamp value + key: trait type + value: integer value } - - TraitType - - trait definition - - { id: trait id - desc: description of trait - data_type: data type (integer that maps to datatype) + - TraitDatetime + - int trait value + - { event_id: event -> event.id + key: trait type + value: datetime value } + - TraitText + - int trait value + - { event_id: event -> event.id + key: trait type + value: text value + } + - TraitFloat + - int trait value + - { event_id: event -> event.id + key: trait type + value: float value + } + """ CAPABILITIES = utils.update_nested(base.Connection.CAPABILITIES, AVAILABLE_CAPABILITIES) @@ -108,37 +145,6 @@ class Connection(base.Connection): self._engine_facade._session_maker.close_all() engine.dispose() - def _get_or_create_trait_type(self, trait_type, data_type, session=None): - """Find if this trait already exists in the database. - - If it does not, create a new entry in the trait type table. - """ - if session is None: - session = self._engine_facade.get_session() - with session.begin(subtransactions=True): - tt = session.query(models.TraitType).filter( - models.TraitType.desc == trait_type, - models.TraitType.data_type == data_type).first() - if not tt: - tt = models.TraitType(trait_type, data_type) - session.add(tt) - return tt - - def _make_trait(self, trait_model, event, session=None): - """Make a new Trait from a Trait model. - - Doesn't flush or add to session. - """ - trait_type = self._get_or_create_trait_type(trait_model.name, - trait_model.dtype, - session) - value_map = models.Trait._value_map - values = {'t_string': None, 't_float': None, - 't_int': None, 't_datetime': None} - value = trait_model.value - values[value_map[trait_model.dtype]] = value - return models.Trait(trait_type, event, **values) - def _get_or_create_event_type(self, event_type, session=None): """Check if an event type with the supplied name is already exists. @@ -154,27 +160,6 @@ class Connection(base.Connection): session.add(et) return et - def _record_event(self, session, event_model): - """Store a single Event, including related Traits.""" - with session.begin(subtransactions=True): - event_type = self._get_or_create_event_type(event_model.event_type, - session=session) - - event = models.Event(event_model.message_id, event_type, - event_model.generated) - session.add(event) - - new_traits = [] - if event_model.traits: - for trait in event_model.traits: - t = self._make_trait(trait, event, session=session) - session.add(t) - new_traits.append(t) - - # Note: we don't flush here, explicitly (unless a new trait or event - # does it). Otherwise, just wait until all the Events are staged. - return event, new_traits - def record_events(self, event_models): """Write the events to SQL database via sqlalchemy. @@ -188,22 +173,43 @@ class Connection(base.Connection): TraitTypes are added along the way. """ session = self._engine_facade.get_session() - events = [] problem_events = [] for event_model in event_models: event = None try: with session.begin(): - event = self._record_event(session, event_model) + event_type = self._get_or_create_event_type( + event_model.event_type, session=session) + event = models.Event(event_model.message_id, event_type, + event_model.generated) + session.add(event) + session.flush() + + if event_model.traits: + trait_map = {} + for trait in event_model.traits: + if trait_map.get(trait.dtype) is None: + trait_map[trait.dtype] = [] + trait_map[trait.dtype].append( + {'event_id': event.id, + 'key': trait.name, + 'value': trait.value}) + for dtype in trait_map.keys(): + model = TRAIT_ID_TO_MODEL[dtype] + session.execute(model.__table__.insert(), + trait_map[dtype]) except dbexc.DBDuplicateEntry as e: LOG.exception(_("Failed to record duplicated event: %s") % e) problem_events.append((api_models.Event.DUPLICATE, event_model)) + except KeyError as e: + LOG.exception(_('Failed to record event: %s') % e) + problem_events.append((api_models.Event.INCOMPATIBLE_TRAIT, + event_model)) except Exception as e: LOG.exception(_('Failed to record event: %s') % e) problem_events.append((api_models.Event.UNKNOWN_PROBLEM, event_model)) - events.append(event) return problem_events def get_events(self, event_filter): @@ -212,10 +218,7 @@ class Connection(base.Connection): :param event_filter: EventFilter instance """ - start = event_filter.start_timestamp - end = event_filter.end_timestamp session = self._engine_facade.get_session() - LOG.debug(_("Getting events that match filter: %s") % event_filter) with session.begin(): event_query = session.query(models.Event) @@ -233,80 +236,98 @@ class Connection(base.Connection): # Build up the where conditions event_filter_conditions = [] if event_filter.message_id: - event_filter_conditions.append(models.Event.message_id == - event_filter.message_id) - if start: - event_filter_conditions.append(models.Event.generated >= start) - if end: - event_filter_conditions.append(models.Event.generated <= end) - + event_filter_conditions.append( + models.Event.message_id == event_filter.message_id) + if event_filter.start_timestamp: + event_filter_conditions.append( + models.Event.generated >= event_filter.start_timestamp) + if event_filter.end_timestamp: + event_filter_conditions.append( + models.Event.generated <= event_filter.end_timestamp) if event_filter_conditions: event_query = (event_query. filter(sa.and_(*event_filter_conditions))) - event_models_dict = {} + trait_subq = None + # Build trait filter if event_filter.traits_filter: + trait_qlist = [] for trait_filter in event_filter.traits_filter: - - # Build a sub query that joins Trait to TraitType - # where the trait name matches - trait_name = trait_filter.pop('key') + key = trait_filter.pop('key') op = trait_filter.pop('op', 'eq') - conditions = [models.Trait.trait_type_id == - models.TraitType.id, - models.TraitType.desc == trait_name] + trait_qlist.append(_build_trait_query( + session, trait_filter.keys()[0], key, + trait_filter.values()[0], op)) + trait_subq = trait_qlist.pop() + if trait_qlist: + trait_subq = trait_subq.intersect(*trait_qlist) + trait_subq = trait_subq.subquery() - for key, value in six.iteritems(trait_filter): - sql_utils.trait_op_condition(conditions, - key, value, op) + query = (session.query(models.Event.id) + .join(models.EventType, + sa.and_(*event_join_conditions))) + if trait_subq is not None: + query = query.join(trait_subq, + trait_subq.c.ev_id == models.Event.id) + if event_filter_conditions: + query = query.filter(sa.and_(*event_filter_conditions)) - trait_query = (session.query(models.Trait.event_id). - join(models.TraitType, - sa.and_(*conditions)).subquery()) + event_list = {} + # get a list of all events that match filters + for (id_, generated, message_id, + desc) in query.add_columns( + models.Event.generated, models.Event.message_id, + models.EventType.desc).order_by( + models.Event.generated).all(): + event_list[id_] = api_models.Event(message_id, desc, + generated, []) + # Query all traits related to events. + # NOTE (gordc): cast is done because pgsql defaults to TEXT when + # handling unknown values such as null. + trait_q = ( + query.join( + models.TraitDatetime, + models.TraitDatetime.event_id == models.Event.id) + .add_columns( + models.TraitDatetime.key, models.TraitDatetime.value, + sa.cast(sa.null(), sa.Integer), + sa.cast(sa.null(), sa.Float(53)), + sa.cast(sa.null(), sa.Text)) + ).union( + query.join( + models.TraitInt, + models.TraitInt.event_id == models.Event.id) + .add_columns(models.TraitInt.key, sa.null(), + models.TraitInt.value, sa.null(), sa.null()), + query.join( + models.TraitFloat, + models.TraitFloat.event_id == models.Event.id) + .add_columns(models.TraitFloat.key, sa.null(), + sa.null(), models.TraitFloat.value, sa.null()), + query.join( + models.TraitText, + models.TraitText.event_id == models.Event.id) + .add_columns(models.TraitText.key, sa.null(), + sa.null(), sa.null(), models.TraitText.value)) - event_query = (event_query. - join(trait_query, models.Event.id == - trait_query.c.event_id)) - else: - # If there are no trait filters, grab the events from the db - query = (session.query(models.Event.id, - models.Event.generated, - models.Event.message_id, - models.EventType.desc). - join(models.EventType, - sa.and_(*event_join_conditions))) - if event_filter_conditions: - query = query.filter(sa.and_(*event_filter_conditions)) - for (id_, generated, message_id, desc_) in query.all(): - event_models_dict[id_] = api_models.Event(message_id, - desc_, - generated, - []) + for id_, key, t_date, t_int, t_float, t_text in trait_q.all(): + if t_int: + dtype = api_models.Trait.INT_TYPE + val = t_int + elif t_float: + dtype = api_models.Trait.FLOAT_TYPE + val = t_float + elif t_date: + dtype = api_models.Trait.DATETIME_TYPE + val = t_date + else: + dtype = api_models.Trait.TEXT_TYPE + val = t_text - # Build event models for the events - event_query = event_query.subquery() - query = (session.query(models.Trait). - join(models.TraitType, models.Trait.trait_type_id == - models.TraitType.id). - join(event_query, models.Trait.event_id == - event_query.c.id)) + trait_model = api_models.Trait(key, dtype, val) + event_list[id_].append_trait(trait_model) - # Now convert the sqlalchemy objects back into Models ... - for trait in query.all(): - event = event_models_dict.get(trait.event_id) - if not event: - event = api_models.Event( - trait.event.message_id, - trait.event.event_type.desc, - trait.event.generated, []) - event_models_dict[trait.event_id] = event - trait_model = api_models.Trait(trait.trait_type.desc, - trait.trait_type.data_type, - trait.get_value()) - event.append_trait(trait_model) - - event_models = event_models_dict.values() - return sorted(event_models, key=operator.attrgetter('generated')) + return event_list.values() def get_event_types(self): """Return all event types as an iterable of strings.""" @@ -327,27 +348,21 @@ class Connection(base.Connection): """ session = self._engine_facade.get_session() - LOG.debug(_("Get traits for %s") % event_type) with session.begin(): - query = (session.query(models.TraitType.desc, - models.TraitType.data_type) - .join(models.Trait, - models.Trait.trait_type_id == - models.TraitType.id) - .join(models.Event, - models.Event.id == - models.Trait.event_id) - .join(models.EventType, - sa.and_(models.EventType.id == - models.Event.id, - models.EventType.desc == - event_type)) - .group_by(models.TraitType.desc, - models.TraitType.data_type) - .distinct()) + for trait_model in [models.TraitText, models.TraitInt, + models.TraitFloat, models.TraitDatetime]: + query = (session.query(trait_model.key) + .join(models.Event, + models.Event.id == trait_model.event_id) + .join(models.EventType, + sa.and_(models.EventType.id == + models.Event.event_type_id, + models.EventType.desc == event_type)) + .distinct()) - for desc_, dtype in query.all(): - yield {'name': desc_, 'data_type': dtype} + dtype = TRAIT_MODEL_TO_ID.get(trait_model) + for row in query.all(): + yield {'name': row[0], 'data_type': dtype} def get_traits(self, event_type, trait_type=None): """Return all trait instances associated with an event_type. @@ -359,22 +374,21 @@ class Connection(base.Connection): session = self._engine_facade.get_session() with session.begin(): - trait_type_filters = [models.TraitType.id == - models.Trait.trait_type_id] - if trait_type: - trait_type_filters.append(models.TraitType.desc == trait_type) + for trait_model in [models.TraitText, models.TraitInt, + models.TraitFloat, models.TraitDatetime]: + query = (session.query(trait_model.key, trait_model.value) + .join(models.Event, + models.Event.id == trait_model.event_id) + .join(models.EventType, + sa.and_(models.EventType.id == + models.Event.event_type_id, + models.EventType.desc == event_type)) + .order_by(trait_model.key)) + if trait_type: + query = query.filter(trait_model.key == trait_type) - query = (session.query(models.Trait) - .join(models.TraitType, sa.and_(*trait_type_filters)) - .join(models.Event, - models.Event.id == models.Trait.event_id) - .join(models.EventType, - sa.and_(models.EventType.id == - models.Event.event_type_id, - models.EventType.desc == event_type))) - - for trait in query.all(): - type = trait.trait_type - yield api_models.Trait(name=type.desc, - dtype=type.data_type, - value=trait.get_value()) + dtype = TRAIT_MODEL_TO_ID.get(trait_model) + for k, v in query.all(): + yield api_models.Trait(name=k, + dtype=dtype, + value=v) diff --git a/ceilometer/event/storage/models.py b/ceilometer/event/storage/models.py index 480643e6c..a43686b04 100644 --- a/ceilometer/event/storage/models.py +++ b/ceilometer/event/storage/models.py @@ -30,6 +30,7 @@ class Event(base.Model): DUPLICATE = 1 UNKNOWN_PROBLEM = 2 + INCOMPATIBLE_TRAIT = 3 def __init__(self, message_id, event_type, generated, traits): """Create a new event. diff --git a/ceilometer/storage/sqlalchemy/migrate_repo/versions/041_expand_event_traits.py b/ceilometer/storage/sqlalchemy/migrate_repo/versions/041_expand_event_traits.py new file mode 100644 index 000000000..332cc56a1 --- /dev/null +++ b/ceilometer/storage/sqlalchemy/migrate_repo/versions/041_expand_event_traits.py @@ -0,0 +1,107 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sa + +from ceilometer.storage.sqlalchemy import models + +tables = [('trait_text', sa.Text, True, 't_string', 1), + ('trait_int', sa.Integer, False, 't_int', 2), + ('trait_float', sa.Float, False, 't_float', 3), + ('trait_datetime', models.PreciseTimestamp(), + False, 't_datetime', 4)] + + +def upgrade(migrate_engine): + meta = sa.MetaData(bind=migrate_engine) + trait = sa.Table('trait', meta, autoload=True) + event = sa.Table('event', meta, autoload=True) + trait_type = sa.Table('trait_type', meta, autoload=True) + for t_name, t_type, t_nullable, col_name, __ in tables: + t_table = sa.Table( + t_name, meta, + sa.Column('event_id', sa.Integer, + sa.ForeignKey(event.c.id), primary_key=True), + sa.Column('key', sa.String(255), primary_key=True), + sa.Column('value', t_type, nullable=t_nullable), + sa.Index('ix_%s_event_id_key' % t_name, + 'event_id', 'key'), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + t_table.create() + query = sa.select( + [trait.c.event_id, + trait_type.c.desc, + trait.c[col_name]]).select_from( + trait.join(trait_type, + trait.c.trait_type_id == trait_type.c.id)).where( + trait.c[col_name] != sa.null()) + if query.alias().select().scalar() is not None: + t_table.insert().from_select( + ['event_id', 'key', 'value'], query).execute() + trait.drop() + trait_type.drop() + + +def downgrade(migrate_engine): + meta = sa.MetaData(bind=migrate_engine) + event = sa.Table('event', meta, autoload=True) + trait_type = sa.Table( + 'trait_type', meta, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('desc', sa.String(255)), + sa.Column('data_type', sa.Integer), + sa.UniqueConstraint('desc', 'data_type', name='tt_unique'), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + trait_type.create() + trait = sa.Table( + 'trait', meta, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('trait_type_id', sa.Integer, sa.ForeignKey(trait_type.c.id)), + sa.Column('event_id', sa.Integer, sa.ForeignKey(event.c.id)), + sa.Column('t_string', sa.String(255), nullable=True, default=None), + sa.Column('t_float', sa.Float(53), nullable=True, default=None), + sa.Column('t_int', sa.Integer, nullable=True, default=None), + sa.Column('t_datetime', models.PreciseTimestamp(), nullable=True, + default=None), + sa.Index('ix_trait_t_int', 't_int'), + sa.Index('ix_trait_t_string', 't_string'), + sa.Index('ix_trait_t_datetime', 't_datetime'), + sa.Index('ix_trait_t_float', 't_float'), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + trait.create() + + for t_name, __, __, col_name, type_id in tables: + table = sa.Table(t_name, meta, autoload=True) + trait_type.insert().from_select([trait_type.c.desc, + trait_type.c.data_type], + sa.select([table.c.key, + type_id]) + .distinct()).execute() + trait.insert().from_select([trait.c['event_id'], + trait.c['trait_type_id'], + trait.c[col_name]], + sa.select([table.c.event_id, + trait_type.c.id, + table.c.value]) + .select_from( + table.join( + trait_type, + table.c.key == trait_type.c.desc)) + ).execute() + table.drop() diff --git a/ceilometer/storage/sqlalchemy/models.py b/ceilometer/storage/sqlalchemy/models.py index d5118234c..18b46f363 100644 --- a/ceilometer/storage/sqlalchemy/models.py +++ b/ceilometer/storage/sqlalchemy/models.py @@ -24,12 +24,10 @@ from sqlalchemy import event, select from sqlalchemy import Float, Boolean, Text, DateTime from sqlalchemy.dialects.mysql import DECIMAL from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import backref from sqlalchemy.orm import deferred from sqlalchemy.orm import relationship from sqlalchemy.types import TypeDecorator -from ceilometer.event.storage import models from ceilometer import utils @@ -335,93 +333,49 @@ class Event(Base): self.generated) -class TraitType(Base): - """Types of event traits. +class TraitText(Base): + """Event text traits.""" - A trait type includes a description and a data type. Uniqueness is - enforced compositely on the data_type and desc fields. This is to - accommodate cases, such as 'generated', which, depending on the - corresponding event, could be a date, a boolean, or a float. - """ - __tablename__ = 'trait_type' + __tablename__ = 'trait_text' __table_args__ = ( - UniqueConstraint('desc', 'data_type', name='tt_unique'), - Index('ix_trait_type', 'desc') + Index('ix_trait_text_event_id_key', 'event_id', 'key'), ) - - id = Column(Integer, primary_key=True) - desc = Column(String(255)) - data_type = Column(Integer) - - def __init__(self, desc, data_type): - self.desc = desc - self.data_type = data_type - - def __repr__(self): - return "" % (self.desc, self.data_type) + event_id = Column(Integer, ForeignKey('event.id'), primary_key=True) + key = Column(Integer, primary_key=True) + value = Column(Text) -class Trait(Base): - __tablename__ = 'trait' +class TraitInt(Base): + """Event integer traits.""" + + __tablename__ = 'trait_int' __table_args__ = ( - Index('ix_trait_t_int', 't_int'), - Index('ix_trait_t_string', 't_string'), - Index('ix_trait_t_datetime', 't_datetime'), - Index('ix_trait_t_float', 't_float'), + Index('ix_trait_int_event_id_key', 'event_id', 'key'), ) - id = Column(Integer, primary_key=True) + event_id = Column(Integer, ForeignKey('event.id'), primary_key=True) + key = Column(Integer, primary_key=True) + value = Column(Integer) - trait_type_id = Column(Integer, ForeignKey('trait_type.id')) - trait_type = relationship("TraitType", backref='traits') - t_string = Column(String(255), nullable=True, default=None) - t_float = Column(Float(53), nullable=True, default=None) - t_int = Column(Integer, nullable=True, default=None) - t_datetime = Column(PreciseTimestamp(), nullable=True, default=None) +class TraitFloat(Base): + """Event float traits.""" - event_id = Column(Integer, ForeignKey('event.id')) - event = relationship("Event", backref=backref('traits', order_by=id)) + __tablename__ = 'trait_float' + __table_args__ = ( + Index('ix_trait_float_event_id_key', 'event_id', 'key'), + ) + event_id = Column(Integer, ForeignKey('event.id'), primary_key=True) + key = Column(Integer, primary_key=True) + value = Column(Float(53)) - _value_map = {models.Trait.TEXT_TYPE: 't_string', - models.Trait.FLOAT_TYPE: 't_float', - models.Trait.INT_TYPE: 't_int', - models.Trait.DATETIME_TYPE: 't_datetime'} - def __init__(self, trait_type, event, t_string=None, - t_float=None, t_int=None, t_datetime=None): - self.trait_type = trait_type - self.t_string = t_string - self.t_float = t_float - self.t_int = t_int - self.t_datetime = t_datetime - self.event = event +class TraitDatetime(Base): + """Event datetime traits.""" - def get_value(self): - if self.trait_type is None: - dtype = None - else: - dtype = self.trait_type.data_type - - if dtype == models.Trait.INT_TYPE: - return self.t_int - if dtype == models.Trait.FLOAT_TYPE: - return self.t_float - if dtype == models.Trait.DATETIME_TYPE: - return self.t_datetime - if dtype == models.Trait.TEXT_TYPE: - return self.t_string - - return None - - def __repr__(self): - name = self.trait_type.desc if self.trait_type else None - data_type = (self.trait_type.data_type if self.trait_type else - models.Trait.NONE_TYPE) - - return "" % (name, - data_type, - self.t_string, - self.t_float, - self.t_int, - self.t_datetime, - self.event) + __tablename__ = 'trait_datetime' + __table_args__ = ( + Index('ix_trait_datetime_event_id_key', 'event_id', 'key'), + ) + event_id = Column(Integer, ForeignKey('event.id'), primary_key=True) + key = Column(Integer, primary_key=True) + value = Column(PreciseTimestamp()) diff --git a/ceilometer/storage/sqlalchemy/utils.py b/ceilometer/storage/sqlalchemy/utils.py index 5465b77aa..13655c4b2 100644 --- a/ceilometer/storage/sqlalchemy/utils.py +++ b/ceilometer/storage/sqlalchemy/utils.py @@ -120,18 +120,3 @@ class QueryTransformer(object): def get_query(self): return self.query - - -trait_models_dict = {'string': models.Trait.t_string, - 'integer': models.Trait.t_int, - 'datetime': models.Trait.t_datetime, - 'float': models.Trait.t_float} - - -def trait_op_condition(conditions, trait_type, value, op='eq'): - trait_model = trait_models_dict[trait_type] - op_dict = {'eq': (trait_model == value), 'lt': (trait_model < value), - 'le': (trait_model <= value), 'gt': (trait_model > value), - 'ge': (trait_model >= value), 'ne': (trait_model != value)} - conditions.append(op_dict[op]) - return conditions diff --git a/ceilometer/tests/storage/test_impl_sqlalchemy.py b/ceilometer/tests/storage/test_impl_sqlalchemy.py index ef9c2b5b5..4dd75df35 100644 --- a/ceilometer/tests/storage/test_impl_sqlalchemy.py +++ b/ceilometer/tests/storage/test_impl_sqlalchemy.py @@ -43,39 +43,6 @@ class CeilometerBaseTest(tests_db.TestBase): self.assertEqual('value', base['key']) -@tests_db.run_with('sqlite', 'mysql', 'pgsql') -class TraitTypeTest(tests_db.TestBase): - # TraitType is a construct specific to sqlalchemy. - # Not applicable to other drivers. - - def test_trait_type_exists(self): - tt1 = self.event_conn._get_or_create_trait_type("foo", 0) - self.assertTrue(tt1.id >= 0) - tt2 = self.event_conn._get_or_create_trait_type("foo", 0) - self.assertEqual(tt2.id, tt1.id) - self.assertEqual(tt2.desc, tt1.desc) - self.assertEqual(tt2.data_type, tt1.data_type) - - def test_new_trait_type(self): - tt1 = self.event_conn._get_or_create_trait_type("foo", 0) - self.assertTrue(tt1.id >= 0) - tt2 = self.event_conn._get_or_create_trait_type("blah", 0) - self.assertNotEqual(tt1.id, tt2.id) - self.assertNotEqual(tt1.desc, tt2.desc) - # Test the method __repr__ returns a string - self.assertTrue(repr.repr(tt2)) - - def test_trait_different_data_type(self): - tt1 = self.event_conn._get_or_create_trait_type("foo", 0) - self.assertTrue(tt1.id >= 0) - tt2 = self.event_conn._get_or_create_trait_type("foo", 1) - self.assertNotEqual(tt1.id, tt2.id) - self.assertEqual(tt2.desc, tt1.desc) - self.assertNotEqual(tt1.data_type, tt2.data_type) - # Test the method __repr__ returns a string - self.assertTrue(repr.repr(tt2)) - - @tests_db.run_with('sqlite', 'mysql', 'pgsql') class EventTypeTest(tests_db.TestBase): # EventType is a construct specific to sqlalchemy @@ -104,65 +71,49 @@ class MyException(Exception): @tests_db.run_with('sqlite', 'mysql', 'pgsql') class EventTest(tests_db.TestBase): + def _verify_data(self, trait, trait_table): + now = datetime.datetime.utcnow() + ev = models.Event('1', 'name', now, [trait]) + self.event_conn.record_events([ev]) + session = self.event_conn._engine_facade.get_session() + t_tables = [sql_models.TraitText, sql_models.TraitFloat, + sql_models.TraitInt, sql_models.TraitDatetime] + for table in t_tables: + if table == trait_table: + self.assertEqual(1, session.query(table).count()) + else: + self.assertEqual(0, session.query(table).count()) + def test_string_traits(self): model = models.Trait("Foo", models.Trait.TEXT_TYPE, "my_text") - trait = self.event_conn._make_trait(model, None) - self.assertEqual(models.Trait.TEXT_TYPE, trait.trait_type.data_type) - self.assertIsNone(trait.t_float) - self.assertIsNone(trait.t_int) - self.assertIsNone(trait.t_datetime) - self.assertEqual("my_text", trait.t_string) - self.assertIsNotNone(trait.trait_type.desc) + self._verify_data(model, sql_models.TraitText) def test_int_traits(self): model = models.Trait("Foo", models.Trait.INT_TYPE, 100) - trait = self.event_conn._make_trait(model, None) - self.assertEqual(models.Trait.INT_TYPE, trait.trait_type.data_type) - self.assertIsNone(trait.t_float) - self.assertIsNone(trait.t_string) - self.assertIsNone(trait.t_datetime) - self.assertEqual(100, trait.t_int) - self.assertIsNotNone(trait.trait_type.desc) + self._verify_data(model, sql_models.TraitInt) def test_float_traits(self): model = models.Trait("Foo", models.Trait.FLOAT_TYPE, 123.456) - trait = self.event_conn._make_trait(model, None) - self.assertEqual(models.Trait.FLOAT_TYPE, trait.trait_type.data_type) - self.assertIsNone(trait.t_int) - self.assertIsNone(trait.t_string) - self.assertIsNone(trait.t_datetime) - self.assertEqual(123.456, trait.t_float) - self.assertIsNotNone(trait.trait_type.desc) + self._verify_data(model, sql_models.TraitFloat) def test_datetime_traits(self): now = datetime.datetime.utcnow() model = models.Trait("Foo", models.Trait.DATETIME_TYPE, now) - trait = self.event_conn._make_trait(model, None) - self.assertEqual(models.Trait.DATETIME_TYPE, - trait.trait_type.data_type) - self.assertIsNone(trait.t_int) - self.assertIsNone(trait.t_string) - self.assertIsNone(trait.t_float) - self.assertEqual(now, trait.t_datetime) - self.assertIsNotNone(trait.trait_type.desc) + self._verify_data(model, sql_models.TraitDatetime) def test_bad_event(self): now = datetime.datetime.utcnow() m = [models.Event("1", "Foo", now, []), models.Event("2", "Zoo", now, [])] - with mock.patch.object(self.event_conn, "_record_event") as mock_save: + with mock.patch.object(self.event_conn, + "_get_or_create_event_type") as mock_save: mock_save.side_effect = MyException("Boom") problem_events = self.event_conn.record_events(m) self.assertEqual(2, len(problem_events)) for bad, event in problem_events: self.assertEqual(bad, models.Event.UNKNOWN_PROBLEM) - def test_get_none_value_traits(self): - model = sql_models.Trait(None, None, 5) - self.assertIsNone(model.get_value()) - self.assertTrue(repr.repr(model)) - def test_event_repr(self): ev = sql_models.Event('msg_id', None, False) ev.id = 100