Merge "modify events sql schema to reduce empty columns"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
3ad53cc903
@ -14,13 +14,11 @@
|
||||
"""SQLAlchemy storage backend."""
|
||||
|
||||
from __future__ import absolute_import
|
||||
import operator
|
||||
import os
|
||||
|
||||
from oslo.db import exception as dbexc
|
||||
from oslo.db.sqlalchemy import session as db_session
|
||||
from oslo_config import cfg
|
||||
import six
|
||||
import sqlalchemy as sa
|
||||
|
||||
from ceilometer.event.storage import base
|
||||
@ -28,7 +26,6 @@ from ceilometer.event.storage import models as api_models
|
||||
from ceilometer.i18n import _
|
||||
from ceilometer.openstack.common import log
|
||||
from ceilometer.storage.sqlalchemy import models
|
||||
from ceilometer.storage.sqlalchemy import utils as sql_utils
|
||||
from ceilometer import utils
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
@ -44,6 +41,36 @@ AVAILABLE_STORAGE_CAPABILITIES = {
|
||||
}
|
||||
|
||||
|
||||
TRAIT_MAPLIST = [(api_models.Trait.NONE_TYPE, models.TraitText),
|
||||
(api_models.Trait.TEXT_TYPE, models.TraitText),
|
||||
(api_models.Trait.INT_TYPE, models.TraitInt),
|
||||
(api_models.Trait.FLOAT_TYPE, models.TraitFloat),
|
||||
(api_models.Trait.DATETIME_TYPE, models.TraitDatetime)]
|
||||
|
||||
|
||||
TRAIT_ID_TO_MODEL = dict((x, y) for x, y in TRAIT_MAPLIST)
|
||||
TRAIT_MODEL_TO_ID = dict((y, x) for x, y in TRAIT_MAPLIST)
|
||||
|
||||
|
||||
trait_models_dict = {'string': models.TraitText,
|
||||
'integer': models.TraitInt,
|
||||
'datetime': models.TraitDatetime,
|
||||
'float': models.TraitFloat}
|
||||
|
||||
|
||||
def _build_trait_query(session, trait_type, key, value, op='eq'):
|
||||
trait_model = trait_models_dict[trait_type]
|
||||
op_dict = {'eq': (trait_model.value == value),
|
||||
'lt': (trait_model.value < value),
|
||||
'le': (trait_model.value <= value),
|
||||
'gt': (trait_model.value > value),
|
||||
'ge': (trait_model.value >= value),
|
||||
'ne': (trait_model.value != value)}
|
||||
conditions = [trait_model.key == key, op_dict[op]]
|
||||
return (session.query(trait_model.event_id.label('ev_id'))
|
||||
.filter(*conditions))
|
||||
|
||||
|
||||
class Connection(base.Connection):
|
||||
"""Put the event data into a SQLAlchemy database.
|
||||
|
||||
@ -61,21 +88,31 @@ class Connection(base.Connection):
|
||||
generated = timestamp of event
|
||||
event_type_id = event type -> eventtype.id
|
||||
}
|
||||
- Trait
|
||||
- trait value
|
||||
- TraitInt
|
||||
- int trait value
|
||||
- { event_id: event -> event.id
|
||||
trait_type_id: trait type -> traittype.id
|
||||
t_string: string value
|
||||
t_float: float value
|
||||
t_int: integer value
|
||||
t_datetime: timestamp value
|
||||
key: trait type
|
||||
value: integer value
|
||||
}
|
||||
- TraitType
|
||||
- trait definition
|
||||
- { id: trait id
|
||||
desc: description of trait
|
||||
data_type: data type (integer that maps to datatype)
|
||||
- TraitDatetime
|
||||
- int trait value
|
||||
- { event_id: event -> event.id
|
||||
key: trait type
|
||||
value: datetime value
|
||||
}
|
||||
- TraitText
|
||||
- int trait value
|
||||
- { event_id: event -> event.id
|
||||
key: trait type
|
||||
value: text value
|
||||
}
|
||||
- TraitFloat
|
||||
- int trait value
|
||||
- { event_id: event -> event.id
|
||||
key: trait type
|
||||
value: float value
|
||||
}
|
||||
|
||||
"""
|
||||
CAPABILITIES = utils.update_nested(base.Connection.CAPABILITIES,
|
||||
AVAILABLE_CAPABILITIES)
|
||||
@ -108,37 +145,6 @@ class Connection(base.Connection):
|
||||
self._engine_facade._session_maker.close_all()
|
||||
engine.dispose()
|
||||
|
||||
def _get_or_create_trait_type(self, trait_type, data_type, session=None):
|
||||
"""Find if this trait already exists in the database.
|
||||
|
||||
If it does not, create a new entry in the trait type table.
|
||||
"""
|
||||
if session is None:
|
||||
session = self._engine_facade.get_session()
|
||||
with session.begin(subtransactions=True):
|
||||
tt = session.query(models.TraitType).filter(
|
||||
models.TraitType.desc == trait_type,
|
||||
models.TraitType.data_type == data_type).first()
|
||||
if not tt:
|
||||
tt = models.TraitType(trait_type, data_type)
|
||||
session.add(tt)
|
||||
return tt
|
||||
|
||||
def _make_trait(self, trait_model, event, session=None):
|
||||
"""Make a new Trait from a Trait model.
|
||||
|
||||
Doesn't flush or add to session.
|
||||
"""
|
||||
trait_type = self._get_or_create_trait_type(trait_model.name,
|
||||
trait_model.dtype,
|
||||
session)
|
||||
value_map = models.Trait._value_map
|
||||
values = {'t_string': None, 't_float': None,
|
||||
't_int': None, 't_datetime': None}
|
||||
value = trait_model.value
|
||||
values[value_map[trait_model.dtype]] = value
|
||||
return models.Trait(trait_type, event, **values)
|
||||
|
||||
def _get_or_create_event_type(self, event_type, session=None):
|
||||
"""Check if an event type with the supplied name is already exists.
|
||||
|
||||
@ -154,27 +160,6 @@ class Connection(base.Connection):
|
||||
session.add(et)
|
||||
return et
|
||||
|
||||
def _record_event(self, session, event_model):
|
||||
"""Store a single Event, including related Traits."""
|
||||
with session.begin(subtransactions=True):
|
||||
event_type = self._get_or_create_event_type(event_model.event_type,
|
||||
session=session)
|
||||
|
||||
event = models.Event(event_model.message_id, event_type,
|
||||
event_model.generated)
|
||||
session.add(event)
|
||||
|
||||
new_traits = []
|
||||
if event_model.traits:
|
||||
for trait in event_model.traits:
|
||||
t = self._make_trait(trait, event, session=session)
|
||||
session.add(t)
|
||||
new_traits.append(t)
|
||||
|
||||
# Note: we don't flush here, explicitly (unless a new trait or event
|
||||
# does it). Otherwise, just wait until all the Events are staged.
|
||||
return event, new_traits
|
||||
|
||||
def record_events(self, event_models):
|
||||
"""Write the events to SQL database via sqlalchemy.
|
||||
|
||||
@ -188,22 +173,43 @@ class Connection(base.Connection):
|
||||
TraitTypes are added along the way.
|
||||
"""
|
||||
session = self._engine_facade.get_session()
|
||||
events = []
|
||||
problem_events = []
|
||||
for event_model in event_models:
|
||||
event = None
|
||||
try:
|
||||
with session.begin():
|
||||
event = self._record_event(session, event_model)
|
||||
event_type = self._get_or_create_event_type(
|
||||
event_model.event_type, session=session)
|
||||
event = models.Event(event_model.message_id, event_type,
|
||||
event_model.generated)
|
||||
session.add(event)
|
||||
session.flush()
|
||||
|
||||
if event_model.traits:
|
||||
trait_map = {}
|
||||
for trait in event_model.traits:
|
||||
if trait_map.get(trait.dtype) is None:
|
||||
trait_map[trait.dtype] = []
|
||||
trait_map[trait.dtype].append(
|
||||
{'event_id': event.id,
|
||||
'key': trait.name,
|
||||
'value': trait.value})
|
||||
for dtype in trait_map.keys():
|
||||
model = TRAIT_ID_TO_MODEL[dtype]
|
||||
session.execute(model.__table__.insert(),
|
||||
trait_map[dtype])
|
||||
except dbexc.DBDuplicateEntry as e:
|
||||
LOG.exception(_("Failed to record duplicated event: %s") % e)
|
||||
problem_events.append((api_models.Event.DUPLICATE,
|
||||
event_model))
|
||||
except KeyError as e:
|
||||
LOG.exception(_('Failed to record event: %s') % e)
|
||||
problem_events.append((api_models.Event.INCOMPATIBLE_TRAIT,
|
||||
event_model))
|
||||
except Exception as e:
|
||||
LOG.exception(_('Failed to record event: %s') % e)
|
||||
problem_events.append((api_models.Event.UNKNOWN_PROBLEM,
|
||||
event_model))
|
||||
events.append(event)
|
||||
return problem_events
|
||||
|
||||
def get_events(self, event_filter):
|
||||
@ -212,10 +218,7 @@ class Connection(base.Connection):
|
||||
:param event_filter: EventFilter instance
|
||||
"""
|
||||
|
||||
start = event_filter.start_timestamp
|
||||
end = event_filter.end_timestamp
|
||||
session = self._engine_facade.get_session()
|
||||
LOG.debug(_("Getting events that match filter: %s") % event_filter)
|
||||
with session.begin():
|
||||
event_query = session.query(models.Event)
|
||||
|
||||
@ -233,80 +236,98 @@ class Connection(base.Connection):
|
||||
# Build up the where conditions
|
||||
event_filter_conditions = []
|
||||
if event_filter.message_id:
|
||||
event_filter_conditions.append(models.Event.message_id ==
|
||||
event_filter.message_id)
|
||||
if start:
|
||||
event_filter_conditions.append(models.Event.generated >= start)
|
||||
if end:
|
||||
event_filter_conditions.append(models.Event.generated <= end)
|
||||
|
||||
event_filter_conditions.append(
|
||||
models.Event.message_id == event_filter.message_id)
|
||||
if event_filter.start_timestamp:
|
||||
event_filter_conditions.append(
|
||||
models.Event.generated >= event_filter.start_timestamp)
|
||||
if event_filter.end_timestamp:
|
||||
event_filter_conditions.append(
|
||||
models.Event.generated <= event_filter.end_timestamp)
|
||||
if event_filter_conditions:
|
||||
event_query = (event_query.
|
||||
filter(sa.and_(*event_filter_conditions)))
|
||||
|
||||
event_models_dict = {}
|
||||
trait_subq = None
|
||||
# Build trait filter
|
||||
if event_filter.traits_filter:
|
||||
trait_qlist = []
|
||||
for trait_filter in event_filter.traits_filter:
|
||||
|
||||
# Build a sub query that joins Trait to TraitType
|
||||
# where the trait name matches
|
||||
trait_name = trait_filter.pop('key')
|
||||
key = trait_filter.pop('key')
|
||||
op = trait_filter.pop('op', 'eq')
|
||||
conditions = [models.Trait.trait_type_id ==
|
||||
models.TraitType.id,
|
||||
models.TraitType.desc == trait_name]
|
||||
trait_qlist.append(_build_trait_query(
|
||||
session, trait_filter.keys()[0], key,
|
||||
trait_filter.values()[0], op))
|
||||
trait_subq = trait_qlist.pop()
|
||||
if trait_qlist:
|
||||
trait_subq = trait_subq.intersect(*trait_qlist)
|
||||
trait_subq = trait_subq.subquery()
|
||||
|
||||
for key, value in six.iteritems(trait_filter):
|
||||
sql_utils.trait_op_condition(conditions,
|
||||
key, value, op)
|
||||
query = (session.query(models.Event.id)
|
||||
.join(models.EventType,
|
||||
sa.and_(*event_join_conditions)))
|
||||
if trait_subq is not None:
|
||||
query = query.join(trait_subq,
|
||||
trait_subq.c.ev_id == models.Event.id)
|
||||
if event_filter_conditions:
|
||||
query = query.filter(sa.and_(*event_filter_conditions))
|
||||
|
||||
trait_query = (session.query(models.Trait.event_id).
|
||||
join(models.TraitType,
|
||||
sa.and_(*conditions)).subquery())
|
||||
event_list = {}
|
||||
# get a list of all events that match filters
|
||||
for (id_, generated, message_id,
|
||||
desc) in query.add_columns(
|
||||
models.Event.generated, models.Event.message_id,
|
||||
models.EventType.desc).order_by(
|
||||
models.Event.generated).all():
|
||||
event_list[id_] = api_models.Event(message_id, desc,
|
||||
generated, [])
|
||||
# Query all traits related to events.
|
||||
# NOTE (gordc): cast is done because pgsql defaults to TEXT when
|
||||
# handling unknown values such as null.
|
||||
trait_q = (
|
||||
query.join(
|
||||
models.TraitDatetime,
|
||||
models.TraitDatetime.event_id == models.Event.id)
|
||||
.add_columns(
|
||||
models.TraitDatetime.key, models.TraitDatetime.value,
|
||||
sa.cast(sa.null(), sa.Integer),
|
||||
sa.cast(sa.null(), sa.Float(53)),
|
||||
sa.cast(sa.null(), sa.Text))
|
||||
).union(
|
||||
query.join(
|
||||
models.TraitInt,
|
||||
models.TraitInt.event_id == models.Event.id)
|
||||
.add_columns(models.TraitInt.key, sa.null(),
|
||||
models.TraitInt.value, sa.null(), sa.null()),
|
||||
query.join(
|
||||
models.TraitFloat,
|
||||
models.TraitFloat.event_id == models.Event.id)
|
||||
.add_columns(models.TraitFloat.key, sa.null(),
|
||||
sa.null(), models.TraitFloat.value, sa.null()),
|
||||
query.join(
|
||||
models.TraitText,
|
||||
models.TraitText.event_id == models.Event.id)
|
||||
.add_columns(models.TraitText.key, sa.null(),
|
||||
sa.null(), sa.null(), models.TraitText.value))
|
||||
|
||||
event_query = (event_query.
|
||||
join(trait_query, models.Event.id ==
|
||||
trait_query.c.event_id))
|
||||
else:
|
||||
# If there are no trait filters, grab the events from the db
|
||||
query = (session.query(models.Event.id,
|
||||
models.Event.generated,
|
||||
models.Event.message_id,
|
||||
models.EventType.desc).
|
||||
join(models.EventType,
|
||||
sa.and_(*event_join_conditions)))
|
||||
if event_filter_conditions:
|
||||
query = query.filter(sa.and_(*event_filter_conditions))
|
||||
for (id_, generated, message_id, desc_) in query.all():
|
||||
event_models_dict[id_] = api_models.Event(message_id,
|
||||
desc_,
|
||||
generated,
|
||||
[])
|
||||
for id_, key, t_date, t_int, t_float, t_text in trait_q.all():
|
||||
if t_int:
|
||||
dtype = api_models.Trait.INT_TYPE
|
||||
val = t_int
|
||||
elif t_float:
|
||||
dtype = api_models.Trait.FLOAT_TYPE
|
||||
val = t_float
|
||||
elif t_date:
|
||||
dtype = api_models.Trait.DATETIME_TYPE
|
||||
val = t_date
|
||||
else:
|
||||
dtype = api_models.Trait.TEXT_TYPE
|
||||
val = t_text
|
||||
|
||||
# Build event models for the events
|
||||
event_query = event_query.subquery()
|
||||
query = (session.query(models.Trait).
|
||||
join(models.TraitType, models.Trait.trait_type_id ==
|
||||
models.TraitType.id).
|
||||
join(event_query, models.Trait.event_id ==
|
||||
event_query.c.id))
|
||||
trait_model = api_models.Trait(key, dtype, val)
|
||||
event_list[id_].append_trait(trait_model)
|
||||
|
||||
# Now convert the sqlalchemy objects back into Models ...
|
||||
for trait in query.all():
|
||||
event = event_models_dict.get(trait.event_id)
|
||||
if not event:
|
||||
event = api_models.Event(
|
||||
trait.event.message_id,
|
||||
trait.event.event_type.desc,
|
||||
trait.event.generated, [])
|
||||
event_models_dict[trait.event_id] = event
|
||||
trait_model = api_models.Trait(trait.trait_type.desc,
|
||||
trait.trait_type.data_type,
|
||||
trait.get_value())
|
||||
event.append_trait(trait_model)
|
||||
|
||||
event_models = event_models_dict.values()
|
||||
return sorted(event_models, key=operator.attrgetter('generated'))
|
||||
return event_list.values()
|
||||
|
||||
def get_event_types(self):
|
||||
"""Return all event types as an iterable of strings."""
|
||||
@ -327,27 +348,21 @@ class Connection(base.Connection):
|
||||
"""
|
||||
session = self._engine_facade.get_session()
|
||||
|
||||
LOG.debug(_("Get traits for %s") % event_type)
|
||||
with session.begin():
|
||||
query = (session.query(models.TraitType.desc,
|
||||
models.TraitType.data_type)
|
||||
.join(models.Trait,
|
||||
models.Trait.trait_type_id ==
|
||||
models.TraitType.id)
|
||||
.join(models.Event,
|
||||
models.Event.id ==
|
||||
models.Trait.event_id)
|
||||
.join(models.EventType,
|
||||
sa.and_(models.EventType.id ==
|
||||
models.Event.id,
|
||||
models.EventType.desc ==
|
||||
event_type))
|
||||
.group_by(models.TraitType.desc,
|
||||
models.TraitType.data_type)
|
||||
.distinct())
|
||||
for trait_model in [models.TraitText, models.TraitInt,
|
||||
models.TraitFloat, models.TraitDatetime]:
|
||||
query = (session.query(trait_model.key)
|
||||
.join(models.Event,
|
||||
models.Event.id == trait_model.event_id)
|
||||
.join(models.EventType,
|
||||
sa.and_(models.EventType.id ==
|
||||
models.Event.event_type_id,
|
||||
models.EventType.desc == event_type))
|
||||
.distinct())
|
||||
|
||||
for desc_, dtype in query.all():
|
||||
yield {'name': desc_, 'data_type': dtype}
|
||||
dtype = TRAIT_MODEL_TO_ID.get(trait_model)
|
||||
for row in query.all():
|
||||
yield {'name': row[0], 'data_type': dtype}
|
||||
|
||||
def get_traits(self, event_type, trait_type=None):
|
||||
"""Return all trait instances associated with an event_type.
|
||||
@ -359,22 +374,21 @@ class Connection(base.Connection):
|
||||
|
||||
session = self._engine_facade.get_session()
|
||||
with session.begin():
|
||||
trait_type_filters = [models.TraitType.id ==
|
||||
models.Trait.trait_type_id]
|
||||
if trait_type:
|
||||
trait_type_filters.append(models.TraitType.desc == trait_type)
|
||||
for trait_model in [models.TraitText, models.TraitInt,
|
||||
models.TraitFloat, models.TraitDatetime]:
|
||||
query = (session.query(trait_model.key, trait_model.value)
|
||||
.join(models.Event,
|
||||
models.Event.id == trait_model.event_id)
|
||||
.join(models.EventType,
|
||||
sa.and_(models.EventType.id ==
|
||||
models.Event.event_type_id,
|
||||
models.EventType.desc == event_type))
|
||||
.order_by(trait_model.key))
|
||||
if trait_type:
|
||||
query = query.filter(trait_model.key == trait_type)
|
||||
|
||||
query = (session.query(models.Trait)
|
||||
.join(models.TraitType, sa.and_(*trait_type_filters))
|
||||
.join(models.Event,
|
||||
models.Event.id == models.Trait.event_id)
|
||||
.join(models.EventType,
|
||||
sa.and_(models.EventType.id ==
|
||||
models.Event.event_type_id,
|
||||
models.EventType.desc == event_type)))
|
||||
|
||||
for trait in query.all():
|
||||
type = trait.trait_type
|
||||
yield api_models.Trait(name=type.desc,
|
||||
dtype=type.data_type,
|
||||
value=trait.get_value())
|
||||
dtype = TRAIT_MODEL_TO_ID.get(trait_model)
|
||||
for k, v in query.all():
|
||||
yield api_models.Trait(name=k,
|
||||
dtype=dtype,
|
||||
value=v)
|
||||
|
||||
@ -31,6 +31,7 @@ class Event(base.Model):
|
||||
|
||||
DUPLICATE = 1
|
||||
UNKNOWN_PROBLEM = 2
|
||||
INCOMPATIBLE_TRAIT = 3
|
||||
|
||||
def __init__(self, message_id, event_type, generated, traits):
|
||||
"""Create a new event.
|
||||
|
||||
@ -0,0 +1,107 @@
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from ceilometer.storage.sqlalchemy import models
|
||||
|
||||
tables = [('trait_text', sa.Text, True, 't_string', 1),
|
||||
('trait_int', sa.Integer, False, 't_int', 2),
|
||||
('trait_float', sa.Float, False, 't_float', 3),
|
||||
('trait_datetime', models.PreciseTimestamp(),
|
||||
False, 't_datetime', 4)]
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
meta = sa.MetaData(bind=migrate_engine)
|
||||
trait = sa.Table('trait', meta, autoload=True)
|
||||
event = sa.Table('event', meta, autoload=True)
|
||||
trait_type = sa.Table('trait_type', meta, autoload=True)
|
||||
for t_name, t_type, t_nullable, col_name, __ in tables:
|
||||
t_table = sa.Table(
|
||||
t_name, meta,
|
||||
sa.Column('event_id', sa.Integer,
|
||||
sa.ForeignKey(event.c.id), primary_key=True),
|
||||
sa.Column('key', sa.String(255), primary_key=True),
|
||||
sa.Column('value', t_type, nullable=t_nullable),
|
||||
sa.Index('ix_%s_event_id_key' % t_name,
|
||||
'event_id', 'key'),
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8',
|
||||
)
|
||||
t_table.create()
|
||||
query = sa.select(
|
||||
[trait.c.event_id,
|
||||
trait_type.c.desc,
|
||||
trait.c[col_name]]).select_from(
|
||||
trait.join(trait_type,
|
||||
trait.c.trait_type_id == trait_type.c.id)).where(
|
||||
trait.c[col_name] != sa.null())
|
||||
if query.alias().select().scalar() is not None:
|
||||
t_table.insert().from_select(
|
||||
['event_id', 'key', 'value'], query).execute()
|
||||
trait.drop()
|
||||
trait_type.drop()
|
||||
|
||||
|
||||
def downgrade(migrate_engine):
|
||||
meta = sa.MetaData(bind=migrate_engine)
|
||||
event = sa.Table('event', meta, autoload=True)
|
||||
trait_type = sa.Table(
|
||||
'trait_type', meta,
|
||||
sa.Column('id', sa.Integer, primary_key=True),
|
||||
sa.Column('desc', sa.String(255)),
|
||||
sa.Column('data_type', sa.Integer),
|
||||
sa.UniqueConstraint('desc', 'data_type', name='tt_unique'),
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8',
|
||||
)
|
||||
trait_type.create()
|
||||
trait = sa.Table(
|
||||
'trait', meta,
|
||||
sa.Column('id', sa.Integer, primary_key=True),
|
||||
sa.Column('trait_type_id', sa.Integer, sa.ForeignKey(trait_type.c.id)),
|
||||
sa.Column('event_id', sa.Integer, sa.ForeignKey(event.c.id)),
|
||||
sa.Column('t_string', sa.String(255), nullable=True, default=None),
|
||||
sa.Column('t_float', sa.Float(53), nullable=True, default=None),
|
||||
sa.Column('t_int', sa.Integer, nullable=True, default=None),
|
||||
sa.Column('t_datetime', models.PreciseTimestamp(), nullable=True,
|
||||
default=None),
|
||||
sa.Index('ix_trait_t_int', 't_int'),
|
||||
sa.Index('ix_trait_t_string', 't_string'),
|
||||
sa.Index('ix_trait_t_datetime', 't_datetime'),
|
||||
sa.Index('ix_trait_t_float', 't_float'),
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8',
|
||||
)
|
||||
trait.create()
|
||||
|
||||
for t_name, __, __, col_name, type_id in tables:
|
||||
table = sa.Table(t_name, meta, autoload=True)
|
||||
trait_type.insert().from_select([trait_type.c.desc,
|
||||
trait_type.c.data_type],
|
||||
sa.select([table.c.key,
|
||||
type_id])
|
||||
.distinct()).execute()
|
||||
trait.insert().from_select([trait.c['event_id'],
|
||||
trait.c['trait_type_id'],
|
||||
trait.c[col_name]],
|
||||
sa.select([table.c.event_id,
|
||||
trait_type.c.id,
|
||||
table.c.value])
|
||||
.select_from(
|
||||
table.join(
|
||||
trait_type,
|
||||
table.c.key == trait_type.c.desc))
|
||||
).execute()
|
||||
table.drop()
|
||||
@ -24,12 +24,10 @@ from sqlalchemy import event, select
|
||||
from sqlalchemy import Float, Boolean, Text, DateTime
|
||||
from sqlalchemy.dialects.mysql import DECIMAL
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import backref
|
||||
from sqlalchemy.orm import deferred
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.types import TypeDecorator
|
||||
|
||||
from ceilometer.event.storage import models
|
||||
from ceilometer import utils
|
||||
|
||||
|
||||
@ -335,93 +333,49 @@ class Event(Base):
|
||||
self.generated)
|
||||
|
||||
|
||||
class TraitType(Base):
|
||||
"""Types of event traits.
|
||||
class TraitText(Base):
|
||||
"""Event text traits."""
|
||||
|
||||
A trait type includes a description and a data type. Uniqueness is
|
||||
enforced compositely on the data_type and desc fields. This is to
|
||||
accommodate cases, such as 'generated', which, depending on the
|
||||
corresponding event, could be a date, a boolean, or a float.
|
||||
"""
|
||||
__tablename__ = 'trait_type'
|
||||
__tablename__ = 'trait_text'
|
||||
__table_args__ = (
|
||||
UniqueConstraint('desc', 'data_type', name='tt_unique'),
|
||||
Index('ix_trait_type', 'desc')
|
||||
Index('ix_trait_text_event_id_key', 'event_id', 'key'),
|
||||
)
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
desc = Column(String(255))
|
||||
data_type = Column(Integer)
|
||||
|
||||
def __init__(self, desc, data_type):
|
||||
self.desc = desc
|
||||
self.data_type = data_type
|
||||
|
||||
def __repr__(self):
|
||||
return "<TraitType: %s:%d>" % (self.desc, self.data_type)
|
||||
event_id = Column(Integer, ForeignKey('event.id'), primary_key=True)
|
||||
key = Column(Integer, primary_key=True)
|
||||
value = Column(Text)
|
||||
|
||||
|
||||
class Trait(Base):
|
||||
__tablename__ = 'trait'
|
||||
class TraitInt(Base):
|
||||
"""Event integer traits."""
|
||||
|
||||
__tablename__ = 'trait_int'
|
||||
__table_args__ = (
|
||||
Index('ix_trait_t_int', 't_int'),
|
||||
Index('ix_trait_t_string', 't_string'),
|
||||
Index('ix_trait_t_datetime', 't_datetime'),
|
||||
Index('ix_trait_t_float', 't_float'),
|
||||
Index('ix_trait_int_event_id_key', 'event_id', 'key'),
|
||||
)
|
||||
id = Column(Integer, primary_key=True)
|
||||
event_id = Column(Integer, ForeignKey('event.id'), primary_key=True)
|
||||
key = Column(Integer, primary_key=True)
|
||||
value = Column(Integer)
|
||||
|
||||
trait_type_id = Column(Integer, ForeignKey('trait_type.id'))
|
||||
trait_type = relationship("TraitType", backref='traits')
|
||||
|
||||
t_string = Column(String(255), nullable=True, default=None)
|
||||
t_float = Column(Float(53), nullable=True, default=None)
|
||||
t_int = Column(Integer, nullable=True, default=None)
|
||||
t_datetime = Column(PreciseTimestamp(), nullable=True, default=None)
|
||||
class TraitFloat(Base):
|
||||
"""Event float traits."""
|
||||
|
||||
event_id = Column(Integer, ForeignKey('event.id'))
|
||||
event = relationship("Event", backref=backref('traits', order_by=id))
|
||||
__tablename__ = 'trait_float'
|
||||
__table_args__ = (
|
||||
Index('ix_trait_float_event_id_key', 'event_id', 'key'),
|
||||
)
|
||||
event_id = Column(Integer, ForeignKey('event.id'), primary_key=True)
|
||||
key = Column(Integer, primary_key=True)
|
||||
value = Column(Float(53))
|
||||
|
||||
_value_map = {models.Trait.TEXT_TYPE: 't_string',
|
||||
models.Trait.FLOAT_TYPE: 't_float',
|
||||
models.Trait.INT_TYPE: 't_int',
|
||||
models.Trait.DATETIME_TYPE: 't_datetime'}
|
||||
|
||||
def __init__(self, trait_type, event, t_string=None,
|
||||
t_float=None, t_int=None, t_datetime=None):
|
||||
self.trait_type = trait_type
|
||||
self.t_string = t_string
|
||||
self.t_float = t_float
|
||||
self.t_int = t_int
|
||||
self.t_datetime = t_datetime
|
||||
self.event = event
|
||||
class TraitDatetime(Base):
|
||||
"""Event datetime traits."""
|
||||
|
||||
def get_value(self):
|
||||
if self.trait_type is None:
|
||||
dtype = None
|
||||
else:
|
||||
dtype = self.trait_type.data_type
|
||||
|
||||
if dtype == models.Trait.INT_TYPE:
|
||||
return self.t_int
|
||||
if dtype == models.Trait.FLOAT_TYPE:
|
||||
return self.t_float
|
||||
if dtype == models.Trait.DATETIME_TYPE:
|
||||
return self.t_datetime
|
||||
if dtype == models.Trait.TEXT_TYPE:
|
||||
return self.t_string
|
||||
|
||||
return None
|
||||
|
||||
def __repr__(self):
|
||||
name = self.trait_type.desc if self.trait_type else None
|
||||
data_type = (self.trait_type.data_type if self.trait_type else
|
||||
models.Trait.NONE_TYPE)
|
||||
|
||||
return "<Trait(%s) %d=%s/%s/%s/%s on %s>" % (name,
|
||||
data_type,
|
||||
self.t_string,
|
||||
self.t_float,
|
||||
self.t_int,
|
||||
self.t_datetime,
|
||||
self.event)
|
||||
__tablename__ = 'trait_datetime'
|
||||
__table_args__ = (
|
||||
Index('ix_trait_datetime_event_id_key', 'event_id', 'key'),
|
||||
)
|
||||
event_id = Column(Integer, ForeignKey('event.id'), primary_key=True)
|
||||
key = Column(Integer, primary_key=True)
|
||||
value = Column(PreciseTimestamp())
|
||||
|
||||
@ -120,18 +120,3 @@ class QueryTransformer(object):
|
||||
|
||||
def get_query(self):
|
||||
return self.query
|
||||
|
||||
|
||||
trait_models_dict = {'string': models.Trait.t_string,
|
||||
'integer': models.Trait.t_int,
|
||||
'datetime': models.Trait.t_datetime,
|
||||
'float': models.Trait.t_float}
|
||||
|
||||
|
||||
def trait_op_condition(conditions, trait_type, value, op='eq'):
|
||||
trait_model = trait_models_dict[trait_type]
|
||||
op_dict = {'eq': (trait_model == value), 'lt': (trait_model < value),
|
||||
'le': (trait_model <= value), 'gt': (trait_model > value),
|
||||
'ge': (trait_model >= value), 'ne': (trait_model != value)}
|
||||
conditions.append(op_dict[op])
|
||||
return conditions
|
||||
|
||||
@ -43,39 +43,6 @@ class CeilometerBaseTest(tests_db.TestBase):
|
||||
self.assertEqual('value', base['key'])
|
||||
|
||||
|
||||
@tests_db.run_with('sqlite', 'mysql', 'pgsql')
|
||||
class TraitTypeTest(tests_db.TestBase):
|
||||
# TraitType is a construct specific to sqlalchemy.
|
||||
# Not applicable to other drivers.
|
||||
|
||||
def test_trait_type_exists(self):
|
||||
tt1 = self.event_conn._get_or_create_trait_type("foo", 0)
|
||||
self.assertTrue(tt1.id >= 0)
|
||||
tt2 = self.event_conn._get_or_create_trait_type("foo", 0)
|
||||
self.assertEqual(tt2.id, tt1.id)
|
||||
self.assertEqual(tt2.desc, tt1.desc)
|
||||
self.assertEqual(tt2.data_type, tt1.data_type)
|
||||
|
||||
def test_new_trait_type(self):
|
||||
tt1 = self.event_conn._get_or_create_trait_type("foo", 0)
|
||||
self.assertTrue(tt1.id >= 0)
|
||||
tt2 = self.event_conn._get_or_create_trait_type("blah", 0)
|
||||
self.assertNotEqual(tt1.id, tt2.id)
|
||||
self.assertNotEqual(tt1.desc, tt2.desc)
|
||||
# Test the method __repr__ returns a string
|
||||
self.assertTrue(repr.repr(tt2))
|
||||
|
||||
def test_trait_different_data_type(self):
|
||||
tt1 = self.event_conn._get_or_create_trait_type("foo", 0)
|
||||
self.assertTrue(tt1.id >= 0)
|
||||
tt2 = self.event_conn._get_or_create_trait_type("foo", 1)
|
||||
self.assertNotEqual(tt1.id, tt2.id)
|
||||
self.assertEqual(tt2.desc, tt1.desc)
|
||||
self.assertNotEqual(tt1.data_type, tt2.data_type)
|
||||
# Test the method __repr__ returns a string
|
||||
self.assertTrue(repr.repr(tt2))
|
||||
|
||||
|
||||
@tests_db.run_with('sqlite', 'mysql', 'pgsql')
|
||||
class EventTypeTest(tests_db.TestBase):
|
||||
# EventType is a construct specific to sqlalchemy
|
||||
@ -104,65 +71,49 @@ class MyException(Exception):
|
||||
|
||||
@tests_db.run_with('sqlite', 'mysql', 'pgsql')
|
||||
class EventTest(tests_db.TestBase):
|
||||
def _verify_data(self, trait, trait_table):
|
||||
now = datetime.datetime.utcnow()
|
||||
ev = models.Event('1', 'name', now, [trait])
|
||||
self.event_conn.record_events([ev])
|
||||
session = self.event_conn._engine_facade.get_session()
|
||||
t_tables = [sql_models.TraitText, sql_models.TraitFloat,
|
||||
sql_models.TraitInt, sql_models.TraitDatetime]
|
||||
for table in t_tables:
|
||||
if table == trait_table:
|
||||
self.assertEqual(1, session.query(table).count())
|
||||
else:
|
||||
self.assertEqual(0, session.query(table).count())
|
||||
|
||||
def test_string_traits(self):
|
||||
model = models.Trait("Foo", models.Trait.TEXT_TYPE, "my_text")
|
||||
trait = self.event_conn._make_trait(model, None)
|
||||
self.assertEqual(models.Trait.TEXT_TYPE, trait.trait_type.data_type)
|
||||
self.assertIsNone(trait.t_float)
|
||||
self.assertIsNone(trait.t_int)
|
||||
self.assertIsNone(trait.t_datetime)
|
||||
self.assertEqual("my_text", trait.t_string)
|
||||
self.assertIsNotNone(trait.trait_type.desc)
|
||||
self._verify_data(model, sql_models.TraitText)
|
||||
|
||||
def test_int_traits(self):
|
||||
model = models.Trait("Foo", models.Trait.INT_TYPE, 100)
|
||||
trait = self.event_conn._make_trait(model, None)
|
||||
self.assertEqual(models.Trait.INT_TYPE, trait.trait_type.data_type)
|
||||
self.assertIsNone(trait.t_float)
|
||||
self.assertIsNone(trait.t_string)
|
||||
self.assertIsNone(trait.t_datetime)
|
||||
self.assertEqual(100, trait.t_int)
|
||||
self.assertIsNotNone(trait.trait_type.desc)
|
||||
self._verify_data(model, sql_models.TraitInt)
|
||||
|
||||
def test_float_traits(self):
|
||||
model = models.Trait("Foo", models.Trait.FLOAT_TYPE, 123.456)
|
||||
trait = self.event_conn._make_trait(model, None)
|
||||
self.assertEqual(models.Trait.FLOAT_TYPE, trait.trait_type.data_type)
|
||||
self.assertIsNone(trait.t_int)
|
||||
self.assertIsNone(trait.t_string)
|
||||
self.assertIsNone(trait.t_datetime)
|
||||
self.assertEqual(123.456, trait.t_float)
|
||||
self.assertIsNotNone(trait.trait_type.desc)
|
||||
self._verify_data(model, sql_models.TraitFloat)
|
||||
|
||||
def test_datetime_traits(self):
|
||||
now = datetime.datetime.utcnow()
|
||||
model = models.Trait("Foo", models.Trait.DATETIME_TYPE, now)
|
||||
trait = self.event_conn._make_trait(model, None)
|
||||
self.assertEqual(models.Trait.DATETIME_TYPE,
|
||||
trait.trait_type.data_type)
|
||||
self.assertIsNone(trait.t_int)
|
||||
self.assertIsNone(trait.t_string)
|
||||
self.assertIsNone(trait.t_float)
|
||||
self.assertEqual(now, trait.t_datetime)
|
||||
self.assertIsNotNone(trait.trait_type.desc)
|
||||
self._verify_data(model, sql_models.TraitDatetime)
|
||||
|
||||
def test_bad_event(self):
|
||||
now = datetime.datetime.utcnow()
|
||||
m = [models.Event("1", "Foo", now, []),
|
||||
models.Event("2", "Zoo", now, [])]
|
||||
|
||||
with mock.patch.object(self.event_conn, "_record_event") as mock_save:
|
||||
with mock.patch.object(self.event_conn,
|
||||
"_get_or_create_event_type") as mock_save:
|
||||
mock_save.side_effect = MyException("Boom")
|
||||
problem_events = self.event_conn.record_events(m)
|
||||
self.assertEqual(2, len(problem_events))
|
||||
for bad, event in problem_events:
|
||||
self.assertEqual(bad, models.Event.UNKNOWN_PROBLEM)
|
||||
|
||||
def test_get_none_value_traits(self):
|
||||
model = sql_models.Trait(None, None, 5)
|
||||
self.assertIsNone(model.get_value())
|
||||
self.assertTrue(repr.repr(model))
|
||||
|
||||
def test_event_repr(self):
|
||||
ev = sql_models.Event('msg_id', None, False)
|
||||
ev.id = 100
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user