openstack/aodh
Code Issues Proposed changes

Convert event timestamp to PrecisionTimestamp

Browse Source 
Before the timestamps for event and trait are stored in a Float
In mysql, the default precision won't work well, this one are rounded.

This patch, change the Float by the PrecisionTimestamp Column Type, like
the sample timestamp that demand the same precision.

And ensure that get_events always return datetime object.

Change-Id: I1f986784b07afa50bdbb7faa9499e7b614da8ec1
This commit is contained in:
Mehdi Abaakouk 2013-12-16 19:06:55 +01:00 committed by John Herndon
parent d8257c670f
commit 7194e92568
5 changed files with 109 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
ceilometer/storage/impl_sqlalchemy.py
View File

@ -863,8 +863,6 @@ class Connection(base.Connection):
values = {'t_string': None, 't_float': None,
't_int': None, 't_datetime': None}
value = trait_model.value
if trait_model.dtype == api_models.Trait.DATETIME_TYPE:
value = utils.dt_to_decimal(value)
values[value_map[trait_model.dtype]] = value
return models.Trait(trait_type, event, **values)
@ -893,8 +891,8 @@ class Connection(base.Connection):
event_type = cls._get_or_create_event_type(event_model.event_type,
session=session)
generated = utils.dt_to_decimal(event_model.generated)
event = models.Event(event_model.message_id, event_type, generated)
event = models.Event(event_model.message_id, event_type,
event_model.generated)
session.add(event)
new_traits = []
@ -944,8 +942,8 @@ class Connection(base.Connection):
:param event_filter: EventFilter instance
"""
start = utils.dt_to_decimal(event_filter.start_time)
end = utils.dt_to_decimal(event_filter.end_time)
start = event_filter.start_time
end = event_filter.end_time
session = sqlalchemy_session.get_session()
with session.begin():
event_query = session.query(models.Event)
@ -992,8 +990,7 @@ class Connection(base.Connection):
elif key == 't_int':
conditions.append(models.Trait.t_int == value)
elif key == 't_datetime':
dt = utils.dt_to_decimal(value)
conditions.append(models.Trait.t_datetime == dt)
conditions.append(models.Trait.t_datetime == value)
elif key == 't_float':
conditions.append(models.Trait.t_float == value)
@ -1030,11 +1027,10 @@ class Connection(base.Connection):
for trait in query.all():
event = event_models_dict.get(trait.event_id)
if not event:
generated = utils.decimal_to_dt(trait.event.generated)
event = api_models.Event(
trait.event.message_id,
trait.event.event_type.desc,
generated, [])
trait.event.generated, [])
event_models_dict[trait.event_id] = event
trait_model = api_models.Trait(trait.trait_type.desc,
trait.trait_type.data_type,
@ -1111,7 +1107,7 @@ class Connection(base.Connection):
.join(models.EventType,
and_(models.EventType.id ==
models.Event.event_type_id,
models.EventType.desc == event_type)))
models.EventType.desc == event_type)))
for trait in query.all():
type = trait.trait_type

81
ceilometer/storage/sqlalchemy/migrate_repo/versions/024_event_use_floatingprecision.py Normal file
View File

@ -0,0 +1,81 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2013 eNovance SAS <licensing@enovance.com>
#
# Author: Mehdi Abaakouk <mehdi.abaakouk@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sa
from ceilometer.storage.sqlalchemy import models
def _paged(query, size):
offset = 0
while True:
page = query.offset(offset).limit(size).execute()
if page.rowcount <= 0:
# There are no more rows
break
for row in page:
yield row
offset += size
def _convert_data_type(table, col, from_t, to_t, pk_attr='id', index=False):
temp_col_n = 'convert_data_type_temp_col'
# Override column we're going to convert with from_t, since the type we're
# replacing could be custom and we need to tell SQLALchemy how to perform
# CRUD operations with it.
table = sa.Table(table.name, table.metadata, sa.Column(col, from_t),
extend_existing=True)
sa.Column(temp_col_n, to_t).create(table)
key_attr = getattr(table.c, pk_attr)
orig_col = getattr(table.c, col)
new_col = getattr(table.c, temp_col_n)
query = sa.select([key_attr, orig_col])
for key, value in _paged(query, 1000):
table.update().where(key_attr == key)\
.values({temp_col_n: value}).execute()
orig_col.drop()
new_col.alter(name=col)
if index:
sa.Index('ix_%s_%s' % (table.name, col), new_col).create()
def upgrade(migrate_engine):
if migrate_engine.name == 'mysql':
meta = sa.MetaData(bind=migrate_engine)
event = sa.Table('event', meta, autoload=True)
_convert_data_type(event, 'generated', sa.Float(),
models.PreciseTimestamp(),
pk_attr='id', index=True)
trait = sa.Table('trait', meta, autoload=True)
_convert_data_type(trait, 't_datetime', sa.Float(),
models.PreciseTimestamp(),
pk_attr='id', index=True)
def downgrade(migrate_engine):
if migrate_engine.name == 'mysql':
meta = sa.MetaData(bind=migrate_engine)
event = sa.Table('event', meta, autoload=True)
_convert_data_type(event, 'generated', models.PreciseTimestamp(),
sa.Float(), pk_attr='id', index=True)
trait = sa.Table('trait', meta, autoload=True)
_convert_data_type(trait, 't_datetime', models.PreciseTimestamp(),
sa.Float(), pk_attr='id', index=True)

6
ceilometer/storage/sqlalchemy/models.py
View File

@ -320,7 +320,7 @@ class Event(Base):
)
id = Column(Integer, primary_key=True)
message_id = Column(String(50), unique=True)
generated = Column(Float(asdecimal=True))
generated = Column(PreciseTimestamp())
event_type_id = Column(Integer, ForeignKey('event_type.id'))
event_type = relationship("EventType", backref=backref('event_type'))
@ -379,7 +379,7 @@ class Trait(Base):
t_string = Column(String(255), nullable=True, default=None)
t_float = Column(Float, nullable=True, default=None)
t_int = Column(Integer, nullable=True, default=None)
t_datetime = Column(Float(asdecimal=True), nullable=True, default=None)
t_datetime = Column(PreciseTimestamp(), nullable=True, default=None)
event_id = Column(Integer, ForeignKey('event.id'))
event = relationship("Event", backref=backref('event', order_by=id))
@ -409,7 +409,7 @@ class Trait(Base):
if dtype == api_models.Trait.FLOAT_TYPE:
return self.t_float
if dtype == api_models.Trait.DATETIME_TYPE:
return utils.decimal_to_dt(self.t_datetime)
return self.t_datetime
if dtype == api_models.Trait.TEXT_TYPE:
return self.t_string

3
ceilometer/tests/storage/test_impl_sqlalchemy.py
View File

@ -34,7 +34,6 @@ from ceilometer.storage import models
from ceilometer.storage.sqlalchemy import models as sql_models
from ceilometer.tests import db as tests_db
from ceilometer.tests.storage import test_storage_scenarios as scenarios
from ceilometer import utils
class EventTestBase(tests_db.TestBase):
@ -147,7 +146,7 @@ class EventTest(EventTestBase):
self.assertIsNone(trait.t_int)
self.assertIsNone(trait.t_string)
self.assertIsNone(trait.t_float)
self.assertEqual(trait.t_datetime, utils.dt_to_decimal(now))
self.assertEqual(trait.t_datetime, now)
self.assertIsNotNone(trait.trait_type.desc)
def test_bad_event(self):

20
ceilometer/tests/storage/test_storage_scenarios.py
View File

@ -2145,7 +2145,7 @@ class EventTest(EventTestBase):
class GetEventTest(EventTestBase):
def prepare_data(self):
event_models = []
self.event_models = []
base = 0
self.start = datetime.datetime(2013, 12, 31, 5, 0)
now = self.start
@ -2160,14 +2160,28 @@ class GetEventTest(EventTestBase):
('trait_C', models.Trait.FLOAT_TYPE,
float(base) + 0.123456),
('trait_D', models.Trait.DATETIME_TYPE, now)]]
event_models.append(
self.event_models.append(
models.Event("id_%s_%d" % (event_type, base),
event_type, now, trait_models))
base += 100
now = now + datetime.timedelta(hours=1)
self.end = now
self.conn.record_events(event_models)
self.conn.record_events(self.event_models)
def test_generated_is_datetime(self):
event_filter = storage.EventFilter(self.start, self.end)
events = self.conn.get_events(event_filter)
self.assertEqual(6, len(events))
for i, event in enumerate(events):
self.assertTrue(isinstance(event.generated, datetime.datetime))
self.assertEqual(event.generated,
self.event_models[i].generated)
model_traits = self.event_models[i].traits
for j, trait in enumerate(event.traits):
if trait.dtype == models.Trait.DATETIME_TYPE:
self.assertTrue(isinstance(trait.value, datetime.datetime))
self.assertEqual(trait.value, model_traits[j].value)
def test_simple_get(self):
event_filter = storage.EventFilter(self.start, self.end)