isolate event storage models

move event storage models to event module. this is the first step
to separating metering and event data.

Change-Id: I6d720690a958f5b3accf22db66c830bfa48ed9ae
Partially-Implements: blueprint dedicated-event-db
This commit is contained in:
gordon chung 2014-10-01 16:18:42 -04:00
parent 1278d067ef
commit 303da461f5
16 changed files with 198 additions and 175 deletions

View File

@ -51,6 +51,7 @@ import ceilometer
from ceilometer.alarm import service as alarm_service
from ceilometer.alarm.storage import models as alarm_models
from ceilometer.api import acl
from ceilometer.event.storage import models as event_models
from ceilometer import messaging
from ceilometer.openstack.common import context
from ceilometer.openstack.common.gettextutils import _
@ -2279,9 +2280,9 @@ class Trait(_Base):
if isinstance(trait, Trait):
return trait
value = (six.text_type(trait.value)
if not trait.dtype == storage.models.Trait.DATETIME_TYPE
if not trait.dtype == event_models.Trait.DATETIME_TYPE
else trait.value.isoformat())
trait_type = storage.models.Trait.get_name_by_type(trait.dtype)
trait_type = event_models.Trait.get_name_by_type(trait.dtype)
return Trait(name=trait.name, type=trait_type, value=value)
@classmethod
@ -2395,7 +2396,7 @@ class TraitsController(rest.RestController):
:param event_type: Event type to filter traits by
"""
get_trait_name = storage.models.Trait.get_name_by_type
get_trait_name = event_models.Trait.get_name_by_type
return [TraitDescription(name=t['name'],
type=get_trait_name(t['data_type']))
for t in pecan.request.storage_conn

View File

@ -24,9 +24,9 @@ from oslo.utils import timeutils
import six
import yaml
from ceilometer.event.storage import models
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log
from ceilometer.storage import models
OPTS = [
cfg.StrOpt('definitions_cfg_file',

View File

@ -24,9 +24,9 @@ from stevedore import extension
import ceilometer
from ceilometer import dispatcher
from ceilometer.event import converter as event_converter
from ceilometer.event.storage import models
from ceilometer import messaging
from ceilometer.openstack.common.gettextutils import _
from ceilometer.storage import models
LOG = logging.getLogger(__name__)

View File

View File

@ -0,0 +1,106 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Model classes for use in the events storage API.
"""
from oslo.utils import timeutils
from ceilometer.storage import base
class Event(base.Model):
"""A raw event from the source system. Events have Traits.
Metrics will be derived from one or more Events.
"""
DUPLICATE = 1
UNKNOWN_PROBLEM = 2
def __init__(self, message_id, event_type, generated, traits):
"""Create a new event.
:param message_id: Unique ID for the message this event
stemmed from. This is different than
the Event ID, which comes from the
underlying storage system.
:param event_type: The type of the event.
:param generated: UTC time for when the event occurred.
:param traits: list of Traits on this Event.
"""
base.Model.__init__(self, message_id=message_id, event_type=event_type,
generated=generated, traits=traits)
def append_trait(self, trait_model):
self.traits.append(trait_model)
def __repr__(self):
trait_list = []
if self.traits:
trait_list = [str(trait) for trait in self.traits]
return ("<Event: %s, %s, %s, %s>" %
(self.message_id, self.event_type, self.generated,
" ".join(trait_list)))
class Trait(base.Model):
"""A Trait is a key/value pair of data on an Event.
The value is variant record of basic data types (int, date, float, etc).
"""
NONE_TYPE = 0
TEXT_TYPE = 1
INT_TYPE = 2
FLOAT_TYPE = 3
DATETIME_TYPE = 4
type_names = {
NONE_TYPE: "none",
TEXT_TYPE: "string",
INT_TYPE: "integer",
FLOAT_TYPE: "float",
DATETIME_TYPE: "datetime"
}
def __init__(self, name, dtype, value):
if not dtype:
dtype = Trait.NONE_TYPE
base.Model.__init__(self, name=name, dtype=dtype, value=value)
def __repr__(self):
return "<Trait: %s %d %s>" % (self.name, self.dtype, self.value)
def get_type_name(self):
return self.get_name_by_type(self.dtype)
@classmethod
def get_type_by_name(cls, type_name):
return getattr(cls, '%s_TYPE' % type_name.upper(), None)
@classmethod
def get_type_names(cls):
return cls.type_names.values()
@classmethod
def get_name_by_type(cls, type_id):
return cls.type_names.get(type_id, "none")
@classmethod
def convert_value(cls, trait_type, value):
if trait_type is cls.INT_TYPE:
return int(value)
if trait_type is cls.FLOAT_TYPE:
return float(value)
if trait_type is cls.DATETIME_TYPE:
return timeutils.normalize_time(timeutils.parse_isotime(value))
return str(value)

View File

@ -23,6 +23,7 @@ from oslo.utils import timeutils
from six.moves.urllib import parse as urlparse
import ceilometer
from ceilometer.event.storage import models as ev_models
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log
from ceilometer.storage import base
@ -529,7 +530,7 @@ class Connection(base.Connection):
events_table.put(row, record)
except Exception as ex:
LOG.debug(_("Failed to record event: %s") % ex)
problem_events.append((models.Event.UNKNOWN_PROBLEM,
problem_events.append((ev_models.Event.UNKNOWN_PROBLEM,
event_model))
return problem_events
@ -553,12 +554,12 @@ class Connection(base.Connection):
if (not key.startswith('event_type')
and not key.startswith('timestamp')):
trait_name, trait_dtype = key.rsplit('+', 1)
traits.append(models.Trait(name=trait_name,
dtype=int(trait_dtype),
value=value))
traits.append(ev_models.Trait(name=trait_name,
dtype=int(trait_dtype),
value=value))
ts, mess = event_id.split('_', 1)
yield models.Event(
yield ev_models.Event(
message_id=mess,
event_type=events_dict['event_type'],
generated=events_dict['timestamp'],
@ -608,7 +609,7 @@ class Connection(base.Connection):
# proposed that certain trait name could have only one
# trait type.
trait_names.add(trait_name)
data_type = models.Trait.type_names[int(trait_type)]
data_type = ev_models.Trait.type_names[int(trait_type)]
yield {'name': trait_name, 'data_type': data_type}
def get_traits(self, event_type, trait_type=None):
@ -629,5 +630,5 @@ class Connection(base.Connection):
if (not key.startswith('event_type') and
not key.startswith('timestamp')):
trait_name, trait_type = key.rsplit('+', 1)
yield models.Trait(name=trait_name,
dtype=int(trait_type), value=value)
yield ev_models.Trait(name=trait_name,
dtype=int(trait_type), value=value)

View File

@ -34,6 +34,7 @@ from sqlalchemy import func
from sqlalchemy.orm import aliased
import ceilometer
from ceilometer.event.storage import models as ev_models
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import jsonutils
from ceilometer.openstack.common import log
@ -806,11 +807,11 @@ class Connection(base.Connection):
event = self._record_event(session, event_model)
except dbexc.DBDuplicateEntry as e:
LOG.exception(_("Failed to record duplicated event: %s") % e)
problem_events.append((api_models.Event.DUPLICATE,
problem_events.append((ev_models.Event.DUPLICATE,
event_model))
except Exception as e:
LOG.exception(_('Failed to record event: %s') % e)
problem_events.append((api_models.Event.UNKNOWN_PROBLEM,
problem_events.append((ev_models.Event.UNKNOWN_PROBLEM,
event_model))
events.append(event)
return problem_events
@ -886,10 +887,10 @@ class Connection(base.Connection):
if event_filter_conditions:
query = query.filter(and_(*event_filter_conditions))
for (id_, generated, message_id, desc_) in query.all():
event_models_dict[id_] = api_models.Event(message_id,
desc_,
generated,
[])
event_models_dict[id_] = ev_models.Event(message_id,
desc_,
generated,
[])
# Build event models for the events
event_query = event_query.subquery()
@ -903,14 +904,14 @@ class Connection(base.Connection):
for trait in query.all():
event = event_models_dict.get(trait.event_id)
if not event:
event = api_models.Event(
event = ev_models.Event(
trait.event.message_id,
trait.event.event_type.desc,
trait.event.generated, [])
event_models_dict[trait.event_id] = event
trait_model = api_models.Trait(trait.trait_type.desc,
trait.trait_type.data_type,
trait.get_value())
trait_model = ev_models.Trait(trait.trait_type.desc,
trait.trait_type.data_type,
trait.get_value())
event.append_trait(trait_model)
event_models = event_models_dict.values()
@ -983,6 +984,6 @@ class Connection(base.Connection):
for trait in query.all():
type = trait.trait_type
yield api_models.Trait(name=type.desc,
dtype=type.data_type,
value=trait.get_value())
yield ev_models.Trait(name=type.desc,
dtype=type.data_type,
value=trait.get_value())

View File

@ -16,100 +16,9 @@
# under the License.
"""Model classes for use in the storage API.
"""
from oslo.utils import timeutils
from ceilometer.storage import base
class Event(base.Model):
"""A raw event from the source system. Events have Traits.
Metrics will be derived from one or more Events.
"""
DUPLICATE = 1
UNKNOWN_PROBLEM = 2
def __init__(self, message_id, event_type, generated, traits):
"""Create a new event.
:param message_id: Unique ID for the message this event
stemmed from. This is different than
the Event ID, which comes from the
underlying storage system.
:param event_type: The type of the event.
:param generated: UTC time for when the event occurred.
:param traits: list of Traits on this Event.
"""
base.Model.__init__(self, message_id=message_id, event_type=event_type,
generated=generated, traits=traits)
def append_trait(self, trait_model):
self.traits.append(trait_model)
def __repr__(self):
trait_list = []
if self.traits:
trait_list = [str(trait) for trait in self.traits]
return ("<Event: %s, %s, %s, %s>" %
(self.message_id, self.event_type, self.generated,
" ".join(trait_list)))
class Trait(base.Model):
"""A Trait is a key/value pair of data on an Event.
The value is variant record of basic data types (int, date, float, etc).
"""
NONE_TYPE = 0
TEXT_TYPE = 1
INT_TYPE = 2
FLOAT_TYPE = 3
DATETIME_TYPE = 4
type_names = {
NONE_TYPE: "none",
TEXT_TYPE: "string",
INT_TYPE: "integer",
FLOAT_TYPE: "float",
DATETIME_TYPE: "datetime"
}
def __init__(self, name, dtype, value):
if not dtype:
dtype = Trait.NONE_TYPE
base.Model.__init__(self, name=name, dtype=dtype, value=value)
def __repr__(self):
return "<Trait: %s %d %s>" % (self.name, self.dtype, self.value)
def get_type_name(self):
return self.get_name_by_type(self.dtype)
@classmethod
def get_type_by_name(cls, type_name):
return getattr(cls, '%s_TYPE' % type_name.upper(), None)
@classmethod
def get_type_names(cls):
return cls.type_names.values()
@classmethod
def get_name_by_type(cls, type_id):
return cls.type_names.get(type_id, "none")
@classmethod
def convert_value(cls, trait_type, value):
if trait_type is cls.INT_TYPE:
return int(value)
if trait_type is cls.FLOAT_TYPE:
return float(value)
if trait_type is cls.DATETIME_TYPE:
return timeutils.normalize_time(timeutils.parse_isotime(value))
return str(value)
class Resource(base.Model):
"""Something for which sample data has been collected."""

View File

@ -20,6 +20,7 @@
import pymongo
import ceilometer
from ceilometer.event.storage import models as ev_models
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log
from ceilometer.storage import base
@ -138,11 +139,11 @@ class Connection(base.Connection):
'traits': traits})
except pymongo.errors.DuplicateKeyError as ex:
LOG.exception(_("Failed to record duplicated event: %s") % ex)
problem_events.append((models.Event.DUPLICATE,
problem_events.append((ev_models.Event.DUPLICATE,
event_model))
except Exception as ex:
LOG.exception(_("Failed to record event: %s") % ex)
problem_events.append((models.Event.UNKNOWN_PROBLEM,
problem_events.append((ev_models.Event.UNKNOWN_PROBLEM,
event_model))
return problem_events
@ -156,13 +157,14 @@ class Connection(base.Connection):
for event in self.db.event.find(q):
traits = []
for trait in event['traits']:
traits.append(models.Trait(name=trait['trait_name'],
dtype=int(trait['trait_type']),
value=trait['trait_value']))
yield models.Event(message_id=event['_id'],
event_type=event['event_type'],
generated=event['timestamp'],
traits=traits)
traits.append(
ev_models.Trait(name=trait['trait_name'],
dtype=int(trait['trait_type']),
value=trait['trait_value']))
yield ev_models.Event(message_id=event['_id'],
event_type=event['event_type'],
generated=event['timestamp'],
traits=traits)
def get_event_types(self):
"""Return all event types as an iter of strings."""
@ -217,9 +219,9 @@ class Connection(base.Connection):
})
for event in events:
for trait in event['traits']:
yield models.Trait(name=trait['trait_name'],
dtype=trait['trait_type'],
value=trait['trait_value'])
yield ev_models.Trait(name=trait['trait_name'],
dtype=trait['trait_type'],
value=trait['trait_value'])
def query_samples(self, filter_expr=None, orderby=None, limit=None):
if limit == 0:

View File

@ -32,7 +32,7 @@ from sqlalchemy.orm import deferred
from sqlalchemy.orm import relationship
from sqlalchemy.types import TypeDecorator
from ceilometer.storage import models as api_models
from ceilometer.event.storage import models
from ceilometer import utils
@ -380,10 +380,10 @@ class Trait(Base):
event_id = Column(Integer, ForeignKey('event.id'))
event = relationship("Event", backref=backref('traits', order_by=id))
_value_map = {api_models.Trait.TEXT_TYPE: 't_string',
api_models.Trait.FLOAT_TYPE: 't_float',
api_models.Trait.INT_TYPE: 't_int',
api_models.Trait.DATETIME_TYPE: 't_datetime'}
_value_map = {models.Trait.TEXT_TYPE: 't_string',
models.Trait.FLOAT_TYPE: 't_float',
models.Trait.INT_TYPE: 't_int',
models.Trait.DATETIME_TYPE: 't_datetime'}
def __init__(self, trait_type, event, t_string=None,
t_float=None, t_int=None, t_datetime=None):
@ -400,13 +400,13 @@ class Trait(Base):
else:
dtype = self.trait_type.data_type
if dtype == api_models.Trait.INT_TYPE:
if dtype == models.Trait.INT_TYPE:
return self.t_int
if dtype == api_models.Trait.FLOAT_TYPE:
if dtype == models.Trait.FLOAT_TYPE:
return self.t_float
if dtype == api_models.Trait.DATETIME_TYPE:
if dtype == models.Trait.DATETIME_TYPE:
return self.t_datetime
if dtype == api_models.Trait.TEXT_TYPE:
if dtype == models.Trait.TEXT_TYPE:
return self.t_string
return None
@ -414,7 +414,7 @@ class Trait(Base):
def __repr__(self):
name = self.trait_type.desc if self.trait_type else None
data_type = (self.trait_type.data_type if self.trait_type else
api_models.Trait.NONE_TYPE)
models.Trait.NONE_TYPE)
return "<Trait(%s) %d=%s/%s/%s/%s on %s>" % (name,
data_type,

View File

@ -19,7 +19,7 @@ import datetime
from oslo.utils import timeutils
import webtest.app
from ceilometer.storage import models
from ceilometer.event.storage import models
from ceilometer.tests.api import v2
from ceilometer.tests import db as tests_db

View File

@ -23,7 +23,7 @@ from oslo.config import cfg as oslo_cfg
import six
from ceilometer.event import converter
from ceilometer.storage import models
from ceilometer.event.storage import models
from ceilometer.tests import base

View File

@ -24,7 +24,7 @@ from stevedore import extension
import ceilometer
from ceilometer.event import endpoint as event_endpoint
from ceilometer.storage import models
from ceilometer.event.storage import models
from ceilometer.tests import base as tests_base
TEST_NOTICE_CTXT = {

View File

@ -29,8 +29,8 @@ import mock
from oslo.utils import timeutils
from ceilometer.alarm.storage import impl_sqlalchemy as impl_sqla_alarm
from ceilometer.event.storage import models
from ceilometer.storage import impl_sqlalchemy
from ceilometer.storage import models
from ceilometer.storage.sqlalchemy import models as sql_models
from ceilometer.tests import base as test_base
from ceilometer.tests import db as tests_db

View File

@ -20,6 +20,7 @@ import datetime
from oslotest import base as testbase
from ceilometer.alarm.storage import models as alarm_models
from ceilometer.event.storage import models as event_models
from ceilometer.storage import base
from ceilometer.storage import models
@ -58,7 +59,7 @@ class ModelTest(testbase.BaseTestCase):
d)
def test_event_repr_no_traits(self):
x = models.Event("1", "name", "now", None)
x = event_models.Event("1", "name", "now", None)
self.assertEqual("<Event: 1, name, now, >", repr(x))
def test_get_field_names_of_sample(self):
@ -93,21 +94,21 @@ class ModelTest(testbase.BaseTestCase):
class TestTraitModel(testbase.BaseTestCase):
def test_convert_value(self):
v = models.Trait.convert_value(
models.Trait.INT_TYPE, '10')
v = event_models.Trait.convert_value(
event_models.Trait.INT_TYPE, '10')
self.assertEqual(10, v)
self.assertIsInstance(v, int)
v = models.Trait.convert_value(
models.Trait.FLOAT_TYPE, '10')
v = event_models.Trait.convert_value(
event_models.Trait.FLOAT_TYPE, '10')
self.assertEqual(10.0, v)
self.assertIsInstance(v, float)
v = models.Trait.convert_value(
models.Trait.DATETIME_TYPE, '2013-08-08 21:05:37.123456')
v = event_models.Trait.convert_value(
event_models.Trait.DATETIME_TYPE, '2013-08-08 21:05:37.123456')
self.assertEqual(datetime.datetime(2013, 8, 8, 21, 5, 37, 123456), v)
self.assertIsInstance(v, datetime.datetime)
v = models.Trait.convert_value(
models.Trait.TEXT_TYPE, 10)
v = event_models.Trait.convert_value(
event_models.Trait.TEXT_TYPE, 10)
self.assertEqual("10", v)
self.assertIsInstance(v, str)

View File

@ -27,11 +27,11 @@ from oslo.utils import timeutils
import ceilometer
from ceilometer.alarm.storage import models as alarm_models
from ceilometer.event.storage import models as event_models
from ceilometer.publisher import utils
from ceilometer import sample
from ceilometer import storage
from ceilometer.storage import base
from ceilometer.storage import models
from ceilometer.tests import db as tests_db
@ -2721,12 +2721,12 @@ class EventTest(EventTestBase):
@tests_db.run_with('sqlite', 'mongodb', 'db2')
def test_duplicate_message_id(self):
now = datetime.datetime.utcnow()
m = [models.Event("1", "Foo", now, None),
models.Event("1", "Zoo", now, [])]
m = [event_models.Event("1", "Foo", now, None),
event_models.Event("1", "Zoo", now, [])]
problem_events = self.conn.record_events(m)
self.assertEqual(1, len(problem_events))
bad = problem_events[0]
self.assertEqual(models.Event.DUPLICATE, bad[0])
self.assertEqual(event_models.Event.DUPLICATE, bad[0])
class GetEventTest(EventTestBase):
@ -2736,18 +2736,19 @@ class GetEventTest(EventTestBase):
self.start = datetime.datetime(2013, 12, 31, 5, 0)
now = self.start
for event_type in ['Foo', 'Bar', 'Zoo', 'Foo', 'Bar', 'Zoo']:
trait_models = [models.Trait(name, dtype, value)
trait_models = [event_models.Trait(name, dtype, value)
for name, dtype, value in [
('trait_A', models.Trait.TEXT_TYPE,
('trait_A', event_models.Trait.TEXT_TYPE,
"my_%s_text" % event_type),
('trait_B', models.Trait.INT_TYPE,
('trait_B', event_models.Trait.INT_TYPE,
base + 1),
('trait_C', models.Trait.FLOAT_TYPE,
('trait_C', event_models.Trait.FLOAT_TYPE,
float(base) + 0.123456),
('trait_D', models.Trait.DATETIME_TYPE, now)]]
('trait_D', event_models.Trait.DATETIME_TYPE,
now)]]
self.event_models.append(
models.Event("id_%s_%d" % (event_type, base),
event_type, now, trait_models))
event_models.Event("id_%s_%d" % (event_type, base),
event_type, now, trait_models))
base += 100
now = now + datetime.timedelta(hours=1)
self.end = now
@ -2764,7 +2765,7 @@ class GetEventTest(EventTestBase):
self.event_models[i].generated)
model_traits = self.event_models[i].traits
for j, trait in enumerate(event.traits):
if trait.dtype == models.Trait.DATETIME_TYPE:
if trait.dtype == event_models.Trait.DATETIME_TYPE:
self.assertIsInstance(trait.value, datetime.datetime)
self.assertEqual(trait.value, model_traits[j].value)
@ -3035,13 +3036,13 @@ class GetEventTest(EventTestBase):
trait_dict[trait.name] = trait.dtype
self.assertTrue("trait_A" in trait_dict)
self.assertEqual(models.Trait.TEXT_TYPE, trait_dict["trait_A"])
self.assertEqual(event_models.Trait.TEXT_TYPE, trait_dict["trait_A"])
self.assertTrue("trait_B" in trait_dict)
self.assertEqual(models.Trait.INT_TYPE, trait_dict["trait_B"])
self.assertEqual(event_models.Trait.INT_TYPE, trait_dict["trait_B"])
self.assertTrue("trait_C" in trait_dict)
self.assertEqual(models.Trait.FLOAT_TYPE, trait_dict["trait_C"])
self.assertEqual(event_models.Trait.FLOAT_TYPE, trait_dict["trait_C"])
self.assertTrue("trait_D" in trait_dict)
self.assertEqual(models.Trait.DATETIME_TYPE,
self.assertEqual(event_models.Trait.DATETIME_TYPE,
trait_dict["trait_D"])
def test_get_all_traits(self):
@ -3050,10 +3051,11 @@ class GetEventTest(EventTestBase):
self.assertEqual(8, len(traits))
trait = traits[0]
self.assertEqual("trait_A", trait.name)
self.assertEqual(models.Trait.TEXT_TYPE, trait.dtype)
self.assertEqual(event_models.Trait.TEXT_TYPE, trait.dtype)
def test_simple_get_event_no_traits(self):
new_events = [models.Event("id_notraits", "NoTraits", self.start, [])]
new_events = [event_models.Event("id_notraits", "NoTraits",
self.start, [])]
bad_events = self.conn.record_events(new_events)
event_filter = storage.EventFilter(self.start, self.end, "NoTraits")
events = [event for event in self.conn.get_events(event_filter)]
@ -3069,10 +3071,10 @@ class GetEventTest(EventTestBase):
self.assertEqual(6, len(events))
def test_get_by_message_id(self):
new_events = [models.Event("id_testid",
"MessageIDTest",
self.start,
[])]
new_events = [event_models.Event("id_testid",
"MessageIDTest",
self.start,
[])]
bad_events = self.conn.record_events(new_events)
event_filter = storage.EventFilter(message_id="id_testid")