Merge "Add Event API"

ceilometer/api/controllers/v2.py

@@ -26,6 +26,7 @@ import ast
 import base64
 import copy
 import datetime
+import functools
 import inspect
 import json
 import uuid
@@ -179,9 +180,19 @@ class Link(_Base):
 
 
 class Query(_Base):
-    """Sample query filter.
+    """Query filter.
     """
 
+    # The data types supported by the query.
+    _supported_types = ['integer', 'float', 'string', 'boolean']
+
+    # Functions to convert the data field to the correct type.
+    _type_converters = {'integer': int,
+                        'float': float,
+                        'boolean': strutils.bool_from_string,
+                        'string': six.text_type,
+                        'datetime': timeutils.parse_isotime}
+
     _op = None  # provide a default
 
     def get_op(self):
@@ -249,28 +260,22 @@ class Query(_Base):
                             ' automatically') % (self.value)
                     LOG.debug(msg)
             else:
-                if type == 'integer':
+                if type not in self._supported_types:
-                    converted_value = int(self.value)
+                    # Types must be explicitly declared so the
-                elif type == 'float':
+                    # correct type converter may be used. Subclasses
-                    converted_value = float(self.value)
+                    # of Query may define _supported_types and
-                elif type == 'boolean':
+                    # _type_converters to define their own types.
-                    converted_value = strutils.bool_from_string(self.value)
-                elif type == 'string':
-                    converted_value = self.value
-                else:
-                    # For now, this method only support integer, float,
-                    # boolean and and string as the metadata type. A TypeError
-                    # will be raised for any other type.
                     raise TypeError()
+                converted_value = self._type_converters[type](self.value)
         except ValueError:
-            msg = _('Failed to convert the metadata value %(value)s'
+            msg = _('Failed to convert the value %(value)s'
                     ' to the expected data type %(type)s.') % \
                 {'value': self.value, 'type': type}
             raise ClientSideError(msg)
         except TypeError:
-            msg = _('The data type %s is not supported. The supported'
+            msg = _('The data type %(type)s is not supported. The supported'
-                    ' data type list is: integer, float, boolean and'
+                    ' data type list is: %(supported)s') % \
-                    ' string.') % (type)
+                {'type': type, 'supported': self._supported_types}
             raise ClientSideError(msg)
         except Exception:
             msg = _('Unexpected exception converting %(value)s to'
@@ -1591,6 +1596,227 @@ class AlarmsController(rest.RestController):
                 for m in pecan.request.storage_conn.get_alarms(**kwargs)]
 
 
+class TraitDescription(_Base):
+    """A description of a trait, with no associated value."""
+
+    type = wtypes.text
+    "the data type, defaults to string"
+
+    name = wtypes.text
+    "the name of the trait"
+
+    @classmethod
+    def sample(cls):
+        return cls(name='service',
+                   type='string'
+                   )
+
+
+class EventQuery(Query):
+    """Query arguments for Event Queries."""
+
+    _supported_types = ['integer', 'float', 'string', 'datetime']
+
+    type = wsme.wsattr(wtypes.text, default='string')
+    "the type of the trait filter, defaults to string"
+
+    def __repr__(self):
+        # for logging calls
+        return '<EventQuery %r %s %r %s>' % (self.field,
+                                             self.op,
+                                             self._get_value_as_type(),
+                                             self.type)
+
+
+class Trait(_Base):
+    """A Trait associated with an event."""
+
+    name = wtypes.text
+    "The name of the trait"
+
+    value = wtypes.text
+    "the value of the trait"
+
+    type = wtypes.text
+    "the type of the trait (string, integer, float or datetime)"
+
+    @classmethod
+    def sample(cls):
+        return cls(name='service',
+                   type='string',
+                   value='compute.hostname'
+                   )
+
+
+class Event(_Base):
+    """A System event."""
+
+    message_id = wtypes.text
+    "The message ID for the notification"
+
+    event_type = wtypes.text
+    "The type of the event"
+
+    _traits = None
+
+    def get_traits(self):
+        return self._traits
+
+    def set_traits(self, traits):
+        self._traits = {}
+        for trait in traits:
+            if trait.dtype == storage.models.Trait.DATETIME_TYPE:
+                value = trait.value.isoformat()
+            else:
+                value = six.text_type(trait.value)
+            self._traits[trait.name] = value
+
+    traits = wsme.wsproperty({wtypes.text: wtypes.text},
+                             get_traits,
+                             set_traits)
+    "Event specific properties"
+
+    generated = datetime.datetime
+    "The time the event occurred"
+
+    @classmethod
+    def sample(cls):
+        return cls(
+            event_type='compute.instance.update',
+            generated='2013-11-11T20:00:00',
+            message_id='94834db1-8f1b-404d-b2ec-c35901f1b7f0',
+            traits={
+                'request_id': 'req-4e2d67b8-31a4-48af-bb2f-9df72a353a72',
+                'service': 'conductor.tem-devstack-01',
+                'tenant_id': '7f13f2b17917463b9ee21aa92c4b36d6'
+            }
+        )
+
+
+def requires_admin(func):
+
+    @functools.wraps(func)
+    def wrapped(*args, **kwargs):
+        usr_limit, proj_limit = acl.get_limited_to(pecan.request.headers)
+        # If User and Project are None, you have full access.
+        if usr_limit and proj_limit:
+            raise ClientSideError(_("Not Authorized"), status_code=403)
+        return func(*args, **kwargs)
+
+    return wrapped
+
+
+def _event_query_to_event_filter(q):
+    evt_model_filter = {
+        'event_type': None,
+        'message_id': None,
+        'start_time': None,
+        'end_time': None
+    }
+    traits_filter = []
+
+    for i in q:
+        # FIXME(herndon): Support for operators other than
+        # 'eq' will come later.
+        if i.op != 'eq':
+            error = _("operator %s not supported") % i.op
+            raise ClientSideError(error)
+        if i.field in evt_model_filter:
+            evt_model_filter[i.field] = i.value
+        else:
+            traits_filter.append({"key": i.field,
+                                  i.type: i._get_value_as_type()})
+    return storage.EventFilter(traits_filter=traits_filter, **evt_model_filter)
+
+
+class TraitsController(rest.RestController):
+    """Works on Event Traits."""
+
+    @requires_admin
+    @wsme_pecan.wsexpose([Trait], wtypes.text, wtypes.text)
+    def get_one(self, event_type, trait_name):
+        """Return all instances of a trait for an event type.
+
+        :param event_type: Event type to filter traits by
+        :param trait_name: Trait to return values for
+        """
+        LOG.debug(_("Getting traits for %s") % event_type)
+        return [Trait(name=t.name, type=t.get_type_name(), value=t.value)
+                for t in pecan.request.storage_conn
+                .get_traits(event_type, trait_name)]
+
+    @requires_admin
+    @wsme_pecan.wsexpose([TraitDescription], wtypes.text)
+    def get_all(self, event_type):
+        """Return all trait names for an event type.
+
+        :param event_type: Event type to filter traits by
+        """
+        get_trait_name = storage.models.Trait.get_name_by_type
+        return [TraitDescription(name=t['name'],
+                                 type=get_trait_name(t['data_type']))
+                for t in pecan.request.storage_conn
+                .get_trait_types(event_type)]
+
+
+class EventTypesController(rest.RestController):
+    """Works on Event Types in the system."""
+
+    traits = TraitsController()
+
+    # FIXME(herndon): due to a bug in pecan, making this method
+    # get_all instead of get will hide the traits subcontroller.
+    # https://bugs.launchpad.net/pecan/+bug/1262277
+    @requires_admin
+    @wsme_pecan.wsexpose([unicode])
+    def get(self):
+        """Get all event types.
+        """
+        return list(pecan.request.storage_conn.get_event_types())
+
+
+class EventsController(rest.RestController):
+    """Works on Events."""
+
+    @requires_admin
+    @wsme_pecan.wsexpose([Event], [EventQuery])
+    def get_all(self, q=[]):
+        """Return all events matching the query filters.
+
+        :param q: Filter arguments for which Events to return
+        """
+        event_filter = _event_query_to_event_filter(q)
+        return [Event(message_id=event.message_id,
+                      event_type=event.event_type,
+                      generated=event.generated,
+                      traits=event.traits)
+                for event in
+                pecan.request.storage_conn.get_events(event_filter)]
+
+    @requires_admin
+    @wsme_pecan.wsexpose(Event, wtypes.text)
+    def get_one(self, message_id):
+        """Return a single event with the given message id.
+
+        :param message_id: Message ID of the Event to be returned
+        """
+        event_filter = storage.EventFilter(message_id=message_id)
+        events = pecan.request.storage_conn.get_events(event_filter)
+        if len(events) == 0:
+            raise EntityNotFound(_("Event"), message_id)
+
+        if len(events) > 1:
+            LOG.error(_("More than one event with "
+                        "id %s returned from storage driver") % message_id)
+
+        event = events[0]
+
+        return Event(message_id=event.message_id,
+                     event_type=event.event_type,
+                     generated=event.generated,
+                     traits=event.traits)
+
+
 class V2Controller(object):
     """Version 2 API controller root."""
 
@@ -1598,3 +1824,5 @@ class V2Controller(object):
     meters = MetersController()
     samples = SamplesController()
     alarms = AlarmsController()
+    event_types = EventTypesController()
+    events = EventsController()

ceilometer/storage/__init__.py

@@ -21,6 +21,7 @@
 import urlparse
 
 from oslo.config import cfg
+import six
 from stevedore import driver
 
 from ceilometer.openstack.common.gettextutils import _  # noqa
@@ -124,10 +125,10 @@ class EventFilter(object):
                    This parameter is a list of dictionaries that specify
                    trait values:
                     {'key': <key>,
-                    't_string': <value>,
+                    'string': <value>,
-                    't_int': <value>,
+                    'integer': <value>,
-                    't_datetime': <value>
+                    'datetime': <value>,
-                    't_float': <value>,
+                    'float': <value>,
                     'op': <eq, lt, le, ne, gt or ge> }
     """
 
@@ -139,6 +140,16 @@ class EventFilter(object):
         self.event_type = event_type
         self.traits_filter = traits_filter
 
+    def __repr__(self):
+        return ("<EventFilter(start_time: %s,"
+                " end_time: %s,"
+                " event_type: %s,"
+                " traits: %s)>" %
+                (self.start_time,
+                 self.end_time,
+                 self.event_type,
+                 six.text_type(self.traits_filter)))
+
 
 def dbsync():
     service.prepare_service()

ceilometer/storage/impl_sqlalchemy.py

@@ -949,6 +949,7 @@ class Connection(base.Connection):
         start = event_filter.start_time
         end = event_filter.end_time
         session = sqlalchemy_session.get_session()
+        LOG.debug(_("Getting events that match filter: %s") % event_filter)
         with session.begin():
             event_query = session.query(models.Event)
 
@@ -989,13 +990,13 @@ class Connection(base.Connection):
                                   models.TraitType.desc == trait_name]
 
                     for key, value in trait_filter.iteritems():
-                        if key == 't_string':
+                        if key == 'string':
                             conditions.append(models.Trait.t_string == value)
-                        elif key == 't_int':
+                        elif key == 'integer':
                             conditions.append(models.Trait.t_int == value)
-                        elif key == 't_datetime':
+                        elif key == 'datetime':
                             conditions.append(models.Trait.t_datetime == value)
-                        elif key == 't_float':
+                        elif key == 'float':
                             conditions.append(models.Trait.t_float == value)
 
                     trait_query = session.query(models.Trait.event_id)\
@@ -1067,6 +1068,7 @@ class Connection(base.Connection):
         """
         session = sqlalchemy_session.get_session()
 
+        LOG.debug(_("Get traits for %s") % event_type)
         with session.begin():
             query = (session.query(models.TraitType.desc,
                                    models.TraitType.data_type)

ceilometer/storage/models.py

@@ -91,6 +91,14 @@ class Trait(Model):
     FLOAT_TYPE = 3
     DATETIME_TYPE = 4
 
+    type_names = {
+        NONE_TYPE: "none",
+        TEXT_TYPE: "string",
+        INT_TYPE: "integer",
+        FLOAT_TYPE: "float",
+        DATETIME_TYPE: "datetime"
+    }
+
     def __init__(self, name, dtype, value):
         if not dtype:
             dtype = Trait.NONE_TYPE
@@ -99,10 +107,21 @@ class Trait(Model):
     def __repr__(self):
         return "<Trait: %s %d %s>" % (self.name, self.dtype, self.value)
 
+    def get_type_name(self):
+        return self.get_name_by_type(self.dtype)
+
     @classmethod
     def get_type_by_name(cls, type_name):
         return getattr(cls, '%s_TYPE' % type_name.upper(), None)
 
+    @classmethod
+    def get_type_names(cls):
+        return cls.type_names.values()
+
+    @classmethod
+    def get_name_by_type(cls, type_id):
+        return cls.type_names.get(type_id, "none")
+
     @classmethod
     def convert_value(cls, trait_type, value):
         if trait_type is cls.INT_TYPE:

ceilometer/tests/api/v2/test_acl_scenarios.py

@@ -23,6 +23,7 @@ import json
 import testscenarios
 
 from ceilometer.api import acl
+from ceilometer.api.controllers import v2 as v2_api
 from ceilometer.openstack.common import timeutils
 from ceilometer.publisher import rpc
 from ceilometer import sample
@@ -205,3 +206,25 @@ class TestAPIACL(FunctionalTest,
                                  'value': 'project-naughty',
                                  }])
         self.assertEqual(data.status_int, 401)
+
+    def test_non_admin_get_events(self):
+
+        # NOTE(herndon): wsme does not handle the  error that is being
+        # raised in by requires_admin dues to the decorator ordering. wsme
+        # does not play nice with other decorators, and so requires_admin
+        # must call wsme.wsexpose, and not the other way arou. The
+        # implication is that I can't look at the status code in the
+        # return value. Work around is to catch the exception here and
+        # verify that the status code is correct.
+
+        try:
+            # Intentionally *not* using assertRaises here so I can look
+            # at the status code of the exception.
+            self.get_json('/event_types', expect_errors=True,
+                          headers={"X-Roles": "Member",
+                                   "X-Auth-Token": VALID_TOKEN2,
+                                   "X-Project-Id": "project-good"})
+        except v2_api.ClientSideError as ex:
+            self.assertEqual(403, ex.code)
+        else:
+            self.fail()

ceilometer/tests/api/v2/test_event_scenarios.py

@@ -0,0 +1,216 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Test event, event_type and trait retrieval."""
+
+import datetime
+import testscenarios
+
+
+from ceilometer.openstack.common import timeutils
+from ceilometer.storage import models
+from ceilometer.tests.api.v2 import FunctionalTest
+from ceilometer.tests import db as tests_db
+
+load_tests = testscenarios.load_tests_apply_scenarios
+headers = {"X-Roles": "admin"}
+
+
+class EventTestBase(FunctionalTest,
+                    tests_db.MixinTestsWithBackendScenarios):
+
+    def setUp(self):
+        super(EventTestBase, self).setUp()
+        self._generate_models()
+
+    def _generate_models(self):
+        event_models = []
+        base = 0
+        self.trait_time = datetime.datetime(2013, 12, 31, 5, 0)
+        for event_type in ['Foo', 'Bar', 'Zoo']:
+            trait_models = \
+                [models.Trait(name, type, value)
+                    for name, type, value in [
+                        ('trait_A', models.Trait.TEXT_TYPE,
+                            "my_%s_text" % event_type),
+                        ('trait_B', models.Trait.INT_TYPE,
+                            base + 1),
+                        ('trait_C', models.Trait.FLOAT_TYPE,
+                            float(base) + 0.123456),
+                        ('trait_D', models.Trait.DATETIME_TYPE,
+                            self.trait_time)]]
+
+            # Message ID for test will be 'base'. So, message ID for the first
+            # event will be '0', the second '100', and so on.
+            event_models.append(
+                models.Event(message_id=str(base),
+                             event_type=event_type,
+                             generated=self.trait_time,
+                             traits=trait_models))
+            base += 100
+
+        self.conn.record_events(event_models)
+
+
+class TestEventTypeAPI(EventTestBase):
+
+    PATH = '/event_types'
+
+    def test_event_types(self):
+        data = self.get_json(self.PATH, headers=headers)
+        for event_type in ['Foo', 'Bar', 'Zoo']:
+            self.assertTrue(event_type in data)
+
+
+class TestTraitAPI(EventTestBase):
+
+    PATH = '/event_types/%s/traits'
+
+    def test_get_traits_for_event(self):
+        path = self.PATH % "Foo"
+        data = self.get_json(path, headers=headers)
+
+        self.assertEqual(4, len(data))
+
+    def test_get_traits_for_non_existent_event(self):
+        path = self.PATH % "NO_SUCH_EVENT_TYPE"
+        data = self.get_json(path, headers=headers)
+
+        self.assertEqual(data, [])
+
+    def test_get_trait_data_for_event(self):
+        path = (self.PATH % "Foo") + "/trait_A"
+        data = self.get_json(path, headers=headers)
+
+        self.assertEqual(len(data), 1)
+
+        trait = data[0]
+        self.assertEqual(trait['name'], "trait_A")
+
+    def test_get_trait_data_for_non_existent_event(self):
+        path = (self.PATH % "NO_SUCH_EVENT") + "/trait_A"
+        data = self.get_json(path, headers=headers)
+
+        self.assertEqual(data, [])
+
+    def test_get_trait_data_for_non_existent_trait(self):
+        path = (self.PATH % "Foo") + "/no_such_trait"
+        data = self.get_json(path, headers=headers)
+
+        self.assertEqual(data, [])
+
+
+class TestEventAPI(EventTestBase):
+
+    PATH = '/events'
+
+    def test_get_events(self):
+        data = self.get_json(self.PATH, headers=headers)
+        self.assertEqual(len(data), 3)
+        # We expect to get native UTC generated time back
+        expected_generated = timeutils.strtime(
+            at=timeutils.normalize_time(self.trait_time),
+            fmt=timeutils._ISO8601_TIME_FORMAT)
+        for event in data:
+            self.assertTrue(event['event_type'] in ['Foo', 'Bar', 'Zoo'])
+            self.assertEqual(4, len(event['traits']))
+            self.assertEqual(event['generated'], expected_generated)
+            for trait_name in ['trait_A', 'trait_B',
+                               'trait_C', 'trait_D']:
+                self.assertTrue(trait_name in event['traits'])
+
+    def test_get_event_by_message_id(self):
+        event = self.get_json(self.PATH + "/100", headers=headers)
+        expected_traits = {'trait_D': '2013-12-31T05:00:00',
+                           'trait_B': '101',
+                           'trait_C': '100.123456',
+                           'trait_A': 'my_Bar_text'}
+        self.assertEqual('100', event['message_id'])
+        self.assertEqual('Bar', event['event_type'])
+        self.assertEqual('2013-12-31T05:00:00', event['generated'])
+        self.assertEqual(expected_traits, event['traits'])
+
+    def test_get_event_by_message_id_no_such_id(self):
+        data = self.get_json(self.PATH + "/DNE", headers=headers,
+                             expect_errors=True)
+        self.assertEqual(404, data.status_int)
+
+    def test_get_events_filter_event_type(self):
+        data = self.get_json(self.PATH, headers=headers,
+                             q=[{'field': 'event_type',
+                                 'value': 'Foo'}])
+        self.assertEqual(1, len(data))
+
+    def test_get_events_filter_text_trait(self):
+        data = self.get_json(self.PATH, headers=headers,
+                             q=[{'field': 'trait_A',
+                                 'value': 'my_Foo_text',
+                                 'type': 'string'}])
+        self.assertEqual(1, len(data))
+        self.assertEqual(data[0]['event_type'], 'Foo')
+
+    def test_get_events_filter_int_trait(self):
+        data = self.get_json(self.PATH, headers=headers,
+                             q=[{'field': 'trait_B',
+                                 'value': '101',
+                                 'type': 'integer'}])
+        self.assertEqual(1, len(data))
+        self.assertEqual(data[0]['event_type'], 'Bar')
+        self.assertEqual(int(data[0]['traits']['trait_B']), 101)
+
+    def test_get_events_filter_float_trait(self):
+        data = self.get_json(self.PATH, headers=headers,
+                             q=[{'field': 'trait_C',
+                                 'value': '200.123456',
+                                 'type': 'float'}])
+        self.assertEqual(1, len(data))
+        self.assertEqual(data[0]['event_type'], 'Zoo')
+        self.assertEqual(float(data[0]['traits']['trait_C']), 200.123456)
+
+    def test_get_events_filter_datetime_trait(self):
+        data = self.get_json(self.PATH, headers=headers,
+                             q=[{'field': 'trait_D',
+                                 'value': self.trait_time.isoformat(),
+                                 'type': 'datetime'}])
+        self.assertEqual(3, len(data))
+        self.assertEqual(data[0]['traits']['trait_D'],
+                         self.trait_time.isoformat())
+
+    def test_get_events_multiple_filters(self):
+        data = self.get_json(self.PATH, headers=headers,
+                             q=[{'field': 'trait_B',
+                                 'value': '1',
+                                 'type': 'integer'},
+                                {'field': 'trait_A',
+                                 'value': 'my_Foo_text',
+                                 'type': 'string'}])
+        self.assertEqual(1, len(data))
+        self.assertEqual(data[0]['event_type'], 'Foo')
+
+    def test_get_events_multiple_filters_no_matches(self):
+        data = self.get_json(self.PATH, headers=headers,
+                             q=[{'field': 'trait_B',
+                                 'value': '101',
+                                 'type': 'integer'},
+                                {'field': 'trait_A',
+                                 'value': 'my_Foo_text',
+                                 'type': 'string'}])
+
+        self.assertEqual(0, len(data))
+
+    def test_get_events_not_filters(self):
+        data = self.get_json(self.PATH, headers=headers,
+                             q=[])
+        self.assertEqual(3, len(data))

ceilometer/tests/storage/test_storage_scenarios.py

@@ -2234,7 +2234,7 @@ class GetEventTest(EventTestBase):
                 self.assertEqual(expected_val, trait.value)
 
     def test_get_event_trait_filter(self):
-        trait_filters = [{'key': 'trait_B', 't_int': 101}]
+        trait_filters = [{'key': 'trait_B', 'integer': 101}]
         event_filter = storage.EventFilter(self.start, self.end,
                                            traits_filter=trait_filters)
         events = self.conn.get_events(event_filter)
@@ -2243,8 +2243,8 @@ class GetEventTest(EventTestBase):
         self.assertEqual(4, len(events[0].traits))
 
     def test_get_event_multiple_trait_filter(self):
-        trait_filters = [{'key': 'trait_B', 't_int': 1},
+        trait_filters = [{'key': 'trait_B', 'integer': 1},
-                         {'key': 'trait_A', 't_string': 'my_Foo_text'}]
+                         {'key': 'trait_A', 'string': 'my_Foo_text'}]
         event_filter = storage.EventFilter(self.start, self.end,
                                            traits_filter=trait_filters)
         events = self.conn.get_events(event_filter)
@@ -2253,8 +2253,8 @@ class GetEventTest(EventTestBase):
         self.assertEqual(4, len(events[0].traits))
 
     def test_get_event_multiple_trait_filter_expect_none(self):
-        trait_filters = [{'key': 'trait_B', 't_int': 1},
+        trait_filters = [{'key': 'trait_B', 'integer': 1},
-                         {'key': 'trait_A', 't_string': 'my_Zoo_text'}]
+                         {'key': 'trait_A', 'string': 'my_Zoo_text'}]
         event_filter = storage.EventFilter(self.start, self.end,
                                            traits_filter=trait_filters)
         events = self.conn.get_events(event_filter)