Merge "Add configuration-driven conversion to Events"

bin/ceilometer-test-event.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2013 Rackspace Hosting.
+#
+# Author: Monsyne Dragon <mdragon@rackspace.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Command line tool help you debug your event definitions.
+
+Feed it a list of test notifications in json format, and it will show
+you what events will be generated.
+"""
+
+import json
+import sys
+
+from oslo.config import cfg
+from stevedore import extension
+
+from ceilometer.event import converter
+from ceilometer import service
+
+
+cfg.CONF.register_cli_opts([
+    cfg.StrOpt('input-file',
+               short='i',
+               help='File to read test notifications from.'
+               ' (Containing a json list of notifications.)'
+               ' defaults to stdin.'),
+    cfg.StrOpt('output-file',
+               short='o',
+               help='File to write results to. Defaults to stdout'),
+])
+
+TYPES = {1: 'text',
+         2: 'int',
+         3: 'float',
+         4: 'datetime'}
+
+
+service.prepare_service()
+
+config_file = converter.get_config_file()
+output_file = cfg.CONF.output_file
+input_file = cfg.CONF.input_file
+
+if output_file is None:
+    out = sys.stdout
+else:
+    out = open(output_file, 'w')
+
+if input_file is None:
+    notifications = json.load(sys.stdin)
+else:
+    with open(input_file, 'r') as f:
+        notifications = json.load(f)
+
+out.write("Definitions file: %s\n" % config_file)
+out.write("Notifications tested: %s\n" % len(notifications))
+
+event_converter = converter.setup_events(
+    extension.ExtensionManager(
+        namespace='ceilometer.event.trait_plugin'))
+
+for notification in notifications:
+    event = event_converter.to_event(notification)
+    if event is None:
+        out.write("Dropped notification: %s\n" %
+                  notification['message_id'])
+        continue
+    out.write("Event: %s at %s\n" % (event.event_name, event.generated))
+    for trait in event.traits:
+        dtype = TYPES[trait.dtype]
+        out.write("    Trait: name: %s, type: %s, value: %s\n" % (
+            trait.name, dtype, trait.value))

ceilometer/event/converter.py

@@ -0,0 +1,397 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2013 Rackspace Hosting.
+#
+# Author: Monsyne Dragon <mdragon@rackspace.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import fnmatch
+import os
+
+import jsonpath_rw
+from oslo.config import cfg
+import six
+import yaml
+
+from ceilometer.openstack.common import log
+from ceilometer.openstack.common import timeutils
+from ceilometer.storage import models
+
+OPTS = [
+    cfg.StrOpt('definitions_cfg_file',
+               default="event_definitions.yaml",
+               help="Configuration file for event definitions"
+               ),
+    cfg.BoolOpt('drop_unmatched_notifications',
+                default=False,
+                help='Drop notifications if no event definition matches. '
+                '(Otherwise, we convert them with just the default traits)'),
+
+]
+
+cfg.CONF.register_opts(OPTS, group='event')
+
+LOG = log.getLogger(__name__)
+
+
+class EventDefinitionException(Exception):
+    def __init__(self, message, definition_cfg):
+        super(EventDefinitionException, self).__init__(message)
+        self.definition_cfg = definition_cfg
+
+    def __str__(self):
+        return '%s %s: %s' % (self.__class__.__name__,
+                              self.definition_cfg, self.message)
+
+
+class TraitDefinition(object):
+
+    def __init__(self, name, trait_cfg, plugin_manager):
+        self.cfg = trait_cfg
+        self.name = name
+
+        type_name = trait_cfg.get('type', 'text')
+
+        if 'plugin' in trait_cfg:
+            plugin_cfg = trait_cfg['plugin']
+            if isinstance(plugin_cfg, six.string_types):
+                plugin_name = plugin_cfg
+                plugin_params = {}
+            else:
+                try:
+                    plugin_name = plugin_cfg['name']
+                except KeyError:
+                    raise EventDefinitionException(
+                        _('Plugin specified, but no plugin name supplied for '
+                          'trait %s') % name, self.cfg)
+                plugin_params = plugin_cfg.get('parameters')
+                if plugin_params is None:
+                    plugin_params = {}
+            try:
+                plugin_ext = plugin_manager[plugin_name]
+            except KeyError:
+                raise EventDefinitionException(
+                    _('No plugin named %(plugin)s available for '
+                      'trait %(trait)s') % dict(plugin=plugin_name,
+                                                trait=name), self.cfg)
+            plugin_class = plugin_ext.plugin
+            self.plugin = plugin_class(**plugin_params)
+        else:
+            self.plugin = None
+
+        if 'fields' not in trait_cfg:
+            raise EventDefinitionException(
+                _("Required field in trait definition not specified: "
+                  "'%s'") % 'fields',
+                self.cfg)
+
+        fields = trait_cfg['fields']
+        if not isinstance(fields, six.string_types):
+            # NOTE(mdragon): if not a string, we assume a list.
+            if len(fields) == 1:
+                fields = fields[0]
+            else:
+                fields = '|'.join('(%s)' % path for path in fields)
+        try:
+            self.fields = jsonpath_rw.parse(fields)
+        except Exception as e:
+            raise EventDefinitionException(
+                _("Parse error in JSONPath specification "
+                  "'%(jsonpath)s' for %(trait)s: %(err)s")
+                % dict(jsonpath=fields, trait=name, err=e), self.cfg)
+        self.trait_type = models.Trait.get_type_by_name(type_name)
+        if self.trait_type is None:
+            raise EventDefinitionException(
+                _("Invalid trait type '%(type)s' for trait %(trait)s")
+                % dict(type=type_name, trait=name), self.cfg)
+
+    def _get_path(self, match):
+        if match.context is not None:
+            for path_element in self._get_path(match.context):
+                yield path_element
+            yield str(match.path)
+
+    def to_trait(self, notification_body):
+        values = [match for match in self.fields.find(notification_body)
+                  if match.value is not None]
+
+        if self.plugin is not None:
+            value_map = [('.'.join(self._get_path(match)), match.value) for
+                         match in values]
+            value = self.plugin.trait_value(value_map)
+        else:
+            value = values[0].value if values else None
+
+        if value is None:
+            return None
+
+        # NOTE(mdragon): some openstack projects (mostly Nova) emit ''
+        # for null fields for things like dates.
+        if self.trait_type != models.Trait.TEXT_TYPE and value == '':
+            return None
+
+        value = models.Trait.convert_value(self.trait_type, value)
+        return models.Trait(self.name, self.trait_type, value)
+
+
+class EventDefinition(object):
+
+    DEFAULT_TRAITS = dict(
+        service=dict(type='text', fields='publisher_id'),
+        request_id=dict(type='text', fields='_context_request_id'),
+        tenant_id=dict(type='text', fields=['payload.tenant_id',
+                                            '_context_tenant']),
+    )
+
+    def __init__(self, definition_cfg, trait_plugin_mgr):
+        self._included_types = []
+        self._excluded_types = []
+        self.traits = dict()
+        self.cfg = definition_cfg
+
+        try:
+            event_type = definition_cfg['event_type']
+            traits = definition_cfg['traits']
+        except KeyError as err:
+            raise EventDefinitionException(
+                _("Required field %s not specified") % err.args[0], self.cfg)
+
+        if isinstance(event_type, six.string_types):
+            event_type = [event_type]
+
+        for t in event_type:
+            if t.startswith('!'):
+                self._excluded_types.append(t[1:])
+            else:
+                self._included_types.append(t)
+
+        if self._excluded_types and not self._included_types:
+            self._included_types.append('*')
+
+        for trait_name in self.DEFAULT_TRAITS:
+            self.traits[trait_name] = TraitDefinition(
+                trait_name,
+                self.DEFAULT_TRAITS[trait_name],
+                trait_plugin_mgr)
+        for trait_name in traits:
+            self.traits[trait_name] = TraitDefinition(
+                trait_name,
+                traits[trait_name],
+                trait_plugin_mgr)
+
+    def included_type(self, event_type):
+        for t in self._included_types:
+            if fnmatch.fnmatch(event_type, t):
+                return True
+        return False
+
+    def excluded_type(self, event_type):
+        for t in self._excluded_types:
+            if fnmatch.fnmatch(event_type, t):
+                return True
+        return False
+
+    def match_type(self, event_type):
+        return (self.included_type(event_type)
+                and not self.excluded_type(event_type))
+
+    @property
+    def is_catchall(self):
+        return '*' in self._included_types and not self._excluded_types
+
+    @staticmethod
+    def _extract_when(body):
+        """Extract the generated datetime from the notification.
+        """
+        # NOTE: I am keeping the logic the same as it was in the collector,
+        # However, *ALL* notifications should have a 'timestamp' field, it's
+        # part of the notification envelope spec. If this was put here because
+        # some openstack project is generating notifications without a
+        # timestamp, then that needs to be filed as a bug with the offending
+        # project (mdragon)
+        when = body.get('timestamp', body.get('_context_timestamp'))
+        if when:
+            return timeutils.normalize_time(timeutils.parse_isotime(when))
+
+        return timeutils.utcnow()
+
+    def to_event(self, notification_body):
+        event_type = notification_body['event_type']
+        message_id = notification_body['message_id']
+        when = self._extract_when(notification_body)
+
+        traits = (self.traits[t].to_trait(notification_body)
+                  for t in self.traits)
+        # Only accept non-None value traits ...
+        traits = [trait for trait in traits if trait is not None]
+        event = models.Event(message_id, event_type, when, traits)
+        return event
+
+
+class NotificationEventsConverter(object):
+    """Notification Event Converter
+
+    The NotificationEventsConverter handles the conversion of Notifications
+    from openstack systems into Ceilometer Events.
+
+    The conversion is handled according to event definitions in a config file.
+
+    The config is a list of event definitions. Order is significant, a
+    notification will be processed according to the LAST definition that
+    matches it's event_type. (We use the last matching definition because that
+    allows you to use YAML merge syntax in the definitions file.)
+    Each definition is a dictionary with the following keys (all are required):
+        event_type: this is a list of notification event_types this definition
+                    will handle. These can be wildcarded with unix shell glob
+                    (not regex!) wildcards.
+                    An exclusion listing (starting with a '!') will exclude any
+                    types listed from matching. If ONLY exclusions are listed,
+                    the definition will match anything not matching the
+                    exclusions.
+                    This item can also be a string, which will be taken as
+                    equivalent to 1 item list.
+
+                    Examples:
+                    *   ['compute.instance.exists'] will only match
+                            compute.intance.exists notifications
+                    *   "compute.instance.exists"   Same as above.
+                    *   ["image.create", "image.delete"]  will match
+                         image.create and image.delete, but not anything else.
+                    *   'compute.instance.*" will match
+                        compute.instance.create.start but not image.upload
+                    *   ['*.start','*.end', '!scheduler.*'] will match
+                        compute.instance.create.start, and image.delete.end,
+                        but NOT compute.instance.exists or
+                        scheduler.run_instance.start
+                    *   '!image.*' matches any notification except image
+                        notifications.
+                    *   ['*', '!image.*']  same as above.
+        traits:  dictionary, The keys are trait names, the values are the trait
+                 definitions
+            Each trait definiton is a dictionary with the following keys:
+                type (optional): The data type for this trait. (as a string)
+                    Valid options are: 'text', 'int', 'float' and 'datetime'
+                    defaults to 'text' if not specified.
+                fields:  a path specification for the field(s) in the
+                    notification you wish to extract. The paths can be
+                    specified with a dot syntax (e.g. 'payload.host').
+                    dictionary syntax (e.g. 'payload[host]') is also supported.
+                    in either case, if the key for the field you are looking
+                    for contains special charecters, like '.', it will need to
+                    be quoted (with double or single quotes) like so:
+
+                          "payload.image_meta.'org.openstack__1__architecture'"
+
+                    The syntax used for the field specification is a variant
+                    of JSONPath, and is fairly flexible.
+                    (see: https://github.com/kennknowles/python-jsonpath-rw
+                    for more info)  Specifications can be written to match
+                    multiple possible fields, the value for the trait will
+                    be derived from the matching fields that exist and have
+                    a non-null (i.e. is not None) values in the notification.
+                    By default the value will be the first such field.
+                    (plugins can alter that, if they wish)
+
+                    This configuration value is normally a string, for
+                    convenience, it can be specified as a list of
+                    specifications, which will be OR'ed together (a union
+                    query in jsonpath terms)
+                plugin (optional): (dictionary) with the following keys:
+                    name: (string) name of a plugin to load
+                    parameters: (optional) Dictionary of keyword args to pass
+                                to the plugin on initialization.
+                                See documentation on each plugin to see what
+                                arguments it accepts.
+                    For convenience, this value can also be specified as a
+                    string, which is interpreted as a plugin name, which will
+                    be loaded with no parameters.
+
+    """
+
+    def __init__(self, events_config, trait_plugin_mgr, add_catchall=True):
+        self.definitions = [
+            EventDefinition(event_def, trait_plugin_mgr)
+            for event_def in reversed(events_config)]
+        if add_catchall and not any(d.is_catchall for d in self.definitions):
+            event_def = dict(event_type='*', traits={})
+            self.definitions.append(EventDefinition(event_def,
+                                                    trait_plugin_mgr))
+
+    def to_event(self, notification_body):
+        event_type = notification_body['event_type']
+        message_id = notification_body['message_id']
+        edef = None
+        for d in self.definitions:
+            if d.match_type(event_type):
+                edef = d
+                break
+
+        if edef is None:
+            msg = (_('Dropping Notification %(type)s (uuid:%(msgid)s)')
+                   % dict(type=event_type, msgid=message_id))
+            if cfg.CONF.event.drop_unmatched_notifications:
+                LOG.debug(msg)
+            else:
+                # If drop_unmatched_notifications is False, this should
+                # never happen. (mdragon)
+                LOG.error(msg)
+            return None
+
+        return edef.to_event(notification_body)
+
+
+def get_config_file():
+    config_file = cfg.CONF.event.definitions_cfg_file
+    if not os.path.exists(config_file):
+        config_file = cfg.CONF.find_file(config_file)
+    return config_file
+
+
+def setup_events(trait_plugin_mgr):
+    """Setup the event definitions from yaml config file."""
+    config_file = get_config_file()
+    if config_file is not None:
+        LOG.debug(_("Event Definitions configuration file: %s"), config_file)
+
+        with open(config_file) as cf:
+            config = cf.read()
+
+        try:
+            events_config = yaml.safe_load(config)
+        except yaml.YAMLError as err:
+            if hasattr(err, 'problem_mark'):
+                mark = err.problem_mark
+                errmsg = (_("Invalid YAML syntax in Event Definitions file "
+                            "%(file)s at line: %(line)s, column: %(column)s.")
+                          % dict(file=config_file,
+                                 line=mark.line+1,
+                                 column=mark.column+1))
+            else:
+                errmsg = (_("YAML error reading Event Definitions file "
+                            "%(file)s")
+                          % dict(file=config_file))
+            LOG.error(errmsg)
+            raise
+
+    else:
+        LOG.debug(_("No Event Definitions configuration file found!"
+                  " Using default config."))
+        events_config = []
+
+    LOG.info(_("Event Definitions: %s"), events_config)
+
+    allow_drop = cfg.CONF.event.drop_unmatched_notifications
+    return NotificationEventsConverter(events_config,
+                                       trait_plugin_mgr,
+                                       add_catchall=not allow_drop)

ceilometer/event/trait_plugins.py

@@ -0,0 +1,158 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2013 Rackspace Hosting.
+#
+# Author: Monsyne Dragon <mdragon@rackspace.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class TraitPluginBase(object):
+    """Base class for plugins that convert notification fields to
+       Trait values.
+    """
+
+    def __init__(self, **kw):
+        """Setup the trait plugin.
+
+        For each Trait definition a plugin is used on in a conversion
+        definition, a new instance of the plugin will be created, and
+        initialized with the parameters (if any) specified in the
+        config file.
+
+        :param kw: the parameters specified in the event definitions file.
+
+        """
+        super(TraitPluginBase, self).__init__()
+
+    @abc.abstractmethod
+    def trait_value(self, match_list):
+        """Convert a set of fields to a Trait value.
+
+        This method is called each time a trait is attempted to be extracted
+        from a notification. It will be called *even if* no matching fields
+        are found in the notification (in that case, the match_list will be
+        empty). If this method returns None, the trait *will not* be added to
+        the event. Any other value returned by this method will be used as
+        the value for the trait. Values returned will be coerced to the
+        appropriate type for the trait.
+
+        :param match_list: A list (may be empty if no matches) of *tuples*.
+                          Each tuple is (field_path, value) where field_path
+                          is the jsonpath for that specific field,
+
+        Example:
+            trait's fields definition: ['payload.foobar',
+                                        'payload.baz',
+                                        'payload.thing.*']
+            notification body:
+                        {
+                            'message_id': '12345',
+                            'publisher':    'someservice.host',
+                            'payload':  {
+                                            'foobar': 'test',
+                                            'thing':    {
+                                                            'bar': 12,
+                                                            'boing': 13,
+                                                        }
+                                        }
+                        }
+            match_list will be: [('payload.foobar','test'),
+                                 ('payload.thing.bar',12),
+                                 ('payload.thing.boing',13)]
+
+        Here is a plugin that emulates the default (no plugin) behavior:
+
+        class DefaultPlugin(TraitPluginBase):
+            "Plugin that returns the first field value"
+
+            def __init__(self, **kw):
+                super(DefaultPlugin, self).__init__()
+
+            def trait_value(self, match_list):
+                if not match_list:
+                    return None
+                return match_list[0][1]
+        """
+
+
+class SplitterTraitPlugin(TraitPluginBase):
+    """Plugin that splits a piece off of a string value."""
+
+    def __init__(self, separator=".", segment=0, max_split=None, **kw):
+        """Setup how do split the field.
+
+        :param  separator:  String to split on. default "."
+        :param  segment:    Which segment to return. (int) default 0
+        :param  max_split: Limit number of splits. Default: None (no limit)
+        """
+        self.separator = separator
+        self.segment = segment
+        self.max_split = max_split
+        super(SplitterTraitPlugin, self).__init__(**kw)
+
+    def trait_value(self, match_list):
+        if not match_list:
+            return None
+        value = str(match_list[0][1])
+        if self.max_split is not None:
+            values = value.split(self.separator, self.max_split)
+        else:
+            values = value.split(self.separator)
+        try:
+            return values[self.segment]
+        except IndexError:
+            return None
+
+
+class BitfieldTraitPlugin(TraitPluginBase):
+    """Plugin to set flags on a bitfield."""
+    def __init__(self, initial_bitfield=0, flags=None, **kw):
+        """Setup bitfield trait.
+
+        :param initial_bitfield: (int) initial value for the bitfield
+                                 Flags that are set will be OR'ed with this.
+        :param flags: List of dictionaries defining bitflags to set depending
+                      on data in the notification. Each one has the following
+                      keys:
+                            path: jsonpath of field to match.
+                            bit: (int) number of bit to set (lsb is bit 0)
+                            value: set bit if corrosponding field's value
+                                   matches this. If value is not provided,
+                                   bit will be set if the field exists (and
+                                   is non-null), regardless of it's value.
+
+        """
+        self.initial_bitfield = initial_bitfield
+        if flags is None:
+            flags = []
+        self.flags = flags
+        super(BitfieldTraitPlugin, self).__init__(**kw)
+
+    def trait_value(self, match_list):
+        matches = dict(match_list)
+        bitfield = self.initial_bitfield
+        for flagdef in self.flags:
+            path = flagdef['path']
+            bit = 2 ** int(flagdef['bit'])
+            if path in matches:
+                if 'value' in flagdef:
+                    if matches[path] == flagdef['value']:
+                        bitfield |= bit
+                else:
+                    bitfield |= bit
+        return bitfield

ceilometer/notification.py

@@ -19,12 +19,12 @@
 from oslo.config import cfg
 from stevedore import extension
 
+from ceilometer.event import converter as event_converter
 from ceilometer.openstack.common import context
 from ceilometer.openstack.common.gettextutils import _  # noqa
 from ceilometer.openstack.common import log
 from ceilometer.openstack.common.rpc import service as rpc_service
 from ceilometer.openstack.common import service as os_service
-from ceilometer.openstack.common import timeutils
 from ceilometer import pipeline
 from ceilometer import service
 from ceilometer.storage import models
@@ -73,6 +73,11 @@ class NotificationService(service.DispatchedService, rpc_service.Service):
             ),
         )
 
+        LOG.debug('loading event definitions')
+        self.event_converter = event_converter.setup_events(
+            extension.ExtensionManager(
+                namespace='ceilometer.event.trait_plugin'))
+
         self.notification_manager = \
             extension.ExtensionManager(
                 namespace=self.NOTIFICATION_NAMESPACE,
@@ -133,52 +138,24 @@ class NotificationService(service.DispatchedService, rpc_service.Service):
         if cfg.CONF.notification.store_events:
             self._message_to_event(notification)
 
-    @staticmethod
-    def _extract_when(body):
-        """Extract the generated datetime from the notification.
-        """
-        when = body.get('timestamp', body.get('_context_timestamp'))
-        if when:
-            return timeutils.normalize_time(timeutils.parse_isotime(when))
-
-        return timeutils.utcnow()
-
     def _message_to_event(self, body):
         """Convert message to Ceilometer Event.
 
-        NOTE: this is currently based on the Nova notification format.
-        We will need to make this driver-based to support other formats.
-
         NOTE: the rpc layer currently rips out the notification
         delivery_info, which is critical to determining the
         source of the notification. This will have to get added back later.
         """
-        message_id = body.get('message_id')
+        event = self.event_converter.to_event(body)
-        event_type = body['event_type']
-        when = self._extract_when(body)
-        LOG.debug(_('Saving event "%s"'), event_type)
 
-        publisher = body.get('publisher_id')
+        if event is not None:
-        request_id = body.get('_context_request_id')
+            LOG.debug('Saving event "%s"', event.event_type)
-        tenant_id = body.get('_context_tenant')
+            problem_events = []
-
+            for dispatcher in self.dispatcher_manager:
-        text = models.Trait.TEXT_TYPE
+                problem_events.extend(dispatcher.obj.record_events(event))
-        all_traits = [models.Trait('service', text, publisher),
+            if models.Event.UNKNOWN_PROBLEM in [x[0] for x in problem_events]:
-                      models.Trait('request_id', text, request_id),
+                # Don't ack the message, raise to requeue it
-                      models.Trait('tenant_id', text, tenant_id),
+                # if ack_on_error = False
-                      ]
+                raise UnableToSaveEventException()
-        # Only store non-None value traits ...
-        traits = [trait for trait in all_traits if trait.value is not None]
-
-        event = models.Event(message_id, event_type, when, traits)
-
-        problem_events = []
-        for dispatcher in self.dispatcher_manager:
-            problem_events.extend(dispatcher.obj.record_events(event))
-        if models.Event.UNKNOWN_PROBLEM in [x[0] for x in problem_events]:
-            # Don't ack the message, raise to requeue it
-            # if ack_on_error = False
-            raise UnableToSaveEventException()
 
     def _process_notification_for_ext(self, ext, notification):
         """Wrapper for calling pipelines when a notification arrives

ceilometer/storage/models.py

@@ -18,6 +18,8 @@
 """Model classes for use in the storage API.
 """
 
+from ceilometer.openstack.common import timeutils
+
 
 class Model(object):
     """Base class for storage API models.
@@ -97,6 +99,20 @@ class Trait(Model):
     def __repr__(self):
         return "<Trait: %s %d %s>" % (self.name, self.dtype, self.value)
 
+    @classmethod
+    def get_type_by_name(cls, type_name):
+        return getattr(cls, '%s_TYPE' % type_name.upper(), None)
+
+    @classmethod
+    def convert_value(cls, trait_type, value):
+        if trait_type is cls.INT_TYPE:
+            return int(value)
+        if trait_type is cls.FLOAT_TYPE:
+            return float(value)
+        if trait_type is cls.DATETIME_TYPE:
+            return timeutils.normalize_time(timeutils.parse_isotime(value))
+        return str(value)
+
 
 class Resource(Model):
     """Something for which sample data has been collected.

ceilometer/tests/event/test_converter.py

@@ -0,0 +1,734 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2013 Rackspace Hosting.
+#
+# Author: Monsyne Dragon <mdragon@rackspace.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import datetime
+
+import jsonpath_rw
+import mock
+from oslo.config import cfg as oslo_cfg
+import six
+
+from ceilometer.event import converter
+from ceilometer.openstack.common import timeutils
+from ceilometer.storage import models
+from ceilometer.tests import base
+
+
+class ConverterBase(base.BaseTestCase):
+    def _create_test_notification(self, event_type, message_id, **kw):
+        return dict(event_type=event_type,
+                    message_id=message_id,
+                    priority="INFO",
+                    publisher_id="compute.host-1-2-3",
+                    timestamp="2013-08-08 21:06:37.803826",
+                    payload=kw,
+                    )
+
+    def assertIsValidEvent(self, event, notification):
+        self.assertIsNot(
+            None, event,
+            "Notification dropped unexpectedly:"
+            " %s" % str(notification))
+        self.assertIsInstance(event, models.Event)
+
+    def assertIsNotValidEvent(self, event, notification):
+        self.assertIs(
+            None, event,
+            "Notification NOT dropped when expected to be dropped:"
+            " %s" % str(notification))
+
+    def assertHasTrait(self, event, name, value=None, dtype=None):
+        traits = [trait for trait in event.traits if trait.name == name]
+        self.assertTrue(
+            len(traits) > 0,
+            "Trait %s not found in event %s" % (name, event))
+        trait = traits[0]
+        if value is not None:
+            self.assertEqual(trait.value, value)
+        if dtype is not None:
+            self.assertEqual(trait.dtype, dtype)
+            if dtype == models.Trait.INT_TYPE:
+                self.assertIsInstance(trait.value, int)
+            elif dtype == models.Trait.FLOAT_TYPE:
+                self.assertIsInstance(trait.value, float)
+            elif dtype == models.Trait.DATETIME_TYPE:
+                self.assertIsInstance(trait.value, datetime.datetime)
+            elif dtype == models.Trait.TEXT_TYPE:
+                self.assertIsInstance(trait.value, six.string_types)
+
+    def assertDoesNotHaveTrait(self, event, name):
+        traits = [trait for trait in event.traits if trait.name == name]
+        self.assertEqual(
+            len(traits), 0,
+            "Extra Trait %s found in event %s" % (name, event))
+
+    def assertHasDefaultTraits(self, event):
+        text = models.Trait.TEXT_TYPE
+        self.assertHasTrait(event, 'service', dtype=text)
+
+    def _cmp_tree(self, this, other):
+        if hasattr(this, 'right') and hasattr(other, 'right'):
+            return (self._cmp_tree(this.right, other.right) and
+                    self._cmp_tree(this.left, other.left))
+        if not hasattr(this, 'right') and not hasattr(other, 'right'):
+            return this == other
+        return False
+
+    def assertPathsEqual(self, path1, path2):
+        self.assertTrue(self._cmp_tree(path1, path2),
+                        'JSONPaths not equivalent %s %s' % (path1, path2))
+
+
+class TestTraitDefinition(ConverterBase):
+
+    def setUp(self):
+        super(TestTraitDefinition, self).setUp()
+        self.n1 = self._create_test_notification(
+            "test.thing",
+            "uuid-for-notif-0001",
+            instance_uuid="uuid-for-instance-0001",
+            instance_id="id-for-instance-0001",
+            instance_uuid2=None,
+            instance_id2=None,
+            host='host-1-2-3',
+            bogus_date='',
+            image_meta=dict(
+                        disk_gb='20',
+                        thing='whatzit'),
+            foobar=50)
+
+        self.ext1 = mock.MagicMock(name='mock_test_plugin')
+        self.test_plugin_class = self.ext1.plugin
+        self.test_plugin = self.test_plugin_class()
+        self.test_plugin.trait_value.return_value = 'foobar'
+        self.ext1.reset_mock()
+
+        self.ext2 = mock.MagicMock(name='mock_nothing_plugin')
+        self.nothing_plugin_class = self.ext2.plugin
+        self.nothing_plugin = self.nothing_plugin_class()
+        self.nothing_plugin.trait_value.return_value = None
+        self.ext2.reset_mock()
+
+        self.fake_plugin_mgr = dict(test=self.ext1, nothing=self.ext2)
+
+    def test_to_trait_with_plugin(self):
+        cfg = dict(type='text',
+                   fields=['payload.instance_id', 'payload.instance_uuid'],
+                   plugin=dict(name='test'))
+
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIsInstance(t, models.Trait)
+        self.assertEqual(t.name, 'test_trait')
+        self.assertEqual(t.dtype, models.Trait.TEXT_TYPE)
+        self.assertEqual(t.value, 'foobar')
+        self.test_plugin_class.assert_called_once_with()
+        self.test_plugin.trait_value.assert_called_once_with([
+            ('payload.instance_id', 'id-for-instance-0001'),
+            ('payload.instance_uuid', 'uuid-for-instance-0001')])
+
+    def test_to_trait_null_match_with_plugin(self):
+        cfg = dict(type='text',
+                   fields=['payload.nothere', 'payload.bogus'],
+                   plugin=dict(name='test'))
+
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIsInstance(t, models.Trait)
+        self.assertEqual(t.name, 'test_trait')
+        self.assertEqual(t.dtype, models.Trait.TEXT_TYPE)
+        self.assertEqual(t.value, 'foobar')
+        self.test_plugin_class.assert_called_once_with()
+        self.test_plugin.trait_value.assert_called_once_with([])
+
+    def test_to_trait_with_plugin_null(self):
+        cfg = dict(type='text',
+                   fields=['payload.instance_id', 'payload.instance_uuid'],
+                   plugin=dict(name='nothing'))
+
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIs(None, t)
+        self.nothing_plugin_class.assert_called_once_with()
+        self.nothing_plugin.trait_value.assert_called_once_with([
+            ('payload.instance_id', 'id-for-instance-0001'),
+            ('payload.instance_uuid', 'uuid-for-instance-0001')])
+
+    def test_to_trait_with_plugin_with_parameters(self):
+        cfg = dict(type='text',
+                   fields=['payload.instance_id', 'payload.instance_uuid'],
+                   plugin=dict(name='test', parameters=dict(a=1, b='foo')))
+
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIsInstance(t, models.Trait)
+        self.assertEqual(t.name, 'test_trait')
+        self.assertEqual(t.dtype, models.Trait.TEXT_TYPE)
+        self.assertEqual(t.value, 'foobar')
+        self.test_plugin_class.assert_called_once_with(a=1, b='foo')
+        self.test_plugin.trait_value.assert_called_once_with([
+            ('payload.instance_id', 'id-for-instance-0001'),
+            ('payload.instance_uuid', 'uuid-for-instance-0001')])
+
+    def test_to_trait(self):
+        cfg = dict(type='text', fields='payload.instance_id')
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIsInstance(t, models.Trait)
+        self.assertEqual(t.name, 'test_trait')
+        self.assertEqual(t.dtype, models.Trait.TEXT_TYPE)
+        self.assertEqual(t.value, 'id-for-instance-0001')
+
+        cfg = dict(type='int', fields='payload.image_meta.disk_gb')
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIsInstance(t, models.Trait)
+        self.assertEqual(t.name, 'test_trait')
+        self.assertEqual(t.dtype, models.Trait.INT_TYPE)
+        self.assertEqual(t.value, 20)
+
+    def test_to_trait_multiple(self):
+        cfg = dict(type='text', fields=['payload.instance_id',
+                                        'payload.instance_uuid'])
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIsInstance(t, models.Trait)
+        self.assertEqual(t.value, 'id-for-instance-0001')
+
+        cfg = dict(type='text', fields=['payload.instance_uuid',
+                                        'payload.instance_id'])
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIsInstance(t, models.Trait)
+        self.assertEqual(t.value, 'uuid-for-instance-0001')
+
+    def test_to_trait_multiple_different_nesting(self):
+        cfg = dict(type='int', fields=['payload.foobar',
+                   'payload.image_meta.disk_gb'])
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIsInstance(t, models.Trait)
+        self.assertEqual(t.value, 50)
+
+        cfg = dict(type='int', fields=['payload.image_meta.disk_gb',
+                   'payload.foobar'])
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIsInstance(t, models.Trait)
+        self.assertEqual(t.value, 20)
+
+    def test_to_trait_some_null_multiple(self):
+        cfg = dict(type='text', fields=['payload.instance_id2',
+                                        'payload.instance_uuid'])
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIsInstance(t, models.Trait)
+        self.assertEqual(t.value, 'uuid-for-instance-0001')
+
+    def test_to_trait_some_missing_multiple(self):
+        cfg = dict(type='text', fields=['payload.not_here_boss',
+                                        'payload.instance_uuid'])
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIsInstance(t, models.Trait)
+        self.assertEqual(t.value, 'uuid-for-instance-0001')
+
+    def test_to_trait_missing(self):
+        cfg = dict(type='text', fields='payload.not_here_boss')
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIs(None, t)
+
+    def test_to_trait_null(self):
+        cfg = dict(type='text', fields='payload.instance_id2')
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIs(None, t)
+
+    def test_to_trait_empty_nontext(self):
+        cfg = dict(type='datetime', fields='payload.bogus_date')
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIs(None, t)
+
+    def test_to_trait_multiple_null_missing(self):
+        cfg = dict(type='text', fields=['payload.not_here_boss',
+                                        'payload.instance_id2'])
+        tdef = converter.TraitDefinition('test_trait', cfg,
+                                         self.fake_plugin_mgr)
+        t = tdef.to_trait(self.n1)
+        self.assertIs(None, t)
+
+    def test_missing_fields_config(self):
+        self.assertRaises(converter.EventDefinitionException,
+                          converter.TraitDefinition,
+                          'bogus_trait',
+                          dict(),
+                          self.fake_plugin_mgr)
+
+    def test_string_fields_config(self):
+        cfg = dict(fields='payload.test')
+        t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr)
+        self.assertPathsEqual(t.fields, jsonpath_rw.parse('payload.test'))
+
+    def test_list_fields_config(self):
+        cfg = dict(fields=['payload.test', 'payload.other'])
+        t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr)
+        self.assertPathsEqual(
+            t.fields,
+            jsonpath_rw.parse('(payload.test)|(payload.other)'))
+
+    def test_invalid_path_config(self):
+        #test invalid jsonpath...
+        cfg = dict(fields='payload.bogus(')
+        self.assertRaises(converter.EventDefinitionException,
+                          converter.TraitDefinition,
+                          'bogus_trait',
+                          cfg,
+                          self.fake_plugin_mgr)
+
+    def test_invalid_plugin_config(self):
+        #test invalid jsonpath...
+        cfg = dict(fields='payload.test', plugin=dict(bogus="true"))
+        self.assertRaises(converter.EventDefinitionException,
+                          converter.TraitDefinition,
+                          'test_trait',
+                          cfg,
+                          self.fake_plugin_mgr)
+
+    def test_unknown_plugin(self):
+        #test invalid jsonpath...
+        cfg = dict(fields='payload.test', plugin=dict(name='bogus'))
+        self.assertRaises(converter.EventDefinitionException,
+                          converter.TraitDefinition,
+                          'test_trait',
+                          cfg,
+                          self.fake_plugin_mgr)
+
+    def test_type_config(self):
+        cfg = dict(type='text', fields='payload.test')
+        t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr)
+        self.assertEqual(t.trait_type, models.Trait.TEXT_TYPE)
+
+        cfg = dict(type='int', fields='payload.test')
+        t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr)
+        self.assertEqual(t.trait_type, models.Trait.INT_TYPE)
+
+        cfg = dict(type='float', fields='payload.test')
+        t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr)
+        self.assertEqual(t.trait_type, models.Trait.FLOAT_TYPE)
+
+        cfg = dict(type='datetime', fields='payload.test')
+        t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr)
+        self.assertEqual(t.trait_type, models.Trait.DATETIME_TYPE)
+
+    def test_invalid_type_config(self):
+        #test invalid jsonpath...
+        cfg = dict(type='bogus', fields='payload.test')
+        self.assertRaises(converter.EventDefinitionException,
+                          converter.TraitDefinition,
+                          'bogus_trait',
+                          cfg,
+                          self.fake_plugin_mgr)
+
+
+class TestEventDefinition(ConverterBase):
+
+    def setUp(self):
+        super(TestEventDefinition, self).setUp()
+
+        self.traits_cfg = {
+            'instance_id': {
+                'type': 'text',
+                'fields': ['payload.instance_uuid',
+                           'payload.instance_id'],
+            },
+            'host': {
+                'type': 'text',
+                'fields': 'payload.host',
+            },
+        }
+
+        self.test_notification1 = self._create_test_notification(
+            "test.thing",
+            "uuid-for-notif-0001",
+            instance_id="uuid-for-instance-0001",
+            host='host-1-2-3')
+
+        self.test_notification2 = self._create_test_notification(
+            "test.thing",
+            "uuid-for-notif-0002",
+            instance_id="uuid-for-instance-0002")
+
+        self.test_notification3 = self._create_test_notification(
+            "test.thing",
+            "uuid-for-notif-0003",
+            instance_id="uuid-for-instance-0003",
+            host=None)
+        self.fake_plugin_mgr = {}
+
+    def test_to_event(self):
+        dtype = models.Trait.TEXT_TYPE
+        cfg = dict(event_type='test.thing', traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+
+        e = edef.to_event(self.test_notification1)
+        self.assertEqual(e.event_type, 'test.thing')
+        self.assertEqual(e.generated,
+                         datetime.datetime(2013, 8, 8, 21, 6, 37, 803826))
+
+        self.assertHasDefaultTraits(e)
+        self.assertHasTrait(e, 'host', value='host-1-2-3', dtype=dtype)
+        self.assertHasTrait(e, 'instance_id',
+                            value='uuid-for-instance-0001',
+                            dtype=dtype)
+
+    def test_to_event_missing_trait(self):
+        dtype = models.Trait.TEXT_TYPE
+        cfg = dict(event_type='test.thing', traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+
+        e = edef.to_event(self.test_notification2)
+
+        self.assertHasDefaultTraits(e)
+        self.assertHasTrait(e, 'instance_id',
+                            value='uuid-for-instance-0002',
+                            dtype=dtype)
+        self.assertDoesNotHaveTrait(e, 'host')
+
+    def test_to_event_null_trait(self):
+        dtype = models.Trait.TEXT_TYPE
+        cfg = dict(event_type='test.thing', traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+
+        e = edef.to_event(self.test_notification3)
+
+        self.assertHasDefaultTraits(e)
+        self.assertHasTrait(e, 'instance_id',
+                            value='uuid-for-instance-0003',
+                            dtype=dtype)
+        self.assertDoesNotHaveTrait(e, 'host')
+
+    def test_bogus_cfg_no_traits(self):
+        bogus = dict(event_type='test.foo')
+        self.assertRaises(converter.EventDefinitionException,
+                          converter.EventDefinition,
+                          bogus,
+                          self.fake_plugin_mgr)
+
+    def test_bogus_cfg_no_type(self):
+        bogus = dict(traits=self.traits_cfg)
+        self.assertRaises(converter.EventDefinitionException,
+                          converter.EventDefinition,
+                          bogus,
+                          self.fake_plugin_mgr)
+
+    def test_included_type_string(self):
+        cfg = dict(event_type='test.thing', traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        self.assertEqual(len(edef._included_types), 1)
+        self.assertEqual(edef._included_types[0], 'test.thing')
+        self.assertEqual(len(edef._excluded_types), 0)
+        self.assertTrue(edef.included_type('test.thing'))
+        self.assertFalse(edef.excluded_type('test.thing'))
+        self.assertTrue(edef.match_type('test.thing'))
+        self.assertFalse(edef.match_type('random.thing'))
+
+    def test_included_type_list(self):
+        cfg = dict(event_type=['test.thing', 'other.thing'],
+                   traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        self.assertEqual(len(edef._included_types), 2)
+        self.assertEqual(len(edef._excluded_types), 0)
+        self.assertTrue(edef.included_type('test.thing'))
+        self.assertTrue(edef.included_type('other.thing'))
+        self.assertFalse(edef.excluded_type('test.thing'))
+        self.assertTrue(edef.match_type('test.thing'))
+        self.assertTrue(edef.match_type('other.thing'))
+        self.assertFalse(edef.match_type('random.thing'))
+
+    def test_excluded_type_string(self):
+        cfg = dict(event_type='!test.thing', traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        self.assertEqual(len(edef._included_types), 1)
+        self.assertEqual(edef._included_types[0], '*')
+        self.assertEqual(edef._excluded_types[0], 'test.thing')
+        self.assertEqual(len(edef._excluded_types), 1)
+        self.assertEqual(edef._excluded_types[0], 'test.thing')
+        self.assertTrue(edef.excluded_type('test.thing'))
+        self.assertTrue(edef.included_type('random.thing'))
+        self.assertFalse(edef.match_type('test.thing'))
+        self.assertTrue(edef.match_type('random.thing'))
+
+    def test_excluded_type_list(self):
+        cfg = dict(event_type=['!test.thing', '!other.thing'],
+                   traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        self.assertEqual(len(edef._included_types), 1)
+        self.assertEqual(len(edef._excluded_types), 2)
+        self.assertTrue(edef.excluded_type('test.thing'))
+        self.assertTrue(edef.excluded_type('other.thing'))
+        self.assertFalse(edef.excluded_type('random.thing'))
+        self.assertFalse(edef.match_type('test.thing'))
+        self.assertFalse(edef.match_type('other.thing'))
+        self.assertTrue(edef.match_type('random.thing'))
+
+    def test_mixed_type_list(self):
+        cfg = dict(event_type=['*.thing', '!test.thing', '!other.thing'],
+                   traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        self.assertEqual(len(edef._included_types), 1)
+        self.assertEqual(len(edef._excluded_types), 2)
+        self.assertTrue(edef.excluded_type('test.thing'))
+        self.assertTrue(edef.excluded_type('other.thing'))
+        self.assertFalse(edef.excluded_type('random.thing'))
+        self.assertFalse(edef.match_type('test.thing'))
+        self.assertFalse(edef.match_type('other.thing'))
+        self.assertFalse(edef.match_type('random.whatzit'))
+        self.assertTrue(edef.match_type('random.thing'))
+
+    def test_catchall(self):
+        cfg = dict(event_type=['*.thing', '!test.thing', '!other.thing'],
+                   traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        self.assertFalse(edef.is_catchall)
+
+        cfg = dict(event_type=['!other.thing'],
+                   traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        self.assertFalse(edef.is_catchall)
+
+        cfg = dict(event_type=['other.thing'],
+                   traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        self.assertFalse(edef.is_catchall)
+
+        cfg = dict(event_type=['*', '!other.thing'],
+                   traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        self.assertFalse(edef.is_catchall)
+
+        cfg = dict(event_type=['*'],
+                   traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        self.assertTrue(edef.is_catchall)
+
+        cfg = dict(event_type=['*', 'foo'],
+                   traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        self.assertTrue(edef.is_catchall)
+
+    def test_extract_when(self):
+        now = timeutils.utcnow()
+        modified = now + datetime.timedelta(minutes=1)
+        timeutils.set_time_override(now)
+
+        body = {"timestamp": str(modified)}
+        when = converter.EventDefinition._extract_when(body)
+        self.assertTimestampEqual(modified, when)
+
+        body = {"_context_timestamp": str(modified)}
+        when = converter.EventDefinition._extract_when(body)
+        self.assertTimestampEqual(modified, when)
+
+        then = now + datetime.timedelta(hours=1)
+        body = {"timestamp": str(modified), "_context_timestamp": str(then)}
+        when = converter.EventDefinition._extract_when(body)
+        self.assertTimestampEqual(modified, when)
+
+        when = converter.EventDefinition._extract_when({})
+        self.assertTimestampEqual(now, when)
+
+    def test_default_traits(self):
+        cfg = dict(event_type='test.thing', traits={})
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        default_traits = converter.EventDefinition.DEFAULT_TRAITS.keys()
+        traits = set(edef.traits.keys())
+        for dt in default_traits:
+            self.assertIn(dt, traits)
+        self.assertEqual(len(edef.traits),
+                         len(converter.EventDefinition.DEFAULT_TRAITS))
+
+    def test_traits(self):
+        cfg = dict(event_type='test.thing', traits=self.traits_cfg)
+        edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
+        default_traits = converter.EventDefinition.DEFAULT_TRAITS.keys()
+        traits = set(edef.traits.keys())
+        for dt in default_traits:
+            self.assertIn(dt, traits)
+        self.assertIn('host', traits)
+        self.assertIn('instance_id', traits)
+        self.assertEqual(len(edef.traits),
+                         len(converter.EventDefinition.DEFAULT_TRAITS) + 2)
+
+
+class TestNotificationConverter(ConverterBase):
+
+    def setUp(self):
+        super(TestNotificationConverter, self).setUp()
+
+        self.valid_event_def1 = [{
+            'event_type': 'compute.instance.create.*',
+            'traits': {
+                'instance_id': {
+                    'type': 'text',
+                    'fields': ['payload.instance_uuid',
+                               'payload.instance_id'],
+                },
+                'host': {
+                    'type': 'text',
+                    'fields': 'payload.host',
+                },
+            },
+        }]
+
+        self.test_notification1 = self._create_test_notification(
+            "compute.instance.create.start",
+            "uuid-for-notif-0001",
+            instance_id="uuid-for-instance-0001",
+            host='host-1-2-3')
+        self.test_notification2 = self._create_test_notification(
+            "bogus.notification.from.mars",
+            "uuid-for-notif-0002",
+            weird='true',
+            host='cydonia')
+        self.fake_plugin_mgr = {}
+
+    def test_converter_missing_keys(self):
+        # test a malformed notification
+        now = timeutils.utcnow()
+        timeutils.set_time_override(now)
+        c = converter.NotificationEventsConverter(
+            [],
+            self.fake_plugin_mgr,
+            add_catchall=True)
+        message = {'event_type': "foo",
+                   'message_id': "abc",
+                   'publisher_id': "1"}
+        e = c.to_event(message)
+        self.assertIsValidEvent(e, message)
+        self.assertEqual(len(e.traits), 1)
+        self.assertEqual("foo", e.event_type)
+        self.assertEqual(now, e.generated)
+
+    def test_converter_with_catchall(self):
+        c = converter.NotificationEventsConverter(
+            self.valid_event_def1,
+            self.fake_plugin_mgr,
+            add_catchall=True)
+        self.assertEqual(len(c.definitions), 2)
+        e = c.to_event(self.test_notification1)
+        self.assertIsValidEvent(e, self.test_notification1)
+        self.assertEqual(len(e.traits), 3)
+        self.assertHasDefaultTraits(e)
+        self.assertHasTrait(e, 'instance_id')
+        self.assertHasTrait(e, 'host')
+
+        e = c.to_event(self.test_notification2)
+        self.assertIsValidEvent(e, self.test_notification2)
+        self.assertEqual(len(e.traits), 1)
+        self.assertHasDefaultTraits(e)
+        self.assertDoesNotHaveTrait(e, 'instance_id')
+        self.assertDoesNotHaveTrait(e, 'host')
+
+    def test_converter_without_catchall(self):
+        c = converter.NotificationEventsConverter(
+            self.valid_event_def1,
+            self.fake_plugin_mgr,
+            add_catchall=False)
+        self.assertEqual(len(c.definitions), 1)
+        e = c.to_event(self.test_notification1)
+        self.assertIsValidEvent(e, self.test_notification1)
+        self.assertEqual(len(e.traits), 3)
+        self.assertHasDefaultTraits(e)
+        self.assertHasTrait(e, 'instance_id')
+        self.assertHasTrait(e, 'host')
+
+        e = c.to_event(self.test_notification2)
+        self.assertIsNotValidEvent(e, self.test_notification2)
+
+    def test_converter_empty_cfg_with_catchall(self):
+        c = converter.NotificationEventsConverter(
+            [],
+            self.fake_plugin_mgr,
+            add_catchall=True)
+        self.assertEqual(len(c.definitions), 1)
+        e = c.to_event(self.test_notification1)
+        self.assertIsValidEvent(e, self.test_notification1)
+        self.assertEqual(len(e.traits), 1)
+        self.assertHasDefaultTraits(e)
+
+        e = c.to_event(self.test_notification2)
+        self.assertIsValidEvent(e, self.test_notification2)
+        self.assertEqual(len(e.traits), 1)
+        self.assertHasDefaultTraits(e)
+
+    def test_converter_empty_cfg_without_catchall(self):
+        c = converter.NotificationEventsConverter(
+            [],
+            self.fake_plugin_mgr,
+            add_catchall=False)
+        self.assertEqual(len(c.definitions), 0)
+        e = c.to_event(self.test_notification1)
+        self.assertIsNotValidEvent(e, self.test_notification1)
+
+        e = c.to_event(self.test_notification2)
+        self.assertIsNotValidEvent(e, self.test_notification2)
+
+    def test_setup_events_default_config(self):
+
+        def mock_exists(path):
+            return False
+
+        def mock_get_config_file():
+            return None
+
+        with mock.patch('ceilometer.event.converter.get_config_file',
+                        mock_get_config_file):
+
+            oslo_cfg.CONF.set_override('drop_unmatched_notifications',
+                                       False, group='event')
+
+            with mock.patch('os.path.exists', mock_exists):
+                c = converter.setup_events(self.fake_plugin_mgr)
+            self.assertIsInstance(c, converter.NotificationEventsConverter)
+            self.assertEqual(len(c.definitions), 1)
+            self.assertTrue(c.definitions[0].is_catchall)
+
+            oslo_cfg.CONF.set_override('drop_unmatched_notifications',
+                                       True, group='event')
+
+            with mock.patch('os.path.exists', mock_exists):
+                c = converter.setup_events(self.fake_plugin_mgr)
+            self.assertIsInstance(c, converter.NotificationEventsConverter)
+            self.assertEqual(len(c.definitions), 0)

ceilometer/tests/event/test_trait_plugins.py

@@ -0,0 +1,118 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2013 Rackspace Hosting.
+#
+# Author: Monsyne Dragon <mdragon@rackspace.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ceilometer.event import trait_plugins
+from ceilometer.openstack.common import test
+
+
+class TestSplitterPlugin(test.BaseTestCase):
+
+    def setUp(self):
+        super(TestSplitterPlugin, self).setUp()
+        self.pclass = trait_plugins.SplitterTraitPlugin
+
+    def test_split(self):
+        param = dict(separator='-', segment=0)
+        plugin = self.pclass(**param)
+        match_list = [('test.thing', 'test-foobar-baz')]
+        value = plugin.trait_value(match_list)
+        self.assertEqual(value, 'test')
+
+        param = dict(separator='-', segment=1)
+        plugin = self.pclass(**param)
+        match_list = [('test.thing', 'test-foobar-baz')]
+        value = plugin.trait_value(match_list)
+        self.assertEqual(value, 'foobar')
+
+        param = dict(separator='-', segment=1, max_split=1)
+        plugin = self.pclass(**param)
+        match_list = [('test.thing', 'test-foobar-baz')]
+        value = plugin.trait_value(match_list)
+        self.assertEqual(value, 'foobar-baz')
+
+    def test_no_sep(self):
+        param = dict(separator='-', segment=0)
+        plugin = self.pclass(**param)
+        match_list = [('test.thing', 'test.foobar.baz')]
+        value = plugin.trait_value(match_list)
+        self.assertEqual(value, 'test.foobar.baz')
+
+    def test_no_segment(self):
+        param = dict(separator='-', segment=5)
+        plugin = self.pclass(**param)
+        match_list = [('test.thing', 'test-foobar-baz')]
+        value = plugin.trait_value(match_list)
+        self.assertIs(None, value)
+
+    def test_no_match(self):
+        param = dict(separator='-', segment=0)
+        plugin = self.pclass(**param)
+        match_list = []
+        value = plugin.trait_value(match_list)
+        self.assertIs(None, value)
+
+
+class TestBitfieldPlugin(test.BaseTestCase):
+
+    def setUp(self):
+        super(TestBitfieldPlugin, self).setUp()
+        self.pclass = trait_plugins.BitfieldTraitPlugin
+        self.init = 0
+        self.params = dict(initial_bitfield=self.init,
+                           flags=[dict(path='payload.foo', bit=0, value=42),
+                                  dict(path='payload.foo', bit=1, value=12),
+                                  dict(path='payload.thud', bit=1, value=23),
+                                  dict(path='thingy.boink', bit=4),
+                                  dict(path='thingy.quux', bit=6,
+                                       value="wokka"),
+                                  dict(path='payload.bar', bit=10,
+                                       value='test')])
+
+    def test_bitfield(self):
+        match_list = [('payload.foo', 12),
+                      ('payload.bar', 'test'),
+                      ('thingy.boink', 'testagain')]
+
+        plugin = self.pclass(**self.params)
+        value = plugin.trait_value(match_list)
+        self.assertEqual(value, 0x412)
+
+    def test_initial(self):
+        match_list = [('payload.foo', 12),
+                      ('payload.bar', 'test'),
+                      ('thingy.boink', 'testagain')]
+        self.params['initial_bitfield'] = 0x2000
+        plugin = self.pclass(**self.params)
+        value = plugin.trait_value(match_list)
+        self.assertEqual(value, 0x2412)
+
+    def test_no_match(self):
+        match_list = []
+        plugin = self.pclass(**self.params)
+        value = plugin.trait_value(match_list)
+        self.assertEqual(value, self.init)
+
+    def test_multi(self):
+        match_list = [('payload.foo', 12),
+                      ('payload.thud', 23),
+                      ('payload.bar', 'test'),
+                      ('thingy.boink', 'testagain')]
+
+        plugin = self.pclass(**self.params)
+        value = plugin.trait_value(match_list)
+        self.assertEqual(value, 0x412)

ceilometer/tests/storage/test_models.py

@@ -16,6 +16,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import datetime
+
 from ceilometer.openstack.common import test
 from ceilometer.storage import models
 
@@ -54,3 +56,26 @@ class ModelTest(test.BaseTestCase):
     def test_event_repr_no_traits(self):
         x = models.Event("1", "name", "now", None)
         self.assertEqual("<Event: 1, name, now, >", repr(x))
+
+
+class TestTraitModel(test.BaseTestCase):
+
+    def test_convert_value(self):
+        v = models.Trait.convert_value(
+            models.Trait.INT_TYPE, '10')
+        self.assertEqual(v, 10)
+        self.assertIsInstance(v, int)
+        v = models.Trait.convert_value(
+            models.Trait.FLOAT_TYPE, '10')
+        self.assertEqual(v, 10.0)
+        self.assertIsInstance(v, float)
+
+        v = models.Trait.convert_value(
+            models.Trait.DATETIME_TYPE, '2013-08-08 21:05:37.123456')
+        self.assertEqual(v, datetime.datetime(2013, 8, 8, 21, 5, 37, 123456))
+        self.assertIsInstance(v, datetime.datetime)
+
+        v = models.Trait.convert_value(
+            models.Trait.TEXT_TYPE, 10)
+        self.assertEqual(v, "10")
+        self.assertIsInstance(v, str)

ceilometer/tests/test_collector.py

@@ -147,6 +147,7 @@ class TestCollector(tests_base.BaseTestCase):
         self._verify_udp_socket(udp_socket)
 
     @patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
+    @patch('ceilometer.event.converter.setup_events', mock.MagicMock())
     def test_init_host(self):
         # If we try to create a real RPC connection, init_host() never
         # returns. Mock it out so we can establish the service

ceilometer/tests/test_notification.py

@@ -17,7 +17,6 @@
 # under the License.
 """Tests for Ceilometer notify daemon."""
 
-import datetime
 import mock
 
 from stevedore import extension
@@ -26,7 +25,6 @@ from stevedore.tests import manager as test_manager
 from ceilometer.compute import notifications
 from ceilometer import notification
 from ceilometer.openstack.common.fixture import config
-from ceilometer.openstack.common import timeutils
 from ceilometer.storage import models
 from ceilometer.tests import base as tests_base
 
@@ -89,6 +87,7 @@ class TestNotification(tests_base.BaseTestCase):
         self.CONF.set_override("connection", "log://", group='database')
 
     @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
+    @mock.patch('ceilometer.event.converter.setup_events', mock.MagicMock())
     def test_process_notification(self):
         # If we try to create a real RPC connection, init_host() never
         # returns. Mock it out so we can establish the service
@@ -124,33 +123,12 @@ class TestNotification(tests_base.BaseTestCase):
             self.srv.process_notification({})
             self.assertTrue(fake_msg_to_event.called)
 
-    def test_message_to_event_missing_keys(self):
-        now = timeutils.utcnow()
-        timeutils.set_time_override(now)
-        message = {'event_type': "foo",
-                   'message_id': "abc",
-                   'publisher_id': "1"}
-
-        mock_dispatcher = mock.MagicMock()
-        self.srv.dispatcher_manager = test_manager.TestExtensionManager(
-            [extension.Extension('test',
-                                 None,
-                                 None,
-                                 mock_dispatcher
-                                 ),
-             ])
-
-        self.srv._message_to_event(message)
-        events = mock_dispatcher.record_events.call_args[0]
-        self.assertEqual(1, len(events))
-        event = events[0]
-        self.assertEqual("foo", event.event_type)
-        self.assertEqual(now, event.generated)
-        self.assertEqual(1, len(event.traits))
-
     def test_message_to_event_duplicate(self):
         self.CONF.set_override("store_events", True, group="notification")
         mock_dispatcher = mock.MagicMock()
+        self.srv.event_converter = mock.MagicMock()
+        self.srv.event_converter.to_event.return_value = mock.MagicMock(
+            event_type='test.test')
         self.srv.dispatcher_manager = test_manager.TestExtensionManager(
             [extension.Extension('test',
                                  None,
@@ -166,6 +144,9 @@ class TestNotification(tests_base.BaseTestCase):
     def test_message_to_event_bad_event(self):
         self.CONF.set_override("store_events", True, group="notification")
         mock_dispatcher = mock.MagicMock()
+        self.srv.event_converter = mock.MagicMock()
+        self.srv.event_converter.to_event.return_value = mock.MagicMock(
+            event_type='test.test')
         self.srv.dispatcher_manager = test_manager.TestExtensionManager(
             [extension.Extension('test',
                                  None,
@@ -178,24 +159,3 @@ class TestNotification(tests_base.BaseTestCase):
         message = {'event_type': "foo", 'message_id': "abc"}
         self.assertRaises(notification.UnableToSaveEventException,
                           self.srv._message_to_event, message)
-
-    def test_extract_when(self):
-        now = timeutils.utcnow()
-        modified = now + datetime.timedelta(minutes=1)
-        timeutils.set_time_override(now)
-
-        body = {"timestamp": str(modified)}
-        when = notification.NotificationService._extract_when(body)
-        self.assertTimestampEqual(modified, when)
-
-        body = {"_context_timestamp": str(modified)}
-        when = notification.NotificationService._extract_when(body)
-        self.assertTimestampEqual(modified, when)
-
-        then = now + datetime.timedelta(hours=1)
-        body = {"timestamp": str(modified), "_context_timestamp": str(then)}
-        when = notification.NotificationService._extract_when(body)
-        self.assertTimestampEqual(modified, when)
-
-        when = notification.NotificationService._extract_when({})
-        self.assertTimestampEqual(now, when)

doc/source/configuration.rst

@@ -169,6 +169,22 @@ database_connection          hbase://$hbase-thrift-server:9090     Database conn
     the Ceilometer services that use the database to allow the changes to take
     affect, i.e. the collector and API services.
 
+Event Conversion
+================
+
+The following options in the [event] configuration section affect the extraction of Event data from notifications.
+
+==================================  ======================================  ==============================================================
+Parameter                           Default                                 Note
+==================================  ======================================  ==============================================================
+drop_unmatched_notifications        False                                   If set to True, then notifications with no matching event
+                                                                            definition will be dropped.
+                                                                            (Notifications will *only* be dropped if this is True)
+definitions_cfg_file                event_definitions.yaml                  Name of event definitions config file (yaml format)
+==================================  ======================================  ==============================================================
+
+
+
 General options
 ===============
 

doc/source/events.rst

@@ -0,0 +1,266 @@
+..
+      Copyright 2013 Rackspace Hosting.
+
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+=============================
+ Events and Event Processing
+=============================
+
+Events vs. Samples
+==================
+
+In addition to Meters, and related Sample data, Ceilometer can also process
+Events.  While a Sample represents a single numeric datapoint, driving a Meter
+that represents the changes in that value over time, an Event represents the
+state of an object in an OpenStack service (such as an Instance in Nova, or
+an Image in Glance) at a point in time when something of interest has occurred.
+This can include non-numeric data, such as an instance's flavor, or network
+address.
+
+In general, Events let you know when something has changed about an
+object in an OpenStack system, such as the resize of an instance, or creation
+of an image.
+
+While Samples can be relatively cheap (small),
+disposable (losing an individual sample datapoint won't matter much),
+and fast, Events are larger, more informative, and should be handled more
+consistently (you do not want to lose one).
+
+Event Structure
+===============
+
+Events contain the following information:
+
+event_type
+    A dotted string defining what event occured, such as "compute.instance.resize.start"
+
+message_id
+    A UUID for this event.
+
+generated
+    A timestamp of when the event occurred on the source system.
+
+traits
+    A flat mapping of key-value pairs.
+    The event's Traits contain most of the details of the event. Traits are
+    typed, and can be strings, ints, floats, or datetimes.
+
+
+Events from Notifications
+=========================
+
+Events are primarily created via the notifications system in OpenStack.
+OpenStack systems, such as Nova, Glance, Neutron, etc. will emit
+notifications in a JSON format to message queue when some notable action is
+taken by that system. Ceilometer will consume such notifications from the
+message queue, and process them.
+
+The general philosophy of notifications in OpenStack is to emit any and all
+data someone might need, and let the consumer filter out what they are not
+interested in. In order to make processing simpler and more efficient,
+the notifications are stored and processed within Ceilometer as Events.
+The notification payload, which can be an arbitrarily complex JSON data
+structure, is converted to a flat set of key-value pairs known as Traits.
+This conversion is specified by a config file, so that only the specific
+fields within the notification that are actually needed for processing the
+event will have to be stored as Traits.
+
+Note that the Event format is meant for efficient processing and querying,
+there are other means available for archiving notifications (i.e. for audit
+purposes, etc), possibly to different datastores.
+
+Converting Notifications to Events
+----------------------------------
+
+In order to make it easier to allow users to extract what they need,
+the conversion from Notifications to Events is driven by a
+configuration file (specified by the flag definitions_cfg_file_ in
+ceilometer.conf).
+
+This includes descriptions of how to map fields in the notification body
+to Traits, and optional plugins for doing any programmatic translations
+(splitting a string, forcing case, etc.)
+
+The mapping of notifications to events is defined per event_type, which
+can be wildcarded. Traits are added to events if the corresponding fields
+in the notification exist and are non-null. (As a special case, an empty
+string is considered null for non-text traits. This is due to some openstack
+projects (mostly Nova) using empty string for null dates.)
+
+If the definitions file is not present, a warning will be logged, but an empty
+set of definitions will be assumed. By default, any notifications that
+do not have an event definition in the definitions file for them will be
+converted to events with a set of minimal, default traits.  This can be
+changed by setting the flag drop_unmatched_notifications_ in the
+ceilometer.conf file. If this is set to True, then any notifications
+that don't have events defined for them in the file will be dropped.
+This can be what you want, the notification system is quite chatty by design
+(notifications philosophy is "tell us everything, we'll ignore what we don't
+need"), so you may want to ignore the noisier ones if you don't use them.
+
+.. _definitions_cfg_file: configuration.html#event-conversion
+.. _drop_unmatched_notifications: configuration.html#event-conversion
+
+There is a set of default traits (all are TEXT type) that will be added to
+all events if the notification has the relevant data:
+
+* service:  (All notifications should have this) notification's publisher
+* tenant_id
+* request_id
+
+These do not have to be specified in the event definition, they are
+automatically added, but their definitions can be overridden for a given
+event_type.
+
+Definitions file format
+-----------------------
+
+The event definitions file is in YAML format. It consists of a list of event
+definitions, which are mappings. Order is significant, the list of definitions
+is scanned in *reverse* order (last definition in the file to the first),
+to find a definition which matches the notification's event_type.  That
+definition will be used to generate the Event. The reverse ordering is done
+because it is common to want to have a more general wildcarded definition
+(such as "compute.instance.*" ) with a set of traits common to all of those
+events, with a few more specific event definitions (like
+"compute.instance.exists") afterward that have all of the above traits, plus
+a few more. This lets you put the general definition first, followed by the
+specific ones, and use YAML mapping include syntax to avoid copying all of the
+trait definitions.
+
+Event Definitions
+-----------------
+
+Each event definition is a mapping with two keys (both required):
+
+event_type
+    This is a list (or a string, which will be taken as a 1 element
+    list) of event_types this definition will handle. These can be
+    wildcarded with unix shell glob syntax. An exclusion listing
+    (starting with a '!') will exclude any types listed from matching.
+    If ONLY exclusions are listed, the definition will match anything
+    not matching the exclusions.
+traits
+    This is a mapping, the keys are the trait names, and the values are
+    trait definitions.
+
+Trait Definitions
+-----------------
+
+Each trait definition is a mapping with the following keys:
+
+type
+    (optional) The data type for this trait. (as a string). Valid
+    options are: *text*, *int*, *float*, and *datetime*.
+    defaults to *text* if not specified.
+fields
+    A path specification for the field(s) in the notification you wish
+    to extract for this trait. Specifications can be written to match
+    multiple possible fields, the value for the trait will be derived
+    from the matching fields that exist and have a non-null values in
+    the notification. By default the value will be the first such field.
+    (plugins can alter that, if they wish). This is normally a string,
+    but, for convenience, it can be specified as a list of
+    specifications, which will match the fields for all of them. (See
+    `Field Path Specifications`_ for more info on this syntax.)
+plugin
+    (optional) This is a mapping (For convenience, this value can also
+    be specified as a string, which is interpreted as the name of a
+    plugin to be loaded with no parameters) with the following keys
+
+    name
+        (string) name of a plugin to load
+
+    parameters
+        (optional) Mapping of keyword arguments to pass to the plugin on
+        initialization. (See documentation on each plugin to see what
+        arguments it accepts.)
+
+Field Path Specifications
+-------------------------
+
+The path specifications define which fields in the JSON notification
+body are extracted to provide the value for a given trait.  The paths
+can be specified with a dot syntax (e.g. "payload.host"). Square
+bracket syntax (e.g. "payload[host]") is also supported. In either
+case, if the key for the field you are looking for contains special
+characters, like '.', it will need to be quoted (with double or single
+quotes) like so:
+
+          payload.image_meta.'org.openstack__1__architecture'
+
+The syntax used for the field specification is a variant of JSONPath,
+and is fairly flexible. (see: https://github.com/kennknowles/python-jsonpath-rw for more info)
+
+Example Definitions file
+------------------------
+
+::
+
+    ---
+    - event_type: compute.instance.*
+      traits: &instance_traits
+        user_id:
+          fields: payload.user_id
+        instance_id:
+          fields: payload.instance_id
+        host:
+          fields: publisher_id
+          plugin:
+            name: split
+            parameters:
+              segment: 1
+              max_split: 1
+        service_name:
+          fields: publisher_id
+          plugin: split
+        instance_type_id:
+          type: int
+          fields: payload.instance_type_id
+        os_architecture:
+          fields: payload.image_meta.'org.openstack__1__architecture'
+        launched_at:
+          type: datetime
+          fields: payload.launched_at
+        deleted_at:
+          type: datetime
+          fields: payload.deleted_at
+    - event_type:
+        - compute.instance.exists
+        - compute.instance.update
+      traits:
+        <<: *instance_traits
+        audit_period_beginning:
+          type: datetime
+          fields: payload.audit_period_beginning
+        audit_period_ending:
+          type: datetime
+          fields: payload.audit_period_ending
+
+Trait plugins
+-------------
+
+Trait plugins can be used to do simple programmatic conversions on the value in
+a notification field, like splitting a string, lowercasing a value, converting
+a screwball date into ISO format, or the like. They are initialized with the
+parameters from the trait definition, if any, which can customize their
+behavior for a given trait. They are called with a list of all matching fields
+from the notification, so they can derive a value from multiple fields. The
+plugin will be called even if there is no fields found matching the field
+path(s), this lets a plugin set a default value, if needed. A plugin can also
+reject a value by returning *None*, which will cause the trait not to be
+added. If the plugin returns anything other than *None*, the trait's value
+will be set from whatever the plugin returned (coerced to the appropriate type
+for the trait).
+

doc/source/index.rst

@@ -48,6 +48,7 @@ Table of contents
 
    architecture
    measurements
+   events
    install/index
    configuration
    webapi/index

etc/ceilometer/ceilometer.conf.sample

@@ -709,6 +709,17 @@
 #record_history=true
 
 
+[matchmaker_ring]
+
+#
+# Options defined in ceilometer.openstack.common.rpc.matchmaker_ring
+#
+
+# Matchmaker ring file (JSON) (string value)
+# Deprecated group/name - [DEFAULT]/matchmaker_ringfile
+#ringfile=/etc/oslo/matchmaker_ring.json
+
+
 [rpc_notifier2]
 
 #
@@ -803,15 +814,19 @@
 #udp_port=4952
 
 
-[matchmaker_ring]
+[event]
 
 #
-# Options defined in ceilometer.openstack.common.rpc.matchmaker_ring
+# Options defined in ceilometer.event.converter
 #
 
-# Matchmaker ring file (JSON) (string value)
+# Configuration file for event definitions (string value)
-# Deprecated group/name - [DEFAULT]/matchmaker_ringfile
+#definitions_cfg_file=event_definitions.yaml
-#ringfile=/etc/oslo/matchmaker_ring.json
+
+# Drop notifications if no event definition matches.
+# (Otherwise, we convert them with just the default traits)
+# (boolean value)
+#drop_unmatched_notifications=false
 
 
 [matchmaker_redis]

etc/ceilometer/event_definitions.yaml

@@ -0,0 +1,63 @@
+---
+- event_type: compute.instance.*
+  traits: &instance_traits
+    tenant_id:
+      fields: payload.tenant_id
+    user_id:
+      fields: payload.user_id
+    instance_id:
+      fields: payload.instance_id
+    host:
+      fields: publisher_id
+      plugin:
+        name: split
+        parameters:
+          segment: 1
+          max_split: 1
+    service:
+      fields: publisher_id
+      plugin: split
+    memory_mb:
+      type: int
+      fields: payload.memory_mb
+    disk_gb:
+      type: int
+      fields: payload.disk_gb
+    root_gb:
+      type: int
+      fields: payload.root_gb
+    ephemeral_gb:
+      type: int
+      fields: payload.ephemeral_gb
+    vcpus:
+      type: int
+      fields: payload.vcpus
+    instance_type_id:
+      type: int
+      fields: payload.instance_type_id
+    instance_type:
+      fields: payload.instance_type
+    state:
+      fields: payload.state
+    os_architecture:
+      fields: payload.image_meta.'org.openstack__1__architecture'
+    os_version:
+      fields: payload.image_meta.'org.openstack__1__os_version'
+    os_distro:
+      fields: payload.image_meta.'org.openstack__1__os_distro'
+    launched_at:
+      type: datetime
+      fields: payload.launched_at
+    deleted_at:
+      type: datetime
+      fields: payload.deleted_at
+- event_type: compute.instance.exists
+  traits:
+    <<: *instance_traits
+    audit_period_beginning:
+      type: datetime
+      fields: payload.audit_period_beginning
+    audit_period_ending:
+      type: datetime
+      fields: payload.audit_period_ending
+

requirements.txt

@@ -5,6 +5,7 @@ eventlet>=0.13.0
 Flask>=0.10,<1.0
 happybase>=0.4
 iso8601>=0.1.8
+jsonpath-rw>=1.2.0,<2.0
 kombu>=2.4.8
 lockfile>=0.8
 lxml>=2.3

setup.cfg

@@ -112,6 +112,10 @@ ceilometer.alarm.notifier =
     http = ceilometer.alarm.notifier.rest:RestAlarmNotifier
     https = ceilometer.alarm.notifier.rest:RestAlarmNotifier
 
+ceilometer.event.trait_plugin =
+    split = ceilometer.event.trait_plugins:SplitterTraitPlugin
+    bitfield = ceilometer.event.trait_plugins:BitfieldTraitPlugin
+
 paste.filter_factory =
     swift = ceilometer.objectstore.swift_middleware:filter_factory