Add openstack.common.{context,notifier,log} and update .rpc

Change-Id: I04e47ffc2eed86483e0c902532f20ef02d1d0641
Signed-off-by: Julien Danjou <julien@danjou.info>

ceilometer/agent/manager.py

@@ -20,7 +20,7 @@ import pkg_resources
 
 from nova import manager
 
-from ceilometer import log
+from ceilometer.openstack.common import log
 from ceilometer import publish
 
 

ceilometer/collector/dispatcher.py

@@ -21,7 +21,7 @@ and publish the results.
 
 import pkg_resources
 
-from ceilometer import log
+from ceilometer.openstack.common import log
 
 LOG = log.getLogger(__name__)
 

ceilometer/collector/manager.py

@@ -20,19 +20,19 @@ from nova import context
 from nova import flags
 from nova import manager
 
-from ceilometer import log
 from ceilometer import meter
 from ceilometer import publish
 from ceilometer import rpc
 from ceilometer import storage
 from ceilometer.collector import dispatcher
 from ceilometer.openstack.common import cfg
+from ceilometer.openstack.common import log
 from ceilometer.openstack.common import timeutils
 from ceilometer.openstack.common.rpc import dispatcher as rpc_dispatcher
 
 # FIXME(dhellmann): There must be another way to do this.
 # Import rabbit_notifier to register notification_topics flag
-import nova.notifier.rabbit_notifier
+import ceilometer.openstack.common.notifier.rabbit_notifier
 try:
     import nova.openstack.common.rpc as nova_rpc
 except ImportError:

ceilometer/compute/libvirt.py

@@ -21,10 +21,10 @@ from lxml import etree
 from nova import flags
 import nova.virt.connection
 
-from ceilometer import log
 from ceilometer import counter
 from ceilometer import plugin
 from ceilometer.compute import instance as compute_instance
+from ceilometer.openstack.common import log
 from ceilometer.openstack.common import timeutils
 
 FLAGS = flags.FLAGS

ceilometer/compute/network.py

@@ -18,7 +18,7 @@
 
 from nova import exception
 
-from ceilometer import log
+from ceilometer.openstack.common import log
 
 from .. import counter
 from .. import plugin

ceilometer/log.py

@@ -1,102 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# Copyright © 2012 eNovance <licensing@enovance.com>
-#
-# Author: Julien Danjou <julien@danjou.info>
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import os
-import inspect
-import logging
-import logging.config
-import traceback
-import sys
-
-from ceilometer.openstack.common import cfg
-
-cfg.CONF.register_opts(
-    [
-        cfg.StrOpt('log_level',
-                   default="debug",
-                   help='Log level',
-                   ),
-        cfg.StrOpt('logging_default_format_string',
-                   default='%(asctime)s %(levelname)s %(name)s: %(message)s',
-                   help='format string to use for log messages'),
-        ]
-    )
-
-
-def _get_binary_name():
-    return os.path.basename(inspect.stack()[-1][1])
-
-
-def _get_log_file_path(binary=None):
-    if cfg.CONF.log_file and not cfg.CONF.log_dir:
-        return cfg.CONF.log_file
-
-    if cfg.CONF.log_file and cfg.CONF.log_dir:
-        return os.path.join(cfg.CONF.log_file,
-                            cfg.CONF.log_file)
-
-    if cfg.CONF.log_dir:
-        binary = binary or _get_binary_name()
-        return '%s.log' % (os.path.join(cfg.CONF.log_dir, binary),)
-
-
-def getLogger(name='ceilometer'):
-    return logging.getLogger(name)
-
-
-def _setup_default_logger(logger_name):
-    """Configure a single logger."""
-    root = getLogger(logger_name)
-    for handler in root.handlers:
-        root.removeHandler(handler)
-    logpath = _get_log_file_path()
-    if logpath:
-        filelog = logging.handlers.WatchedFileHandler(logpath)
-        filelog.setFormatter(
-            logging.Formatter(cfg.CONF.logging_default_format_string))
-        root.addHandler(filelog)
-
-        mode = int(FLAGS.logfile_mode, 8)
-        st = os.stat(logpath)
-        if st.st_mode != (stat.S_IFREG | mode):
-            os.chmod(logpath, mode)
-    else:
-        streamlog = logging.StreamHandler(sys.stdout)
-        streamlog.setFormatter(
-            logging.Formatter(cfg.CONF.logging_default_format_string))
-        root.addHandler(streamlog)
-
-    if cfg.CONF.log_level:
-        root.setLevel(logging.getLevelName(cfg.CONF.log_level.upper()))
-
-
-def setup():
-    if cfg.CONF.log_config:
-        try:
-            logging.config.fileConfig(cfg.CONF.log_config)
-        except Exception:
-            traceback.print_exc()
-            raise
-    else:
-        # Strip any existing log handlers to avoid seeing duplicate
-        # messages on the console.
-        root = getLogger(None)
-        for handler in root.handlers:
-            root.removeHandler(handler)
-        _setup_default_logger('ceilometer')
-        _setup_default_logger('nova')

ceilometer/openstack/common/context.py

@@ -0,0 +1,81 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Simple class that stores security context information in the web request.
+
+Projects should subclass this class if they wish to enhance the request
+context or provide additional information in their specific WSGI pipeline.
+"""
+
+import itertools
+import uuid
+
+
+def generate_request_id():
+    return 'req-' + str(uuid.uuid4())
+
+
+class RequestContext(object):
+
+    """
+    Stores information about the security context under which the user
+    accesses the system, as well as additional request information.
+    """
+
+    def __init__(self, auth_tok=None, user=None, tenant=None, is_admin=False,
+                 read_only=False, show_deleted=False, request_id=None):
+        self.auth_tok = auth_tok
+        self.user = user
+        self.tenant = tenant
+        self.is_admin = is_admin
+        self.read_only = read_only
+        self.show_deleted = show_deleted
+        if not request_id:
+            request_id = generate_request_id()
+        self.request_id = request_id
+
+    def to_dict(self):
+        return {'user': self.user,
+                'tenant': self.tenant,
+                'is_admin': self.is_admin,
+                'read_only': self.read_only,
+                'show_deleted': self.show_deleted,
+                'auth_token': self.auth_tok,
+                'request_id': self.request_id}
+
+
+def get_admin_context(show_deleted="no"):
+    context = RequestContext(None,
+                             tenant=None,
+                             is_admin=True,
+                             show_deleted=show_deleted)
+    return context
+
+
+def get_context_from_function_and_args(function, args, kwargs):
+    """Find an arg of type RequestContext and return it.
+
+       This is useful in a couple of decorators where we don't
+       know much about the function we're wrapping.
+    """
+
+    for arg in itertools.chain(kwargs.values(), args):
+        if isinstance(arg, RequestContext):
+            return arg
+
+    return None

ceilometer/openstack/common/log.py

@@ -0,0 +1,458 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Openstack logging handler.
+
+This module adds to logging functionality by adding the option to specify
+a context object when calling the various log methods.  If the context object
+is not specified, default formatting is used. Additionally, an instance uuid
+may be passed as part of the log message, which is intended to make it easier
+for admins to find messages related to a specific instance.
+
+It also allows setting of formatting information through conf.
+
+"""
+
+import cStringIO
+import inspect
+import itertools
+import logging
+import logging.config
+import logging.handlers
+import os
+import stat
+import sys
+import traceback
+
+from ceilometer.openstack.common import cfg
+from ceilometer.openstack.common import jsonutils
+from ceilometer.openstack.common import local
+from ceilometer.openstack.common import notifier
+
+
+log_opts = [
+    cfg.StrOpt('logging_context_format_string',
+               default='%(asctime)s %(levelname)s %(name)s [%(request_id)s '
+                       '%(user_id)s %(project_id)s] %(instance)s'
+                       '%(message)s',
+               help='format string to use for log messages with context'),
+    cfg.StrOpt('logging_default_format_string',
+               default='%(asctime)s %(levelname)s %(name)s [-] %(instance)s'
+                       '%(message)s',
+               help='format string to use for log messages without context'),
+    cfg.StrOpt('logging_debug_format_suffix',
+               default='from (pid=%(process)d) %(funcName)s '
+                       '%(pathname)s:%(lineno)d',
+               help='data to append to log format when level is DEBUG'),
+    cfg.StrOpt('logging_exception_prefix',
+               default='%(asctime)s TRACE %(name)s %(instance)s',
+               help='prefix each line of exception output with this format'),
+    cfg.ListOpt('default_log_levels',
+                default=[
+                  'amqplib=WARN',
+                  'sqlalchemy=WARN',
+                  'boto=WARN',
+                  'suds=INFO',
+                  'keystone=INFO',
+                  'eventlet.wsgi.server=WARN'
+                  ],
+                help='list of logger=LEVEL pairs'),
+    cfg.BoolOpt('publish_errors',
+                default=False,
+                help='publish error events'),
+
+    # NOTE(mikal): there are two options here because sometimes we are handed
+    # a full instance (and could include more information), and other times we
+    # are just handed a UUID for the instance.
+    cfg.StrOpt('instance_format',
+               default='[instance: %(uuid)s] ',
+               help='If an instance is passed with the log message, format '
+                    'it like this'),
+    cfg.StrOpt('instance_uuid_format',
+               default='[instance: %(uuid)s] ',
+               help='If an instance UUID is passed with the log message, '
+                    'format it like this'),
+    ]
+
+
+generic_log_opts = [
+    cfg.StrOpt('logdir',
+               default=None,
+               help='Log output to a per-service log file in named directory'),
+    cfg.StrOpt('logfile',
+               default=None,
+               help='Log output to a named file'),
+    cfg.BoolOpt('use_stderr',
+                default=True,
+                help='Log output to standard error'),
+    cfg.StrOpt('logfile_mode',
+               default='0644',
+               help='Default file mode used when creating log files'),
+    ]
+
+
+CONF = cfg.CONF
+CONF.register_opts(generic_log_opts)
+CONF.register_opts(log_opts)
+
+# our new audit level
+# NOTE(jkoelker) Since we synthesized an audit level, make the logging
+#                module aware of it so it acts like other levels.
+logging.AUDIT = logging.INFO + 1
+logging.addLevelName(logging.AUDIT, 'AUDIT')
+
+
+try:
+    NullHandler = logging.NullHandler
+except AttributeError:  # NOTE(jkoelker) NullHandler added in Python 2.7
+    class NullHandler(logging.Handler):
+        def handle(self, record):
+            pass
+
+        def emit(self, record):
+            pass
+
+        def createLock(self):
+            self.lock = None
+
+
+def _dictify_context(context):
+    if context is None:
+        return None
+    if not isinstance(context, dict) and getattr(context, 'to_dict', None):
+        context = context.to_dict()
+    return context
+
+
+def _get_binary_name():
+    return os.path.basename(inspect.stack()[-1][1])
+
+
+def _get_log_file_path(binary=None):
+    logfile = CONF.log_file or CONF.logfile
+    logdir = CONF.log_dir or CONF.logdir
+
+    if logfile and not logdir:
+        return logfile
+
+    if logfile and logdir:
+        return os.path.join(logdir, logfile)
+
+    if logdir:
+        binary = binary or _get_binary_name()
+        return '%s.log' % (os.path.join(logdir, binary),)
+
+
+class ContextAdapter(logging.LoggerAdapter):
+    warn = logging.LoggerAdapter.warning
+
+    def __init__(self, logger, project_name, version_string):
+        self.logger = logger
+        self.project = project_name
+        self.version = version_string
+
+    def audit(self, msg, *args, **kwargs):
+        self.log(logging.AUDIT, msg, *args, **kwargs)
+
+    def process(self, msg, kwargs):
+        if 'extra' not in kwargs:
+            kwargs['extra'] = {}
+        extra = kwargs['extra']
+
+        context = kwargs.pop('context', None)
+        if not context:
+            context = getattr(local.store, 'context', None)
+        if context:
+            extra.update(_dictify_context(context))
+
+        instance = kwargs.pop('instance', None)
+        instance_extra = ''
+        if instance:
+            instance_extra = CONF.instance_format % instance
+        else:
+            instance_uuid = kwargs.pop('instance_uuid', None)
+            if instance_uuid:
+                instance_extra = (CONF.instance_uuid_format
+                                  % {'uuid': instance_uuid})
+        extra.update({'instance': instance_extra})
+
+        extra.update({"project": self.project})
+        extra.update({"version": self.version})
+        extra['extra'] = extra.copy()
+        return msg, kwargs
+
+
+class JSONFormatter(logging.Formatter):
+    def __init__(self, fmt=None, datefmt=None):
+        # NOTE(jkoelker) we ignore the fmt argument, but its still there
+        #                since logging.config.fileConfig passes it.
+        self.datefmt = datefmt
+
+    def formatException(self, ei, strip_newlines=True):
+        lines = traceback.format_exception(*ei)
+        if strip_newlines:
+            lines = [itertools.ifilter(lambda x: x,
+                                      line.rstrip().splitlines())
+                    for line in lines]
+            lines = list(itertools.chain(*lines))
+        return lines
+
+    def format(self, record):
+        message = {'message': record.getMessage(),
+                   'asctime': self.formatTime(record, self.datefmt),
+                   'name': record.name,
+                   'msg': record.msg,
+                   'args': record.args,
+                   'levelname': record.levelname,
+                   'levelno': record.levelno,
+                   'pathname': record.pathname,
+                   'filename': record.filename,
+                   'module': record.module,
+                   'lineno': record.lineno,
+                   'funcname': record.funcName,
+                   'created': record.created,
+                   'msecs': record.msecs,
+                   'relative_created': record.relativeCreated,
+                   'thread': record.thread,
+                   'thread_name': record.threadName,
+                   'process_name': record.processName,
+                   'process': record.process,
+                   'traceback': None}
+
+        if hasattr(record, 'extra'):
+            message['extra'] = record.extra
+
+        if record.exc_info:
+            message['traceback'] = self.formatException(record.exc_info)
+
+        return jsonutils.dumps(message)
+
+
+class PublishErrorsHandler(logging.Handler):
+    def emit(self, record):
+        if 'list_notifier_drivers' in CONF:
+            if ('ceilometer.openstack.common.notifier.log_notifier' in
+                CONF.list_notifier_drivers):
+                return
+        notifier.api.notify(None, 'error.publisher',
+                                 'error_notification',
+                                 notifier.api.ERROR,
+                                 dict(error=record.msg))
+
+
+def handle_exception(type, value, tb):
+    extra = {}
+    if CONF.verbose:
+        extra['exc_info'] = (type, value, tb)
+    getLogger().critical(str(value), **extra)
+
+
+def setup(product_name):
+    """Setup logging."""
+    sys.excepthook = handle_exception
+
+    if CONF.log_config:
+        try:
+            logging.config.fileConfig(CONF.log_config)
+        except Exception:
+            traceback.print_exc()
+            raise
+    else:
+        _setup_logging_from_conf(product_name)
+
+
+def _find_facility_from_conf():
+    facility_names = logging.handlers.SysLogHandler.facility_names
+    facility = getattr(logging.handlers.SysLogHandler,
+                       CONF.syslog_log_facility,
+                       None)
+
+    if facility is None and CONF.syslog_log_facility in facility_names:
+        facility = facility_names.get(CONF.syslog_log_facility)
+
+    if facility is None:
+        valid_facilities = facility_names.keys()
+        consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON',
+                  'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS',
+                  'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP',
+                  'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3',
+                  'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7']
+        valid_facilities.extend(consts)
+        raise TypeError(_('syslog facility must be one of: %s') %
+                        ', '.join("'%s'" % fac
+                                  for fac in valid_facilities))
+
+    return facility
+
+
+def _setup_logging_from_conf(product_name):
+    log_root = getLogger(product_name).logger
+    for handler in log_root.handlers:
+        log_root.removeHandler(handler)
+
+    if CONF.use_syslog:
+        facility = _find_facility_from_conf()
+        syslog = logging.handlers.SysLogHandler(address='/dev/log',
+                                                facility=facility)
+        log_root.addHandler(syslog)
+
+    logpath = _get_log_file_path()
+    if logpath:
+        filelog = logging.handlers.WatchedFileHandler(logpath)
+        log_root.addHandler(filelog)
+
+        mode = int(CONF.logfile_mode, 8)
+        st = os.stat(logpath)
+        if st.st_mode != (stat.S_IFREG | mode):
+            os.chmod(logpath, mode)
+
+    if CONF.use_stderr:
+        streamlog = ColorHandler()
+        log_root.addHandler(streamlog)
+
+    elif not CONF.log_file:
+        # pass sys.stdout as a positional argument
+        # python2.6 calls the argument strm, in 2.7 it's stream
+        streamlog = logging.StreamHandler(sys.stdout)
+        log_root.addHandler(streamlog)
+
+    if CONF.publish_errors:
+        log_root.addHandler(PublishErrorsHandler(logging.ERROR))
+
+    for handler in log_root.handlers:
+        datefmt = CONF.log_date_format
+        if CONF.log_format:
+            handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
+                                                   datefmt=datefmt))
+        handler.setFormatter(LegacyFormatter(datefmt=datefmt))
+
+    if CONF.verbose or CONF.debug:
+        log_root.setLevel(logging.DEBUG)
+    else:
+        log_root.setLevel(logging.INFO)
+
+    level = logging.NOTSET
+    for pair in CONF.default_log_levels:
+        mod, _sep, level_name = pair.partition('=')
+        level = logging.getLevelName(level_name)
+        logger = logging.getLogger(mod)
+        logger.setLevel(level)
+        for handler in log_root.handlers:
+            logger.addHandler(handler)
+
+    # NOTE(jkoelker) Clear the handlers for the root logger that was setup
+    #                by basicConfig in nova/__init__.py and install the
+    #                NullHandler.
+    root = logging.getLogger()
+    for handler in root.handlers:
+        root.removeHandler(handler)
+    handler = NullHandler()
+    handler.setFormatter(logging.Formatter())
+    root.addHandler(handler)
+
+
+_loggers = {}
+
+
+def getLogger(name='unknown', version='unknown'):
+    if name not in _loggers:
+        _loggers[name] = ContextAdapter(logging.getLogger(name),
+                                        name,
+                                        version)
+    return _loggers[name]
+
+
+class WritableLogger(object):
+    """A thin wrapper that responds to `write` and logs."""
+
+    def __init__(self, logger, level=logging.INFO):
+        self.logger = logger
+        self.level = level
+
+    def write(self, msg):
+        self.logger.log(self.level, msg)
+
+
+class LegacyFormatter(logging.Formatter):
+    """A context.RequestContext aware formatter configured through flags.
+
+    The flags used to set format strings are: logging_context_format_string
+    and logging_default_format_string.  You can also specify
+    logging_debug_format_suffix to append extra formatting if the log level is
+    debug.
+
+    For information about what variables are available for the formatter see:
+    http://docs.python.org/library/logging.html#formatter
+
+    """
+
+    def format(self, record):
+        """Uses contextstring if request_id is set, otherwise default."""
+        if 'instance' not in record.__dict__:
+            record.__dict__['instance'] = ''
+
+        if record.__dict__.get('request_id', None):
+            self._fmt = CONF.logging_context_format_string
+        else:
+            self._fmt = CONF.logging_default_format_string
+
+        if (record.levelno == logging.DEBUG and
+            CONF.logging_debug_format_suffix):
+            self._fmt += " " + CONF.logging_debug_format_suffix
+
+        # Cache this on the record, Logger will respect our formated copy
+        if record.exc_info:
+            record.exc_text = self.formatException(record.exc_info, record)
+        return logging.Formatter.format(self, record)
+
+    def formatException(self, exc_info, record=None):
+        """Format exception output with CONF.logging_exception_prefix."""
+        if not record:
+            return logging.Formatter.formatException(self, exc_info)
+
+        stringbuffer = cStringIO.StringIO()
+        traceback.print_exception(exc_info[0], exc_info[1], exc_info[2],
+                                  None, stringbuffer)
+        lines = stringbuffer.getvalue().split('\n')
+        stringbuffer.close()
+
+        if CONF.logging_exception_prefix.find('%(asctime)') != -1:
+            record.asctime = self.formatTime(record, self.datefmt)
+
+        formatted_lines = []
+        for line in lines:
+            pl = CONF.logging_exception_prefix % record.__dict__
+            fl = '%s%s' % (pl, line)
+            formatted_lines.append(fl)
+        return '\n'.join(formatted_lines)
+
+
+class ColorHandler(logging.StreamHandler):
+    LEVEL_COLORS = {
+        logging.DEBUG: '\033[00;32m',  # GREEN
+        logging.INFO: '\033[00;36m',  # CYAN
+        logging.AUDIT: '\033[01;36m',  # BOLD CYAN
+        logging.WARN: '\033[01;33m',  # BOLD YELLOW
+        logging.ERROR: '\033[01;31m',  # BOLD RED
+        logging.CRITICAL: '\033[01;31m',  # BOLD RED
+    }
+
+    def format(self, record):
+        record.color = self.LEVEL_COLORS[record.levelno]
+        return logging.StreamHandler.format(self, record)

ceilometer/openstack/common/notifier/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

ceilometer/openstack/common/notifier/api.py

@@ -0,0 +1,141 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import inspect
+import uuid
+
+from ceilometer.openstack.common import cfg
+from ceilometer.openstack.common import context
+from ceilometer.openstack.common import importutils
+from ceilometer.openstack.common import jsonutils
+from ceilometer.openstack.common import log as logging
+from ceilometer.openstack.common import timeutils
+
+
+LOG = logging.getLogger(__name__)
+
+notifier_opts = [
+    cfg.StrOpt('notification_driver',
+               default='ceilometer.openstack.common.notifier.no_op_notifier',
+               help='Default driver for sending notifications'),
+    cfg.StrOpt('default_notification_level',
+               default='INFO',
+               help='Default notification level for outgoing notifications'),
+    cfg.StrOpt('default_publisher_id',
+               default='$host',
+               help='Default publisher_id for outgoing notifications'),
+    ]
+
+CONF = cfg.CONF
+CONF.register_opts(notifier_opts)
+
+WARN = 'WARN'
+INFO = 'INFO'
+ERROR = 'ERROR'
+CRITICAL = 'CRITICAL'
+DEBUG = 'DEBUG'
+
+log_levels = (DEBUG, WARN, INFO, ERROR, CRITICAL)
+
+
+class BadPriorityException(Exception):
+    pass
+
+
+def notify_decorator(name, fn):
+    """ decorator for notify which is used from utils.monkey_patch()
+
+        :param name: name of the function
+        :param function: - object of the function
+        :returns: function -- decorated function
+
+    """
+    def wrapped_func(*args, **kwarg):
+        body = {}
+        body['args'] = []
+        body['kwarg'] = {}
+        for arg in args:
+            body['args'].append(arg)
+        for key in kwarg:
+            body['kwarg'][key] = kwarg[key]
+
+        ctxt = context.get_context_from_function_and_args(fn, args, kwarg)
+        notify(ctxt,
+               CONF.default_publisher_id,
+               name,
+               CONF.default_notification_level,
+               body)
+        return fn(*args, **kwarg)
+    return wrapped_func
+
+
+def publisher_id(service, host=None):
+    if not host:
+        host = CONF.host
+    return "%s.%s" % (service, host)
+
+
+def notify(context, publisher_id, event_type, priority, payload):
+    """Sends a notification using the specified driver
+
+    :param publisher_id: the source worker_type.host of the message
+    :param event_type:   the literal type of event (ex. Instance Creation)
+    :param priority:     patterned after the enumeration of Python logging
+                         levels in the set (DEBUG, WARN, INFO, ERROR, CRITICAL)
+    :param payload:       A python dictionary of attributes
+
+    Outgoing message format includes the above parameters, and appends the
+    following:
+
+    message_id
+      a UUID representing the id for this notification
+
+    timestamp
+      the GMT timestamp the notification was sent at
+
+    The composite message will be constructed as a dictionary of the above
+    attributes, which will then be sent via the transport mechanism defined
+    by the driver.
+
+    Message example::
+
+        {'message_id': str(uuid.uuid4()),
+         'publisher_id': 'compute.host1',
+         'timestamp': timeutils.utcnow(),
+         'priority': 'WARN',
+         'event_type': 'compute.create_instance',
+         'payload': {'instance_id': 12, ... }}
+
+    """
+    if priority not in log_levels:
+        raise BadPriorityException(
+                 _('%s not in valid priorities') % priority)
+
+    # Ensure everything is JSON serializable.
+    payload = jsonutils.to_primitive(payload, convert_instances=True)
+
+    driver = importutils.import_module(CONF.notification_driver)
+    msg = dict(message_id=str(uuid.uuid4()),
+                   publisher_id=publisher_id,
+                   event_type=event_type,
+                   priority=priority,
+                   payload=payload,
+                   timestamp=str(timeutils.utcnow()))
+    try:
+        driver.notify(context, msg)
+    except Exception, e:
+        LOG.exception(_("Problem '%(e)s' attempting to "
+                        "send to notification system. Payload=%(payload)s") %
+                        locals())

ceilometer/openstack/common/notifier/list_notifier.py

@@ -0,0 +1,116 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from ceilometer.openstack.common import cfg
+from ceilometer.openstack.common import importutils
+from ceilometer.openstack.common import log as logging
+
+
+list_notifier_drivers_opt = cfg.MultiStrOpt('list_notifier_drivers',
+        default=['ceilometer.openstack.common.notifier.no_op_notifier'],
+        help='List of drivers to send notifications')
+
+CONF = cfg.CONF
+CONF.register_opt(list_notifier_drivers_opt)
+
+LOG = logging.getLogger(__name__)
+
+drivers = None
+
+
+class ImportFailureNotifier(object):
+    """Noisily re-raises some exception over-and-over when notify is called."""
+
+    def __init__(self, exception):
+        self.exception = exception
+
+    def notify(self, context, message):
+        raise self.exception
+
+
+def _get_drivers():
+    """Instantiates and returns drivers based on the flag values."""
+    global drivers
+    if drivers is None:
+        drivers = []
+        for notification_driver in CONF.list_notifier_drivers:
+            try:
+                drivers.append(importutils.import_module(notification_driver))
+            except ImportError as e:
+                drivers.append(ImportFailureNotifier(e))
+    return drivers
+
+
+def add_driver(notification_driver):
+    """Add a notification driver at runtime."""
+    # Make sure the driver list is initialized.
+    _get_drivers()
+    if isinstance(notification_driver, basestring):
+        # Load and add
+        try:
+            drivers.append(importutils.import_module(notification_driver))
+        except ImportError as e:
+            drivers.append(ImportFailureNotifier(e))
+    else:
+        # Driver is already loaded; just add the object.
+        drivers.append(notification_driver)
+
+
+def _object_name(obj):
+    name = []
+    if hasattr(obj, '__module__'):
+        name.append(obj.__module__)
+    if hasattr(obj, '__name__'):
+        name.append(obj.__name__)
+    else:
+        name.append(obj.__class__.__name__)
+    return '.'.join(name)
+
+
+def remove_driver(notification_driver):
+    """Remove a notification driver at runtime."""
+    # Make sure the driver list is initialized.
+    _get_drivers()
+    removed = False
+    if notification_driver in drivers:
+        # We're removing an object.  Easy.
+        drivers.remove(notification_driver)
+        removed = True
+    else:
+        # We're removing a driver by name.  Search for it.
+        for driver in drivers:
+            if _object_name(driver) == notification_driver:
+                drivers.remove(driver)
+                removed = True
+
+    if not removed:
+        raise ValueError("Cannot remove; %s is not in list" %
+                         notification_driver)
+
+
+def notify(context, message):
+    """Passes notification to multiple notifiers in a list."""
+    for driver in _get_drivers():
+        try:
+            driver.notify(context, message)
+        except Exception as e:
+            LOG.exception(_("Problem '%(e)s' attempting to send to "
+                            "notification driver %(driver)s."), locals())
+
+
+def _reset_drivers():
+    """Used by unit tests to reset the drivers."""
+    global drivers
+    drivers = None

ceilometer/openstack/common/notifier/log_notifier.py

@@ -0,0 +1,35 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from ceilometer.openstack.common import cfg
+from ceilometer.openstack.common import jsonutils
+from ceilometer.openstack.common import log as logging
+
+
+CONF = cfg.CONF
+
+
+def notify(_context, message):
+    """Notifies the recipient of the desired event given the model.
+    Log notifications using openstack's default logging system"""
+
+    priority = message.get('priority',
+                           CONF.default_notification_level)
+    priority = priority.lower()
+    logger = logging.getLogger(
+        'ceilometer.openstack.common.notification.%s'
+        % message['event_type'])
+    getattr(logger, priority)(jsonutils.dumps(message))

ceilometer/openstack/common/notifier/no_op_notifier.py

@@ -0,0 +1,19 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+def notify(_context, message):
+    """Notifies the recipient of the desired event given the model"""
+    pass

ceilometer/openstack/common/notifier/rabbit_notifier.py

@@ -0,0 +1,45 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from ceilometer.openstack.common import cfg
+from ceilometer.openstack.common import context as req_context
+from ceilometer.openstack.common import log as logging
+from ceilometer.openstack.common import rpc
+
+LOG = logging.getLogger(__name__)
+
+notification_topic_opt = cfg.ListOpt('notification_topics',
+        default=['notifications', ],
+        help='AMQP topic used for openstack notifications')
+
+CONF = cfg.CONF
+CONF.register_opt(notification_topic_opt)
+
+
+def notify(context, message):
+    """Sends a notification to the RabbitMQ"""
+    if not context:
+        context = req_context.get_admin_context()
+    priority = message.get('priority',
+                           CONF.default_notification_level)
+    priority = priority.lower()
+    for topic in CONF.notification_topics:
+        topic = '%s.%s' % (topic, priority)
+        try:
+            rpc.notify(context, topic, message)
+        except Exception, e:
+            LOG.exception(_("Could not send notification to %(topic)s. "
+                            "Payload=%(message)s"), locals())

ceilometer/openstack/common/notifier/test_notifier.py

@@ -0,0 +1,22 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+NOTIFICATIONS = []
+
+
+def notify(_context, message):
+    """Test notifier, stores notifications in memory for unittests."""
+    NOTIFICATIONS.append(message)

ceilometer/openstack/common/rpc/__init__.py

@@ -47,7 +47,8 @@ rpc_opts = [
                help='Seconds to wait before a cast expires (TTL). '
                     'Only supported by impl_zmq.'),
     cfg.ListOpt('allowed_rpc_exception_modules',
-                default=['openstack.common.exception', 'nova.exception'],
+                default=['ceilometer.openstack.common.exception',
+                         'nova.exception'],
                 help='Modules of exceptions that are permitted to be recreated'
                      'upon receiving exception data from an rpc call.'),
     cfg.StrOpt('control_exchange',

ceilometer/openstack/common/rpc/dispatcher.py

@@ -92,14 +92,20 @@ class RpcDispatcher(object):
         if not version:
             version = '1.0'
 
+        had_compatible = False
         for proxyobj in self.callbacks:
             if hasattr(proxyobj, 'RPC_API_VERSION'):
                 rpc_api_version = proxyobj.RPC_API_VERSION
             else:
                 rpc_api_version = '1.0'
+            is_compatible = self._is_compatible(rpc_api_version, version)
+            had_compatible = had_compatible or is_compatible
             if not hasattr(proxyobj, method):
                 continue
-            if self._is_compatible(rpc_api_version, version):
+            if is_compatible:
                 return getattr(proxyobj, method)(ctxt, **kwargs)
 
+        if had_compatible:
+            raise AttributeError("No such RPC function '%s'" % method)
+        else:
             raise rpc_common.UnsupportedRpcVersion(version=version)

ceilometer/openstack/common/rpc/impl_fake.py

@@ -18,11 +18,11 @@ queues.  Casts will block, but this is very useful for tests.
 """
 
 import inspect
-import json
 import time
 
 import eventlet
 
+from ceilometer.openstack.common import jsonutils
 from ceilometer.openstack.common.rpc import common as rpc_common
 
 CONSUMERS = {}
@@ -121,7 +121,7 @@ def create_connection(conf, new=True):
 
 def check_serialize(msg):
     """Make sure a message intended for rpc can be serialized."""
-    json.dumps(msg)
+    jsonutils.dumps(msg)
 
 
 def multicall(conf, context, topic, msg, timeout=None):

ceilometer/openstack/common/rpc/impl_zmq.py

@@ -14,8 +14,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
 import pprint
+import socket
 import string
 import sys
 import types
@@ -28,6 +28,7 @@ import greenlet
 from ceilometer.openstack.common import cfg
 from ceilometer.openstack.common.gettextutils import _
 from ceilometer.openstack.common import importutils
+from ceilometer.openstack.common import jsonutils
 from ceilometer.openstack.common.rpc import common as rpc_common
 
 
@@ -47,7 +48,8 @@ zmq_opts = [
 
     # The module.Class to use for matchmaking.
     cfg.StrOpt('rpc_zmq_matchmaker',
-               default='openstack.common.rpc.matchmaker.MatchMakerLocalhost',
+               default='ceilometer.'
+               'openstack.common.rpc.matchmaker.MatchMakerLocalhost',
                help='MatchMaker driver'),
 
     # The following port is unassigned by IANA as of 2012-05-21
@@ -59,6 +61,9 @@ zmq_opts = [
 
     cfg.StrOpt('rpc_zmq_ipc_dir', default='/var/run/openstack',
                help='Directory for holding IPC sockets'),
+    cfg.StrOpt('rpc_zmq_host', default=socket.gethostname(),
+               help='Name of this node. Must be a valid hostname, FQDN, or '
+                    'IP address')
 ]
 
 
@@ -76,7 +81,7 @@ def _serialize(data):
     Error if a developer passes us bad data.
     """
     try:
-        return str(json.dumps(data, ensure_ascii=True))
+        return str(jsonutils.dumps(data, ensure_ascii=True))
     except TypeError:
         LOG.error(_("JSON serialization failed."))
         raise
@@ -87,7 +92,7 @@ def _deserialize(data):
     Deserialization wrapper
     """
     LOG.debug(_("Deserializing: %s"), data)
-    return json.loads(data)
+    return jsonutils.loads(data)
 
 
 class ZmqSocket(object):
@@ -119,11 +124,12 @@ class ZmqSocket(object):
         for f in do_sub:
             self.subscribe(f)
 
-        LOG.debug(_("Connecting to %{addr}s with %{type}s"
+        str_data = {'addr': addr, 'type': self.socket_s(),
-                    "\n-> Subscribed to %{subscribe}s"
+                    'subscribe': subscribe, 'bind': bind}
-                    "\n-> bind: %{bind}s"),
+
-                  {'addr': addr, 'type': self.socket_s(),
+        LOG.debug(_("Connecting to %(addr)s with %(type)s"), str_data)
-                   'subscribe': subscribe, 'bind': bind})
+        LOG.debug(_("-> Subscribed to %(subscribe)s"),  str_data)
+        LOG.debug(_("-> bind: %(bind)s"), str_data)
 
         try:
             if bind:
@@ -542,8 +548,7 @@ def _call(addr, context, msg_id, topic, msg, timeout=None):
     msg_id = str(uuid.uuid4().hex)
 
     # Replies always come into the reply service.
-    # We require that FLAGS.host is a FQDN, IP, or resolvable hostname.
+    reply_topic = "zmq_replies.%s" % FLAGS.rpc_zmq_host
-    reply_topic = "zmq_replies.%s" % FLAGS.host
 
     LOG.debug(_("Creating payload"))
     # Curry the original request into a reply method.
@@ -707,7 +712,7 @@ def register_opts(conf):
         if mm_path[-1][0] not in string.ascii_uppercase:
             LOG.error(_("Matchmaker could not be loaded.\n"
                       "rpc_zmq_matchmaker is not a class."))
-            raise
+            raise RPCException(_("Error loading Matchmaker."))
 
         mm_impl = importutils.import_module(mm_module)
         mm_constructor = getattr(mm_impl, mm_class)

ceilometer/publish.py

@@ -18,9 +18,8 @@
 """Publish a counter using the preferred RPC mechanism.
 """
 
-from ceilometer import log
+from ceilometer.openstack.common import log
 from ceilometer.openstack.common import rpc
-
 from ceilometer.openstack.common import cfg
 from ceilometer import meter
 

ceilometer/service.py

@@ -19,7 +19,7 @@
 
 from nova import flags
 
-from ceilometer import log
+from ceilometer.openstack.common import log
 from ceilometer.openstack.common import cfg
 
 cfg.CONF.register_opts([
@@ -36,4 +36,4 @@ def prepare_service(argv=[]):
     # still using the Service object out of nova directly. We need to
     # move that into openstack.common.
     flags.FLAGS(argv[1:])
-    log.setup()
+    log.setup('ceilometer')

ceilometer/storage/__init__.py

@@ -20,7 +20,7 @@
 
 import pkg_resources
 
-from ceilometer import log
+from ceilometer.openstack.common import log
 from ceilometer.openstack.common import cfg
 
 LOG = log.getLogger(__name__)

ceilometer/storage/base.py

@@ -20,7 +20,7 @@
 
 import abc
 
-from ceilometer import log
+from ceilometer.openstack.common import log
 
 LOG = log.getLogger(__name__)
 

ceilometer/storage/impl_log.py

@@ -18,7 +18,7 @@
 """Simple logging storage backend.
 """
 
-from ceilometer import log
+from ceilometer.openstack.common import log
 from ceilometer.storage import base
 
 LOG = log.getLogger(__name__)

ceilometer/storage/impl_mongodb.py

@@ -20,7 +20,7 @@
 
 import datetime
 
-from ceilometer import log
+from ceilometer.openstack.common import log
 from ceilometer.openstack.common import cfg
 from ceilometer.storage import base
 

openstack-common.conf

@@ -1,3 +1,3 @@
 [DEFAULT]
-modules=cfg,iniparser,rpc,importutils,excutils,local,jsonutils,gettextutils,timeutils
+modules=cfg,iniparser,rpc,importutils,excutils,local,jsonutils,gettextutils,timeutils,notifier,context,log
 base=ceilometer

tests/test_log.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-#
-# Copyright © 2012 eNovance <licensing@enovance.com>
-#
-# Author: Julien Danjou <julien@danjou.info>
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import unittest
-import logging
-
-from ceilometer import log
-from ceilometer.openstack.common import cfg
-
-
-class LoggerTestCase(unittest.TestCase):
-    def setUp(self):
-        super(LoggerTestCase, self).setUp()
-        self.log = log.getLogger()
-
-    def test_log_level(self):
-        cfg.CONF.log_level = "info"
-        log.setup()
-        self.assertEqual(logging.INFO, self.log.getEffectiveLevel())
-
-    def test_child_log_level(self):
-        cfg.CONF.log_level = "info"
-        log.setup()
-        self.assertEqual(logging.INFO, log.getLogger('ceilometer.foobar').getEffectiveLevel())
-
-
-class LogfilePathTestCase(unittest.TestCase):
-    def test_log_path_logdir(self):
-        cfg.CONF.log_dir = '/some/path'
-        cfg.CONF.log_file = None
-        self.assertEquals(log._get_log_file_path(binary='foo-bar'),
-                          '/some/path/foo-bar.log')
-
-    def test_log_path_logfile(self):
-        cfg.CONF.log_file = '/some/path/foo-bar.log'
-        self.assertEquals(log._get_log_file_path(binary='foo-bar'),
-                          '/some/path/foo-bar.log')
-
-    def test_log_path_none(self):
-        cfg.CONF.log_dir = None
-        cfg.CONF.log_file = None
-        self.assertTrue(log._get_log_file_path(binary='foo-bar') is None)
-
-    def test_log_path_logfile_overrides_logdir(self):
-        cfg.CONF.log_dir = '/some/other/path'
-        cfg.CONF.log_file = '/some/path/foo-bar.log'
-        self.assertEquals(log._get_log_file_path(binary='foo-bar'),
-                          '/some/path/foo-bar.log')