From 0e7eb412876bf75703add76bde0e4490aa1f5d34 Mon Sep 17 00:00:00 2001 From: Shuangtai Tian Date: Tue, 18 Feb 2014 18:40:40 +0800 Subject: [PATCH] Sync common modules from olso and remove py3kcompat Module py3kcompat was removed from oslo-incubator, use six directly. Change-Id: I4011680b1f7438a4cf5e0da453fe3d04ae460da8 --- marconi/openstack/common/cache/cache.py | 9 +- marconi/openstack/common/gettextutils.py | 37 +++++- marconi/openstack/common/importutils.py | 7 + marconi/openstack/common/lockutils.py | 23 +++- marconi/openstack/common/log.py | 120 +++++++++++++++--- .../openstack/common/py3kcompat/__init__.py | 0 .../openstack/common/py3kcompat/urlutils.py | 67 ---------- 7 files changed, 167 insertions(+), 96 deletions(-) delete mode 100644 marconi/openstack/common/py3kcompat/__init__.py delete mode 100644 marconi/openstack/common/py3kcompat/urlutils.py diff --git a/marconi/openstack/common/cache/cache.py b/marconi/openstack/common/cache/cache.py index 297172cac..ab29b9a60 100644 --- a/marconi/openstack/common/cache/cache.py +++ b/marconi/openstack/common/cache/cache.py @@ -20,10 +20,9 @@ Supported configuration options: `key_namespace`: Namespace under which keys will be created. """ +from six.moves.urllib import parse from stevedore import driver -from marconi.openstack.common.py3kcompat import urlutils - def _get_olso_configs(): """Returns the oslo.config options to register.""" @@ -36,7 +35,7 @@ def _get_olso_configs(): return [ cfg.StrOpt('cache_url', default='memory://', - help='Url to connect to the cache backend.') + help='URL to connect to the cache back end.') ] @@ -58,7 +57,7 @@ def get_cache(url='memory://'): :param conf: Configuration instance to use """ - parsed = urlutils.urlparse(url) + parsed = parse.urlparse(url) backend = parsed.scheme query = parsed.query @@ -69,7 +68,7 @@ def get_cache(url='memory://'): # http://hg.python.org/cpython/rev/79e6ff3d9afd if not query and '?' in parsed.path: query = parsed.path.split('?', 1)[-1] - parameters = urlutils.parse_qsl(query) + parameters = parse.parse_qsl(query) kwargs = {'options': dict(parameters)} mgr = driver.DriverManager('marconi.openstack.common.cache.backends', backend, diff --git a/marconi/openstack/common/gettextutils.py b/marconi/openstack/common/gettextutils.py index 5eb86bab3..f3c0bf5c9 100644 --- a/marconi/openstack/common/gettextutils.py +++ b/marconi/openstack/common/gettextutils.py @@ -23,6 +23,7 @@ Usual usage in an openstack.common module: """ import copy +import functools import gettext import locale from logging import handlers @@ -35,6 +36,17 @@ import six _localedir = os.environ.get('marconi'.upper() + '_LOCALEDIR') _t = gettext.translation('marconi', localedir=_localedir, fallback=True) +# We use separate translation catalogs for each log level, so set up a +# mapping between the log level name and the translator. The domain +# for the log level is project_name + "-log-" + log_level so messages +# for each level end up in their own catalog. +_t_log_levels = dict( + (level, gettext.translation('marconi' + '-log-' + level, + localedir=_localedir, + fallback=True)) + for level in ['info', 'warning', 'error', 'critical'] +) + _AVAILABLE_LANGUAGES = {} USE_LAZY = False @@ -60,6 +72,28 @@ def _(msg): return _t.ugettext(msg) +def _log_translation(msg, level): + """Build a single translation of a log message + """ + if USE_LAZY: + return Message(msg, domain='marconi' + '-log-' + level) + else: + translator = _t_log_levels[level] + if six.PY3: + return translator.gettext(msg) + return translator.ugettext(msg) + +# Translators for log levels. +# +# The abbreviated names are meant to reflect the usual use of a short +# name like '_'. The "L" is for "log" and the other letter comes from +# the level. +_LI = functools.partial(_log_translation, level='info') +_LW = functools.partial(_log_translation, level='warning') +_LE = functools.partial(_log_translation, level='error') +_LC = functools.partial(_log_translation, level='critical') + + def install(domain, lazy=False): """Install a _() function using the given translation domain. @@ -118,7 +152,8 @@ class Message(six.text_type): and can be treated as such. """ - def __new__(cls, msgid, msgtext=None, params=None, domain='marconi', *args): + def __new__(cls, msgid, msgtext=None, params=None, + domain='marconi', *args): """Create a new Message object. In order for translation to work gettext requires a message ID, this diff --git a/marconi/openstack/common/importutils.py b/marconi/openstack/common/importutils.py index 4fd9ae2bc..081f8a389 100644 --- a/marconi/openstack/common/importutils.py +++ b/marconi/openstack/common/importutils.py @@ -58,6 +58,13 @@ def import_module(import_str): return sys.modules[import_str] +def import_versioned_module(version, submodule=None): + module = 'marconi.v%s' % version + if submodule: + module = '.'.join((module, submodule)) + return import_module(module) + + def try_import(import_str, default=None): """Try to import a module and if it fails return default.""" try: diff --git a/marconi/openstack/common/lockutils.py b/marconi/openstack/common/lockutils.py index 594367de8..ee69be400 100644 --- a/marconi/openstack/common/lockutils.py +++ b/marconi/openstack/common/lockutils.py @@ -74,7 +74,7 @@ class _InterProcessLock(object): self.lockfile = None self.fname = name - def __enter__(self): + def acquire(self): basedir = os.path.dirname(self.fname) if not os.path.exists(basedir): @@ -91,23 +91,36 @@ class _InterProcessLock(object): # to have a laughable 10 attempts "blocking" mechanism. self.trylock() LOG.debug(_('Got file lock "%s"'), self.fname) - return self + return True except IOError as e: if e.errno in (errno.EACCES, errno.EAGAIN): # external locks synchronise things like iptables # updates - give it some time to prevent busy spinning time.sleep(0.01) else: - raise + raise threading.ThreadError(_("Unable to acquire lock on" + " `%(filename)s` due to" + " %(exception)s") % + { + 'filename': self.fname, + 'exception': e, + }) - def __exit__(self, exc_type, exc_val, exc_tb): + def __enter__(self): + self.acquire() + return self + + def release(self): try: self.unlock() self.lockfile.close() + LOG.debug(_('Released file lock "%s"'), self.fname) except IOError: LOG.exception(_("Could not release the acquired lock `%s`"), self.fname) - LOG.debug(_('Released file lock "%s"'), self.fname) + + def __exit__(self, exc_type, exc_val, exc_tb): + self.release() def trylock(self): raise NotImplementedError() diff --git a/marconi/openstack/common/log.py b/marconi/openstack/common/log.py index 7aa637e85..9e058935b 100644 --- a/marconi/openstack/common/log.py +++ b/marconi/openstack/common/log.py @@ -115,10 +115,21 @@ logging_cli_opts = [ '--log-file paths'), cfg.BoolOpt('use-syslog', default=False, - help='Use syslog for logging.'), + help='Use syslog for logging. ' + 'Existing syslog format is DEPRECATED during I, ' + 'and then will be changed in J to honor RFC5424'), + cfg.BoolOpt('use-syslog-rfc-format', + # TODO(bogdando) remove or use True after existing + # syslog format deprecation in J + default=False, + help='(Optional) Use syslog rfc5424 format for logging. ' + 'If enabled, will add APP-NAME (RFC5424) before the ' + 'MSG part of the syslog message. The old format ' + 'without APP-NAME is deprecated in I, ' + 'and will be removed in J.'), cfg.StrOpt('syslog-log-facility', default='LOG_USER', - help='syslog facility to receive log lines') + help='Syslog facility to receive log lines') ] generic_log_opts = [ @@ -132,18 +143,18 @@ log_opts = [ default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' '%(name)s [%(request_id)s %(user_identity)s] ' '%(instance)s%(message)s', - help='format string to use for log messages with context'), + help='Format string to use for log messages with context'), cfg.StrOpt('logging_default_format_string', default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' '%(name)s [-] %(instance)s%(message)s', - help='format string to use for log messages without context'), + help='Format string to use for log messages without context'), cfg.StrOpt('logging_debug_format_suffix', default='%(funcName)s %(pathname)s:%(lineno)d', - help='data to append to log format when level is DEBUG'), + help='Data to append to log format when level is DEBUG'), cfg.StrOpt('logging_exception_prefix', default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s ' '%(instance)s', - help='prefix each line of exception output with this format'), + help='Prefix each line of exception output with this format'), cfg.ListOpt('default_log_levels', default=[ 'amqp=WARN', @@ -153,14 +164,15 @@ log_opts = [ 'sqlalchemy=WARN', 'suds=INFO', 'iso8601=WARN', + 'requests.packages.urllib3.connectionpool=WARN' ], - help='list of logger=LEVEL pairs'), + help='List of logger=LEVEL pairs'), cfg.BoolOpt('publish_errors', default=False, - help='publish error events'), + help='Publish error events'), cfg.BoolOpt('fatal_deprecations', default=False, - help='make deprecations fatal'), + help='Make deprecations fatal'), # NOTE(mikal): there are two options here because sometimes we are handed # a full instance (and could include more information), and other times we @@ -292,18 +304,39 @@ class ContextAdapter(BaseLoggerAdapter): self.logger = logger self.project = project_name self.version = version_string + self._deprecated_messages_sent = dict() @property def handlers(self): return self.logger.handlers def deprecated(self, msg, *args, **kwargs): + """Call this method when a deprecated feature is used. + + If the system is configured for fatal deprecations then the message + is logged at the 'critical' level and :class:`DeprecatedConfig` will + be raised. + + Otherwise, the message will be logged (once) at the 'warn' level. + + :raises: :class:`DeprecatedConfig` if the system is configured for + fatal deprecations. + + """ stdmsg = _("Deprecated: %s") % msg if CONF.fatal_deprecations: self.critical(stdmsg, *args, **kwargs) raise DeprecatedConfig(msg=stdmsg) - else: - self.warn(stdmsg, *args, **kwargs) + + # Using a list because a tuple with dict can't be stored in a set. + sent_args = self._deprecated_messages_sent.setdefault(msg, list()) + + if args in sent_args: + # Already logged this message, so don't log it again. + return + + sent_args.append(args) + self.warn(stdmsg, *args, **kwargs) def process(self, msg, kwargs): # NOTE(mrodden): catch any Message/other object and @@ -420,12 +453,12 @@ def _load_log_config(log_config_append): raise LogConfigError(log_config_append, str(exc)) -def setup(product_name): +def setup(product_name, version='unknown'): """Setup logging.""" if CONF.log_config_append: _load_log_config(CONF.log_config_append) else: - _setup_logging_from_conf() + _setup_logging_from_conf(product_name, version) sys.excepthook = _create_logging_excepthook(product_name) @@ -459,15 +492,32 @@ def _find_facility_from_conf(): return facility -def _setup_logging_from_conf(): +class RFCSysLogHandler(logging.handlers.SysLogHandler): + def __init__(self, *args, **kwargs): + self.binary_name = _get_binary_name() + super(RFCSysLogHandler, self).__init__(*args, **kwargs) + + def format(self, record): + msg = super(RFCSysLogHandler, self).format(record) + msg = self.binary_name + ' ' + msg + return msg + + +def _setup_logging_from_conf(project, version): log_root = getLogger(None).logger for handler in log_root.handlers: log_root.removeHandler(handler) if CONF.use_syslog: facility = _find_facility_from_conf() - syslog = logging.handlers.SysLogHandler(address='/dev/log', - facility=facility) + # TODO(bogdando) use the format provided by RFCSysLogHandler + # after existing syslog format deprecation in J + if CONF.use_syslog_rfc_format: + syslog = RFCSysLogHandler(address='/dev/log', + facility=facility) + else: + syslog = logging.handlers.SysLogHandler(address='/dev/log', + facility=facility) log_root.addHandler(syslog) logpath = _get_log_file_path() @@ -501,7 +551,9 @@ def _setup_logging_from_conf(): log_root.info('Deprecated: log_format is now deprecated and will ' 'be removed in the next release') else: - handler.setFormatter(ContextFormatter(datefmt=datefmt)) + handler.setFormatter(ContextFormatter(project=project, + version=version, + datefmt=datefmt)) if CONF.debug: log_root.setLevel(logging.DEBUG) @@ -545,7 +597,7 @@ class WritableLogger(object): self.level = level def write(self, msg): - self.logger.log(self.level, msg) + self.logger.log(self.level, msg.rstrip()) class ContextFormatter(logging.Formatter): @@ -559,10 +611,42 @@ class ContextFormatter(logging.Formatter): For information about what variables are available for the formatter see: http://docs.python.org/library/logging.html#formatter + If available, uses the context value stored in TLS - local.store.context + """ + def __init__(self, *args, **kwargs): + """Initialize ContextFormatter instance + + Takes additional keyword arguments which can be used in the message + format string. + + :keyword project: project name + :type project: string + :keyword version: project version + :type version: string + + """ + + self.project = kwargs.pop('project', 'unknown') + self.version = kwargs.pop('version', 'unknown') + + logging.Formatter.__init__(self, *args, **kwargs) + def format(self, record): """Uses contextstring if request_id is set, otherwise default.""" + + # store project info + record.project = self.project + record.version = self.version + + # store request info + context = getattr(local.store, 'context', None) + if context: + d = _dictify_context(context) + for k, v in d.items(): + setattr(record, k, v) + # NOTE(sdague): default the fancier formatting params # to an empty string so we don't throw an exception if # they get used diff --git a/marconi/openstack/common/py3kcompat/__init__.py b/marconi/openstack/common/py3kcompat/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/marconi/openstack/common/py3kcompat/urlutils.py b/marconi/openstack/common/py3kcompat/urlutils.py deleted file mode 100644 index 84e457a44..000000000 --- a/marconi/openstack/common/py3kcompat/urlutils.py +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright 2013 Canonical Ltd. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -""" -Python2/Python3 compatibility layer for OpenStack -""" - -import six - -if six.PY3: - # python3 - import urllib.error - import urllib.parse - import urllib.request - - urlencode = urllib.parse.urlencode - urljoin = urllib.parse.urljoin - quote = urllib.parse.quote - quote_plus = urllib.parse.quote_plus - parse_qsl = urllib.parse.parse_qsl - unquote = urllib.parse.unquote - unquote_plus = urllib.parse.unquote_plus - urlparse = urllib.parse.urlparse - urlsplit = urllib.parse.urlsplit - urlunsplit = urllib.parse.urlunsplit - SplitResult = urllib.parse.SplitResult - - urlopen = urllib.request.urlopen - URLError = urllib.error.URLError - pathname2url = urllib.request.pathname2url -else: - # python2 - import urllib - import urllib2 - import urlparse - - urlencode = urllib.urlencode - quote = urllib.quote - quote_plus = urllib.quote_plus - unquote = urllib.unquote - unquote_plus = urllib.unquote_plus - - parse = urlparse - parse_qsl = parse.parse_qsl - urljoin = parse.urljoin - urlparse = parse.urlparse - urlsplit = parse.urlsplit - urlunsplit = parse.urlunsplit - SplitResult = parse.SplitResult - - urlopen = urllib2.urlopen - URLError = urllib2.URLError - pathname2url = urllib.pathname2url