diff --git a/oslo/messaging/openstack/common/excutils.py b/oslo/messaging/openstack/common/excutils.py index 6d9bb8486..7ad06d046 100644 --- a/oslo/messaging/openstack/common/excutils.py +++ b/oslo/messaging/openstack/common/excutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # Copyright 2012, Red Hat, Inc. # @@ -24,6 +22,8 @@ import sys import time import traceback +import six + from oslo.messaging.openstack.common.gettextutils import _ # noqa @@ -42,13 +42,13 @@ class save_and_reraise_exception(object): In some cases the caller may not want to re-raise the exception, and for those circumstances this context provides a reraise flag that - can be used to suppress the exception. For example: + can be used to suppress the exception. For example:: - except Exception: - with save_and_reraise_exception() as ctxt: - decide_if_need_reraise() - if not should_be_reraised: - ctxt.reraise = False + except Exception: + with save_and_reraise_exception() as ctxt: + decide_if_need_reraise() + if not should_be_reraised: + ctxt.reraise = False """ def __init__(self): self.reraise = True @@ -65,7 +65,7 @@ class save_and_reraise_exception(object): self.tb)) return False if self.reraise: - raise self.type_, self.value, self.tb + six.reraise(self.type_, self.value, self.tb) def forever_retry_uncaught_exceptions(infunc): @@ -77,7 +77,8 @@ def forever_retry_uncaught_exceptions(infunc): try: return infunc(*args, **kwargs) except Exception as exc: - if exc.message == last_exc_message: + this_exc_message = six.u(str(exc)) + if this_exc_message == last_exc_message: exc_count += 1 else: exc_count = 1 @@ -85,12 +86,12 @@ def forever_retry_uncaught_exceptions(infunc): # the exception message changes cur_time = int(time.time()) if (cur_time - last_log_time > 60 or - exc.message != last_exc_message): + this_exc_message != last_exc_message): logging.exception( _('Unexpected exception occurred %d time(s)... ' 'retrying.') % exc_count) last_log_time = cur_time - last_exc_message = exc.message + last_exc_message = this_exc_message exc_count = 0 # This should be a very rare event. In case it isn't, do # a sleep. diff --git a/oslo/messaging/openstack/common/fixture/moxstubout.py b/oslo/messaging/openstack/common/fixture/moxstubout.py index f277fdd73..e8c031f08 100644 --- a/oslo/messaging/openstack/common/fixture/moxstubout.py +++ b/oslo/messaging/openstack/common/fixture/moxstubout.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2013 Hewlett-Packard Development Company, L.P. @@ -19,7 +17,6 @@ import fixtures import mox -import stubout class MoxStubout(fixtures.Fixture): @@ -30,8 +27,6 @@ class MoxStubout(fixtures.Fixture): # emulate some of the mox stuff, we can't use the metaclass # because it screws with our generators self.mox = mox.Mox() - self.stubs = stubout.StubOutForTesting() + self.stubs = self.mox.stubs self.addCleanup(self.mox.UnsetStubs) - self.addCleanup(self.stubs.UnsetAll) - self.addCleanup(self.stubs.SmartUnsetAll) self.addCleanup(self.mox.VerifyAll) diff --git a/oslo/messaging/openstack/common/gettextutils.py b/oslo/messaging/openstack/common/gettextutils.py index 49755c0ce..ec432f4ef 100644 --- a/oslo/messaging/openstack/common/gettextutils.py +++ b/oslo/messaging/openstack/common/gettextutils.py @@ -1,8 +1,6 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Red Hat, Inc. -# All Rights Reserved. # Copyright 2013 IBM Corp. +# All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -21,27 +19,51 @@ gettext for openstack-common modules. Usual usage in an openstack.common module: - from oslo.messaging.openstack.common.gettextutils import _ + from oslo.messaging.openstack.common.gettextutils import _ # noqa """ import copy import gettext -import logging.handlers +import logging import os import re -import UserString +try: + import UserString as _userString +except ImportError: + import collections as _userString +from babel import localedata import six _localedir = os.environ.get('oslo.messaging'.upper() + '_LOCALEDIR') _t = gettext.translation('oslo.messaging', localedir=_localedir, fallback=True) +_AVAILABLE_LANGUAGES = {} +USE_LAZY = False + + +def enable_lazy(): + """Convenience function for configuring _() to use lazy gettext + + Call this at the start of execution to enable the gettextutils._ + function to use lazy gettext functionality. This is useful if + your project is importing _ directly instead of using the + gettextutils.install() way of importing the _ function. + """ + global USE_LAZY + USE_LAZY = True + def _(msg): - return _t.ugettext(msg) + if USE_LAZY: + return Message(msg, 'oslo.messaging') + else: + if six.PY3: + return _t.gettext(msg) + return _t.ugettext(msg) -def install(domain): +def install(domain, lazy=False): """Install a _() function using the given translation domain. Given a translation domain, install a _() function using gettext's @@ -51,52 +73,60 @@ def install(domain): overriding the default localedir (e.g. /usr/share/locale) using a translation-domain-specific environment variable (e.g. NOVA_LOCALEDIR). + + :param domain: the translation domain + :param lazy: indicates whether or not to install the lazy _() function. + The lazy _() introduces a way to do deferred translation + of messages by installing a _ that builds Message objects, + instead of strings, which can then be lazily translated into + any available locale. """ - gettext.install(domain, - localedir=os.environ.get(domain.upper() + '_LOCALEDIR'), - unicode=True) + if lazy: + # NOTE(mrodden): Lazy gettext functionality. + # + # The following introduces a deferred way to do translations on + # messages in OpenStack. We override the standard _() function + # and % (format string) operation to build Message objects that can + # later be translated when we have more information. + # + # Also included below is an example LocaleHandler that translates + # Messages to an associated locale, effectively allowing many logs, + # each with their own locale. + + def _lazy_gettext(msg): + """Create and return a Message object. + + Lazy gettext function for a given domain, it is a factory method + for a project/module to get a lazy gettext function for its own + translation domain (i.e. nova, glance, cinder, etc.) + + Message encapsulates a string so that we can translate + it later when needed. + """ + return Message(msg, domain) + + from six import moves + moves.builtins.__dict__['_'] = _lazy_gettext + else: + localedir = '%s_LOCALEDIR' % domain.upper() + if six.PY3: + gettext.install(domain, + localedir=os.environ.get(localedir)) + else: + gettext.install(domain, + localedir=os.environ.get(localedir), + unicode=True) -""" -Lazy gettext functionality. - -The following is an attempt to introduce a deferred way -to do translations on messages in OpenStack. We attempt to -override the standard _() function and % (format string) operation -to build Message objects that can later be translated when we have -more information. Also included is an example LogHandler that -translates Messages to an associated locale, effectively allowing -many logs, each with their own locale. -""" - - -def get_lazy_gettext(domain): - """Assemble and return a lazy gettext function for a given domain. - - Factory method for a project/module to get a lazy gettext function - for its own translation domain (i.e. nova, glance, cinder, etc.) - """ - - def _lazy_gettext(msg): - """Create and return a Message object. - - Message encapsulates a string so that we can translate it later when - needed. - """ - return Message(msg, domain) - - return _lazy_gettext - - -class Message(UserString.UserString, object): +class Message(_userString.UserString, object): """Class used to encapsulate translatable messages.""" def __init__(self, msg, domain): # _msg is the gettext msgid and should never change self._msg = msg self._left_extra_msg = '' self._right_extra_msg = '' + self._locale = None self.params = None - self.locale = None self.domain = domain @property @@ -116,8 +146,13 @@ class Message(UserString.UserString, object): localedir=localedir, fallback=True) + if six.PY3: + ugettext = lang.gettext + else: + ugettext = lang.ugettext + full_msg = (self._left_extra_msg + - lang.ugettext(self._msg) + + ugettext(self._msg) + self._right_extra_msg) if self.params is not None: @@ -125,12 +160,39 @@ class Message(UserString.UserString, object): return six.text_type(full_msg) + @property + def locale(self): + return self._locale + + @locale.setter + def locale(self, value): + self._locale = value + if not self.params: + return + + # This Message object may have been constructed with one or more + # Message objects as substitution parameters, given as a single + # Message, or a tuple or Map containing some, so when setting the + # locale for this Message we need to set it for those Messages too. + if isinstance(self.params, Message): + self.params.locale = value + return + if isinstance(self.params, tuple): + for param in self.params: + if isinstance(param, Message): + param.locale = value + return + if isinstance(self.params, dict): + for param in self.params.values(): + if isinstance(param, Message): + param.locale = value + def _save_dictionary_parameter(self, dict_param): full_msg = self.data # look for %(blah) fields in string; # ignore %% and deal with the # case where % is first character on the line - keys = re.findall('(?:[^%]|^)%\((\w*)\)[a-z]', full_msg) + keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', full_msg) # if we don't find any %(blah) blocks but have a %s if not keys and re.findall('(?:[^%]|^)%[a-z]', full_msg): @@ -143,7 +205,7 @@ class Message(UserString.UserString, object): params[key] = copy.deepcopy(dict_param[key]) except TypeError: # cast uncopyable thing to unicode string - params[key] = unicode(dict_param[key]) + params[key] = six.text_type(dict_param[key]) return params @@ -162,7 +224,7 @@ class Message(UserString.UserString, object): try: self.params = copy.deepcopy(other) except TypeError: - self.params = unicode(other) + self.params = six.text_type(other) return self @@ -171,11 +233,13 @@ class Message(UserString.UserString, object): return self.data def __str__(self): + if six.PY3: + return self.__unicode__() return self.data.encode('utf-8') def __getstate__(self): to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg', - 'domain', 'params', 'locale'] + 'domain', 'params', '_locale'] new_dict = self.__dict__.fromkeys(to_copy) for attr in to_copy: new_dict[attr] = copy.deepcopy(self.__dict__[attr]) @@ -229,7 +293,55 @@ class Message(UserString.UserString, object): if name in ops: return getattr(self.data, name) else: - return UserString.UserString.__getattribute__(self, name) + return _userString.UserString.__getattribute__(self, name) + + +def get_available_languages(domain): + """Lists the available languages for the given translation domain. + + :param domain: the domain to get languages for + """ + if domain in _AVAILABLE_LANGUAGES: + return copy.copy(_AVAILABLE_LANGUAGES[domain]) + + localedir = '%s_LOCALEDIR' % domain.upper() + find = lambda x: gettext.find(domain, + localedir=os.environ.get(localedir), + languages=[x]) + + # NOTE(mrodden): en_US should always be available (and first in case + # order matters) since our in-line message strings are en_US + language_list = ['en_US'] + # NOTE(luisg): Babel <1.0 used a function called list(), which was + # renamed to locale_identifiers() in >=1.0, the requirements master list + # requires >=0.9.6, uncapped, so defensively work with both. We can remove + # this check when the master list updates to >=1.0, and update all projects + list_identifiers = (getattr(localedata, 'list', None) or + getattr(localedata, 'locale_identifiers')) + locale_identifiers = list_identifiers() + for i in locale_identifiers: + if find(i) is not None: + language_list.append(i) + _AVAILABLE_LANGUAGES[domain] = language_list + return copy.copy(language_list) + + +def get_localized_message(message, user_locale): + """Gets a localized version of the given message in the given locale. + + If the message is not a Message object the message is returned as-is. + If the locale is None the message is translated to the default locale. + + :returns: the translated message in unicode, or the original message if + it could not be translated + """ + translated = message + if isinstance(message, Message): + original_locale = message.locale + message.locale = user_locale + translated = six.text_type(message) + message.locale = original_locale + return translated class LocaleHandler(logging.Handler): diff --git a/oslo/messaging/openstack/common/importutils.py b/oslo/messaging/openstack/common/importutils.py index 7a303f93f..4fd9ae2bc 100644 --- a/oslo/messaging/openstack/common/importutils.py +++ b/oslo/messaging/openstack/common/importutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/oslo/messaging/openstack/common/jsonutils.py b/oslo/messaging/openstack/common/jsonutils.py index 261326b95..24f3784d8 100644 --- a/oslo/messaging/openstack/common/jsonutils.py +++ b/oslo/messaging/openstack/common/jsonutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2011 Justin Santa Barbara @@ -38,13 +36,23 @@ import functools import inspect import itertools import json -import types -import xmlrpclib +try: + import xmlrpclib +except ImportError: + # NOTE(jaypipes): xmlrpclib was renamed to xmlrpc.client in Python3 + # however the function and object call signatures + # remained the same. This whole try/except block should + # be removed and replaced with a call to six.moves once + # six 1.4.2 is released. See http://bit.ly/1bqrVzu + import xmlrpc.client as xmlrpclib import six +from oslo.messaging.openstack.common import gettextutils +from oslo.messaging.openstack.common import importutils from oslo.messaging.openstack.common import timeutils +netaddr = importutils.try_import("netaddr") _nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod, inspect.isfunction, inspect.isgeneratorfunction, @@ -52,7 +60,8 @@ _nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod, inspect.iscode, inspect.isbuiltin, inspect.isroutine, inspect.isabstract] -_simple_types = (types.NoneType, int, basestring, bool, float, long) +_simple_types = (six.string_types + six.integer_types + + (type(None), bool, float)) def to_primitive(value, convert_instances=False, convert_datetime=True, @@ -117,7 +126,7 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, level=level, max_depth=max_depth) if isinstance(value, dict): - return dict((k, recursive(v)) for k, v in value.iteritems()) + return dict((k, recursive(v)) for k, v in six.iteritems(value)) elif isinstance(value, (list, tuple)): return [recursive(lv) for lv in value] @@ -129,6 +138,8 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, if convert_datetime and isinstance(value, datetime.datetime): return timeutils.strtime(value) + elif isinstance(value, gettextutils.Message): + return value.data elif hasattr(value, 'iteritems'): return recursive(dict(value.iteritems()), level=level + 1) elif hasattr(value, '__iter__'): @@ -137,6 +148,8 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, # Likely an instance of something. Watch for cycles. # Ignore class member vars. return recursive(value.__dict__, level=level + 1) + elif netaddr and isinstance(value, netaddr.IPAddress): + return six.text_type(value) else: if any(test(value) for test in _nasty_type_tests): return six.text_type(value) diff --git a/oslo/messaging/openstack/common/network_utils.py b/oslo/messaging/openstack/common/network_utils.py index dbed1ceb4..401763838 100644 --- a/oslo/messaging/openstack/common/network_utils.py +++ b/oslo/messaging/openstack/common/network_utils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 OpenStack Foundation. # All Rights Reserved. # @@ -19,7 +17,7 @@ Network-related utilities and helper functions. """ -import urlparse +from oslo.messaging.openstack.common.py3kcompat import urlutils def parse_host_port(address, default_port=None): @@ -72,10 +70,10 @@ def urlsplit(url, scheme='', allow_fragments=True): The parameters are the same as urlparse.urlsplit. """ - scheme, netloc, path, query, fragment = urlparse.urlsplit( + scheme, netloc, path, query, fragment = urlutils.urlsplit( url, scheme, allow_fragments) if allow_fragments and '#' in path: path, fragment = path.split('#', 1) if '?' in path: path, query = path.split('?', 1) - return urlparse.SplitResult(scheme, netloc, path, query, fragment) + return urlutils.SplitResult(scheme, netloc, path, query, fragment) diff --git a/oslo/messaging/openstack/common/sslutils.py b/oslo/messaging/openstack/common/sslutils.py index 5181289a6..11df3e5ec 100644 --- a/oslo/messaging/openstack/common/sslutils.py +++ b/oslo/messaging/openstack/common/sslutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2013 IBM Corp. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/oslo/messaging/openstack/common/timeutils.py b/oslo/messaging/openstack/common/timeutils.py index ac2441bcb..d8cf5395b 100644 --- a/oslo/messaging/openstack/common/timeutils.py +++ b/oslo/messaging/openstack/common/timeutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -21,8 +19,10 @@ Time related utilities and helper functions. import calendar import datetime +import time import iso8601 +import six # ISO 8601 extended time format with microseconds @@ -48,9 +48,9 @@ def parse_isotime(timestr): try: return iso8601.parse_date(timestr) except iso8601.ParseError as e: - raise ValueError(e.message) + raise ValueError(six.text_type(e)) except TypeError as e: - raise ValueError(e.message) + raise ValueError(six.text_type(e)) def strtime(at=None, fmt=PERFECT_TIME_FORMAT): @@ -75,20 +75,31 @@ def normalize_time(timestamp): def is_older_than(before, seconds): """Return True if before is older than seconds.""" - if isinstance(before, basestring): + if isinstance(before, six.string_types): before = parse_strtime(before).replace(tzinfo=None) + else: + before = before.replace(tzinfo=None) + return utcnow() - before > datetime.timedelta(seconds=seconds) def is_newer_than(after, seconds): """Return True if after is newer than seconds.""" - if isinstance(after, basestring): + if isinstance(after, six.string_types): after = parse_strtime(after).replace(tzinfo=None) + else: + after = after.replace(tzinfo=None) + return after - utcnow() > datetime.timedelta(seconds=seconds) def utcnow_ts(): """Timestamp version of our utcnow function.""" + if utcnow.override_time is None: + # NOTE(kgriffs): This is several times faster + # than going through calendar.timegm(...) + return int(time.time()) + return calendar.timegm(utcnow().timetuple()) @@ -110,12 +121,15 @@ def iso8601_from_timestamp(timestamp): utcnow.override_time = None -def set_time_override(override_time=datetime.datetime.utcnow()): +def set_time_override(override_time=None): """Overrides utils.utcnow. Make it return a constant time or a list thereof, one at a time. + + :param override_time: datetime instance or list thereof. If not + given, defaults to the current UTC time. """ - utcnow.override_time = override_time + utcnow.override_time = override_time or datetime.datetime.utcnow() def advance_time_delta(timedelta): @@ -168,6 +182,15 @@ def delta_seconds(before, after): datetime objects (as a float, to microsecond resolution). """ delta = after - before + return total_seconds(delta) + + +def total_seconds(delta): + """Return the total seconds of datetime.timedelta object. + + Compute total seconds of datetime.timedelta, datetime.timedelta + doesn't have method total_seconds in Python2.6, calculate it manually. + """ try: return delta.total_seconds() except AttributeError: diff --git a/requirements.txt b/requirements.txt index 135718159..5fe5fdfd6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,6 @@ six>=1.4.1 # FIXME(markmc): remove this when the drivers no longer # import eventlet eventlet>=0.13.0 + +# used by openstack/common/gettextutils.py +Babel>=1.3