x/vmware-nsx
Code Issues Proposed changes

Updated fileutils and its dependencies

Browse Source 
This is to avoid fileutils dependency on excutils that are now moved to
oslo.utils.

The following changes are included:

* neutron/openstack/common/__init__.py
  6b048e7 Let oslotest manage the six.move setting for mox

 * neutron/openstack/common/_i18n.py
  9ce1d96 Fix i18n import
  5d40e14 Remove code that moved to oslo.i18n

 * neutron/openstack/common/fileutils.py
  6ff6b4b Switch oslo-incubator to use oslo.utils and remove old modules
  2b966f9 Fix deletion of cached file for policy enforcer
  9c88dc3 file_open: fixed docstring to refer to open() instead of file()
  6c7407b fileutils: port to Python 3
  fcf517d Update oslo log messages with translation domains

 * neutron/openstack/common/log.py
  6c706c5 Delete graduated serialization files
  5d40e14 Remove code that moved to oslo.i18n
  6ff6b4b Switch oslo-incubator to use oslo.utils and remove old modules
  aa74411 log: add missing space in error message
  037dee0 Set stevedore log level to WARN by default
  37c0091 Add unicode coercion of logged messages to ContextFormatter
  6614413 Correct coercion of logged message to unicode
  1188d88 Except socket.error if syslog isn't running
  ac995be Fix E126 pep8 errors
  631f880 Set keystonemiddleware and routes.middleware to log on WARN level
  726d00a Adjust oslo logging to provide adapter is enabled for
  433fa0b Make logging_context_format_string optional in log.set_defaults
  ac92c06 Add default log level for websocket
  5fd77eb Ability to customize default_log_levels for each project
  4d9328c Python 3: enable tests/unit/test_log.py
  cb5a804 Move `mask_password` to strutils

Note: cb5a804 is partially included; that's ok because we don't use the
moved function in Neutron.

Change-Id: I3bfcaff2620b368d807e9468bb7abc01d6471661
This commit is contained in:
Ihar Hrachyshka 2014-10-14 14:22:09 +02:00
parent 14818ee060
commit 90efdc25ac
5 changed files with 131 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
neutron/openstack/common/__init__.py
View File

@ -1,17 +0,0 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))

40
neutron/openstack/common/_i18n.py Normal file
View File

@ -0,0 +1,40 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""oslo.i18n integration module.
See http://docs.openstack.org/developer/oslo.i18n/usage.html
"""
import oslo.i18n
# NOTE(dhellmann): This reference to o-s-l-o will be replaced by the
# application name when this module is synced into the separate
# repository. It is OK to have more than one translation function
# using the same domain, since there will still only be one message
# catalog.
_translators = oslo.i18n.TranslatorFactory(domain='neutron')
# The primary translation function using the well-known name "_"
_ = _translators.primary
# Translators for log levels.
#
# The abbreviated names are meant to reflect the usual use of a short
# name like '_'. The "L" is for "log" and the other letter comes from
# the level.
_LI = _translators.log_info
_LW = _translators.log_warning
_LE = _translators.log_error
_LC = _translators.log_critical

26
neutron/openstack/common/fileutils.py
View File

@ -13,14 +13,13 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import contextlib import contextlib
import errno import errno
import os import os
import tempfile import tempfile
from neutron.openstack.common import excutils from oslo.utils import excutils
from neutron.openstack.common.gettextutils import _
from neutron.openstack.common import log as logging from neutron.openstack.common import log as logging
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
@ -52,15 +51,15 @@ def read_cached_file(filename, force_reload=False):
""" """
global _FILE_CACHE global _FILE_CACHE
if force_reload and filename in _FILE_CACHE: if force_reload:
del _FILE_CACHE[filename] delete_cached_file(filename)
reloaded = False reloaded = False
mtime = os.path.getmtime(filename) mtime = os.path.getmtime(filename)
cache_info = _FILE_CACHE.setdefault(filename, {}) cache_info = _FILE_CACHE.setdefault(filename, {})
if not cache_info or mtime > cache_info.get('mtime', 0): if not cache_info or mtime > cache_info.get('mtime', 0):
LOG.debug(_("Reloading cached file %s") % filename) LOG.debug("Reloading cached file %s" % filename)
with open(filename) as fap: with open(filename) as fap:
cache_info['data'] = fap.read() cache_info['data'] = fap.read()
cache_info['mtime'] = mtime cache_info['mtime'] = mtime
@ -68,6 +67,17 @@ def read_cached_file(filename, force_reload=False):
return (reloaded, cache_info['data']) return (reloaded, cache_info['data'])
def delete_cached_file(filename):
"""Delete cached file if present.
:param filename: filename to delete
"""
global _FILE_CACHE
if filename in _FILE_CACHE:
del _FILE_CACHE[filename]
def delete_if_exists(path, remove=os.unlink): def delete_if_exists(path, remove=os.unlink):
"""Delete a file, but ignore file not found error. """Delete a file, but ignore file not found error.
@ -101,13 +111,13 @@ def remove_path_on_error(path, remove=delete_if_exists):
def file_open(*args, **kwargs): def file_open(*args, **kwargs):
"""Open file """Open file
see built-in file() documentation for more details see built-in open() documentation for more details
Note: The reason this is kept in a separate module is to easily Note: The reason this is kept in a separate module is to easily
be able to provide a stub module that doesn't alter system be able to provide a stub module that doesn't alter system
state at all (for unit tests) state at all (for unit tests)
""" """
return file(*args, **kwargs) return open(*args, **kwargs)
def write_to_tempfile(content, path=None, suffix='', prefix='tmp'): def write_to_tempfile(content, path=None, suffix='', prefix='tmp'):

162
neutron/openstack/common/log.py
View File

@ -33,42 +33,24 @@ import logging
import logging.config import logging.config
import logging.handlers import logging.handlers
import os import os
import re import socket
import sys import sys
import traceback import traceback
from oslo.config import cfg from oslo.config import cfg
from oslo.serialization import jsonutils
from oslo.utils import importutils
import six import six
from six import moves from six import moves
from neutron.openstack.common.gettextutils import _ _PY26 = sys.version_info[0:2] == (2, 6)
from neutron.openstack.common import importutils
from neutron.openstack.common import jsonutils from neutron.openstack.common._i18n import _
from neutron.openstack.common import local from neutron.openstack.common import local
_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
# NOTE(ldbragst): Let's build a list of regex objects using the list of
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
# to the list of _SANITIZE_KEYS and we can generate regular expressions
# for XML and JSON automatically.
_SANITIZE_PATTERNS = []
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
r'(<%(key)s>).*?(</%(key)s>)',
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
'.*?([\'"])',
r'(%(key)s\s*--?[A-z]+\s*).*?([\s])']
for key in _SANITIZE_KEYS:
for pattern in _FORMAT_PATTERNS:
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
_SANITIZE_PATTERNS.append(reg_ex)
common_cli_opts = [ common_cli_opts = [
cfg.BoolOpt('debug', cfg.BoolOpt('debug',
@ -138,6 +120,14 @@ generic_log_opts = [
help='Log output to standard error.') help='Log output to standard error.')
] ]
DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
'oslo.messaging=INFO', 'iso8601=WARN',
'requests.packages.urllib3.connectionpool=WARN',
'urllib3.connectionpool=WARN', 'websocket=WARN',
"keystonemiddleware=WARN", "routes.middleware=WARN",
"stevedore=WARN"]
log_opts = [ log_opts = [
cfg.StrOpt('logging_context_format_string', cfg.StrOpt('logging_context_format_string',
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
@ -156,17 +146,7 @@ log_opts = [
'%(instance)s', '%(instance)s',
help='Prefix each line of exception output with this format.'), help='Prefix each line of exception output with this format.'),
cfg.ListOpt('default_log_levels', cfg.ListOpt('default_log_levels',
default=[ default=DEFAULT_LOG_LEVELS,
'amqp=WARN',
'amqplib=WARN',
'boto=WARN',
'qpid=WARN',
'sqlalchemy=WARN',
'suds=INFO',
'oslo.messaging=INFO',
'iso8601=WARN',
'requests.packages.urllib3.connectionpool=WARN'
],
help='List of logger=LEVEL pairs.'), help='List of logger=LEVEL pairs.'),
cfg.BoolOpt('publish_errors', cfg.BoolOpt('publish_errors',
default=False, default=False,
@ -181,11 +161,11 @@ log_opts = [
cfg.StrOpt('instance_format', cfg.StrOpt('instance_format',
default='[instance: %(uuid)s] ', default='[instance: %(uuid)s] ',
help='The format for an instance that is passed with the log ' help='The format for an instance that is passed with the log '
'message. '), 'message.'),
cfg.StrOpt('instance_uuid_format', cfg.StrOpt('instance_uuid_format',
default='[instance: %(uuid)s] ', default='[instance: %(uuid)s] ',
help='The format for an instance UUID that is passed with the ' help='The format for an instance UUID that is passed with the '
'log message. '), 'log message.'),
] ]
CONF = cfg.CONF CONF = cfg.CONF
@ -244,45 +224,20 @@ def _get_log_file_path(binary=None):
return None return None
def mask_password(message, secret="***"):
"""Replace password with 'secret' in message.
:param message: The string which includes security information.
:param secret: value with which to replace passwords.
:returns: The unicode value of message with the password fields masked.
For example:
>>> mask_password("'adminPass' : 'aaaaa'")
"'adminPass' : '***'"
>>> mask_password("'admin_pass' : 'aaaaa'")
"'admin_pass' : '***'"
>>> mask_password('"password" : "aaaaa"')
'"password" : "***"'
>>> mask_password("'original_password' : 'aaaaa'")
"'original_password' : '***'"
>>> mask_password("u'original_password' : u'aaaaa'")
"u'original_password' : u'***'"
"""
message = six.text_type(message)
# NOTE(ldbragst): Check to see if anything in message contains any key
# specified in _SANITIZE_KEYS, if not then just return the message since
# we don't have to mask any passwords.
if not any(key in message for key in _SANITIZE_KEYS):
return message
secret = r'\g<1>' + secret + r'\g<2>'
for pattern in _SANITIZE_PATTERNS:
message = re.sub(pattern, secret, message)
return message
class BaseLoggerAdapter(logging.LoggerAdapter): class BaseLoggerAdapter(logging.LoggerAdapter):
def audit(self, msg, *args, **kwargs): def audit(self, msg, *args, **kwargs):
self.log(logging.AUDIT, msg, *args, **kwargs) self.log(logging.AUDIT, msg, *args, **kwargs)
def isEnabledFor(self, level):
if _PY26:
# This method was added in python 2.7 (and it does the exact
# same logic, so we need to do the exact same logic so that
# python 2.6 has this capability as well).
return self.logger.isEnabledFor(level)
else:
return super(BaseLoggerAdapter, self).isEnabledFor(level)
class LazyAdapter(BaseLoggerAdapter): class LazyAdapter(BaseLoggerAdapter):
def __init__(self, name='unknown', version='unknown'): def __init__(self, name='unknown', version='unknown'):
@ -295,6 +250,11 @@ class LazyAdapter(BaseLoggerAdapter):
def logger(self): def logger(self):
if not self._logger: if not self._logger:
self._logger = getLogger(self.name, self.version) self._logger = getLogger(self.name, self.version)
if six.PY3:
# In Python 3, the code fails because the 'manager' attribute
# cannot be found when using a LoggerAdapter as the
# underlying logger. Work around this issue.
self._logger.manager = self._logger.logger.manager
return self._logger return self._logger
@ -340,11 +300,10 @@ class ContextAdapter(BaseLoggerAdapter):
self.warn(stdmsg, *args, **kwargs) self.warn(stdmsg, *args, **kwargs)
def process(self, msg, kwargs): def process(self, msg, kwargs):
# NOTE(mrodden): catch any Message/other object and # NOTE(jecarey): If msg is not unicode, coerce it into unicode
# coerce to unicode before they can get # before it can get to the python logging and
# to the python logging and possibly # possibly cause string encoding trouble
# cause string encoding trouble if not isinstance(msg, six.text_type):
if not isinstance(msg, six.string_types):
msg = six.text_type(msg) msg = six.text_type(msg)
if 'extra' not in kwargs: if 'extra' not in kwargs:
@ -448,7 +407,7 @@ def _load_log_config(log_config_append):
try: try:
logging.config.fileConfig(log_config_append, logging.config.fileConfig(log_config_append,
disable_existing_loggers=False) disable_existing_loggers=False)
except moves.configparser.Error as exc: except (moves.configparser.Error, KeyError) as exc:
raise LogConfigError(log_config_append, six.text_type(exc)) raise LogConfigError(log_config_append, six.text_type(exc))
@ -461,9 +420,20 @@ def setup(product_name, version='unknown'):
sys.excepthook = _create_logging_excepthook(product_name) sys.excepthook = _create_logging_excepthook(product_name)
def set_defaults(logging_context_format_string): def set_defaults(logging_context_format_string=None,
cfg.set_defaults( default_log_levels=None):
log_opts, logging_context_format_string=logging_context_format_string) # Just in case the caller is not setting the
# default_log_level. This is insurance because
# we introduced the default_log_level parameter
# later in a backwards in-compatible change
if default_log_levels is not None:
cfg.set_defaults(
log_opts,
default_log_levels=default_log_levels)
if logging_context_format_string is not None:
cfg.set_defaults(
log_opts,
logging_context_format_string=logging_context_format_string)
def _find_facility_from_conf(): def _find_facility_from_conf():
@ -512,18 +482,6 @@ def _setup_logging_from_conf(project, version):
for handler in log_root.handlers: for handler in log_root.handlers:
log_root.removeHandler(handler) log_root.removeHandler(handler)
if CONF.use_syslog:
facility = _find_facility_from_conf()
# TODO(bogdando) use the format provided by RFCSysLogHandler
# after existing syslog format deprecation in J
if CONF.use_syslog_rfc_format:
syslog = RFCSysLogHandler(address='/dev/log',
facility=facility)
else:
syslog = logging.handlers.SysLogHandler(address='/dev/log',
facility=facility)
log_root.addHandler(syslog)
logpath = _get_log_file_path() logpath = _get_log_file_path()
if logpath: if logpath:
filelog = logging.handlers.WatchedFileHandler(logpath) filelog = logging.handlers.WatchedFileHandler(logpath)
@ -582,6 +540,20 @@ def _setup_logging_from_conf(project, version):
else: else:
logger.setLevel(level_name) logger.setLevel(level_name)
if CONF.use_syslog:
try:
facility = _find_facility_from_conf()
# TODO(bogdando) use the format provided by RFCSysLogHandler
# after existing syslog format deprecation in J
if CONF.use_syslog_rfc_format:
syslog = RFCSysLogHandler(facility=facility)
else:
syslog = logging.handlers.SysLogHandler(facility=facility)
log_root.addHandler(syslog)
except socket.error:
log_root.error('Unable to add syslog handler. Verify that syslog '
'is running.')
_loggers = {} _loggers = {}
@ -651,6 +623,12 @@ class ContextFormatter(logging.Formatter):
def format(self, record): def format(self, record):
"""Uses contextstring if request_id is set, otherwise default.""" """Uses contextstring if request_id is set, otherwise default."""
# NOTE(jecarey): If msg is not unicode, coerce it into unicode
# before it can get to the python logging and
# possibly cause string encoding trouble
if not isinstance(record.msg, six.text_type):
record.msg = six.text_type(record.msg)
# store project info # store project info
record.project = self.project record.project = self.project
record.version = self.version record.version = self.version

3
requirements.txt
View File

@ -27,7 +27,10 @@ six>=1.7.0
stevedore>=1.0.0 # Apache-2.0 stevedore>=1.0.0 # Apache-2.0
oslo.config>=1.4.0 # Apache-2.0 oslo.config>=1.4.0 # Apache-2.0
oslo.db>=1.0.0 # Apache-2.0 oslo.db>=1.0.0 # Apache-2.0
oslo.i18n>=1.0.0 # Apache-2.0
oslo.messaging>=1.4.0 oslo.messaging>=1.4.0
oslo.rootwrap>=1.3.0 oslo.rootwrap>=1.3.0
oslo.serialization>=1.0.0 # Apache-2.0
oslo.utils>=1.0.0 # Apache-2.0
python-novaclient>=2.18.0 python-novaclient>=2.18.0