From 733f28113304241380e1050539c7dba72cf0e6e4 Mon Sep 17 00:00:00 2001 From: Gordon Chung Date: Tue, 4 Feb 2014 17:09:12 -0500 Subject: [PATCH] sync oslo common code sync code with commit-id: 5e5a51b3ddc25b3242e8f4f4b5e5357942618662 Change-Id: If62f3bba14cdc74c78685b2fdf5aa769c4158a50 --- .../openstack/common/config/generator.py | 7 ++-- .../openstack/common/db/sqlalchemy/session.py | 19 +++++---- .../common/db/sqlalchemy/test_migrations.py | 2 +- .../openstack/common/db/sqlalchemy/utils.py | 3 +- ceilometer/openstack/common/lockutils.py | 2 +- ceilometer/openstack/common/log.py | 42 +++++++++++++++++-- ceilometer/openstack/common/policy.py | 7 +++- ceilometer/openstack/common/service.py | 17 ++++---- tools/config/generate_sample.sh | 20 +++++++-- 9 files changed, 88 insertions(+), 31 deletions(-) diff --git a/ceilometer/openstack/common/config/generator.py b/ceilometer/openstack/common/config/generator.py index 8b2ae6518..cb2e06d2a 100644 --- a/ceilometer/openstack/common/config/generator.py +++ b/ceilometer/openstack/common/config/generator.py @@ -65,6 +65,7 @@ def generate(argv): parser = argparse.ArgumentParser( description='generate sample configuration file', ) + parser.add_argument('-m', dest='modules', action='append') parser.add_argument('-l', dest='libraries', action='append') parser.add_argument('srcfiles', nargs='*') parsed_args = parser.parse_args(argv) @@ -84,10 +85,8 @@ def generate(argv): # The options list is a list of (module, options) tuples opts_by_group = {'DEFAULT': []} - extra_modules = os.getenv("CEILOMETER_CONFIG_GENERATOR_EXTRA_MODULES", "") - if extra_modules: - for module_name in extra_modules.split(','): - module_name = module_name.strip() + if parsed_args.modules: + for module_name in parsed_args.modules: module = _import_module(module_name) if module: for group, opts in _list_opts(module): diff --git a/ceilometer/openstack/common/db/sqlalchemy/session.py b/ceilometer/openstack/common/db/sqlalchemy/session.py index ce407816b..213070255 100644 --- a/ceilometer/openstack/common/db/sqlalchemy/session.py +++ b/ceilometer/openstack/common/db/sqlalchemy/session.py @@ -271,6 +271,7 @@ Efficient use of soft deletes: """ import functools +import logging import os.path import re import time @@ -285,7 +286,6 @@ from sqlalchemy.sql.expression import literal_column from ceilometer.openstack.common.db import exception from ceilometer.openstack.common.gettextutils import _ -from ceilometer.openstack.common import log as logging from ceilometer.openstack.common import timeutils sqlite_db_opts = [ @@ -737,13 +737,16 @@ def create_engine(sql_connection, sqlite_fk=False, if engine.name in ['mysql', 'ibm_db_sa']: callback = functools.partial(_ping_listener, engine) sqlalchemy.event.listen(engine, 'checkout', callback) - if mysql_traditional_mode: - sqlalchemy.event.listen(engine, 'checkout', _set_mode_traditional) - else: - LOG.warning(_("This application has not enabled MySQL traditional" - " mode, which means silent data corruption may" - " occur. Please encourage the application" - " developers to enable this mode.")) + if engine.name == 'mysql': + if mysql_traditional_mode: + sqlalchemy.event.listen(engine, 'checkout', + _set_mode_traditional) + else: + LOG.warning(_("This application has not enabled MySQL " + "traditional mode, which means silent " + "data corruption may occur. " + "Please encourage the application " + "developers to enable this mode.")) elif 'sqlite' in connection_dict.drivername: if not CONF.sqlite_synchronous: sqlalchemy.event.listen(engine, 'connect', diff --git a/ceilometer/openstack/common/db/sqlalchemy/test_migrations.py b/ceilometer/openstack/common/db/sqlalchemy/test_migrations.py index 124b9cc42..767a0fa60 100644 --- a/ceilometer/openstack/common/db/sqlalchemy/test_migrations.py +++ b/ceilometer/openstack/common/db/sqlalchemy/test_migrations.py @@ -15,6 +15,7 @@ # under the License. import functools +import logging import os import subprocess @@ -25,7 +26,6 @@ import sqlalchemy.exc from ceilometer.openstack.common.db.sqlalchemy import utils from ceilometer.openstack.common.gettextutils import _ -from ceilometer.openstack.common import log as logging from ceilometer.openstack.common.py3kcompat import urlutils from ceilometer.openstack.common import test diff --git a/ceilometer/openstack/common/db/sqlalchemy/utils.py b/ceilometer/openstack/common/db/sqlalchemy/utils.py index 08d64ede8..c141c7de3 100644 --- a/ceilometer/openstack/common/db/sqlalchemy/utils.py +++ b/ceilometer/openstack/common/db/sqlalchemy/utils.py @@ -16,6 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. +import logging import re from migrate.changeset import UniqueConstraint @@ -37,8 +38,6 @@ from sqlalchemy import Table from sqlalchemy.types import NullType from ceilometer.openstack.common.gettextutils import _ - -from ceilometer.openstack.common import log as logging from ceilometer.openstack.common import timeutils diff --git a/ceilometer/openstack/common/lockutils.py b/ceilometer/openstack/common/lockutils.py index e2db5b3ae..4db095a90 100644 --- a/ceilometer/openstack/common/lockutils.py +++ b/ceilometer/openstack/common/lockutils.py @@ -114,10 +114,10 @@ class _InterProcessLock(object): try: self.unlock() self.lockfile.close() + LOG.debug(_('Released file lock "%s"'), self.fname) except IOError: LOG.exception(_("Could not release the acquired lock `%s`"), self.fname) - LOG.debug(_('Released file lock "%s"'), self.fname) def __exit__(self, exc_type, exc_val, exc_tb): self.release() diff --git a/ceilometer/openstack/common/log.py b/ceilometer/openstack/common/log.py index 37b09a834..4205e65b2 100644 --- a/ceilometer/openstack/common/log.py +++ b/ceilometer/openstack/common/log.py @@ -432,12 +432,12 @@ def _load_log_config(log_config_append): raise LogConfigError(log_config_append, str(exc)) -def setup(product_name): +def setup(product_name, version='unknown'): """Setup logging.""" if CONF.log_config_append: _load_log_config(CONF.log_config_append) else: - _setup_logging_from_conf() + _setup_logging_from_conf(product_name, version) sys.excepthook = _create_logging_excepthook(product_name) @@ -482,7 +482,7 @@ class RFCSysLogHandler(logging.handlers.SysLogHandler): return msg -def _setup_logging_from_conf(): +def _setup_logging_from_conf(project, version): log_root = getLogger(None).logger for handler in log_root.handlers: log_root.removeHandler(handler) @@ -530,7 +530,9 @@ def _setup_logging_from_conf(): log_root.info('Deprecated: log_format is now deprecated and will ' 'be removed in the next release') else: - handler.setFormatter(ContextFormatter(datefmt=datefmt)) + handler.setFormatter(ContextFormatter(project=project, + version=version, + datefmt=datefmt)) if CONF.debug: log_root.setLevel(logging.DEBUG) @@ -588,10 +590,42 @@ class ContextFormatter(logging.Formatter): For information about what variables are available for the formatter see: http://docs.python.org/library/logging.html#formatter + If available, uses the context value stored in TLS - local.store.context + """ + def __init__(self, *args, **kwargs): + """Initialize ContextFormatter instance + + Takes additional keyword arguments which can be used in the message + format string. + + :keyword project: project name + :type project: string + :keyword version: project version + :type version: string + + """ + + self.project = kwargs.pop('project', 'unknown') + self.version = kwargs.pop('version', 'unknown') + + logging.Formatter.__init__(self, *args, **kwargs) + def format(self, record): """Uses contextstring if request_id is set, otherwise default.""" + + # store project info + record.project = self.project + record.version = self.version + + # store request info + context = getattr(local.store, 'context', None) + if context: + d = _dictify_context(context) + for k, v in d.items(): + setattr(record, k, v) + # NOTE(sdague): default the fancier formatting params # to an empty string so we don't throw an exception if # they get used diff --git a/ceilometer/openstack/common/policy.py b/ceilometer/openstack/common/policy.py index 23ab9e9ab..ae83ba517 100644 --- a/ceilometer/openstack/common/policy.py +++ b/ceilometer/openstack/common/policy.py @@ -119,11 +119,16 @@ class Rules(dict): # If the default rule isn't actually defined, do something # reasonably intelligent - if not self.default_rule or self.default_rule not in self: + if not self.default_rule: raise KeyError(key) if isinstance(self.default_rule, BaseCheck): return self.default_rule + + # We need to check this or we can get infinite recursion + if self.default_rule not in self: + raise KeyError(key) + elif isinstance(self.default_rule, six.string_types): return self[self.default_rule] diff --git a/ceilometer/openstack/common/service.py b/ceilometer/openstack/common/service.py index 1ef002cfa..3a5cbc48b 100644 --- a/ceilometer/openstack/common/service.py +++ b/ceilometer/openstack/common/service.py @@ -23,7 +23,6 @@ import os import random import signal import sys -import threading import time try: @@ -35,6 +34,7 @@ except ImportError: UnsupportedOperation = None import eventlet +from eventlet import event from oslo.config import cfg from ceilometer.openstack.common import eventlet_backdoor @@ -419,10 +419,11 @@ class Service(object): self.tg = threadgroup.ThreadGroup(threads) # signal that the service is done shutting itself down: - self._done = threading.Event() + self._done = event.Event() def reset(self): - self._done = threading.Event() + # NOTE(Fengqian): docs for Event.reset() recommend against using it + self._done = event.Event() def start(self): pass @@ -431,7 +432,8 @@ class Service(object): self.tg.stop() self.tg.wait() # Signal that service cleanup is done: - self._done.set() + if not self._done.ready(): + self._done.send() def wait(self): self._done.wait() @@ -442,7 +444,7 @@ class Services(object): def __init__(self): self.services = [] self.tg = threadgroup.ThreadGroup() - self.done = threading.Event() + self.done = event.Event() def add(self, service): self.services.append(service) @@ -456,7 +458,8 @@ class Services(object): # Each service has performed cleanup, now signal that the run_service # wrapper threads can now die: - self.done.set() + if not self.done.ready(): + self.done.send() # reap threads: self.tg.stop() @@ -466,7 +469,7 @@ class Services(object): def restart(self): self.stop() - self.done = threading.Event() + self.done = event.Event() for restart_service in self.services: restart_service.reset() self.tg.add_thread(self.run_service, restart_service, self.done) diff --git a/tools/config/generate_sample.sh b/tools/config/generate_sample.sh index 521effd61..5ccfc399d 100755 --- a/tools/config/generate_sample.sh +++ b/tools/config/generate_sample.sh @@ -4,8 +4,8 @@ print_hint() { echo "Try \`${0##*/} --help' for more information." >&2 } -PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:l:o: \ - --long help,base-dir:,package-name:,output-dir:,library: -- "$@") +PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \ + --long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@") if [ $? != 0 ] ; then print_hint ; exit 1 ; fi @@ -21,6 +21,7 @@ while true; do echo "-b, --base-dir=DIR project base directory" echo "-p, --package-name=NAME project package name" echo "-o, --output-dir=DIR file output directory" + echo "-m, --module=MOD extra python module to interrogate for options" echo "-l, --library=LIB extra library that registers options for discovery" exit 0 ;; @@ -39,6 +40,11 @@ while true; do OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'` shift ;; + -m|--module) + shift + MODULES="$MODULES -m $1" + shift + ;; -l|--library) shift LIBRARIES="$LIBRARIES -l $1" @@ -89,6 +95,14 @@ then source "$RC_FILE" fi +for mod in ${CEILOMETER_CONFIG_GENERATOR_EXTRA_MODULES}; do + MODULES="$MODULES -m $mod" +done + +for lib in ${CEILOMETER_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do + LIBRARIES="$LIBRARIES -l $lib" +done + export EVENTLET_NO_GREENDNS=yes OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs) @@ -96,7 +110,7 @@ OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs) DEFAULT_MODULEPATH=ceilometer.openstack.common.config.generator MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH} OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample -python -m $MODULEPATH $LIBRARIES $FILES > $OUTPUTFILE +python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE # Hook to allow projects to append custom config file snippets CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)