sync oslo common code

sync code with commit-id: 5e5a51b3ddc25b3242e8f4f4b5e5357942618662

Change-Id: If62f3bba14cdc74c78685b2fdf5aa769c4158a50

ceilometer/openstack/common/config/generator.py

@@ -65,6 +65,7 @@ def generate(argv):
     parser = argparse.ArgumentParser(
         description='generate sample configuration file',
     )
+    parser.add_argument('-m', dest='modules', action='append')
     parser.add_argument('-l', dest='libraries', action='append')
     parser.add_argument('srcfiles', nargs='*')
     parsed_args = parser.parse_args(argv)
@@ -84,10 +85,8 @@ def generate(argv):
     # The options list is a list of (module, options) tuples
     opts_by_group = {'DEFAULT': []}
 
-    extra_modules = os.getenv("CEILOMETER_CONFIG_GENERATOR_EXTRA_MODULES", "")
+    if parsed_args.modules:
-    if extra_modules:
+        for module_name in parsed_args.modules:
-        for module_name in extra_modules.split(','):
-            module_name = module_name.strip()
             module = _import_module(module_name)
             if module:
                 for group, opts in _list_opts(module):

ceilometer/openstack/common/db/sqlalchemy/session.py

@@ -271,6 +271,7 @@ Efficient use of soft deletes:
 """
 
 import functools
+import logging
 import os.path
 import re
 import time
@@ -285,7 +286,6 @@ from sqlalchemy.sql.expression import literal_column
 
 from ceilometer.openstack.common.db import exception
 from ceilometer.openstack.common.gettextutils import _
-from ceilometer.openstack.common import log as logging
 from ceilometer.openstack.common import timeutils
 
 sqlite_db_opts = [
@@ -737,13 +737,16 @@ def create_engine(sql_connection, sqlite_fk=False,
     if engine.name in ['mysql', 'ibm_db_sa']:
         callback = functools.partial(_ping_listener, engine)
         sqlalchemy.event.listen(engine, 'checkout', callback)
+        if engine.name == 'mysql':
             if mysql_traditional_mode:
-            sqlalchemy.event.listen(engine, 'checkout', _set_mode_traditional)
+                sqlalchemy.event.listen(engine, 'checkout',
+                                        _set_mode_traditional)
             else:
-            LOG.warning(_("This application has not enabled MySQL traditional"
+                LOG.warning(_("This application has not enabled MySQL "
-                          " mode, which means silent data corruption may"
+                              "traditional mode, which means silent "
-                          " occur. Please encourage the application"
+                              "data corruption may occur. "
-                          " developers to enable this mode."))
+                              "Please encourage the application "
+                              "developers to enable this mode."))
     elif 'sqlite' in connection_dict.drivername:
         if not CONF.sqlite_synchronous:
             sqlalchemy.event.listen(engine, 'connect',

ceilometer/openstack/common/db/sqlalchemy/test_migrations.py

@@ -15,6 +15,7 @@
 #    under the License.
 
 import functools
+import logging
 import os
 import subprocess
 
@@ -25,7 +26,6 @@ import sqlalchemy.exc
 
 from ceilometer.openstack.common.db.sqlalchemy import utils
 from ceilometer.openstack.common.gettextutils import _
-from ceilometer.openstack.common import log as logging
 from ceilometer.openstack.common.py3kcompat import urlutils
 from ceilometer.openstack.common import test
 

ceilometer/openstack/common/db/sqlalchemy/utils.py

@@ -16,6 +16,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import logging
 import re
 
 from migrate.changeset import UniqueConstraint
@@ -37,8 +38,6 @@ from sqlalchemy import Table
 from sqlalchemy.types import NullType
 
 from ceilometer.openstack.common.gettextutils import _
-
-from ceilometer.openstack.common import log as logging
 from ceilometer.openstack.common import timeutils
 
 

ceilometer/openstack/common/lockutils.py

@@ -114,10 +114,10 @@ class _InterProcessLock(object):
         try:
             self.unlock()
             self.lockfile.close()
+            LOG.debug(_('Released file lock "%s"'), self.fname)
         except IOError:
             LOG.exception(_("Could not release the acquired lock `%s`"),
                           self.fname)
-        LOG.debug(_('Released file lock "%s"'), self.fname)
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.release()

ceilometer/openstack/common/log.py

@@ -432,12 +432,12 @@ def _load_log_config(log_config_append):
         raise LogConfigError(log_config_append, str(exc))
 
 
-def setup(product_name):
+def setup(product_name, version='unknown'):
     """Setup logging."""
     if CONF.log_config_append:
         _load_log_config(CONF.log_config_append)
     else:
-        _setup_logging_from_conf()
+        _setup_logging_from_conf(product_name, version)
     sys.excepthook = _create_logging_excepthook(product_name)
 
 
@@ -482,7 +482,7 @@ class RFCSysLogHandler(logging.handlers.SysLogHandler):
         return msg
 
 
-def _setup_logging_from_conf():
+def _setup_logging_from_conf(project, version):
     log_root = getLogger(None).logger
     for handler in log_root.handlers:
         log_root.removeHandler(handler)
@@ -530,7 +530,9 @@ def _setup_logging_from_conf():
             log_root.info('Deprecated: log_format is now deprecated and will '
                           'be removed in the next release')
         else:
-            handler.setFormatter(ContextFormatter(datefmt=datefmt))
+            handler.setFormatter(ContextFormatter(project=project,
+                                                  version=version,
+                                                  datefmt=datefmt))
 
     if CONF.debug:
         log_root.setLevel(logging.DEBUG)
@@ -588,10 +590,42 @@ class ContextFormatter(logging.Formatter):
     For information about what variables are available for the formatter see:
     http://docs.python.org/library/logging.html#formatter
 
+    If available, uses the context value stored in TLS - local.store.context
+
     """
 
+    def __init__(self, *args, **kwargs):
+        """Initialize ContextFormatter instance
+
+        Takes additional keyword arguments which can be used in the message
+        format string.
+
+        :keyword project: project name
+        :type project: string
+        :keyword version: project version
+        :type version: string
+
+        """
+
+        self.project = kwargs.pop('project', 'unknown')
+        self.version = kwargs.pop('version', 'unknown')
+
+        logging.Formatter.__init__(self, *args, **kwargs)
+
     def format(self, record):
         """Uses contextstring if request_id is set, otherwise default."""
+
+        # store project info
+        record.project = self.project
+        record.version = self.version
+
+        # store request info
+        context = getattr(local.store, 'context', None)
+        if context:
+            d = _dictify_context(context)
+            for k, v in d.items():
+                setattr(record, k, v)
+
         # NOTE(sdague): default the fancier formatting params
         # to an empty string so we don't throw an exception if
         # they get used

ceilometer/openstack/common/policy.py

@@ -119,11 +119,16 @@ class Rules(dict):
 
         # If the default rule isn't actually defined, do something
         # reasonably intelligent
-        if not self.default_rule or self.default_rule not in self:
+        if not self.default_rule:
             raise KeyError(key)
 
         if isinstance(self.default_rule, BaseCheck):
             return self.default_rule
+
+        # We need to check this or we can get infinite recursion
+        if self.default_rule not in self:
+            raise KeyError(key)
+
         elif isinstance(self.default_rule, six.string_types):
             return self[self.default_rule]
 

ceilometer/openstack/common/service.py

@@ -23,7 +23,6 @@ import os
 import random
 import signal
 import sys
-import threading
 import time
 
 try:
@@ -35,6 +34,7 @@ except ImportError:
     UnsupportedOperation = None
 
 import eventlet
+from eventlet import event
 from oslo.config import cfg
 
 from ceilometer.openstack.common import eventlet_backdoor
@@ -419,10 +419,11 @@ class Service(object):
         self.tg = threadgroup.ThreadGroup(threads)
 
         # signal that the service is done shutting itself down:
-        self._done = threading.Event()
+        self._done = event.Event()
 
     def reset(self):
-        self._done = threading.Event()
+        # NOTE(Fengqian): docs for Event.reset() recommend against using it
+        self._done = event.Event()
 
     def start(self):
         pass
@@ -431,7 +432,8 @@ class Service(object):
         self.tg.stop()
         self.tg.wait()
         # Signal that service cleanup is done:
-        self._done.set()
+        if not self._done.ready():
+            self._done.send()
 
     def wait(self):
         self._done.wait()
@@ -442,7 +444,7 @@ class Services(object):
     def __init__(self):
         self.services = []
         self.tg = threadgroup.ThreadGroup()
-        self.done = threading.Event()
+        self.done = event.Event()
 
     def add(self, service):
         self.services.append(service)
@@ -456,7 +458,8 @@ class Services(object):
 
         # Each service has performed cleanup, now signal that the run_service
         # wrapper threads can now die:
-        self.done.set()
+        if not self.done.ready():
+            self.done.send()
 
         # reap threads:
         self.tg.stop()
@@ -466,7 +469,7 @@ class Services(object):
 
     def restart(self):
         self.stop()
-        self.done = threading.Event()
+        self.done = event.Event()
         for restart_service in self.services:
             restart_service.reset()
             self.tg.add_thread(self.run_service, restart_service, self.done)

tools/config/generate_sample.sh

@@ -4,8 +4,8 @@ print_hint() {
     echo "Try \`${0##*/} --help' for more information." >&2
 }
 
-PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:l:o: \
+PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \
-                 --long help,base-dir:,package-name:,output-dir:,library: -- "$@")
+                 --long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@")
 
 if [ $? != 0 ] ; then print_hint ; exit 1 ; fi
 
@@ -21,6 +21,7 @@ while true; do
             echo "-b, --base-dir=DIR        project base directory"
             echo "-p, --package-name=NAME   project package name"
             echo "-o, --output-dir=DIR      file output directory"
+            echo "-m, --module=MOD          extra python module to interrogate for options"
             echo "-l, --library=LIB         extra library that registers options for discovery"
             exit 0
             ;;
@@ -39,6 +40,11 @@ while true; do
             OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'`
             shift
             ;;
+        -m|--module)
+            shift
+            MODULES="$MODULES -m $1"
+            shift
+            ;;
         -l|--library)
             shift
             LIBRARIES="$LIBRARIES -l $1"
@@ -89,6 +95,14 @@ then
     source "$RC_FILE"
 fi
 
+for mod in ${CEILOMETER_CONFIG_GENERATOR_EXTRA_MODULES}; do
+    MODULES="$MODULES -m $mod"
+done
+
+for lib in ${CEILOMETER_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do
+    LIBRARIES="$LIBRARIES -l $lib"
+done
+
 export EVENTLET_NO_GREENDNS=yes
 
 OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
@@ -96,7 +110,7 @@ OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
 DEFAULT_MODULEPATH=ceilometer.openstack.common.config.generator
 MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
 OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample
-python -m $MODULEPATH $LIBRARIES $FILES > $OUTPUTFILE
+python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE
 
 # Hook to allow projects to append custom config file snippets
 CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)