Merge "sync with oslo-incubator"

ceilometer/openstack/common/config/generator.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 SINA Corporation
 # All Rights Reserved.
 #
@@ -28,6 +26,7 @@ import sys
 import textwrap
 
 from oslo.config import cfg
+import six
 
 from ceilometer.openstack.common import gettextutils
 from ceilometer.openstack.common import importutils
@@ -78,12 +77,15 @@ def generate(srcfiles):
     # The options list is a list of (module, options) tuples
     opts_by_group = {'DEFAULT': []}
 
-    for module_name in os.getenv(
+    extra_modules = os.getenv("CEILOMETER_CONFIG_GENERATOR_EXTRA_MODULES", "")
-            "OSLO_CONFIG_GENERATOR_EXTRA_MODULES", "").split(','):
+    if extra_modules:
+        for module_name in extra_modules.split(','):
+            module_name = module_name.strip()
             module = _import_module(module_name)
             if module:
                 for group, opts in _list_opts(module):
-                opts_by_group.setdefault(group, []).append((module_name, opts))
+                    opts_by_group.setdefault(group, []).append((module_name,
+                                                                opts))
 
     for pkg_name in pkg_names:
         mods = mods_by_pkg.get(pkg_name)
@@ -94,7 +96,7 @@ def generate(srcfiles):
 
             mod_obj = _import_module(mod_str)
             if not mod_obj:
-                continue
+                raise RuntimeError("Unable to import module %s" % mod_str)
 
             for group, opts in _list_opts(mod_obj):
                 opts_by_group.setdefault(group, []).append((mod_str, opts))
@@ -111,10 +113,8 @@ def _import_module(mod_str):
             return sys.modules[mod_str[4:]]
         else:
             return importutils.import_module(mod_str)
-    except ImportError as ie:
+    except Exception as e:
-        sys.stderr.write("%s\n" % str(ie))
+        sys.stderr.write("Error importing module %s: %s\n" % (mod_str, str(e)))
-        return None
-    except Exception:
         return None
 
 
@@ -221,11 +221,19 @@ def _print_opt(opt):
         sys.exit(1)
     opt_help += ' (' + OPT_TYPES[opt_type] + ')'
     print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH)))
+    if opt.deprecated_opts:
+        for deprecated_opt in opt.deprecated_opts:
+            if deprecated_opt.name:
+                deprecated_group = (deprecated_opt.group if
+                                    deprecated_opt.group else "DEFAULT")
+                print('# Deprecated group/name - [%s]/%s' %
+                      (deprecated_group,
+                       deprecated_opt.name))
     try:
         if opt_default is None:
             print('#%s=<None>' % opt_name)
         elif opt_type == STROPT:
-            assert(isinstance(opt_default, basestring))
+            assert(isinstance(opt_default, six.string_types))
             print('#%s=%s' % (opt_name, _sanitize_default(opt_name,
                                                           opt_default)))
         elif opt_type == BOOLOPT:

ceilometer/openstack/common/context.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -23,12 +21,11 @@ context or provide additional information in their specific WSGI pipeline.
 """
 
 import itertools
-
+import uuid
-from ceilometer.openstack.common import uuidutils
 
 
 def generate_request_id():
-    return 'req-%s' % uuidutils.generate_uuid()
+    return 'req-%s' % str(uuid.uuid4())
 
 
 class RequestContext(object):

ceilometer/openstack/common/db/__init__.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 Cloudscaling Group, Inc
 # All Rights Reserved.
 #

ceilometer/openstack/common/db/api.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright (c) 2013 Rackspace Hosting
 # All Rights Reserved.
 #

ceilometer/openstack/common/db/exception.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # All Rights Reserved.
@@ -49,3 +47,8 @@ class DbMigrationError(DBError):
     """Wraps migration specific exception."""
     def __init__(self, message=None):
         super(DbMigrationError, self).__init__(str(message))
+
+
+class DBConnectionError(DBError):
+    """Wraps connection specific exception."""
+    pass

ceilometer/openstack/common/db/sqlalchemy/__init__.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 Cloudscaling Group, Inc
 # All Rights Reserved.
 #

ceilometer/openstack/common/db/sqlalchemy/models.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright (c) 2011 X.commerce, a business unit of eBay Inc.
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.

ceilometer/openstack/common/db/sqlalchemy/provision.py

@@ -0,0 +1,187 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Mirantis.inc
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Provision test environment for specific DB backends"""
+
+import argparse
+import os
+import random
+import string
+
+import sqlalchemy
+
+from ceilometer.openstack.common.db import exception as exc
+
+
+SQL_CONNECTION = os.getenv('OS_TEST_DBAPI_ADMIN_CONNECTION', 'sqlite://')
+
+
+def _gen_credentials(*names):
+    """Generate credentials."""
+    auth_dict = {}
+    for name in names:
+        val = ''.join(random.choice(string.lowercase) for i in xrange(10))
+        auth_dict[name] = val
+    return auth_dict
+
+
+def _get_engine(uri=SQL_CONNECTION):
+    """Engine creation
+
+    By default the uri is SQL_CONNECTION which is admin credentials.
+    Call the function without arguments to get admin connection. Admin
+    connection required to create temporary user and database for each
+    particular test. Otherwise use existing connection to recreate connection
+    to the temporary database.
+    """
+    return sqlalchemy.create_engine(uri, poolclass=sqlalchemy.pool.NullPool)
+
+
+def _execute_sql(engine, sql, driver):
+    """Initialize connection, execute sql query and close it."""
+    try:
+        with engine.connect() as conn:
+            if driver == 'postgresql':
+                conn.connection.set_isolation_level(0)
+            for s in sql:
+                conn.execute(s)
+    except sqlalchemy.exc.OperationalError:
+        msg = ('%s does not match database admin '
+               'credentials or database does not exist.')
+        raise exc.DBConnectionError(msg % SQL_CONNECTION)
+
+
+def create_database(engine):
+    """Provide temporary user and database for each particular test."""
+    driver = engine.name
+
+    auth = _gen_credentials('database', 'user', 'passwd')
+
+    sqls = {
+        'mysql': [
+            "drop database if exists %(database)s;",
+            "grant all on %(database)s.* to '%(user)s'@'localhost'"
+            " identified by '%(passwd)s';",
+            "create database %(database)s;",
+        ],
+        'postgresql': [
+            "drop database if exists %(database)s;",
+            "drop user if exists %(user)s;",
+            "create user %(user)s with password '%(passwd)s';",
+            "create database %(database)s owner %(user)s;",
+        ]
+    }
+
+    if driver == 'sqlite':
+        return 'sqlite:////tmp/%s' % auth['database']
+
+    try:
+        sql_rows = sqls[driver]
+    except KeyError:
+        raise ValueError('Unsupported RDBMS %s' % driver)
+    sql_query = map(lambda x: x % auth, sql_rows)
+
+    _execute_sql(engine, sql_query, driver)
+
+    params = auth.copy()
+    params['backend'] = driver
+    return "%(backend)s://%(user)s:%(passwd)s@localhost/%(database)s" % params
+
+
+def drop_database(engine, current_uri):
+    """Drop temporary database and user after each particular test."""
+    engine = _get_engine(current_uri)
+    admin_engine = _get_engine()
+    driver = engine.name
+    auth = {'database': engine.url.database, 'user': engine.url.username}
+
+    if driver == 'sqlite':
+        try:
+            os.remove(auth['database'])
+        except OSError:
+            pass
+        return
+
+    sqls = {
+        'mysql': [
+            "drop database if exists %(database)s;",
+            "drop user '%(user)s'@'localhost';",
+        ],
+        'postgresql': [
+            "drop database if exists %(database)s;",
+            "drop user if exists %(user)s;",
+        ]
+    }
+
+    try:
+        sql_rows = sqls[driver]
+    except KeyError:
+        raise ValueError('Unsupported RDBMS %s' % driver)
+    sql_query = map(lambda x: x % auth, sql_rows)
+
+    _execute_sql(admin_engine, sql_query, driver)
+
+
+def main():
+    """Controller to handle commands
+
+    ::create: Create test user and database with random names.
+    ::drop: Drop user and database created by previous command.
+    """
+    parser = argparse.ArgumentParser(
+        description='Controller to handle database creation and dropping'
+        ' commands.',
+        epilog='Under normal circumstances is not used directly.'
+        ' Used in .testr.conf to automate test database creation'
+        ' and dropping processes.')
+    subparsers = parser.add_subparsers(
+        help='Subcommands to manipulate temporary test databases.')
+
+    create = subparsers.add_parser(
+        'create',
+        help='Create temporary test '
+        'databases and users.')
+    create.set_defaults(which='create')
+    create.add_argument(
+        'instances_count',
+        type=int,
+        help='Number of databases to create.')
+
+    drop = subparsers.add_parser(
+        'drop',
+        help='Drop temporary test databases and users.')
+    drop.set_defaults(which='drop')
+    drop.add_argument(
+        'instances',
+        nargs='+',
+        help='List of databases uri to be dropped.')
+
+    args = parser.parse_args()
+
+    engine = _get_engine()
+    which = args.which
+
+    if which == "create":
+        for i in range(int(args.instances_count)):
+            print(create_database(engine))
+    elif which == "drop":
+        for db in args.instances:
+            drop_database(engine, db)
+
+
+if __name__ == "__main__":
+    main()

ceilometer/openstack/common/db/sqlalchemy/session.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # All Rights Reserved.
@@ -623,7 +621,8 @@ def _is_db_connection_error(args):
     """Return True if error in connecting to db."""
     # NOTE(adam_g): This is currently MySQL specific and needs to be extended
     #               to support Postgres and others.
-    conn_err_codes = ('2002', '2003', '2006')
+    # For the db2, the error code is -30081 since the db2 is still not ready
+    conn_err_codes = ('2002', '2003', '2006', '-30081')
     for err_code in conn_err_codes:
         if args.find(err_code) != -1:
             return True
@@ -753,25 +752,25 @@ def _patch_mysqldb_with_stacktrace_comments():
 
     def _do_query(self, q):
         stack = ''
-        for file, line, method, function in traceback.extract_stack():
+        for filename, line, method, function in traceback.extract_stack():
             # exclude various common things from trace
-            if file.endswith('session.py') and method == '_do_query':
+            if filename.endswith('session.py') and method == '_do_query':
                 continue
-            if file.endswith('api.py') and method == 'wrapper':
+            if filename.endswith('api.py') and method == 'wrapper':
                 continue
-            if file.endswith('utils.py') and method == '_inner':
+            if filename.endswith('utils.py') and method == '_inner':
                 continue
-            if file.endswith('exception.py') and method == '_wrap':
+            if filename.endswith('exception.py') and method == '_wrap':
                 continue
             # db/api is just a wrapper around db/sqlalchemy/api
-            if file.endswith('db/api.py'):
+            if filename.endswith('db/api.py'):
                 continue
             # only trace inside ceilometer
-            index = file.rfind('ceilometer')
+            index = filename.rfind('ceilometer')
             if index == -1:
                 continue
             stack += "File:%s:%s Method:%s() Line:%s | " \
-                     % (file[index:], line, method, function)
+                     % (filename[index:], line, method, function)
 
         # strip trailing " | " from stack
         if stack:

ceilometer/openstack/common/db/sqlalchemy/test_migrations.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010-2011 OpenStack Foundation
 # Copyright 2012-2013 IBM Corp.
 # All Rights Reserved.
@@ -16,17 +14,18 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-
-import commands
 import ConfigParser
+import functools
 import os
-import urlparse
 
+import lockfile
 import sqlalchemy
 import sqlalchemy.exc
 
-from ceilometer.openstack.common import lockutils
+from ceilometer.openstack.common.gettextutils import _
 from ceilometer.openstack.common import log as logging
+from ceilometer.openstack.common import processutils
+from ceilometer.openstack.common.py3kcompat import urlutils
 from ceilometer.openstack.common import test
 
 LOG = logging.getLogger(__name__)
@@ -93,6 +92,22 @@ def get_db_connection_info(conn_pieces):
     return (user, password, database, host)
 
 
+def _set_db_lock(lock_path=None, lock_prefix=None):
+    def decorator(f):
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            try:
+                path = lock_path or os.environ.get("CEILOMETER_LOCK_PATH")
+                lock = lockfile.FileLock(os.path.join(path, lock_prefix))
+                with lock:
+                    LOG.debug(_('Got lock "%s"') % f.__name__)
+                    return f(*args, **kwargs)
+            finally:
+                LOG.debug(_('Lock released "%s"') % f.__name__)
+        return wrapper
+    return decorator
+
+
 class BaseMigrationTestCase(test.BaseTestCase):
     """Base class fort testing of migration utils."""
 
@@ -143,12 +158,13 @@ class BaseMigrationTestCase(test.BaseTestCase):
         super(BaseMigrationTestCase, self).tearDown()
 
     def execute_cmd(self, cmd=None):
-        status, output = commands.getstatusoutput(cmd)
+        out, err = processutils.trycmd(cmd, shell=True, discard_warnings=True)
+        output = out or err
         LOG.debug(output)
-        self.assertEqual(0, status,
+        self.assertEqual('', err,
                          "Failed to run: %s\n%s" % (cmd, output))
 
-    @lockutils.synchronized('pgadmin', 'tests-', external=True)
+    @_set_db_lock('pgadmin', 'tests-')
     def _reset_pg(self, conn_pieces):
         (user, password, database, host) = get_db_connection_info(conn_pieces)
         os.environ['PGPASSWORD'] = password
@@ -173,7 +189,7 @@ class BaseMigrationTestCase(test.BaseTestCase):
     def _reset_databases(self):
         for key, engine in self.engines.items():
             conn_string = self.test_databases[key]
-            conn_pieces = urlparse.urlparse(conn_string)
+            conn_pieces = urlutils.urlparse(conn_string)
             engine.dispose()
             if conn_string.startswith('sqlite'):
                 # We can just delete the SQLite database, which is

ceilometer/openstack/common/db/sqlalchemy/utils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # Copyright 2010-2011 OpenStack Foundation.

ceilometer/openstack/common/deprecated/wsgi.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -24,6 +22,8 @@ import datetime
 import errno
 import socket
 import time
+from xml.dom import minidom
+from xml.parsers import expat
 
 import eventlet.wsgi
 from oslo.config import cfg
@@ -32,8 +32,6 @@ import routes.middleware
 import six
 import webob.dec
 import webob.exc
-from xml.dom import minidom
-from xml.parsers import expat
 
 from ceilometer.openstack.common.gettextutils import _  # noqa
 from ceilometer.openstack.common import jsonutils
@@ -42,6 +40,7 @@ from ceilometer.openstack.common import service
 from ceilometer.openstack.common import sslutils
 from ceilometer.openstack.common import xmlutils
 
+
 socket_opts = [
     cfg.IntOpt('backlog',
                default=4096,

ceilometer/openstack/common/eventlet_backdoor.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright (c) 2012 OpenStack Foundation.
 # Administrator of the National Aeronautics and Space Administration.
 # All Rights Reserved.

ceilometer/openstack/common/excutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # Copyright 2012, Red Hat, Inc.
 #

ceilometer/openstack/common/fileutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #

ceilometer/openstack/common/fixture/config.py

@@ -1,4 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # Copyright 2013 Mirantis, Inc.
 # Copyright 2013 OpenStack Foundation
@@ -30,7 +29,7 @@ class Config(fixtures.Fixture):
     the specified configuration option group.
 
     All overrides are automatically cleared at the end of the current
-    test by the reset() method, which is registred by addCleanup().
+    test by the reset() method, which is registered by addCleanup().
     """
 
     def __init__(self, conf=cfg.CONF):

ceilometer/openstack/common/fixture/lockutils.py

@@ -0,0 +1,51 @@
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import fixtures
+
+from ceilometer.openstack.common.lockutils import lock
+
+
+class LockFixture(fixtures.Fixture):
+    """External locking fixture.
+
+    This fixture is basically an alternative to the synchronized decorator with
+    the external flag so that tearDowns and addCleanups will be included in
+    the lock context for locking between tests. The fixture is recommended to
+    be the first line in a test method, like so::
+
+        def test_method(self):
+            self.useFixture(LockFixture)
+                ...
+
+    or the first line in setUp if all the test methods in the class are
+    required to be serialized. Something like::
+
+        class TestCase(testtools.testcase):
+            def setUp(self):
+                self.useFixture(LockFixture)
+                super(TestCase, self).setUp()
+                    ...
+
+    This is because addCleanups are put on a LIFO queue that gets run after the
+    test method exits. (either by completing or raising an exception)
+    """
+    def __init__(self, name, lock_file_prefix=None):
+        self.mgr = lock(name, lock_file_prefix, True)
+
+    def setUp(self):
+        super(LockFixture, self).setUp()
+        self.addCleanup(self.mgr.__exit__, None, None, None)
+        self.mgr.__enter__()

ceilometer/openstack/common/fixture/mockpatch.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # Copyright 2013 Hewlett-Packard Development Company, L.P.

ceilometer/openstack/common/fixture/moxstubout.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # Copyright 2013 Hewlett-Packard Development Company, L.P.
@@ -19,7 +17,6 @@
 
 import fixtures
 import mox
-import stubout
 
 
 class MoxStubout(fixtures.Fixture):
@@ -30,8 +27,6 @@ class MoxStubout(fixtures.Fixture):
         # emulate some of the mox stuff, we can't use the metaclass
         # because it screws with our generators
         self.mox = mox.Mox()
-        self.stubs = stubout.StubOutForTesting()
+        self.stubs = self.mox.stubs
         self.addCleanup(self.mox.UnsetStubs)
-        self.addCleanup(self.stubs.UnsetAll)
-        self.addCleanup(self.stubs.SmartUnsetAll)
         self.addCleanup(self.mox.VerifyAll)

ceilometer/openstack/common/gettextutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 Red Hat, Inc.
 # Copyright 2013 IBM Corp.
 # All Rights Reserved.
@@ -317,7 +315,7 @@ def get_available_languages(domain):
     # NOTE(luisg): Babel <1.0 used a function called list(), which was
     # renamed to locale_identifiers() in >=1.0, the requirements master list
     # requires >=0.9.6, uncapped, so defensively work with both. We can remove
-    # this check when the master list updates to >=1.0, and all projects udpate
+    # this check when the master list updates to >=1.0, and update all projects
     list_identifiers = (getattr(localedata, 'list', None) or
                         getattr(localedata, 'locale_identifiers'))
     locale_identifiers = list_identifiers()
@@ -329,13 +327,21 @@ def get_available_languages(domain):
 
 
 def get_localized_message(message, user_locale):
-    """Gets a localized version of the given message in the given locale."""
+    """Gets a localized version of the given message in the given locale.
+
+    If the message is not a Message object the message is returned as-is.
+    If the locale is None the message is translated to the default locale.
+
+    :returns: the translated message in unicode, or the original message if
+              it could not be translated
+    """
+    translated = message
     if isinstance(message, Message):
-        if user_locale:
+        original_locale = message.locale
         message.locale = user_locale
-        return six.text_type(message)
+        translated = six.text_type(message)
-    else:
+        message.locale = original_locale
-        return message
+    return translated
 
 
 class LocaleHandler(logging.Handler):

ceilometer/openstack/common/importutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #

ceilometer/openstack/common/jsonutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # Copyright 2011 Justin Santa Barbara

ceilometer/openstack/common/local.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #

ceilometer/openstack/common/lockutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -20,11 +18,14 @@ import contextlib
 import errno
 import functools
 import os
+import shutil
+import subprocess
+import sys
+import tempfile
 import threading
 import time
 import weakref
 
-import fixtures
 from oslo.config import cfg
 
 from ceilometer.openstack.common import fileutils
@@ -40,6 +41,7 @@ util_opts = [
     cfg.BoolOpt('disable_process_locking', default=False,
                 help='Whether to disable inter-process locks'),
     cfg.StrOpt('lock_path',
+               default=os.environ.get("CEILOMETER_LOCK_PATH"),
                help=('Directory to use for lock files.'))
 ]
 
@@ -132,6 +134,7 @@ else:
     InterProcessLock = _PosixLock
 
 _semaphores = weakref.WeakValueDictionary()
+_semaphores_lock = threading.Lock()
 
 
 @contextlib.contextmanager
@@ -154,14 +157,11 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None):
     special location for external lock files to live. If nothing is set, then
     CONF.lock_path is used as a default.
     """
-    # NOTE(soren): If we ever go natively threaded, this will be racy.
+    with _semaphores_lock:
-    #              See http://stackoverflow.com/questions/5390569/dyn
+        try:
-    #              amically-allocating-and-destroying-mutexes
+            sem = _semaphores[name]
-    sem = _semaphores.get(name, threading.Semaphore())
+        except KeyError:
-    if name not in _semaphores:
+            sem = threading.Semaphore()
-        # this check is not racy - we're already holding ref locally
-        # so GC won't remove the item and there was no IO switch
-        # (only valid in greenthreads)
             _semaphores[name] = sem
 
     with sem:
@@ -242,11 +242,12 @@ def synchronized(name, lock_file_prefix=None, external=False, lock_path=None):
     def wrap(f):
         @functools.wraps(f)
         def inner(*args, **kwargs):
+            try:
                 with lock(name, lock_file_prefix, external, lock_path):
                     LOG.debug(_('Got semaphore / lock "%(function)s"'),
                               {'function': f.__name__})
                     return f(*args, **kwargs)
-
+            finally:
                 LOG.debug(_('Semaphore / lock released "%(function)s"'),
                           {'function': f.__name__})
         return inner
@@ -278,34 +279,25 @@ def synchronized_with_prefix(lock_file_prefix):
     return functools.partial(synchronized, lock_file_prefix=lock_file_prefix)
 
 
-class LockFixture(fixtures.Fixture):
+def main(argv):
-    """External locking fixture.
+    """Create a dir for locks and pass it to command from arguments
 
-    This fixture is basically an alternative to the synchronized decorator with
+    If you run this:
-    the external flag so that tearDowns and addCleanups will be included in
+    python -m openstack.common.lockutils python setup.py testr <etc>
-    the lock context for locking between tests. The fixture is recommended to
-    be the first line in a test method, like so::
 
-        def test_method(self):
+    a temporary directory will be created for all your locks and passed to all
-            self.useFixture(LockFixture)
+    your tests in an environment variable. The temporary dir will be deleted
-                ...
+    afterwards and the return value will be preserved.
-
-    or the first line in setUp if all the test methods in the class are
-    required to be serialized. Something like::
-
-        class TestCase(testtools.testcase):
-            def setUp(self):
-                self.useFixture(LockFixture)
-                super(TestCase, self).setUp()
-                    ...
-
-    This is because addCleanups are put on a LIFO queue that gets run after the
-    test method exits. (either by completing or raising an exception)
     """
-    def __init__(self, name, lock_file_prefix=None):
-        self.mgr = lock(name, lock_file_prefix, True)
 
-    def setUp(self):
+    lock_dir = tempfile.mkdtemp()
-        super(LockFixture, self).setUp()
+    os.environ["CEILOMETER_LOCK_PATH"] = lock_dir
-        self.addCleanup(self.mgr.__exit__, None, None, None)
+    try:
-        self.mgr.__enter__()
+        ret_val = subprocess.call(argv[1:])
+    finally:
+        shutil.rmtree(lock_dir, ignore_errors=True)
+    return ret_val
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

ceilometer/openstack/common/log.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
@@ -35,6 +33,7 @@ import logging
 import logging.config
 import logging.handlers
 import os
+import re
 import sys
 import traceback
 
@@ -50,6 +49,24 @@ from ceilometer.openstack.common import local
 
 _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
 
+_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
+
+# NOTE(ldbragst): Let's build a list of regex objects using the list of
+# _SANITIZE_KEYS we already have. This way, we only have to add the new key
+# to the list of _SANITIZE_KEYS and we can generate regular expressions
+# for XML and JSON automatically.
+_SANITIZE_PATTERNS = []
+_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
+                    r'(<%(key)s>).*?(</%(key)s>)',
+                    r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
+                    r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])']
+
+for key in _SANITIZE_KEYS:
+    for pattern in _FORMAT_PATTERNS:
+        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
+        _SANITIZE_PATTERNS.append(reg_ex)
+
+
 common_cli_opts = [
     cfg.BoolOpt('debug',
                 short='d',
@@ -136,6 +153,7 @@ log_opts = [
                     'qpid=WARN',
                     'sqlalchemy=WARN',
                     'suds=INFO',
+                    'iso8601=WARN',
                 ],
                 help='list of logger=LEVEL pairs'),
     cfg.BoolOpt('publish_errors',
@@ -214,6 +232,39 @@ def _get_log_file_path(binary=None):
     return None
 
 
+def mask_password(message, secret="***"):
+    """Replace password with 'secret' in message.
+
+    :param message: The string which includes security information.
+    :param secret: value with which to replace passwords, defaults to "***".
+    :returns: The unicode value of message with the password fields masked.
+
+    For example:
+    >>> mask_password("'adminPass' : 'aaaaa'")
+    "'adminPass' : '***'"
+    >>> mask_password("'admin_pass' : 'aaaaa'")
+    "'admin_pass' : '***'"
+    >>> mask_password('"password" : "aaaaa"')
+    '"password" : "***"'
+    >>> mask_password("'original_password' : 'aaaaa'")
+    "'original_password' : '***'"
+    >>> mask_password("u'original_password' :   u'aaaaa'")
+    "u'original_password' :   u'***'"
+    """
+    message = six.text_type(message)
+
+    # NOTE(ldbragst): Check to see if anything in message contains any key
+    # specified in _SANITIZE_KEYS, if not then just return the message since
+    # we don't have to mask any passwords.
+    if not any(key in message for key in _SANITIZE_KEYS):
+        return message
+
+    secret = r'\g<1>' + secret + r'\g<2>'
+    for pattern in _SANITIZE_PATTERNS:
+        message = re.sub(pattern, secret, message)
+    return message
+
+
 class BaseLoggerAdapter(logging.LoggerAdapter):
 
     def audit(self, msg, *args, **kwargs):
@@ -298,7 +349,7 @@ class JSONFormatter(logging.Formatter):
     def formatException(self, ei, strip_newlines=True):
         lines = traceback.format_exception(*ei)
         if strip_newlines:
-            lines = [itertools.ifilter(
+            lines = [moves.filter(
                 lambda x: x,
                 line.rstrip().splitlines()) for line in lines]
             lines = list(itertools.chain(*lines))
@@ -336,10 +387,10 @@ class JSONFormatter(logging.Formatter):
 
 
 def _create_logging_excepthook(product_name):
-    def logging_excepthook(type, value, tb):
+    def logging_excepthook(exc_type, value, tb):
         extra = {}
         if CONF.verbose:
-            extra['exc_info'] = (type, value, tb)
+            extra['exc_info'] = (exc_type, value, tb)
         getLogger(product_name).critical(str(value), **extra)
     return logging_excepthook
 
@@ -424,7 +475,7 @@ def _setup_logging_from_conf():
         streamlog = ColorHandler()
         log_root.addHandler(streamlog)
 
-    elif not CONF.log_file:
+    elif not logpath:
         # pass sys.stdout as a positional argument
         # python2.6 calls the argument strm, in 2.7 it's stream
         streamlog = logging.StreamHandler(sys.stdout)

ceilometer/openstack/common/log_handler.py

@@ -1,6 +1,4 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
+# Copyright 2013 IBM Corp.
-
-# Copyright 2013 SoftLayer Technologies, an IBM Company
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -15,7 +13,7 @@
 #    under the License.
 import logging
 
-from nova.openstack.common import notifier
+from ceilometer.openstack.common import notifier
 
 from oslo.config import cfg
 

ceilometer/openstack/common/loopingcall.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # Copyright 2011 Justin Santa Barbara

ceilometer/openstack/common/middleware/audit.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright (c) 2013 OpenStack Foundation
 # All Rights Reserved.
 #

ceilometer/openstack/common/middleware/context.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -55,7 +53,7 @@ def filter_factory(global_conf, **local_conf):
     conf = global_conf.copy()
     conf.update(local_conf)
 
-    def filter(app):
+    def f(app):
         return ContextMiddleware(app, conf)
 
-    return filter
+    return f

ceilometer/openstack/common/middleware/correlation_id.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright (c) 2013 Rackspace Hosting
 # All Rights Reserved.
 #
@@ -17,13 +15,14 @@
 
 """Middleware that attaches a correlation id to WSGI request"""
 
+import uuid
+
 from ceilometer.openstack.common.middleware import base
-from ceilometer.openstack.common import uuidutils
 
 
 class CorrelationIdMiddleware(base.Middleware):
 
     def process_request(self, req):
         correlation_id = (req.headers.get("X_CORRELATION_ID") or
-                          uuidutils.generate_uuid())
+                          str(uuid.uuid4()))
         req.headers['X_CORRELATION_ID'] = correlation_id

ceilometer/openstack/common/middleware/notifier.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright (c) 2013 eNovance
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -118,7 +116,7 @@ class RequestNotifier(base.Middleware):
             try:
                 response = req.get_response(self.application)
             except Exception:
-                type, value, traceback = sys.exc_info()
+                exc_type, value, traceback = sys.exc_info()
                 self.process_response(req, None, value, traceback)
                 raise
             else:

ceilometer/openstack/common/middleware/sizelimit.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright (c) 2012 Red Hat, Inc.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

ceilometer/openstack/common/network_utils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -19,7 +17,7 @@
 Network-related utilities and helper functions.
 """
 
-import urlparse
+from ceilometer.openstack.common.py3kcompat import urlutils
 
 
 def parse_host_port(address, default_port=None):
@@ -72,10 +70,10 @@ def urlsplit(url, scheme='', allow_fragments=True):
 
     The parameters are the same as urlparse.urlsplit.
     """
-    scheme, netloc, path, query, fragment = urlparse.urlsplit(
+    scheme, netloc, path, query, fragment = urlutils.urlsplit(
         url, scheme, allow_fragments)
     if allow_fragments and '#' in path:
         path, fragment = path.split('#', 1)
     if '?' in path:
         path, query = path.split('?', 1)
-    return urlparse.SplitResult(scheme, netloc, path, query, fragment)
+    return urlutils.SplitResult(scheme, netloc, path, query, fragment)

ceilometer/openstack/common/policy.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright (c) 2012 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -505,7 +503,7 @@ def _parse_list_rule(rule):
             continue
 
         # Handle bare strings
-        if isinstance(inner_rule, basestring):
+        if isinstance(inner_rule, six.string_types):
             inner_rule = [inner_rule]
 
         # Parse the inner rules into Check objects
@@ -763,7 +761,7 @@ def parse_rule(rule):
     """Parses a policy rule into a tree of Check objects."""
 
     # If the rule is a string, it's in the policy language
-    if isinstance(rule, basestring):
+    if isinstance(rule, six.string_types):
         return _parse_text_rule(rule)
     return _parse_list_rule(rule)
 

ceilometer/openstack/common/processutils.py

@@ -0,0 +1,248 @@
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+System-level utilities and helper functions.
+"""
+
+import logging as stdlib_logging
+import os
+import random
+import shlex
+import signal
+
+from eventlet.green import subprocess
+from eventlet import greenthread
+
+from ceilometer.openstack.common.gettextutils import _  # noqa
+from ceilometer.openstack.common import log as logging
+
+
+LOG = logging.getLogger(__name__)
+
+
+class InvalidArgumentError(Exception):
+    def __init__(self, message=None):
+        super(InvalidArgumentError, self).__init__(message)
+
+
+class UnknownArgumentError(Exception):
+    def __init__(self, message=None):
+        super(UnknownArgumentError, self).__init__(message)
+
+
+class ProcessExecutionError(Exception):
+    def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
+                 description=None):
+        self.exit_code = exit_code
+        self.stderr = stderr
+        self.stdout = stdout
+        self.cmd = cmd
+        self.description = description
+
+        if description is None:
+            description = "Unexpected error while running command."
+        if exit_code is None:
+            exit_code = '-'
+        message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r"
+                   % (description, cmd, exit_code, stdout, stderr))
+        super(ProcessExecutionError, self).__init__(message)
+
+
+class NoRootWrapSpecified(Exception):
+    def __init__(self, message=None):
+        super(NoRootWrapSpecified, self).__init__(message)
+
+
+def _subprocess_setup():
+    # Python installs a SIGPIPE handler by default. This is usually not what
+    # non-Python subprocesses expect.
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+
+def execute(*cmd, **kwargs):
+    """Helper method to shell out and execute a command through subprocess.
+
+    Allows optional retry.
+
+    :param cmd:             Passed to subprocess.Popen.
+    :type cmd:              string
+    :param process_input:   Send to opened process.
+    :type proces_input:     string
+    :param check_exit_code: Single bool, int, or list of allowed exit
+                            codes.  Defaults to [0].  Raise
+                            :class:`ProcessExecutionError` unless
+                            program exits with one of these code.
+    :type check_exit_code:  boolean, int, or [int]
+    :param delay_on_retry:  True | False. Defaults to True. If set to True,
+                            wait a short amount of time before retrying.
+    :type delay_on_retry:   boolean
+    :param attempts:        How many times to retry cmd.
+    :type attempts:         int
+    :param run_as_root:     True | False. Defaults to False. If set to True,
+                            the command is prefixed by the command specified
+                            in the root_helper kwarg.
+    :type run_as_root:      boolean
+    :param root_helper:     command to prefix to commands called with
+                            run_as_root=True
+    :type root_helper:      string
+    :param shell:           whether or not there should be a shell used to
+                            execute this command. Defaults to false.
+    :type shell:            boolean
+    :param loglevel:        log level for execute commands.
+    :type loglevel:         int.  (Should be stdlib_logging.DEBUG or
+                            stdlib_logging.INFO)
+    :returns:               (stdout, stderr) from process execution
+    :raises:                :class:`UnknownArgumentError` on
+                            receiving unknown arguments
+    :raises:                :class:`ProcessExecutionError`
+    """
+
+    process_input = kwargs.pop('process_input', None)
+    check_exit_code = kwargs.pop('check_exit_code', [0])
+    ignore_exit_code = False
+    delay_on_retry = kwargs.pop('delay_on_retry', True)
+    attempts = kwargs.pop('attempts', 1)
+    run_as_root = kwargs.pop('run_as_root', False)
+    root_helper = kwargs.pop('root_helper', '')
+    shell = kwargs.pop('shell', False)
+    loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
+
+    if isinstance(check_exit_code, bool):
+        ignore_exit_code = not check_exit_code
+        check_exit_code = [0]
+    elif isinstance(check_exit_code, int):
+        check_exit_code = [check_exit_code]
+
+    if kwargs:
+        raise UnknownArgumentError(_('Got unknown keyword args '
+                                     'to utils.execute: %r') % kwargs)
+
+    if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
+        if not root_helper:
+            raise NoRootWrapSpecified(
+                message=('Command requested root, but did not specify a root '
+                         'helper.'))
+        cmd = shlex.split(root_helper) + list(cmd)
+
+    cmd = map(str, cmd)
+
+    while attempts > 0:
+        attempts -= 1
+        try:
+            LOG.log(loglevel, _('Running cmd (subprocess): %s'), ' '.join(cmd))
+            _PIPE = subprocess.PIPE  # pylint: disable=E1101
+
+            if os.name == 'nt':
+                preexec_fn = None
+                close_fds = False
+            else:
+                preexec_fn = _subprocess_setup
+                close_fds = True
+
+            obj = subprocess.Popen(cmd,
+                                   stdin=_PIPE,
+                                   stdout=_PIPE,
+                                   stderr=_PIPE,
+                                   close_fds=close_fds,
+                                   preexec_fn=preexec_fn,
+                                   shell=shell)
+            result = None
+            if process_input is not None:
+                result = obj.communicate(process_input)
+            else:
+                result = obj.communicate()
+            obj.stdin.close()  # pylint: disable=E1101
+            _returncode = obj.returncode  # pylint: disable=E1101
+            LOG.log(loglevel, _('Result was %s') % _returncode)
+            if not ignore_exit_code and _returncode not in check_exit_code:
+                (stdout, stderr) = result
+                raise ProcessExecutionError(exit_code=_returncode,
+                                            stdout=stdout,
+                                            stderr=stderr,
+                                            cmd=' '.join(cmd))
+            return result
+        except ProcessExecutionError:
+            if not attempts:
+                raise
+            else:
+                LOG.log(loglevel, _('%r failed. Retrying.'), cmd)
+                if delay_on_retry:
+                    greenthread.sleep(random.randint(20, 200) / 100.0)
+        finally:
+            # NOTE(termie): this appears to be necessary to let the subprocess
+            #               call clean something up in between calls, without
+            #               it two execute calls in a row hangs the second one
+            greenthread.sleep(0)
+
+
+def trycmd(*args, **kwargs):
+    """A wrapper around execute() to more easily handle warnings and errors.
+
+    Returns an (out, err) tuple of strings containing the output of
+    the command's stdout and stderr.  If 'err' is not empty then the
+    command can be considered to have failed.
+
+    :discard_warnings   True | False. Defaults to False. If set to True,
+                        then for succeeding commands, stderr is cleared
+
+    """
+    discard_warnings = kwargs.pop('discard_warnings', False)
+
+    try:
+        out, err = execute(*args, **kwargs)
+        failed = False
+    except ProcessExecutionError as exn:
+        out, err = '', str(exn)
+        failed = True
+
+    if not failed and discard_warnings and err:
+        # Handle commands that output to stderr but otherwise succeed
+        err = ''
+
+    return out, err
+
+
+def ssh_execute(ssh, cmd, process_input=None,
+                addl_env=None, check_exit_code=True):
+    LOG.debug(_('Running cmd (SSH): %s'), cmd)
+    if addl_env:
+        raise InvalidArgumentError(_('Environment not supported over SSH'))
+
+    if process_input:
+        # This is (probably) fixable if we need it...
+        raise InvalidArgumentError(_('process_input not supported over SSH'))
+
+    stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd)
+    channel = stdout_stream.channel
+
+    # NOTE(justinsb): This seems suspicious...
+    # ...other SSH clients have buffering issues with this approach
+    stdout = stdout_stream.read()
+    stderr = stderr_stream.read()
+    stdin_stream.close()
+
+    exit_status = channel.recv_exit_status()
+
+    # exit_status == -1 if no exit code was returned
+    if exit_status != -1:
+        LOG.debug(_('Result was %s') % exit_status)
+        if check_exit_code and exit_status != 0:
+            raise ProcessExecutionError(exit_code=exit_status,
+                                        stdout=stdout,
+                                        stderr=stderr,
+                                        cmd=cmd)
+
+    return (stdout, stderr)

ceilometer/openstack/common/py3kcompat/__init__.py

@@ -1,6 +1,5 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
-
+# Copyright 2013 Canonical Ltd.
-# Copyright (c) 2012 Intel Corporation.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -14,26 +13,4 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-
+#
-"""
-UUID related utilities and helper functions.
-"""
-
-import uuid
-
-
-def generate_uuid():
-    return str(uuid.uuid4())
-
-
-def is_uuid_like(val):
-    """Returns validation of a value as a UUID.
-
-    For our purposes, a UUID is a canonical form string:
-    aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
-
-    """
-    try:
-        return str(uuid.UUID(val)) == val
-    except (TypeError, ValueError, AttributeError):
-        return False

ceilometer/openstack/common/py3kcompat/urlutils.py

@@ -0,0 +1,63 @@
+#
+# Copyright 2013 Canonical Ltd.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""
+Python2/Python3 compatibility layer for OpenStack
+"""
+
+import six
+
+if six.PY3:
+    # python3
+    import urllib.error
+    import urllib.parse
+    import urllib.request
+
+    urlencode = urllib.parse.urlencode
+    urljoin = urllib.parse.urljoin
+    quote = urllib.parse.quote
+    parse_qsl = urllib.parse.parse_qsl
+    unquote = urllib.parse.unquote
+    urlparse = urllib.parse.urlparse
+    urlsplit = urllib.parse.urlsplit
+    urlunsplit = urllib.parse.urlunsplit
+    SplitResult = urllib.parse.SplitResult
+
+    urlopen = urllib.request.urlopen
+    URLError = urllib.error.URLError
+    pathname2url = urllib.request.pathname2url
+else:
+    # python2
+    import urllib
+    import urllib2
+    import urlparse
+
+    urlencode = urllib.urlencode
+    quote = urllib.quote
+    unquote = urllib.unquote
+
+    parse = urlparse
+    parse_qsl = parse.parse_qsl
+    urljoin = parse.urljoin
+    urlparse = parse.urlparse
+    urlsplit = parse.urlsplit
+    urlunsplit = parse.urlunsplit
+    SplitResult = parse.SplitResult
+
+    urlopen = urllib2.urlopen
+    URLError = urllib2.URLError
+    pathname2url = urllib.pathname2url

ceilometer/openstack/common/rpc/__init__.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # All Rights Reserved.
@@ -227,7 +225,7 @@ def notify(context, topic, msg, envelope=False):
 
 
 def cleanup():
-    """Clean up resoruces in use by implementation.
+    """Clean up resources in use by implementation.
 
     Clean up any resources that have been allocated by the RPC implementation.
     This is typically open connections to a messaging service.  This function

ceilometer/openstack/common/rpc/amqp.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # All Rights Reserved.
@@ -20,9 +18,9 @@
 """
 Shared code between AMQP based openstack.common.rpc implementations.
 
-The code in this module is shared between the rpc implemenations based on AMQP.
+The code in this module is shared between the rpc implementations based on
-Specifically, this includes impl_kombu and impl_qpid.  impl_carrot also uses
+AMQP. Specifically, this includes impl_kombu and impl_qpid. impl_carrot also
-AMQP, but is deprecated and predates this code.
+uses AMQP, but is deprecated and predates this code.
 """
 
 import collections
@@ -189,7 +187,7 @@ class ReplyProxy(ConnectionContext):
     def __init__(self, conf, connection_pool):
         self._call_waiters = {}
         self._num_call_waiters = 0
-        self._num_call_waiters_wrn_threshhold = 10
+        self._num_call_waiters_wrn_threshold = 10
         self._reply_q = 'reply_' + uuid.uuid4().hex
         super(ReplyProxy, self).__init__(conf, connection_pool, pooled=False)
         self.declare_direct_consumer(self._reply_q, self._process_data)
@@ -208,11 +206,11 @@ class ReplyProxy(ConnectionContext):
 
     def add_call_waiter(self, waiter, msg_id):
         self._num_call_waiters += 1
-        if self._num_call_waiters > self._num_call_waiters_wrn_threshhold:
+        if self._num_call_waiters > self._num_call_waiters_wrn_threshold:
             LOG.warn(_('Number of call waiters is greater than warning '
-                       'threshhold: %d. There could be a MulticallProxyWaiter '
+                       'threshold: %d. There could be a MulticallProxyWaiter '
-                       'leak.') % self._num_call_waiters_wrn_threshhold)
+                       'leak.') % self._num_call_waiters_wrn_threshold)
-            self._num_call_waiters_wrn_threshhold *= 2
+            self._num_call_waiters_wrn_threshold *= 2
         self._call_waiters[msg_id] = waiter
 
     def del_call_waiter(self, msg_id):
@@ -241,7 +239,7 @@ def msg_reply(conf, msg_id, reply_q, connection_pool, reply=None,
         _add_unique_id(msg)
         # If a reply_q exists, add the msg_id to the reply and pass the
         # reply_q to direct_send() to use it as the response queue.
-        # Otherwise use the msg_id for backward compatibilty.
+        # Otherwise use the msg_id for backward compatibility.
         if reply_q:
             msg['_msg_id'] = msg_id
             conn.direct_send(reply_q, rpc_common.serialize_msg(msg))

ceilometer/openstack/common/rpc/common.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # All Rights Reserved.
@@ -266,7 +264,7 @@ def _safe_log(log_func, msg, msg_data):
 
     def _fix_passwords(d):
         """Sanitizes the password fields in the dictionary."""
-        for k in d.iterkeys():
+        for k in six.iterkeys(d):
             if k.lower().find('password') != -1:
                 d[k] = '<SANITIZED>'
             elif k.lower() in SANITIZE:

ceilometer/openstack/common/rpc/dispatcher.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 Red Hat, Inc.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

ceilometer/openstack/common/rpc/impl_fake.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 #    Copyright 2011 OpenStack Foundation
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

ceilometer/openstack/common/rpc/impl_kombu.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 #    Copyright 2011 OpenStack Foundation
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -28,6 +26,7 @@ import kombu.connection
 import kombu.entity
 import kombu.messaging
 from oslo.config import cfg
+import six
 
 from ceilometer.openstack.common import excutils
 from ceilometer.openstack.common.gettextutils import _  # noqa
@@ -625,7 +624,7 @@ class Connection(object):
 
         def _declare_consumer():
             consumer = consumer_cls(self.conf, self.channel, topic, callback,
-                                    self.consumer_num.next())
+                                    six.next(self.consumer_num))
             self.consumers.append(consumer)
             return consumer
 
@@ -732,7 +731,7 @@ class Connection(object):
         it = self.iterconsume(limit=limit)
         while True:
             try:
-                it.next()
+                six.next(it)
             except StopIteration:
                 return
 

ceilometer/openstack/common/rpc/impl_qpid.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 #    Copyright 2011 OpenStack Foundation
 #    Copyright 2011 - 2012, Red Hat, Inc.
 #
@@ -23,6 +21,7 @@ import uuid
 import eventlet
 import greenlet
 from oslo.config import cfg
+import six
 
 from ceilometer.openstack.common import excutils
 from ceilometer.openstack.common.gettextutils import _  # noqa
@@ -152,7 +151,7 @@ class ConsumerBase(object):
         self.connect(session)
 
     def connect(self, session):
-        """Declare the reciever on connect."""
+        """Declare the receiver on connect."""
         self._declare_receiver(session)
 
     def reconnect(self, session):
@@ -395,7 +394,7 @@ class DirectPublisher(Publisher):
 class TopicPublisher(Publisher):
     """Publisher class for 'topic'."""
     def __init__(self, conf, session, topic):
-        """init a 'topic' publisher.
+        """Init a 'topic' publisher.
         """
         exchange_name = rpc_amqp.get_control_exchange(conf)
 
@@ -412,7 +411,7 @@ class TopicPublisher(Publisher):
 class FanoutPublisher(Publisher):
     """Publisher class for 'fanout'."""
     def __init__(self, conf, session, topic):
-        """init a 'fanout' publisher.
+        """Init a 'fanout' publisher.
         """
 
         if conf.qpid_topology_version == 1:
@@ -431,7 +430,7 @@ class FanoutPublisher(Publisher):
 class NotifyPublisher(Publisher):
     """Publisher class for notifications."""
     def __init__(self, conf, session, topic):
-        """init a 'topic' publisher.
+        """Init a 'topic' publisher.
         """
         exchange_name = rpc_amqp.get_control_exchange(conf)
         node_opts = {"durable": True}
@@ -539,7 +538,7 @@ class Connection(object):
             consumers = self.consumers
             self.consumers = {}
 
-            for consumer in consumers.itervalues():
+            for consumer in six.itervalues(consumers):
                 consumer.reconnect(self.session)
                 self._register_consumer(consumer)
 
@@ -697,7 +696,7 @@ class Connection(object):
         it = self.iterconsume(limit=limit)
         while True:
             try:
-                it.next()
+                six.next(it)
             except StopIteration:
                 return
 

ceilometer/openstack/common/rpc/impl_zmq.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 #    Copyright 2011 Cloudscaling Group, Inc
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -25,6 +23,8 @@ import uuid
 import eventlet
 import greenlet
 from oslo.config import cfg
+import six
+from six import moves
 
 from ceilometer.openstack.common import excutils
 from ceilometer.openstack.common.gettextutils import _  # noqa
@@ -192,7 +192,7 @@ class ZmqSocket(object):
             # it would be much worse if some of the code calling this
             # were to fail. For now, lets log, and later evaluate
             # if we can safely raise here.
-            LOG.error("ZeroMQ socket could not be closed.")
+            LOG.error(_("ZeroMQ socket could not be closed."))
         self.sock = None
 
     def recv(self, **kwargs):
@@ -221,7 +221,7 @@ class ZmqClient(object):
             return
 
         rpc_envelope = rpc_common.serialize_msg(data[1], envelope)
-        zmq_msg = reduce(lambda x, y: x + y, rpc_envelope.items())
+        zmq_msg = moves.reduce(lambda x, y: x + y, rpc_envelope.items())
         self.outq.send(map(bytes,
                        (msg_id, topic, 'impl_zmq_v2', data[0]) + zmq_msg))
 
@@ -523,8 +523,8 @@ def unflatten_envelope(packenv):
     h = {}
     try:
         while True:
-            k = i.next()
+            k = six.next(i)
-            h[k] = i.next()
+            h[k] = six.next(i)
     except StopIteration:
         return h
 

ceilometer/openstack/common/rpc/matchmaker.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 #    Copyright 2011 Cloudscaling Group, Inc
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

ceilometer/openstack/common/rpc/matchmaker_redis.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 #    Copyright 2013 Cloudscaling Group, Inc
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

ceilometer/openstack/common/rpc/matchmaker_ring.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 #    Copyright 2011-2013 Cloudscaling Group, Inc
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

ceilometer/openstack/common/rpc/proxy.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012-2013 Red Hat, Inc.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -21,7 +19,6 @@ For more information about rpc API version numbers, see:
     rpc/dispatcher.py
 """
 
-
 from ceilometer.openstack.common import rpc
 from ceilometer.openstack.common.rpc import common as rpc_common
 from ceilometer.openstack.common.rpc import serializer as rpc_serializer
@@ -36,7 +33,7 @@ class RpcProxy(object):
     rpc API.
     """
 
-    # The default namespace, which can be overriden in a subclass.
+    # The default namespace, which can be overridden in a subclass.
     RPC_API_NAMESPACE = None
 
     def __init__(self, topic, default_version, version_cap=None,

ceilometer/openstack/common/rpc/securemessage.py

@@ -1,521 +0,0 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright 2013 Red Hat, Inc.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import base64
-import collections
-import os
-import struct
-import time
-
-import requests
-
-from oslo.config import cfg
-
-from ceilometer.openstack.common.crypto import utils as cryptoutils
-from ceilometer.openstack.common import jsonutils
-from ceilometer.openstack.common import log as logging
-
-secure_message_opts = [
-    cfg.BoolOpt('enabled', default=True,
-                help='Whether Secure Messaging (Signing) is enabled,'
-                     ' defaults to enabled'),
-    cfg.BoolOpt('enforced', default=False,
-                help='Whether Secure Messaging (Signing) is enforced,'
-                     ' defaults to not enforced'),
-    cfg.BoolOpt('encrypt', default=False,
-                help='Whether Secure Messaging (Encryption) is enabled,'
-                     ' defaults to not enabled'),
-    cfg.StrOpt('secret_keys_file',
-               help='Path to the file containing the keys, takes precedence'
-                    ' over secret_key'),
-    cfg.MultiStrOpt('secret_key',
-                    help='A list of keys: (ex: name:<base64 encoded key>),'
-                         ' ignored if secret_keys_file is set'),
-    cfg.StrOpt('kds_endpoint',
-               help='KDS endpoint (ex: http://kds.example.com:35357/v3)'),
-]
-secure_message_group = cfg.OptGroup('secure_messages',
-                                    title='Secure Messaging options')
-
-LOG = logging.getLogger(__name__)
-
-
-class SecureMessageException(Exception):
-    """Generic Exception for Secure Messages."""
-
-    msg = "An unknown Secure Message related exception occurred."
-
-    def __init__(self, msg=None):
-        if msg is None:
-            msg = self.msg
-        super(SecureMessageException, self).__init__(msg)
-
-
-class SharedKeyNotFound(SecureMessageException):
-    """No shared key was found and no other external authentication mechanism
-    is available.
-    """
-
-    msg = "Shared Key for [%s] Not Found. (%s)"
-
-    def __init__(self, name, errmsg):
-        super(SharedKeyNotFound, self).__init__(self.msg % (name, errmsg))
-
-
-class InvalidMetadata(SecureMessageException):
-    """The metadata is invalid."""
-
-    msg = "Invalid metadata: %s"
-
-    def __init__(self, err):
-        super(InvalidMetadata, self).__init__(self.msg % err)
-
-
-class InvalidSignature(SecureMessageException):
-    """Signature validation failed."""
-
-    msg = "Failed to validate signature (source=%s, destination=%s)"
-
-    def __init__(self, src, dst):
-        super(InvalidSignature, self).__init__(self.msg % (src, dst))
-
-
-class UnknownDestinationName(SecureMessageException):
-    """The Destination name is unknown to us."""
-
-    msg = "Invalid destination name (%s)"
-
-    def __init__(self, name):
-        super(UnknownDestinationName, self).__init__(self.msg % name)
-
-
-class InvalidEncryptedTicket(SecureMessageException):
-    """The Encrypted Ticket could not be successfully handled."""
-
-    msg = "Invalid Ticket (source=%s, destination=%s)"
-
-    def __init__(self, src, dst):
-        super(InvalidEncryptedTicket, self).__init__(self.msg % (src, dst))
-
-
-class InvalidExpiredTicket(SecureMessageException):
-    """The ticket received is already expired."""
-
-    msg = "Expired ticket (source=%s, destination=%s)"
-
-    def __init__(self, src, dst):
-        super(InvalidExpiredTicket, self).__init__(self.msg % (src, dst))
-
-
-class CommunicationError(SecureMessageException):
-    """The Communication with the KDS failed."""
-
-    msg = "Communication Error (target=%s): %s"
-
-    def __init__(self, target, errmsg):
-        super(CommunicationError, self).__init__(self.msg % (target, errmsg))
-
-
-class InvalidArgument(SecureMessageException):
-    """Bad initialization argument."""
-
-    msg = "Invalid argument: %s"
-
-    def __init__(self, errmsg):
-        super(InvalidArgument, self).__init__(self.msg % errmsg)
-
-
-Ticket = collections.namedtuple('Ticket', ['skey', 'ekey', 'esek'])
-
-
-class KeyStore(object):
-    """A storage class for Signing and Encryption Keys.
-
-    This class creates an object that holds Generic Keys like Signing
-    Keys, Encryption Keys, Encrypted SEK Tickets ...
-    """
-
-    def __init__(self):
-        self._kvps = dict()
-
-    def _get_key_name(self, source, target, ktype):
-        return (source, target, ktype)
-
-    def _put(self, src, dst, ktype, expiration, data):
-        name = self._get_key_name(src, dst, ktype)
-        self._kvps[name] = (expiration, data)
-
-    def _get(self, src, dst, ktype):
-        name = self._get_key_name(src, dst, ktype)
-        if name in self._kvps:
-            expiration, data = self._kvps[name]
-            if expiration > time.time():
-                return data
-            else:
-                del self._kvps[name]
-
-        return None
-
-    def clear(self):
-        """Wipes the store clear of all data."""
-        self._kvps.clear()
-
-    def put_ticket(self, source, target, skey, ekey, esek, expiration):
-        """Puts a sek pair in the cache.
-
-        :param source: Client name
-        :param target: Target name
-        :param skey: The Signing Key
-        :param ekey: The Encription Key
-        :param esek: The token encrypted with the target key
-        :param expiration: Expiration time in seconds since Epoch
-        """
-        keys = Ticket(skey, ekey, esek)
-        self._put(source, target, 'ticket', expiration, keys)
-
-    def get_ticket(self, source, target):
-        """Returns a Ticket (skey, ekey, esek) namedtuple for the
-           source/target pair.
-        """
-        return self._get(source, target, 'ticket')
-
-
-_KEY_STORE = KeyStore()
-
-
-class _KDSClient(object):
-
-    USER_AGENT = 'oslo-incubator/rpc'
-
-    def __init__(self, endpoint=None, timeout=None):
-        """A KDS Client class."""
-
-        self._endpoint = endpoint
-        if timeout is not None:
-            self.timeout = float(timeout)
-        else:
-            self.timeout = None
-
-    def _do_get(self, url, request):
-        req_kwargs = dict()
-        req_kwargs['headers'] = dict()
-        req_kwargs['headers']['User-Agent'] = self.USER_AGENT
-        req_kwargs['headers']['Content-Type'] = 'application/json'
-        req_kwargs['data'] = jsonutils.dumps({'request': request})
-        if self.timeout is not None:
-            req_kwargs['timeout'] = self.timeout
-
-        try:
-            resp = requests.get(url, **req_kwargs)
-        except requests.ConnectionError as e:
-            err = "Unable to establish connection. %s" % e
-            raise CommunicationError(url, err)
-
-        return resp
-
-    def _get_reply(self, url, resp):
-        if resp.text:
-            try:
-                body = jsonutils.loads(resp.text)
-                reply = body['reply']
-            except (KeyError, TypeError, ValueError):
-                msg = "Failed to decode reply: %s" % resp.text
-                raise CommunicationError(url, msg)
-        else:
-            msg = "No reply data was returned."
-            raise CommunicationError(url, msg)
-
-        return reply
-
-    def _get_ticket(self, request, url=None, redirects=10):
-        """Send an HTTP request.
-
-        Wraps around 'requests' to handle redirects and common errors.
-        """
-        if url is None:
-            if not self._endpoint:
-                raise CommunicationError(url, 'Endpoint not configured')
-            url = self._endpoint + '/kds/ticket'
-
-        while redirects:
-            resp = self._do_get(url, request)
-            if resp.status_code in (301, 302, 305):
-                # Redirected. Reissue the request to the new location.
-                url = resp.headers['location']
-                redirects -= 1
-                continue
-            elif resp.status_code != 200:
-                msg = "Request returned failure status: %s (%s)"
-                err = msg % (resp.status_code, resp.text)
-                raise CommunicationError(url, err)
-
-            return self._get_reply(url, resp)
-
-        raise CommunicationError(url, "Too many redirections, giving up!")
-
-    def get_ticket(self, source, target, crypto, key):
-
-        # prepare metadata
-        md = {'requestor': source,
-              'target': target,
-              'timestamp': time.time(),
-              'nonce': struct.unpack('Q', os.urandom(8))[0]}
-        metadata = base64.b64encode(jsonutils.dumps(md))
-
-        # sign metadata
-        signature = crypto.sign(key, metadata)
-
-        # HTTP request
-        reply = self._get_ticket({'metadata': metadata,
-                                  'signature': signature})
-
-        # verify reply
-        signature = crypto.sign(key, (reply['metadata'] + reply['ticket']))
-        if signature != reply['signature']:
-            raise InvalidEncryptedTicket(md['source'], md['destination'])
-        md = jsonutils.loads(base64.b64decode(reply['metadata']))
-        if ((md['source'] != source or
-             md['destination'] != target or
-             md['expiration'] < time.time())):
-            raise InvalidEncryptedTicket(md['source'], md['destination'])
-
-        # return ticket data
-        tkt = jsonutils.loads(crypto.decrypt(key, reply['ticket']))
-
-        return tkt, md['expiration']
-
-
-# we need to keep a global nonce, as this value should never repeat non
-# matter how many SecureMessage objects we create
-_NONCE = None
-
-
-def _get_nonce():
-    """We keep a single counter per instance, as it is so huge we can't
-    possibly cycle through within 1/100 of a second anyway.
-    """
-
-    global _NONCE
-    # Lazy initialize, for now get a random value, multiply by 2^32 and
-    # use it as the nonce base. The counter itself will rotate after
-    # 2^32 increments.
-    if _NONCE is None:
-        _NONCE = [struct.unpack('I', os.urandom(4))[0], 0]
-
-    # Increment counter and wrap at 2^32
-    _NONCE[1] += 1
-    if _NONCE[1] > 0xffffffff:
-        _NONCE[1] = 0
-
-    # Return base + counter
-    return long((_NONCE[0] * 0xffffffff)) + _NONCE[1]
-
-
-class SecureMessage(object):
-    """A Secure Message object.
-
-    This class creates a signing/encryption facility for RPC messages.
-    It encapsulates all the necessary crypto primitives to insulate
-    regular code from the intricacies of message authentication, validation
-    and optionally encryption.
-
-    :param topic: The topic name of the queue
-    :param host: The server name, together with the topic it forms a unique
-                 name that is used to source signing keys, and verify
-                 incoming messages.
-    :param conf: a ConfigOpts object
-    :param key: (optional) explicitly pass in endpoint private key.
-                  If not provided it will be sourced from the service config
-    :param key_store: (optional) Storage class for local caching
-    :param encrypt: (defaults to False) Whether to encrypt messages
-    :param enctype: (defaults to AES) Cipher to use
-    :param hashtype: (defaults to SHA256) Hash function to use for signatures
-    """
-
-    def __init__(self, topic, host, conf, key=None, key_store=None,
-                 encrypt=None, enctype='AES', hashtype='SHA256'):
-
-        conf.register_group(secure_message_group)
-        conf.register_opts(secure_message_opts, group='secure_messages')
-
-        self._name = '%s.%s' % (topic, host)
-        self._key = key
-        self._conf = conf.secure_messages
-        self._encrypt = self._conf.encrypt if (encrypt is None) else encrypt
-        self._crypto = cryptoutils.SymmetricCrypto(enctype, hashtype)
-        self._hkdf = cryptoutils.HKDF(hashtype)
-        self._kds = _KDSClient(self._conf.kds_endpoint)
-
-        if self._key is None:
-            self._key = self._init_key(topic, self._name)
-        if self._key is None:
-            err = "Secret Key (or key file) is missing or malformed"
-            raise SharedKeyNotFound(self._name, err)
-
-        self._key_store = key_store or _KEY_STORE
-
-    def _init_key(self, topic, name):
-        keys = None
-        if self._conf.secret_keys_file:
-            with open(self._conf.secret_keys_file, 'r') as f:
-                keys = f.readlines()
-        elif self._conf.secret_key:
-            keys = self._conf.secret_key
-
-        if keys is None:
-            return None
-
-        for k in keys:
-            if k[0] == '#':
-                continue
-            if ':' not in k:
-                break
-            svc, key = k.split(':', 1)
-            if svc == topic or svc == name:
-                return base64.b64decode(key)
-
-        return None
-
-    def _split_key(self, key, size):
-        sig_key = key[:size]
-        enc_key = key[size:]
-        return sig_key, enc_key
-
-    def _decode_esek(self, key, source, target, timestamp, esek):
-        """This function decrypts the esek buffer passed in and returns a
-        KeyStore to be used to check and decrypt the received message.
-
-        :param key: The key to use to decrypt the ticket (esek)
-        :param source: The name of the source service
-        :param traget: The name of the target service
-        :param timestamp: The incoming message timestamp
-        :param esek: a base64 encoded encrypted block containing a JSON string
-        """
-        rkey = None
-
-        try:
-            s = self._crypto.decrypt(key, esek)
-            j = jsonutils.loads(s)
-
-            rkey = base64.b64decode(j['key'])
-            expiration = j['timestamp'] + j['ttl']
-            if j['timestamp'] > timestamp or timestamp > expiration:
-                raise InvalidExpiredTicket(source, target)
-
-        except Exception:
-            raise InvalidEncryptedTicket(source, target)
-
-        info = '%s,%s,%s' % (source, target, str(j['timestamp']))
-
-        sek = self._hkdf.expand(rkey, info, len(key) * 2)
-
-        return self._split_key(sek, len(key))
-
-    def _get_ticket(self, target):
-        """This function will check if we already have a SEK for the specified
-        target in the cache, or will go and try to fetch a new SEK from the key
-        server.
-
-        :param target: The name of the target service
-        """
-        ticket = self._key_store.get_ticket(self._name, target)
-
-        if ticket is not None:
-            return ticket
-
-        tkt, expiration = self._kds.get_ticket(self._name, target,
-                                               self._crypto, self._key)
-
-        self._key_store.put_ticket(self._name, target,
-                                   base64.b64decode(tkt['skey']),
-                                   base64.b64decode(tkt['ekey']),
-                                   tkt['esek'], expiration)
-        return self._key_store.get_ticket(self._name, target)
-
-    def encode(self, version, target, json_msg):
-        """This is the main encoding function.
-
-        It takes a target and a message and returns a tuple consisting of a
-        JSON serialized metadata object, a JSON serialized (and optionally
-        encrypted) message, and a signature.
-
-        :param version: the current envelope version
-        :param target: The name of the target service (usually with hostname)
-        :param json_msg: a serialized json message object
-        """
-        ticket = self._get_ticket(target)
-
-        metadata = jsonutils.dumps({'source': self._name,
-                                    'destination': target,
-                                    'timestamp': time.time(),
-                                    'nonce': _get_nonce(),
-                                    'esek': ticket.esek,
-                                    'encryption': self._encrypt})
-
-        message = json_msg
-        if self._encrypt:
-            message = self._crypto.encrypt(ticket.ekey, message)
-
-        signature = self._crypto.sign(ticket.skey,
-                                      version + metadata + message)
-
-        return (metadata, message, signature)
-
-    def decode(self, version, metadata, message, signature):
-        """This is the main decoding function.
-
-        It takes a version, metadata, message and signature strings and
-        returns a tuple with a (decrypted) message and metadata or raises
-        an exception in case of error.
-
-        :param version: the current envelope version
-        :param metadata: a JSON serialized object with metadata for validation
-        :param message: a JSON serialized (base64 encoded encrypted) message
-        :param signature: a base64 encoded signature
-        """
-        md = jsonutils.loads(metadata)
-
-        check_args = ('source', 'destination', 'timestamp',
-                      'nonce', 'esek', 'encryption')
-        for arg in check_args:
-            if arg not in md:
-                raise InvalidMetadata('Missing metadata "%s"' % arg)
-
-        if md['destination'] != self._name:
-            # TODO(simo) handle group keys by checking target
-            raise UnknownDestinationName(md['destination'])
-
-        try:
-            skey, ekey = self._decode_esek(self._key,
-                                           md['source'], md['destination'],
-                                           md['timestamp'], md['esek'])
-        except InvalidExpiredTicket:
-            raise
-        except Exception:
-            raise InvalidMetadata('Failed to decode ESEK for %s/%s' % (
-                                  md['source'], md['destination']))
-
-        sig = self._crypto.sign(skey, version + metadata + message)
-
-        if sig != signature:
-            raise InvalidSignature(md['source'], md['destination'])
-
-        if md['encryption'] is True:
-            msg = self._crypto.decrypt(ekey, message)
-        else:
-            msg = message
-
-        return (md, msg)

ceilometer/openstack/common/rpc/serializer.py

@@ -15,6 +15,7 @@
 """Provides the definition of an RPC serialization handler"""
 
 import abc
+
 import six
 
 

ceilometer/openstack/common/rpc/service.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # All Rights Reserved.

ceilometer/openstack/common/rpc/zmq_receiver.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 #    Copyright 2011 OpenStack Foundation
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

ceilometer/openstack/common/service.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # Copyright 2011 Justin Santa Barbara
@@ -20,6 +18,7 @@
 """Generic Node base class for all workers that run on hosts."""
 
 import errno
+import logging as std_logging
 import os
 import random
 import signal
@@ -28,7 +27,6 @@ import time
 
 import eventlet
 from eventlet import event
-import logging as std_logging
 from oslo.config import cfg
 
 from ceilometer.openstack.common import eventlet_backdoor
@@ -129,7 +127,7 @@ class ServiceLauncher(Launcher):
     def handle_signal(self):
         _set_signals_handler(self._handle_signal)
 
-    def _wait_for_exit_or_signal(self):
+    def _wait_for_exit_or_signal(self, ready_callback=None):
         status = None
         signo = 0
 
@@ -137,6 +135,8 @@ class ServiceLauncher(Launcher):
         CONF.log_opt_values(LOG, std_logging.DEBUG)
 
         try:
+            if ready_callback:
+                ready_callback()
             super(ServiceLauncher, self).wait()
         except SignalExit as exc:
             signame = _signo_to_signame(exc.signo)
@@ -156,10 +156,10 @@ class ServiceLauncher(Launcher):
 
         return status, signo
 
-    def wait(self):
+    def wait(self, ready_callback=None):
         while True:
             self.handle_signal()
-            status, signo = self._wait_for_exit_or_signal()
+            status, signo = self._wait_for_exit_or_signal(ready_callback)
             if not _is_sighup(signo):
                 return status
             self.restart()
@@ -218,7 +218,7 @@ class ProcessLauncher(object):
         signal.signal(signal.SIGINT, signal.SIG_IGN)
 
     def _child_wait_for_exit_or_signal(self, launcher):
-        status = None
+        status = 0
         signo = 0
 
         # NOTE(johannes): All exceptions are caught to ensure this

ceilometer/openstack/common/sslutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2013 IBM Corp.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

ceilometer/openstack/common/strutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -101,7 +99,7 @@ def safe_decode(text, incoming=None, errors='strict'):
         values http://docs.python.org/2/library/codecs.html
     :returns: text or a unicode `incoming` encoded
                 representation of it.
-    :raises TypeError: If text is not an isntance of str
+    :raises TypeError: If text is not an instance of str
     """
     if not isinstance(text, six.string_types):
         raise TypeError("%s can't be decoded" % type(text))
@@ -144,7 +142,7 @@ def safe_encode(text, incoming=None,
         values http://docs.python.org/2/library/codecs.html
     :returns: text or a bytestring `encoding` encoded
                 representation of it.
-    :raises TypeError: If text is not an isntance of str
+    :raises TypeError: If text is not an instance of str
     """
     if not isinstance(text, six.string_types):
         raise TypeError("%s can't be encoded" % type(text))

ceilometer/openstack/common/test.py

@@ -1,6 +1,4 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
-
-# Copyright 2010-2011 OpenStack Foundation
 # All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -22,6 +20,8 @@ import os
 import fixtures
 import testtools
 
+_TRUE_VALUES = ('True', 'true', '1', 'yes')
+
 
 class BaseTestCase(testtools.TestCase):
 
@@ -31,6 +31,7 @@ class BaseTestCase(testtools.TestCase):
         self._fake_output()
         self.useFixture(fixtures.FakeLogger('ceilometer.openstack.common'))
         self.useFixture(fixtures.NestedTempfile())
+        self.useFixture(fixtures.TempHomeDir())
 
     def _set_timeout(self):
         test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
@@ -43,11 +44,9 @@ class BaseTestCase(testtools.TestCase):
             self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
 
     def _fake_output(self):
-        if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
+        if os.environ.get('OS_STDOUT_CAPTURE') in _TRUE_VALUES:
-                os.environ.get('OS_STDOUT_CAPTURE') == '1'):
             stdout = self.useFixture(fixtures.StringStream('stdout')).stream
             self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
-        if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
+        if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES:
-                os.environ.get('OS_STDERR_CAPTURE') == '1'):
             stderr = self.useFixture(fixtures.StringStream('stderr')).stream
             self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))

ceilometer/openstack/common/threadgroup.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 Red Hat, Inc.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -48,6 +46,9 @@ class Thread(object):
     def wait(self):
         return self.thread.wait()
 
+    def link(self, func, *args, **kwargs):
+        self.thread.link(func, *args, **kwargs)
+
 
 class ThreadGroup(object):
     """The point of the ThreadGroup classis to:
@@ -79,6 +80,7 @@ class ThreadGroup(object):
         gt = self.pool.spawn(callback, *args, **kwargs)
         th = Thread(gt, self)
         self.threads.append(th)
+        return th
 
     def thread_done(self, thread):
         self.threads.remove(thread)

ceilometer/openstack/common/timeutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -50,9 +48,9 @@ def parse_isotime(timestr):
     try:
         return iso8601.parse_date(timestr)
     except iso8601.ParseError as e:
-        raise ValueError(unicode(e))
+        raise ValueError(six.text_type(e))
     except TypeError as e:
-        raise ValueError(unicode(e))
+        raise ValueError(six.text_type(e))
 
 
 def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
@@ -178,6 +176,15 @@ def delta_seconds(before, after):
     datetime objects (as a float, to microsecond resolution).
     """
     delta = after - before
+    return total_seconds(delta)
+
+
+def total_seconds(delta):
+    """Return the total seconds of datetime.timedelta object.
+
+    Compute total seconds of datetime.timedelta, datetime.timedelta
+    doesn't have method total_seconds in Python2.6, calculate it manually.
+    """
     try:
         return delta.total_seconds()
     except AttributeError:

ceilometer/openstack/common/versionutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright (c) 2013 OpenStack Foundation
 # All Rights Reserved.
 #

ceilometer/openstack/common/xmlutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2013 IBM Corp.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

etc/ceilometer/ceilometer.conf.sample

@@ -24,6 +24,7 @@
 #
 
 # Source for samples emited on this instance (string value)
+# Deprecated group/name - [DEFAULT]/counter_source
 #sample_source=openstack
 
 
@@ -108,6 +109,7 @@
 #
 
 # Exchange name for Neutron notifications (string value)
+# Deprecated group/name - [DEFAULT]/quantum_control_exchange
 #neutron_control_exchange=neutron
 
 
@@ -202,7 +204,7 @@
 #logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s
 
 # list of logger=LEVEL pairs (list value)
-#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,keystone=INFO,qpid=WARN,sqlalchemy=WARN,suds=INFO
+#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,keystone=INFO,qpid=WARN,sqlalchemy=WARN,suds=INFO,iso8601=WARN
 
 # publish error events (boolean value)
 #publish_errors=false
@@ -223,6 +225,7 @@
 # configuration to any other existing logging options. Please
 # see the Python logging module documentation for details on
 # logging configuration files. (string value)
+# Deprecated group/name - [DEFAULT]/log_config
 #log_config_append=<None>
 
 # DEPRECATED. A logging.Formatter log message format string
@@ -238,10 +241,12 @@
 
 # (Optional) Name of log file to output to. If no default is
 # set, logging will go to stdout. (string value)
+# Deprecated group/name - [DEFAULT]/logfile
 #log_file=<None>
 
 # (Optional) The base directory used for relative --log-file
 # paths (string value)
+# Deprecated group/name - [DEFAULT]/logdir
 #log_dir=<None>
 
 # Use syslog for logging. (boolean value)
@@ -257,6 +262,7 @@
 
 # the maximum body size per each request(bytes) (integer
 # value)
+# Deprecated group/name - [DEFAULT]/osapi_max_request_body_size
 #max_request_body_size=114688
 
 
@@ -319,8 +325,8 @@
 # by impl_zmq. (integer value)
 #rpc_cast_timeout=30
 
-# Modules of exceptions that are permitted to be recreatedupon
+# Modules of exceptions that are permitted to be recreated
-# receiving exception data from an rpc call. (list value)
+# upon receiving exception data from an rpc call. (list value)
 #allowed_rpc_exception_modules=nova.exception,cinder.exception,exceptions
 
 # If passed, use a fake RabbitMQ provider (boolean value)
@@ -336,6 +342,7 @@
 #
 
 # Use durable queues in amqp. (boolean value)
+# Deprecated group/name - [DEFAULT]/rabbit_durable_queues
 #amqp_durable_queues=false
 
 # Auto-delete queues in amqp. (boolean value)
@@ -551,10 +558,12 @@
 #
 
 # The backend to use for db (string value)
+# Deprecated group/name - [DEFAULT]/db_backend
 #backend=sqlalchemy
 
 # Enable the experimental use of thread pooling for all DB API
 # calls (boolean value)
+# Deprecated group/name - [DEFAULT]/dbapi_use_tpool
 #use_tpool=false
 
 
@@ -564,6 +573,9 @@
 
 # The SQLAlchemy connection string used to connect to the
 # database (string value)
+# Deprecated group/name - [DEFAULT]/sql_connection
+# Deprecated group/name - [DATABASE]/sql_connection
+# Deprecated group/name - [sql]/connection
 #connection=sqlite:////ceilometer/openstack/common/db/$sqlite_db
 
 # The SQLAlchemy connection string used to connect to the
@@ -572,38 +584,53 @@
 
 # timeout before idle sql connections are reaped (integer
 # value)
+# Deprecated group/name - [DEFAULT]/sql_idle_timeout
+# Deprecated group/name - [DATABASE]/sql_idle_timeout
 #idle_timeout=3600
 
 # Minimum number of SQL connections to keep open in a pool
 # (integer value)
+# Deprecated group/name - [DEFAULT]/sql_min_pool_size
+# Deprecated group/name - [DATABASE]/sql_min_pool_size
 #min_pool_size=1
 
 # Maximum number of SQL connections to keep open in a pool
 # (integer value)
+# Deprecated group/name - [DEFAULT]/sql_max_pool_size
+# Deprecated group/name - [DATABASE]/sql_max_pool_size
 #max_pool_size=<None>
 
 # maximum db connection retries during startup. (setting -1
 # implies an infinite retry count) (integer value)
+# Deprecated group/name - [DEFAULT]/sql_max_retries
+# Deprecated group/name - [DATABASE]/sql_max_retries
 #max_retries=10
 
 # interval between retries of opening a sql connection
 # (integer value)
+# Deprecated group/name - [DEFAULT]/sql_retry_interval
+# Deprecated group/name - [DATABASE]/reconnect_interval
 #retry_interval=10
 
 # If set, use this value for max_overflow with sqlalchemy
 # (integer value)
+# Deprecated group/name - [DEFAULT]/sql_max_overflow
+# Deprecated group/name - [DATABASE]/sqlalchemy_max_overflow
 #max_overflow=<None>
 
 # Verbosity of SQL debugging information. 0=None,
 # 100=Everything (integer value)
+# Deprecated group/name - [DEFAULT]/sql_connection_debug
 #connection_debug=0
 
 # Add python stack traces to SQL as comment strings (boolean
 # value)
+# Deprecated group/name - [DEFAULT]/sql_connection_trace
 #connection_trace=false
 
 # If set, use this value for pool_timeout with sqlalchemy
 # (integer value)
+# Deprecated group/name - [DATABASE]/sqlalchemy_pool_timeout
 #pool_timeout=<None>
 
 
@@ -667,6 +694,7 @@
 # Period of evaluation cycle, should be >= than configured
 # pipeline interval for collection of underlying metrics.
 # (integer value)
+# Deprecated group/name - [alarm]/threshold_evaluation_interval
 #evaluation_interval=60
 
 # Class to launch as alarm evaluation service (string value)
@@ -698,6 +726,7 @@
 #
 
 # The port for the ceilometer API server (integer value)
+# Deprecated group/name - [DEFAULT]/metering_api_port
 #port=8777
 
 # The listen IP for the ceilometer API server (string value)
@@ -760,119 +789,6 @@
 #backup_count=0
 
 
-[keystone_authtoken]
-
-#
-# Options defined in keystoneclient.middleware.auth_token
-#
-
-# Prefix to prepend at the beginning of the path (string
-# value)
-#auth_admin_prefix=
-
-# Host providing the admin Identity API endpoint (string
-# value)
-#auth_host=127.0.0.1
-
-# Port of the admin Identity API endpoint (integer value)
-#auth_port=35357
-
-# Protocol of the admin Identity API endpoint(http or https)
-# (string value)
-#auth_protocol=https
-
-# Complete public Identity API endpoint (string value)
-#auth_uri=<None>
-
-# API version of the admin Identity API endpoint (string
-# value)
-#auth_version=<None>
-
-# Do not handle authorization requests within the middleware,
-# but delegate the authorization decision to downstream WSGI
-# components (boolean value)
-#delay_auth_decision=false
-
-# Request timeout value for communicating with Identity API
-# server. (boolean value)
-#http_connect_timeout=<None>
-
-# How many times are we trying to reconnect when communicating
-# with Identity API Server. (integer value)
-#http_request_max_retries=3
-
-# Allows to pass in the name of a fake http_handler callback
-# function used instead of httplib.HTTPConnection or
-# httplib.HTTPSConnection. Useful for unit testing where
-# network is not available. (string value)
-#http_handler=<None>
-
-# Single shared secret with the Keystone configuration used
-# for bootstrapping a Keystone installation, or otherwise
-# bypassing the normal authentication process. (string value)
-#admin_token=<None>
-
-# Keystone account username (string value)
-#admin_user=<None>
-
-# Keystone account password (string value)
-#admin_password=<None>
-
-# Keystone service account tenant name to validate user tokens
-# (string value)
-#admin_tenant_name=admin
-
-# Env key for the swift cache (string value)
-#cache=<None>
-
-# Required if Keystone server requires client certificate
-# (string value)
-#certfile=<None>
-
-# Required if Keystone server requires client certificate
-# (string value)
-#keyfile=<None>
-
-# A PEM encoded Certificate Authority to use when verifying
-# HTTPs connections. Defaults to system CAs. (string value)
-#cafile=<None>
-
-# Verify HTTPS connections. (boolean value)
-#insecure=false
-
-# Directory used to cache files related to PKI tokens (string
-# value)
-#signing_dir=<None>
-
-# If defined, the memcache server(s) to use for caching (list
-# value)
-#memcached_servers=<None>
-
-# In order to prevent excessive requests and validations, the
-# middleware uses an in-memory cache for the tokens the
-# Keystone API returns. This is only valid if memcache_servers
-# is defined. Set to -1 to disable caching completely.
-# (integer value)
-#token_cache_time=300
-
-# Value only used for unit testing (integer value)
-#revocation_cache_time=1
-
-# (optional) if defined, indicate whether token data should be
-# authenticated or authenticated and encrypted. Acceptable
-# values are MAC or ENCRYPT.  If MAC, token data is
-# authenticated (with HMAC) in the cache. If ENCRYPT, token
-# data is encrypted and authenticated in the cache. If the
-# value is not one of these options or empty, auth_token will
-# raise an exception on initialization. (string value)
-#memcache_security_strategy=<None>
-
-# (optional, mandatory if memcache_security_strategy is
-# defined) this string is used for key derivation. (string
-# value)
-#memcache_secret_key=<None>
-
-
 [collector]
 
 #
@@ -894,6 +810,7 @@
 #
 
 # Matchmaker ring file (JSON) (string value)
+# Deprecated group/name - [DEFAULT]/matchmaker_ringfile
 #ringfile=/etc/oslo/matchmaker_ring.json
 
 

requirements.txt

@@ -19,6 +19,7 @@ python-novaclient>=2.15.0
 python-keystoneclient>=0.4.1
 python-ceilometerclient>=1.0.6
 python-swiftclient>=1.5
+lockfile>=0.8
 lxml>=2.3
 requests>=1.1
 six>=1.4.1

tools/config/generate_sample.sh

@@ -73,6 +73,7 @@ then
 fi
 
 BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'`
+find $TARGETDIR -type f -name "*.pyc" -delete
 FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \
         -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u)
 
@@ -86,7 +87,13 @@ export EVENTLET_NO_GREENDNS=yes
 
 OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
 [ "$OS_VARS" ] && eval "unset \$OS_VARS"
-
+DEFAULT_MODULEPATH=ceilometer.openstack.common.config.generator
-MODULEPATH=ceilometer.openstack.common.config.generator
+MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
 OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample
 python -m $MODULEPATH $FILES > $OUTPUTFILE
+
+# Hook to allow projects to append custom config file snippets
+CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)
+for CONCAT_FILE in $CONCAT_FILES; do
+    cat $CONCAT_FILE >> $OUTPUTFILE
+done