From fba8c39b622b8cb50a51d58b674ca7cddff35da1 Mon Sep 17 00:00:00 2001 From: Gordon Chung Date: Fri, 29 Nov 2013 12:41:47 -0500 Subject: [PATCH] sync with oslo-incubator - py3 compatibility fixes - drop uuidutils and rpc/securemessage - lockutils fix - config generator fix synched with commit-id: 0d2546b228f13a42355bc54e82256bf17a2eb478 Change-Id: I2c2b69da1b7d834b42c185bb8e2667ce748df33a --- .../openstack/common/config/generator.py | 36 +- ceilometer/openstack/common/context.py | 7 +- ceilometer/openstack/common/db/__init__.py | 2 - ceilometer/openstack/common/db/api.py | 2 - ceilometer/openstack/common/db/exception.py | 7 +- .../common/db/sqlalchemy/__init__.py | 2 - .../openstack/common/db/sqlalchemy/models.py | 2 - .../common/db/sqlalchemy/provision.py | 187 +++++++ .../openstack/common/db/sqlalchemy/session.py | 21 +- .../common/db/sqlalchemy/test_migrations.py | 36 +- .../openstack/common/db/sqlalchemy/utils.py | 2 - .../openstack/common/deprecated/wsgi.py | 7 +- .../openstack/common/eventlet_backdoor.py | 2 - ceilometer/openstack/common/excutils.py | 2 - ceilometer/openstack/common/fileutils.py | 2 - ceilometer/openstack/common/fixture/config.py | 3 +- .../openstack/common/fixture/lockutils.py | 51 ++ .../openstack/common/fixture/mockpatch.py | 2 - .../openstack/common/fixture/moxstubout.py | 7 +- ceilometer/openstack/common/gettextutils.py | 24 +- ceilometer/openstack/common/importutils.py | 2 - ceilometer/openstack/common/jsonutils.py | 2 - ceilometer/openstack/common/local.py | 2 - ceilometer/openstack/common/lockutils.py | 82 ++- ceilometer/openstack/common/log.py | 63 ++- ceilometer/openstack/common/log_handler.py | 6 +- ceilometer/openstack/common/loopingcall.py | 2 - .../openstack/common/middleware/audit.py | 2 - .../openstack/common/middleware/context.py | 6 +- .../common/middleware/correlation_id.py | 7 +- .../openstack/common/middleware/notifier.py | 4 +- .../openstack/common/middleware/sizelimit.py | 2 - ceilometer/openstack/common/network_utils.py | 8 +- ceilometer/openstack/common/policy.py | 6 +- ceilometer/openstack/common/processutils.py | 248 +++++++++ .../{uuidutils.py => py3kcompat/__init__.py} | 29 +- .../openstack/common/py3kcompat/urlutils.py | 63 +++ ceilometer/openstack/common/rpc/__init__.py | 6 +- ceilometer/openstack/common/rpc/amqp.py | 20 +- ceilometer/openstack/common/rpc/common.py | 4 +- ceilometer/openstack/common/rpc/dispatcher.py | 2 - ceilometer/openstack/common/rpc/impl_fake.py | 2 - ceilometer/openstack/common/rpc/impl_kombu.py | 7 +- ceilometer/openstack/common/rpc/impl_qpid.py | 15 +- ceilometer/openstack/common/rpc/impl_zmq.py | 12 +- ceilometer/openstack/common/rpc/matchmaker.py | 2 - .../openstack/common/rpc/matchmaker_redis.py | 2 - .../openstack/common/rpc/matchmaker_ring.py | 2 - ceilometer/openstack/common/rpc/proxy.py | 5 +- .../openstack/common/rpc/securemessage.py | 521 ------------------ ceilometer/openstack/common/rpc/serializer.py | 1 + ceilometer/openstack/common/rpc/service.py | 2 - .../openstack/common/rpc/zmq_receiver.py | 2 - ceilometer/openstack/common/service.py | 14 +- ceilometer/openstack/common/sslutils.py | 2 - ceilometer/openstack/common/strutils.py | 6 +- ceilometer/openstack/common/test.py | 31 +- ceilometer/openstack/common/threadgroup.py | 6 +- ceilometer/openstack/common/timeutils.py | 15 +- ceilometer/openstack/common/versionutils.py | 2 - ceilometer/openstack/common/xmlutils.py | 2 - etc/ceilometer/ceilometer.conf.sample | 149 ++--- requirements.txt | 1 + tools/config/generate_sample.sh | 11 +- 64 files changed, 856 insertions(+), 926 deletions(-) create mode 100644 ceilometer/openstack/common/db/sqlalchemy/provision.py create mode 100644 ceilometer/openstack/common/fixture/lockutils.py create mode 100644 ceilometer/openstack/common/processutils.py rename ceilometer/openstack/common/{uuidutils.py => py3kcompat/__init__.py} (53%) create mode 100644 ceilometer/openstack/common/py3kcompat/urlutils.py delete mode 100644 ceilometer/openstack/common/rpc/securemessage.py diff --git a/ceilometer/openstack/common/config/generator.py b/ceilometer/openstack/common/config/generator.py index 20fe4dd1e..e5183b7b4 100644 --- a/ceilometer/openstack/common/config/generator.py +++ b/ceilometer/openstack/common/config/generator.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 SINA Corporation # All Rights Reserved. # @@ -28,6 +26,7 @@ import sys import textwrap from oslo.config import cfg +import six from ceilometer.openstack.common import gettextutils from ceilometer.openstack.common import importutils @@ -78,12 +77,15 @@ def generate(srcfiles): # The options list is a list of (module, options) tuples opts_by_group = {'DEFAULT': []} - for module_name in os.getenv( - "OSLO_CONFIG_GENERATOR_EXTRA_MODULES", "").split(','): - module = _import_module(module_name) - if module: - for group, opts in _list_opts(module): - opts_by_group.setdefault(group, []).append((module_name, opts)) + extra_modules = os.getenv("CEILOMETER_CONFIG_GENERATOR_EXTRA_MODULES", "") + if extra_modules: + for module_name in extra_modules.split(','): + module_name = module_name.strip() + module = _import_module(module_name) + if module: + for group, opts in _list_opts(module): + opts_by_group.setdefault(group, []).append((module_name, + opts)) for pkg_name in pkg_names: mods = mods_by_pkg.get(pkg_name) @@ -94,7 +96,7 @@ def generate(srcfiles): mod_obj = _import_module(mod_str) if not mod_obj: - continue + raise RuntimeError("Unable to import module %s" % mod_str) for group, opts in _list_opts(mod_obj): opts_by_group.setdefault(group, []).append((mod_str, opts)) @@ -111,10 +113,8 @@ def _import_module(mod_str): return sys.modules[mod_str[4:]] else: return importutils.import_module(mod_str) - except ImportError as ie: - sys.stderr.write("%s\n" % str(ie)) - return None - except Exception: + except Exception as e: + sys.stderr.write("Error importing module %s: %s\n" % (mod_str, str(e))) return None @@ -221,11 +221,19 @@ def _print_opt(opt): sys.exit(1) opt_help += ' (' + OPT_TYPES[opt_type] + ')' print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH))) + if opt.deprecated_opts: + for deprecated_opt in opt.deprecated_opts: + if deprecated_opt.name: + deprecated_group = (deprecated_opt.group if + deprecated_opt.group else "DEFAULT") + print('# Deprecated group/name - [%s]/%s' % + (deprecated_group, + deprecated_opt.name)) try: if opt_default is None: print('#%s=' % opt_name) elif opt_type == STROPT: - assert(isinstance(opt_default, basestring)) + assert(isinstance(opt_default, six.string_types)) print('#%s=%s' % (opt_name, _sanitize_default(opt_name, opt_default))) elif opt_type == BOOLOPT: diff --git a/ceilometer/openstack/common/context.py b/ceilometer/openstack/common/context.py index fe7847b20..2e46d7024 100644 --- a/ceilometer/openstack/common/context.py +++ b/ceilometer/openstack/common/context.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -23,12 +21,11 @@ context or provide additional information in their specific WSGI pipeline. """ import itertools - -from ceilometer.openstack.common import uuidutils +import uuid def generate_request_id(): - return 'req-%s' % uuidutils.generate_uuid() + return 'req-%s' % str(uuid.uuid4()) class RequestContext(object): diff --git a/ceilometer/openstack/common/db/__init__.py b/ceilometer/openstack/common/db/__init__.py index 1b9b60dec..5f5273f3e 100644 --- a/ceilometer/openstack/common/db/__init__.py +++ b/ceilometer/openstack/common/db/__init__.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Cloudscaling Group, Inc # All Rights Reserved. # diff --git a/ceilometer/openstack/common/db/api.py b/ceilometer/openstack/common/db/api.py index e39e18bd0..94bb5dea5 100644 --- a/ceilometer/openstack/common/db/api.py +++ b/ceilometer/openstack/common/db/api.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2013 Rackspace Hosting # All Rights Reserved. # diff --git a/ceilometer/openstack/common/db/exception.py b/ceilometer/openstack/common/db/exception.py index 81bab2d25..572f28902 100644 --- a/ceilometer/openstack/common/db/exception.py +++ b/ceilometer/openstack/common/db/exception.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -49,3 +47,8 @@ class DbMigrationError(DBError): """Wraps migration specific exception.""" def __init__(self, message=None): super(DbMigrationError, self).__init__(str(message)) + + +class DBConnectionError(DBError): + """Wraps connection specific exception.""" + pass diff --git a/ceilometer/openstack/common/db/sqlalchemy/__init__.py b/ceilometer/openstack/common/db/sqlalchemy/__init__.py index 1b9b60dec..5f5273f3e 100644 --- a/ceilometer/openstack/common/db/sqlalchemy/__init__.py +++ b/ceilometer/openstack/common/db/sqlalchemy/__init__.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Cloudscaling Group, Inc # All Rights Reserved. # diff --git a/ceilometer/openstack/common/db/sqlalchemy/models.py b/ceilometer/openstack/common/db/sqlalchemy/models.py index d6d90a3cb..6c4debc3c 100644 --- a/ceilometer/openstack/common/db/sqlalchemy/models.py +++ b/ceilometer/openstack/common/db/sqlalchemy/models.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2011 X.commerce, a business unit of eBay Inc. # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. diff --git a/ceilometer/openstack/common/db/sqlalchemy/provision.py b/ceilometer/openstack/common/db/sqlalchemy/provision.py new file mode 100644 index 000000000..a16ed9218 --- /dev/null +++ b/ceilometer/openstack/common/db/sqlalchemy/provision.py @@ -0,0 +1,187 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Mirantis.inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provision test environment for specific DB backends""" + +import argparse +import os +import random +import string + +import sqlalchemy + +from ceilometer.openstack.common.db import exception as exc + + +SQL_CONNECTION = os.getenv('OS_TEST_DBAPI_ADMIN_CONNECTION', 'sqlite://') + + +def _gen_credentials(*names): + """Generate credentials.""" + auth_dict = {} + for name in names: + val = ''.join(random.choice(string.lowercase) for i in xrange(10)) + auth_dict[name] = val + return auth_dict + + +def _get_engine(uri=SQL_CONNECTION): + """Engine creation + + By default the uri is SQL_CONNECTION which is admin credentials. + Call the function without arguments to get admin connection. Admin + connection required to create temporary user and database for each + particular test. Otherwise use existing connection to recreate connection + to the temporary database. + """ + return sqlalchemy.create_engine(uri, poolclass=sqlalchemy.pool.NullPool) + + +def _execute_sql(engine, sql, driver): + """Initialize connection, execute sql query and close it.""" + try: + with engine.connect() as conn: + if driver == 'postgresql': + conn.connection.set_isolation_level(0) + for s in sql: + conn.execute(s) + except sqlalchemy.exc.OperationalError: + msg = ('%s does not match database admin ' + 'credentials or database does not exist.') + raise exc.DBConnectionError(msg % SQL_CONNECTION) + + +def create_database(engine): + """Provide temporary user and database for each particular test.""" + driver = engine.name + + auth = _gen_credentials('database', 'user', 'passwd') + + sqls = { + 'mysql': [ + "drop database if exists %(database)s;", + "grant all on %(database)s.* to '%(user)s'@'localhost'" + " identified by '%(passwd)s';", + "create database %(database)s;", + ], + 'postgresql': [ + "drop database if exists %(database)s;", + "drop user if exists %(user)s;", + "create user %(user)s with password '%(passwd)s';", + "create database %(database)s owner %(user)s;", + ] + } + + if driver == 'sqlite': + return 'sqlite:////tmp/%s' % auth['database'] + + try: + sql_rows = sqls[driver] + except KeyError: + raise ValueError('Unsupported RDBMS %s' % driver) + sql_query = map(lambda x: x % auth, sql_rows) + + _execute_sql(engine, sql_query, driver) + + params = auth.copy() + params['backend'] = driver + return "%(backend)s://%(user)s:%(passwd)s@localhost/%(database)s" % params + + +def drop_database(engine, current_uri): + """Drop temporary database and user after each particular test.""" + engine = _get_engine(current_uri) + admin_engine = _get_engine() + driver = engine.name + auth = {'database': engine.url.database, 'user': engine.url.username} + + if driver == 'sqlite': + try: + os.remove(auth['database']) + except OSError: + pass + return + + sqls = { + 'mysql': [ + "drop database if exists %(database)s;", + "drop user '%(user)s'@'localhost';", + ], + 'postgresql': [ + "drop database if exists %(database)s;", + "drop user if exists %(user)s;", + ] + } + + try: + sql_rows = sqls[driver] + except KeyError: + raise ValueError('Unsupported RDBMS %s' % driver) + sql_query = map(lambda x: x % auth, sql_rows) + + _execute_sql(admin_engine, sql_query, driver) + + +def main(): + """Controller to handle commands + + ::create: Create test user and database with random names. + ::drop: Drop user and database created by previous command. + """ + parser = argparse.ArgumentParser( + description='Controller to handle database creation and dropping' + ' commands.', + epilog='Under normal circumstances is not used directly.' + ' Used in .testr.conf to automate test database creation' + ' and dropping processes.') + subparsers = parser.add_subparsers( + help='Subcommands to manipulate temporary test databases.') + + create = subparsers.add_parser( + 'create', + help='Create temporary test ' + 'databases and users.') + create.set_defaults(which='create') + create.add_argument( + 'instances_count', + type=int, + help='Number of databases to create.') + + drop = subparsers.add_parser( + 'drop', + help='Drop temporary test databases and users.') + drop.set_defaults(which='drop') + drop.add_argument( + 'instances', + nargs='+', + help='List of databases uri to be dropped.') + + args = parser.parse_args() + + engine = _get_engine() + which = args.which + + if which == "create": + for i in range(int(args.instances_count)): + print(create_database(engine)) + elif which == "drop": + for db in args.instances: + drop_database(engine, db) + + +if __name__ == "__main__": + main() diff --git a/ceilometer/openstack/common/db/sqlalchemy/session.py b/ceilometer/openstack/common/db/sqlalchemy/session.py index b6c365804..861b8e06f 100644 --- a/ceilometer/openstack/common/db/sqlalchemy/session.py +++ b/ceilometer/openstack/common/db/sqlalchemy/session.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -623,7 +621,8 @@ def _is_db_connection_error(args): """Return True if error in connecting to db.""" # NOTE(adam_g): This is currently MySQL specific and needs to be extended # to support Postgres and others. - conn_err_codes = ('2002', '2003', '2006') + # For the db2, the error code is -30081 since the db2 is still not ready + conn_err_codes = ('2002', '2003', '2006', '-30081') for err_code in conn_err_codes: if args.find(err_code) != -1: return True @@ -753,25 +752,25 @@ def _patch_mysqldb_with_stacktrace_comments(): def _do_query(self, q): stack = '' - for file, line, method, function in traceback.extract_stack(): + for filename, line, method, function in traceback.extract_stack(): # exclude various common things from trace - if file.endswith('session.py') and method == '_do_query': + if filename.endswith('session.py') and method == '_do_query': continue - if file.endswith('api.py') and method == 'wrapper': + if filename.endswith('api.py') and method == 'wrapper': continue - if file.endswith('utils.py') and method == '_inner': + if filename.endswith('utils.py') and method == '_inner': continue - if file.endswith('exception.py') and method == '_wrap': + if filename.endswith('exception.py') and method == '_wrap': continue # db/api is just a wrapper around db/sqlalchemy/api - if file.endswith('db/api.py'): + if filename.endswith('db/api.py'): continue # only trace inside ceilometer - index = file.rfind('ceilometer') + index = filename.rfind('ceilometer') if index == -1: continue stack += "File:%s:%s Method:%s() Line:%s | " \ - % (file[index:], line, method, function) + % (filename[index:], line, method, function) # strip trailing " | " from stack if stack: diff --git a/ceilometer/openstack/common/db/sqlalchemy/test_migrations.py b/ceilometer/openstack/common/db/sqlalchemy/test_migrations.py index 0bed212c8..9b9be8620 100644 --- a/ceilometer/openstack/common/db/sqlalchemy/test_migrations.py +++ b/ceilometer/openstack/common/db/sqlalchemy/test_migrations.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010-2011 OpenStack Foundation # Copyright 2012-2013 IBM Corp. # All Rights Reserved. @@ -16,17 +14,18 @@ # License for the specific language governing permissions and limitations # under the License. - -import commands import ConfigParser +import functools import os -import urlparse +import lockfile import sqlalchemy import sqlalchemy.exc -from ceilometer.openstack.common import lockutils +from ceilometer.openstack.common.gettextutils import _ from ceilometer.openstack.common import log as logging +from ceilometer.openstack.common import processutils +from ceilometer.openstack.common.py3kcompat import urlutils from ceilometer.openstack.common import test LOG = logging.getLogger(__name__) @@ -93,6 +92,22 @@ def get_db_connection_info(conn_pieces): return (user, password, database, host) +def _set_db_lock(lock_path=None, lock_prefix=None): + def decorator(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + try: + path = lock_path or os.environ.get("CEILOMETER_LOCK_PATH") + lock = lockfile.FileLock(os.path.join(path, lock_prefix)) + with lock: + LOG.debug(_('Got lock "%s"') % f.__name__) + return f(*args, **kwargs) + finally: + LOG.debug(_('Lock released "%s"') % f.__name__) + return wrapper + return decorator + + class BaseMigrationTestCase(test.BaseTestCase): """Base class fort testing of migration utils.""" @@ -143,12 +158,13 @@ class BaseMigrationTestCase(test.BaseTestCase): super(BaseMigrationTestCase, self).tearDown() def execute_cmd(self, cmd=None): - status, output = commands.getstatusoutput(cmd) + out, err = processutils.trycmd(cmd, shell=True, discard_warnings=True) + output = out or err LOG.debug(output) - self.assertEqual(0, status, + self.assertEqual('', err, "Failed to run: %s\n%s" % (cmd, output)) - @lockutils.synchronized('pgadmin', 'tests-', external=True) + @_set_db_lock('pgadmin', 'tests-') def _reset_pg(self, conn_pieces): (user, password, database, host) = get_db_connection_info(conn_pieces) os.environ['PGPASSWORD'] = password @@ -173,7 +189,7 @@ class BaseMigrationTestCase(test.BaseTestCase): def _reset_databases(self): for key, engine in self.engines.items(): conn_string = self.test_databases[key] - conn_pieces = urlparse.urlparse(conn_string) + conn_pieces = urlutils.urlparse(conn_string) engine.dispose() if conn_string.startswith('sqlite'): # We can just delete the SQLite database, which is diff --git a/ceilometer/openstack/common/db/sqlalchemy/utils.py b/ceilometer/openstack/common/db/sqlalchemy/utils.py index a9c073dfa..530e5600f 100644 --- a/ceilometer/openstack/common/db/sqlalchemy/utils.py +++ b/ceilometer/openstack/common/db/sqlalchemy/utils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2010-2011 OpenStack Foundation. diff --git a/ceilometer/openstack/common/deprecated/wsgi.py b/ceilometer/openstack/common/deprecated/wsgi.py index 718e180db..3cacad736 100644 --- a/ceilometer/openstack/common/deprecated/wsgi.py +++ b/ceilometer/openstack/common/deprecated/wsgi.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -24,6 +22,8 @@ import datetime import errno import socket import time +from xml.dom import minidom +from xml.parsers import expat import eventlet.wsgi from oslo.config import cfg @@ -32,8 +32,6 @@ import routes.middleware import six import webob.dec import webob.exc -from xml.dom import minidom -from xml.parsers import expat from ceilometer.openstack.common.gettextutils import _ # noqa from ceilometer.openstack.common import jsonutils @@ -42,6 +40,7 @@ from ceilometer.openstack.common import service from ceilometer.openstack.common import sslutils from ceilometer.openstack.common import xmlutils + socket_opts = [ cfg.IntOpt('backlog', default=4096, diff --git a/ceilometer/openstack/common/eventlet_backdoor.py b/ceilometer/openstack/common/eventlet_backdoor.py index 552c4b10c..693bf81a4 100644 --- a/ceilometer/openstack/common/eventlet_backdoor.py +++ b/ceilometer/openstack/common/eventlet_backdoor.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2012 OpenStack Foundation. # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. diff --git a/ceilometer/openstack/common/excutils.py b/ceilometer/openstack/common/excutils.py index 771275b29..c49734ea2 100644 --- a/ceilometer/openstack/common/excutils.py +++ b/ceilometer/openstack/common/excutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # Copyright 2012, Red Hat, Inc. # diff --git a/ceilometer/openstack/common/fileutils.py b/ceilometer/openstack/common/fileutils.py index 3eb5576e6..986a09ca6 100644 --- a/ceilometer/openstack/common/fileutils.py +++ b/ceilometer/openstack/common/fileutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/ceilometer/openstack/common/fixture/config.py b/ceilometer/openstack/common/fixture/config.py index 7b044ef74..0bf90ff7a 100644 --- a/ceilometer/openstack/common/fixture/config.py +++ b/ceilometer/openstack/common/fixture/config.py @@ -1,4 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 # # Copyright 2013 Mirantis, Inc. # Copyright 2013 OpenStack Foundation @@ -30,7 +29,7 @@ class Config(fixtures.Fixture): the specified configuration option group. All overrides are automatically cleared at the end of the current - test by the reset() method, which is registred by addCleanup(). + test by the reset() method, which is registered by addCleanup(). """ def __init__(self, conf=cfg.CONF): diff --git a/ceilometer/openstack/common/fixture/lockutils.py b/ceilometer/openstack/common/fixture/lockutils.py new file mode 100644 index 000000000..f9f969600 --- /dev/null +++ b/ceilometer/openstack/common/fixture/lockutils.py @@ -0,0 +1,51 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures + +from ceilometer.openstack.common.lockutils import lock + + +class LockFixture(fixtures.Fixture): + """External locking fixture. + + This fixture is basically an alternative to the synchronized decorator with + the external flag so that tearDowns and addCleanups will be included in + the lock context for locking between tests. The fixture is recommended to + be the first line in a test method, like so:: + + def test_method(self): + self.useFixture(LockFixture) + ... + + or the first line in setUp if all the test methods in the class are + required to be serialized. Something like:: + + class TestCase(testtools.testcase): + def setUp(self): + self.useFixture(LockFixture) + super(TestCase, self).setUp() + ... + + This is because addCleanups are put on a LIFO queue that gets run after the + test method exits. (either by completing or raising an exception) + """ + def __init__(self, name, lock_file_prefix=None): + self.mgr = lock(name, lock_file_prefix, True) + + def setUp(self): + super(LockFixture, self).setUp() + self.addCleanup(self.mgr.__exit__, None, None, None) + self.mgr.__enter__() diff --git a/ceilometer/openstack/common/fixture/mockpatch.py b/ceilometer/openstack/common/fixture/mockpatch.py index cd0d6ca6b..858e77cd0 100644 --- a/ceilometer/openstack/common/fixture/mockpatch.py +++ b/ceilometer/openstack/common/fixture/mockpatch.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2013 Hewlett-Packard Development Company, L.P. diff --git a/ceilometer/openstack/common/fixture/moxstubout.py b/ceilometer/openstack/common/fixture/moxstubout.py index f277fdd73..e8c031f08 100644 --- a/ceilometer/openstack/common/fixture/moxstubout.py +++ b/ceilometer/openstack/common/fixture/moxstubout.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2013 Hewlett-Packard Development Company, L.P. @@ -19,7 +17,6 @@ import fixtures import mox -import stubout class MoxStubout(fixtures.Fixture): @@ -30,8 +27,6 @@ class MoxStubout(fixtures.Fixture): # emulate some of the mox stuff, we can't use the metaclass # because it screws with our generators self.mox = mox.Mox() - self.stubs = stubout.StubOutForTesting() + self.stubs = self.mox.stubs self.addCleanup(self.mox.UnsetStubs) - self.addCleanup(self.stubs.UnsetAll) - self.addCleanup(self.stubs.SmartUnsetAll) self.addCleanup(self.mox.VerifyAll) diff --git a/ceilometer/openstack/common/gettextutils.py b/ceilometer/openstack/common/gettextutils.py index 12c8fa8b8..0cd85124d 100644 --- a/ceilometer/openstack/common/gettextutils.py +++ b/ceilometer/openstack/common/gettextutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Red Hat, Inc. # Copyright 2013 IBM Corp. # All Rights Reserved. @@ -317,7 +315,7 @@ def get_available_languages(domain): # NOTE(luisg): Babel <1.0 used a function called list(), which was # renamed to locale_identifiers() in >=1.0, the requirements master list # requires >=0.9.6, uncapped, so defensively work with both. We can remove - # this check when the master list updates to >=1.0, and all projects udpate + # this check when the master list updates to >=1.0, and update all projects list_identifiers = (getattr(localedata, 'list', None) or getattr(localedata, 'locale_identifiers')) locale_identifiers = list_identifiers() @@ -329,13 +327,21 @@ def get_available_languages(domain): def get_localized_message(message, user_locale): - """Gets a localized version of the given message in the given locale.""" + """Gets a localized version of the given message in the given locale. + + If the message is not a Message object the message is returned as-is. + If the locale is None the message is translated to the default locale. + + :returns: the translated message in unicode, or the original message if + it could not be translated + """ + translated = message if isinstance(message, Message): - if user_locale: - message.locale = user_locale - return six.text_type(message) - else: - return message + original_locale = message.locale + message.locale = user_locale + translated = six.text_type(message) + message.locale = original_locale + return translated class LocaleHandler(logging.Handler): diff --git a/ceilometer/openstack/common/importutils.py b/ceilometer/openstack/common/importutils.py index 7a303f93f..4fd9ae2bc 100644 --- a/ceilometer/openstack/common/importutils.py +++ b/ceilometer/openstack/common/importutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/ceilometer/openstack/common/jsonutils.py b/ceilometer/openstack/common/jsonutils.py index 88b01c0e7..85ca4d2cb 100644 --- a/ceilometer/openstack/common/jsonutils.py +++ b/ceilometer/openstack/common/jsonutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2011 Justin Santa Barbara diff --git a/ceilometer/openstack/common/local.py b/ceilometer/openstack/common/local.py index e82f17d0f..0819d5b97 100644 --- a/ceilometer/openstack/common/local.py +++ b/ceilometer/openstack/common/local.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/ceilometer/openstack/common/lockutils.py b/ceilometer/openstack/common/lockutils.py index 937027911..f88e5bbbf 100644 --- a/ceilometer/openstack/common/lockutils.py +++ b/ceilometer/openstack/common/lockutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -20,11 +18,14 @@ import contextlib import errno import functools import os +import shutil +import subprocess +import sys +import tempfile import threading import time import weakref -import fixtures from oslo.config import cfg from ceilometer.openstack.common import fileutils @@ -40,6 +41,7 @@ util_opts = [ cfg.BoolOpt('disable_process_locking', default=False, help='Whether to disable inter-process locks'), cfg.StrOpt('lock_path', + default=os.environ.get("CEILOMETER_LOCK_PATH"), help=('Directory to use for lock files.')) ] @@ -132,6 +134,7 @@ else: InterProcessLock = _PosixLock _semaphores = weakref.WeakValueDictionary() +_semaphores_lock = threading.Lock() @contextlib.contextmanager @@ -154,15 +157,12 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None): special location for external lock files to live. If nothing is set, then CONF.lock_path is used as a default. """ - # NOTE(soren): If we ever go natively threaded, this will be racy. - # See http://stackoverflow.com/questions/5390569/dyn - # amically-allocating-and-destroying-mutexes - sem = _semaphores.get(name, threading.Semaphore()) - if name not in _semaphores: - # this check is not racy - we're already holding ref locally - # so GC won't remove the item and there was no IO switch - # (only valid in greenthreads) - _semaphores[name] = sem + with _semaphores_lock: + try: + sem = _semaphores[name] + except KeyError: + sem = threading.Semaphore() + _semaphores[name] = sem with sem: LOG.debug(_('Got semaphore "%(lock)s"'), {'lock': name}) @@ -242,13 +242,14 @@ def synchronized(name, lock_file_prefix=None, external=False, lock_path=None): def wrap(f): @functools.wraps(f) def inner(*args, **kwargs): - with lock(name, lock_file_prefix, external, lock_path): - LOG.debug(_('Got semaphore / lock "%(function)s"'), + try: + with lock(name, lock_file_prefix, external, lock_path): + LOG.debug(_('Got semaphore / lock "%(function)s"'), + {'function': f.__name__}) + return f(*args, **kwargs) + finally: + LOG.debug(_('Semaphore / lock released "%(function)s"'), {'function': f.__name__}) - return f(*args, **kwargs) - - LOG.debug(_('Semaphore / lock released "%(function)s"'), - {'function': f.__name__}) return inner return wrap @@ -278,34 +279,25 @@ def synchronized_with_prefix(lock_file_prefix): return functools.partial(synchronized, lock_file_prefix=lock_file_prefix) -class LockFixture(fixtures.Fixture): - """External locking fixture. +def main(argv): + """Create a dir for locks and pass it to command from arguments - This fixture is basically an alternative to the synchronized decorator with - the external flag so that tearDowns and addCleanups will be included in - the lock context for locking between tests. The fixture is recommended to - be the first line in a test method, like so:: + If you run this: + python -m openstack.common.lockutils python setup.py testr - def test_method(self): - self.useFixture(LockFixture) - ... - - or the first line in setUp if all the test methods in the class are - required to be serialized. Something like:: - - class TestCase(testtools.testcase): - def setUp(self): - self.useFixture(LockFixture) - super(TestCase, self).setUp() - ... - - This is because addCleanups are put on a LIFO queue that gets run after the - test method exits. (either by completing or raising an exception) + a temporary directory will be created for all your locks and passed to all + your tests in an environment variable. The temporary dir will be deleted + afterwards and the return value will be preserved. """ - def __init__(self, name, lock_file_prefix=None): - self.mgr = lock(name, lock_file_prefix, True) - def setUp(self): - super(LockFixture, self).setUp() - self.addCleanup(self.mgr.__exit__, None, None, None) - self.mgr.__enter__() + lock_dir = tempfile.mkdtemp() + os.environ["CEILOMETER_LOCK_PATH"] = lock_dir + try: + ret_val = subprocess.call(argv[1:]) + finally: + shutil.rmtree(lock_dir, ignore_errors=True) + return ret_val + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/ceilometer/openstack/common/log.py b/ceilometer/openstack/common/log.py index a29746cec..def624a4a 100644 --- a/ceilometer/openstack/common/log.py +++ b/ceilometer/openstack/common/log.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. @@ -35,6 +33,7 @@ import logging import logging.config import logging.handlers import os +import re import sys import traceback @@ -50,6 +49,24 @@ from ceilometer.openstack.common import local _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" +_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] + +# NOTE(ldbragst): Let's build a list of regex objects using the list of +# _SANITIZE_KEYS we already have. This way, we only have to add the new key +# to the list of _SANITIZE_KEYS and we can generate regular expressions +# for XML and JSON automatically. +_SANITIZE_PATTERNS = [] +_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', + r'(<%(key)s>).*?()', + r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', + r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])'] + +for key in _SANITIZE_KEYS: + for pattern in _FORMAT_PATTERNS: + reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) + _SANITIZE_PATTERNS.append(reg_ex) + + common_cli_opts = [ cfg.BoolOpt('debug', short='d', @@ -136,6 +153,7 @@ log_opts = [ 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', + 'iso8601=WARN', ], help='list of logger=LEVEL pairs'), cfg.BoolOpt('publish_errors', @@ -214,6 +232,39 @@ def _get_log_file_path(binary=None): return None +def mask_password(message, secret="***"): + """Replace password with 'secret' in message. + + :param message: The string which includes security information. + :param secret: value with which to replace passwords, defaults to "***". + :returns: The unicode value of message with the password fields masked. + + For example: + >>> mask_password("'adminPass' : 'aaaaa'") + "'adminPass' : '***'" + >>> mask_password("'admin_pass' : 'aaaaa'") + "'admin_pass' : '***'" + >>> mask_password('"password" : "aaaaa"') + '"password" : "***"' + >>> mask_password("'original_password' : 'aaaaa'") + "'original_password' : '***'" + >>> mask_password("u'original_password' : u'aaaaa'") + "u'original_password' : u'***'" + """ + message = six.text_type(message) + + # NOTE(ldbragst): Check to see if anything in message contains any key + # specified in _SANITIZE_KEYS, if not then just return the message since + # we don't have to mask any passwords. + if not any(key in message for key in _SANITIZE_KEYS): + return message + + secret = r'\g<1>' + secret + r'\g<2>' + for pattern in _SANITIZE_PATTERNS: + message = re.sub(pattern, secret, message) + return message + + class BaseLoggerAdapter(logging.LoggerAdapter): def audit(self, msg, *args, **kwargs): @@ -298,7 +349,7 @@ class JSONFormatter(logging.Formatter): def formatException(self, ei, strip_newlines=True): lines = traceback.format_exception(*ei) if strip_newlines: - lines = [itertools.ifilter( + lines = [moves.filter( lambda x: x, line.rstrip().splitlines()) for line in lines] lines = list(itertools.chain(*lines)) @@ -336,10 +387,10 @@ class JSONFormatter(logging.Formatter): def _create_logging_excepthook(product_name): - def logging_excepthook(type, value, tb): + def logging_excepthook(exc_type, value, tb): extra = {} if CONF.verbose: - extra['exc_info'] = (type, value, tb) + extra['exc_info'] = (exc_type, value, tb) getLogger(product_name).critical(str(value), **extra) return logging_excepthook @@ -424,7 +475,7 @@ def _setup_logging_from_conf(): streamlog = ColorHandler() log_root.addHandler(streamlog) - elif not CONF.log_file: + elif not logpath: # pass sys.stdout as a positional argument # python2.6 calls the argument strm, in 2.7 it's stream streamlog = logging.StreamHandler(sys.stdout) diff --git a/ceilometer/openstack/common/log_handler.py b/ceilometer/openstack/common/log_handler.py index 09fb4fb85..6ba52fd23 100644 --- a/ceilometer/openstack/common/log_handler.py +++ b/ceilometer/openstack/common/log_handler.py @@ -1,6 +1,4 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2013 SoftLayer Technologies, an IBM Company +# Copyright 2013 IBM Corp. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -15,7 +13,7 @@ # under the License. import logging -from nova.openstack.common import notifier +from ceilometer.openstack.common import notifier from oslo.config import cfg diff --git a/ceilometer/openstack/common/loopingcall.py b/ceilometer/openstack/common/loopingcall.py index fdd606d2b..877296e36 100644 --- a/ceilometer/openstack/common/loopingcall.py +++ b/ceilometer/openstack/common/loopingcall.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2011 Justin Santa Barbara diff --git a/ceilometer/openstack/common/middleware/audit.py b/ceilometer/openstack/common/middleware/audit.py index bb69e313a..863ddb2d2 100644 --- a/ceilometer/openstack/common/middleware/audit.py +++ b/ceilometer/openstack/common/middleware/audit.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2013 OpenStack Foundation # All Rights Reserved. # diff --git a/ceilometer/openstack/common/middleware/context.py b/ceilometer/openstack/common/middleware/context.py index da664942f..78dc144af 100644 --- a/ceilometer/openstack/common/middleware/context.py +++ b/ceilometer/openstack/common/middleware/context.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -55,7 +53,7 @@ def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) - def filter(app): + def f(app): return ContextMiddleware(app, conf) - return filter + return f diff --git a/ceilometer/openstack/common/middleware/correlation_id.py b/ceilometer/openstack/common/middleware/correlation_id.py index af35f8d7a..f0560a5c2 100644 --- a/ceilometer/openstack/common/middleware/correlation_id.py +++ b/ceilometer/openstack/common/middleware/correlation_id.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2013 Rackspace Hosting # All Rights Reserved. # @@ -17,13 +15,14 @@ """Middleware that attaches a correlation id to WSGI request""" +import uuid + from ceilometer.openstack.common.middleware import base -from ceilometer.openstack.common import uuidutils class CorrelationIdMiddleware(base.Middleware): def process_request(self, req): correlation_id = (req.headers.get("X_CORRELATION_ID") or - uuidutils.generate_uuid()) + str(uuid.uuid4())) req.headers['X_CORRELATION_ID'] = correlation_id diff --git a/ceilometer/openstack/common/middleware/notifier.py b/ceilometer/openstack/common/middleware/notifier.py index 183016da8..d618fc9b8 100644 --- a/ceilometer/openstack/common/middleware/notifier.py +++ b/ceilometer/openstack/common/middleware/notifier.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2013 eNovance # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -118,7 +116,7 @@ class RequestNotifier(base.Middleware): try: response = req.get_response(self.application) except Exception: - type, value, traceback = sys.exc_info() + exc_type, value, traceback = sys.exc_info() self.process_response(req, None, value, traceback) raise else: diff --git a/ceilometer/openstack/common/middleware/sizelimit.py b/ceilometer/openstack/common/middleware/sizelimit.py index 3fba43842..c959d7483 100644 --- a/ceilometer/openstack/common/middleware/sizelimit.py +++ b/ceilometer/openstack/common/middleware/sizelimit.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/ceilometer/openstack/common/network_utils.py b/ceilometer/openstack/common/network_utils.py index dbed1ceb4..85b061b48 100644 --- a/ceilometer/openstack/common/network_utils.py +++ b/ceilometer/openstack/common/network_utils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 OpenStack Foundation. # All Rights Reserved. # @@ -19,7 +17,7 @@ Network-related utilities and helper functions. """ -import urlparse +from ceilometer.openstack.common.py3kcompat import urlutils def parse_host_port(address, default_port=None): @@ -72,10 +70,10 @@ def urlsplit(url, scheme='', allow_fragments=True): The parameters are the same as urlparse.urlsplit. """ - scheme, netloc, path, query, fragment = urlparse.urlsplit( + scheme, netloc, path, query, fragment = urlutils.urlsplit( url, scheme, allow_fragments) if allow_fragments and '#' in path: path, fragment = path.split('#', 1) if '?' in path: path, query = path.split('?', 1) - return urlparse.SplitResult(scheme, netloc, path, query, fragment) + return urlutils.SplitResult(scheme, netloc, path, query, fragment) diff --git a/ceilometer/openstack/common/policy.py b/ceilometer/openstack/common/policy.py index c0168d8e2..f48e46b09 100644 --- a/ceilometer/openstack/common/policy.py +++ b/ceilometer/openstack/common/policy.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2012 OpenStack Foundation. # All Rights Reserved. # @@ -505,7 +503,7 @@ def _parse_list_rule(rule): continue # Handle bare strings - if isinstance(inner_rule, basestring): + if isinstance(inner_rule, six.string_types): inner_rule = [inner_rule] # Parse the inner rules into Check objects @@ -763,7 +761,7 @@ def parse_rule(rule): """Parses a policy rule into a tree of Check objects.""" # If the rule is a string, it's in the policy language - if isinstance(rule, basestring): + if isinstance(rule, six.string_types): return _parse_text_rule(rule) return _parse_list_rule(rule) diff --git a/ceilometer/openstack/common/processutils.py b/ceilometer/openstack/common/processutils.py new file mode 100644 index 000000000..c03131ebc --- /dev/null +++ b/ceilometer/openstack/common/processutils.py @@ -0,0 +1,248 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +System-level utilities and helper functions. +""" + +import logging as stdlib_logging +import os +import random +import shlex +import signal + +from eventlet.green import subprocess +from eventlet import greenthread + +from ceilometer.openstack.common.gettextutils import _ # noqa +from ceilometer.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + + +class InvalidArgumentError(Exception): + def __init__(self, message=None): + super(InvalidArgumentError, self).__init__(message) + + +class UnknownArgumentError(Exception): + def __init__(self, message=None): + super(UnknownArgumentError, self).__init__(message) + + +class ProcessExecutionError(Exception): + def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + self.exit_code = exit_code + self.stderr = stderr + self.stdout = stdout + self.cmd = cmd + self.description = description + + if description is None: + description = "Unexpected error while running command." + if exit_code is None: + exit_code = '-' + message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" + % (description, cmd, exit_code, stdout, stderr)) + super(ProcessExecutionError, self).__init__(message) + + +class NoRootWrapSpecified(Exception): + def __init__(self, message=None): + super(NoRootWrapSpecified, self).__init__(message) + + +def _subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + +def execute(*cmd, **kwargs): + """Helper method to shell out and execute a command through subprocess. + + Allows optional retry. + + :param cmd: Passed to subprocess.Popen. + :type cmd: string + :param process_input: Send to opened process. + :type proces_input: string + :param check_exit_code: Single bool, int, or list of allowed exit + codes. Defaults to [0]. Raise + :class:`ProcessExecutionError` unless + program exits with one of these code. + :type check_exit_code: boolean, int, or [int] + :param delay_on_retry: True | False. Defaults to True. If set to True, + wait a short amount of time before retrying. + :type delay_on_retry: boolean + :param attempts: How many times to retry cmd. + :type attempts: int + :param run_as_root: True | False. Defaults to False. If set to True, + the command is prefixed by the command specified + in the root_helper kwarg. + :type run_as_root: boolean + :param root_helper: command to prefix to commands called with + run_as_root=True + :type root_helper: string + :param shell: whether or not there should be a shell used to + execute this command. Defaults to false. + :type shell: boolean + :param loglevel: log level for execute commands. + :type loglevel: int. (Should be stdlib_logging.DEBUG or + stdlib_logging.INFO) + :returns: (stdout, stderr) from process execution + :raises: :class:`UnknownArgumentError` on + receiving unknown arguments + :raises: :class:`ProcessExecutionError` + """ + + process_input = kwargs.pop('process_input', None) + check_exit_code = kwargs.pop('check_exit_code', [0]) + ignore_exit_code = False + delay_on_retry = kwargs.pop('delay_on_retry', True) + attempts = kwargs.pop('attempts', 1) + run_as_root = kwargs.pop('run_as_root', False) + root_helper = kwargs.pop('root_helper', '') + shell = kwargs.pop('shell', False) + loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG) + + if isinstance(check_exit_code, bool): + ignore_exit_code = not check_exit_code + check_exit_code = [0] + elif isinstance(check_exit_code, int): + check_exit_code = [check_exit_code] + + if kwargs: + raise UnknownArgumentError(_('Got unknown keyword args ' + 'to utils.execute: %r') % kwargs) + + if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0: + if not root_helper: + raise NoRootWrapSpecified( + message=('Command requested root, but did not specify a root ' + 'helper.')) + cmd = shlex.split(root_helper) + list(cmd) + + cmd = map(str, cmd) + + while attempts > 0: + attempts -= 1 + try: + LOG.log(loglevel, _('Running cmd (subprocess): %s'), ' '.join(cmd)) + _PIPE = subprocess.PIPE # pylint: disable=E1101 + + if os.name == 'nt': + preexec_fn = None + close_fds = False + else: + preexec_fn = _subprocess_setup + close_fds = True + + obj = subprocess.Popen(cmd, + stdin=_PIPE, + stdout=_PIPE, + stderr=_PIPE, + close_fds=close_fds, + preexec_fn=preexec_fn, + shell=shell) + result = None + if process_input is not None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + obj.stdin.close() # pylint: disable=E1101 + _returncode = obj.returncode # pylint: disable=E1101 + LOG.log(loglevel, _('Result was %s') % _returncode) + if not ignore_exit_code and _returncode not in check_exit_code: + (stdout, stderr) = result + raise ProcessExecutionError(exit_code=_returncode, + stdout=stdout, + stderr=stderr, + cmd=' '.join(cmd)) + return result + except ProcessExecutionError: + if not attempts: + raise + else: + LOG.log(loglevel, _('%r failed. Retrying.'), cmd) + if delay_on_retry: + greenthread.sleep(random.randint(20, 200) / 100.0) + finally: + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one + greenthread.sleep(0) + + +def trycmd(*args, **kwargs): + """A wrapper around execute() to more easily handle warnings and errors. + + Returns an (out, err) tuple of strings containing the output of + the command's stdout and stderr. If 'err' is not empty then the + command can be considered to have failed. + + :discard_warnings True | False. Defaults to False. If set to True, + then for succeeding commands, stderr is cleared + + """ + discard_warnings = kwargs.pop('discard_warnings', False) + + try: + out, err = execute(*args, **kwargs) + failed = False + except ProcessExecutionError as exn: + out, err = '', str(exn) + failed = True + + if not failed and discard_warnings and err: + # Handle commands that output to stderr but otherwise succeed + err = '' + + return out, err + + +def ssh_execute(ssh, cmd, process_input=None, + addl_env=None, check_exit_code=True): + LOG.debug(_('Running cmd (SSH): %s'), cmd) + if addl_env: + raise InvalidArgumentError(_('Environment not supported over SSH')) + + if process_input: + # This is (probably) fixable if we need it... + raise InvalidArgumentError(_('process_input not supported over SSH')) + + stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd) + channel = stdout_stream.channel + + # NOTE(justinsb): This seems suspicious... + # ...other SSH clients have buffering issues with this approach + stdout = stdout_stream.read() + stderr = stderr_stream.read() + stdin_stream.close() + + exit_status = channel.recv_exit_status() + + # exit_status == -1 if no exit code was returned + if exit_status != -1: + LOG.debug(_('Result was %s') % exit_status) + if check_exit_code and exit_status != 0: + raise ProcessExecutionError(exit_code=exit_status, + stdout=stdout, + stderr=stderr, + cmd=cmd) + + return (stdout, stderr) diff --git a/ceilometer/openstack/common/uuidutils.py b/ceilometer/openstack/common/py3kcompat/__init__.py similarity index 53% rename from ceilometer/openstack/common/uuidutils.py rename to ceilometer/openstack/common/py3kcompat/__init__.py index 7608acb94..97ae4e34a 100644 --- a/ceilometer/openstack/common/uuidutils.py +++ b/ceilometer/openstack/common/py3kcompat/__init__.py @@ -1,6 +1,5 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2012 Intel Corporation. +# +# Copyright 2013 Canonical Ltd. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -14,26 +13,4 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -""" -UUID related utilities and helper functions. -""" - -import uuid - - -def generate_uuid(): - return str(uuid.uuid4()) - - -def is_uuid_like(val): - """Returns validation of a value as a UUID. - - For our purposes, a UUID is a canonical form string: - aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa - - """ - try: - return str(uuid.UUID(val)) == val - except (TypeError, ValueError, AttributeError): - return False +# diff --git a/ceilometer/openstack/common/py3kcompat/urlutils.py b/ceilometer/openstack/common/py3kcompat/urlutils.py new file mode 100644 index 000000000..51e18111a --- /dev/null +++ b/ceilometer/openstack/common/py3kcompat/urlutils.py @@ -0,0 +1,63 @@ +# +# Copyright 2013 Canonical Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +""" +Python2/Python3 compatibility layer for OpenStack +""" + +import six + +if six.PY3: + # python3 + import urllib.error + import urllib.parse + import urllib.request + + urlencode = urllib.parse.urlencode + urljoin = urllib.parse.urljoin + quote = urllib.parse.quote + parse_qsl = urllib.parse.parse_qsl + unquote = urllib.parse.unquote + urlparse = urllib.parse.urlparse + urlsplit = urllib.parse.urlsplit + urlunsplit = urllib.parse.urlunsplit + SplitResult = urllib.parse.SplitResult + + urlopen = urllib.request.urlopen + URLError = urllib.error.URLError + pathname2url = urllib.request.pathname2url +else: + # python2 + import urllib + import urllib2 + import urlparse + + urlencode = urllib.urlencode + quote = urllib.quote + unquote = urllib.unquote + + parse = urlparse + parse_qsl = parse.parse_qsl + urljoin = parse.urljoin + urlparse = parse.urlparse + urlsplit = parse.urlsplit + urlunsplit = parse.urlunsplit + SplitResult = parse.SplitResult + + urlopen = urllib2.urlopen + URLError = urllib2.URLError + pathname2url = urllib.pathname2url diff --git a/ceilometer/openstack/common/rpc/__init__.py b/ceilometer/openstack/common/rpc/__init__.py index e9dee1287..735b19df4 100644 --- a/ceilometer/openstack/common/rpc/__init__.py +++ b/ceilometer/openstack/common/rpc/__init__.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -61,7 +59,7 @@ rpc_opts = [ 'exceptions', ], help='Modules of exceptions that are permitted to be recreated' - 'upon receiving exception data from an rpc call.'), + ' upon receiving exception data from an rpc call.'), cfg.BoolOpt('fake_rabbit', default=False, help='If passed, use a fake RabbitMQ provider'), @@ -227,7 +225,7 @@ def notify(context, topic, msg, envelope=False): def cleanup(): - """Clean up resoruces in use by implementation. + """Clean up resources in use by implementation. Clean up any resources that have been allocated by the RPC implementation. This is typically open connections to a messaging service. This function diff --git a/ceilometer/openstack/common/rpc/amqp.py b/ceilometer/openstack/common/rpc/amqp.py index 8ed1be0bc..7f326e8a7 100644 --- a/ceilometer/openstack/common/rpc/amqp.py +++ b/ceilometer/openstack/common/rpc/amqp.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -20,9 +18,9 @@ """ Shared code between AMQP based openstack.common.rpc implementations. -The code in this module is shared between the rpc implemenations based on AMQP. -Specifically, this includes impl_kombu and impl_qpid. impl_carrot also uses -AMQP, but is deprecated and predates this code. +The code in this module is shared between the rpc implementations based on +AMQP. Specifically, this includes impl_kombu and impl_qpid. impl_carrot also +uses AMQP, but is deprecated and predates this code. """ import collections @@ -189,7 +187,7 @@ class ReplyProxy(ConnectionContext): def __init__(self, conf, connection_pool): self._call_waiters = {} self._num_call_waiters = 0 - self._num_call_waiters_wrn_threshhold = 10 + self._num_call_waiters_wrn_threshold = 10 self._reply_q = 'reply_' + uuid.uuid4().hex super(ReplyProxy, self).__init__(conf, connection_pool, pooled=False) self.declare_direct_consumer(self._reply_q, self._process_data) @@ -208,11 +206,11 @@ class ReplyProxy(ConnectionContext): def add_call_waiter(self, waiter, msg_id): self._num_call_waiters += 1 - if self._num_call_waiters > self._num_call_waiters_wrn_threshhold: + if self._num_call_waiters > self._num_call_waiters_wrn_threshold: LOG.warn(_('Number of call waiters is greater than warning ' - 'threshhold: %d. There could be a MulticallProxyWaiter ' - 'leak.') % self._num_call_waiters_wrn_threshhold) - self._num_call_waiters_wrn_threshhold *= 2 + 'threshold: %d. There could be a MulticallProxyWaiter ' + 'leak.') % self._num_call_waiters_wrn_threshold) + self._num_call_waiters_wrn_threshold *= 2 self._call_waiters[msg_id] = waiter def del_call_waiter(self, msg_id): @@ -241,7 +239,7 @@ def msg_reply(conf, msg_id, reply_q, connection_pool, reply=None, _add_unique_id(msg) # If a reply_q exists, add the msg_id to the reply and pass the # reply_q to direct_send() to use it as the response queue. - # Otherwise use the msg_id for backward compatibilty. + # Otherwise use the msg_id for backward compatibility. if reply_q: msg['_msg_id'] = msg_id conn.direct_send(reply_q, rpc_common.serialize_msg(msg)) diff --git a/ceilometer/openstack/common/rpc/common.py b/ceilometer/openstack/common/rpc/common.py index 8e3c7b3c5..6d0343240 100644 --- a/ceilometer/openstack/common/rpc/common.py +++ b/ceilometer/openstack/common/rpc/common.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -266,7 +264,7 @@ def _safe_log(log_func, msg, msg_data): def _fix_passwords(d): """Sanitizes the password fields in the dictionary.""" - for k in d.iterkeys(): + for k in six.iterkeys(d): if k.lower().find('password') != -1: d[k] = '' elif k.lower() in SANITIZE: diff --git a/ceilometer/openstack/common/rpc/dispatcher.py b/ceilometer/openstack/common/rpc/dispatcher.py index ff41de5b2..bce940e70 100644 --- a/ceilometer/openstack/common/rpc/dispatcher.py +++ b/ceilometer/openstack/common/rpc/dispatcher.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/ceilometer/openstack/common/rpc/impl_fake.py b/ceilometer/openstack/common/rpc/impl_fake.py index 16caadc45..4902f93f0 100644 --- a/ceilometer/openstack/common/rpc/impl_fake.py +++ b/ceilometer/openstack/common/rpc/impl_fake.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/ceilometer/openstack/common/rpc/impl_kombu.py b/ceilometer/openstack/common/rpc/impl_kombu.py index cf55a9f00..6e099e3d8 100644 --- a/ceilometer/openstack/common/rpc/impl_kombu.py +++ b/ceilometer/openstack/common/rpc/impl_kombu.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -28,6 +26,7 @@ import kombu.connection import kombu.entity import kombu.messaging from oslo.config import cfg +import six from ceilometer.openstack.common import excutils from ceilometer.openstack.common.gettextutils import _ # noqa @@ -625,7 +624,7 @@ class Connection(object): def _declare_consumer(): consumer = consumer_cls(self.conf, self.channel, topic, callback, - self.consumer_num.next()) + six.next(self.consumer_num)) self.consumers.append(consumer) return consumer @@ -732,7 +731,7 @@ class Connection(object): it = self.iterconsume(limit=limit) while True: try: - it.next() + six.next(it) except StopIteration: return diff --git a/ceilometer/openstack/common/rpc/impl_qpid.py b/ceilometer/openstack/common/rpc/impl_qpid.py index 5ffee500d..9b82bf1ad 100644 --- a/ceilometer/openstack/common/rpc/impl_qpid.py +++ b/ceilometer/openstack/common/rpc/impl_qpid.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation # Copyright 2011 - 2012, Red Hat, Inc. # @@ -23,6 +21,7 @@ import uuid import eventlet import greenlet from oslo.config import cfg +import six from ceilometer.openstack.common import excutils from ceilometer.openstack.common.gettextutils import _ # noqa @@ -152,7 +151,7 @@ class ConsumerBase(object): self.connect(session) def connect(self, session): - """Declare the reciever on connect.""" + """Declare the receiver on connect.""" self._declare_receiver(session) def reconnect(self, session): @@ -395,7 +394,7 @@ class DirectPublisher(Publisher): class TopicPublisher(Publisher): """Publisher class for 'topic'.""" def __init__(self, conf, session, topic): - """init a 'topic' publisher. + """Init a 'topic' publisher. """ exchange_name = rpc_amqp.get_control_exchange(conf) @@ -412,7 +411,7 @@ class TopicPublisher(Publisher): class FanoutPublisher(Publisher): """Publisher class for 'fanout'.""" def __init__(self, conf, session, topic): - """init a 'fanout' publisher. + """Init a 'fanout' publisher. """ if conf.qpid_topology_version == 1: @@ -431,7 +430,7 @@ class FanoutPublisher(Publisher): class NotifyPublisher(Publisher): """Publisher class for notifications.""" def __init__(self, conf, session, topic): - """init a 'topic' publisher. + """Init a 'topic' publisher. """ exchange_name = rpc_amqp.get_control_exchange(conf) node_opts = {"durable": True} @@ -539,7 +538,7 @@ class Connection(object): consumers = self.consumers self.consumers = {} - for consumer in consumers.itervalues(): + for consumer in six.itervalues(consumers): consumer.reconnect(self.session) self._register_consumer(consumer) @@ -697,7 +696,7 @@ class Connection(object): it = self.iterconsume(limit=limit) while True: try: - it.next() + six.next(it) except StopIteration: return diff --git a/ceilometer/openstack/common/rpc/impl_zmq.py b/ceilometer/openstack/common/rpc/impl_zmq.py index d260d115d..862c259a7 100644 --- a/ceilometer/openstack/common/rpc/impl_zmq.py +++ b/ceilometer/openstack/common/rpc/impl_zmq.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 Cloudscaling Group, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -25,6 +23,8 @@ import uuid import eventlet import greenlet from oslo.config import cfg +import six +from six import moves from ceilometer.openstack.common import excutils from ceilometer.openstack.common.gettextutils import _ # noqa @@ -192,7 +192,7 @@ class ZmqSocket(object): # it would be much worse if some of the code calling this # were to fail. For now, lets log, and later evaluate # if we can safely raise here. - LOG.error("ZeroMQ socket could not be closed.") + LOG.error(_("ZeroMQ socket could not be closed.")) self.sock = None def recv(self, **kwargs): @@ -221,7 +221,7 @@ class ZmqClient(object): return rpc_envelope = rpc_common.serialize_msg(data[1], envelope) - zmq_msg = reduce(lambda x, y: x + y, rpc_envelope.items()) + zmq_msg = moves.reduce(lambda x, y: x + y, rpc_envelope.items()) self.outq.send(map(bytes, (msg_id, topic, 'impl_zmq_v2', data[0]) + zmq_msg)) @@ -523,8 +523,8 @@ def unflatten_envelope(packenv): h = {} try: while True: - k = i.next() - h[k] = i.next() + k = six.next(i) + h[k] = six.next(i) except StopIteration: return h diff --git a/ceilometer/openstack/common/rpc/matchmaker.py b/ceilometer/openstack/common/rpc/matchmaker.py index 504b0eb2a..d76ac8db5 100644 --- a/ceilometer/openstack/common/rpc/matchmaker.py +++ b/ceilometer/openstack/common/rpc/matchmaker.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 Cloudscaling Group, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/ceilometer/openstack/common/rpc/matchmaker_redis.py b/ceilometer/openstack/common/rpc/matchmaker_redis.py index 68e2ccc10..a190a14ec 100644 --- a/ceilometer/openstack/common/rpc/matchmaker_redis.py +++ b/ceilometer/openstack/common/rpc/matchmaker_redis.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2013 Cloudscaling Group, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/ceilometer/openstack/common/rpc/matchmaker_ring.py b/ceilometer/openstack/common/rpc/matchmaker_ring.py index 2b4ef3a8c..e74c6d4c6 100644 --- a/ceilometer/openstack/common/rpc/matchmaker_ring.py +++ b/ceilometer/openstack/common/rpc/matchmaker_ring.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011-2013 Cloudscaling Group, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/ceilometer/openstack/common/rpc/proxy.py b/ceilometer/openstack/common/rpc/proxy.py index de3d317bb..6eb8cb247 100644 --- a/ceilometer/openstack/common/rpc/proxy.py +++ b/ceilometer/openstack/common/rpc/proxy.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012-2013 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -21,7 +19,6 @@ For more information about rpc API version numbers, see: rpc/dispatcher.py """ - from ceilometer.openstack.common import rpc from ceilometer.openstack.common.rpc import common as rpc_common from ceilometer.openstack.common.rpc import serializer as rpc_serializer @@ -36,7 +33,7 @@ class RpcProxy(object): rpc API. """ - # The default namespace, which can be overriden in a subclass. + # The default namespace, which can be overridden in a subclass. RPC_API_NAMESPACE = None def __init__(self, topic, default_version, version_cap=None, diff --git a/ceilometer/openstack/common/rpc/securemessage.py b/ceilometer/openstack/common/rpc/securemessage.py deleted file mode 100644 index 9f0869bbe..000000000 --- a/ceilometer/openstack/common/rpc/securemessage.py +++ /dev/null @@ -1,521 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import base64 -import collections -import os -import struct -import time - -import requests - -from oslo.config import cfg - -from ceilometer.openstack.common.crypto import utils as cryptoutils -from ceilometer.openstack.common import jsonutils -from ceilometer.openstack.common import log as logging - -secure_message_opts = [ - cfg.BoolOpt('enabled', default=True, - help='Whether Secure Messaging (Signing) is enabled,' - ' defaults to enabled'), - cfg.BoolOpt('enforced', default=False, - help='Whether Secure Messaging (Signing) is enforced,' - ' defaults to not enforced'), - cfg.BoolOpt('encrypt', default=False, - help='Whether Secure Messaging (Encryption) is enabled,' - ' defaults to not enabled'), - cfg.StrOpt('secret_keys_file', - help='Path to the file containing the keys, takes precedence' - ' over secret_key'), - cfg.MultiStrOpt('secret_key', - help='A list of keys: (ex: name:),' - ' ignored if secret_keys_file is set'), - cfg.StrOpt('kds_endpoint', - help='KDS endpoint (ex: http://kds.example.com:35357/v3)'), -] -secure_message_group = cfg.OptGroup('secure_messages', - title='Secure Messaging options') - -LOG = logging.getLogger(__name__) - - -class SecureMessageException(Exception): - """Generic Exception for Secure Messages.""" - - msg = "An unknown Secure Message related exception occurred." - - def __init__(self, msg=None): - if msg is None: - msg = self.msg - super(SecureMessageException, self).__init__(msg) - - -class SharedKeyNotFound(SecureMessageException): - """No shared key was found and no other external authentication mechanism - is available. - """ - - msg = "Shared Key for [%s] Not Found. (%s)" - - def __init__(self, name, errmsg): - super(SharedKeyNotFound, self).__init__(self.msg % (name, errmsg)) - - -class InvalidMetadata(SecureMessageException): - """The metadata is invalid.""" - - msg = "Invalid metadata: %s" - - def __init__(self, err): - super(InvalidMetadata, self).__init__(self.msg % err) - - -class InvalidSignature(SecureMessageException): - """Signature validation failed.""" - - msg = "Failed to validate signature (source=%s, destination=%s)" - - def __init__(self, src, dst): - super(InvalidSignature, self).__init__(self.msg % (src, dst)) - - -class UnknownDestinationName(SecureMessageException): - """The Destination name is unknown to us.""" - - msg = "Invalid destination name (%s)" - - def __init__(self, name): - super(UnknownDestinationName, self).__init__(self.msg % name) - - -class InvalidEncryptedTicket(SecureMessageException): - """The Encrypted Ticket could not be successfully handled.""" - - msg = "Invalid Ticket (source=%s, destination=%s)" - - def __init__(self, src, dst): - super(InvalidEncryptedTicket, self).__init__(self.msg % (src, dst)) - - -class InvalidExpiredTicket(SecureMessageException): - """The ticket received is already expired.""" - - msg = "Expired ticket (source=%s, destination=%s)" - - def __init__(self, src, dst): - super(InvalidExpiredTicket, self).__init__(self.msg % (src, dst)) - - -class CommunicationError(SecureMessageException): - """The Communication with the KDS failed.""" - - msg = "Communication Error (target=%s): %s" - - def __init__(self, target, errmsg): - super(CommunicationError, self).__init__(self.msg % (target, errmsg)) - - -class InvalidArgument(SecureMessageException): - """Bad initialization argument.""" - - msg = "Invalid argument: %s" - - def __init__(self, errmsg): - super(InvalidArgument, self).__init__(self.msg % errmsg) - - -Ticket = collections.namedtuple('Ticket', ['skey', 'ekey', 'esek']) - - -class KeyStore(object): - """A storage class for Signing and Encryption Keys. - - This class creates an object that holds Generic Keys like Signing - Keys, Encryption Keys, Encrypted SEK Tickets ... - """ - - def __init__(self): - self._kvps = dict() - - def _get_key_name(self, source, target, ktype): - return (source, target, ktype) - - def _put(self, src, dst, ktype, expiration, data): - name = self._get_key_name(src, dst, ktype) - self._kvps[name] = (expiration, data) - - def _get(self, src, dst, ktype): - name = self._get_key_name(src, dst, ktype) - if name in self._kvps: - expiration, data = self._kvps[name] - if expiration > time.time(): - return data - else: - del self._kvps[name] - - return None - - def clear(self): - """Wipes the store clear of all data.""" - self._kvps.clear() - - def put_ticket(self, source, target, skey, ekey, esek, expiration): - """Puts a sek pair in the cache. - - :param source: Client name - :param target: Target name - :param skey: The Signing Key - :param ekey: The Encription Key - :param esek: The token encrypted with the target key - :param expiration: Expiration time in seconds since Epoch - """ - keys = Ticket(skey, ekey, esek) - self._put(source, target, 'ticket', expiration, keys) - - def get_ticket(self, source, target): - """Returns a Ticket (skey, ekey, esek) namedtuple for the - source/target pair. - """ - return self._get(source, target, 'ticket') - - -_KEY_STORE = KeyStore() - - -class _KDSClient(object): - - USER_AGENT = 'oslo-incubator/rpc' - - def __init__(self, endpoint=None, timeout=None): - """A KDS Client class.""" - - self._endpoint = endpoint - if timeout is not None: - self.timeout = float(timeout) - else: - self.timeout = None - - def _do_get(self, url, request): - req_kwargs = dict() - req_kwargs['headers'] = dict() - req_kwargs['headers']['User-Agent'] = self.USER_AGENT - req_kwargs['headers']['Content-Type'] = 'application/json' - req_kwargs['data'] = jsonutils.dumps({'request': request}) - if self.timeout is not None: - req_kwargs['timeout'] = self.timeout - - try: - resp = requests.get(url, **req_kwargs) - except requests.ConnectionError as e: - err = "Unable to establish connection. %s" % e - raise CommunicationError(url, err) - - return resp - - def _get_reply(self, url, resp): - if resp.text: - try: - body = jsonutils.loads(resp.text) - reply = body['reply'] - except (KeyError, TypeError, ValueError): - msg = "Failed to decode reply: %s" % resp.text - raise CommunicationError(url, msg) - else: - msg = "No reply data was returned." - raise CommunicationError(url, msg) - - return reply - - def _get_ticket(self, request, url=None, redirects=10): - """Send an HTTP request. - - Wraps around 'requests' to handle redirects and common errors. - """ - if url is None: - if not self._endpoint: - raise CommunicationError(url, 'Endpoint not configured') - url = self._endpoint + '/kds/ticket' - - while redirects: - resp = self._do_get(url, request) - if resp.status_code in (301, 302, 305): - # Redirected. Reissue the request to the new location. - url = resp.headers['location'] - redirects -= 1 - continue - elif resp.status_code != 200: - msg = "Request returned failure status: %s (%s)" - err = msg % (resp.status_code, resp.text) - raise CommunicationError(url, err) - - return self._get_reply(url, resp) - - raise CommunicationError(url, "Too many redirections, giving up!") - - def get_ticket(self, source, target, crypto, key): - - # prepare metadata - md = {'requestor': source, - 'target': target, - 'timestamp': time.time(), - 'nonce': struct.unpack('Q', os.urandom(8))[0]} - metadata = base64.b64encode(jsonutils.dumps(md)) - - # sign metadata - signature = crypto.sign(key, metadata) - - # HTTP request - reply = self._get_ticket({'metadata': metadata, - 'signature': signature}) - - # verify reply - signature = crypto.sign(key, (reply['metadata'] + reply['ticket'])) - if signature != reply['signature']: - raise InvalidEncryptedTicket(md['source'], md['destination']) - md = jsonutils.loads(base64.b64decode(reply['metadata'])) - if ((md['source'] != source or - md['destination'] != target or - md['expiration'] < time.time())): - raise InvalidEncryptedTicket(md['source'], md['destination']) - - # return ticket data - tkt = jsonutils.loads(crypto.decrypt(key, reply['ticket'])) - - return tkt, md['expiration'] - - -# we need to keep a global nonce, as this value should never repeat non -# matter how many SecureMessage objects we create -_NONCE = None - - -def _get_nonce(): - """We keep a single counter per instance, as it is so huge we can't - possibly cycle through within 1/100 of a second anyway. - """ - - global _NONCE - # Lazy initialize, for now get a random value, multiply by 2^32 and - # use it as the nonce base. The counter itself will rotate after - # 2^32 increments. - if _NONCE is None: - _NONCE = [struct.unpack('I', os.urandom(4))[0], 0] - - # Increment counter and wrap at 2^32 - _NONCE[1] += 1 - if _NONCE[1] > 0xffffffff: - _NONCE[1] = 0 - - # Return base + counter - return long((_NONCE[0] * 0xffffffff)) + _NONCE[1] - - -class SecureMessage(object): - """A Secure Message object. - - This class creates a signing/encryption facility for RPC messages. - It encapsulates all the necessary crypto primitives to insulate - regular code from the intricacies of message authentication, validation - and optionally encryption. - - :param topic: The topic name of the queue - :param host: The server name, together with the topic it forms a unique - name that is used to source signing keys, and verify - incoming messages. - :param conf: a ConfigOpts object - :param key: (optional) explicitly pass in endpoint private key. - If not provided it will be sourced from the service config - :param key_store: (optional) Storage class for local caching - :param encrypt: (defaults to False) Whether to encrypt messages - :param enctype: (defaults to AES) Cipher to use - :param hashtype: (defaults to SHA256) Hash function to use for signatures - """ - - def __init__(self, topic, host, conf, key=None, key_store=None, - encrypt=None, enctype='AES', hashtype='SHA256'): - - conf.register_group(secure_message_group) - conf.register_opts(secure_message_opts, group='secure_messages') - - self._name = '%s.%s' % (topic, host) - self._key = key - self._conf = conf.secure_messages - self._encrypt = self._conf.encrypt if (encrypt is None) else encrypt - self._crypto = cryptoutils.SymmetricCrypto(enctype, hashtype) - self._hkdf = cryptoutils.HKDF(hashtype) - self._kds = _KDSClient(self._conf.kds_endpoint) - - if self._key is None: - self._key = self._init_key(topic, self._name) - if self._key is None: - err = "Secret Key (or key file) is missing or malformed" - raise SharedKeyNotFound(self._name, err) - - self._key_store = key_store or _KEY_STORE - - def _init_key(self, topic, name): - keys = None - if self._conf.secret_keys_file: - with open(self._conf.secret_keys_file, 'r') as f: - keys = f.readlines() - elif self._conf.secret_key: - keys = self._conf.secret_key - - if keys is None: - return None - - for k in keys: - if k[0] == '#': - continue - if ':' not in k: - break - svc, key = k.split(':', 1) - if svc == topic or svc == name: - return base64.b64decode(key) - - return None - - def _split_key(self, key, size): - sig_key = key[:size] - enc_key = key[size:] - return sig_key, enc_key - - def _decode_esek(self, key, source, target, timestamp, esek): - """This function decrypts the esek buffer passed in and returns a - KeyStore to be used to check and decrypt the received message. - - :param key: The key to use to decrypt the ticket (esek) - :param source: The name of the source service - :param traget: The name of the target service - :param timestamp: The incoming message timestamp - :param esek: a base64 encoded encrypted block containing a JSON string - """ - rkey = None - - try: - s = self._crypto.decrypt(key, esek) - j = jsonutils.loads(s) - - rkey = base64.b64decode(j['key']) - expiration = j['timestamp'] + j['ttl'] - if j['timestamp'] > timestamp or timestamp > expiration: - raise InvalidExpiredTicket(source, target) - - except Exception: - raise InvalidEncryptedTicket(source, target) - - info = '%s,%s,%s' % (source, target, str(j['timestamp'])) - - sek = self._hkdf.expand(rkey, info, len(key) * 2) - - return self._split_key(sek, len(key)) - - def _get_ticket(self, target): - """This function will check if we already have a SEK for the specified - target in the cache, or will go and try to fetch a new SEK from the key - server. - - :param target: The name of the target service - """ - ticket = self._key_store.get_ticket(self._name, target) - - if ticket is not None: - return ticket - - tkt, expiration = self._kds.get_ticket(self._name, target, - self._crypto, self._key) - - self._key_store.put_ticket(self._name, target, - base64.b64decode(tkt['skey']), - base64.b64decode(tkt['ekey']), - tkt['esek'], expiration) - return self._key_store.get_ticket(self._name, target) - - def encode(self, version, target, json_msg): - """This is the main encoding function. - - It takes a target and a message and returns a tuple consisting of a - JSON serialized metadata object, a JSON serialized (and optionally - encrypted) message, and a signature. - - :param version: the current envelope version - :param target: The name of the target service (usually with hostname) - :param json_msg: a serialized json message object - """ - ticket = self._get_ticket(target) - - metadata = jsonutils.dumps({'source': self._name, - 'destination': target, - 'timestamp': time.time(), - 'nonce': _get_nonce(), - 'esek': ticket.esek, - 'encryption': self._encrypt}) - - message = json_msg - if self._encrypt: - message = self._crypto.encrypt(ticket.ekey, message) - - signature = self._crypto.sign(ticket.skey, - version + metadata + message) - - return (metadata, message, signature) - - def decode(self, version, metadata, message, signature): - """This is the main decoding function. - - It takes a version, metadata, message and signature strings and - returns a tuple with a (decrypted) message and metadata or raises - an exception in case of error. - - :param version: the current envelope version - :param metadata: a JSON serialized object with metadata for validation - :param message: a JSON serialized (base64 encoded encrypted) message - :param signature: a base64 encoded signature - """ - md = jsonutils.loads(metadata) - - check_args = ('source', 'destination', 'timestamp', - 'nonce', 'esek', 'encryption') - for arg in check_args: - if arg not in md: - raise InvalidMetadata('Missing metadata "%s"' % arg) - - if md['destination'] != self._name: - # TODO(simo) handle group keys by checking target - raise UnknownDestinationName(md['destination']) - - try: - skey, ekey = self._decode_esek(self._key, - md['source'], md['destination'], - md['timestamp'], md['esek']) - except InvalidExpiredTicket: - raise - except Exception: - raise InvalidMetadata('Failed to decode ESEK for %s/%s' % ( - md['source'], md['destination'])) - - sig = self._crypto.sign(skey, version + metadata + message) - - if sig != signature: - raise InvalidSignature(md['source'], md['destination']) - - if md['encryption'] is True: - msg = self._crypto.decrypt(ekey, message) - else: - msg = message - - return (md, msg) diff --git a/ceilometer/openstack/common/rpc/serializer.py b/ceilometer/openstack/common/rpc/serializer.py index 5fd346d65..9bc6e2a3a 100644 --- a/ceilometer/openstack/common/rpc/serializer.py +++ b/ceilometer/openstack/common/rpc/serializer.py @@ -15,6 +15,7 @@ """Provides the definition of an RPC serialization handler""" import abc + import six diff --git a/ceilometer/openstack/common/rpc/service.py b/ceilometer/openstack/common/rpc/service.py index 7ffb83a38..65b284bca 100644 --- a/ceilometer/openstack/common/rpc/service.py +++ b/ceilometer/openstack/common/rpc/service.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. diff --git a/ceilometer/openstack/common/rpc/zmq_receiver.py b/ceilometer/openstack/common/rpc/zmq_receiver.py index 14e869b2e..654b52ab7 100644 --- a/ceilometer/openstack/common/rpc/zmq_receiver.py +++ b/ceilometer/openstack/common/rpc/zmq_receiver.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/ceilometer/openstack/common/service.py b/ceilometer/openstack/common/service.py index 574dd9096..670f9cd3c 100644 --- a/ceilometer/openstack/common/service.py +++ b/ceilometer/openstack/common/service.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2011 Justin Santa Barbara @@ -20,6 +18,7 @@ """Generic Node base class for all workers that run on hosts.""" import errno +import logging as std_logging import os import random import signal @@ -28,7 +27,6 @@ import time import eventlet from eventlet import event -import logging as std_logging from oslo.config import cfg from ceilometer.openstack.common import eventlet_backdoor @@ -129,7 +127,7 @@ class ServiceLauncher(Launcher): def handle_signal(self): _set_signals_handler(self._handle_signal) - def _wait_for_exit_or_signal(self): + def _wait_for_exit_or_signal(self, ready_callback=None): status = None signo = 0 @@ -137,6 +135,8 @@ class ServiceLauncher(Launcher): CONF.log_opt_values(LOG, std_logging.DEBUG) try: + if ready_callback: + ready_callback() super(ServiceLauncher, self).wait() except SignalExit as exc: signame = _signo_to_signame(exc.signo) @@ -156,10 +156,10 @@ class ServiceLauncher(Launcher): return status, signo - def wait(self): + def wait(self, ready_callback=None): while True: self.handle_signal() - status, signo = self._wait_for_exit_or_signal() + status, signo = self._wait_for_exit_or_signal(ready_callback) if not _is_sighup(signo): return status self.restart() @@ -218,7 +218,7 @@ class ProcessLauncher(object): signal.signal(signal.SIGINT, signal.SIG_IGN) def _child_wait_for_exit_or_signal(self, launcher): - status = None + status = 0 signo = 0 # NOTE(johannes): All exceptions are caught to ensure this diff --git a/ceilometer/openstack/common/sslutils.py b/ceilometer/openstack/common/sslutils.py index d67d05440..e29938460 100644 --- a/ceilometer/openstack/common/sslutils.py +++ b/ceilometer/openstack/common/sslutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2013 IBM Corp. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/ceilometer/openstack/common/strutils.py b/ceilometer/openstack/common/strutils.py index 6ca349281..ac18323ca 100644 --- a/ceilometer/openstack/common/strutils.py +++ b/ceilometer/openstack/common/strutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -101,7 +99,7 @@ def safe_decode(text, incoming=None, errors='strict'): values http://docs.python.org/2/library/codecs.html :returns: text or a unicode `incoming` encoded representation of it. - :raises TypeError: If text is not an isntance of str + :raises TypeError: If text is not an instance of str """ if not isinstance(text, six.string_types): raise TypeError("%s can't be decoded" % type(text)) @@ -144,7 +142,7 @@ def safe_encode(text, incoming=None, values http://docs.python.org/2/library/codecs.html :returns: text or a bytestring `encoding` encoded representation of it. - :raises TypeError: If text is not an isntance of str + :raises TypeError: If text is not an instance of str """ if not isinstance(text, six.string_types): raise TypeError("%s can't be encoded" % type(text)) diff --git a/ceilometer/openstack/common/test.py b/ceilometer/openstack/common/test.py index 00088cee4..9e62caae1 100644 --- a/ceilometer/openstack/common/test.py +++ b/ceilometer/openstack/common/test.py @@ -1,19 +1,17 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010-2011 OpenStack Foundation +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # All Rights Reserved. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. """Common utilities used in testing""" @@ -22,6 +20,8 @@ import os import fixtures import testtools +_TRUE_VALUES = ('True', 'true', '1', 'yes') + class BaseTestCase(testtools.TestCase): @@ -31,6 +31,7 @@ class BaseTestCase(testtools.TestCase): self._fake_output() self.useFixture(fixtures.FakeLogger('ceilometer.openstack.common')) self.useFixture(fixtures.NestedTempfile()) + self.useFixture(fixtures.TempHomeDir()) def _set_timeout(self): test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0) @@ -43,11 +44,9 @@ class BaseTestCase(testtools.TestCase): self.useFixture(fixtures.Timeout(test_timeout, gentle=True)) def _fake_output(self): - if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or - os.environ.get('OS_STDOUT_CAPTURE') == '1'): + if os.environ.get('OS_STDOUT_CAPTURE') in _TRUE_VALUES: stdout = self.useFixture(fixtures.StringStream('stdout')).stream self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout)) - if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or - os.environ.get('OS_STDERR_CAPTURE') == '1'): + if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES: stderr = self.useFixture(fixtures.StringStream('stderr')).stream self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) diff --git a/ceilometer/openstack/common/threadgroup.py b/ceilometer/openstack/common/threadgroup.py index a060897d4..97ccba97f 100644 --- a/ceilometer/openstack/common/threadgroup.py +++ b/ceilometer/openstack/common/threadgroup.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -48,6 +46,9 @@ class Thread(object): def wait(self): return self.thread.wait() + def link(self, func, *args, **kwargs): + self.thread.link(func, *args, **kwargs) + class ThreadGroup(object): """The point of the ThreadGroup classis to: @@ -79,6 +80,7 @@ class ThreadGroup(object): gt = self.pool.spawn(callback, *args, **kwargs) th = Thread(gt, self) self.threads.append(th) + return th def thread_done(self, thread): self.threads.remove(thread) diff --git a/ceilometer/openstack/common/timeutils.py b/ceilometer/openstack/common/timeutils.py index 98d877d59..c8b0b1539 100644 --- a/ceilometer/openstack/common/timeutils.py +++ b/ceilometer/openstack/common/timeutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -50,9 +48,9 @@ def parse_isotime(timestr): try: return iso8601.parse_date(timestr) except iso8601.ParseError as e: - raise ValueError(unicode(e)) + raise ValueError(six.text_type(e)) except TypeError as e: - raise ValueError(unicode(e)) + raise ValueError(six.text_type(e)) def strtime(at=None, fmt=PERFECT_TIME_FORMAT): @@ -178,6 +176,15 @@ def delta_seconds(before, after): datetime objects (as a float, to microsecond resolution). """ delta = after - before + return total_seconds(delta) + + +def total_seconds(delta): + """Return the total seconds of datetime.timedelta object. + + Compute total seconds of datetime.timedelta, datetime.timedelta + doesn't have method total_seconds in Python2.6, calculate it manually. + """ try: return delta.total_seconds() except AttributeError: diff --git a/ceilometer/openstack/common/versionutils.py b/ceilometer/openstack/common/versionutils.py index f7b1f8a82..f8dc13ea8 100644 --- a/ceilometer/openstack/common/versionutils.py +++ b/ceilometer/openstack/common/versionutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2013 OpenStack Foundation # All Rights Reserved. # diff --git a/ceilometer/openstack/common/xmlutils.py b/ceilometer/openstack/common/xmlutils.py index b131d3e2e..1231a5902 100644 --- a/ceilometer/openstack/common/xmlutils.py +++ b/ceilometer/openstack/common/xmlutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2013 IBM Corp. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/etc/ceilometer/ceilometer.conf.sample b/etc/ceilometer/ceilometer.conf.sample index bae6f727b..d546cedca 100644 --- a/etc/ceilometer/ceilometer.conf.sample +++ b/etc/ceilometer/ceilometer.conf.sample @@ -24,6 +24,7 @@ # # Source for samples emited on this instance (string value) +# Deprecated group/name - [DEFAULT]/counter_source #sample_source=openstack @@ -108,6 +109,7 @@ # # Exchange name for Neutron notifications (string value) +# Deprecated group/name - [DEFAULT]/quantum_control_exchange #neutron_control_exchange=neutron @@ -202,7 +204,7 @@ #logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s # list of logger=LEVEL pairs (list value) -#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,keystone=INFO,qpid=WARN,sqlalchemy=WARN,suds=INFO +#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,keystone=INFO,qpid=WARN,sqlalchemy=WARN,suds=INFO,iso8601=WARN # publish error events (boolean value) #publish_errors=false @@ -223,6 +225,7 @@ # configuration to any other existing logging options. Please # see the Python logging module documentation for details on # logging configuration files. (string value) +# Deprecated group/name - [DEFAULT]/log_config #log_config_append= # DEPRECATED. A logging.Formatter log message format string @@ -238,10 +241,12 @@ # (Optional) Name of log file to output to. If no default is # set, logging will go to stdout. (string value) +# Deprecated group/name - [DEFAULT]/logfile #log_file= # (Optional) The base directory used for relative --log-file # paths (string value) +# Deprecated group/name - [DEFAULT]/logdir #log_dir= # Use syslog for logging. (boolean value) @@ -257,6 +262,7 @@ # the maximum body size per each request(bytes) (integer # value) +# Deprecated group/name - [DEFAULT]/osapi_max_request_body_size #max_request_body_size=114688 @@ -319,8 +325,8 @@ # by impl_zmq. (integer value) #rpc_cast_timeout=30 -# Modules of exceptions that are permitted to be recreatedupon -# receiving exception data from an rpc call. (list value) +# Modules of exceptions that are permitted to be recreated +# upon receiving exception data from an rpc call. (list value) #allowed_rpc_exception_modules=nova.exception,cinder.exception,exceptions # If passed, use a fake RabbitMQ provider (boolean value) @@ -336,6 +342,7 @@ # # Use durable queues in amqp. (boolean value) +# Deprecated group/name - [DEFAULT]/rabbit_durable_queues #amqp_durable_queues=false # Auto-delete queues in amqp. (boolean value) @@ -551,10 +558,12 @@ # # The backend to use for db (string value) +# Deprecated group/name - [DEFAULT]/db_backend #backend=sqlalchemy # Enable the experimental use of thread pooling for all DB API # calls (boolean value) +# Deprecated group/name - [DEFAULT]/dbapi_use_tpool #use_tpool=false @@ -564,6 +573,9 @@ # The SQLAlchemy connection string used to connect to the # database (string value) +# Deprecated group/name - [DEFAULT]/sql_connection +# Deprecated group/name - [DATABASE]/sql_connection +# Deprecated group/name - [sql]/connection #connection=sqlite:////ceilometer/openstack/common/db/$sqlite_db # The SQLAlchemy connection string used to connect to the @@ -572,38 +584,53 @@ # timeout before idle sql connections are reaped (integer # value) +# Deprecated group/name - [DEFAULT]/sql_idle_timeout +# Deprecated group/name - [DATABASE]/sql_idle_timeout #idle_timeout=3600 # Minimum number of SQL connections to keep open in a pool # (integer value) +# Deprecated group/name - [DEFAULT]/sql_min_pool_size +# Deprecated group/name - [DATABASE]/sql_min_pool_size #min_pool_size=1 # Maximum number of SQL connections to keep open in a pool # (integer value) +# Deprecated group/name - [DEFAULT]/sql_max_pool_size +# Deprecated group/name - [DATABASE]/sql_max_pool_size #max_pool_size= # maximum db connection retries during startup. (setting -1 # implies an infinite retry count) (integer value) +# Deprecated group/name - [DEFAULT]/sql_max_retries +# Deprecated group/name - [DATABASE]/sql_max_retries #max_retries=10 # interval between retries of opening a sql connection # (integer value) +# Deprecated group/name - [DEFAULT]/sql_retry_interval +# Deprecated group/name - [DATABASE]/reconnect_interval #retry_interval=10 # If set, use this value for max_overflow with sqlalchemy # (integer value) +# Deprecated group/name - [DEFAULT]/sql_max_overflow +# Deprecated group/name - [DATABASE]/sqlalchemy_max_overflow #max_overflow= # Verbosity of SQL debugging information. 0=None, # 100=Everything (integer value) +# Deprecated group/name - [DEFAULT]/sql_connection_debug #connection_debug=0 # Add python stack traces to SQL as comment strings (boolean # value) +# Deprecated group/name - [DEFAULT]/sql_connection_trace #connection_trace=false # If set, use this value for pool_timeout with sqlalchemy # (integer value) +# Deprecated group/name - [DATABASE]/sqlalchemy_pool_timeout #pool_timeout= @@ -667,6 +694,7 @@ # Period of evaluation cycle, should be >= than configured # pipeline interval for collection of underlying metrics. # (integer value) +# Deprecated group/name - [alarm]/threshold_evaluation_interval #evaluation_interval=60 # Class to launch as alarm evaluation service (string value) @@ -698,6 +726,7 @@ # # The port for the ceilometer API server (integer value) +# Deprecated group/name - [DEFAULT]/metering_api_port #port=8777 # The listen IP for the ceilometer API server (string value) @@ -760,119 +789,6 @@ #backup_count=0 -[keystone_authtoken] - -# -# Options defined in keystoneclient.middleware.auth_token -# - -# Prefix to prepend at the beginning of the path (string -# value) -#auth_admin_prefix= - -# Host providing the admin Identity API endpoint (string -# value) -#auth_host=127.0.0.1 - -# Port of the admin Identity API endpoint (integer value) -#auth_port=35357 - -# Protocol of the admin Identity API endpoint(http or https) -# (string value) -#auth_protocol=https - -# Complete public Identity API endpoint (string value) -#auth_uri= - -# API version of the admin Identity API endpoint (string -# value) -#auth_version= - -# Do not handle authorization requests within the middleware, -# but delegate the authorization decision to downstream WSGI -# components (boolean value) -#delay_auth_decision=false - -# Request timeout value for communicating with Identity API -# server. (boolean value) -#http_connect_timeout= - -# How many times are we trying to reconnect when communicating -# with Identity API Server. (integer value) -#http_request_max_retries=3 - -# Allows to pass in the name of a fake http_handler callback -# function used instead of httplib.HTTPConnection or -# httplib.HTTPSConnection. Useful for unit testing where -# network is not available. (string value) -#http_handler= - -# Single shared secret with the Keystone configuration used -# for bootstrapping a Keystone installation, or otherwise -# bypassing the normal authentication process. (string value) -#admin_token= - -# Keystone account username (string value) -#admin_user= - -# Keystone account password (string value) -#admin_password= - -# Keystone service account tenant name to validate user tokens -# (string value) -#admin_tenant_name=admin - -# Env key for the swift cache (string value) -#cache= - -# Required if Keystone server requires client certificate -# (string value) -#certfile= - -# Required if Keystone server requires client certificate -# (string value) -#keyfile= - -# A PEM encoded Certificate Authority to use when verifying -# HTTPs connections. Defaults to system CAs. (string value) -#cafile= - -# Verify HTTPS connections. (boolean value) -#insecure=false - -# Directory used to cache files related to PKI tokens (string -# value) -#signing_dir= - -# If defined, the memcache server(s) to use for caching (list -# value) -#memcached_servers= - -# In order to prevent excessive requests and validations, the -# middleware uses an in-memory cache for the tokens the -# Keystone API returns. This is only valid if memcache_servers -# is defined. Set to -1 to disable caching completely. -# (integer value) -#token_cache_time=300 - -# Value only used for unit testing (integer value) -#revocation_cache_time=1 - -# (optional) if defined, indicate whether token data should be -# authenticated or authenticated and encrypted. Acceptable -# values are MAC or ENCRYPT. If MAC, token data is -# authenticated (with HMAC) in the cache. If ENCRYPT, token -# data is encrypted and authenticated in the cache. If the -# value is not one of these options or empty, auth_token will -# raise an exception on initialization. (string value) -#memcache_security_strategy= - -# (optional, mandatory if memcache_security_strategy is -# defined) this string is used for key derivation. (string -# value) -#memcache_secret_key= - - [collector] # @@ -894,6 +810,7 @@ # # Matchmaker ring file (JSON) (string value) +# Deprecated group/name - [DEFAULT]/matchmaker_ringfile #ringfile=/etc/oslo/matchmaker_ring.json diff --git a/requirements.txt b/requirements.txt index a4d4ad16b..847190b3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ python-novaclient>=2.15.0 python-keystoneclient>=0.4.1 python-ceilometerclient>=1.0.6 python-swiftclient>=1.5 +lockfile>=0.8 lxml>=2.3 requests>=1.1 six>=1.4.1 diff --git a/tools/config/generate_sample.sh b/tools/config/generate_sample.sh index d28a24d2d..cfc40dc10 100755 --- a/tools/config/generate_sample.sh +++ b/tools/config/generate_sample.sh @@ -73,6 +73,7 @@ then fi BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'` +find $TARGETDIR -type f -name "*.pyc" -delete FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \ -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u) @@ -86,7 +87,13 @@ export EVENTLET_NO_GREENDNS=yes OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs) [ "$OS_VARS" ] && eval "unset \$OS_VARS" - -MODULEPATH=ceilometer.openstack.common.config.generator +DEFAULT_MODULEPATH=ceilometer.openstack.common.config.generator +MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH} OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample python -m $MODULEPATH $FILES > $OUTPUTFILE + +# Hook to allow projects to append custom config file snippets +CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null) +for CONCAT_FILE in $CONCAT_FILES; do + cat $CONCAT_FILE >> $OUTPUTFILE +done