openstack/aodh
Code Issues Proposed changes

Merge "Update oslo"

Browse Source
This commit is contained in:
Jenkins 2014-01-22 11:10:37 +00:00 committed by Gerrit Code Review
commit 9229bf12fa
60 changed files with 944 additions and 753 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ceilometer/api/middleware.py
View File

@ -104,7 +104,7 @@ class ParsableErrorMiddleware(object):
if error is not None:
for fault_string in fault.findall('faultstring'):
fault_string.text = (
gettextutils.get_localized_message(
gettextutils.translate(
error, user_locale))
body = ['<error_message>' + etree.tostring(fault)
+ '</error_message>']
@ -118,7 +118,7 @@ class ParsableErrorMiddleware(object):
fault = json.loads('\n'.join(app_iter))
if error is not None and 'faultstring' in fault:
fault['faultstring'] = (
gettextutils.get_localized_message(
gettextutils.translate(
error, user_locale))
body = [json.dumps({'error_message': fault})]
except ValueError as err:

10
ceilometer/openstack/common/config/generator.py
View File

@ -67,10 +67,8 @@ def generate(srcfiles):
os.path.basename(filepath).split('.')[0]])
mods_by_pkg.setdefault(pkg_name, list()).append(mod_str)
# NOTE(lzyeval): place top level modules before packages
pkg_names = filter(lambda x: x.endswith(PY_EXT), mods_by_pkg.keys())
pkg_names.sort()
ext_names = filter(lambda x: x not in pkg_names, mods_by_pkg.keys())
ext_names.sort()
pkg_names = sorted(pkg for pkg in mods_by_pkg if pkg.endswith(PY_EXT))
ext_names = sorted(pkg for pkg in mods_by_pkg if pkg not in pkg_names)
pkg_names.extend(ext_names)
# opts_by_group is a mapping of group name to an options list
@ -120,7 +118,7 @@ def _import_module(mod_str):
def _is_in_group(opt, group):
"Check if opt is in group."
for key, value in group._opts.items():
for value in group._opts.values():
# NOTE(llu): Temporary workaround for bug #1262148, wait until
# newly released oslo.config support '==' operator.
if not(value['opt'] != opt):
@ -134,7 +132,7 @@ def _guess_groups(opt, mod_obj):
return 'DEFAULT'
# what other groups is it in?
for key, value in cfg.CONF.items():
for value in cfg.CONF.values():
if isinstance(value, cfg.CONF.GroupAttr):
if _is_in_group(opt, value._group):
return value._group.name

21
ceilometer/openstack/common/context.py
View File

@ -36,12 +36,18 @@ class RequestContext(object):
accesses the system, as well as additional request information.
"""
def __init__(self, auth_token=None, user=None, tenant=None, is_admin=False,
user_idt_format = '{user} {tenant} {domain} {user_domain} {p_domain}'
def __init__(self, auth_token=None, user=None, tenant=None, domain=None,
user_domain=None, project_domain=None, is_admin=False,
read_only=False, show_deleted=False, request_id=None,
instance_uuid=None):
self.auth_token = auth_token
self.user = user
self.tenant = tenant
self.domain = domain
self.user_domain = user_domain
self.project_domain = project_domain
self.is_admin = is_admin
self.read_only = read_only
self.show_deleted = show_deleted
@ -51,14 +57,25 @@ class RequestContext(object):
self.request_id = request_id
def to_dict(self):
user_idt = (
self.user_idt_format.format(user=self.user or '-',
tenant=self.tenant or '-',
domain=self.domain or '-',
user_domain=self.user_domain or '-',
p_domain=self.project_domain or '-'))
return {'user': self.user,
'tenant': self.tenant,
'domain': self.domain,
'user_domain': self.user_domain,
'project_domain': self.project_domain,
'is_admin': self.is_admin,
'read_only': self.read_only,
'show_deleted': self.show_deleted,
'auth_token': self.auth_token,
'request_id': self.request_id,
'instance_uuid': self.instance_uuid}
'instance_uuid': self.instance_uuid,
'user_identity': user_idt}
def get_admin_context(show_deleted=False):

14
ceilometer/openstack/common/db/__init__.py
View File

@ -1,14 +0,0 @@
# Copyright 2012 Cloudscaling Group, Inc
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

51
ceilometer/openstack/common/db/api.py
View File

@ -19,27 +19,15 @@ Supported configuration options:
The following two parameters are in the 'database' group:
`backend`: DB backend name or full module path to DB backend module.
`use_tpool`: Enable thread pooling of DB API calls.
A DB backend module should implement a method named 'get_backend' which
takes no arguments. The method can return any object that implements DB
API methods.
*NOTE*: There are bugs in eventlet when using tpool combined with
threading locks. The python logging module happens to use such locks. To
work around this issue, be sure to specify thread=False with
eventlet.monkey_patch().
A bug for eventlet has been filed here:
https://bitbucket.org/eventlet/eventlet/issue/137/
"""
import functools
from oslo.config import cfg
from ceilometer.openstack.common import importutils
from ceilometer.openstack.common import lockutils
db_opts = [
@ -48,12 +36,6 @@ db_opts = [
deprecated_name='db_backend',
deprecated_group='DEFAULT',
help='The backend to use for db'),
cfg.BoolOpt('use_tpool',
default=False,
deprecated_name='dbapi_use_tpool',
deprecated_group='DEFAULT',
help='Enable the experimental use of thread pooling for '
'all DB API calls')
]
CONF = cfg.CONF
@ -64,41 +46,12 @@ class DBAPI(object):
def __init__(self, backend_mapping=None):
if backend_mapping is None:
backend_mapping = {}
self.__backend = None
self.__backend_mapping = backend_mapping
@lockutils.synchronized('dbapi_backend', 'ceilometer-')
def __get_backend(self):
"""Get the actual backend. May be a module or an instance of
a class. Doesn't matter to us. We do this synchronized as it's
possible multiple greenthreads started very quickly trying to do
DB calls and eventlet can switch threads before self.__backend gets
assigned.
"""
if self.__backend:
# Another thread assigned it
return self.__backend
backend_name = CONF.database.backend
self.__use_tpool = CONF.database.use_tpool
if self.__use_tpool:
from eventlet import tpool
self.__tpool = tpool
# Import the untranslated name if we don't have a
# mapping.
backend_path = self.__backend_mapping.get(backend_name,
backend_name)
backend_path = backend_mapping.get(backend_name, backend_name)
backend_mod = importutils.import_module(backend_path)
self.__backend = backend_mod.get_backend()
return self.__backend
def __getattr__(self, key):
backend = self.__backend or self.__get_backend()
attr = getattr(backend, key)
if not self.__use_tpool or not hasattr(attr, '__call__'):
return attr
def tpool_wrapper(*args, **kwargs):
return self.__tpool.execute(attr, *args, **kwargs)
functools.update_wrapper(tpool_wrapper, attr)
return tpool_wrapper
return getattr(self.__backend, key)

2
ceilometer/openstack/common/db/exception.py
View File

@ -16,7 +16,7 @@
"""DB related custom exceptions."""
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
class DBError(Exception):

14
ceilometer/openstack/common/db/sqlalchemy/__init__.py
View File

@ -1,14 +0,0 @@
# Copyright 2012 Cloudscaling Group, Inc
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

52
ceilometer/openstack/common/db/sqlalchemy/migration.py
View File

@ -36,53 +36,25 @@
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
import distutils.version as dist_version
import os
import re
import migrate
from migrate.changeset import ansisql
from migrate.changeset.databases import sqlite
from migrate.versioning import util as migrate_util
from migrate import exceptions as versioning_exceptions
from migrate.versioning import api as versioning_api
from migrate.versioning.repository import Repository
import sqlalchemy
from sqlalchemy.schema import UniqueConstraint
from ceilometer.openstack.common.db import exception
from ceilometer.openstack.common.db.sqlalchemy import session as db_session
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
@migrate_util.decorator
def patched_with_engine(f, *a, **kw):
url = a[0]
engine = migrate_util.construct_engine(url, **kw)
try:
kw['engine'] = engine
return f(*a, **kw)
finally:
if isinstance(engine, migrate_util.Engine) and engine is not url:
migrate_util.log.debug('Disposing SQLAlchemy engine %s', engine)
engine.dispose()
# TODO(jkoelker) When migrate 0.7.3 is released and nova depends
# on that version or higher, this can be removed
MIN_PKG_VERSION = dist_version.StrictVersion('0.7.3')
if (not hasattr(migrate, '__version__') or
dist_version.StrictVersion(migrate.__version__) < MIN_PKG_VERSION):
migrate_util.with_engine = patched_with_engine
# NOTE(jkoelker) Delay importing migrate until we are patched
from migrate import exceptions as versioning_exceptions
from migrate.versioning import api as versioning_api
from migrate.versioning.repository import Repository
_REPOSITORY = None
get_engine = db_session.get_engine
@ -245,10 +217,11 @@ def db_version(abs_path, init_version):
db_version_control(abs_path, init_version)
return versioning_api.db_version(get_engine(), repository)
else:
# Some pre-Essex DB's may not be version controlled.
# Require them to upgrade using Essex first.
raise exception.DbMigrationError(
message=_("Upgrade DB using Essex release first."))
message=_(
"The database is not under version control, but has "
"tables. Please stamp the current version of the schema "
"manually."))
def db_version_control(abs_path, version=None):
@ -270,9 +243,6 @@ def _find_migrate_repo(abs_path):
:param abs_path: Absolute path to migrate repository
"""
global _REPOSITORY
if not os.path.exists(abs_path):
raise exception.DbMigrationError("Path %s not found" % abs_path)
if _REPOSITORY is None:
_REPOSITORY = Repository(abs_path)
return _REPOSITORY
return Repository(abs_path)

19
ceilometer/openstack/common/db/sqlalchemy/models.py
View File

@ -39,13 +39,13 @@ class ModelBase(object):
if not session:
session = sa.get_session()
# NOTE(boris-42): This part of code should be look like:
# sesssion.add(self)
# session.add(self)
# session.flush()
# But there is a bug in sqlalchemy and eventlet that
# raises NoneType exception if there is no running
# transaction and rollback is called. As long as
# sqlalchemy has this bug we have to create transaction
# explicity.
# explicitly.
with session.begin(subtransactions=True):
session.add(self)
session.flush()
@ -59,7 +59,16 @@ class ModelBase(object):
def get(self, key, default=None):
return getattr(self, key, default)
def _get_extra_keys(self):
@property
def _extra_keys(self):
"""Specifies custom fields
Subclasses can override this property to return a list
of custom fields that should be included in their dict
representation.
For reference check tests/db/sqlalchemy/test_models.py
"""
return []
def __iter__(self):
@ -67,7 +76,7 @@ class ModelBase(object):
# NOTE(russellb): Allow models to specify other keys that can be looked
# up, beyond the actual db columns. An example would be the 'name'
# property for an Instance.
columns.extend(self._get_extra_keys())
columns.extend(self._extra_keys)
self._i = iter(columns)
return self
@ -89,7 +98,7 @@ class ModelBase(object):
joined = dict([(k, v) for k, v in six.iteritems(self.__dict__)
if not k[0] == '_'])
local.update(joined)
return local.iteritems()
return six.iteritems(local)
class TimestampMixin(object):

6
ceilometer/openstack/common/db/sqlalchemy/provision.py
View File

@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 Mirantis.inc
# All Rights Reserved.
#
@ -22,6 +20,7 @@ import os
import random
import string
from six import moves
import sqlalchemy
from ceilometer.openstack.common.db import exception as exc
@ -34,7 +33,8 @@ def _gen_credentials(*names):
"""Generate credentials."""
auth_dict = {}
for name in names:
val = ''.join(random.choice(string.lowercase) for i in xrange(10))
val = ''.join(random.choice(string.ascii_lowercase)
for i in moves.range(10))
auth_dict[name] = val
return auth_dict

167
ceilometer/openstack/common/db/sqlalchemy/session.py
View File

@ -21,7 +21,7 @@ Initializing:
* Call set_defaults with the minimal of the following kwargs:
sql_connection, sqlite_db
Example:
Example::
session.set_defaults(
sql_connection="sqlite:///var/lib/ceilometer/sqlite.db",
@ -42,17 +42,17 @@ Recommended ways to use sessions within this framework:
functionality should be handled at a logical level. For an example, look at
the code around quotas and reservation_rollback().
Examples:
Examples::
def get_foo(context, foo):
return model_query(context, models.Foo).\
filter_by(foo=foo).\
first()
return (model_query(context, models.Foo).
filter_by(foo=foo).
first())
def update_foo(context, id, newfoo):
model_query(context, models.Foo).\
filter_by(id=id).\
update({'foo': newfoo})
(model_query(context, models.Foo).
filter_by(id=id).
update({'foo': newfoo}))
def create_foo(context, values):
foo_ref = models.Foo()
@ -66,14 +66,21 @@ Recommended ways to use sessions within this framework:
handler will take care of calling flush() and commit() for you.
If using this approach, you should not explicitly call flush() or commit().
Any error within the context of the session will cause the session to emit
a ROLLBACK. If the connection is dropped before this is possible, the
database will implicitly rollback the transaction.
a ROLLBACK. Database Errors like IntegrityError will be raised in
session's __exit__ handler, and any try/except within the context managed
by session will not be triggered. And catching other non-database errors in
the session will not trigger the ROLLBACK, so exception handlers should
always be outside the session, unless the developer wants to do a partial
commit on purpose. If the connection is dropped before this is possible,
the database will implicitly roll back the transaction.
Note: statements in the session scope will not be automatically retried.
If you create models within the session, they need to be added, but you
do not need to call model.save()
::
def create_many_foo(context, foos):
session = get_session()
with session.begin():
@ -85,33 +92,50 @@ Recommended ways to use sessions within this framework:
def update_bar(context, foo_id, newbar):
session = get_session()
with session.begin():
foo_ref = model_query(context, models.Foo, session).\
filter_by(id=foo_id).\
first()
model_query(context, models.Bar, session).\
filter_by(id=foo_ref['bar_id']).\
update({'bar': newbar})
foo_ref = (model_query(context, models.Foo, session).
filter_by(id=foo_id).
first())
(model_query(context, models.Bar, session).
filter_by(id=foo_ref['bar_id']).
update({'bar': newbar}))
Note: update_bar is a trivially simple example of using "with session.begin".
Whereas create_many_foo is a good example of when a transaction is needed,
it is always best to use as few queries as possible. The two queries in
update_bar can be better expressed using a single query which avoids
the need for an explicit transaction. It can be expressed like so:
the need for an explicit transaction. It can be expressed like so::
def update_bar(context, foo_id, newbar):
subq = model_query(context, models.Foo.id).\
filter_by(id=foo_id).\
limit(1).\
subquery()
model_query(context, models.Bar).\
filter_by(id=subq.as_scalar()).\
update({'bar': newbar})
subq = (model_query(context, models.Foo.id).
filter_by(id=foo_id).
limit(1).
subquery())
(model_query(context, models.Bar).
filter_by(id=subq.as_scalar()).
update({'bar': newbar}))
For reference, this emits approximagely the following SQL statement:
For reference, this emits approximately the following SQL statement::
UPDATE bar SET bar = ${newbar}
WHERE id=(SELECT bar_id FROM foo WHERE id = ${foo_id} LIMIT 1);
Note: create_duplicate_foo is a trivially simple example of catching an
exception while using "with session.begin". Here create two duplicate
instances with same primary key, must catch the exception out of context
managed by a single session:
def create_duplicate_foo(context):
foo1 = models.Foo()
foo2 = models.Foo()
foo1.id = foo2.id = 1
session = get_session()
try:
with session.begin():
session.add(foo1)
session.add(foo2)
except exception.DBDuplicateEntry as e:
handle_error(e)
* Passing an active session between methods. Sessions should only be passed
to private methods. The private method must use a subtransaction; otherwise
SQLAlchemy will throw an error when you call session.begin() on an existing
@ -127,6 +151,8 @@ Recommended ways to use sessions within this framework:
becomes less clear in this situation. When this is needed for code clarity,
it should be clearly documented.
::
def myfunc(foo):
session = get_session()
with session.begin():
@ -171,7 +197,7 @@ There are some things which it is best to avoid:
Enabling soft deletes:
* To use/enable soft-deletes, the SoftDeleteMixin must be added
to your model class. For example:
to your model class. For example::
class NovaBase(models.SoftDeleteMixin, models.ModelBase):
pass
@ -179,14 +205,15 @@ Enabling soft deletes:
Efficient use of soft deletes:
* There are two possible ways to mark a record as deleted:
* There are two possible ways to mark a record as deleted::
model.soft_delete() and query.soft_delete().
model.soft_delete() method works with single already fetched entry.
query.soft_delete() makes only one db request for all entries that correspond
to query.
* In almost all cases you should use query.soft_delete(). Some examples:
* In almost all cases you should use query.soft_delete(). Some examples::
def soft_delete_bar():
count = model_query(BarModel).find(some_condition).soft_delete()
@ -197,9 +224,9 @@ Efficient use of soft deletes:
if session is None:
session = get_session()
with session.begin(subtransactions=True):
count = model_query(BarModel).\
find(some_condition).\
soft_delete(synchronize_session=True)
count = (model_query(BarModel).
find(some_condition).
soft_delete(synchronize_session=True))
# Here synchronize_session is required, because we
# don't know what is going on in outer session.
if count == 0:
@ -209,6 +236,8 @@ Efficient use of soft deletes:
you fetch a single record, work with it, and mark it as deleted in the same
transaction.
::
def soft_delete_bar_model():
session = get_session()
with session.begin():
@ -217,13 +246,13 @@ Efficient use of soft deletes:
bar_ref.soft_delete(session=session)
However, if you need to work with all entries that correspond to query and
then soft delete them you should use query.soft_delete() method:
then soft delete them you should use query.soft_delete() method::
def soft_delete_multi_models():
session = get_session()
with session.begin():
query = model_query(BarModel, session=session).\
find(some_condition)
query = (model_query(BarModel, session=session).
find(some_condition))
model_refs = query.all()
# Work with model_refs
query.soft_delete(synchronize_session=False)
@ -234,6 +263,8 @@ Efficient use of soft deletes:
which issues a single query. Using model.soft_delete(), as in the following
example, is very inefficient.
::
for bar_ref in bar_refs:
bar_ref.soft_delete(session=session)
# This will produce count(bar_refs) db requests.
@ -247,14 +278,13 @@ import time
from oslo.config import cfg
import six
from sqlalchemy import exc as sqla_exc
import sqlalchemy.interfaces
from sqlalchemy.interfaces import PoolListener
import sqlalchemy.orm
from sqlalchemy.pool import NullPool, StaticPool
from sqlalchemy.sql.expression import literal_column
from ceilometer.openstack.common.db import exception
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common import timeutils
@ -274,6 +304,7 @@ database_opts = [
'../', '$sqlite_db')),
help='The SQLAlchemy connection string used to connect to the '
'database',
secret=True,
deprecated_opts=[cfg.DeprecatedOpt('sql_connection',
group='DEFAULT'),
cfg.DeprecatedOpt('sql_connection',
@ -282,6 +313,7 @@ database_opts = [
group='sql'), ]),
cfg.StrOpt('slave_connection',
default='',
secret=True,
help='The SQLAlchemy connection string used to connect to the '
'slave database'),
cfg.IntOpt('idle_timeout',
@ -289,7 +321,9 @@ database_opts = [
deprecated_opts=[cfg.DeprecatedOpt('sql_idle_timeout',
group='DEFAULT'),
cfg.DeprecatedOpt('sql_idle_timeout',
group='DATABASE')],
group='DATABASE'),
cfg.DeprecatedOpt('idle_timeout',
group='sql')],
help='timeout before idle sql connections are reaped'),
cfg.IntOpt('min_pool_size',
default=1,
@ -407,8 +441,8 @@ class SqliteForeignKeysListener(PoolListener):
dbapi_con.execute('pragma foreign_keys=ON')
def get_session(autocommit=True, expire_on_commit=False,
sqlite_fk=False, slave_session=False):
def get_session(autocommit=True, expire_on_commit=False, sqlite_fk=False,
slave_session=False, mysql_traditional_mode=False):
"""Return a SQLAlchemy session."""
global _MAKER
global _SLAVE_MAKER
@ -418,7 +452,8 @@ def get_session(autocommit=True, expire_on_commit=False,
maker = _SLAVE_MAKER
if maker is None:
engine = get_engine(sqlite_fk=sqlite_fk, slave_engine=slave_session)
engine = get_engine(sqlite_fk=sqlite_fk, slave_engine=slave_session,
mysql_traditional_mode=mysql_traditional_mode)
maker = get_maker(engine, autocommit, expire_on_commit)
if slave_session:
@ -437,6 +472,11 @@ def get_session(autocommit=True, expire_on_commit=False,
# 1 column - (IntegrityError) column c1 is not unique
# N columns - (IntegrityError) column c1, c2, ..., N are not unique
#
# sqlite since 3.7.16:
# 1 column - (IntegrityError) UNIQUE constraint failed: k1
#
# N columns - (IntegrityError) UNIQUE constraint failed: k1, k2
#
# postgres:
# 1 column - (IntegrityError) duplicate key value violates unique
# constraint "users_c1_key"
@ -449,9 +489,10 @@ def get_session(autocommit=True, expire_on_commit=False,
# N columns - (IntegrityError) (1062, "Duplicate entry 'values joined
# with -' for key 'name_of_our_constraint'")
_DUP_KEY_RE_DB = {
"sqlite": re.compile(r"^.*columns?([^)]+)(is|are)\s+not\s+unique$"),
"postgresql": re.compile(r"^.*duplicate\s+key.*\"([^\"]+)\"\s*\n.*$"),
"mysql": re.compile(r"^.*\(1062,.*'([^\']+)'\"\)$")
"sqlite": (re.compile(r"^.*columns?([^)]+)(is|are)\s+not\s+unique$"),
re.compile(r"^.*UNIQUE\s+constraint\s+failed:\s+(.+)$")),
"postgresql": (re.compile(r"^.*duplicate\s+key.*\"([^\"]+)\"\s*\n.*$"),),
"mysql": (re.compile(r"^.*\(1062,.*'([^\']+)'\"\)$"),)
}
@ -481,10 +522,14 @@ def _raise_if_duplicate_entry_error(integrity_error, engine_name):
# SQLAlchemy can differ when using unicode() and accessing .message.
# An audit across all three supported engines will be necessary to
# ensure there are no regressions.
m = _DUP_KEY_RE_DB[engine_name].match(integrity_error.message)
if not m:
for pattern in _DUP_KEY_RE_DB[engine_name]:
match = pattern.match(integrity_error.message)
if match:
break
else:
return
columns = m.group(1)
columns = match.group(1)
if engine_name == "sqlite":
columns = columns.strip().split(", ")
@ -553,7 +598,8 @@ def _wrap_db_error(f):
return _wrap
def get_engine(sqlite_fk=False, slave_engine=False):
def get_engine(sqlite_fk=False, slave_engine=False,
mysql_traditional_mode=False):
"""Return a SQLAlchemy engine."""
global _ENGINE
global _SLAVE_ENGINE
@ -565,8 +611,8 @@ def get_engine(sqlite_fk=False, slave_engine=False):
db_uri = CONF.database.slave_connection
if engine is None:
engine = create_engine(db_uri,
sqlite_fk=sqlite_fk)
engine = create_engine(db_uri, sqlite_fk=sqlite_fk,
mysql_traditional_mode=mysql_traditional_mode)
if slave_engine:
_SLAVE_ENGINE = engine
else:
@ -623,6 +669,17 @@ def _ping_listener(engine, dbapi_conn, connection_rec, connection_proxy):
raise
def _set_mode_traditional(dbapi_con, connection_rec, connection_proxy):
"""Set engine mode to 'traditional'.
Required to prevent silent truncates at insert or update operations
under MySQL. By default MySQL truncates inserted string if it longer
than a declared field just with warning. That is fraught with data
corruption.
"""
dbapi_con.cursor().execute("SET SESSION sql_mode = TRADITIONAL;")
def _is_db_connection_error(args):
"""Return True if error in connecting to db."""
# NOTE(adam_g): This is currently MySQL specific and needs to be extended
@ -635,7 +692,8 @@ def _is_db_connection_error(args):
return False
def create_engine(sql_connection, sqlite_fk=False):
def create_engine(sql_connection, sqlite_fk=False,
mysql_traditional_mode=False):
"""Return a new SQLAlchemy engine."""
# NOTE(geekinutah): At this point we could be connecting to the normal
# db handle or the slave db handle. Things like
@ -679,6 +737,13 @@ def create_engine(sql_connection, sqlite_fk=False):
if engine.name in ['mysql', 'ibm_db_sa']:
callback = functools.partial(_ping_listener, engine)
sqlalchemy.event.listen(engine, 'checkout', callback)
if mysql_traditional_mode:
sqlalchemy.event.listen(engine, 'checkout', _set_mode_traditional)
else:
LOG.warning(_("This application has not enabled MySQL traditional"
" mode, which means silent data corruption may"
" occur. Please encourage the application"
" developers to enable this mode."))
elif 'sqlite' in connection_dict.drivername:
if not CONF.sqlite_synchronous:
sqlalchemy.event.listen(engine, 'connect',
@ -700,7 +765,7 @@ def create_engine(sql_connection, sqlite_fk=False):
remaining = 'infinite'
while True:
msg = _('SQL connection failed. %s attempts left.')
LOG.warn(msg % remaining)
LOG.warning(msg % remaining)
if remaining != 'infinite':
remaining -= 1
time.sleep(CONF.database.retry_interval)

17
ceilometer/openstack/common/db/sqlalchemy/test_migrations.py
View File

@ -14,17 +14,17 @@
# License for the specific language governing permissions and limitations
# under the License.
import ConfigParser
import functools
import os
import subprocess
import lockfile
from six import moves
import sqlalchemy
import sqlalchemy.exc
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common import processutils
from ceilometer.openstack.common.py3kcompat import urlutils
from ceilometer.openstack.common import test
@ -130,13 +130,13 @@ class BaseMigrationTestCase(test.BaseTestCase):
# once. No need to re-run this on each test...
LOG.debug('config_path is %s' % self.CONFIG_FILE_PATH)
if os.path.exists(self.CONFIG_FILE_PATH):
cp = ConfigParser.RawConfigParser()
cp = moves.configparser.RawConfigParser()
try:
cp.read(self.CONFIG_FILE_PATH)
defaults = cp.defaults()
for key, value in defaults.items():
self.test_databases[key] = value
except ConfigParser.ParsingError as e:
except moves.configparser.ParsingError as e:
self.fail("Failed to read test_migrations.conf config "
"file. Got error: %s" % e)
else:
@ -158,13 +158,13 @@ class BaseMigrationTestCase(test.BaseTestCase):
super(BaseMigrationTestCase, self).tearDown()
def execute_cmd(self, cmd=None):
out, err = processutils.trycmd(cmd, shell=True, discard_warnings=True)
output = out or err
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = process.communicate()[0]
LOG.debug(output)
self.assertEqual('', err,
self.assertEqual(0, process.returncode,
"Failed to run: %s\n%s" % (cmd, output))
@_set_db_lock('pgadmin', 'tests-')
def _reset_pg(self, conn_pieces):
(user, password, database, host) = get_db_connection_info(conn_pieces)
os.environ['PGPASSWORD'] = password
@ -186,6 +186,7 @@ class BaseMigrationTestCase(test.BaseTestCase):
os.unsetenv('PGPASSWORD')
os.unsetenv('PGUSER')
@_set_db_lock(lock_prefix='migration_tests-')
def _reset_databases(self):
for key, engine in self.engines.items():
conn_string = self.test_databases[key]

8
ceilometer/openstack/common/db/sqlalchemy/utils.py
View File

@ -36,7 +36,7 @@ from sqlalchemy import String
from sqlalchemy import Table
from sqlalchemy.types import NullType
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common import timeutils
@ -94,7 +94,7 @@ def paginate_query(query, model, limit, sort_keys, marker=None,
if 'id' not in sort_keys:
# TODO(justinsb): If this ever gives a false-positive, check
# the actual primary key, rather than assuming its id
LOG.warn(_('Id not in sort_keys; is sort_keys unique?'))
LOG.warning(_('Id not in sort_keys; is sort_keys unique?'))
assert(not (sort_dir and sort_dirs))
@ -133,9 +133,9 @@ def paginate_query(query, model, limit, sort_keys, marker=None,
# Build up an array of sort criteria as in the docstring
criteria_list = []
for i in range(0, len(sort_keys)):
for i in range(len(sort_keys)):
crit_attrs = []
for j in range(0, i):
for j in range(i):
model_attr = getattr(model, sort_keys[j])
crit_attrs.append((model_attr == marker_values[j]))

4
ceilometer/openstack/common/eventlet_backdoor.py
View File

@ -29,7 +29,7 @@ import eventlet.backdoor
import greenlet
from oslo.config import cfg
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging
help_for_backdoor_port = (
@ -64,7 +64,7 @@ def _dont_use_this():
def _find_objects(t):
return filter(lambda o: isinstance(o, t), gc.get_objects())
return [o for o in gc.get_objects() if isinstance(o, t)]
def _print_greenthreads():

14
ceilometer/openstack/common/excutils.py
View File

@ -24,7 +24,7 @@ import traceback
import six
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
class save_and_reraise_exception(object):
@ -42,13 +42,13 @@ class save_and_reraise_exception(object):
In some cases the caller may not want to re-raise the exception, and
for those circumstances this context provides a reraise flag that
can be used to suppress the exception. For example:
can be used to suppress the exception. For example::
except Exception:
with save_and_reraise_exception() as ctxt:
decide_if_need_reraise()
if not should_be_reraised:
ctxt.reraise = False
except Exception:
with save_and_reraise_exception() as ctxt:
decide_if_need_reraise()
if not should_be_reraised:
ctxt.reraise = False
"""
def __init__(self):
self.reraise = True

2
ceilometer/openstack/common/fileutils.py
View File

@ -20,7 +20,7 @@ import os
import tempfile
from ceilometer.openstack.common import excutils
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging
LOG = logging.getLogger(__name__)

4
ceilometer/openstack/common/fixture/lockutils.py
View File

@ -15,7 +15,7 @@
import fixtures
from ceilometer.openstack.common.lockutils import lock
from ceilometer.openstack.common import lockutils
class LockFixture(fixtures.Fixture):
@ -43,7 +43,7 @@ class LockFixture(fixtures.Fixture):
test method exits. (either by completing or raising an exception)
"""
def __init__(self, name, lock_file_prefix=None):
self.mgr = lock(name, lock_file_prefix, True)
self.mgr = lockutils.lock(name, lock_file_prefix, True)
def setUp(self):
super(LockFixture, self).setUp()

418
ceilometer/openstack/common/gettextutils.py
View File

@ -19,18 +19,15 @@ gettext for openstack-common modules.
Usual usage in an openstack.common module:
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
"""
import copy
import gettext
import logging
import locale
from logging import handlers
import os
import re
try:
import UserString as _userString
except ImportError:
import collections as _userString
from babel import localedata
import six
@ -56,7 +53,7 @@ def enable_lazy():
def _(msg):
if USE_LAZY:
return Message(msg, 'ceilometer')
return Message(msg, domain='ceilometer')
else:
if six.PY3:
return _t.gettext(msg)
@ -88,11 +85,6 @@ def install(domain, lazy=False):
# messages in OpenStack. We override the standard _() function
# and % (format string) operation to build Message objects that can
# later be translated when we have more information.
#
# Also included below is an example LocaleHandler that translates
# Messages to an associated locale, effectively allowing many logs,
# each with their own locale.
def _lazy_gettext(msg):
"""Create and return a Message object.
@ -103,7 +95,7 @@ def install(domain, lazy=False):
Message encapsulates a string so that we can translate
it later when needed.
"""
return Message(msg, domain)
return Message(msg, domain=domain)
from six import moves
moves.builtins.__dict__['_'] = _lazy_gettext
@ -118,182 +110,158 @@ def install(domain, lazy=False):
unicode=True)
class Message(_userString.UserString, object):
"""Class used to encapsulate translatable messages."""
def __init__(self, msg, domain):
# _msg is the gettext msgid and should never change
self._msg = msg
self._left_extra_msg = ''
self._right_extra_msg = ''
self._locale = None
self.params = None
self.domain = domain
class Message(six.text_type):
"""A Message object is a unicode object that can be translated.
@property
def data(self):
# NOTE(mrodden): this should always resolve to a unicode string
# that best represents the state of the message currently
Translation of Message is done explicitly using the translate() method.
For all non-translation intents and purposes, a Message is simply unicode,
and can be treated as such.
"""
localedir = os.environ.get(self.domain.upper() + '_LOCALEDIR')
if self.locale:
lang = gettext.translation(self.domain,
localedir=localedir,
languages=[self.locale],
fallback=True)
else:
# use system locale for translations
lang = gettext.translation(self.domain,
localedir=localedir,
fallback=True)
def __new__(cls, msgid, msgtext=None, params=None, domain='ceilometer', *args):
"""Create a new Message object.
In order for translation to work gettext requires a message ID, this
msgid will be used as the base unicode text. It is also possible
for the msgid and the base unicode text to be different by passing
the msgtext parameter.
"""
# If the base msgtext is not given, we use the default translation
# of the msgid (which is in English) just in case the system locale is
# not English, so that the base text will be in that locale by default.
if not msgtext:
msgtext = Message._translate_msgid(msgid, domain)
# We want to initialize the parent unicode with the actual object that
# would have been plain unicode if 'Message' was not enabled.
msg = super(Message, cls).__new__(cls, msgtext)
msg.msgid = msgid
msg.domain = domain
msg.params = params
return msg
def translate(self, desired_locale=None):
"""Translate this message to the desired locale.
:param desired_locale: The desired locale to translate the message to,
if no locale is provided the message will be
translated to the system's default locale.
:returns: the translated message in unicode
"""
translated_message = Message._translate_msgid(self.msgid,
self.domain,
desired_locale)
if self.params is None:
# No need for more translation
return translated_message
# This Message object may have been formatted with one or more
# Message objects as substitution arguments, given either as a single
# argument, part of a tuple, or as one or more values in a dictionary.
# When translating this Message we need to translate those Messages too
translated_params = _translate_args(self.params, desired_locale)
translated_message = translated_message % translated_params
return translated_message
@staticmethod
def _translate_msgid(msgid, domain, desired_locale=None):
if not desired_locale:
system_locale = locale.getdefaultlocale()
# If the system locale is not available to the runtime use English
if not system_locale[0]:
desired_locale = 'en_US'
else:
desired_locale = system_locale[0]
locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR')
lang = gettext.translation(domain,
localedir=locale_dir,
languages=[desired_locale],
fallback=True)
if six.PY3:
ugettext = lang.gettext
translator = lang.gettext
else:
ugettext = lang.ugettext
translator = lang.ugettext
full_msg = (self._left_extra_msg +
ugettext(self._msg) +
self._right_extra_msg)
translated_message = translator(msgid)
return translated_message
if self.params is not None:
full_msg = full_msg % self.params
def __mod__(self, other):
# When we mod a Message we want the actual operation to be performed
# by the parent class (i.e. unicode()), the only thing we do here is
# save the original msgid and the parameters in case of a translation
unicode_mod = super(Message, self).__mod__(other)
modded = Message(self.msgid,
msgtext=unicode_mod,
params=self._sanitize_mod_params(other),
domain=self.domain)
return modded
return six.text_type(full_msg)
def _sanitize_mod_params(self, other):
"""Sanitize the object being modded with this Message.
@property
def locale(self):
return self._locale
- Add support for modding 'None' so translation supports it
- Trim the modded object, which can be a large dictionary, to only
those keys that would actually be used in a translation
- Snapshot the object being modded, in case the message is
translated, it will be used as it was when the Message was created
"""
if other is None:
params = (other,)
elif isinstance(other, dict):
params = self._trim_dictionary_parameters(other)
else:
params = self._copy_param(other)
return params
@locale.setter
def locale(self, value):
self._locale = value
if not self.params:
return
def _trim_dictionary_parameters(self, dict_param):
"""Return a dict that only has matching entries in the msgid."""
# NOTE(luisg): Here we trim down the dictionary passed as parameters
# to avoid carrying a lot of unnecessary weight around in the message
# object, for example if someone passes in Message() % locals() but
# only some params are used, and additionally we prevent errors for
# non-deepcopyable objects by unicoding() them.
# This Message object may have been constructed with one or more
# Message objects as substitution parameters, given as a single
# Message, or a tuple or Map containing some, so when setting the
# locale for this Message we need to set it for those Messages too.
if isinstance(self.params, Message):
self.params.locale = value
return
if isinstance(self.params, tuple):
for param in self.params:
if isinstance(param, Message):
param.locale = value
return
if isinstance(self.params, dict):
for param in self.params.values():
if isinstance(param, Message):
param.locale = value
# Look for %(param) keys in msgid;
# Skip %% and deal with the case where % is first character on the line
keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', self.msgid)
def _save_dictionary_parameter(self, dict_param):
full_msg = self.data
# look for %(blah) fields in string;
# ignore %% and deal with the
# case where % is first character on the line
keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', full_msg)
# if we don't find any %(blah) blocks but have a %s
if not keys and re.findall('(?:[^%]|^)%[a-z]', full_msg):
# apparently the full dictionary is the parameter
params = copy.deepcopy(dict_param)
# If we don't find any %(param) keys but have a %s
if not keys and re.findall('(?:[^%]|^)%[a-z]', self.msgid):
# Apparently the full dictionary is the parameter
params = self._copy_param(dict_param)
else:
params = {}
for key in keys:
try:
params[key] = copy.deepcopy(dict_param[key])
except TypeError:
# cast uncopyable thing to unicode string
params[key] = six.text_type(dict_param[key])
params[key] = self._copy_param(dict_param[key])
return params
def _save_parameters(self, other):
# we check for None later to see if
# we actually have parameters to inject,
# so encapsulate if our parameter is actually None
if other is None:
self.params = (other, )
elif isinstance(other, dict):
self.params = self._save_dictionary_parameter(other)
else:
# fallback to casting to unicode,
# this will handle the problematic python code-like
# objects that cannot be deep-copied
try:
self.params = copy.deepcopy(other)
except TypeError:
self.params = six.text_type(other)
def _copy_param(self, param):
try:
return copy.deepcopy(param)
except TypeError:
# Fallback to casting to unicode this will handle the
# python code-like objects that can't be deep-copied
return six.text_type(param)
return self
# overrides to be more string-like
def __unicode__(self):
return self.data
def __str__(self):
if six.PY3:
return self.__unicode__()
return self.data.encode('utf-8')
def __getstate__(self):
to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg',
'domain', 'params', '_locale']
new_dict = self.__dict__.fromkeys(to_copy)
for attr in to_copy:
new_dict[attr] = copy.deepcopy(self.__dict__[attr])
return new_dict
def __setstate__(self, state):
for (k, v) in state.items():
setattr(self, k, v)
# operator overloads
def __add__(self, other):
copied = copy.deepcopy(self)
copied._right_extra_msg += other.__str__()
return copied
msg = _('Message objects do not support addition.')
raise TypeError(msg)
def __radd__(self, other):
copied = copy.deepcopy(self)
copied._left_extra_msg += other.__str__()
return copied
return self.__add__(other)
def __mod__(self, other):
# do a format string to catch and raise
# any possible KeyErrors from missing parameters
self.data % other
copied = copy.deepcopy(self)
return copied._save_parameters(other)
def __mul__(self, other):
return self.data * other
def __rmul__(self, other):
return other * self.data
def __getitem__(self, key):
return self.data[key]
def __getslice__(self, start, end):
return self.data.__getslice__(start, end)
def __getattribute__(self, name):
# NOTE(mrodden): handle lossy operations that we can't deal with yet
# These override the UserString implementation, since UserString
# uses our __class__ attribute to try and build a new message
# after running the inner data string through the operation.
# At that point, we have lost the gettext message id and can just
# safely resolve to a string instead.
ops = ['capitalize', 'center', 'decode', 'encode',
'expandtabs', 'ljust', 'lstrip', 'replace', 'rjust', 'rstrip',
'strip', 'swapcase', 'title', 'translate', 'upper', 'zfill']
if name in ops:
return getattr(self.data, name)
else:
return _userString.UserString.__getattribute__(self, name)
def __str__(self):
# NOTE(luisg): Logging in python 2.6 tries to str() log records,
# and it expects specifically a UnicodeError in order to proceed.
msg = _('Message objects do not support str() because they may '
'contain non-ascii characters. '
'Please use unicode() or translate() instead.')
raise UnicodeError(msg)
def get_available_languages(domain):
@ -326,46 +294,118 @@ def get_available_languages(domain):
return copy.copy(language_list)
def get_localized_message(message, user_locale):
"""Gets a localized version of the given message in the given locale.
def translate(obj, desired_locale=None):
"""Gets the translated unicode representation of the given object.
If the message is not a Message object the message is returned as-is.
If the locale is None the message is translated to the default locale.
If the object is not translatable it is returned as-is.
If the locale is None the object is translated to the system locale.
:returns: the translated message in unicode, or the original message if
:param obj: the object to translate
:param desired_locale: the locale to translate the message to, if None the
default system locale will be used
:returns: the translated object in unicode, or the original object if
it could not be translated
"""
translated = message
message = obj
if not isinstance(message, Message):
# If the object to translate is not already translatable,
# let's first get its unicode representation
message = six.text_type(obj)
if isinstance(message, Message):
original_locale = message.locale
message.locale = user_locale
translated = six.text_type(message)
message.locale = original_locale
return translated
# Even after unicoding() we still need to check if we are
# running with translatable unicode before translating
return message.translate(desired_locale)
return obj
class LocaleHandler(logging.Handler):
"""Handler that can have a locale associated to translate Messages.
def _translate_args(args, desired_locale=None):
"""Translates all the translatable elements of the given arguments object.
A quick example of how to utilize the Message class above.
LocaleHandler takes a locale and a target logging.Handler object
to forward LogRecord objects to after translating the internal Message.
This method is used for translating the translatable values in method
arguments which include values of tuples or dictionaries.
If the object is not a tuple or a dictionary the object itself is
translated if it is translatable.
If the locale is None the object is translated to the system locale.
:param args: the args to translate
:param desired_locale: the locale to translate the args to, if None the
default system locale will be used
:returns: a new args object with the translated contents of the original
"""
if isinstance(args, tuple):
return tuple(translate(v, desired_locale) for v in args)
if isinstance(args, dict):
translated_dict = {}
for (k, v) in six.iteritems(args):
translated_v = translate(v, desired_locale)
translated_dict[k] = translated_v
return translated_dict
return translate(args, desired_locale)
class TranslationHandler(handlers.MemoryHandler):
"""Handler that translates records before logging them.
The TranslationHandler takes a locale and a target logging.Handler object
to forward LogRecord objects to after translating them. This handler
depends on Message objects being logged, instead of regular strings.
The handler can be configured declaratively in the logging.conf as follows:
[handlers]
keys = translatedlog, translator
[handler_translatedlog]
class = handlers.WatchedFileHandler
args = ('/var/log/api-localized.log',)
formatter = context
[handler_translator]
class = openstack.common.log.TranslationHandler
target = translatedlog
args = ('zh_CN',)
If the specified locale is not available in the system, the handler will
log in the default locale.
"""
def __init__(self, locale, target):
"""Initialize a LocaleHandler
def __init__(self, locale=None, target=None):
"""Initialize a TranslationHandler
:param locale: locale to use for translating messages
:param target: logging.Handler object to forward
LogRecord objects to after translation
"""
logging.Handler.__init__(self)
# NOTE(luisg): In order to allow this handler to be a wrapper for
# other handlers, such as a FileHandler, and still be able to
# configure it using logging.conf, this handler has to extend
# MemoryHandler because only the MemoryHandlers' logging.conf
# parsing is implemented such that it accepts a target handler.
handlers.MemoryHandler.__init__(self, capacity=0, target=target)
self.locale = locale
self.target = target
def setFormatter(self, fmt):
self.target.setFormatter(fmt)
def emit(self, record):
if isinstance(record.msg, Message):
# set the locale and resolve to a string
record.msg.locale = self.locale
# We save the message from the original record to restore it
# after translation, so other handlers are not affected by this
original_msg = record.msg
original_args = record.args
try:
self._translate_and_log_record(record)
finally:
record.msg = original_msg
record.args = original_args
def _translate_and_log_record(self, record):
record.msg = translate(record.msg, self.locale)
# In addition to translating the message, we also need to translate
# arguments that were passed to the log method that were not part
# of the main message e.g., log.info(_('Some message %s'), this_one))
record.args = _translate_args(record.args, self.locale)
self.target.emit(record)

12
ceilometer/openstack/common/jsonutils.py
View File

@ -39,8 +39,12 @@ import json
try:
import xmlrpclib
except ImportError:
# NOTE(jd): xmlrpclib is not shipped with Python 3
xmlrpclib = None
# NOTE(jaypipes): xmlrpclib was renamed to xmlrpc.client in Python3
# however the function and object call signatures
# remained the same. This whole try/except block should
# be removed and replaced with a call to six.moves once
# six 1.4.2 is released. See http://bit.ly/1bqrVzu
import xmlrpc.client as xmlrpclib
import six
@ -122,14 +126,14 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
level=level,
max_depth=max_depth)
if isinstance(value, dict):
return dict((k, recursive(v)) for k, v in value.iteritems())
return dict((k, recursive(v)) for k, v in six.iteritems(value))
elif isinstance(value, (list, tuple)):
return [recursive(lv) for lv in value]
# It's not clear why xmlrpclib created their own DateTime type, but
# for our purposes, make it a datetime type which is explicitly
# handled
if xmlrpclib and isinstance(value, xmlrpclib.DateTime):
if isinstance(value, xmlrpclib.DateTime):
value = datetime.datetime(*tuple(value.timetuple())[:6])
if convert_datetime and isinstance(value, datetime.datetime):

120
ceilometer/openstack/common/lockutils.py
View File

@ -29,7 +29,7 @@ import weakref
from oslo.config import cfg
from ceilometer.openstack.common import fileutils
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import local
from ceilometer.openstack.common import log as logging
@ -138,25 +138,46 @@ _semaphores_lock = threading.Lock()
@contextlib.contextmanager
def lock(name, lock_file_prefix=None, external=False, lock_path=None):
"""Context based lock
def external_lock(name, lock_file_prefix=None, lock_path=None):
with internal_lock(name):
LOG.debug(_('Attempting to grab file lock "%(lock)s"'),
{'lock': name})
This function yields a `threading.Semaphore` instance (if we don't use
eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is
True, in which case, it'll yield an InterProcessLock instance.
# We need a copy of lock_path because it is non-local
local_lock_path = lock_path or CONF.lock_path
if not local_lock_path:
raise cfg.RequiredOptError('lock_path')
:param lock_file_prefix: The lock_file_prefix argument is used to provide
lock files on disk with a meaningful prefix.
if not os.path.exists(local_lock_path):
fileutils.ensure_tree(local_lock_path)
LOG.info(_('Created lock path: %s'), local_lock_path)
:param external: The external keyword argument denotes whether this lock
should work across multiple processes. This means that if two different
workers both run a a method decorated with @synchronized('mylock',
external=True), only one of them will execute at a time.
def add_prefix(name, prefix):
if not prefix:
return name
sep = '' if prefix.endswith('-') else '-'
return '%s%s%s' % (prefix, sep, name)
:param lock_path: The lock_path keyword argument is used to specify a
special location for external lock files to live. If nothing is set, then
CONF.lock_path is used as a default.
"""
# NOTE(mikal): the lock name cannot contain directory
# separators
lock_file_name = add_prefix(name.replace(os.sep, '_'),
lock_file_prefix)
lock_file_path = os.path.join(local_lock_path, lock_file_name)
try:
lock = InterProcessLock(lock_file_path)
with lock as lock:
LOG.debug(_('Got file lock "%(lock)s" at %(path)s'),
{'lock': name, 'path': lock_file_path})
yield lock
finally:
LOG.debug(_('Released file lock "%(lock)s" at %(path)s'),
{'lock': name, 'path': lock_file_path})
@contextlib.contextmanager
def internal_lock(name):
with _semaphores_lock:
try:
sem = _semaphores[name]
@ -173,48 +194,39 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None):
local.strong_store.locks_held.append(name)
try:
if external and not CONF.disable_process_locking:
LOG.debug(_('Attempting to grab file lock "%(lock)s"'),
{'lock': name})
# We need a copy of lock_path because it is non-local
local_lock_path = lock_path or CONF.lock_path
if not local_lock_path:
raise cfg.RequiredOptError('lock_path')
if not os.path.exists(local_lock_path):
fileutils.ensure_tree(local_lock_path)
LOG.info(_('Created lock path: %s'), local_lock_path)
def add_prefix(name, prefix):
if not prefix:
return name
sep = '' if prefix.endswith('-') else '-'
return '%s%s%s' % (prefix, sep, name)
# NOTE(mikal): the lock name cannot contain directory
# separators
lock_file_name = add_prefix(name.replace(os.sep, '_'),
lock_file_prefix)
lock_file_path = os.path.join(local_lock_path, lock_file_name)
try:
lock = InterProcessLock(lock_file_path)
with lock as lock:
LOG.debug(_('Got file lock "%(lock)s" at %(path)s'),
{'lock': name, 'path': lock_file_path})
yield lock
finally:
LOG.debug(_('Released file lock "%(lock)s" at %(path)s'),
{'lock': name, 'path': lock_file_path})
else:
yield sem
yield sem
finally:
local.strong_store.locks_held.remove(name)
@contextlib.contextmanager
def lock(name, lock_file_prefix=None, external=False, lock_path=None):
"""Context based lock
This function yields a `threading.Semaphore` instance (if we don't use
eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is
True, in which case, it'll yield an InterProcessLock instance.
:param lock_file_prefix: The lock_file_prefix argument is used to provide
lock files on disk with a meaningful prefix.
:param external: The external keyword argument denotes whether this lock
should work across multiple processes. This means that if two different
workers both run a a method decorated with @synchronized('mylock',
external=True), only one of them will execute at a time.
:param lock_path: The lock_path keyword argument is used to specify a
special location for external lock files to live. If nothing is set, then
CONF.lock_path is used as a default.
"""
if external and not CONF.disable_process_locking:
with external_lock(name, lock_file_prefix, lock_path) as lock:
yield lock
else:
with internal_lock(name) as lock:
yield lock
def synchronized(name, lock_file_prefix=None, external=False, lock_path=None):
"""Synchronization decorator.

18
ceilometer/openstack/common/log.py
View File

@ -41,7 +41,7 @@ from oslo.config import cfg
import six
from six import moves
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import importutils
from ceilometer.openstack.common import jsonutils
from ceilometer.openstack.common import local
@ -130,7 +130,7 @@ generic_log_opts = [
log_opts = [
cfg.StrOpt('logging_context_format_string',
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
'%(name)s [%(request_id)s %(user)s %(tenant)s] '
'%(name)s [%(request_id)s %(user_identity)s] '
'%(instance)s%(message)s',
help='format string to use for log messages with context'),
cfg.StrOpt('logging_default_format_string',
@ -149,7 +149,6 @@ log_opts = [
'amqp=WARN',
'amqplib=WARN',
'boto=WARN',
'keystone=INFO',
'qpid=WARN',
'sqlalchemy=WARN',
'suds=INFO',
@ -236,10 +235,11 @@ def mask_password(message, secret="***"):
"""Replace password with 'secret' in message.
:param message: The string which includes security information.
:param secret: value with which to replace passwords, defaults to "***".
:param secret: value with which to replace passwords.
:returns: The unicode value of message with the password fields masked.
For example:
>>> mask_password("'adminPass' : 'aaaaa'")
"'adminPass' : '***'"
>>> mask_password("'admin_pass' : 'aaaaa'")
@ -332,10 +332,12 @@ class ContextAdapter(BaseLoggerAdapter):
elif instance_uuid:
instance_extra = (CONF.instance_uuid_format
% {'uuid': instance_uuid})
extra.update({'instance': instance_extra})
extra['instance'] = instance_extra
extra.update({"project": self.project})
extra.update({"version": self.version})
extra.setdefault('user_identity', kwargs.pop('user_identity', None))
extra['project'] = self.project
extra['version'] = self.version
extra['extra'] = extra.copy()
return msg, kwargs
@ -389,7 +391,7 @@ class JSONFormatter(logging.Formatter):
def _create_logging_excepthook(product_name):
def logging_excepthook(exc_type, value, tb):
extra = {}
if CONF.verbose:
if CONF.verbose or CONF.debug:
extra['exc_info'] = (exc_type, value, tb)
getLogger(product_name).critical(str(value), **extra)
return logging_excepthook

4
ceilometer/openstack/common/log_handler.py
View File

@ -13,10 +13,10 @@
# under the License.
import logging
from ceilometer.openstack.common import notifier
from oslo.config import cfg
from ceilometer.openstack.common import notifier
class PublishErrorsHandler(logging.Handler):
def emit(self, record):

2
ceilometer/openstack/common/loopingcall.py
View File

@ -20,7 +20,7 @@ import sys
from eventlet import event
from eventlet import greenthread
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common import timeutils

43
ceilometer/openstack/common/middleware/catch_errors.py Normal file
View File

@ -0,0 +1,43 @@
# Copyright (c) 2013 NEC Corporation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Middleware that provides high-level error handling.
It catches all exceptions from subsequent applications in WSGI pipeline
to hide internal errors from API response.
"""
import webob.dec
import webob.exc
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common.middleware import base
LOG = logging.getLogger(__name__)
class CatchErrorsMiddleware(base.Middleware):
@webob.dec.wsgify
def __call__(self, req):
try:
response = req.get_response(self.application)
except Exception:
LOG.exception(_('An error occurred during '
'processing the request: %s'))
response = webob.exc.HTTPInternalServerError()
return response

3
ceilometer/openstack/common/middleware/debug.py
View File

@ -18,6 +18,7 @@ from __future__ import print_function
import sys
import six
import webob.dec
from ceilometer.openstack.common.middleware import base
@ -39,7 +40,7 @@ class Debug(base.Middleware):
resp = req.get_response(self.application)
print(("*" * 40) + " RESPONSE HEADERS")
for (key, value) in resp.headers.iteritems():
for (key, value) in six.iteritems(resp.headers):
print(key, "=", value)
print()

5
ceilometer/openstack/common/middleware/notifier.py
View File

@ -19,10 +19,11 @@ import os.path
import sys
import traceback as tb
import six
import webob.dec
from ceilometer.openstack.common import context
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common.middleware import base
from ceilometer.openstack.common.notifier import api
@ -65,7 +66,7 @@ class RequestNotifier(base.Middleware):
include them.
"""
return dict((k, v) for k, v in environ.iteritems()
return dict((k, v) for k, v in six.iteritems(environ)
if k.isupper())
@log_and_ignore_error

38
ceilometer/openstack/common/middleware/request_id.py Normal file
View File

@ -0,0 +1,38 @@
# Copyright (c) 2013 NEC Corporation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Middleware that ensures request ID.
It ensures to assign request ID for each API request and set it to
request environment. The request ID is also added to API response.
"""
from ceilometer.openstack.common import context
from ceilometer.openstack.common.middleware import base
ENV_REQUEST_ID = 'openstack.request_id'
HTTP_RESP_HEADER_REQUEST_ID = 'x-openstack-request-id'
class RequestIdMiddleware(base.Middleware):
def process_request(self, req):
self.req_id = context.generate_request_id()
req.environ[ENV_REQUEST_ID] = self.req_id
def process_response(self, response):
response.headers.add(HTTP_RESP_HEADER_REQUEST_ID, self.req_id)
return response

5
ceilometer/openstack/common/middleware/sizelimit.py
View File

@ -20,8 +20,7 @@ from oslo.config import cfg
import webob.dec
import webob.exc
from ceilometer.openstack.common.deprecated import wsgi
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common.middleware import base
@ -69,7 +68,7 @@ class LimitingReader(object):
class RequestBodySizeLimiter(base.Middleware):
"""Limit the size of incoming requests."""
@webob.dec.wsgify(RequestClass=wsgi.Request)
@webob.dec.wsgify
def __call__(self, req):
if req.content_length > CONF.max_request_body_size:
msg = _("Request is too large.")

14
ceilometer/openstack/common/notifier/__init__.py
View File

@ -1,14 +0,0 @@
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

2
ceilometer/openstack/common/notifier/api.py
View File

@ -19,7 +19,7 @@ import uuid
from oslo.config import cfg
from ceilometer.openstack.common import context
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import importutils
from ceilometer.openstack.common import jsonutils
from ceilometer.openstack.common import log as logging

77
ceilometer/openstack/common/notifier/proxy.py Normal file
View File

@ -0,0 +1,77 @@
# Copyright 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
A temporary helper which emulates ceilometer.messaging.Notifier.
This helper method allows us to do the tedious porting to the new Notifier API
as a standalone commit so that the commit which switches us to ceilometer.messaging
is smaller and easier to review. This file will be removed as part of that
commit.
"""
from oslo.config import cfg
from ceilometer.openstack.common.notifier import api as notifier_api
CONF = cfg.CONF
class Notifier(object):
def __init__(self, publisher_id):
super(Notifier, self).__init__()
self.publisher_id = publisher_id
_marker = object()
def prepare(self, publisher_id=_marker):
ret = self.__class__(self.publisher_id)
if publisher_id is not self._marker:
ret.publisher_id = publisher_id
return ret
def _notify(self, ctxt, event_type, payload, priority):
notifier_api.notify(ctxt,
self.publisher_id,
event_type,
priority,
payload)
def audit(self, ctxt, event_type, payload):
# No audit in old notifier.
self._notify(ctxt, event_type, payload, 'INFO')
def debug(self, ctxt, event_type, payload):
self._notify(ctxt, event_type, payload, 'DEBUG')
def info(self, ctxt, event_type, payload):
self._notify(ctxt, event_type, payload, 'INFO')
def warn(self, ctxt, event_type, payload):
self._notify(ctxt, event_type, payload, 'WARN')
warning = warn
def error(self, ctxt, event_type, payload):
self._notify(ctxt, event_type, payload, 'ERROR')
def critical(self, ctxt, event_type, payload):
self._notify(ctxt, event_type, payload, 'CRITICAL')
def get_notifier(service=None, host=None, publisher_id=None):
if not publisher_id:
publisher_id = "%s.%s" % (service, host or CONF.host)
return Notifier(publisher_id)

2
ceilometer/openstack/common/notifier/rpc_notifier.py
View File

@ -16,7 +16,7 @@
from oslo.config import cfg
from ceilometer.openstack.common import context as req_context
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common import rpc

2
ceilometer/openstack/common/notifier/rpc_notifier2.py
View File

@ -18,7 +18,7 @@
from oslo.config import cfg
from ceilometer.openstack.common import context as req_context
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common import rpc

10
ceilometer/openstack/common/policy.py
View File

@ -56,16 +56,16 @@ as it allows particular rules to be explicitly disabled.
import abc
import re
import urllib
import urllib2
from oslo.config import cfg
import six
from ceilometer.openstack.common import fileutils
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import jsonutils
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common.py3kcompat import urlutils
policy_opts = [
cfg.StrOpt('policy_file',
@ -824,8 +824,8 @@ class HttpCheck(Check):
url = ('http:' + self.match) % target
data = {'target': jsonutils.dumps(target),
'credentials': jsonutils.dumps(creds)}
post_data = urllib.urlencode(data)
f = urllib2.urlopen(url, post_data)
post_data = urlutils.urlencode(data)
f = urlutils.urlopen(url, post_data)
return f.read() == "True"

16
ceilometer/openstack/common/py3kcompat/__init__.py
View File

@ -1,16 +0,0 @@
#
# Copyright 2013 Canonical Ltd.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#

4
ceilometer/openstack/common/py3kcompat/urlutils.py
View File

@ -30,8 +30,10 @@ if six.PY3:
urlencode = urllib.parse.urlencode
urljoin = urllib.parse.urljoin
quote = urllib.parse.quote
quote_plus = urllib.parse.quote_plus
parse_qsl = urllib.parse.parse_qsl
unquote = urllib.parse.unquote
unquote_plus = urllib.parse.unquote_plus
urlparse = urllib.parse.urlparse
urlsplit = urllib.parse.urlsplit
urlunsplit = urllib.parse.urlunsplit
@ -48,7 +50,9 @@ else:
urlencode = urllib.urlencode
quote = urllib.quote
quote_plus = urllib.quote_plus
unquote = urllib.unquote
unquote_plus = urllib.unquote_plus
parse = urlparse
parse_qsl = parse.parse_qsl

2
ceilometer/openstack/common/rpc/__init__.py
View File

@ -27,7 +27,7 @@ import inspect
from oslo.config import cfg
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import importutils
from ceilometer.openstack.common import local
from ceilometer.openstack.common import log as logging

11
ceilometer/openstack/common/rpc/amqp.py
View File

@ -33,9 +33,11 @@ from eventlet import pools
from eventlet import queue
from eventlet import semaphore
from oslo.config import cfg
import six
from ceilometer.openstack.common import excutils
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import local
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common.rpc import common as rpc_common
@ -300,10 +302,11 @@ def pack_context(msg, context):
"""
if isinstance(context, dict):
context_d = dict([('_context_%s' % key, value)
for (key, value) in context.iteritems()])
for (key, value) in six.iteritems(context)])
else:
context_d = dict([('_context_%s' % key, value)
for (key, value) in context.to_dict().iteritems()])
for (key, value) in
six.iteritems(context.to_dict())])
msg.update(context_d)
@ -398,7 +401,7 @@ class CallbackWrapper(_ThreadPoolWithWait):
if self.wait_for_consumers:
self.pool.waitall()
if self.exc_info:
raise self.exc_info[1], None, self.exc_info[2]
six.reraise(self.exc_info[1], None, self.exc_info[2])
class ProxyCallback(_ThreadPoolWithWait):

12
ceilometer/openstack/common/rpc/common.py
View File

@ -22,7 +22,7 @@ import traceback
from oslo.config import cfg
import six
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import importutils
from ceilometer.openstack.common import jsonutils
from ceilometer.openstack.common import local
@ -34,6 +34,7 @@ CONF = cfg.CONF
LOG = logging.getLogger(__name__)
_RPC_ENVELOPE_VERSION = '2.0'
'''RPC Envelope Version.
This version number applies to the top level structure of messages sent out.
@ -46,7 +47,7 @@ This version number applies to the message envelope that is used in the
serialization done inside the rpc layer. See serialize_msg() and
deserialize_msg().
The current message format (version 2.0) is very simple. It is:
The current message format (version 2.0) is very simple. It is::
{
'oslo.version': <RPC Envelope Version as a String>,
@ -64,7 +65,6 @@ We will JSON encode the application message payload. The message envelope,
which includes the JSON encoded application message body, will be passed down
to the messaging libraries as a dict.
'''
_RPC_ENVELOPE_VERSION = '2.0'
_VERSION_KEY = 'oslo.version'
_MESSAGE_KEY = 'oslo.message'
@ -86,7 +86,7 @@ class RPCException(Exception):
# kwargs doesn't match a variable in the message
# log the issue and the kwargs
LOG.exception(_('Exception in string format operation'))
for name, value in kwargs.iteritems():
for name, value in six.iteritems(kwargs):
LOG.error("%s: %s" % (name, value))
# at least get the core message out if something happened
message = self.msg_fmt
@ -269,6 +269,10 @@ def _safe_log(log_func, msg, msg_data):
d[k] = '<SANITIZED>'
elif k.lower() in SANITIZE:
d[k] = '<SANITIZED>'
elif isinstance(d[k], list):
for e in d[k]:
if isinstance(e, dict):
_fix_passwords(e)
elif isinstance(d[k], dict):
_fix_passwords(d[k])
return d

4
ceilometer/openstack/common/rpc/dispatcher.py
View File

@ -81,6 +81,8 @@ On the client side, the same changes should be made as in example 1. The
minimum version that supports the new parameter should be specified.
"""
import six
from ceilometer.openstack.common.rpc import common as rpc_common
from ceilometer.openstack.common.rpc import serializer as rpc_serializer
@ -119,7 +121,7 @@ class RpcDispatcher(object):
:returns: A new set of deserialized args
"""
new_kwargs = dict()
for argname, arg in kwargs.iteritems():
for argname, arg in six.iteritems(kwargs):
new_kwargs[argname] = self.serializer.deserialize_entity(context,
arg)
return new_kwargs

3
ceilometer/openstack/common/rpc/impl_fake.py
View File

@ -24,6 +24,7 @@ import json
import time
import eventlet
import six
from ceilometer.openstack.common.rpc import common as rpc_common
@ -67,7 +68,7 @@ class Consumer(object):
# Caller might have called ctxt.reply() manually
for (reply, failure) in ctxt._response:
if failure:
raise failure[0], failure[1], failure[2]
six.reraise(failure[0], failure[1], failure[2])
res.append(reply)
# if ending not 'sent'...we might have more data to
# return from the function itself

4
ceilometer/openstack/common/rpc/impl_kombu.py
View File

@ -29,7 +29,7 @@ from oslo.config import cfg
import six
from ceilometer.openstack.common import excutils
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import network_utils
from ceilometer.openstack.common.rpc import amqp as rpc_amqp
from ceilometer.openstack.common.rpc import common as rpc_common
@ -445,7 +445,7 @@ class Connection(object):
'virtual_host': self.conf.rabbit_virtual_host,
}
for sp_key, value in server_params.iteritems():
for sp_key, value in six.iteritems(server_params):
p_key = server_params_to_kombu_params.get(sp_key, sp_key)
params[p_key] = value

2
ceilometer/openstack/common/rpc/impl_qpid.py
View File

@ -23,7 +23,7 @@ from oslo.config import cfg
import six
from ceilometer.openstack.common import excutils
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import importutils
from ceilometer.openstack.common import jsonutils
from ceilometer.openstack.common import log as logging

2
ceilometer/openstack/common/rpc/impl_zmq.py
View File

@ -27,7 +27,7 @@ import six
from six import moves
from ceilometer.openstack.common import excutils
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import importutils
from ceilometer.openstack.common import jsonutils
from ceilometer.openstack.common.rpc import common as rpc_common

2
ceilometer/openstack/common/rpc/matchmaker.py
View File

@ -21,7 +21,7 @@ import contextlib
import eventlet
from oslo.config import cfg
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging

2
ceilometer/openstack/common/rpc/matchmaker_redis.py
View File

@ -93,7 +93,7 @@ class MatchMakerRedis(mm_common.HeartbeatMatchMakerBase):
if not redis:
raise ImportError("Failed to import module redis.")
self.redis = redis.StrictRedis(
self.redis = redis.Redis(
host=CONF.matchmaker_redis.host,
port=CONF.matchmaker_redis.port,
password=CONF.matchmaker_redis.password)

2
ceilometer/openstack/common/rpc/matchmaker_ring.py
View File

@ -21,7 +21,7 @@ import json
from oslo.config import cfg
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common.rpc import matchmaker as mm

4
ceilometer/openstack/common/rpc/proxy.py
View File

@ -19,6 +19,8 @@ For more information about rpc API version numbers, see:
rpc/dispatcher.py
"""
import six
from ceilometer.openstack.common import rpc
from ceilometer.openstack.common.rpc import common as rpc_common
from ceilometer.openstack.common.rpc import serializer as rpc_serializer
@ -97,7 +99,7 @@ class RpcProxy(object):
:returns: A new set of serialized arguments
"""
new_kwargs = dict()
for argname, arg in kwargs.iteritems():
for argname, arg in six.iteritems(kwargs):
new_kwargs[argname] = self.serializer.serialize_entity(context,
arg)
return new_kwargs

2
ceilometer/openstack/common/rpc/service.py
View File

@ -15,7 +15,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common import rpc
from ceilometer.openstack.common.rpc import dispatcher as rpc_dispatcher

71
ceilometer/openstack/common/service.py
View File

@ -23,14 +23,22 @@ import os
import random
import signal
import sys
import threading
import time
try:
# Importing just the symbol here because the io module does not
# exist in Python 2.6.
from io import UnsupportedOperation # noqa
except ImportError:
# Python 2.6
UnsupportedOperation = None
import eventlet
from eventlet import event
from oslo.config import cfg
from ceilometer.openstack.common import eventlet_backdoor
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import importutils
from ceilometer.openstack.common import log as logging
from ceilometer.openstack.common import threadgroup
@ -45,8 +53,32 @@ def _sighup_supported():
return hasattr(signal, 'SIGHUP')
def _is_sighup(signo):
return _sighup_supported() and signo == signal.SIGHUP
def _is_daemon():
# The process group for a foreground process will match the
# process group of the controlling terminal. If those values do
# not match, or ioctl() fails on the stdout file handle, we assume
# the process is running in the background as a daemon.
# http://www.gnu.org/software/bash/manual/bashref.html#Job-Control-Basics
try:
is_daemon = os.getpgrp() != os.tcgetpgrp(sys.stdout.fileno())
except OSError as err:
if err.errno == errno.ENOTTY:
# Assume we are a daemon because there is no terminal.
is_daemon = True
else:
raise
except UnsupportedOperation:
# Could not get the fileno for stdout, so we must be a daemon.
is_daemon = True
return is_daemon
def _is_sighup_and_daemon(signo):
if not (_sighup_supported() and signo == signal.SIGHUP):
# Avoid checking if we are a daemon, because the signal isn't
# SIGHUP.
return False
return _is_daemon()
def _signo_to_signame(signo):
@ -160,7 +192,7 @@ class ServiceLauncher(Launcher):
while True:
self.handle_signal()
status, signo = self._wait_for_exit_or_signal(ready_callback)
if not _is_sighup(signo):
if not _is_sighup_and_daemon(signo):
return status
self.restart()
@ -174,10 +206,16 @@ class ServiceWrapper(object):
class ProcessLauncher(object):
def __init__(self):
def __init__(self, wait_interval=0.01):
"""Constructor.
:param wait_interval: The interval to sleep for between checks
of child process exit.
"""
self.children = {}
self.sigcaught = None
self.running = True
self.wait_interval = wait_interval
rfd, self.writepipe = os.pipe()
self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r')
self.handle_signal()
@ -280,7 +318,7 @@ class ProcessLauncher(object):
while True:
self._child_process_handle_signal()
status, signo = self._child_wait_for_exit_or_signal(launcher)
if not _is_sighup(signo):
if not _is_sighup_and_daemon(signo):
break
launcher.restart()
@ -335,7 +373,7 @@ class ProcessLauncher(object):
# Yield to other threads if no children have exited
# Sleep for a short time to avoid excessive CPU usage
# (see bug #1095346)
eventlet.greenthread.sleep(.01)
eventlet.greenthread.sleep(self.wait_interval)
continue
while self.running and len(wrap.children) < wrap.workers:
self._start_child(wrap)
@ -352,7 +390,7 @@ class ProcessLauncher(object):
if self.sigcaught:
signame = _signo_to_signame(self.sigcaught)
LOG.info(_('Caught %s, stopping children'), signame)
if not _is_sighup(self.sigcaught):
if not _is_sighup_and_daemon(self.sigcaught):
break
for pid in self.children:
@ -381,11 +419,10 @@ class Service(object):
self.tg = threadgroup.ThreadGroup(threads)
# signal that the service is done shutting itself down:
self._done = event.Event()
self._done = threading.Event()
def reset(self):
# NOTE(Fengqian): docs for Event.reset() recommend against using it
self._done = event.Event()
self._done = threading.Event()
def start(self):
pass
@ -394,8 +431,7 @@ class Service(object):
self.tg.stop()
self.tg.wait()
# Signal that service cleanup is done:
if not self._done.ready():
self._done.send()
self._done.set()
def wait(self):
self._done.wait()
@ -406,7 +442,7 @@ class Services(object):
def __init__(self):
self.services = []
self.tg = threadgroup.ThreadGroup()
self.done = event.Event()
self.done = threading.Event()
def add(self, service):
self.services.append(service)
@ -420,8 +456,7 @@ class Services(object):
# Each service has performed cleanup, now signal that the run_service
# wrapper threads can now die:
if not self.done.ready():
self.done.send()
self.done.set()
# reap threads:
self.tg.stop()
@ -431,7 +466,7 @@ class Services(object):
def restart(self):
self.stop()
self.done = event.Event()
self.done = threading.Event()
for restart_service in self.services:
restart_service.reset()
self.tg.add_thread(self.run_service, restart_service, self.done)

2
ceilometer/openstack/common/sslutils.py
View File

@ -17,7 +17,7 @@ import ssl
from oslo.config import cfg
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
ssl_opts = [

18
ceilometer/openstack/common/strutils.py
View File

@ -23,7 +23,7 @@ import unicodedata
import six
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common.gettextutils import _
# Used for looking up extensions of text
@ -58,12 +58,12 @@ def int_from_bool_as_string(subject):
return bool_from_string(subject) and 1 or 0
def bool_from_string(subject, strict=False):
def bool_from_string(subject, strict=False, default=False):
"""Interpret a string as a boolean.
A case-insensitive match is performed such that strings matching 't',
'true', 'on', 'y', 'yes', or '1' are considered True and, when
`strict=False`, anything else is considered False.
`strict=False`, anything else returns the value specified by 'default'.
Useful for JSON-decoded stuff and config file parsing.
@ -88,7 +88,7 @@ def bool_from_string(subject, strict=False):
'acceptable': acceptable}
raise ValueError(msg)
else:
return False
return default
def safe_decode(text, incoming=None, errors='strict'):
@ -152,11 +152,17 @@ def safe_encode(text, incoming=None,
sys.getdefaultencoding())
if isinstance(text, six.text_type):
return text.encode(encoding, errors)
if six.PY3:
return text.encode(encoding, errors).decode(incoming)
else:
return text.encode(encoding, errors)
elif text and encoding != incoming:
# Decode text before encoding it with `encoding`
text = safe_decode(text, incoming, errors)
return text.encode(encoding, errors)
if six.PY3:
return text.encode(encoding, errors).decode(incoming)
else:
return text.encode(encoding, errors)
return text

2
ceilometer/openstack/common/test.py
View File

@ -29,7 +29,7 @@ class BaseTestCase(testtools.TestCase):
super(BaseTestCase, self).setUp()
self._set_timeout()
self._fake_output()
self.useFixture(fixtures.FakeLogger('ceilometer.openstack.common'))
self.useFixture(fixtures.FakeLogger())
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())

10
ceilometer/openstack/common/threadgroup.py
View File

@ -87,7 +87,10 @@ class ThreadGroup(object):
def stop(self):
current = greenthread.getcurrent()
for x in self.threads:
# Iterate over a copy of self.threads so thread_done doesn't
# modify the list while we're iterating
for x in self.threads[:]:
if x is current:
# don't kill the current thread.
continue
@ -112,7 +115,10 @@ class ThreadGroup(object):
except Exception as ex:
LOG.exception(ex)
current = greenthread.getcurrent()
for x in self.threads:
# Iterate over a copy of self.threads so thread_done doesn't
# modify the list while we're iterating
for x in self.threads[:]:
if x is current:
continue
try:

10
ceilometer/openstack/common/timeutils.py
View File

@ -77,6 +77,9 @@ def is_older_than(before, seconds):
"""Return True if before is older than seconds."""
if isinstance(before, six.string_types):
before = parse_strtime(before).replace(tzinfo=None)
else:
before = before.replace(tzinfo=None)
return utcnow() - before > datetime.timedelta(seconds=seconds)
@ -84,6 +87,9 @@ def is_newer_than(after, seconds):
"""Return True if after is newer than seconds."""
if isinstance(after, six.string_types):
after = parse_strtime(after).replace(tzinfo=None)
else:
after = after.replace(tzinfo=None)
return after - utcnow() > datetime.timedelta(seconds=seconds)
@ -195,8 +201,8 @@ def total_seconds(delta):
def is_soon(dt, window):
"""Determines if time is going to happen in the next window seconds.
:params dt: the time
:params window: minimum seconds to remain to consider the time not soon
:param dt: the time
:param window: minimum seconds to remain to consider the time not soon
:return: True if expiration is within the given duration
"""

105
ceilometer/openstack/common/versionutils.py
View File

@ -17,8 +17,113 @@
Helpers for comparing version strings.
"""
import functools
import pkg_resources
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log as logging
LOG = logging.getLogger(__name__)
class deprecated(object):
"""A decorator to mark callables as deprecated.
This decorator logs a deprecation message when the callable it decorates is
used. The message will include the release where the callable was
deprecated, the release where it may be removed and possibly an optional
replacement.
Examples:
1. Specifying the required deprecated release
>>> @deprecated(as_of=deprecated.ICEHOUSE)
... def a(): pass
2. Specifying a replacement:
>>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()')
... def b(): pass
3. Specifying the release where the functionality may be removed:
>>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=+1)
... def c(): pass
"""
FOLSOM = 'F'
GRIZZLY = 'G'
HAVANA = 'H'
ICEHOUSE = 'I'
_RELEASES = {
'F': 'Folsom',
'G': 'Grizzly',
'H': 'Havana',
'I': 'Icehouse',
}
_deprecated_msg_with_alternative = _(
'%(what)s is deprecated as of %(as_of)s in favor of '
'%(in_favor_of)s and may be removed in %(remove_in)s.')
_deprecated_msg_no_alternative = _(
'%(what)s is deprecated as of %(as_of)s and may be '
'removed in %(remove_in)s. It will not be superseded.')
def __init__(self, as_of, in_favor_of=None, remove_in=2, what=None):
"""Initialize decorator
:param as_of: the release deprecating the callable. Constants
are define in this class for convenience.
:param in_favor_of: the replacement for the callable (optional)
:param remove_in: an integer specifying how many releases to wait
before removing (default: 2)
:param what: name of the thing being deprecated (default: the
callable's name)
"""
self.as_of = as_of
self.in_favor_of = in_favor_of
self.remove_in = remove_in
self.what = what
def __call__(self, func):
if not self.what:
self.what = func.__name__ + '()'
@functools.wraps(func)
def wrapped(*args, **kwargs):
msg, details = self._build_message()
LOG.deprecated(msg, details)
return func(*args, **kwargs)
return wrapped
def _get_safe_to_remove_release(self, release):
# TODO(dstanek): this method will have to be reimplemented once
# when we get to the X release because once we get to the Y
# release, what is Y+2?
new_release = chr(ord(release) + self.remove_in)
if new_release in self._RELEASES:
return self._RELEASES[new_release]
else:
return new_release
def _build_message(self):
details = dict(what=self.what,
as_of=self._RELEASES[self.as_of],
remove_in=self._get_safe_to_remove_release(self.as_of))
if self.in_favor_of:
details['in_favor_of'] = self.in_favor_of
msg = self._deprecated_msg_with_alternative
else:
msg = self._deprecated_msg_no_alternative
return msg, details
def is_compatible(requested_version, current_version, same_major=True):
"""Determine whether `requested_version` is satisfied by

14
ceilometer/tests/api/v2/test_app.py
View File

@ -91,7 +91,7 @@ class TestApiMiddleware(FunctionalTest):
no_lang_translated_error = 'No lang translated error'
en_US_translated_error = 'en-US translated error'
def _fake_get_localized_message(self, message, user_locale):
def _fake_translate(self, message, user_locale):
if user_locale is None:
return self.no_lang_translated_error
else:
@ -140,8 +140,8 @@ class TestApiMiddleware(FunctionalTest):
def test_json_parsable_error_middleware_translation_400(self):
# Ensure translated messages get placed properly into json faults
with mock.patch.object(gettextutils, 'get_localized_message',
side_effect=self._fake_get_localized_message):
with mock.patch.object(gettextutils, 'translate',
side_effect=self._fake_translate):
response = self.post_json('/alarms', params={'name': 'foobar',
'type': 'threshold'},
expect_errors=True,
@ -175,8 +175,8 @@ class TestApiMiddleware(FunctionalTest):
def test_xml_parsable_error_middleware_translation_400(self):
# Ensure translated messages get placed properly into xml faults
with mock.patch.object(gettextutils, 'get_localized_message',
side_effect=self._fake_get_localized_message):
with mock.patch.object(gettextutils, 'translate',
side_effect=self._fake_translate):
response = self.post_json('/alarms', params={'name': 'foobar',
'type': 'threshold'},
expect_errors=True,
@ -192,8 +192,8 @@ class TestApiMiddleware(FunctionalTest):
def test_best_match_language(self):
# Ensure that we are actually invoking language negotiation
with mock.patch.object(gettextutils, 'get_localized_message',
side_effect=self._fake_get_localized_message):
with mock.patch.object(gettextutils, 'translate',
side_effect=self._fake_translate):
response = self.post_json('/alarms', params={'name': 'foobar',
'type': 'threshold'},
expect_errors=True,

167
ceilometer/tests/test_service.py
View File

@ -1,167 +0,0 @@
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
#
# Author: Julien Danjou <julien@danjou.info>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import shutil
import signal
import subprocess
import threading
import time
import yaml
from ceilometer.openstack.common import fileutils
from ceilometer.openstack.common import test
from ceilometer import service
from ceilometer.tests import base
class ServiceTestCase(test.BaseTestCase):
def test_prepare_service(self):
service.prepare_service([])
#NOTE(Fengqian): I have to set up a thread to parse the ouput of
#subprocess.Popen. Because readline() may block the process in
#some conditions.
class ParseOutput(threading.Thread):
def __init__(self, input_stream, str_flag):
super(ParseOutput, self).__init__()
self.input_stream = input_stream
self.str_flag = str_flag
self.ret_stream = None
self.ret = False
self.thread_stop = False
def run(self):
while not self.thread_stop:
next_line = self.input_stream.readline()
if next_line == '':
break
if self.str_flag in next_line:
self.ret = True
self.ret_stream = next_line[(next_line.find(self.str_flag) +
len(self.str_flag)):]
self.stop()
def stop(self):
self.thread_stop = True
class ServiceRestartTest(base.BaseTestCase):
def setUp(self):
super(ServiceRestartTest, self).setUp()
self.pipeline_cfg_file = fileutils.write_to_tempfile(content='',
prefix='pipeline',
suffix='.yaml')
shutil.copy(self.path_get('etc/ceilometer/pipeline.yaml'),
self.pipeline_cfg_file)
self.pipelinecfg_read_from_file()
policy_file = self.path_get('etc/ceilometer/policy.json')
content = "[DEFAULT]\n"\
"rpc_backend=ceilometer.openstack.common.rpc.impl_fake\n"\
"auth_strategy=noauth\n"\
"debug=true\n"\
"pipeline_cfg_file={0}\n"\
"policy_file={1}\n"\
"[database]\n"\
"connection=log://localhost\n".format(self.pipeline_cfg_file,
policy_file)
self.tempfile = fileutils.write_to_tempfile(content=content,
prefix='ceilometer',
suffix='.conf')
def _modify_pipeline_file(self):
with open(self.pipeline_cfg_file, 'w') as pipe_fd:
pipe_fd.truncate()
pipe_fd.write(yaml.safe_dump(self.pipeline_cfg[1]))
def pipelinecfg_read_from_file(self):
with open(self.pipeline_cfg_file) as fd:
data = fd.read()
self.pipeline_cfg = yaml.safe_load(data)
def tearDown(self):
super(ServiceRestartTest, self).tearDown()
self.sub.kill()
self.sub.wait()
os.remove(self.pipeline_cfg_file)
os.remove(self.tempfile)
@staticmethod
def _check_process_alive(pid):
try:
os.kill(pid, 0)
except OSError:
return False
return True
def check_process_alive(self):
cond = lambda: self._check_process_alive(self.sub.pid)
return self._wait(cond, 60)
def parse_output(self, str_flag, timeout=3):
parse = ParseOutput(self.sub.stderr, str_flag)
parse.start()
parse.join(timeout)
parse.stop()
return parse
@staticmethod
def _wait(cond, timeout):
start = time.time()
while not cond():
if time.time() - start > timeout:
break
time.sleep(.1)
return cond()
def _spawn_service(self, cmd, conf_file=None):
if conf_file is None:
conf_file = self.tempfile
self.sub = subprocess.Popen([cmd, '--config-file=%s' % conf_file],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
#NOTE(Fengqian): Parse the output to see if the service started
self.assertTrue(self.parse_output("Starting").ret)
self.check_process_alive()
def _service_restart(self, cmd):
self._spawn_service(cmd)
self.assertTrue(self.sub.pid)
#NOTE(Fengqian): Modify the pipleline configure file to see
#if the file is reloaded correctly.
self._modify_pipeline_file()
self.pipelinecfg_read_from_file()
os.kill(self.sub.pid, signal.SIGHUP)
self.assertTrue(self.check_process_alive())
self.assertTrue(self.parse_output("Caught SIGHUP").ret)
self.assertEqual(self.pipeline_cfg,
yaml.safe_load(
self.parse_output("Pipeline config: ").ret_stream))
def test_compute_service_restart(self):
self._service_restart('ceilometer-agent-compute')
def test_central_service_restart(self):
self._service_restart('ceilometer-agent-central')

10
etc/ceilometer/ceilometer.conf.sample
View File

@ -189,7 +189,7 @@
# format string to use for log messages with context (string
# value)
#logging_context_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user)s %(tenant)s] %(instance)s%(message)s
#logging_context_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
# format string to use for log messages without context
# (string value)
@ -204,7 +204,7 @@
#logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s
# list of logger=LEVEL pairs (list value)
#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,keystone=INFO,qpid=WARN,sqlalchemy=WARN,suds=INFO,iso8601=WARN
#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,iso8601=WARN
# publish error events (boolean value)
#publish_errors=false
@ -608,11 +608,6 @@
# Deprecated group/name - [DEFAULT]/db_backend
#backend=sqlalchemy
# Enable the experimental use of thread pooling for all DB API
# calls (boolean value)
# Deprecated group/name - [DEFAULT]/dbapi_use_tpool
#use_tpool=false
#
# Options defined in ceilometer.openstack.common.db.sqlalchemy.session
@ -633,6 +628,7 @@
# value)
# Deprecated group/name - [DEFAULT]/sql_idle_timeout
# Deprecated group/name - [DATABASE]/sql_idle_timeout
# Deprecated group/name - [sql]/idle_timeout
#idle_timeout=3600
# Minimum number of SQL connections to keep open in a pool

32
tools/config/check_uptodate.sh
View File

@ -1,9 +1,25 @@
#!/bin/sh
TEMPDIR=`mktemp -d /tmp/ceilometer-check-config-XXXXXX`
CFGFILE=ceilometer.conf.sample
tools/config/generate_sample.sh -b ./ -p ceilometer -o $TEMPDIR
if ! diff $TEMPDIR/$CFGFILE etc/ceilometer/$CFGFILE
then
echo "E: ceilometer.conf.sample is not up to date, please run tools/config/generate_sample.sh"
exit 42
#!/usr/bin/env bash
PROJECT_NAME=${PROJECT_NAME:-ceilometer}
CFGFILE_NAME=${PROJECT_NAME}.conf.sample
if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then
CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME}
elif [ -e etc/${CFGFILE_NAME} ]; then
CFGFILE=etc/${CFGFILE_NAME}
else
echo "${0##*/}: can not find config file"
exit 1
fi
TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX`
trap "rm -rf $TEMPDIR" EXIT
tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR}
if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE}
then
echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date."
echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh."
exit 1
fi