diff --git a/etc/trove/api-paste.ini b/etc/trove/api-paste.ini index 64c3a1445a..009125f7ca 100644 --- a/etc/trove/api-paste.ini +++ b/etc/trove/api-paste.ini @@ -7,7 +7,7 @@ use = call:trove.common.wsgi:versioned_urlmap paste.app_factory = trove.versions:app_factory [pipeline:troveapi] -pipeline = faultwrapper authtoken authorization contextwrapper ratelimit extensions troveapp +pipeline = faultwrapper osprofiler authtoken authorization contextwrapper ratelimit extensions troveapp #pipeline = debug extensions troveapp [filter:extensions] @@ -28,6 +28,11 @@ paste.filter_factory = trove.common.wsgi:FaultWrapper.factory [filter:ratelimit] paste.filter_factory = trove.common.limits:RateLimitingMiddleware.factory +[filter:osprofiler] +paste.filter_factory = osprofiler.web:WsgiMiddleware.factory +hmac_keys = SECRET_KEY +enabled = yes + [app:troveapp] paste.app_factory = trove.common.api:app_factory diff --git a/etc/trove/trove-conductor.conf.sample b/etc/trove/trove-conductor.conf.sample index aaab8966b5..e9a6a90827 100644 --- a/etc/trove/trove-conductor.conf.sample +++ b/etc/trove/trove-conductor.conf.sample @@ -46,3 +46,9 @@ rabbit_password=f7999d1955c5014aa32c # The manager class to use for conductor. (string value) conductor_manager = trove.conductor.manager.Manager + +[profiler] +# If False fully disable profiling feature. +#enabled = False +# If False doesn't trace SQL requests. +#trace_sqlalchemy = True diff --git a/etc/trove/trove-guestagent.conf.sample b/etc/trove/trove-guestagent.conf.sample index a928d5d9db..3948947d17 100644 --- a/etc/trove/trove-guestagent.conf.sample +++ b/etc/trove/trove-guestagent.conf.sample @@ -105,6 +105,11 @@ root_grant_option = True log_dir = /var/log/trove/ log_file = logfile.txt +[profiler] +# If False fully disable profiling feature. +#enabled = False +# If False doesn't trace SQL requests. +#trace_sqlalchemy = True # ========== Datastore Specific Configuration Options ========== diff --git a/etc/trove/trove-taskmanager.conf.sample b/etc/trove/trove-taskmanager.conf.sample index 3c29b1a5e9..a805341b79 100644 --- a/etc/trove/trove-taskmanager.conf.sample +++ b/etc/trove/trove-taskmanager.conf.sample @@ -206,6 +206,12 @@ pydev_debug = disabled # its own oslo group with defined in it: # - tcp_ports; upd_ports; +[profiler] +# If False fully disable profiling feature. +#enabled = False +# If False doesn't trace SQL requests. +#trace_sqlalchemy = True + [mysql] # Format (single port or port range): A, B-C # where C greater than B diff --git a/etc/trove/trove.conf.sample b/etc/trove/trove.conf.sample index dcfa5dbad6..af93c3b699 100644 --- a/etc/trove/trove.conf.sample +++ b/etc/trove/trove.conf.sample @@ -194,6 +194,12 @@ api_paste_config = api-paste.ini # accessible. The existence of those setting and files will # enable SSL. +[profiler] +# If False fully disable profiling feature. +#enabled = False +# If False doesn't trace SQL requests. +#trace_sqlalchemy = True + [ssl] #cert_file = /path/to/server.crt diff --git a/requirements.txt b/requirements.txt index 6e6b3b29dd..21dcda47f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,4 +38,4 @@ six>=1.9.0 stevedore>=1.1.0 # Apache-2.0 ordereddict oslo.messaging>=1.6.0 # Apache-2.0 - +osprofiler>=0.3.0 # Apache-2.0 diff --git a/trove/cmd/api.py b/trove/cmd/api.py index d119aca57a..2e5a14ad32 100755 --- a/trove/cmd/api.py +++ b/trove/cmd/api.py @@ -14,11 +14,13 @@ # under the License. from oslo_concurrency import processutils from trove.cmd.common import with_initialize +from trove.common import profile @with_initialize def main(CONF): from trove.common import wsgi + profile.setup_profiler('api', CONF.host) conf_file = CONF.find_file(CONF.api_paste_config) workers = CONF.trove_api_workers or processutils.get_worker_count() launcher = wsgi.launch('trove', CONF.bind_port, conf_file, diff --git a/trove/common/cfg.py b/trove/common/cfg.py index 188a4ea9a5..e0d31cfccd 100644 --- a/trove/common/cfg.py +++ b/trove/common/cfg.py @@ -392,6 +392,18 @@ common_opts = [ 'become active.'), ] +# Profiling specific option groups + +profiler_group = cfg.OptGroup( + 'profiler', title='Profiler options', + help="Oslo option group designed for profiler") +profiler_opts = [ + cfg.BoolOpt("enabled", default=False, + help="If False fully disable profiling feature."), + cfg.BoolOpt("trace_sqlalchemy", default=True, + help="If False doesn't trace SQL requests.") +] + # Datastore specific option groups # Mysql @@ -777,6 +789,9 @@ CONF = cfg.CONF CONF.register_opts(path_opts) CONF.register_opts(common_opts) +CONF.register_group(profiler_group) +CONF.register_opts(profiler_opts, profiler_group) + CONF.register_group(mysql_group) CONF.register_group(percona_group) CONF.register_group(redis_group) diff --git a/trove/common/profile.py b/trove/common/profile.py new file mode 100644 index 0000000000..8f6c1c5d1f --- /dev/null +++ b/trove/common/profile.py @@ -0,0 +1,48 @@ +# Copyright 2015 IBM Corp. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo import messaging +from osprofiler import notifier +from osprofiler import web + +from trove.common import cfg +from trove.common import i18n +from trove.openstack.common import context +from trove.openstack.common import log as logging +from trove import rpc + + +_LW = i18n._LW +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +def setup_profiler(binary, host): + if CONF.profiler.enabled: + _notifier = notifier.create( + "Messaging", messaging, context.get_admin_context().to_dict(), + rpc.TRANSPORT, "trove", binary, host) + notifier.set(_notifier) + LOG.warn(_LW("The OpenStack Profiler is enabled. Using one of the " + "hmac_keys specified in the api-paste.ini file " + "(typically in /etc/trove), a trace can be made of all " + "requests. Only an admin user can retrieve the trace " + "information, however.\n" + "To disable the profiler, add the following to the " + "configuration file:\n" + "[profiler]\n" + "enabled=false")) + else: + web.disable() diff --git a/trove/common/rpc/service.py b/trove/common/rpc/service.py index 5e5c021274..4608d85eed 100644 --- a/trove/common/rpc/service.py +++ b/trove/common/rpc/service.py @@ -20,14 +20,17 @@ import inspect import os from oslo import messaging from oslo.utils import importutils +from osprofiler import profiler from trove.openstack.common.gettextutils import _ from trove.openstack.common import log as logging from trove.openstack.common import loopingcall from trove.openstack.common import service from trove.common import cfg +from trove.common import profile from trove import rpc + CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -40,10 +43,12 @@ class RpcService(service.Service): self.host = host or CONF.host self.binary = binary or os.path.basename(inspect.stack()[-1][1]) self.topic = topic or self.binary.rpartition('trove-')[2] - self.manager_impl = importutils.import_object(manager) + _manager = importutils.import_object(manager) + self.manager_impl = profiler.trace_cls("rpc")(_manager) self.report_interval = CONF.report_interval self.rpc_api_version = rpc_api_version or \ self.manager_impl.RPC_API_VERSION + profile.setup_profiler(self.binary, self.host) def start(self): LOG.debug("Creating RPC server for service %s", self.topic) diff --git a/trove/db/sqlalchemy/session.py b/trove/db/sqlalchemy/session.py index 54b271bec0..140e814339 100644 --- a/trove/db/sqlalchemy/session.py +++ b/trove/db/sqlalchemy/session.py @@ -14,6 +14,8 @@ # under the License. import contextlib +import osprofiler.sqlalchemy +import sqlalchemy from sqlalchemy import create_engine from sqlalchemy import MetaData from sqlalchemy.orm import sessionmaker @@ -77,7 +79,10 @@ def _create_engine(options): "echo": CONF.sql_query_log } LOG.info(_("Creating SQLAlchemy engine with args: %s") % engine_args) - return create_engine(options['sql_connection'], **engine_args) + db_engine = create_engine(options['sql_connection'], **engine_args) + if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy: + osprofiler.sqlalchemy.add_tracing(sqlalchemy, db_engine, "db") + return db_engine def get_session(autocommit=True, expire_on_commit=False): diff --git a/trove/rpc.py b/trove/rpc.py index cff7d1d0b6..151da301fd 100644 --- a/trove/rpc.py +++ b/trove/rpc.py @@ -30,12 +30,16 @@ __all__ = [ 'TRANSPORT_ALIASES', ] + from oslo.config import cfg from oslo import messaging +from osprofiler import profiler + from trove.common.context import TroveContext import trove.common.exception from trove.openstack.common import jsonutils + CONF = cfg.CONF TRANSPORT = None NOTIFIER = None @@ -115,9 +119,21 @@ class RequestContextSerializer(messaging.Serializer): return self._base.deserialize_entity(context, entity) def serialize_context(self, context): - return context.to_dict() + _context = context.to_dict() + prof = profiler.get() + if prof: + trace_info = { + "hmac_key": prof.hmac_key, + "base_id": prof.get_base_id(), + "parent_id": prof.get_id() + } + _context.update({"trace_info": trace_info}) + return _context def deserialize_context(self, context): + trace_info = context.pop("trace_info", None) + if trace_info: + profiler.init(**trace_info) return TroveContext.from_dict(context)