From feb409eda0f04aacc02bb1ff5679cb6345c7a15a Mon Sep 17 00:00:00 2001 From: Rob Raymond Date: Thu, 27 Feb 2014 17:29:16 -0700 Subject: [PATCH] Replace hard coded WSGI application creation This change replaces the hard coded WSGI app creation with a pipeline of WSGI apps declared in a configuration file. Paste Deploy was used to create the pipeline since it is used by many other OpenStack projects and it is an active project with new contributors and supports Python 3. Dependency on Paste is localized so switching to another library would not be hard if OpenStack moves to another package in the future. The changes are small but the changes for the tests were large since many acl tests were assuming a hard coded WSGI app creation. blueprint declarative-filters Change-Id: I5ce05eab980271873269eca2945dc809f2923045 --- ceilometer/api/acl.py | 21 -------- ceilometer/api/app.py | 48 ++++++++++++++----- ceilometer/api/app.wsgi | 4 +- ceilometer/api/v1/app.py | 9 +--- ceilometer/tests/api/__init__.py | 8 ++-- ceilometer/tests/api/v1/test_app.py | 14 ++++-- ceilometer/tests/api/v2/test_acl_scenarios.py | 9 +++- ceilometer/tests/api/v2/test_app.py | 13 +++-- ceilometer/tests/test_bin.py | 15 +++++- etc/ceilometer/api_paste.ini | 15 ++++++ requirements.txt | 1 + 11 files changed, 100 insertions(+), 57 deletions(-) create mode 100644 etc/ceilometer/api_paste.ini diff --git a/ceilometer/api/acl.py b/ceilometer/api/acl.py index 7d8f077ad..3179ef507 100644 --- a/ceilometer/api/acl.py +++ b/ceilometer/api/acl.py @@ -19,29 +19,8 @@ """Access Control Lists (ACL's) control access the API server.""" from ceilometer.openstack.common import policy -from keystoneclient.middleware import auth_token -from oslo.config import cfg - _ENFORCER = None -OPT_GROUP_NAME = 'keystone_authtoken' - - -def register_opts(conf): - """Register keystoneclient middleware options - """ - conf.register_opts(auth_token.opts, - group=OPT_GROUP_NAME) - auth_token.CONF = conf - - -register_opts(cfg.CONF) - - -def install(app, conf): - """Install ACL check on application.""" - return auth_token.AuthProtocol(app, - conf=dict(conf.get(OPT_GROUP_NAME))) def get_limited_to(headers): diff --git a/ceilometer/api/app.py b/ceilometer/api/app.py index 74b47ee1d..49ceacfb0 100644 --- a/ceilometer/api/app.py +++ b/ceilometer/api/app.py @@ -23,9 +23,9 @@ from wsgiref import simple_server import netaddr from oslo.config import cfg +from paste import deploy import pecan -from ceilometer.api import acl from ceilometer.api import config as api_config from ceilometer.api import hooks from ceilometer.api import middleware @@ -35,12 +35,13 @@ from ceilometer import storage LOG = log.getLogger(__name__) auth_opts = [ - cfg.StrOpt('auth_strategy', - default='keystone', - help='The strategy to use for auth: noauth or keystone.'), cfg.BoolOpt('enable_v1_api', default=True, help='Deploy the deprecated v1 API.'), + cfg.StrOpt('api_paste_config', + default="api_paste.ini", + help="Configuration file for WSGI definition of API." + ), ] CONF = cfg.CONF @@ -80,9 +81,6 @@ def setup_app(pecan_config=None, extra_hooks=None): guess_content_type_from_ext=False ) - if getattr(pecan_config.app, 'enable_acl', True): - return acl.install(app, cfg.CONF) - return app @@ -90,10 +88,9 @@ class VersionSelectorApplication(object): def __init__(self): pc = get_pecan_config() pc.app.debug = CONF.debug - pc.app.enable_acl = (CONF.auth_strategy == 'keystone') if cfg.CONF.enable_v1_api: from ceilometer.api.v1 import app as v1app - self.v1 = v1app.make_app(cfg.CONF, enable_acl=pc.app.enable_acl) + self.v1 = v1app.make_app(cfg.CONF) else: def not_found(environ, start_response): start_response('404 Not Found', []) @@ -136,14 +133,36 @@ def get_handler_cls(): return CeilometerHandler -def build_server(): +def load_app(): # Build the WSGI app - root = VersionSelectorApplication() + cfg_file = cfg.CONF.api_paste_config + LOG.info("WSGI config requested: %s" % cfg_file) + if not os.path.exists(cfg_file): + # this code is to work around chicken-egg dependency between + # ceilometer gate jobs use of devstack and this change. + # The gate job uses devstack to run tempest. + # devstack does not copy api_paste.ini into /etc/ceilometer because it + # is introduced in this change. Once this is merged, we will change + # devstack to copy api_paste.ini and once that is merged will remove + # this code. + root = os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..', 'etc', 'ceilometer' + ) + ) + cfg_file = os.path.join(root, cfg_file) + if not os.path.exists(cfg_file): + raise Exception('api_paste_config file not found') + LOG.info("Full WSGI config used: %s" % cfg_file) + return deploy.loadapp("config:" + cfg_file) + +def build_server(): + app = load_app() # Create the WSGI server and start it host, port = cfg.CONF.api.host, cfg.CONF.api.port server_cls = get_server_cls(host) - srv = simple_server.make_server(host, port, root, + + srv = simple_server.make_server(host, port, app, server_cls, get_handler_cls()) LOG.info(_('Starting server in PID %s') % os.getpid()) @@ -157,4 +176,9 @@ def build_server(): else: LOG.info(_("serving on http://%(host)s:%(port)s") % ( {'host': host, 'port': port})) + return srv + + +def app_factory(global_config, **local_conf): + return VersionSelectorApplication() diff --git a/ceilometer/api/app.wsgi b/ceilometer/api/app.wsgi index 7ce62c2ca..4108e95c2 100644 --- a/ceilometer/api/app.wsgi +++ b/ceilometer/api/app.wsgi @@ -20,11 +20,9 @@ See http://pecan.readthedocs.org/en/latest/deployment.html for details. """ - from ceilometer import service from ceilometer.api import app # Initialize the oslo configuration library and logging service.prepare_service([]) - -application = app.VersionSelectorApplication() +application = app.load_app() diff --git a/ceilometer/api/v1/app.py b/ceilometer/api/v1/app.py index d383bfa63..9c496d937 100644 --- a/ceilometer/api/v1/app.py +++ b/ceilometer/api/v1/app.py @@ -20,7 +20,6 @@ import flask from oslo.config import cfg -from ceilometer.api import acl from ceilometer.api.v1 import blueprint as v1_blueprint from ceilometer.openstack.common import jsonutils from ceilometer import storage @@ -32,7 +31,7 @@ class JSONEncoder(flask.json.JSONEncoder): return jsonutils.to_primitive(o) -def make_app(conf, enable_acl=True, attach_storage=True, +def make_app(conf, attach_storage=True, sources_file='sources.json'): app = flask.Flask('ceilometer.api') app.register_blueprint(v1_blueprint.blueprint, url_prefix='/v1') @@ -56,11 +55,7 @@ def make_app(conf, enable_acl=True, attach_storage=True, flask.request.storage_conn = \ storage.get_connection(conf) - # Install the middleware wrapper - if enable_acl: - app.wsgi_app = acl.install(app.wsgi_app, conf) - return app # For documentation -app = make_app(cfg.CONF, enable_acl=False, attach_storage=False) +app = make_app(cfg.CONF, attach_storage=False) diff --git a/ceilometer/tests/api/__init__.py b/ceilometer/tests/api/__init__.py index 8d0f21322..5e165bcb6 100644 --- a/ceilometer/tests/api/__init__.py +++ b/ceilometer/tests/api/__init__.py @@ -23,7 +23,6 @@ import pecan import pecan.testing from six.moves import urllib -from ceilometer.api import acl from ceilometer.api.v1 import app as v1_app from ceilometer.api.v1 import blueprint as v1_blueprint from ceilometer import messaging @@ -31,6 +30,8 @@ from ceilometer.openstack.common import jsonutils from ceilometer import service from ceilometer.tests import db as db_test_base +OPT_GROUP_NAME = 'keystone_authtoken' + class TestBase(db_test_base.TestBase): """Use only for v1 API tests. @@ -42,12 +43,11 @@ class TestBase(db_test_base.TestBase): self.addCleanup(messaging.cleanup) service.prepare_service([]) self.CONF.set_override("auth_version", - "v2.0", group=acl.OPT_GROUP_NAME) + "v2.0", group=OPT_GROUP_NAME) self.CONF.set_override("policy_file", self.path_get('etc/ceilometer/policy.json')) sources_file = self.path_get('ceilometer/tests/sources.json') self.app = v1_app.make_app(self.CONF, - enable_acl=False, attach_storage=False, sources_file=sources_file) @@ -90,7 +90,7 @@ class FunctionalTest(db_test_base.TestBase): self.addCleanup(messaging.cleanup) super(FunctionalTest, self).setUp() self.CONF.set_override("auth_version", "v2.0", - group=acl.OPT_GROUP_NAME) + group=OPT_GROUP_NAME) self.CONF.set_override("policy_file", self.path_get('etc/ceilometer/policy.json')) self.app = self._make_app() diff --git a/ceilometer/tests/api/v1/test_app.py b/ceilometer/tests/api/v1/test_app.py index 4252b65be..d708ca2c5 100644 --- a/ceilometer/tests/api/v1/test_app.py +++ b/ceilometer/tests/api/v1/test_app.py @@ -19,13 +19,15 @@ """ import os -from ceilometer.api import acl +from keystoneclient.middleware import auth_token + from ceilometer.api.v1 import app from ceilometer import messaging from ceilometer.openstack.common import fileutils from ceilometer.openstack.common.fixture import config from ceilometer.openstack.common import test from ceilometer import service +from ceilometer.tests import api as acl class TestApp(test.BaseTestCase): @@ -44,7 +46,10 @@ class TestApp(test.BaseTestCase): self.CONF.set_override("auth_uri", None, group=acl.OPT_GROUP_NAME) api_app = app.make_app(self.CONF, attach_storage=False) - self.assertTrue(api_app.wsgi_app.auth_uri.startswith('file')) + conf = dict(self.CONF.get(acl.OPT_GROUP_NAME)) + api_app = auth_token.AuthProtocol(api_app, + conf=conf) + self.assertTrue(api_app.auth_uri.startswith('file')) def test_keystone_middleware_parse_conffile(self): content = "[{0}]\nauth_protocol = file"\ @@ -55,5 +60,8 @@ class TestApp(test.BaseTestCase): service.prepare_service(['ceilometer-api', '--config-file=%s' % tmpfile]) api_app = app.make_app(self.CONF, attach_storage=False) - self.assertTrue(api_app.wsgi_app.auth_uri.startswith('file')) + conf = dict(self.CONF.get(acl.OPT_GROUP_NAME)) + api_app = auth_token.AuthProtocol(api_app, + conf=conf) + self.assertTrue(api_app.auth_uri.startswith('file')) os.unlink(tmpfile) diff --git a/ceilometer/tests/api/v2/test_acl_scenarios.py b/ceilometer/tests/api/v2/test_acl_scenarios.py index a4320ede4..0cd27b3ed 100644 --- a/ceilometer/tests/api/v2/test_acl_scenarios.py +++ b/ceilometer/tests/api/v2/test_acl_scenarios.py @@ -20,11 +20,14 @@ import datetime import json -from ceilometer.api import acl +import webtest + +from ceilometer.api import app from ceilometer.api.controllers import v2 as v2_api from ceilometer.openstack.common import timeutils from ceilometer.publisher import utils from ceilometer import sample +from ceilometer.tests import api as acl from ceilometer.tests.api.v2 import FunctionalTest from ceilometer.tests import db as tests_db @@ -115,7 +118,9 @@ class TestAPIACL(FunctionalTest, def _make_app(self): self.CONF.set_override("cache", "fake.cache", group=acl.OPT_GROUP_NAME) - return super(TestAPIACL, self)._make_app(enable_acl=True) + file_name = self.path_get('etc/ceilometer/api_paste.ini') + self.CONF.set_override("api_paste_config", file_name) + return webtest.TestApp(app.load_app()) def test_non_authenticated(self): response = self.get_json('/meters', expect_errors=True) diff --git a/ceilometer/tests/api/v2/test_app.py b/ceilometer/tests/api/v2/test_app.py index 895d5804e..5469f4fbb 100644 --- a/ceilometer/tests/api/v2/test_app.py +++ b/ceilometer/tests/api/v2/test_app.py @@ -24,12 +24,12 @@ import os import mock import wsme -from ceilometer.api import acl from ceilometer.api import app from ceilometer.openstack.common import fileutils from ceilometer.openstack.common.fixture import config from ceilometer.openstack.common import gettextutils from ceilometer import service +from ceilometer.tests import api as acl from ceilometer.tests.api.v2 import FunctionalTest from ceilometer.tests import base from ceilometer.tests import db as tests_db @@ -50,18 +50,23 @@ class TestApp(base.BaseTestCase): self.path_get("etc/ceilometer/pipeline.yaml")) self.CONF.set_override('connection', "log://", group="database") self.CONF.set_override("auth_uri", None, group=acl.OPT_GROUP_NAME) + file_name = self.path_get('etc/ceilometer/api_paste.ini') + self.CONF.set_override("api_paste_config", file_name) - api_app = app.setup_app() + api_app = app.load_app() self.assertTrue(api_app.auth_uri.startswith('file')) def test_keystone_middleware_parse_conffile(self): pipeline_conf = self.path_get("etc/ceilometer/pipeline.yaml") + api_conf = self.path_get('etc/ceilometer/api_paste.ini') content = "[DEFAULT]\n"\ "rpc_backend = fake\n"\ "pipeline_cfg_file = {0}\n"\ - "[{1}]\n"\ + "api_paste_config = {1}\n"\ + "[{2}]\n"\ "auth_protocol = file\n"\ "auth_version = v2.0\n".format(pipeline_conf, + api_conf, acl.OPT_GROUP_NAME) tmpfile = fileutils.write_to_tempfile(content=content, @@ -70,7 +75,7 @@ class TestApp(base.BaseTestCase): service.prepare_service(['ceilometer-api', '--config-file=%s' % tmpfile]) self.CONF.set_override('connection', "log://", group="database") - api_app = app.setup_app() + api_app = app.load_app() self.assertTrue(api_app.auth_uri.startswith('file')) os.unlink(tmpfile) diff --git a/ceilometer/tests/test_bin.py b/ceilometer/tests/test_bin.py index 00b4b7e1e..5326d4e96 100644 --- a/ceilometer/tests/test_bin.py +++ b/ceilometer/tests/test_bin.py @@ -105,6 +105,17 @@ class BinApiTestCase(base.BaseTestCase): def setUp(self): super(BinApiTestCase, self).setUp() + # create api_paste.ini file without authentication + content = \ + "[pipeline:main]\n"\ + "pipeline = api-server\n"\ + "[app:api-server]\n"\ + "paste.app_factory = ceilometer.api.app:app_factory\n" + self.paste = fileutils.write_to_tempfile(content=content, + prefix='api_paste', + suffix='.ini') + + # create ceilometer.conf file self.api_port = random.randint(10000, 11000) self.http = httplib2.Http() pipeline_cfg_file = self.path_get('etc/ceilometer/pipeline.yaml') @@ -115,11 +126,13 @@ class BinApiTestCase(base.BaseTestCase): "debug=true\n"\ "pipeline_cfg_file={0}\n"\ "policy_file={1}\n"\ + "api_paste_config={2}\n"\ "[api]\n"\ - "port={2}\n"\ + "port={3}\n"\ "[database]\n"\ "connection=log://localhost\n".format(pipeline_cfg_file, policy_file, + self.paste, self.api_port) self.tempfile = fileutils.write_to_tempfile(content=content, diff --git a/etc/ceilometer/api_paste.ini b/etc/ceilometer/api_paste.ini new file mode 100644 index 000000000..6ae6f449c --- /dev/null +++ b/etc/ceilometer/api_paste.ini @@ -0,0 +1,15 @@ +# Ceilometer API WSGI Pipeline +# Define the filters that make up the pipeline for processing WSGI requests +# Note: This pipeline is PasteDeploy's term rather than Ceilometer's pipeline +# used for processing samples + +# Remove authtoken from the pipeline if you don't want to use keystone authentication +[pipeline:main] +pipeline = authtoken api-server + +[app:api-server] +paste.app_factory = ceilometer.api.app:app_factory + +[filter:authtoken] +paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory + diff --git a/requirements.txt b/requirements.txt index 8cca5ea0a..039e26d1d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ msgpack-python netaddr>=0.7.6 oslo.config>=1.2.0 oslo.vmware>=0.2 # Apache-2.0 +PasteDeploy>=1.5.0 pbr>=0.6,<1.0 pecan>=0.4.5 posix_ipc