From 61765db141f058aff2b48c1e5d06b8e44c8042b3 Mon Sep 17 00:00:00 2001 From: Flaper Fesp Date: Tue, 4 Jun 2013 10:02:25 +0200 Subject: [PATCH] Replace gunicorn with wsgiref The patch replaces gunicorn with wsgiref since it doesn't make sense to have gunicorn as dependency. Lets let deployers choose whatever the prefer to use as container. The patch also removes lib/* since marconi_paste is not needed anymore, the wsgi app can now be accessed through: `marconi.transport.wsgi.app:app` Backward incompatible change: bind refers now to the host and a new config variable was introduced to specify the port it should bind to. Fixes bug: #1187280 Implements blueprint: transport-wsgi Change-Id: I9f7767ace5c6553e75e2f4587032d7d64b9537c4 --- etc/marconi.conf-sample | 13 +-- etc/paste.ini-sample | 2 +- lib/__init__.py | 0 marconi/bootstrap.py | 16 ++-- marconi/common/decorators.py | 42 ++++++++++ .../tests/common/__init__.py | 13 +-- marconi/tests/common/test_decorators.py | 77 ++++++++++++++++++ marconi/tests/etc/wsgi_sqlite.conf | 3 +- marconi/tests/test_bootstrap.py | 14 ++-- marconi/tests/transport/wsgi/test_app.py | 40 ---------- marconi/transport/base.py | 10 ++- marconi/transport/wsgi/app.py | 59 ++++---------- marconi/transport/wsgi/driver.py | 79 ++++++++++++------- tools/pip-requires | 1 - 14 files changed, 216 insertions(+), 153 deletions(-) delete mode 100644 lib/__init__.py create mode 100644 marconi/common/decorators.py rename lib/marconi_paste.py => marconi/tests/common/__init__.py (66%) create mode 100644 marconi/tests/common/test_decorators.py delete mode 100644 marconi/tests/transport/wsgi/test_app.py diff --git a/etc/marconi.conf-sample b/etc/marconi.conf-sample index 5f6dac54f..b7419e153 100644 --- a/etc/marconi.conf-sample +++ b/etc/marconi.conf-sample @@ -9,17 +9,8 @@ transport = marconi.transport.wsgi storage = marconi.storage.mongodb [drivers:transport:wsgi] -bind = 0.0.0.0:8888 -; workers = 4 -workers = 1 -; worker_class = sync, gevent, eventlet -worker_class = sync -; user = 1000 -; group = 1000 -; proc_name = marconi -; certfile = cert.crt -; keyfile = cert.key - +bind = 0.0.0.0 +port = 8888 ;[drivers:transport:zmq] ;port = 9999 diff --git a/etc/paste.ini-sample b/etc/paste.ini-sample index 53706bde4..7856c9e37 100644 --- a/etc/paste.ini-sample +++ b/etc/paste.ini-sample @@ -11,4 +11,4 @@ admin_user = %SERVICE_USER% admin_password = %SERVICE_PASSWORD% [app:marconi] -paste.app_factory = lib.marconi_paste:WSGI.app_factory +paste.app_factory = marconi.transport.wsgi.app:app diff --git a/lib/__init__.py b/lib/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/marconi/bootstrap.py b/marconi/bootstrap.py index df07a2d2b..ad03b0f42 100644 --- a/marconi/bootstrap.py +++ b/marconi/bootstrap.py @@ -14,6 +14,7 @@ # limitations under the License. from marconi.common import config +from marconi.common import decorators from marconi.common import exceptions from marconi.openstack.common import importutils @@ -34,14 +35,15 @@ class Bootstrap(object): def __init__(self, config_file=None, cli_args=None): cfg_handle.load(filename=config_file, args=cli_args) - self.storage_module = import_driver(cfg.storage) - self.transport_module = import_driver(cfg.transport) + @decorators.lazy_property(write=False) + def storage(self): + storage_module = import_driver(cfg.storage) + return storage_module.Driver() - self.storage = self.storage_module.Driver() - self.transport = self.transport_module.Driver( - self.storage.queue_controller, - self.storage.message_controller, - self.storage.claim_controller) + @decorators.lazy_property(write=False) + def transport(self): + transport_module = import_driver(cfg.transport) + return transport_module.Driver(self.storage) def run(self): self.transport.listen() diff --git a/marconi/common/decorators.py b/marconi/common/decorators.py new file mode 100644 index 000000000..b1f34aca2 --- /dev/null +++ b/marconi/common/decorators.py @@ -0,0 +1,42 @@ +# Copyright (c) 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def lazy_property(write=False, delete=True): + """Creates a lazy property. + + :param write: Whether this property is "writable" + :param delete: Whether this property can be deleted. + """ + + def wrapper(fn): + attr_name = '_lazy_' + fn.__name__ + + def getter(self): + if not hasattr(self, attr_name): + setattr(self, attr_name, fn(self)) + return getattr(self, attr_name) + + def setter(self, value): + setattr(self, attr_name, value) + + def deleter(self): + delattr(self, attr_name) + + return property(fget=getter, + fset=write and setter, + fdel=delete and deleter, + doc=fn.__doc__) + return wrapper diff --git a/lib/marconi_paste.py b/marconi/tests/common/__init__.py similarity index 66% rename from lib/marconi_paste.py rename to marconi/tests/common/__init__.py index ca9951a1a..cc355f6f5 100644 --- a/lib/marconi_paste.py +++ b/marconi/tests/common/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013 Rackspace, Inc. +# Copyright (c) 2013 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,14 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and -# limitations under the License. - -import marconi - - -class WSGI(object): - @staticmethod - def app_factory(global_config, **local_config): - bootstrap = marconi.Bootstrap() - - return bootstrap.transport.app diff --git a/marconi/tests/common/test_decorators.py b/marconi/tests/common/test_decorators.py new file mode 100644 index 000000000..98c741f0d --- /dev/null +++ b/marconi/tests/common/test_decorators.py @@ -0,0 +1,77 @@ +# Copyright (c) 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from marconi.common import decorators +from marconi.tests import util as testing + + +class TestLazyProperty(testing.TestBase): + + class DecoratedClass(object): + + @decorators.lazy_property(write=True) + def read_write_delete(self): + return True + + @decorators.lazy_property(write=True, delete=False) + def read_write(self): + return True + + @decorators.lazy_property() + def read_delete(self): + return True + + def setUp(self): + super(TestLazyProperty, self).setUp() + self.cls_instance = self.DecoratedClass() + + def test_write_delete(self): + self.assertTrue(self.cls_instance.read_write_delete) + self.assertTrue(hasattr(self.cls_instance, "_lazy_read_write_delete")) + + self.cls_instance.read_write_delete = False + self.assertFalse(self.cls_instance.read_write_delete) + + del self.cls_instance.read_write_delete + self.assertFalse(hasattr(self.cls_instance, "_lazy_read_write_delete")) + + def test_write(self): + self.assertTrue(self.cls_instance.read_write) + self.assertTrue(hasattr(self.cls_instance, "_lazy_read_write")) + + self.cls_instance.read_write = False + self.assertFalse(self.cls_instance.read_write) + + try: + del self.cls_instance.read_write + self.fail() + except TypeError: + # Bool object is not callable + self.assertTrue(hasattr(self.cls_instance, "_lazy_read_write")) + + def test_delete(self): + self.assertTrue(self.cls_instance.read_delete) + self.assertTrue(hasattr(self.cls_instance, "_lazy_read_delete")) + + try: + self.cls_instance.read_delete = False + self.fail() + except TypeError: + # Bool object is not callable + pass + + del self.cls_instance.read_delete + self.assertFalse(hasattr(self.cls_instance, "_lazy_read_delete")) diff --git a/marconi/tests/etc/wsgi_sqlite.conf b/marconi/tests/etc/wsgi_sqlite.conf index ff11a5442..1a6b5ea85 100644 --- a/marconi/tests/etc/wsgi_sqlite.conf +++ b/marconi/tests/etc/wsgi_sqlite.conf @@ -3,5 +3,6 @@ transport = marconi.transport.wsgi storage = marconi.storage.sqlite [drivers:transport:wsgi] -bind = 0.0.0.0:8888 +bind = 0.0.0.0 +port = 8888 workers = 20 diff --git a/marconi/tests/test_bootstrap.py b/marconi/tests/test_bootstrap.py index 4b2068f86..5e8a15186 100644 --- a/marconi/tests/test_bootstrap.py +++ b/marconi/tests/test_bootstrap.py @@ -28,19 +28,21 @@ class TestBootstrap(base.TestBase): self.assertRaises(cfg.ConfigFilesNotFoundError, marconi.Bootstrap, '') def test_storage_invalid(self): + conf_file = 'etc/drivers_storage_invalid.conf' + bootstrap = marconi.Bootstrap(conf_file) self.assertRaises(exceptions.InvalidDriver, - marconi.Bootstrap, - 'etc/drivers_storage_invalid.conf') + lambda: bootstrap.storage) def test_storage_sqlite(self): - bootstrap = marconi.Bootstrap('etc/wsgi_sqlite.conf') - + conf_file = 'etc/wsgi_sqlite.conf' + bootstrap = marconi.Bootstrap(conf_file) self.assertIsInstance(bootstrap.storage, sqlite.Driver) def test_transport_invalid(self): + conf_file = 'etc/drivers_transport_invalid.conf' + bootstrap = marconi.Bootstrap(conf_file) self.assertRaises(exceptions.InvalidDriver, - marconi.Bootstrap, - 'etc/drivers_transport_invalid.conf') + lambda: bootstrap.transport) def test_transport_wsgi(self): bootstrap = marconi.Bootstrap('etc/wsgi_sqlite.conf') diff --git a/marconi/tests/transport/wsgi/test_app.py b/marconi/tests/transport/wsgi/test_app.py deleted file mode 100644 index b3dc1e39a..000000000 --- a/marconi/tests/transport/wsgi/test_app.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import multiprocessing -import signal - -import marconi -from marconi.tests import util -from marconi.transport.wsgi import app - - -class TestApplication(util.TestBase): - - def setUp(self): - super(TestApplication, self).setUp() - - conf_file = self.conf_path('wsgi_sqlite.conf') - boot = marconi.Bootstrap(conf_file) - - self.app = app.Application(boot.transport.app) - - def test_run(self): - server = multiprocessing.Process(target=self.app.run) - server.start() - self.assertTrue(server.is_alive()) - server.terminate() - server.join() - self.assertEquals(server.exitcode, -signal.SIGTERM) diff --git a/marconi/transport/base.py b/marconi/transport/base.py index 1703f3595..fe4df68ec 100644 --- a/marconi/transport/base.py +++ b/marconi/transport/base.py @@ -16,11 +16,17 @@ import abc -class DriverBase: - """Base class for Transport Drivers to document the expected interface.""" +class DriverBase(object): + """Base class for Transport Drivers to document the expected interface. + + :param storage: The storage driver + """ __metaclass__ = abc.ABCMeta + def __init__(self, storage): + self.storage = storage + @abc.abstractmethod def listen(): """Start listening for client requests (self-hosting mode).""" diff --git a/marconi/transport/wsgi/app.py b/marconi/transport/wsgi/app.py index 1a688d487..b5aa757cf 100644 --- a/marconi/transport/wsgi/app.py +++ b/marconi/transport/wsgi/app.py @@ -12,51 +12,20 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Gunicorn Application implementation for Marconi + +"""WSGI App for WSGI Containers + +This app should be used by external WSGI +containers. For example: + + $ gunicorn marconi.transport.wsgi.app:app + +NOTE: As for external containers, it is necessary +to put config files in the standard paths. There's +no common way to specify / pass configuration files +to the WSGI app when it is called from other apps. """ -import gunicorn.app.base as gunicorn -import gunicorn.config as gconfig +from marconi import bootstrap -from marconi.common import config -import marconi.openstack.common.log as logging - - -OPTIONS = { - # Process - "user": None, - "group": None, - "proc_name": "marconi", - - # SSL - "certfile": None, - "keyfile": None, - - # Network - "workers": 1, - "bind": "0.0.0.0:8888", - "worker_class": "sync" -} - -cfg = config.namespace('drivers:transport:wsgi').from_options(**OPTIONS) - -LOG = logging.getLogger(__name__) - - -class Application(gunicorn.Application): - - def __init__(self, wsgi_app, *args, **kwargs): - super(Application, self).__init__(*args, **kwargs) - self.app = wsgi_app - - def load(self): - return self.app - - def load_config(self): - self.cfg = gconfig.Config(self.usage, prog=self.prog) - - for key in OPTIONS: - self.cfg.set(key, getattr(cfg, key)) - - self.logger = LOG +app = bootstrap.Bootstrap().transport.app diff --git a/marconi/transport/wsgi/driver.py b/marconi/transport/wsgi/driver.py index 2b607bda6..88160c459 100644 --- a/marconi/transport/wsgi/driver.py +++ b/marconi/transport/wsgi/driver.py @@ -14,43 +14,68 @@ # limitations under the License. import falcon +from wsgiref import simple_server +from marconi.common import config +import marconi.openstack.common.log as logging from marconi import transport -from marconi.transport.wsgi import app +from marconi.transport.wsgi import claims +from marconi.transport.wsgi import messages +from marconi.transport.wsgi import queues +from marconi.transport.wsgi import stats + +OPTIONS = { + 'bind': '0.0.0.0', + 'port': 8888 +} + +cfg = config.namespace('drivers:transport:wsgi').from_options(**OPTIONS) + +LOG = logging.getLogger(__name__) class Driver(transport.DriverBase): - def __init__(self, queue_controller, message_controller, - claim_controller): + def __init__(self, storage): + super(Driver, self).__init__(storage) - queue_collection = transport.wsgi.queues.CollectionResource( - queue_controller) - queue_item = transport.wsgi.queues.ItemResource(queue_controller) + self.app = falcon.API() - stats_endpoint = transport.wsgi.stats.Resource(queue_controller) + # Queues Endpoints + queue_controller = self.storage.queue_controller + queue_collection = queues.CollectionResource(queue_controller) + self.app.add_route('/v1/{project_id}/queues', queue_collection) - msg_collection = transport.wsgi.messages.CollectionResource( - message_controller) - msg_item = transport.wsgi.messages.ItemResource(message_controller) + queue_item = queues.ItemResource(queue_controller) + self.app.add_route('/v1/{project_id}/queues/{queue_name}', queue_item) - claim_collection = transport.wsgi.claims.CollectionResource( - claim_controller) - claim_item = transport.wsgi.claims.ItemResource(claim_controller) + stats_endpoint = stats.Resource(queue_controller) + self.app.add_route('/v1/{project_id}/queues/{queue_name}' + '/stats', stats_endpoint) - self.app = api = falcon.API() - api.add_route('/v1/{project_id}/queues', queue_collection) - api.add_route('/v1/{project_id}/queues/{queue_name}', queue_item) - api.add_route('/v1/{project_id}/queues/{queue_name}' - '/stats', stats_endpoint) - api.add_route('/v1/{project_id}/queues/{queue_name}' - '/messages', msg_collection) - api.add_route('/v1/{project_id}/queues/{queue_name}' - '/messages/{message_id}', msg_item) - api.add_route('/v1/{project_id}/queues/{queue_name}' - '/claims', claim_collection) - api.add_route('/v1/{project_id}/queues/{queue_name}' - '/claims/{claim_id}', claim_item) + # Messages Endpoints + message_controller = self.storage.message_controller + msg_collection = messages.CollectionResource(message_controller) + self.app.add_route('/v1/{project_id}/queues/{queue_name}' + '/messages', msg_collection) + + msg_item = messages.ItemResource(message_controller) + self.app.add_route('/v1/{project_id}/queues/{queue_name}' + '/messages/{message_id}', msg_item) + + # Claims Endpoints + claim_controller = self.storage.claim_controller + claim_collection = claims.CollectionResource(claim_controller) + self.app.add_route('/v1/{project_id}/queues/{queue_name}' + '/claims', claim_collection) + + claim_item = claims.ItemResource(claim_controller) + self.app.add_route('/v1/{project_id}/queues/{queue_name}' + '/claims/{claim_id}', claim_item) def listen(self): - return app.Application(self.app).run() + msg = _("Serving on host %(bind)s:%(port)s") % {"bind": cfg.bind, + "port": cfg.port} + LOG.debug(msg) + httpd = simple_server.make_server(cfg.bind, cfg.port, self.app) + httpd.serve_forever() diff --git a/tools/pip-requires b/tools/pip-requires index 3889695ea..7cea4c8e2 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -1,7 +1,6 @@ cliff eventlet>=0.9.12 falcon>=0.1.4 -gunicorn iso8601>=0.1.4 msgpack-python oslo.config>=1.1.0