Replace gunicorn with wsgiref

The patch replaces gunicorn with wsgiref since it doesn't make sense to
have gunicorn as dependency. Lets let deployers choose whatever the
prefer to use as container.

The patch also removes lib/* since marconi_paste is not needed anymore,
the wsgi app can now be accessed through:

    `marconi.transport.wsgi.app:app`

Backward incompatible change: bind refers now to the host and a new
config variable was introduced to specify the port it should bind to.

Fixes bug: #1187280
Implements blueprint: transport-wsgi

Change-Id: I9f7767ace5c6553e75e2f4587032d7d64b9537c4

etc/marconi.conf-sample

@@ -9,17 +9,8 @@ transport = marconi.transport.wsgi
 storage = marconi.storage.mongodb
 
 [drivers:transport:wsgi]
-bind = 0.0.0.0:8888
-; workers = 4
-workers = 1
-; worker_class = sync, gevent, eventlet
-worker_class = sync
-; user = 1000
-; group = 1000
-; proc_name = marconi
-; certfile = cert.crt
-; keyfile = cert.key
-
+bind = 0.0.0.0
+port = 8888
 
 ;[drivers:transport:zmq]
 ;port = 9999

etc/paste.ini-sample

@@ -11,4 +11,4 @@ admin_user = %SERVICE_USER%
 admin_password = %SERVICE_PASSWORD%
 
 [app:marconi]
-paste.app_factory = lib.marconi_paste:WSGI.app_factory
+paste.app_factory = marconi.transport.wsgi.app:app

marconi/bootstrap.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from marconi.common import config
+from marconi.common import decorators
 from marconi.common import exceptions
 from marconi.openstack.common import importutils
 
@@ -34,14 +35,15 @@ class Bootstrap(object):
     def __init__(self, config_file=None, cli_args=None):
         cfg_handle.load(filename=config_file, args=cli_args)
 
-        self.storage_module = import_driver(cfg.storage)
-        self.transport_module = import_driver(cfg.transport)
+    @decorators.lazy_property(write=False)
+    def storage(self):
+        storage_module = import_driver(cfg.storage)
+        return storage_module.Driver()
 
-        self.storage = self.storage_module.Driver()
-        self.transport = self.transport_module.Driver(
-            self.storage.queue_controller,
-            self.storage.message_controller,
-            self.storage.claim_controller)
+    @decorators.lazy_property(write=False)
+    def transport(self):
+        transport_module = import_driver(cfg.transport)
+        return transport_module.Driver(self.storage)
 
     def run(self):
         self.transport.listen()

marconi/common/decorators.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2013 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def lazy_property(write=False, delete=True):
+    """Creates a lazy property.
+
+    :param write: Whether this property is "writable"
+    :param delete: Whether this property can be deleted.
+    """
+
+    def wrapper(fn):
+        attr_name = '_lazy_' + fn.__name__
+
+        def getter(self):
+            if not hasattr(self, attr_name):
+                setattr(self, attr_name, fn(self))
+            return getattr(self, attr_name)
+
+        def setter(self,  value):
+            setattr(self, attr_name, value)
+
+        def deleter(self):
+            delattr(self, attr_name)
+
+        return property(fget=getter,
+                        fset=write and setter,
+                        fdel=delete and deleter,
+                        doc=fn.__doc__)
+    return wrapper

marconi/tests/common/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013 Rackspace, Inc.
+# Copyright (c) 2013 Red Hat, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,14 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 # implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
-
-import marconi
-
-
-class WSGI(object):
-    @staticmethod
-    def app_factory(global_config, **local_config):
-        bootstrap = marconi.Bootstrap()
-
-        return bootstrap.transport.app

marconi/tests/common/test_decorators.py

@@ -0,0 +1,77 @@
+# Copyright (c) 2013 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from marconi.common import decorators
+from marconi.tests import util as testing
+
+
+class TestLazyProperty(testing.TestBase):
+
+    class DecoratedClass(object):
+
+        @decorators.lazy_property(write=True)
+        def read_write_delete(self):
+            return True
+
+        @decorators.lazy_property(write=True, delete=False)
+        def read_write(self):
+            return True
+
+        @decorators.lazy_property()
+        def read_delete(self):
+            return True
+
+    def setUp(self):
+        super(TestLazyProperty, self).setUp()
+        self.cls_instance = self.DecoratedClass()
+
+    def test_write_delete(self):
+        self.assertTrue(self.cls_instance.read_write_delete)
+        self.assertTrue(hasattr(self.cls_instance, "_lazy_read_write_delete"))
+
+        self.cls_instance.read_write_delete = False
+        self.assertFalse(self.cls_instance.read_write_delete)
+
+        del self.cls_instance.read_write_delete
+        self.assertFalse(hasattr(self.cls_instance, "_lazy_read_write_delete"))
+
+    def test_write(self):
+        self.assertTrue(self.cls_instance.read_write)
+        self.assertTrue(hasattr(self.cls_instance, "_lazy_read_write"))
+
+        self.cls_instance.read_write = False
+        self.assertFalse(self.cls_instance.read_write)
+
+        try:
+            del self.cls_instance.read_write
+            self.fail()
+        except TypeError:
+            # Bool object is not callable
+            self.assertTrue(hasattr(self.cls_instance, "_lazy_read_write"))
+
+    def test_delete(self):
+        self.assertTrue(self.cls_instance.read_delete)
+        self.assertTrue(hasattr(self.cls_instance, "_lazy_read_delete"))
+
+        try:
+            self.cls_instance.read_delete = False
+            self.fail()
+        except TypeError:
+            # Bool object is not callable
+            pass
+
+        del self.cls_instance.read_delete
+        self.assertFalse(hasattr(self.cls_instance, "_lazy_read_delete"))

marconi/tests/etc/wsgi_sqlite.conf

@@ -3,5 +3,6 @@ transport = marconi.transport.wsgi
 storage = marconi.storage.sqlite
 
 [drivers:transport:wsgi]
-bind = 0.0.0.0:8888
+bind = 0.0.0.0
+port = 8888
 workers = 20

marconi/tests/test_bootstrap.py

@@ -28,19 +28,21 @@ class TestBootstrap(base.TestBase):
         self.assertRaises(cfg.ConfigFilesNotFoundError, marconi.Bootstrap, '')
 
     def test_storage_invalid(self):
+        conf_file = 'etc/drivers_storage_invalid.conf'
+        bootstrap = marconi.Bootstrap(conf_file)
         self.assertRaises(exceptions.InvalidDriver,
-                          marconi.Bootstrap,
-                          'etc/drivers_storage_invalid.conf')
+                          lambda: bootstrap.storage)
 
     def test_storage_sqlite(self):
-        bootstrap = marconi.Bootstrap('etc/wsgi_sqlite.conf')
-
+        conf_file = 'etc/wsgi_sqlite.conf'
+        bootstrap = marconi.Bootstrap(conf_file)
         self.assertIsInstance(bootstrap.storage, sqlite.Driver)
 
     def test_transport_invalid(self):
+        conf_file = 'etc/drivers_transport_invalid.conf'
+        bootstrap = marconi.Bootstrap(conf_file)
         self.assertRaises(exceptions.InvalidDriver,
-                          marconi.Bootstrap,
-                          'etc/drivers_transport_invalid.conf')
+                          lambda: bootstrap.transport)
 
     def test_transport_wsgi(self):
         bootstrap = marconi.Bootstrap('etc/wsgi_sqlite.conf')

marconi/tests/transport/wsgi/test_app.py

@@ -1,40 +0,0 @@
-# Copyright (c) 2013 Red Hat, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import multiprocessing
-import signal
-
-import marconi
-from marconi.tests import util
-from marconi.transport.wsgi import app
-
-
-class TestApplication(util.TestBase):
-
-    def setUp(self):
-        super(TestApplication, self).setUp()
-
-        conf_file = self.conf_path('wsgi_sqlite.conf')
-        boot = marconi.Bootstrap(conf_file)
-
-        self.app = app.Application(boot.transport.app)
-
-    def test_run(self):
-        server = multiprocessing.Process(target=self.app.run)
-        server.start()
-        self.assertTrue(server.is_alive())
-        server.terminate()
-        server.join()
-        self.assertEquals(server.exitcode, -signal.SIGTERM)

marconi/transport/base.py

@@ -16,11 +16,17 @@
 import abc
 
 
-class DriverBase:
-    """Base class for Transport Drivers to document the expected interface."""
+class DriverBase(object):
+    """Base class for Transport Drivers to document the expected interface.
+
+    :param storage: The storage driver
+    """
 
     __metaclass__ = abc.ABCMeta
 
+    def __init__(self, storage):
+        self.storage = storage
+
     @abc.abstractmethod
     def listen():
         """Start listening for client requests (self-hosting mode)."""

marconi/transport/wsgi/app.py

@@ -12,51 +12,20 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
-Gunicorn Application implementation for Marconi
+
+"""WSGI App for WSGI Containers
+
+This app should be used by external WSGI
+containers. For example:
+
+    $ gunicorn marconi.transport.wsgi.app:app
+
+NOTE: As for external containers, it is necessary
+to put config files in the standard paths. There's
+no common way to specify / pass configuration files
+to the WSGI app when it is called from other apps.
 """
 
-import gunicorn.app.base as gunicorn
-import gunicorn.config as gconfig
+from marconi import bootstrap
 
-from marconi.common import config
-import marconi.openstack.common.log as logging
-
-
-OPTIONS = {
-    # Process
-    "user": None,
-    "group": None,
-    "proc_name": "marconi",
-
-    # SSL
-    "certfile": None,
-    "keyfile": None,
-
-    # Network
-    "workers": 1,
-    "bind": "0.0.0.0:8888",
-    "worker_class": "sync"
-}
-
-cfg = config.namespace('drivers:transport:wsgi').from_options(**OPTIONS)
-
-LOG = logging.getLogger(__name__)
-
-
-class Application(gunicorn.Application):
-
-    def __init__(self, wsgi_app, *args, **kwargs):
-        super(Application, self).__init__(*args, **kwargs)
-        self.app = wsgi_app
-
-    def load(self):
-        return self.app
-
-    def load_config(self):
-        self.cfg = gconfig.Config(self.usage, prog=self.prog)
-
-        for key in OPTIONS:
-            self.cfg.set(key, getattr(cfg, key))
-
-        self.logger = LOG
+app = bootstrap.Bootstrap().transport.app

marconi/transport/wsgi/driver.py

@@ -14,43 +14,68 @@
 # limitations under the License.
 
 import falcon
+from wsgiref import simple_server
 
+from marconi.common import config
+import marconi.openstack.common.log as logging
 from marconi import transport
-from marconi.transport.wsgi import app
+from marconi.transport.wsgi import claims
+from marconi.transport.wsgi import messages
+from marconi.transport.wsgi import queues
+from marconi.transport.wsgi import stats
+
+OPTIONS = {
+    'bind': '0.0.0.0',
+    'port': 8888
+}
+
+cfg = config.namespace('drivers:transport:wsgi').from_options(**OPTIONS)
+
+LOG = logging.getLogger(__name__)
 
 
 class Driver(transport.DriverBase):
 
-    def __init__(self, queue_controller, message_controller,
-                 claim_controller):
+    def __init__(self, storage):
+        super(Driver, self).__init__(storage)
 
-        queue_collection = transport.wsgi.queues.CollectionResource(
-            queue_controller)
-        queue_item = transport.wsgi.queues.ItemResource(queue_controller)
+        self.app = falcon.API()
 
-        stats_endpoint = transport.wsgi.stats.Resource(queue_controller)
+        # Queues Endpoints
+        queue_controller = self.storage.queue_controller
+        queue_collection = queues.CollectionResource(queue_controller)
+        self.app.add_route('/v1/{project_id}/queues', queue_collection)
 
-        msg_collection = transport.wsgi.messages.CollectionResource(
-            message_controller)
-        msg_item = transport.wsgi.messages.ItemResource(message_controller)
+        queue_item = queues.ItemResource(queue_controller)
+        self.app.add_route('/v1/{project_id}/queues/{queue_name}', queue_item)
 
-        claim_collection = transport.wsgi.claims.CollectionResource(
-            claim_controller)
-        claim_item = transport.wsgi.claims.ItemResource(claim_controller)
+        stats_endpoint = stats.Resource(queue_controller)
+        self.app.add_route('/v1/{project_id}/queues/{queue_name}'
+                           '/stats', stats_endpoint)
 
-        self.app = api = falcon.API()
-        api.add_route('/v1/{project_id}/queues', queue_collection)
-        api.add_route('/v1/{project_id}/queues/{queue_name}', queue_item)
-        api.add_route('/v1/{project_id}/queues/{queue_name}'
-                      '/stats', stats_endpoint)
-        api.add_route('/v1/{project_id}/queues/{queue_name}'
-                      '/messages', msg_collection)
-        api.add_route('/v1/{project_id}/queues/{queue_name}'
-                      '/messages/{message_id}', msg_item)
-        api.add_route('/v1/{project_id}/queues/{queue_name}'
-                      '/claims', claim_collection)
-        api.add_route('/v1/{project_id}/queues/{queue_name}'
-                      '/claims/{claim_id}', claim_item)
+        # Messages Endpoints
+        message_controller = self.storage.message_controller
+        msg_collection = messages.CollectionResource(message_controller)
+        self.app.add_route('/v1/{project_id}/queues/{queue_name}'
+                           '/messages', msg_collection)
+
+        msg_item = messages.ItemResource(message_controller)
+        self.app.add_route('/v1/{project_id}/queues/{queue_name}'
+                           '/messages/{message_id}', msg_item)
+
+        # Claims Endpoints
+        claim_controller = self.storage.claim_controller
+        claim_collection = claims.CollectionResource(claim_controller)
+        self.app.add_route('/v1/{project_id}/queues/{queue_name}'
+                           '/claims', claim_collection)
+
+        claim_item = claims.ItemResource(claim_controller)
+        self.app.add_route('/v1/{project_id}/queues/{queue_name}'
+                           '/claims/{claim_id}', claim_item)
 
     def listen(self):
-        return app.Application(self.app).run()
+        msg = _("Serving on host %(bind)s:%(port)s") % {"bind": cfg.bind,
+                                                        "port": cfg.port}
+        LOG.debug(msg)
+        httpd = simple_server.make_server(cfg.bind, cfg.port, self.app)
+        httpd.serve_forever()

tools/pip-requires

@@ -1,7 +1,6 @@
 cliff
 eventlet>=0.9.12
 falcon>=0.1.4
-gunicorn
 iso8601>=0.1.4
 msgpack-python
 oslo.config>=1.1.0