Adds read_only middleware

This patch adds a read_only middleware to swift. It gives the ability
to make an entire cluster or individual accounts read only.
When a cluster or an account is in read only mode, requests that would
result in writes to the cluser are not allowed.

DocImpact

Change-Id: I7e0743aecd60b171bbcefcc8b6e1f3fd4cef2478

doc/source/logs.rst

@@ -102,6 +102,7 @@ DLO                     :ref:`dynamic-large-objects`
 LE                      :ref:`list_endpoints`
 KS                      :ref:`keystoneauth`
 RL                      :ref:`ratelimit`
+RO                      :ref:`read_only`
 VW                      :ref:`versioned_writes`
 SSC                     :ref:`copy`
 SYM                     :ref:`symlink`

doc/source/middleware.rst

@@ -299,10 +299,19 @@ Ratelimit
     :members:
     :show-inheritance:
 
+.. _read_only:
+
+Read Only
+=========
+
+.. automodule:: swift.common.middleware.read_only
+    :members:
+    :show-inheritance:
+
 .. _recon:
 
 Recon
-===========
+=====
 
 .. automodule:: swift.common.middleware.recon
     :members:

etc/proxy-server.conf-sample

@@ -679,6 +679,14 @@ use = egg:swift#ratelimit
 # container_listing_ratelimit_10 = 50
 # container_listing_ratelimit_50 = 20
 
+[filter:read_only]
+use = egg:swift#read_only
+# read_only set to true means turn global read only on
+# read_only = false
+# allow_deletes set to true means to allow deletes
+# allow_deletes = false
+# Note: Put after ratelimit in the pipeline.
+
 [filter:domain_remap]
 use = egg:swift#domain_remap
 # You can override the default log routing for this filter here:

setup.cfg

@@ -86,6 +86,7 @@ paste.filter_factory =
     healthcheck = swift.common.middleware.healthcheck:filter_factory
     crossdomain = swift.common.middleware.crossdomain:filter_factory
     memcache = swift.common.middleware.memcache:filter_factory
+    read_only = swift.common.middleware.read_only:filter_factory
     ratelimit = swift.common.middleware.ratelimit:filter_factory
     cname_lookup = swift.common.middleware.cname_lookup:filter_factory
     catch_errors = swift.common.middleware.catch_errors:filter_factory

swift/common/middleware/read_only.py

@@ -0,0 +1,120 @@
+# Copyright (c) 2010-2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from swift.common.constraints import check_account_format
+from swift.common.swob import HTTPMethodNotAllowed, Request
+from swift.common.utils import get_logger, config_true_value
+from swift.proxy.controllers.base import get_info
+
+"""
+=========
+Read Only
+=========
+
+The ability to make an entire cluster or individual accounts read only is
+implemented as pluggable middleware.  When a cluster or an account is in read
+only mode, requests that would result in writes to the cluser are not allowed.
+A 405 is returned on such requests.  "COPY", "DELETE", "POST", and
+"PUT" are the HTTP methods that are considered writes.
+
+-------------
+Configuration
+-------------
+
+All configuration is optional.
+
+============= ======= ====================================================
+Option        Default Description
+------------- ------- ----------------------------------------------------
+read_only     false   Set to 'true' to put the entire cluster in read only
+                      mode.
+allow_deletes false   Set to 'true' to allow deletes.
+============= ======= ====================================================
+
+---------------------------
+Marking Individual Accounts
+---------------------------
+
+If a system administrator wants to mark individual accounts as read only,
+he/she can set X-Account-Sysmeta-Read-Only on an account to 'true'.
+
+If a system administrator wants to allow writes to individual accounts,
+when a cluster is in read only mode, he/she can set
+X-Account-Sysmeta-Read-Only on an account to 'false'.
+
+This header will be hidden from the user, because of the gatekeeper middleware,
+and can only be set using a direct client to the account nodes.
+"""
+
+
+class ReadOnlyMiddleware(object):
+    """
+    Middleware that make an entire cluster or individual accounts read only.
+    """
+
+    def __init__(self, app, conf, logger=None):
+        self.app = app
+        self.logger = logger or get_logger(conf, log_route='read_only')
+        self.read_only = config_true_value(conf.get('read_only'))
+        self.write_methods = ['COPY', 'POST', 'PUT']
+        if not config_true_value(conf.get('allow_deletes')):
+            self.write_methods += ['DELETE']
+
+    def __call__(self, env, start_response):
+        req = Request(env)
+
+        if req.method not in self.write_methods:
+            return self.app(env, start_response)
+
+        try:
+            version, account, container, obj = req.split_path(1, 4, True)
+        except ValueError:
+            return self.app(env, start_response)
+
+        if req.method == 'COPY' and 'Destination-Account' in req.headers:
+            dest_account = req.headers.get('Destination-Account')
+            account = check_account_format(req, dest_account)
+
+        account_read_only = self.account_read_only(req, account)
+        if account_read_only is False:
+            return self.app(env, start_response)
+
+        if self.read_only or account_read_only:
+            return HTTPMethodNotAllowed()(env, start_response)
+
+        return self.app(env, start_response)
+
+    def account_read_only(self, req, account):
+        """
+        Returns None if X-Account-Sysmeta-Read-Only is not set.
+        Returns True or False otherwise.
+        """
+        info = get_info(self.app, req.environ, account, swift_source='RO')
+        read_only = info.get('sysmeta', {}).get('read-only', '')
+        if read_only == '':
+            return None
+        return config_true_value(read_only)
+
+
+def filter_factory(global_conf, **local_conf):
+    """
+    paste.deploy app factory for creating WSGI proxy apps.
+    """
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def read_only_filter(app):
+        return ReadOnlyMiddleware(app, conf)
+
+    return read_only_filter

test/unit/common/middleware/test_read_only.py

@@ -0,0 +1,253 @@
+# Copyright (c) 2010-2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import unittest
+
+from swift.common.middleware import read_only
+from swift.common.swob import Request
+from test.unit import FakeLogger
+
+
+class FakeApp(object):
+    def __call__(self, env, start_response):
+        return ['204 No Content']
+
+
+def start_response(*args):
+    pass
+
+
+read_methods = 'GET HEAD'.split()
+write_methods = 'COPY DELETE POST PUT'.split()
+ro_resp = ['<html><h1>Method Not Allowed</h1><p>The method is not ' +
+           'allowed for this resource.</p></html>']
+
+
+class TestReadOnly(unittest.TestCase):
+    def test_global_read_only_off(self):
+        conf = {
+            'read_only': 'false',
+        }
+
+        ro = read_only.filter_factory(conf)(FakeApp())
+        ro.logger = FakeLogger()
+
+        with mock.patch('swift.common.middleware.read_only.get_info',
+                        return_value={}):
+            for method in read_methods + write_methods:
+                req = Request.blank('/v/a')
+                req.method = method
+                resp = ro(req.environ, start_response)
+                self.assertTrue(resp[0].startswith('204'))
+
+    def test_global_read_only_on(self):
+        conf = {
+            'read_only': 'true',
+        }
+
+        ro = read_only.filter_factory(conf)(FakeApp())
+        ro.logger = FakeLogger()
+
+        with mock.patch('swift.common.middleware.read_only.get_info',
+                        return_value={}):
+            for method in read_methods:
+                req = Request.blank('/v/a')
+                req.method = method
+                resp = ro(req.environ, start_response)
+                self.assertTrue(resp[0].startswith('204'))
+
+            for method in write_methods:
+                req = Request.blank('/v/a')
+                req.method = method
+                resp = ro(req.environ, start_response)
+                self.assertEqual(ro_resp, resp)
+
+    def test_account_read_only_on(self):
+        conf = {}
+
+        ro = read_only.filter_factory(conf)(FakeApp())
+        ro.logger = FakeLogger()
+
+        def get_fake_read_only(*args, **kwargs):
+            return {'sysmeta': {'read-only': 'true'}}
+
+        with mock.patch('swift.common.middleware.read_only.get_info',
+                        get_fake_read_only):
+            for method in read_methods:
+                req = Request.blank('/v/a')
+                req.method = method
+                resp = ro(req.environ, start_response)
+                self.assertTrue(resp[0].startswith('204'))
+
+            for method in write_methods:
+                req = Request.blank('/v/a')
+                req.method = method
+                resp = ro(req.environ, start_response)
+                self.assertEqual(ro_resp, resp)
+
+    def test_account_read_only_off(self):
+        conf = {}
+
+        ro = read_only.filter_factory(conf)(FakeApp())
+        ro.logger = FakeLogger()
+
+        def get_fake_read_only(*args, **kwargs):
+            return {'sysmeta': {'read-only': 'false'}}
+
+        with mock.patch('swift.common.middleware.read_only.get_info',
+                        get_fake_read_only):
+            for method in read_methods + write_methods:
+                req = Request.blank('/v/a')
+                req.method = method
+                resp = ro(req.environ, start_response)
+                self.assertTrue(resp[0].startswith('204'))
+
+    def test_global_read_only_on_account_off(self):
+        conf = {
+            'read_only': 'true',
+        }
+
+        ro = read_only.filter_factory(conf)(FakeApp())
+        ro.logger = FakeLogger()
+
+        def get_fake_read_only(*args, **kwargs):
+            return {'sysmeta': {'read-only': 'false'}}
+
+        with mock.patch('swift.common.middleware.read_only.get_info',
+                        get_fake_read_only):
+            for method in read_methods + write_methods:
+                req = Request.blank('/v/a')
+                req.method = method
+                resp = ro(req.environ, start_response)
+                self.assertTrue(resp[0].startswith('204'))
+
+    def test_global_read_only_on_allow_deletes(self):
+        conf = {
+            'read_only': 'true',
+            'allow_deletes': 'true',
+        }
+
+        ro = read_only.filter_factory(conf)(FakeApp())
+        ro.logger = FakeLogger()
+
+        with mock.patch('swift.common.middleware.read_only.get_info',
+                        return_value={}):
+            req = Request.blank('/v/a')
+            req.method = "DELETE"
+            resp = ro(req.environ, start_response)
+            self.assertTrue(resp[0].startswith('204'))
+
+    def test_account_read_only_on_allow_deletes(self):
+        conf = {
+            'allow_deletes': 'true',
+        }
+
+        ro = read_only.filter_factory(conf)(FakeApp())
+        ro.logger = FakeLogger()
+
+        def get_fake_read_only(*args, **kwargs):
+            return {'sysmeta': {'read-only': 'on'}}
+
+        with mock.patch('swift.common.middleware.read_only.get_info',
+                        get_fake_read_only):
+            req = Request.blank('/v/a')
+            req.method = "DELETE"
+            resp = ro(req.environ, start_response)
+            self.assertTrue(resp[0].startswith('204'))
+
+    def test_global_read_only_on_destination_account_off_on_copy(self):
+        conf = {
+            'read_only': 'true',
+        }
+
+        ro = read_only.filter_factory(conf)(FakeApp())
+        ro.logger = FakeLogger()
+
+        def get_fake_read_only(*args, **kwargs):
+            if 'b' in args:
+                return {'sysmeta': {'read-only': 'false'}}
+            return {}
+
+        with mock.patch('swift.common.middleware.read_only.get_info',
+                        get_fake_read_only):
+            headers = {'Destination-Account': 'b'}
+            req = Request.blank('/v/a', headers=headers)
+            req.method = "COPY"
+            resp = ro(req.environ, start_response)
+            self.assertTrue(resp[0].startswith('204'))
+
+    def test_global_read_only_off_destination_account_on_on_copy(self):
+        conf = {}
+
+        ro = read_only.filter_factory(conf)(FakeApp())
+        ro.logger = FakeLogger()
+
+        def get_fake_read_only(*args, **kwargs):
+            if 'b' in args:
+                return {'sysmeta': {'read-only': 'true'}}
+            return {}
+
+        with mock.patch('swift.common.middleware.read_only.get_info',
+                        get_fake_read_only):
+            headers = {'Destination-Account': 'b'}
+            req = Request.blank('/v/a', headers=headers)
+            req.method = "COPY"
+            resp = ro(req.environ, start_response)
+            self.assertEqual(ro_resp, resp)
+
+    def test_global_read_only_off_src_acct_on_dest_acct_off_on_copy(self):
+        conf = {}
+
+        ro = read_only.filter_factory(conf)(FakeApp())
+        ro.logger = FakeLogger()
+
+        def fake_account_read_only(self, req, account):
+            if account == 'a':
+                return 'on'
+            return ''
+
+        with mock.patch(
+                'swift.common.middleware.read_only.ReadOnlyMiddleware.' +
+                'account_read_only',
+                fake_account_read_only):
+            headers = {'Destination-Account': 'b'}
+            req = Request.blank('/v/a', headers=headers)
+            req.method = "COPY"
+            resp = ro(req.environ, start_response)
+            self.assertTrue(resp[0].startswith('204'))
+
+    def test_global_read_only_off_src_acct_on_dest_acct_on_on_copy(self):
+        conf = {}
+
+        ro = read_only.filter_factory(conf)(FakeApp())
+        ro.logger = FakeLogger()
+
+        def fake_account_read_only(*args, **kwargs):
+            return 'true'
+
+        with mock.patch(
+                'swift.common.middleware.read_only.ReadOnlyMiddleware.' +
+                'account_read_only',
+                fake_account_read_only):
+            headers = {'Destination-Account': 'b'}
+            req = Request.blank('/v/a', headers=headers)
+            req.method = "COPY"
+            resp = ro(req.environ, start_response)
+            self.assertEqual(ro_resp, resp)
+
+
+if __name__ == '__main__':
+    unittest.main()