From e022738734c94c963bbe228a3fd7b2717518f6d2 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Thu, 28 Oct 2010 16:39:42 -0500 Subject: [PATCH 01/15] Added link to RHEL instructions in SAIO, added Google Analytics code --- doc/source/_templates/layout.html | 16 ++++++++++++++++ doc/source/development_saio.rst | 3 +++ doc/source/getting_started.rst | 3 +-- doc/source/index.rst | 2 +- doc/source/ratelimit.rst | 2 +- 5 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 doc/source/_templates/layout.html diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html new file mode 100644 index 0000000000..d170ec41d6 --- /dev/null +++ b/doc/source/_templates/layout.html @@ -0,0 +1,16 @@ +{% extends "!layout.html" %} + +{% block footer %} +{{ super() }} + + +{% endblock %} diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 605081e015..3b031bf6b9 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -17,6 +17,9 @@ virtual machine will emulate running a four node Swift cluster. * Create guest virtual machine from the Ubuntu image. +Additional information about setting up a Swift development snapshot on Fedora, CentOS, or RHEL (Red Hat Enterprise Linux) is available on +the wiki at http://wiki.openstack.org/RhelInstructions. + ----------------------------------------- Installing dependencies and the core code ----------------------------------------- diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 4ce0d934c0..7c7bfc4c16 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -37,5 +37,4 @@ Production ---------- We do not have documentation yet on how to set up and configure Swift for a -production cluster, but hope to begin work on those soon. - +production cluster, but hope to begin work on those soon. \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 66f4d1cc7a..7ebd61ff10 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -40,7 +40,7 @@ Deployment: .. toctree:: :maxdepth: 1 - + deployment_guide admin_guide debian_package_guide diff --git a/doc/source/ratelimit.rst b/doc/source/ratelimit.rst index 37785a9565..80db870773 100644 --- a/doc/source/ratelimit.rst +++ b/doc/source/ratelimit.rst @@ -4,7 +4,7 @@ Rate Limiting Rate limiting in swift is implemented as a pluggable middleware. Rate limiting is performed on requests that result in database writes to the -account and container sqlite dbs. It uses memcached and is dependant on +account and container sqlite dbs. It uses memcached and is dependent on the proxy servers having highly synchronized time. The rate limits are limited by the accuracy of the proxy server clocks. From a6ccc44f893038d12d62a0409849b78ed188d5a6 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Thu, 28 Oct 2010 17:01:50 -0500 Subject: [PATCH 02/15] Updated SAIO to point to RHEL instructions on wiki --- doc/source/development_saio.rst | 3 +++ doc/source/getting_started.rst | 3 +-- doc/source/index.rst | 2 +- doc/source/ratelimit.rst | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 605081e015..3b031bf6b9 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -17,6 +17,9 @@ virtual machine will emulate running a four node Swift cluster. * Create guest virtual machine from the Ubuntu image. +Additional information about setting up a Swift development snapshot on Fedora, CentOS, or RHEL (Red Hat Enterprise Linux) is available on +the wiki at http://wiki.openstack.org/RhelInstructions. + ----------------------------------------- Installing dependencies and the core code ----------------------------------------- diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 4ce0d934c0..7c7bfc4c16 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -37,5 +37,4 @@ Production ---------- We do not have documentation yet on how to set up and configure Swift for a -production cluster, but hope to begin work on those soon. - +production cluster, but hope to begin work on those soon. \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 66f4d1cc7a..7ebd61ff10 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -40,7 +40,7 @@ Deployment: .. toctree:: :maxdepth: 1 - + deployment_guide admin_guide debian_package_guide diff --git a/doc/source/ratelimit.rst b/doc/source/ratelimit.rst index 37785a9565..80db870773 100644 --- a/doc/source/ratelimit.rst +++ b/doc/source/ratelimit.rst @@ -4,7 +4,7 @@ Rate Limiting Rate limiting in swift is implemented as a pluggable middleware. Rate limiting is performed on requests that result in database writes to the -account and container sqlite dbs. It uses memcached and is dependant on +account and container sqlite dbs. It uses memcached and is dependent on the proxy servers having highly synchronized time. The rate limits are limited by the accuracy of the proxy server clocks. From 58f337236096fba28a03023a4aca4850f3734279 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Fri, 29 Oct 2010 01:23:01 +0000 Subject: [PATCH 03/15] shore up accept header parsing --- swift/account/server.py | 12 +-- swift/container/server.py | 13 +-- test/unit/container/test_server.py | 127 ++++++++++++++++++++--------- 3 files changed, 100 insertions(+), 52 deletions(-) diff --git a/swift/account/server.py b/swift/account/server.py index 18bd3f212b..a51bf64054 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -199,12 +199,12 @@ class AccountController(object): except UnicodeDecodeError, err: return HTTPBadRequest(body='parameters not utf8', content_type='text/plain', request=req) - header_format = req.accept.first_match(['text/plain', - 'application/json', - 'application/xml']) - format = query_format if query_format else header_format - if format.startswith('application/'): - format = format[12:] + header_format = req.accept.best_match(['text/plain', + 'application/json', + 'application/xml']) + format = query_format or header_format or 'text/plain' + if '/' in format: + format = format.split('/')[-1] account_list = broker.list_containers_iter(limit, marker, prefix, delimiter) if format == 'json': diff --git a/swift/container/server.py b/swift/container/server.py index ff8dc76684..8ec5573213 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -278,14 +278,15 @@ class ContainerController(object): except UnicodeDecodeError, err: return HTTPBadRequest(body='parameters not utf8', content_type='text/plain', request=req) - header_format = req.accept.first_match(['text/plain', - 'application/json', - 'application/xml']) - format = query_format if query_format else header_format - if format.startswith('application/'): - format = format[12:] + header_format = req.accept.best_match(['text/plain', + 'application/json', + 'application/xml']) + format = query_format or header_format or 'text/plain' + if '/' in format: + format = format.split('/')[-1] container_list = broker.list_objects_iter(limit, marker, prefix, delimiter, path) + print "format is %s" % format if format == 'json': out_content_type = 'application/json' json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s', diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 7aaee688e8..8a2d79c926 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -479,7 +479,7 @@ class TestContainerController(unittest.TestCase): resp = self.controller.GET(req) self.assertEquals(resp.status_int, 412) - def test_GET_format(self): + def test_GET_json(self): # make a container req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) @@ -514,8 +514,80 @@ class TestContainerController(unittest.TestCase): "bytes":0, "content_type":"text/plain", "last_modified":"1970-01-01T00:00:01"}] + req = Request.blank('/sda1/p/a/c?format=json', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'application/json') + result = eval(resp.body) + self.assertEquals(result, json_body) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'application/json' + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'application/json') + result = eval(resp.body) + self.assertEquals(result, json_body) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'application/*' + resp = self.controller.GET(req) + result = eval(resp.body) + self.assertEquals(result, json_body) + + def test_GET_plain(self): + # make a container + req = Request.blank('/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + # fill the container + for i in range(3): + req = Request.blank('/sda1/p/a/plainc/%s'%i, environ= + {'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', + 'HTTP_X_CONTENT_TYPE': 'text/plain', + 'HTTP_X_ETAG': 'x', + 'HTTP_X_SIZE': 0}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + plain_body = '0\n1\n2\n' + + req = Request.blank('/sda1/p/a/plainc', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'text/plain') + self.assertEquals(resp.body, plain_body) + + for accept in ('', 'text/plain', 'application/xml;q=0.8,*/*;q=0.9', + '*/*;q=0.9,application/xml;q=0.8', '*/*'): + req = Request.blank('/sda1/p/a/plainc', + environ={'REQUEST_METHOD': 'GET'}) + req.accept = accept + resp = self.controller.GET(req) + self.assertEquals(resp.body, plain_body) + self.assertEquals(resp.content_type, 'text/plain') + + # test conflicting formats + req = Request.blank('/sda1/p/a/plainc?format=plain', + environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'application/json' + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'text/plain') + self.assertEquals(resp.body, plain_body) + + def test_GET_xml(self): + # make a container + req = Request.blank('/sda1/p/a/xmlc', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + # fill the container + for i in range(3): + req = Request.blank('/sda1/p/a/xmlc/%s'%i, environ= + {'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', + 'HTTP_X_CONTENT_TYPE': 'text/plain', + 'HTTP_X_ETAG': 'x', + 'HTTP_X_SIZE': 0}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) xml_body = '\n' \ - '' \ + '' \ '0x0' \ 'text/plain' \ '1970-01-01T00:00:01' \ @@ -529,46 +601,21 @@ class TestContainerController(unittest.TestCase): '1970-01-01T00:00:01' \ '' \ '' - plain_body = '0\n1\n2\n' - req = Request.blank('/sda1/p/a/c?format=json', environ={'REQUEST_METHOD': 'GET'}) - resp = self.controller.GET(req) - self.assertEquals(resp.content_type, 'application/json') - result = eval(resp.body) - self.assertEquals(result, json_body) - req = Request.blank('/sda1/p/a/c?format=xml', environ={'REQUEST_METHOD': 'GET'}) + # tests + req = Request.blank('/sda1/p/a/xmlc?format=xml', + environ={'REQUEST_METHOD': 'GET'}) resp = self.controller.GET(req) self.assertEquals(resp.content_type, 'application/xml') - result = resp.body - self.assertEquals(result, xml_body) - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) - req.accept = 'application/json' - resp = self.controller.GET(req) - self.assertEquals(resp.content_type, 'application/json') - result = eval(resp.body) - self.assertEquals(result, json_body) - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) - req.accept = '*/*' - resp = self.controller.GET(req) - self.assertEquals(resp.content_type, 'text/plain') - result = resp.body - self.assertEquals(result, plain_body) - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) - req.accept = 'application/*' - resp = self.controller.GET(req) - result = eval(resp.body) - self.assertEquals(result, json_body) - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) - req.accept = 'application/xml' - resp = self.controller.GET(req) - result = resp.body - self.assertEquals(result, xml_body) - # test conflicting formats - req = Request.blank('/sda1/p/a/c?format=plain', environ={'REQUEST_METHOD': 'GET'}) - req.accept = 'application/json' - resp = self.controller.GET(req) - self.assertEquals(resp.content_type, 'text/plain') - result = resp.body - self.assertEquals(result, plain_body) + self.assertEquals(resp.body, xml_body) + + for xml_accept in ('application/xml', 'application/xml;q=1.0,*/*;q=0.9', + '*/*;q=0.9,application/xml;q=1.0'): + req = Request.blank('/sda1/p/a/xmlc', + environ={'REQUEST_METHOD': 'GET'}) + req.accept = xml_accept + resp = self.controller.GET(req) + self.assertEquals(resp.body, xml_body) + self.assertEquals(resp.content_type, 'application/xml') def test_GET_marker(self): # make a container From 655dba5cb28b3459fb4b73418f784332bf7ddc20 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Fri, 29 Oct 2010 01:26:22 +0000 Subject: [PATCH 04/15] missed a debug print --- swift/container/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/swift/container/server.py b/swift/container/server.py index 8ec5573213..7656b6fc75 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -286,7 +286,6 @@ class ContainerController(object): format = format.split('/')[-1] container_list = broker.list_objects_iter(limit, marker, prefix, delimiter, path) - print "format is %s" % format if format == 'json': out_content_type = 'application/json' json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s', From cefcd568cda1b012cafda8baf1262e179734c10e Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Fri, 29 Oct 2010 10:28:19 +0000 Subject: [PATCH 05/15] more tests --- test/unit/container/test_server.py | 60 ++++++++++++++++++------------ 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 8a2d79c926..a80090185c 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -481,16 +481,18 @@ class TestContainerController(unittest.TestCase): def test_GET_json(self): # make a container - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + req = Request.blank('/sda1/p/a/jsonc', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = self.controller.PUT(req) # test an empty container - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req = Request.blank('/sda1/p/a/jsonc?format=json', + environ={'REQUEST_METHOD': 'GET'}) resp = self.controller.GET(req) - self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.status_int, 200) + self.assertEquals(eval(resp.body), []) # fill the container for i in range(3): - req = Request.blank('/sda1/p/a/c/%s'%i, environ= + req = Request.blank('/sda1/p/a/jsonc/%s'%i, environ= {'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', @@ -514,28 +516,33 @@ class TestContainerController(unittest.TestCase): "bytes":0, "content_type":"text/plain", "last_modified":"1970-01-01T00:00:01"}] - req = Request.blank('/sda1/p/a/c?format=json', environ={'REQUEST_METHOD': 'GET'}) + + req = Request.blank('/sda1/p/a/jsonc?format=json', + environ={'REQUEST_METHOD': 'GET'}) resp = self.controller.GET(req) self.assertEquals(resp.content_type, 'application/json') - result = eval(resp.body) - self.assertEquals(result, json_body) - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) - req.accept = 'application/json' - resp = self.controller.GET(req) - self.assertEquals(resp.content_type, 'application/json') - result = eval(resp.body) - self.assertEquals(result, json_body) - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) - req.accept = 'application/*' - resp = self.controller.GET(req) - result = eval(resp.body) - self.assertEquals(result, json_body) + self.assertEquals(eval(resp.body), json_body) + + for accept in ('application/json', 'application/json;q=1.0,*/*;q=0.9', + '*/*;q=0.9,application/json;q=1.0', 'application/*'): + req = Request.blank('/sda1/p/a/jsonc', + environ={'REQUEST_METHOD': 'GET'}) + req.accept = accept + resp = self.controller.GET(req) + self.assertEquals(eval(resp.body), json_body, + 'Invalid body for Accept: %s' % accept) + self.assertEquals(resp.content_type, 'application/json', + 'Invalid content_type for Accept: %s' % accept) def test_GET_plain(self): # make a container req = Request.blank('/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = self.controller.PUT(req) + # test an empty container + req = Request.blank('/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) # fill the container for i in range(3): req = Request.blank('/sda1/p/a/plainc/%s'%i, environ= @@ -555,13 +562,16 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.body, plain_body) for accept in ('', 'text/plain', 'application/xml;q=0.8,*/*;q=0.9', - '*/*;q=0.9,application/xml;q=0.8', '*/*'): + '*/*;q=0.9,application/xml;q=0.8', '*/*', + 'text/plain,application/xml'): req = Request.blank('/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'GET'}) req.accept = accept resp = self.controller.GET(req) - self.assertEquals(resp.body, plain_body) - self.assertEquals(resp.content_type, 'text/plain') + self.assertEquals(resp.body, plain_body, + 'Invalid body for Accept: %s' % accept) + self.assertEquals(resp.content_type, 'text/plain', + 'Invalid content_type for Accept: %s' % accept) # test conflicting formats req = Request.blank('/sda1/p/a/plainc?format=plain', @@ -609,13 +619,15 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.body, xml_body) for xml_accept in ('application/xml', 'application/xml;q=1.0,*/*;q=0.9', - '*/*;q=0.9,application/xml;q=1.0'): + '*/*;q=0.9,application/xml;q=1.0'): req = Request.blank('/sda1/p/a/xmlc', environ={'REQUEST_METHOD': 'GET'}) req.accept = xml_accept resp = self.controller.GET(req) - self.assertEquals(resp.body, xml_body) - self.assertEquals(resp.content_type, 'application/xml') + self.assertEquals(resp.body, xml_body, + 'Invalid body for Accept: %s' % xml_accept) + self.assertEquals(resp.content_type, 'application/xml', + 'Invalid content_type for Accept: %s' % xml_accept) def test_GET_marker(self): # make a container From 1fc40d6c296d36da5d5c48c7c2aba537ed988f1c Mon Sep 17 00:00:00 2001 From: David Goetz Date: Fri, 29 Oct 2010 13:30:34 -0700 Subject: [PATCH 06/15] catching invalid urls and adding tests --- swift/common/middleware/ratelimit.py | 6 +++++- swift/common/utils.py | 2 ++ test/functional/tests.py | 9 +++++++++ test/unit/common/middleware/test_ratelimit.py | 20 +++++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index b13a4a6ab4..707e544ae2 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -14,6 +14,7 @@ import time import eventlet from webob import Request, Response +from webob.exc import HTTPNotFound from swift.common.utils import split_path, cache_from_env, get_logger from swift.proxy.server import get_container_memcache_key @@ -204,7 +205,10 @@ class RateLimitMiddleware(object): req = Request(env) if self.memcache_client is None: self.memcache_client = cache_from_env(env) - version, account, container, obj = split_path(req.path, 1, 4, True) + try: + version, account, container, obj = split_path(req.path, 1, 4, True) + except ValueError: + return HTTPNotFound()(env, start_response) ratelimit_resp = self.handle_ratelimit(req, account, container, obj) if ratelimit_resp is None: return self.app(env, start_response) diff --git a/swift/common/utils.py b/swift/common/utils.py index 5a8bf0e1be..bb635725c8 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -208,6 +208,7 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): trailing data, raises ValueError. :returns: list of segments with a length of maxsegs (non-existant segments will return as None) + :raises: ValueError if given an invalid path """ if not maxsegs: maxsegs = minsegs @@ -622,6 +623,7 @@ def write_pickle(obj, dest, tmp): os.fsync(fd) renamer(tmppath, dest) + def audit_location_generator(devices, datadir, mount_check=True, logger=None): ''' Given a devices path and a data directory, yield (path, device, diff --git a/test/functional/tests.py b/test/functional/tests.py index 6a28d9bb3e..f1ea6232b0 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -170,6 +170,15 @@ class TestAccount(Base): self.assert_status(412) self.assert_body('Bad URL') + def testInvalidPath(self): + was_url = self.env.account.conn.storage_url + self.env.account.conn.storage_url = "/%s" % was_url + self.env.account.conn.make_request('GET') + try: + self.assert_status(404) + finally: + self.env.account.conn.storage_url = was_url + def testPUT(self): self.env.account.conn.make_request('PUT') self.assert_status([403, 405]) diff --git a/test/unit/common/middleware/test_ratelimit.py b/test/unit/common/middleware/test_ratelimit.py index 7bf4a9e445..2f709c4a41 100644 --- a/test/unit/common/middleware/test_ratelimit.py +++ b/test/unit/common/middleware/test_ratelimit.py @@ -366,6 +366,26 @@ class TestRateLimit(unittest.TestCase): time_took = time.time() - begin self.assert_(round(time_took, 1) == .4) + def test_call_invalid_path(self): + env = {'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '//v1/AUTH_1234567890', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '80', + 'swift.cache': FakeMemcache(), + 'SERVER_PROTOCOL': 'HTTP/1.0'} + + app = lambda *args, **kwargs: None + rate_mid = ratelimit.RateLimitMiddleware(app, {}, + logger=FakeLogger()) + + class a_callable(object): + + def __call__(self, *args, **kwargs): + pass + resp = rate_mid.__call__(env, a_callable()) + self.assert_('404 Not Found' in resp[0]) + if __name__ == '__main__': unittest.main() From e03d5bef1f4d3028dcb77c1cb3381ce1c8aeb281 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Fri, 29 Oct 2010 16:30:56 -0500 Subject: [PATCH 07/15] This contains an env variable in conf.py for the build to contain the GA code --- doc/source/{_templates => _ga}/layout.html | 1 + doc/source/conf.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) rename doc/source/{_templates => _ga}/layout.html (99%) diff --git a/doc/source/_templates/layout.html b/doc/source/_ga/layout.html similarity index 99% rename from doc/source/_templates/layout.html rename to doc/source/_ga/layout.html index d170ec41d6..0b72a77ac2 100644 --- a/doc/source/_templates/layout.html +++ b/doc/source/_ga/layout.html @@ -14,3 +14,4 @@ pageTracker._setAllowLinker(true); pageTracker._trackPageview(); } catch(err) {} {% endblock %} + diff --git a/doc/source/conf.py b/doc/source/conf.py index 91596d6b4e..acff18d29b 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -41,7 +41,13 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', todo_include_todos = True # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +# Changing the path so that the Hudson build output contains GA code and the source +# docs do not contain the code so local, offline sphinx builds are "clean." +templates_path = [] +if os.getenv('HUDSON_PUBLISH_DOCS'): + templates_path = ['_ga', '_templates'] +else: + templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' From cc7376c7232416bc41945a7671836ac122bde259 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Mon, 1 Nov 2010 15:04:51 -0500 Subject: [PATCH 08/15] Changing to generic wiki page so no rhel in url --- doc/source/development_saio.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 3b031bf6b9..3a615074ce 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -17,8 +17,8 @@ virtual machine will emulate running a four node Swift cluster. * Create guest virtual machine from the Ubuntu image. -Additional information about setting up a Swift development snapshot on Fedora, CentOS, or RHEL (Red Hat Enterprise Linux) is available on -the wiki at http://wiki.openstack.org/RhelInstructions. +Additional information about setting up a Swift development snapshot on other distributions is +available on the wiki at http://wiki.openstack.org/SAIOInstructions. ----------------------------------------- Installing dependencies and the core code From 4b4a0997a4b0d5803331a429dc09a39b8e452af0 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Mon, 1 Nov 2010 13:26:18 -0700 Subject: [PATCH 09/15] adding new exception type and catching other invalid paths --- swift/account/server.py | 19 +++++++++++------ swift/auth/server.py | 7 ++++-- swift/common/exceptions.py | 3 +++ swift/common/middleware/auth.py | 13 +++++++---- swift/common/middleware/ratelimit.py | 4 ++-- swift/common/utils.py | 12 ++++++----- swift/container/server.py | 14 ++++++------ swift/obj/server.py | 20 ++++++++++------- swift/proxy/server.py | 8 +++++-- swift/stats/access_processor.py | 12 +++++++---- test/unit/auth/test_server.py | 3 ++- test/unit/common/test_utils.py | 32 ++++++++++++++-------------- 12 files changed, 89 insertions(+), 58 deletions(-) diff --git a/swift/account/server.py b/swift/account/server.py index 18bd3f212b..1c29ea84d1 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -34,7 +34,7 @@ from swift.common.utils import get_logger, get_param, hash_path, \ from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.db_replicator import ReplicatorRpc - +from swift.common.exceptions import InvalidPathError DATADIR = 'accounts' @@ -60,7 +60,7 @@ class AccountController(object): """Handle HTTP DELETE request.""" try: drive, part, account = split_path(unquote(req.path), 3) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if self.mount_check and not check_mount(self.root, drive): @@ -77,7 +77,12 @@ class AccountController(object): def PUT(self, req): """Handle HTTP PUT request.""" - drive, part, account, container = split_path(unquote(req.path), 3, 4) + try: + drive, part, account, container = split_path(unquote(req.path), + 3, 4) + except InvalidPathError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain', + request=req) if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) broker = self._get_account_broker(drive, part, account) @@ -130,7 +135,7 @@ class AccountController(object): try: drive, part, account, container = split_path(unquote(req.path), 3, 4) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if self.mount_check and not check_mount(self.root, drive): @@ -161,7 +166,7 @@ class AccountController(object): """Handle HTTP GET request.""" try: drive, part, account = split_path(unquote(req.path), 3) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if self.mount_check and not check_mount(self.root, drive): @@ -252,7 +257,7 @@ class AccountController(object): """ try: post_args = split_path(unquote(req.path), 3) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) drive, partition, hash = post_args @@ -270,7 +275,7 @@ class AccountController(object): """Handle HTTP POST request.""" try: drive, part, account = split_path(unquote(req.path), 3) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if 'x-timestamp' not in req.headers or \ diff --git a/swift/auth/server.py b/swift/auth/server.py index 06e755f767..24c1f9ce9c 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -30,6 +30,7 @@ from webob.exc import HTTPBadRequest, HTTPConflict, HTTPForbidden, \ from swift.common.bufferedhttp import http_connect_raw as http_connect from swift.common.db import get_db_connection from swift.common.utils import get_logger, split_path +from swift.common.exceptions import InvalidPathError class AuthController(object): @@ -415,7 +416,7 @@ YOU HAVE A FEW OPTIONS: """ try: _, token = split_path(request.path, minsegs=2) - except ValueError: + except InvalidPathError: return HTTPBadRequest() # Retrieves (TTL, account, user, cfaccount) if valid, False otherwise validation = self.validate_token(token) @@ -451,7 +452,7 @@ YOU HAVE A FEW OPTIONS: """ try: _, account_name, user_name = split_path(request.path, minsegs=3) - except ValueError: + except InvalidPathError: return HTTPBadRequest() create_reseller_admin = \ request.headers.get('x-auth-user-reseller-admin') == 'true' @@ -607,6 +608,8 @@ YOU HAVE A FEW OPTIONS: else: return HTTPBadRequest(request=env)(env, start_response) response = handler(req) + except InvalidPathError: + return HTTPNotFound()(env, start_response) except: self.logger.exception('ERROR Unhandled exception in ReST request') return HTTPServiceUnavailable(request=req)(env, start_response) diff --git a/swift/common/exceptions.py b/swift/common/exceptions.py index c498e2dae9..504d8f7788 100644 --- a/swift/common/exceptions.py +++ b/swift/common/exceptions.py @@ -52,3 +52,6 @@ class DriveNotMounted(Exception): class LockTimeout(MessageTimeout): pass + +class InvalidPathError(Exception): + pass diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py index a4dd5ac7df..922e6277f7 100644 --- a/swift/common/middleware/auth.py +++ b/swift/common/middleware/auth.py @@ -16,12 +16,12 @@ from time import time from eventlet.timeout import Timeout -from webob.exc import HTTPForbidden, HTTPUnauthorized +from webob.exc import HTTPForbidden, HTTPUnauthorized, HTTPNotFound from swift.common.bufferedhttp import http_connect_raw as http_connect from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed from swift.common.utils import cache_from_env, split_path, TRUE_VALUES - +from swift.common.exceptions import InvalidPathError class DevAuth(object): """Auth Middleware that uses the dev auth server.""" @@ -82,8 +82,11 @@ class DevAuth(object): # With a non-empty reseller_prefix, I would like to be called # back for anonymous access to accounts I know I'm the # definitive auth for. - version, rest = split_path(env.get('PATH_INFO', ''), - 1, 2, True) + try: + version, rest = split_path(env.get('PATH_INFO', ''), + 1, 2, True) + except InvalidPathError, err: + return HTTPNotFound()(env, start_response) if rest and rest.startswith(self.reseller_prefix): # Handle anonymous access to accounts I'm the definitive # auth for. @@ -145,6 +148,8 @@ class DevAuth(object): """ Returns None if the request is authorized to continue or a standard WSGI response callable if not. + + :raises: InvalidPathError (thrown by split_path) if given invalid path """ version, account, container, obj = split_path(req.path, 1, 4, True) if not account or not account.startswith(self.reseller_prefix): diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index 707e544ae2..9ba3ca06f4 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -18,7 +18,7 @@ from webob.exc import HTTPNotFound from swift.common.utils import split_path, cache_from_env, get_logger from swift.proxy.server import get_container_memcache_key - +from swift.common.exceptions import InvalidPathError class MaxSleepTimeHit(Exception): pass @@ -207,7 +207,7 @@ class RateLimitMiddleware(object): self.memcache_client = cache_from_env(env) try: version, account, container, obj = split_path(req.path, 1, 4, True) - except ValueError: + except InvalidPathError: return HTTPNotFound()(env, start_response) ratelimit_resp = self.handle_ratelimit(req, account, container, obj) if ratelimit_resp is None: diff --git a/swift/common/utils.py b/swift/common/utils.py index bb635725c8..4f89a87cef 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -40,7 +40,8 @@ import eventlet from eventlet import greenio, GreenPool, sleep, Timeout, listen from eventlet.green import socket, subprocess, ssl, thread, threading -from swift.common.exceptions import LockTimeout, MessageTimeout +from swift.common.exceptions import LockTimeout, MessageTimeout, \ + InvalidPathError # logging doesn't import patched as cleanly as one would like from logging.handlers import SysLogHandler @@ -208,12 +209,13 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): trailing data, raises ValueError. :returns: list of segments with a length of maxsegs (non-existant segments will return as None) - :raises: ValueError if given an invalid path + :raises: InvalidPathError if given an invalid path """ if not maxsegs: maxsegs = minsegs if minsegs > maxsegs: - raise ValueError('minsegs > maxsegs: %d > %d' % (minsegs, maxsegs)) + raise InvalidPathError('minsegs > maxsegs: %d > %d' % (minsegs, + maxsegs)) if rest_with_last: segs = path.split('/', maxsegs) minsegs += 1 @@ -221,7 +223,7 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): count = len(segs) if segs[0] or count < minsegs or count > maxsegs or \ '' in segs[1:minsegs]: - raise ValueError('Invalid path: %s' % quote(path)) + raise InvalidPathError('Invalid path: %s' % quote(path)) else: minsegs += 1 maxsegs += 1 @@ -229,7 +231,7 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): count = len(segs) if segs[0] or count < minsegs or count > maxsegs + 1 or \ '' in segs[1:minsegs] or (count == maxsegs + 1 and segs[maxsegs]): - raise ValueError('Invalid path: %s' % quote(path)) + raise InvalidPathError('Invalid path: %s' % quote(path)) segs = segs[1:maxsegs] segs.extend([None] * (maxsegs - 1 - len(segs))) return segs diff --git a/swift/container/server.py b/swift/container/server.py index ff8dc76684..a6368118c6 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -35,7 +35,7 @@ from swift.common.utils import get_logger, get_param, hash_path, \ from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.bufferedhttp import http_connect -from swift.common.exceptions import ConnectionTimeout +from swift.common.exceptions import ConnectionTimeout, InvalidPathError from swift.common.db_replicator import ReplicatorRpc DATADIR = 'containers' @@ -130,7 +130,7 @@ class ContainerController(object): try: drive, part, account, container, obj = split_path( unquote(req.path), 4, 5, True) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if 'x-timestamp' not in req.headers or \ @@ -166,7 +166,7 @@ class ContainerController(object): try: drive, part, account, container, obj = split_path( unquote(req.path), 4, 5, True) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if 'x-timestamp' not in req.headers or \ @@ -212,7 +212,7 @@ class ContainerController(object): try: drive, part, account, container, obj = split_path( unquote(req.path), 4, 5, True) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if self.mount_check and not check_mount(self.root, drive): @@ -239,7 +239,7 @@ class ContainerController(object): try: drive, part, account, container, obj = split_path( unquote(req.path), 4, 5, True) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if self.mount_check and not check_mount(self.root, drive): @@ -343,7 +343,7 @@ class ContainerController(object): """ try: post_args = split_path(unquote(req.path), 3) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) drive, partition, hash = post_args @@ -361,7 +361,7 @@ class ContainerController(object): """Handle HTTP POST request.""" try: drive, part, account, container = split_path(unquote(req.path), 4) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if 'x-timestamp' not in req.headers or \ diff --git a/swift/obj/server.py b/swift/obj/server.py index e37fa7e782..9fda3f81ba 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -41,7 +41,7 @@ from swift.common.utils import mkdirs, normalize_timestamp, \ from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, check_mount, \ check_float, check_utf8 -from swift.common.exceptions import ConnectionTimeout +from swift.common.exceptions import ConnectionTimeout, InvalidPathError from swift.obj.replicator import get_hashes, invalidate_hash, \ recalculate_hashes @@ -313,7 +313,7 @@ class ObjectController(object): try: device, partition, account, container, obj = \ split_path(unquote(request.path), 5, 5, True) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), request=request, content_type='text/plain') if 'x-timestamp' not in request.headers or \ @@ -342,7 +342,7 @@ class ObjectController(object): try: device, partition, account, container, obj = \ split_path(unquote(request.path), 5, 5, True) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), request=request, content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): @@ -414,7 +414,7 @@ class ObjectController(object): try: device, partition, account, container, obj = \ split_path(unquote(request.path), 5, 5, True) - except ValueError, err: + except InvalidPathError, err: return HTTPBadRequest(body=str(err), request=request, content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): @@ -474,7 +474,7 @@ class ObjectController(object): try: device, partition, account, container, obj = \ split_path(unquote(request.path), 5, 5, True) - except ValueError, err: + except InvalidPathError, err: resp = HTTPBadRequest(request=request) resp.content_type = 'text/plain' resp.body = str(err) @@ -502,7 +502,7 @@ class ObjectController(object): try: device, partition, account, container, obj = \ split_path(unquote(request.path), 5, 5, True) - except ValueError, e: + except InvalidPathError, e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') if 'x-timestamp' not in request.headers or \ @@ -534,8 +534,12 @@ class ObjectController(object): Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ - device, partition, suffix = split_path( - unquote(request.path), 2, 3, True) + try: + device, partition, suffix = split_path( + unquote(request.path), 2, 3, True) + except InvalidPathError, e: + return HTTPBadRequest(body=str(e), request=request, + content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) path = os.path.join(self.devices, device, DATADIR, partition) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 231aa467f8..ac1f7c4c39 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -39,7 +39,7 @@ from swift.common.constraints import check_metadata, check_object_creation, \ check_utf8, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, \ MAX_FILE_SIZE from swift.common.exceptions import ChunkReadTimeout, \ - ChunkWriteTimeout, ConnectionTimeout + ChunkWriteTimeout, ConnectionTimeout, InvalidPathError def update_headers(response, headers): @@ -1259,6 +1259,8 @@ class BaseApplication(object): :param path: path from request :returns: tuple of (controller class, path dictionary) + + :raises: InvalidPathError (thrown by split_path) if given invalid path """ version, account, container, obj = split_path(path, 1, 4, True) d = dict(version=version, @@ -1297,6 +1299,8 @@ class BaseApplication(object): response = self.handle_request(req)(env, start_response) self.posthooklogger(env, req) return response + except InvalidPathError: + HTTPNotFound()(env, start_response) except: print "EXCEPTION IN __call__: %s: %s" % \ (traceback.format_exc(), env) @@ -1326,7 +1330,7 @@ class BaseApplication(object): try: try: controller, path_parts = self.get_controller(req.path) - except ValueError: + except InvalidPathError: return HTTPNotFound(request=req) if not check_utf8(req.path_info): return HTTPPreconditionFailed(request=req, body='Invalid UTF8') diff --git a/swift/stats/access_processor.py b/swift/stats/access_processor.py index 5d8766b9df..a21b77005a 100644 --- a/swift/stats/access_processor.py +++ b/swift/stats/access_processor.py @@ -18,6 +18,7 @@ from urllib import unquote import copy from swift.common.utils import split_path, get_logger +from swift.common.exceptions import InvalidPathError month_map = '_ Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split() @@ -66,10 +67,13 @@ class AccessLogProcessor(object): self.logger.debug('Bad server name: found "%s" expected "%s"' \ % (server, self.server_name)) return {} - (version, - account, - container_name, - object_name) = split_path(request, 2, 4, True) + try: + (version, account, container_name, object_name) = \ + split_path(request, 2, 4, True) + except InvalidPathError, e: + self.logger.debug( + 'Invalid path: %s from data: %s' % (e, repr(raw_log))) + return {} if container_name is not None: container_name = container_name.split('?', 1)[0] if object_name is not None: diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py index d63f843abe..31e07b7cff 100644 --- a/test/unit/auth/test_server.py +++ b/test/unit/auth/test_server.py @@ -27,6 +27,7 @@ from webob import Request from swift.auth import server as auth_server from swift.common.db import DatabaseConnectionError, get_db_connection from swift.common.utils import get_logger +from swift.common.exceptions import InvalidPathError class TestException(Exception): @@ -241,7 +242,7 @@ class TestAuthServer(unittest.TestCase): len(set(repr(a) for a in cfaccounts) - set(failed)), 2) def test_auth_bad_path(self): - self.assertRaises(ValueError, self.controller.handle_auth, + self.assertRaises(InvalidPathError, self.controller.handle_auth, Request.blank('', environ={'REQUEST_METHOD': 'GET'})) res = self.controller.handle_auth(Request.blank('/bad', environ={'REQUEST_METHOD': 'GET'})) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 92be1077c0..f422226793 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -29,7 +29,7 @@ from StringIO import StringIO from eventlet import sleep from swift.common import utils - +from swift.common.exceptions import InvalidPathError class TestUtils(unittest.TestCase): """ Tests for swift.common.utils """ @@ -87,36 +87,36 @@ class TestUtils(unittest.TestCase): def test_split_path(self): """ Test swift.common.utils.split_account_path """ - self.assertRaises(ValueError, utils.split_path, '') - self.assertRaises(ValueError, utils.split_path, '/') - self.assertRaises(ValueError, utils.split_path, '//') + self.assertRaises(InvalidPathError, utils.split_path, '') + self.assertRaises(InvalidPathError, utils.split_path, '/') + self.assertRaises(InvalidPathError, utils.split_path, '//') self.assertEquals(utils.split_path('/a'), ['a']) - self.assertRaises(ValueError, utils.split_path, '//a') + self.assertRaises(InvalidPathError, utils.split_path, '//a') self.assertEquals(utils.split_path('/a/'), ['a']) - self.assertRaises(ValueError, utils.split_path, '/a/c') - self.assertRaises(ValueError, utils.split_path, '//c') - self.assertRaises(ValueError, utils.split_path, '/a/c/') - self.assertRaises(ValueError, utils.split_path, '/a//') - self.assertRaises(ValueError, utils.split_path, '/a', 2) - self.assertRaises(ValueError, utils.split_path, '/a', 2, 3) - self.assertRaises(ValueError, utils.split_path, '/a', 2, 3, True) + self.assertRaises(InvalidPathError, utils.split_path, '/a/c') + self.assertRaises(InvalidPathError, utils.split_path, '//c') + self.assertRaises(InvalidPathError, utils.split_path, '/a/c/') + self.assertRaises(InvalidPathError, utils.split_path, '/a//') + self.assertRaises(InvalidPathError, utils.split_path, '/a', 2) + self.assertRaises(InvalidPathError, utils.split_path, '/a', 2, 3) + self.assertRaises(InvalidPathError, utils.split_path, '/a', 2, 3, True) self.assertEquals(utils.split_path('/a/c', 2), ['a', 'c']) self.assertEquals(utils.split_path('/a/c/o', 3), ['a', 'c', 'o']) - self.assertRaises(ValueError, utils.split_path, '/a/c/o/r', 3, 3) + self.assertRaises(InvalidPathError, utils.split_path, '/a/c/o/r', 3, 3) self.assertEquals(utils.split_path('/a/c/o/r', 3, 3, True), ['a', 'c', 'o/r']) self.assertEquals(utils.split_path('/a/c', 2, 3, True), ['a', 'c', None]) - self.assertRaises(ValueError, utils.split_path, '/a', 5, 4) + self.assertRaises(InvalidPathError, utils.split_path, '/a', 5, 4) self.assertEquals(utils.split_path('/a/c/', 2), ['a', 'c']) self.assertEquals(utils.split_path('/a/c/', 2, 3), ['a', 'c', '']) try: utils.split_path('o\nn e', 2) - except ValueError, err: + except InvalidPathError, err: self.assertEquals(str(err), 'Invalid path: o%0An%20e') try: utils.split_path('o\nn e', 2, 3, True) - except ValueError, err: + except InvalidPathError, err: self.assertEquals(str(err), 'Invalid path: o%0An%20e') def test_NullLogger(self): From 1f09bb672092911bbac3076da65ad03e4dd8cddc Mon Sep 17 00:00:00 2001 From: David Goetz Date: Mon, 1 Nov 2010 14:47:48 -0700 Subject: [PATCH 10/15] undoing new exception type --- swift/account/server.py | 14 ++++++------ swift/auth/server.py | 15 +++++++------ swift/common/exceptions.py | 3 --- swift/common/middleware/auth.py | 11 +++++----- swift/common/middleware/ratelimit.py | 4 ++-- swift/common/utils.py | 12 +++++------ swift/container/server.py | 14 ++++++------ swift/obj/server.py | 14 ++++++------ swift/proxy/server.py | 8 +++---- swift/stats/access_processor.py | 3 +-- test/unit/auth/test_server.py | 4 ++-- test/unit/common/test_utils.py | 32 ++++++++++++++-------------- 12 files changed, 64 insertions(+), 70 deletions(-) diff --git a/swift/account/server.py b/swift/account/server.py index 1c29ea84d1..005c552b1d 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -34,7 +34,7 @@ from swift.common.utils import get_logger, get_param, hash_path, \ from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.db_replicator import ReplicatorRpc -from swift.common.exceptions import InvalidPathError + DATADIR = 'accounts' @@ -60,7 +60,7 @@ class AccountController(object): """Handle HTTP DELETE request.""" try: drive, part, account = split_path(unquote(req.path), 3) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if self.mount_check and not check_mount(self.root, drive): @@ -80,7 +80,7 @@ class AccountController(object): try: drive, part, account, container = split_path(unquote(req.path), 3, 4) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if self.mount_check and not check_mount(self.root, drive): @@ -135,7 +135,7 @@ class AccountController(object): try: drive, part, account, container = split_path(unquote(req.path), 3, 4) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if self.mount_check and not check_mount(self.root, drive): @@ -166,7 +166,7 @@ class AccountController(object): """Handle HTTP GET request.""" try: drive, part, account = split_path(unquote(req.path), 3) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if self.mount_check and not check_mount(self.root, drive): @@ -257,7 +257,7 @@ class AccountController(object): """ try: post_args = split_path(unquote(req.path), 3) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) drive, partition, hash = post_args @@ -275,7 +275,7 @@ class AccountController(object): """Handle HTTP POST request.""" try: drive, part, account = split_path(unquote(req.path), 3) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if 'x-timestamp' not in req.headers or \ diff --git a/swift/auth/server.py b/swift/auth/server.py index 24c1f9ce9c..2e6d2d7db0 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -30,7 +30,6 @@ from webob.exc import HTTPBadRequest, HTTPConflict, HTTPForbidden, \ from swift.common.bufferedhttp import http_connect_raw as http_connect from swift.common.db import get_db_connection from swift.common.utils import get_logger, split_path -from swift.common.exceptions import InvalidPathError class AuthController(object): @@ -416,7 +415,7 @@ YOU HAVE A FEW OPTIONS: """ try: _, token = split_path(request.path, minsegs=2) - except InvalidPathError: + except ValueError: return HTTPBadRequest() # Retrieves (TTL, account, user, cfaccount) if valid, False otherwise validation = self.validate_token(token) @@ -452,7 +451,7 @@ YOU HAVE A FEW OPTIONS: """ try: _, account_name, user_name = split_path(request.path, minsegs=3) - except InvalidPathError: + except ValueError: return HTTPBadRequest() create_reseller_admin = \ request.headers.get('x-auth-user-reseller-admin') == 'true' @@ -517,8 +516,12 @@ YOU HAVE A FEW OPTIONS: :param request: A webob.Request instance. """ - pathsegs = \ - split_path(request.path, minsegs=1, maxsegs=3, rest_with_last=True) + try: + pathsegs = \ + split_path(request.path, minsegs=1, maxsegs=3, + rest_with_last=True) + except ValueError: + return HTTPBadRequest() if pathsegs[0] == 'v1' and pathsegs[2] == 'auth': account = pathsegs[1] user = request.headers.get('x-storage-user') @@ -608,8 +611,6 @@ YOU HAVE A FEW OPTIONS: else: return HTTPBadRequest(request=env)(env, start_response) response = handler(req) - except InvalidPathError: - return HTTPNotFound()(env, start_response) except: self.logger.exception('ERROR Unhandled exception in ReST request') return HTTPServiceUnavailable(request=req)(env, start_response) diff --git a/swift/common/exceptions.py b/swift/common/exceptions.py index 504d8f7788..c498e2dae9 100644 --- a/swift/common/exceptions.py +++ b/swift/common/exceptions.py @@ -52,6 +52,3 @@ class DriveNotMounted(Exception): class LockTimeout(MessageTimeout): pass - -class InvalidPathError(Exception): - pass diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py index 922e6277f7..b60c95278d 100644 --- a/swift/common/middleware/auth.py +++ b/swift/common/middleware/auth.py @@ -21,7 +21,7 @@ from webob.exc import HTTPForbidden, HTTPUnauthorized, HTTPNotFound from swift.common.bufferedhttp import http_connect_raw as http_connect from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed from swift.common.utils import cache_from_env, split_path, TRUE_VALUES -from swift.common.exceptions import InvalidPathError + class DevAuth(object): """Auth Middleware that uses the dev auth server.""" @@ -85,7 +85,7 @@ class DevAuth(object): try: version, rest = split_path(env.get('PATH_INFO', ''), 1, 2, True) - except InvalidPathError, err: + except ValueError, err: return HTTPNotFound()(env, start_response) if rest and rest.startswith(self.reseller_prefix): # Handle anonymous access to accounts I'm the definitive @@ -148,10 +148,11 @@ class DevAuth(object): """ Returns None if the request is authorized to continue or a standard WSGI response callable if not. - - :raises: InvalidPathError (thrown by split_path) if given invalid path """ - version, account, container, obj = split_path(req.path, 1, 4, True) + try: + version, account, container, obj = split_path(req.path, 1, 4, True) + except ValueError: + return HTTPNotFound(request=req) if not account or not account.startswith(self.reseller_prefix): return self.denied_response(req) user_groups = (req.remote_user or '').split(',') diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index 9ba3ca06f4..707e544ae2 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -18,7 +18,7 @@ from webob.exc import HTTPNotFound from swift.common.utils import split_path, cache_from_env, get_logger from swift.proxy.server import get_container_memcache_key -from swift.common.exceptions import InvalidPathError + class MaxSleepTimeHit(Exception): pass @@ -207,7 +207,7 @@ class RateLimitMiddleware(object): self.memcache_client = cache_from_env(env) try: version, account, container, obj = split_path(req.path, 1, 4, True) - except InvalidPathError: + except ValueError: return HTTPNotFound()(env, start_response) ratelimit_resp = self.handle_ratelimit(req, account, container, obj) if ratelimit_resp is None: diff --git a/swift/common/utils.py b/swift/common/utils.py index 4f89a87cef..bb635725c8 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -40,8 +40,7 @@ import eventlet from eventlet import greenio, GreenPool, sleep, Timeout, listen from eventlet.green import socket, subprocess, ssl, thread, threading -from swift.common.exceptions import LockTimeout, MessageTimeout, \ - InvalidPathError +from swift.common.exceptions import LockTimeout, MessageTimeout # logging doesn't import patched as cleanly as one would like from logging.handlers import SysLogHandler @@ -209,13 +208,12 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): trailing data, raises ValueError. :returns: list of segments with a length of maxsegs (non-existant segments will return as None) - :raises: InvalidPathError if given an invalid path + :raises: ValueError if given an invalid path """ if not maxsegs: maxsegs = minsegs if minsegs > maxsegs: - raise InvalidPathError('minsegs > maxsegs: %d > %d' % (minsegs, - maxsegs)) + raise ValueError('minsegs > maxsegs: %d > %d' % (minsegs, maxsegs)) if rest_with_last: segs = path.split('/', maxsegs) minsegs += 1 @@ -223,7 +221,7 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): count = len(segs) if segs[0] or count < minsegs or count > maxsegs or \ '' in segs[1:minsegs]: - raise InvalidPathError('Invalid path: %s' % quote(path)) + raise ValueError('Invalid path: %s' % quote(path)) else: minsegs += 1 maxsegs += 1 @@ -231,7 +229,7 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): count = len(segs) if segs[0] or count < minsegs or count > maxsegs + 1 or \ '' in segs[1:minsegs] or (count == maxsegs + 1 and segs[maxsegs]): - raise InvalidPathError('Invalid path: %s' % quote(path)) + raise ValueError('Invalid path: %s' % quote(path)) segs = segs[1:maxsegs] segs.extend([None] * (maxsegs - 1 - len(segs))) return segs diff --git a/swift/container/server.py b/swift/container/server.py index a6368118c6..ff8dc76684 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -35,7 +35,7 @@ from swift.common.utils import get_logger, get_param, hash_path, \ from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.bufferedhttp import http_connect -from swift.common.exceptions import ConnectionTimeout, InvalidPathError +from swift.common.exceptions import ConnectionTimeout from swift.common.db_replicator import ReplicatorRpc DATADIR = 'containers' @@ -130,7 +130,7 @@ class ContainerController(object): try: drive, part, account, container, obj = split_path( unquote(req.path), 4, 5, True) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if 'x-timestamp' not in req.headers or \ @@ -166,7 +166,7 @@ class ContainerController(object): try: drive, part, account, container, obj = split_path( unquote(req.path), 4, 5, True) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if 'x-timestamp' not in req.headers or \ @@ -212,7 +212,7 @@ class ContainerController(object): try: drive, part, account, container, obj = split_path( unquote(req.path), 4, 5, True) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if self.mount_check and not check_mount(self.root, drive): @@ -239,7 +239,7 @@ class ContainerController(object): try: drive, part, account, container, obj = split_path( unquote(req.path), 4, 5, True) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if self.mount_check and not check_mount(self.root, drive): @@ -343,7 +343,7 @@ class ContainerController(object): """ try: post_args = split_path(unquote(req.path), 3) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) drive, partition, hash = post_args @@ -361,7 +361,7 @@ class ContainerController(object): """Handle HTTP POST request.""" try: drive, part, account, container = split_path(unquote(req.path), 4) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if 'x-timestamp' not in req.headers or \ diff --git a/swift/obj/server.py b/swift/obj/server.py index 9fda3f81ba..632a0c04cc 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -41,7 +41,7 @@ from swift.common.utils import mkdirs, normalize_timestamp, \ from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, check_mount, \ check_float, check_utf8 -from swift.common.exceptions import ConnectionTimeout, InvalidPathError +from swift.common.exceptions import ConnectionTimeout from swift.obj.replicator import get_hashes, invalidate_hash, \ recalculate_hashes @@ -313,7 +313,7 @@ class ObjectController(object): try: device, partition, account, container, obj = \ split_path(unquote(request.path), 5, 5, True) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), request=request, content_type='text/plain') if 'x-timestamp' not in request.headers or \ @@ -342,7 +342,7 @@ class ObjectController(object): try: device, partition, account, container, obj = \ split_path(unquote(request.path), 5, 5, True) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), request=request, content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): @@ -414,7 +414,7 @@ class ObjectController(object): try: device, partition, account, container, obj = \ split_path(unquote(request.path), 5, 5, True) - except InvalidPathError, err: + except ValueError, err: return HTTPBadRequest(body=str(err), request=request, content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): @@ -474,7 +474,7 @@ class ObjectController(object): try: device, partition, account, container, obj = \ split_path(unquote(request.path), 5, 5, True) - except InvalidPathError, err: + except ValueError, err: resp = HTTPBadRequest(request=request) resp.content_type = 'text/plain' resp.body = str(err) @@ -502,7 +502,7 @@ class ObjectController(object): try: device, partition, account, container, obj = \ split_path(unquote(request.path), 5, 5, True) - except InvalidPathError, e: + except ValueError, e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') if 'x-timestamp' not in request.headers or \ @@ -537,7 +537,7 @@ class ObjectController(object): try: device, partition, suffix = split_path( unquote(request.path), 2, 3, True) - except InvalidPathError, e: + except ValueError, e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): diff --git a/swift/proxy/server.py b/swift/proxy/server.py index ac1f7c4c39..ca7543cc8f 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -39,7 +39,7 @@ from swift.common.constraints import check_metadata, check_object_creation, \ check_utf8, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, \ MAX_FILE_SIZE from swift.common.exceptions import ChunkReadTimeout, \ - ChunkWriteTimeout, ConnectionTimeout, InvalidPathError + ChunkWriteTimeout, ConnectionTimeout def update_headers(response, headers): @@ -1260,7 +1260,7 @@ class BaseApplication(object): :param path: path from request :returns: tuple of (controller class, path dictionary) - :raises: InvalidPathError (thrown by split_path) if given invalid path + :raises: ValueError (thrown by split_path) id given invalid path """ version, account, container, obj = split_path(path, 1, 4, True) d = dict(version=version, @@ -1299,8 +1299,6 @@ class BaseApplication(object): response = self.handle_request(req)(env, start_response) self.posthooklogger(env, req) return response - except InvalidPathError: - HTTPNotFound()(env, start_response) except: print "EXCEPTION IN __call__: %s: %s" % \ (traceback.format_exc(), env) @@ -1330,7 +1328,7 @@ class BaseApplication(object): try: try: controller, path_parts = self.get_controller(req.path) - except InvalidPathError: + except ValueError: return HTTPNotFound(request=req) if not check_utf8(req.path_info): return HTTPPreconditionFailed(request=req, body='Invalid UTF8') diff --git a/swift/stats/access_processor.py b/swift/stats/access_processor.py index a21b77005a..f0ae3a023a 100644 --- a/swift/stats/access_processor.py +++ b/swift/stats/access_processor.py @@ -18,7 +18,6 @@ from urllib import unquote import copy from swift.common.utils import split_path, get_logger -from swift.common.exceptions import InvalidPathError month_map = '_ Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split() @@ -70,7 +69,7 @@ class AccessLogProcessor(object): try: (version, account, container_name, object_name) = \ split_path(request, 2, 4, True) - except InvalidPathError, e: + except ValueError, e: self.logger.debug( 'Invalid path: %s from data: %s' % (e, repr(raw_log))) return {} diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py index 31e07b7cff..cb9070a22f 100644 --- a/test/unit/auth/test_server.py +++ b/test/unit/auth/test_server.py @@ -27,7 +27,6 @@ from webob import Request from swift.auth import server as auth_server from swift.common.db import DatabaseConnectionError, get_db_connection from swift.common.utils import get_logger -from swift.common.exceptions import InvalidPathError class TestException(Exception): @@ -242,8 +241,9 @@ class TestAuthServer(unittest.TestCase): len(set(repr(a) for a in cfaccounts) - set(failed)), 2) def test_auth_bad_path(self): - self.assertRaises(InvalidPathError, self.controller.handle_auth, + res = self.controller.handle_auth( Request.blank('', environ={'REQUEST_METHOD': 'GET'})) + self.assertEquals(res.status_int, 400) res = self.controller.handle_auth(Request.blank('/bad', environ={'REQUEST_METHOD': 'GET'})) self.assertEquals(res.status_int, 400) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index f422226793..92be1077c0 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -29,7 +29,7 @@ from StringIO import StringIO from eventlet import sleep from swift.common import utils -from swift.common.exceptions import InvalidPathError + class TestUtils(unittest.TestCase): """ Tests for swift.common.utils """ @@ -87,36 +87,36 @@ class TestUtils(unittest.TestCase): def test_split_path(self): """ Test swift.common.utils.split_account_path """ - self.assertRaises(InvalidPathError, utils.split_path, '') - self.assertRaises(InvalidPathError, utils.split_path, '/') - self.assertRaises(InvalidPathError, utils.split_path, '//') + self.assertRaises(ValueError, utils.split_path, '') + self.assertRaises(ValueError, utils.split_path, '/') + self.assertRaises(ValueError, utils.split_path, '//') self.assertEquals(utils.split_path('/a'), ['a']) - self.assertRaises(InvalidPathError, utils.split_path, '//a') + self.assertRaises(ValueError, utils.split_path, '//a') self.assertEquals(utils.split_path('/a/'), ['a']) - self.assertRaises(InvalidPathError, utils.split_path, '/a/c') - self.assertRaises(InvalidPathError, utils.split_path, '//c') - self.assertRaises(InvalidPathError, utils.split_path, '/a/c/') - self.assertRaises(InvalidPathError, utils.split_path, '/a//') - self.assertRaises(InvalidPathError, utils.split_path, '/a', 2) - self.assertRaises(InvalidPathError, utils.split_path, '/a', 2, 3) - self.assertRaises(InvalidPathError, utils.split_path, '/a', 2, 3, True) + self.assertRaises(ValueError, utils.split_path, '/a/c') + self.assertRaises(ValueError, utils.split_path, '//c') + self.assertRaises(ValueError, utils.split_path, '/a/c/') + self.assertRaises(ValueError, utils.split_path, '/a//') + self.assertRaises(ValueError, utils.split_path, '/a', 2) + self.assertRaises(ValueError, utils.split_path, '/a', 2, 3) + self.assertRaises(ValueError, utils.split_path, '/a', 2, 3, True) self.assertEquals(utils.split_path('/a/c', 2), ['a', 'c']) self.assertEquals(utils.split_path('/a/c/o', 3), ['a', 'c', 'o']) - self.assertRaises(InvalidPathError, utils.split_path, '/a/c/o/r', 3, 3) + self.assertRaises(ValueError, utils.split_path, '/a/c/o/r', 3, 3) self.assertEquals(utils.split_path('/a/c/o/r', 3, 3, True), ['a', 'c', 'o/r']) self.assertEquals(utils.split_path('/a/c', 2, 3, True), ['a', 'c', None]) - self.assertRaises(InvalidPathError, utils.split_path, '/a', 5, 4) + self.assertRaises(ValueError, utils.split_path, '/a', 5, 4) self.assertEquals(utils.split_path('/a/c/', 2), ['a', 'c']) self.assertEquals(utils.split_path('/a/c/', 2, 3), ['a', 'c', '']) try: utils.split_path('o\nn e', 2) - except InvalidPathError, err: + except ValueError, err: self.assertEquals(str(err), 'Invalid path: o%0An%20e') try: utils.split_path('o\nn e', 2, 3, True) - except InvalidPathError, err: + except ValueError, err: self.assertEquals(str(err), 'Invalid path: o%0An%20e') def test_NullLogger(self): From d722a2884e8fadeb359bab3f404e87e2f5ec50f5 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Mon, 1 Nov 2010 15:13:38 -0700 Subject: [PATCH 11/15] small cleanup stuff --- swift/auth/server.py | 5 ++--- swift/common/middleware/auth.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/swift/auth/server.py b/swift/auth/server.py index 2e6d2d7db0..3e1ef881f4 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -517,9 +517,8 @@ YOU HAVE A FEW OPTIONS: :param request: A webob.Request instance. """ try: - pathsegs = \ - split_path(request.path, minsegs=1, maxsegs=3, - rest_with_last=True) + pathsegs = split_path(request.path, minsegs=1, maxsegs=3, + rest_with_last=True) except ValueError: return HTTPBadRequest() if pathsegs[0] == 'v1' and pathsegs[2] == 'auth': diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py index b60c95278d..1278a4a67a 100644 --- a/swift/common/middleware/auth.py +++ b/swift/common/middleware/auth.py @@ -85,7 +85,7 @@ class DevAuth(object): try: version, rest = split_path(env.get('PATH_INFO', ''), 1, 2, True) - except ValueError, err: + except ValueError: return HTTPNotFound()(env, start_response) if rest and rest.startswith(self.reseller_prefix): # Handle anonymous access to accounts I'm the definitive @@ -106,10 +106,10 @@ class DevAuth(object): def get_groups(self, token, memcache_client=None): """ Get groups for the given token. - + If memcache_client is set, token credentials will be cached appropriately. - + With a cache miss, or no memcache_client, the configurated external authentication server will be queried for the group information. From f33be04dcab96b5d50aa5ec25abf9fdd2e7ca238 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Mon, 1 Nov 2010 17:51:26 -0500 Subject: [PATCH 12/15] Updated SAIO to genericize wiki pointer --- doc/source/development_saio.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 3b031bf6b9..79da61d485 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -17,8 +17,7 @@ virtual machine will emulate running a four node Swift cluster. * Create guest virtual machine from the Ubuntu image. -Additional information about setting up a Swift development snapshot on Fedora, CentOS, or RHEL (Red Hat Enterprise Linux) is available on -the wiki at http://wiki.openstack.org/RhelInstructions. +Additional information about setting up a Swift development snapshot on other distributions is available on the wiki at http://wiki.openstack.org/SAIOInstructions. ----------------------------------------- Installing dependencies and the core code From 86e72dae64f3e646b1c32d9e8f0f74e287387782 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Mon, 1 Nov 2010 18:00:12 -0500 Subject: [PATCH 13/15] Updated SAIO, deleting line to make merge easier --- doc/source/development_saio.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 79da61d485..89934733ff 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -17,7 +17,6 @@ virtual machine will emulate running a four node Swift cluster. * Create guest virtual machine from the Ubuntu image. -Additional information about setting up a Swift development snapshot on other distributions is available on the wiki at http://wiki.openstack.org/SAIOInstructions. ----------------------------------------- Installing dependencies and the core code From 2c70e1b8618196d3f47966cd815c60f4a40fbe85 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Mon, 1 Nov 2010 18:36:40 -0500 Subject: [PATCH 14/15] Reverted SAIO doc --- doc/source/development_saio.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 89934733ff..3a615074ce 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -17,6 +17,8 @@ virtual machine will emulate running a four node Swift cluster. * Create guest virtual machine from the Ubuntu image. +Additional information about setting up a Swift development snapshot on other distributions is +available on the wiki at http://wiki.openstack.org/SAIOInstructions. ----------------------------------------- Installing dependencies and the core code From 5d564a98b06711646f5ca421241230810b9734da Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Tue, 2 Nov 2010 16:04:15 +0000 Subject: [PATCH 15/15] support text/xml --- swift/account/server.py | 19 ++++++++----------- swift/container/server.py | 19 ++++++++----------- test/unit/container/test_server.py | 9 ++++++++- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/swift/account/server.py b/swift/account/server.py index a51bf64054..9b98773897 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -199,16 +199,15 @@ class AccountController(object): except UnicodeDecodeError, err: return HTTPBadRequest(body='parameters not utf8', content_type='text/plain', request=req) - header_format = req.accept.best_match(['text/plain', - 'application/json', - 'application/xml']) - format = query_format or header_format or 'text/plain' - if '/' in format: - format = format.split('/')[-1] + if query_format: + req.accept = 'application/%s' % query_format.lower() + out_content_type = req.accept.best_match( + ['text/plain', 'application/json', + 'application/xml', 'text/xml'], + default_match='text/plain') account_list = broker.list_containers_iter(limit, marker, prefix, delimiter) - if format == 'json': - out_content_type = 'application/json' + if out_content_type == 'application/json': json_pattern = ['"name":%s', '"count":%s', '"bytes":%s'] json_pattern = '{' + ','.join(json_pattern) + '}' json_out = [] @@ -220,8 +219,7 @@ class AccountController(object): json_out.append(json_pattern % (name, object_count, bytes_used)) account_list = '[' + ','.join(json_out) + ']' - elif format == 'xml': - out_content_type = 'application/xml' + elif out_content_type.endswith('/xml'): output_list = ['', '' % account] for (name, object_count, bytes_used, is_subdir) in account_list: @@ -238,7 +236,6 @@ class AccountController(object): else: if not account_list: return HTTPNoContent(request=req, headers=resp_headers) - out_content_type = 'text/plain' account_list = '\n'.join(r[0] for r in account_list) + '\n' ret = Response(body=account_list, request=req, headers=resp_headers) ret.content_type = out_content_type diff --git a/swift/container/server.py b/swift/container/server.py index 7656b6fc75..6dbbc940e0 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -278,16 +278,15 @@ class ContainerController(object): except UnicodeDecodeError, err: return HTTPBadRequest(body='parameters not utf8', content_type='text/plain', request=req) - header_format = req.accept.best_match(['text/plain', - 'application/json', - 'application/xml']) - format = query_format or header_format or 'text/plain' - if '/' in format: - format = format.split('/')[-1] + if query_format: + req.accept = 'application/%s' % query_format.lower() + out_content_type = req.accept.best_match( + ['text/plain', 'application/json', + 'application/xml', 'text/xml'], + default_match='text/plain') container_list = broker.list_objects_iter(limit, marker, prefix, delimiter, path) - if format == 'json': - out_content_type = 'application/json' + if out_content_type == 'application/json': json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s', '"content_type":%s, "last_modified":"%s"'] json_pattern = '{' + ','.join(json_pattern) + '}' @@ -307,8 +306,7 @@ class ContainerController(object): content_type, created_at)) container_list = '[' + ','.join(json_out) + ']' - elif format == 'xml': - out_content_type = 'application/xml' + elif out_content_type.endswith('/xml'): xml_output = [] for (name, created_at, size, content_type, etag) in container_list: # escape name and format date here @@ -330,7 +328,6 @@ class ContainerController(object): else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) - out_content_type = 'text/plain' container_list = '\n'.join(r[0] for r in container_list) + '\n' ret = Response(body=container_list, request=req, headers=resp_headers) ret.content_type = out_content_type diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index a80090185c..fdad12c0f8 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -619,7 +619,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.body, xml_body) for xml_accept in ('application/xml', 'application/xml;q=1.0,*/*;q=0.9', - '*/*;q=0.9,application/xml;q=1.0'): + '*/*;q=0.9,application/xml;q=1.0', 'application/xml,text/xml'): req = Request.blank('/sda1/p/a/xmlc', environ={'REQUEST_METHOD': 'GET'}) req.accept = xml_accept @@ -629,6 +629,13 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.content_type, 'application/xml', 'Invalid content_type for Accept: %s' % xml_accept) + req = Request.blank('/sda1/p/a/xmlc', + environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'text/xml' + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'text/xml') + self.assertEquals(resp.body, xml_body) + def test_GET_marker(self): # make a container req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',