diff --git a/doc/source/_ga/layout.html b/doc/source/_ga/layout.html new file mode 100644 index 0000000000..0b72a77ac2 --- /dev/null +++ b/doc/source/_ga/layout.html @@ -0,0 +1,17 @@ +{% extends "!layout.html" %} + +{% block footer %} +{{ super() }} + + +{% endblock %} + diff --git a/doc/source/conf.py b/doc/source/conf.py index 91596d6b4e..acff18d29b 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -41,7 +41,13 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', todo_include_todos = True # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +# Changing the path so that the Hudson build output contains GA code and the source +# docs do not contain the code so local, offline sphinx builds are "clean." +templates_path = [] +if os.getenv('HUDSON_PUBLISH_DOCS'): + templates_path = ['_ga', '_templates'] +else: + templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 605081e015..3a615074ce 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -17,6 +17,9 @@ virtual machine will emulate running a four node Swift cluster. * Create guest virtual machine from the Ubuntu image. +Additional information about setting up a Swift development snapshot on other distributions is +available on the wiki at http://wiki.openstack.org/SAIOInstructions. + ----------------------------------------- Installing dependencies and the core code ----------------------------------------- diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 4ce0d934c0..7c7bfc4c16 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -37,5 +37,4 @@ Production ---------- We do not have documentation yet on how to set up and configure Swift for a -production cluster, but hope to begin work on those soon. - +production cluster, but hope to begin work on those soon. \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 66f4d1cc7a..7ebd61ff10 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -40,7 +40,7 @@ Deployment: .. toctree:: :maxdepth: 1 - + deployment_guide admin_guide debian_package_guide diff --git a/doc/source/ratelimit.rst b/doc/source/ratelimit.rst index 37785a9565..80db870773 100644 --- a/doc/source/ratelimit.rst +++ b/doc/source/ratelimit.rst @@ -4,7 +4,7 @@ Rate Limiting Rate limiting in swift is implemented as a pluggable middleware. Rate limiting is performed on requests that result in database writes to the -account and container sqlite dbs. It uses memcached and is dependant on +account and container sqlite dbs. It uses memcached and is dependent on the proxy servers having highly synchronized time. The rate limits are limited by the accuracy of the proxy server clocks. diff --git a/swift/account/server.py b/swift/account/server.py index 18bd3f212b..bd10670b9e 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -77,7 +77,12 @@ class AccountController(object): def PUT(self, req): """Handle HTTP PUT request.""" - drive, part, account, container = split_path(unquote(req.path), 3, 4) + try: + drive, part, account, container = split_path(unquote(req.path), + 3, 4) + except ValueError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain', + request=req) if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) broker = self._get_account_broker(drive, part, account) @@ -199,16 +204,15 @@ class AccountController(object): except UnicodeDecodeError, err: return HTTPBadRequest(body='parameters not utf8', content_type='text/plain', request=req) - header_format = req.accept.first_match(['text/plain', - 'application/json', - 'application/xml']) - format = query_format if query_format else header_format - if format.startswith('application/'): - format = format[12:] + if query_format: + req.accept = 'application/%s' % query_format.lower() + out_content_type = req.accept.best_match( + ['text/plain', 'application/json', + 'application/xml', 'text/xml'], + default_match='text/plain') account_list = broker.list_containers_iter(limit, marker, prefix, delimiter) - if format == 'json': - out_content_type = 'application/json' + if out_content_type == 'application/json': json_pattern = ['"name":%s', '"count":%s', '"bytes":%s'] json_pattern = '{' + ','.join(json_pattern) + '}' json_out = [] @@ -220,8 +224,7 @@ class AccountController(object): json_out.append(json_pattern % (name, object_count, bytes_used)) account_list = '[' + ','.join(json_out) + ']' - elif format == 'xml': - out_content_type = 'application/xml' + elif out_content_type.endswith('/xml'): output_list = ['', '' % account] for (name, object_count, bytes_used, is_subdir) in account_list: @@ -238,7 +241,6 @@ class AccountController(object): else: if not account_list: return HTTPNoContent(request=req, headers=resp_headers) - out_content_type = 'text/plain' account_list = '\n'.join(r[0] for r in account_list) + '\n' ret = Response(body=account_list, request=req, headers=resp_headers) ret.content_type = out_content_type diff --git a/swift/auth/server.py b/swift/auth/server.py index 06e755f767..3e1ef881f4 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -516,8 +516,11 @@ YOU HAVE A FEW OPTIONS: :param request: A webob.Request instance. """ - pathsegs = \ - split_path(request.path, minsegs=1, maxsegs=3, rest_with_last=True) + try: + pathsegs = split_path(request.path, minsegs=1, maxsegs=3, + rest_with_last=True) + except ValueError: + return HTTPBadRequest() if pathsegs[0] == 'v1' and pathsegs[2] == 'auth': account = pathsegs[1] user = request.headers.get('x-storage-user') diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py index a4dd5ac7df..1278a4a67a 100644 --- a/swift/common/middleware/auth.py +++ b/swift/common/middleware/auth.py @@ -16,7 +16,7 @@ from time import time from eventlet.timeout import Timeout -from webob.exc import HTTPForbidden, HTTPUnauthorized +from webob.exc import HTTPForbidden, HTTPUnauthorized, HTTPNotFound from swift.common.bufferedhttp import http_connect_raw as http_connect from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed @@ -82,8 +82,11 @@ class DevAuth(object): # With a non-empty reseller_prefix, I would like to be called # back for anonymous access to accounts I know I'm the # definitive auth for. - version, rest = split_path(env.get('PATH_INFO', ''), - 1, 2, True) + try: + version, rest = split_path(env.get('PATH_INFO', ''), + 1, 2, True) + except ValueError: + return HTTPNotFound()(env, start_response) if rest and rest.startswith(self.reseller_prefix): # Handle anonymous access to accounts I'm the definitive # auth for. @@ -103,10 +106,10 @@ class DevAuth(object): def get_groups(self, token, memcache_client=None): """ Get groups for the given token. - + If memcache_client is set, token credentials will be cached appropriately. - + With a cache miss, or no memcache_client, the configurated external authentication server will be queried for the group information. @@ -146,7 +149,10 @@ class DevAuth(object): Returns None if the request is authorized to continue or a standard WSGI response callable if not. """ - version, account, container, obj = split_path(req.path, 1, 4, True) + try: + version, account, container, obj = split_path(req.path, 1, 4, True) + except ValueError: + return HTTPNotFound(request=req) if not account or not account.startswith(self.reseller_prefix): return self.denied_response(req) user_groups = (req.remote_user or '').split(',') diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index b13a4a6ab4..707e544ae2 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -14,6 +14,7 @@ import time import eventlet from webob import Request, Response +from webob.exc import HTTPNotFound from swift.common.utils import split_path, cache_from_env, get_logger from swift.proxy.server import get_container_memcache_key @@ -204,7 +205,10 @@ class RateLimitMiddleware(object): req = Request(env) if self.memcache_client is None: self.memcache_client = cache_from_env(env) - version, account, container, obj = split_path(req.path, 1, 4, True) + try: + version, account, container, obj = split_path(req.path, 1, 4, True) + except ValueError: + return HTTPNotFound()(env, start_response) ratelimit_resp = self.handle_ratelimit(req, account, container, obj) if ratelimit_resp is None: return self.app(env, start_response) diff --git a/swift/common/utils.py b/swift/common/utils.py index 5a8bf0e1be..bb635725c8 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -208,6 +208,7 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): trailing data, raises ValueError. :returns: list of segments with a length of maxsegs (non-existant segments will return as None) + :raises: ValueError if given an invalid path """ if not maxsegs: maxsegs = minsegs @@ -622,6 +623,7 @@ def write_pickle(obj, dest, tmp): os.fsync(fd) renamer(tmppath, dest) + def audit_location_generator(devices, datadir, mount_check=True, logger=None): ''' Given a devices path and a data directory, yield (path, device, diff --git a/swift/container/server.py b/swift/container/server.py index ff8dc76684..6dbbc940e0 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -278,16 +278,15 @@ class ContainerController(object): except UnicodeDecodeError, err: return HTTPBadRequest(body='parameters not utf8', content_type='text/plain', request=req) - header_format = req.accept.first_match(['text/plain', - 'application/json', - 'application/xml']) - format = query_format if query_format else header_format - if format.startswith('application/'): - format = format[12:] + if query_format: + req.accept = 'application/%s' % query_format.lower() + out_content_type = req.accept.best_match( + ['text/plain', 'application/json', + 'application/xml', 'text/xml'], + default_match='text/plain') container_list = broker.list_objects_iter(limit, marker, prefix, delimiter, path) - if format == 'json': - out_content_type = 'application/json' + if out_content_type == 'application/json': json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s', '"content_type":%s, "last_modified":"%s"'] json_pattern = '{' + ','.join(json_pattern) + '}' @@ -307,8 +306,7 @@ class ContainerController(object): content_type, created_at)) container_list = '[' + ','.join(json_out) + ']' - elif format == 'xml': - out_content_type = 'application/xml' + elif out_content_type.endswith('/xml'): xml_output = [] for (name, created_at, size, content_type, etag) in container_list: # escape name and format date here @@ -330,7 +328,6 @@ class ContainerController(object): else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) - out_content_type = 'text/plain' container_list = '\n'.join(r[0] for r in container_list) + '\n' ret = Response(body=container_list, request=req, headers=resp_headers) ret.content_type = out_content_type diff --git a/swift/obj/server.py b/swift/obj/server.py index e37fa7e782..632a0c04cc 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -534,8 +534,12 @@ class ObjectController(object): Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ - device, partition, suffix = split_path( - unquote(request.path), 2, 3, True) + try: + device, partition, suffix = split_path( + unquote(request.path), 2, 3, True) + except ValueError, e: + return HTTPBadRequest(body=str(e), request=request, + content_type='text/plain') if self.mount_check and not check_mount(self.devices, device): return Response(status='507 %s is not mounted' % device) path = os.path.join(self.devices, device, DATADIR, partition) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index f5bb8015c0..e25c53b4a1 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -1261,6 +1261,8 @@ class BaseApplication(object): :param path: path from request :returns: tuple of (controller class, path dictionary) + + :raises: ValueError (thrown by split_path) id given invalid path """ version, account, container, obj = split_path(path, 1, 4, True) d = dict(version=version, diff --git a/swift/stats/access_processor.py b/swift/stats/access_processor.py index 5d8766b9df..f0ae3a023a 100644 --- a/swift/stats/access_processor.py +++ b/swift/stats/access_processor.py @@ -66,10 +66,13 @@ class AccessLogProcessor(object): self.logger.debug('Bad server name: found "%s" expected "%s"' \ % (server, self.server_name)) return {} - (version, - account, - container_name, - object_name) = split_path(request, 2, 4, True) + try: + (version, account, container_name, object_name) = \ + split_path(request, 2, 4, True) + except ValueError, e: + self.logger.debug( + 'Invalid path: %s from data: %s' % (e, repr(raw_log))) + return {} if container_name is not None: container_name = container_name.split('?', 1)[0] if object_name is not None: diff --git a/test/functional/tests.py b/test/functional/tests.py index 6a28d9bb3e..f1ea6232b0 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -170,6 +170,15 @@ class TestAccount(Base): self.assert_status(412) self.assert_body('Bad URL') + def testInvalidPath(self): + was_url = self.env.account.conn.storage_url + self.env.account.conn.storage_url = "/%s" % was_url + self.env.account.conn.make_request('GET') + try: + self.assert_status(404) + finally: + self.env.account.conn.storage_url = was_url + def testPUT(self): self.env.account.conn.make_request('PUT') self.assert_status([403, 405]) diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py index d63f843abe..cb9070a22f 100644 --- a/test/unit/auth/test_server.py +++ b/test/unit/auth/test_server.py @@ -241,8 +241,9 @@ class TestAuthServer(unittest.TestCase): len(set(repr(a) for a in cfaccounts) - set(failed)), 2) def test_auth_bad_path(self): - self.assertRaises(ValueError, self.controller.handle_auth, + res = self.controller.handle_auth( Request.blank('', environ={'REQUEST_METHOD': 'GET'})) + self.assertEquals(res.status_int, 400) res = self.controller.handle_auth(Request.blank('/bad', environ={'REQUEST_METHOD': 'GET'})) self.assertEquals(res.status_int, 400) diff --git a/test/unit/common/middleware/test_ratelimit.py b/test/unit/common/middleware/test_ratelimit.py index 7bf4a9e445..2f709c4a41 100644 --- a/test/unit/common/middleware/test_ratelimit.py +++ b/test/unit/common/middleware/test_ratelimit.py @@ -366,6 +366,26 @@ class TestRateLimit(unittest.TestCase): time_took = time.time() - begin self.assert_(round(time_took, 1) == .4) + def test_call_invalid_path(self): + env = {'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '//v1/AUTH_1234567890', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '80', + 'swift.cache': FakeMemcache(), + 'SERVER_PROTOCOL': 'HTTP/1.0'} + + app = lambda *args, **kwargs: None + rate_mid = ratelimit.RateLimitMiddleware(app, {}, + logger=FakeLogger()) + + class a_callable(object): + + def __call__(self, *args, **kwargs): + pass + resp = rate_mid.__call__(env, a_callable()) + self.assert_('404 Not Found' in resp[0]) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 7aaee688e8..fdad12c0f8 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -479,18 +479,20 @@ class TestContainerController(unittest.TestCase): resp = self.controller.GET(req) self.assertEquals(resp.status_int, 412) - def test_GET_format(self): + def test_GET_json(self): # make a container - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + req = Request.blank('/sda1/p/a/jsonc', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = self.controller.PUT(req) # test an empty container - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req = Request.blank('/sda1/p/a/jsonc?format=json', + environ={'REQUEST_METHOD': 'GET'}) resp = self.controller.GET(req) - self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.status_int, 200) + self.assertEquals(eval(resp.body), []) # fill the container for i in range(3): - req = Request.blank('/sda1/p/a/c/%s'%i, environ= + req = Request.blank('/sda1/p/a/jsonc/%s'%i, environ= {'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', @@ -514,8 +516,88 @@ class TestContainerController(unittest.TestCase): "bytes":0, "content_type":"text/plain", "last_modified":"1970-01-01T00:00:01"}] + + req = Request.blank('/sda1/p/a/jsonc?format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'application/json') + self.assertEquals(eval(resp.body), json_body) + + for accept in ('application/json', 'application/json;q=1.0,*/*;q=0.9', + '*/*;q=0.9,application/json;q=1.0', 'application/*'): + req = Request.blank('/sda1/p/a/jsonc', + environ={'REQUEST_METHOD': 'GET'}) + req.accept = accept + resp = self.controller.GET(req) + self.assertEquals(eval(resp.body), json_body, + 'Invalid body for Accept: %s' % accept) + self.assertEquals(resp.content_type, 'application/json', + 'Invalid content_type for Accept: %s' % accept) + + def test_GET_plain(self): + # make a container + req = Request.blank('/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + # test an empty container + req = Request.blank('/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + # fill the container + for i in range(3): + req = Request.blank('/sda1/p/a/plainc/%s'%i, environ= + {'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', + 'HTTP_X_CONTENT_TYPE': 'text/plain', + 'HTTP_X_ETAG': 'x', + 'HTTP_X_SIZE': 0}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + plain_body = '0\n1\n2\n' + + req = Request.blank('/sda1/p/a/plainc', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'text/plain') + self.assertEquals(resp.body, plain_body) + + for accept in ('', 'text/plain', 'application/xml;q=0.8,*/*;q=0.9', + '*/*;q=0.9,application/xml;q=0.8', '*/*', + 'text/plain,application/xml'): + req = Request.blank('/sda1/p/a/plainc', + environ={'REQUEST_METHOD': 'GET'}) + req.accept = accept + resp = self.controller.GET(req) + self.assertEquals(resp.body, plain_body, + 'Invalid body for Accept: %s' % accept) + self.assertEquals(resp.content_type, 'text/plain', + 'Invalid content_type for Accept: %s' % accept) + + # test conflicting formats + req = Request.blank('/sda1/p/a/plainc?format=plain', + environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'application/json' + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'text/plain') + self.assertEquals(resp.body, plain_body) + + def test_GET_xml(self): + # make a container + req = Request.blank('/sda1/p/a/xmlc', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + # fill the container + for i in range(3): + req = Request.blank('/sda1/p/a/xmlc/%s'%i, environ= + {'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', + 'HTTP_X_CONTENT_TYPE': 'text/plain', + 'HTTP_X_ETAG': 'x', + 'HTTP_X_SIZE': 0}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) xml_body = '\n' \ - '' \ + '' \ '0x0' \ 'text/plain' \ '1970-01-01T00:00:01' \ @@ -529,46 +611,30 @@ class TestContainerController(unittest.TestCase): '1970-01-01T00:00:01' \ '' \ '' - plain_body = '0\n1\n2\n' - req = Request.blank('/sda1/p/a/c?format=json', environ={'REQUEST_METHOD': 'GET'}) - resp = self.controller.GET(req) - self.assertEquals(resp.content_type, 'application/json') - result = eval(resp.body) - self.assertEquals(result, json_body) - req = Request.blank('/sda1/p/a/c?format=xml', environ={'REQUEST_METHOD': 'GET'}) + # tests + req = Request.blank('/sda1/p/a/xmlc?format=xml', + environ={'REQUEST_METHOD': 'GET'}) resp = self.controller.GET(req) self.assertEquals(resp.content_type, 'application/xml') - result = resp.body - self.assertEquals(result, xml_body) - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) - req.accept = 'application/json' + self.assertEquals(resp.body, xml_body) + + for xml_accept in ('application/xml', 'application/xml;q=1.0,*/*;q=0.9', + '*/*;q=0.9,application/xml;q=1.0', 'application/xml,text/xml'): + req = Request.blank('/sda1/p/a/xmlc', + environ={'REQUEST_METHOD': 'GET'}) + req.accept = xml_accept + resp = self.controller.GET(req) + self.assertEquals(resp.body, xml_body, + 'Invalid body for Accept: %s' % xml_accept) + self.assertEquals(resp.content_type, 'application/xml', + 'Invalid content_type for Accept: %s' % xml_accept) + + req = Request.blank('/sda1/p/a/xmlc', + environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'text/xml' resp = self.controller.GET(req) - self.assertEquals(resp.content_type, 'application/json') - result = eval(resp.body) - self.assertEquals(result, json_body) - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) - req.accept = '*/*' - resp = self.controller.GET(req) - self.assertEquals(resp.content_type, 'text/plain') - result = resp.body - self.assertEquals(result, plain_body) - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) - req.accept = 'application/*' - resp = self.controller.GET(req) - result = eval(resp.body) - self.assertEquals(result, json_body) - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) - req.accept = 'application/xml' - resp = self.controller.GET(req) - result = resp.body - self.assertEquals(result, xml_body) - # test conflicting formats - req = Request.blank('/sda1/p/a/c?format=plain', environ={'REQUEST_METHOD': 'GET'}) - req.accept = 'application/json' - resp = self.controller.GET(req) - self.assertEquals(resp.content_type, 'text/plain') - result = resp.body - self.assertEquals(result, plain_body) + self.assertEquals(resp.content_type, 'text/xml') + self.assertEquals(resp.body, xml_body) def test_GET_marker(self): # make a container