From 0149e962d49e7b3b46c088796df509da47293da4 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Fri, 14 Jan 2011 05:43:23 +0000 Subject: [PATCH 1/4] swift3 bucket listing args --- swift/common/middleware/swift3.py | 69 ++++++++++++------ test/unit/common/middleware/test_swift3.py | 81 ++++++++++++++++++++++ 2 files changed, 128 insertions(+), 22 deletions(-) diff --git a/swift/common/middleware/swift3.py b/swift/common/middleware/swift3.py index 979c007538..6f6b830a0f 100644 --- a/swift/common/middleware/swift3.py +++ b/swift/common/middleware/swift3.py @@ -19,6 +19,7 @@ import hmac import base64 import errno from xml.sax.saxutils import escape as xml_escape +import cgi from webob import Request, Response from webob.exc import HTTPNotFound @@ -27,6 +28,9 @@ from simplejson import loads from swift.common.utils import split_path +MAX_BUCKET_LISTING = 1000 + + def get_err_response(code): error_table = {'AccessDenied': (403, 'Access denied'), @@ -82,7 +86,6 @@ class ServiceController(Controller): if status == 401: return get_err_response('AccessDenied') else: - print status, headers, body_iter return get_err_response('InvalidURI') containers = loads(''.join(list(body_iter))) @@ -110,7 +113,19 @@ class BucketController(Controller): env['PATH_INFO'] = '/v1/%s/%s' % (account_name, container_name) def GET(self, env, start_response): - env['QUERY_STRING'] = 'format=json' + if 'QUERY_STRING' in env: + args = dict(cgi.parse_qsl(env['QUERY_STRING'])) + else: + args = {} + max_keys = min(int(args.get('max-keys', MAX_BUCKET_LISTING)), + MAX_BUCKET_LISTING) + env['QUERY_STRING'] = 'format=json&limit=%s' % (max_keys + 1) + if 'marker' in args: + env['QUERY_STRING'] += '&marker=%s' % quote(args['marker']) + if 'prefix' in args: + env['QUERY_STRING'] += '&prefix=%s' % quote(args['prefix']) + if 'delimiter' in args: + env['QUERY_STRING'] += '&delimiter=%s' % quote(args['delimiter']) body_iter = self.app(env, self.do_start_response) status = int(self.response_args[0].split()[0]) headers = dict(self.response_args[1]) @@ -121,25 +136,39 @@ class BucketController(Controller): elif status == 404: return get_err_response('InvalidBucketName') else: - print status, headers, body_iter return get_err_response('InvalidURI') objects = loads(''.join(list(body_iter))) - resp = Response(content_type='text/xml') - resp.status = 200 - resp.body = '' \ - '' \ - '%s' \ - '%s' \ - '' % \ - (self.container_name, - "".join(['%s%s'\ - '%s%sSTANDARD'\ - '' % - (xml_escape(i['name']), i['last_modified'], i['hash'], - i['bytes']) for i in objects])) - return resp + body = ('' + '' + '%s' + '%s' + '%s' + '%s' + '%s' + '%s' + '%s' + '%s' + '' % + ( + xml_escape(args.get('prefix', '')), + xml_escape(args.get('marker', '')), + xml_escape(args.get('delimiter', '')), + 'true' if len(objects) == (max_keys + 1) else 'false', + max_keys, + xml_escape(self.container_name), + "".join(['%s%s%s%sSTA'\ + 'NDARD' % + (xml_escape(i['name']), i['last_modified'], i['hash'], + i['bytes']) + for i in objects[:max_keys] if 'subdir' not in i]), + "".join(['%s' + % xml_escape(i['subdir']) + for i in objects[:max_keys] if 'subdir' in i]) + )) + return Response(body=body, content_type='text/xml') def PUT(self, env, start_response): body_iter = self.app(env, self.do_start_response) @@ -152,7 +181,6 @@ class BucketController(Controller): elif status == 202: return get_err_response('BucketAlreadyExists') else: - print status, headers, body_iter return get_err_response('InvalidURI') resp = Response() @@ -173,7 +201,6 @@ class BucketController(Controller): elif status == 409: return get_err_response('BucketNotEmpty') else: - print status, headers, body_iter return get_err_response('InvalidURI') resp = Response() @@ -229,7 +256,6 @@ class ObjectController(Controller): elif status == 404: return get_err_response('InvalidBucketName') else: - print status, headers, body_iter return get_err_response('InvalidURI') resp = Response() @@ -248,7 +274,6 @@ class ObjectController(Controller): elif status == 404: return get_err_response('NoSuchKey') else: - print status, headers, body_iter return get_err_response('InvalidURI') resp = Response() diff --git a/test/unit/common/middleware/test_swift3.py b/test/unit/common/middleware/test_swift3.py index d4e9aec46a..528c05f7f2 100644 --- a/test/unit/common/middleware/test_swift3.py +++ b/test/unit/common/middleware/test_swift3.py @@ -15,6 +15,7 @@ import unittest from datetime import datetime +import cgi from webob import Request, Response from webob.exc import HTTPUnauthorized, HTTPCreated, HTTPNoContent,\ @@ -299,6 +300,86 @@ class TestSwift3(unittest.TestCase): for i in FakeAppBucket().objects: self.assertTrue(i[0] in names) + def test_bucket_GET_is_truncated(self): + local_app = swift3.filter_factory({})(FakeAppBucket()) + bucket_name = 'junk' + + req = Request.blank('/%s' % bucket_name, + environ={'REQUEST_METHOD': 'GET', + 'QUERY_STRING': 'max-keys=3'}, + headers={'Authorization': 'AUTH_who:password'}) + resp = local_app(req.environ, local_app.app.do_start_response) + dom = xml.dom.minidom.parseString("".join(resp)) + self.assertEquals(dom.getElementsByTagName('IsTruncated')[0]. + childNodes[0].nodeValue, 'false') + + req = Request.blank('/%s' % bucket_name, + environ={'REQUEST_METHOD': 'GET', + 'QUERY_STRING': 'max-keys=2'}, + headers={'Authorization': 'AUTH_who:password'}) + resp = local_app(req.environ, local_app.app.do_start_response) + dom = xml.dom.minidom.parseString("".join(resp)) + self.assertEquals(dom.getElementsByTagName('IsTruncated')[0]. + childNodes[0].nodeValue, 'true') + + def test_bucket_GET_max_keys(self): + class FakeApp(object): + def __call__(self, env, start_response): + self.query_string = env['QUERY_STRING'] + start_response('200 OK', []) + return '[]' + fake_app = FakeApp() + local_app = swift3.filter_factory({})(fake_app) + bucket_name = 'junk' + + req = Request.blank('/%s' % bucket_name, + environ={'REQUEST_METHOD': 'GET', + 'QUERY_STRING': 'max-keys=5'}, + headers={'Authorization': 'AUTH_who:password'}) + resp = local_app(req.environ, lambda *args: None) + dom = xml.dom.minidom.parseString("".join(resp)) + self.assertEquals(dom.getElementsByTagName('MaxKeys')[0]. + childNodes[0].nodeValue, '5') + args = dict(cgi.parse_qsl(fake_app.query_string)) + self.assert_(args['limit'] == '6') + + req = Request.blank('/%s' % bucket_name, + environ={'REQUEST_METHOD': 'GET', + 'QUERY_STRING': 'max-keys=5000'}, + headers={'Authorization': 'AUTH_who:password'}) + resp = local_app(req.environ, lambda *args: None) + dom = xml.dom.minidom.parseString("".join(resp)) + self.assertEquals(dom.getElementsByTagName('MaxKeys')[0]. + childNodes[0].nodeValue, '1000') + args = dict(cgi.parse_qsl(fake_app.query_string)) + self.assertEquals(args['limit'], '1001') + + def test_bucket_GET_passthroughs(self): + class FakeApp(object): + def __call__(self, env, start_response): + self.query_string = env['QUERY_STRING'] + start_response('200 OK', []) + return '[]' + fake_app = FakeApp() + local_app = swift3.filter_factory({})(fake_app) + bucket_name = 'junk' + req = Request.blank('/%s' % bucket_name, + environ={'REQUEST_METHOD': 'GET', 'QUERY_STRING': + 'delimiter=a&marker=b&prefix=c'}, + headers={'Authorization': 'AUTH_who:password'}) + resp = local_app(req.environ, lambda *args: None) + dom = xml.dom.minidom.parseString("".join(resp)) + self.assertEquals(dom.getElementsByTagName('Prefix')[0]. + childNodes[0].nodeValue, 'c') + self.assertEquals(dom.getElementsByTagName('Marker')[0]. + childNodes[0].nodeValue, 'b') + self.assertEquals(dom.getElementsByTagName('Delimiter')[0]. + childNodes[0].nodeValue, 'a') + args = dict(cgi.parse_qsl(fake_app.query_string)) + self.assertEquals(args['delimiter'], 'a') + self.assertEquals(args['marker'], 'b') + self.assertEquals(args['prefix'], 'c') + def test_bucket_PUT_error(self): code = self._test_method_error(FakeAppBucket, 'PUT', '/bucket', 401) self.assertEquals(code, 'AccessDenied') From 15dabb6da0038db0cb0eb49f3c82242146a6ff29 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Fri, 14 Jan 2011 05:47:44 +0000 Subject: [PATCH 2/4] typo --- swift/common/middleware/swift3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/common/middleware/swift3.py b/swift/common/middleware/swift3.py index 6f6b830a0f..5f7159c47c 100644 --- a/swift/common/middleware/swift3.py +++ b/swift/common/middleware/swift3.py @@ -164,7 +164,7 @@ class BucketController(Controller): (xml_escape(i['name']), i['last_modified'], i['hash'], i['bytes']) for i in objects[:max_keys] if 'subdir' not in i]), - "".join(['%s' + "".join(['%s' % xml_escape(i['subdir']) for i in objects[:max_keys] if 'subdir' in i]) )) From bd2125c9de9423cfa06ce403d1214ce4e687dd38 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Fri, 14 Jan 2011 08:26:50 +0000 Subject: [PATCH 3/4] PUT headers --- swift/common/middleware/swift3.py | 19 ++++++++++------ test/unit/common/middleware/test_swift3.py | 25 +++++++++++++++++++++- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/swift/common/middleware/swift3.py b/swift/common/middleware/swift3.py index 5f7159c47c..c3e1b7fb5c 100644 --- a/swift/common/middleware/swift3.py +++ b/swift/common/middleware/swift3.py @@ -166,8 +166,7 @@ class BucketController(Controller): for i in objects[:max_keys] if 'subdir' not in i]), "".join(['%s' % xml_escape(i['subdir']) - for i in objects[:max_keys] if 'subdir' in i]) - )) + for i in objects[:max_keys] if 'subdir' in i]))) return Response(body=body, content_type='text/xml') def PUT(self, env, start_response): @@ -246,6 +245,15 @@ class ObjectController(Controller): return self.GETorHEAD(env, start_response) def PUT(self, env, start_response): + for key, value in env.items(): + if key.startswith('HTTP_X_AMZ_META_'): + del env[key] + env['HTTP_X_OBJECT_META_' + key[16:]] = value + elif key == 'HTTP_CONTENT_MD5': + env['HTTP_ETAG'] = value.decode('base64').encode('hex') + elif key == 'HTTP_X_AMZ_COPY_SOURCE': + env['HTTP_X_OBJECT_COPY'] = value + body_iter = self.app(env, self.do_start_response) status = int(self.response_args[0].split()[0]) headers = dict(self.response_args[1]) @@ -258,10 +266,9 @@ class ObjectController(Controller): else: return get_err_response('InvalidURI') - resp = Response() - resp.etag = headers['etag'] - resp.status = 200 - return resp + etag = headers['etag'] + del headers['etag'] + return Response(status=200, headers=headers, etag=etag) def DELETE(self, env, start_response): body_iter = self.app(env, self.do_start_response) diff --git a/test/unit/common/middleware/test_swift3.py b/test/unit/common/middleware/test_swift3.py index 528c05f7f2..c7c974c965 100644 --- a/test/unit/common/middleware/test_swift3.py +++ b/test/unit/common/middleware/test_swift3.py @@ -470,7 +470,7 @@ class TestSwift3(unittest.TestCase): environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AUTH_who:password', 'x-amz-storage-class': 'REDUCED_REDUNDANCY', - 'Content-MD5': '1b2cf535f27731c974343645a3985328'}) + 'Content-MD5': 'Gyz1NfJ3Mcl0NDZFo5hTKA=='}) req.date = datetime.now() req.content_type = 'text/plain' resp = local_app(req.environ, local_app.app.do_start_response) @@ -480,6 +480,29 @@ class TestSwift3(unittest.TestCase): self.assertEquals(headers['ETag'], "\"%s\"" % local_app.app.response_headers['etag']) + def test_object_PUT_headers(self): + class FakeApp(object): + def __call__(self, env, start_response): + self.req = Request(env) + start_response('200 OK') + start_response([]) + app = FakeApp() + local_app = swift3.filter_factory({})(app) + req = Request.blank('/bucket/object', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AUTH_who:password', + 'X-Amz-Storage-Class': 'REDUCED_REDUNDANCY', + 'X-Amz-Meta-Something': 'oh hai', + 'X-Amz-Copy-Source': '/some/source', + 'Content-MD5': 'ffoHqOWd280dyE1MT4KuoQ=='}) + req.date = datetime.now() + req.content_type = 'text/plain' + resp = local_app(req.environ, lambda *args: None) + self.assertEquals(app.req.headers['ETag'], + '7dfa07a8e59ddbcd1dc84d4c4f82aea1') + self.assertEquals(app.req.headers['X-Object-Meta-Something'], 'oh hai') + self.assertEquals(app.req.headers['X-Object-Copy'], '/some/source') + def test_object_DELETE_error(self): code = self._test_method_error(FakeAppObject, 'DELETE', '/bucket/object', 401) From 5189a3e0b57659fa91b7a974e3ba10c8fc587fe3 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Fri, 14 Jan 2011 08:33:10 +0000 Subject: [PATCH 4/4] simplify put response --- swift/common/middleware/swift3.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/swift/common/middleware/swift3.py b/swift/common/middleware/swift3.py index c3e1b7fb5c..41b7caf792 100644 --- a/swift/common/middleware/swift3.py +++ b/swift/common/middleware/swift3.py @@ -266,9 +266,7 @@ class ObjectController(Controller): else: return get_err_response('InvalidURI') - etag = headers['etag'] - del headers['etag'] - return Response(status=200, headers=headers, etag=etag) + return Response(status=200, etag=headers['etag']) def DELETE(self, env, start_response): body_iter = self.app(env, self.do_start_response)