diff --git a/doc/source/associated_projects.rst b/doc/source/associated_projects.rst index 8b75f39368..fae8920444 100644 --- a/doc/source/associated_projects.rst +++ b/doc/source/associated_projects.rst @@ -49,6 +49,12 @@ Content Distribution Network Integration * `SOS `_ - Swift Origin Server. +Alternative API +--------------- + +* `Swift3 `_ - Amazon S3 API emulation. + + Other ----- diff --git a/doc/source/misc.rst b/doc/source/misc.rst index 57b42ea730..1c1c5ed983 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -140,13 +140,6 @@ Ratelimit :members: :show-inheritance: -Swift3 -====== - -.. automodule:: swift.common.middleware.swift3 - :members: - :show-inheritance: - StaticWeb ========= diff --git a/setup.py b/setup.py index d195d34f6a..6e310483b6 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,6 @@ setup( 'cname_lookup=swift.common.middleware.cname_lookup:filter_factory', 'catch_errors=swift.common.middleware.catch_errors:filter_factory', 'domain_remap=swift.common.middleware.domain_remap:filter_factory', - 'swift3=swift.common.middleware.swift3:filter_factory', 'staticweb=swift.common.middleware.staticweb:filter_factory', 'tempauth=swift.common.middleware.tempauth:filter_factory', 'recon=swift.common.middleware.recon:filter_factory', diff --git a/swift/common/middleware/swift3.py b/swift/common/middleware/swift3.py deleted file mode 100644 index e3390985a7..0000000000 --- a/swift/common/middleware/swift3.py +++ /dev/null @@ -1,480 +0,0 @@ -# Copyright (c) 2010 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -The swift3 middleware will emulate the S3 REST api on top of swift. - -The following opperations are currently supported: - - * GET Service - * DELETE Bucket - * GET Bucket (List Objects) - * PUT Bucket - * DELETE Object - * GET Object - * HEAD Object - * PUT Object - * PUT Object (Copy) - -To add this middleware to your configuration, add the swift3 middleware -in front of the auth middleware, and before any other middleware that -look at swift requests (like rate limiting). - -To set up your client, the access key will be the concatenation of the -account and user strings that should look like test:tester, and the -secret access key is the account password. The host should also point -to the swift storage hostname. It also will have to use the old style -calling format, and not the hostname based container format. - -An example client using the python boto library might look like the -following for an SAIO setup:: - - connection = boto.s3.Connection( - aws_access_key_id='test:tester', - aws_secret_access_key='testing', - port=8080, - host='127.0.0.1', - is_secure=False, - calling_format=boto.s3.connection.OrdinaryCallingFormat()) -""" - -from urllib import unquote, quote -import base64 -from xml.sax.saxutils import escape as xml_escape -import urlparse - -from webob import Request, Response -from simplejson import loads - -from swift.common.utils import split_path -from swift.common.wsgi import WSGIContext -from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \ - HTTP_NO_CONTENT, HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, \ - HTTP_NOT_FOUND, HTTP_CONFLICT, is_success - - -MAX_BUCKET_LISTING = 1000 - - -def get_err_response(code): - """ - Given an HTTP response code, create a properly formatted xml error response - - :param code: error code - :returns: webob.response object - """ - error_table = { - 'AccessDenied': - (HTTP_FORBIDDEN, 'Access denied'), - 'BucketAlreadyExists': - (HTTP_CONFLICT, 'The requested bucket name is not available'), - 'BucketNotEmpty': - (HTTP_CONFLICT, 'The bucket you tried to delete is not empty'), - 'InvalidArgument': - (HTTP_BAD_REQUEST, 'Invalid Argument'), - 'InvalidBucketName': - (HTTP_BAD_REQUEST, 'The specified bucket is not valid'), - 'InvalidURI': - (HTTP_BAD_REQUEST, 'Could not parse the specified URI'), - 'NoSuchBucket': - (HTTP_NOT_FOUND, 'The specified bucket does not exist'), - 'SignatureDoesNotMatch': - (HTTP_FORBIDDEN, 'The calculated request signature does not '\ - 'match your provided one'), - 'NoSuchKey': - (HTTP_NOT_FOUND, 'The resource you requested does not exist')} - - resp = Response(content_type='text/xml') - resp.status = error_table[code][0] - resp.body = error_table[code][1] - resp.body = '\r\n\r\n ' \ - '%s\r\n %s\r\n\r\n' \ - % (code, error_table[code][1]) - return resp - - -def get_acl(account_name): - body = ('' - '' - '%s' - '' - '' - '' - '' - '%s' - '' - 'FULL_CONTROL' - '' - '' - '' % - (account_name, account_name)) - return Response(body=body, content_type="text/plain") - - -def canonical_string(req): - """ - Canonicalize a request to a token that can be signed. - """ - amz_headers = {} - - buf = "%s\n%s\n%s\n" % (req.method, req.headers.get('Content-MD5', ''), - req.headers.get('Content-Type') or '') - - for amz_header in sorted((key.lower() for key in req.headers - if key.lower().startswith('x-amz-'))): - amz_headers[amz_header] = req.headers[amz_header] - - if 'x-amz-date' in amz_headers: - buf += "\n" - elif 'Date' in req.headers: - buf += "%s\n" % req.headers['Date'] - - for k in sorted(key.lower() for key in amz_headers): - buf += "%s:%s\n" % (k, amz_headers[k]) - - path = req.path_qs - if '?' in path: - path, args = path.split('?', 1) - for key in urlparse.parse_qs(args, keep_blank_values=True): - if key in ('acl', 'logging', 'torrent', 'location', - 'requestPayment'): - return "%s%s?%s" % (buf, path, key) - return buf + path - - -class ServiceController(WSGIContext): - """ - Handles account level requests. - """ - def __init__(self, env, app, account_name, token, **kwargs): - WSGIContext.__init__(self, app) - env['HTTP_X_AUTH_TOKEN'] = token - env['PATH_INFO'] = '/v1/%s' % account_name - - def GET(self, env, start_response): - """ - Handle GET Service request - """ - env['QUERY_STRING'] = 'format=json' - body_iter = self._app_call(env) - status = self._get_status_int() - - if status != HTTP_OK: - if status == HTTP_UNAUTHORIZED: - return get_err_response('AccessDenied') - else: - return get_err_response('InvalidURI') - - containers = loads(''.join(list(body_iter))) - # we don't keep the creation time of a backet (s3cmd doesn't - # work without that) so we use something bogus. - body = '' \ - '' \ - '%s' \ - '' \ - % ("".join(['%s' \ - '2009-02-03T16:45:09.000Z' % - xml_escape(i['name']) for i in containers])) - resp = Response(status=HTTP_OK, content_type='application/xml', - body=body) - return resp - - -class BucketController(WSGIContext): - """ - Handles bucket request. - """ - def __init__(self, env, app, account_name, token, container_name, - **kwargs): - WSGIContext.__init__(self, app) - self.container_name = unquote(container_name) - self.account_name = unquote(account_name) - env['HTTP_X_AUTH_TOKEN'] = token - env['PATH_INFO'] = '/v1/%s/%s' % (account_name, container_name) - - def GET(self, env, start_response): - """ - Handle GET Bucket (List Objects) request - """ - if 'QUERY_STRING' in env: - args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1)) - else: - args = {} - max_keys = min(int(args.get('max-keys', MAX_BUCKET_LISTING)), - MAX_BUCKET_LISTING) - env['QUERY_STRING'] = 'format=json&limit=%s' % (max_keys + 1) - if 'marker' in args: - env['QUERY_STRING'] += '&marker=%s' % quote(args['marker']) - if 'prefix' in args: - env['QUERY_STRING'] += '&prefix=%s' % quote(args['prefix']) - if 'delimiter' in args: - env['QUERY_STRING'] += '&delimiter=%s' % quote(args['delimiter']) - body_iter = self._app_call(env) - status = self._get_status_int() - - if status != HTTP_OK: - if status == HTTP_UNAUTHORIZED: - return get_err_response('AccessDenied') - elif status == HTTP_NOT_FOUND: - return get_err_response('NoSuchBucket') - else: - return get_err_response('InvalidURI') - - if 'acl' in args: - return get_acl(self.account_name) - - objects = loads(''.join(list(body_iter))) - body = ('' - '' - '%s' - '%s' - '%s' - '%s' - '%s' - '%s' - '%s' - '%s' - '' % - ( - xml_escape(args.get('prefix', '')), - xml_escape(args.get('marker', '')), - xml_escape(args.get('delimiter', '')), - 'true' if len(objects) == (max_keys + 1) else 'false', - max_keys, - xml_escape(self.container_name), - "".join(['%s%sZ%s%sSTA'\ - 'NDARD' % - (xml_escape(i['name']), i['last_modified'], i['hash'], - i['bytes']) - for i in objects[:max_keys] if 'subdir' not in i]), - "".join(['%s' - % xml_escape(i['subdir']) - for i in objects[:max_keys] if 'subdir' in i]))) - return Response(body=body, content_type='application/xml') - - def PUT(self, env, start_response): - """ - Handle PUT Bucket request - """ - body_iter = self._app_call(env) - status = self._get_status_int() - - if status != HTTP_CREATED: - if status == HTTP_UNAUTHORIZED: - return get_err_response('AccessDenied') - elif status == HTTP_ACCEPTED: - return get_err_response('BucketAlreadyExists') - else: - return get_err_response('InvalidURI') - - resp = Response() - resp.headers.add('Location', self.container_name) - resp.status = HTTP_OK - return resp - - def DELETE(self, env, start_response): - """ - Handle DELETE Bucket request - """ - body_iter = self._app_call(env) - status = self._get_status_int() - - if status != HTTP_NO_CONTENT: - if status == HTTP_UNAUTHORIZED: - return get_err_response('AccessDenied') - elif status == HTTP_NOT_FOUND: - return get_err_response('NoSuchBucket') - elif status == HTTP_CONFLICT: - return get_err_response('BucketNotEmpty') - else: - return get_err_response('InvalidURI') - - resp = Response() - resp.status = HTTP_NO_CONTENT - return resp - - -class ObjectController(WSGIContext): - """ - Handles requests on objects - """ - def __init__(self, env, app, account_name, token, container_name, - object_name, **kwargs): - WSGIContext.__init__(self, app) - self.account_name = unquote(account_name) - self.container_name = unquote(container_name) - env['HTTP_X_AUTH_TOKEN'] = token - env['PATH_INFO'] = '/v1/%s/%s/%s' % (account_name, container_name, - object_name) - - def GETorHEAD(self, env, start_response): - app_iter = self._app_call(env) - status = self._get_status_int() - headers = dict(self._response_headers) - - if is_success(status): - if 'QUERY_STRING' in env: - args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1)) - else: - args = {} - if 'acl' in args: - return get_acl(self.account_name) - - new_hdrs = {} - for key, val in headers.iteritems(): - _key = key.lower() - if _key.startswith('x-object-meta-'): - new_hdrs['x-amz-meta-' + key[14:]] = val - elif _key in ('content-length', 'content-type', - 'content-range', 'content-encoding', - 'etag', 'last-modified'): - new_hdrs[key] = val - return Response(status=status, headers=new_hdrs, app_iter=app_iter) - elif status == HTTP_UNAUTHORIZED: - return get_err_response('AccessDenied') - elif status == HTTP_NOT_FOUND: - return get_err_response('NoSuchKey') - else: - return get_err_response('InvalidURI') - - def HEAD(self, env, start_response): - """ - Handle HEAD Object request - """ - return self.GETorHEAD(env, start_response) - - def GET(self, env, start_response): - """ - Handle GET Object request - """ - return self.GETorHEAD(env, start_response) - - def PUT(self, env, start_response): - """ - Handle PUT Object and PUT Object (Copy) request - """ - for key, value in env.items(): - if key.startswith('HTTP_X_AMZ_META_'): - del env[key] - env['HTTP_X_OBJECT_META_' + key[16:]] = value - elif key == 'HTTP_CONTENT_MD5': - env['HTTP_ETAG'] = value.decode('base64').encode('hex') - elif key == 'HTTP_X_AMZ_COPY_SOURCE': - env['HTTP_X_COPY_FROM'] = value - - body_iter = self._app_call(env) - status = self._get_status_int() - - if status != HTTP_CREATED: - if status == HTTP_UNAUTHORIZED: - return get_err_response('AccessDenied') - elif status == HTTP_NOT_FOUND: - return get_err_response('NoSuchBucket') - else: - return get_err_response('InvalidURI') - - if 'HTTP_X_COPY_FROM' in env: - body = '' \ - '"%s"' \ - '' % self._response_header_value('etag') - return Response(status=HTTP_OK, body=body) - - return Response(status=200, etag=self._response_header_value('etag')) - - def DELETE(self, env, start_response): - """ - Handle DELETE Object request - """ - body_iter = self._app_call(env) - status = self._get_status_int() - - if status != HTTP_NO_CONTENT: - if status == HTTP_UNAUTHORIZED: - return get_err_response('AccessDenied') - elif status == HTTP_NOT_FOUND: - return get_err_response('NoSuchKey') - else: - return get_err_response('InvalidURI') - - resp = Response() - resp.status = HTTP_NO_CONTENT - return resp - - -class Swift3Middleware(object): - """Swift3 S3 compatibility midleware""" - def __init__(self, app, conf, *args, **kwargs): - self.app = app - - def get_controller(self, path): - container, obj = split_path(path, 0, 2, True) - d = dict(container_name=container, object_name=obj) - - if container and obj: - return ObjectController, d - elif container: - return BucketController, d - return ServiceController, d - - def __call__(self, env, start_response): - req = Request(env) - - if 'AWSAccessKeyId' in req.GET: - try: - req.headers['Date'] = req.GET['Expires'] - req.headers['Authorization'] = \ - 'AWS %(AWSAccessKeyId)s:%(Signature)s' % req.GET - except KeyError: - return get_err_response('InvalidArgument')(env, start_response) - - if not 'Authorization' in req.headers: - return self.app(env, start_response) - - try: - account, signature = \ - req.headers['Authorization'].split(' ')[-1].rsplit(':', 1) - except Exception: - return get_err_response('InvalidArgument')(env, start_response) - - try: - controller, path_parts = self.get_controller(req.path) - except ValueError: - return get_err_response('InvalidURI')(env, start_response) - - token = base64.urlsafe_b64encode(canonical_string(req)) - - controller = controller(env, self.app, account, token, **path_parts) - - if hasattr(controller, req.method): - res = getattr(controller, req.method)(env, start_response) - else: - return get_err_response('InvalidURI')(env, start_response) - - return res(env, start_response) - - -def filter_factory(global_conf, **local_conf): - """Standard filter factory to use the middleware with paste.deploy""" - conf = global_conf.copy() - conf.update(local_conf) - - def swift3_filter(app): - return Swift3Middleware(app, conf) - - return swift3_filter diff --git a/test/unit/common/middleware/test_swift3.py b/test/unit/common/middleware/test_swift3.py deleted file mode 100644 index 079f2da183..0000000000 --- a/test/unit/common/middleware/test_swift3.py +++ /dev/null @@ -1,611 +0,0 @@ -# Copyright (c) 2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -from datetime import datetime -import cgi -import hashlib - -from webob import Request, Response -from webob.exc import HTTPUnauthorized, HTTPCreated, HTTPNoContent,\ - HTTPAccepted, HTTPBadRequest, HTTPNotFound, HTTPConflict -import xml.dom.minidom -import simplejson - -from swift.common.middleware import swift3 - - -class FakeApp(object): - def __init__(self): - self.app = self - self.response_args = [] - - def __call__(self, env, start_response): - return "FAKE APP" - - def do_start_response(self, *args): - self.response_args.extend(args) - - -class FakeAppService(FakeApp): - def __init__(self, status=200): - FakeApp.__init__(self) - self.status = status - self.buckets = (('apple', 1, 200), ('orange', 3, 430)) - - def __call__(self, env, start_response): - if self.status == 200: - start_response(Response().status, [('Content-Type', 'text/xml')]) - json_pattern = ['"name":%s', '"count":%s', '"bytes":%s'] - json_pattern = '{' + ','.join(json_pattern) + '}' - json_out = [] - for b in self.buckets: - name = simplejson.dumps(b[0]) - json_out.append(json_pattern % - (name, b[1], b[2])) - account_list = '[' + ','.join(json_out) + ']' - return account_list - elif self.status == 401: - start_response(HTTPUnauthorized().status, []) - else: - start_response(HTTPBadRequest().status, []) - return [] - - -class FakeAppBucket(FakeApp): - def __init__(self, status=200): - FakeApp.__init__(self) - self.status = status - self.objects = (('rose', '2011-01-05T02:19:14.275290', 0, 303), - ('viola', '2011-01-05T02:19:14.275290', 0, 3909), - ('lily', '2011-01-05T02:19:14.275290', 0, 3909)) - - def __call__(self, env, start_response): - if env['REQUEST_METHOD'] == 'GET': - if self.status == 200: - start_response(Response().status, - [('Content-Type', 'text/xml')]) - json_pattern = ['"name":%s', '"last_modified":%s', '"hash":%s', - '"bytes":%s'] - json_pattern = '{' + ','.join(json_pattern) + '}' - json_out = [] - for b in self.objects: - name = simplejson.dumps(b[0]) - time = simplejson.dumps(b[1]) - json_out.append(json_pattern % - (name, time, b[2], b[3])) - account_list = '[' + ','.join(json_out) + ']' - return account_list - elif self.status == 401: - start_response(HTTPUnauthorized().status, []) - elif self.status == 404: - start_response(HTTPNotFound().status, []) - else: - start_response(HTTPBadRequest().status, []) - elif env['REQUEST_METHOD'] == 'PUT': - if self.status == 201: - start_response(HTTPCreated().status, []) - elif self.status == 401: - start_response(HTTPUnauthorized().status, []) - elif self.status == 202: - start_response(HTTPAccepted().status, []) - else: - start_response(HTTPBadRequest().status, []) - elif env['REQUEST_METHOD'] == 'DELETE': - if self.status == 204: - start_response(HTTPNoContent().status, []) - elif self.status == 401: - start_response(HTTPUnauthorized().status, []) - elif self.status == 404: - start_response(HTTPNotFound().status, []) - elif self.status == 409: - start_response(HTTPConflict().status, []) - else: - start_response(HTTPBadRequest().status, []) - return [] - - -class FakeAppObject(FakeApp): - def __init__(self, status=200): - FakeApp.__init__(self) - self.status = status - self.object_body = 'hello' - self.response_headers = {'Content-Type': 'text/html', - 'Content-Length': len(self.object_body), - 'x-object-meta-test': 'swift', - 'etag': '1b2cf535f27731c974343645a3985328', - 'last-modified': '2011-01-05T02:19:14.275290'} - - def __call__(self, env, start_response): - if env['REQUEST_METHOD'] == 'GET' or env['REQUEST_METHOD'] == 'HEAD': - if self.status == 200: - if 'HTTP_RANGE' in env: - resp = Response(body=self.object_body, - conditional_response=True) - return resp(env, start_response) - start_response(Response().status, - self.response_headers.items()) - if env['REQUEST_METHOD'] == 'GET': - return self.object_body - elif self.status == 401: - start_response(HTTPUnauthorized().status, []) - elif self.status == 404: - start_response(HTTPNotFound().status, []) - else: - start_response(HTTPBadRequest().status, []) - elif env['REQUEST_METHOD'] == 'PUT': - if self.status == 201: - start_response(HTTPCreated().status, - [('etag', self.response_headers['etag'])]) - elif self.status == 401: - start_response(HTTPUnauthorized().status, []) - elif self.status == 404: - start_response(HTTPNotFound().status, []) - else: - start_response(HTTPBadRequest().status, []) - elif env['REQUEST_METHOD'] == 'DELETE': - if self.status == 204: - start_response(HTTPNoContent().status, []) - elif self.status == 401: - start_response(HTTPUnauthorized().status, []) - elif self.status == 404: - start_response(HTTPNotFound().status, []) - else: - start_response(HTTPBadRequest().status, []) - return [] - - -def start_response(*args): - pass - - -class TestSwift3(unittest.TestCase): - def setUp(self): - self.app = swift3.filter_factory({})(FakeApp()) - - def test_non_s3_request_passthrough(self): - req = Request.blank('/something') - resp = self.app(req.environ, start_response) - self.assertEquals(resp, 'FAKE APP') - - def test_bad_format_authorization(self): - req = Request.blank('/something', - headers={'Authorization': 'hoge'}) - resp = self.app(req.environ, start_response) - dom = xml.dom.minidom.parseString("".join(resp)) - self.assertEquals(dom.firstChild.nodeName, 'Error') - code = dom.getElementsByTagName('Code')[0].childNodes[0].nodeValue - self.assertEquals(code, 'InvalidArgument') - - def test_bad_method(self): - req = Request.blank('/', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = self.app(req.environ, start_response) - dom = xml.dom.minidom.parseString("".join(resp)) - self.assertEquals(dom.firstChild.nodeName, 'Error') - code = dom.getElementsByTagName('Code')[0].childNodes[0].nodeValue - self.assertEquals(code, 'InvalidURI') - - def _test_method_error(self, cl, method, path, status): - local_app = swift3.filter_factory({})(cl(status)) - req = Request.blank(path, - environ={'REQUEST_METHOD': method}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, start_response) - dom = xml.dom.minidom.parseString("".join(resp)) - self.assertEquals(dom.firstChild.nodeName, 'Error') - return dom.getElementsByTagName('Code')[0].childNodes[0].nodeValue - - def test_service_GET_error(self): - code = self._test_method_error(FakeAppService, 'GET', '/', 401) - self.assertEquals(code, 'AccessDenied') - code = self._test_method_error(FakeAppService, 'GET', '/', 0) - self.assertEquals(code, 'InvalidURI') - - def test_service_GET(self): - local_app = swift3.filter_factory({})(FakeAppService()) - req = Request.blank('/', - environ={'REQUEST_METHOD': 'GET'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, local_app.app.do_start_response) - self.assertEquals(local_app.app.response_args[0].split()[0], '200') - - dom = xml.dom.minidom.parseString("".join(resp)) - self.assertEquals(dom.firstChild.nodeName, 'ListAllMyBucketsResult') - - buckets = [n for n in dom.getElementsByTagName('Bucket')] - listing = [n for n in buckets[0].childNodes if n.nodeName != '#text'] - self.assertEquals(len(listing), 2) - - names = [] - for b in buckets: - if b.childNodes[0].nodeName == 'Name': - names.append(b.childNodes[0].childNodes[0].nodeValue) - - self.assertEquals(len(names), len(FakeAppService().buckets)) - for i in FakeAppService().buckets: - self.assertTrue(i[0] in names) - - def test_bucket_GET_error(self): - code = self._test_method_error(FakeAppBucket, 'GET', '/bucket', 401) - self.assertEquals(code, 'AccessDenied') - code = self._test_method_error(FakeAppBucket, 'GET', '/bucket', 404) - self.assertEquals(code, 'NoSuchBucket') - code = self._test_method_error(FakeAppBucket, 'GET', '/bucket', 0) - self.assertEquals(code, 'InvalidURI') - - def test_bucket_GET(self): - local_app = swift3.filter_factory({})(FakeAppBucket()) - bucket_name = 'junk' - req = Request.blank('/%s' % bucket_name, - environ={'REQUEST_METHOD': 'GET'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, local_app.app.do_start_response) - self.assertEquals(local_app.app.response_args[0].split()[0], '200') - - dom = xml.dom.minidom.parseString("".join(resp)) - self.assertEquals(dom.firstChild.nodeName, 'ListBucketResult') - name = dom.getElementsByTagName('Name')[0].childNodes[0].nodeValue - self.assertEquals(name, bucket_name) - - objects = [n for n in dom.getElementsByTagName('Contents')] - listing = [n for n in objects[0].childNodes if n.nodeName != '#text'] - - names = [] - for o in objects: - if o.childNodes[0].nodeName == 'Key': - names.append(o.childNodes[0].childNodes[0].nodeValue) - if o.childNodes[1].nodeName == 'LastModified': - self.assertTrue( - o.childNodes[1].childNodes[0].nodeValue.endswith('Z')) - - self.assertEquals(len(names), len(FakeAppBucket().objects)) - for i in FakeAppBucket().objects: - self.assertTrue(i[0] in names) - - def test_bucket_GET_is_truncated(self): - local_app = swift3.filter_factory({})(FakeAppBucket()) - bucket_name = 'junk' - - req = Request.blank('/%s' % bucket_name, - environ={'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'max-keys=3'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, local_app.app.do_start_response) - dom = xml.dom.minidom.parseString("".join(resp)) - self.assertEquals(dom.getElementsByTagName('IsTruncated')[0]. - childNodes[0].nodeValue, 'false') - - req = Request.blank('/%s' % bucket_name, - environ={'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'max-keys=2'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, local_app.app.do_start_response) - dom = xml.dom.minidom.parseString("".join(resp)) - self.assertEquals(dom.getElementsByTagName('IsTruncated')[0]. - childNodes[0].nodeValue, 'true') - - def test_bucket_GET_max_keys(self): - class FakeApp(object): - def __call__(self, env, start_response): - self.query_string = env['QUERY_STRING'] - start_response('200 OK', []) - return '[]' - fake_app = FakeApp() - local_app = swift3.filter_factory({})(fake_app) - bucket_name = 'junk' - - req = Request.blank('/%s' % bucket_name, - environ={'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'max-keys=5'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, lambda *args: None) - dom = xml.dom.minidom.parseString("".join(resp)) - self.assertEquals(dom.getElementsByTagName('MaxKeys')[0]. - childNodes[0].nodeValue, '5') - args = dict(cgi.parse_qsl(fake_app.query_string)) - self.assert_(args['limit'] == '6') - - req = Request.blank('/%s' % bucket_name, - environ={'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'max-keys=5000'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, lambda *args: None) - dom = xml.dom.minidom.parseString("".join(resp)) - self.assertEquals(dom.getElementsByTagName('MaxKeys')[0]. - childNodes[0].nodeValue, '1000') - args = dict(cgi.parse_qsl(fake_app.query_string)) - self.assertEquals(args['limit'], '1001') - - def test_bucket_GET_passthroughs(self): - class FakeApp(object): - def __call__(self, env, start_response): - self.query_string = env['QUERY_STRING'] - start_response('200 OK', []) - return '[]' - fake_app = FakeApp() - local_app = swift3.filter_factory({})(fake_app) - bucket_name = 'junk' - req = Request.blank('/%s' % bucket_name, - environ={'REQUEST_METHOD': 'GET', 'QUERY_STRING': - 'delimiter=a&marker=b&prefix=c'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, lambda *args: None) - dom = xml.dom.minidom.parseString("".join(resp)) - self.assertEquals(dom.getElementsByTagName('Prefix')[0]. - childNodes[0].nodeValue, 'c') - self.assertEquals(dom.getElementsByTagName('Marker')[0]. - childNodes[0].nodeValue, 'b') - self.assertEquals(dom.getElementsByTagName('Delimiter')[0]. - childNodes[0].nodeValue, 'a') - args = dict(cgi.parse_qsl(fake_app.query_string)) - self.assertEquals(args['delimiter'], 'a') - self.assertEquals(args['marker'], 'b') - self.assertEquals(args['prefix'], 'c') - - def test_bucket_PUT_error(self): - code = self._test_method_error(FakeAppBucket, 'PUT', '/bucket', 401) - self.assertEquals(code, 'AccessDenied') - code = self._test_method_error(FakeAppBucket, 'PUT', '/bucket', 202) - self.assertEquals(code, 'BucketAlreadyExists') - code = self._test_method_error(FakeAppBucket, 'PUT', '/bucket', 0) - self.assertEquals(code, 'InvalidURI') - - def test_bucket_PUT(self): - local_app = swift3.filter_factory({})(FakeAppBucket(201)) - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, local_app.app.do_start_response) - self.assertEquals(local_app.app.response_args[0].split()[0], '200') - - def test_bucket_DELETE_error(self): - code = self._test_method_error(FakeAppBucket, 'DELETE', '/bucket', 401) - self.assertEquals(code, 'AccessDenied') - code = self._test_method_error(FakeAppBucket, 'DELETE', '/bucket', 404) - self.assertEquals(code, 'NoSuchBucket') - code = self._test_method_error(FakeAppBucket, 'DELETE', '/bucket', 409) - self.assertEquals(code, 'BucketNotEmpty') - code = self._test_method_error(FakeAppBucket, 'DELETE', '/bucket', 0) - self.assertEquals(code, 'InvalidURI') - - def test_bucket_DELETE(self): - local_app = swift3.filter_factory({})(FakeAppBucket(204)) - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, local_app.app.do_start_response) - self.assertEquals(local_app.app.response_args[0].split()[0], '204') - - def _check_acl(self, owner, resp): - dom = xml.dom.minidom.parseString("".join(resp)) - self.assertEquals(dom.firstChild.nodeName, 'AccessControlPolicy') - name = dom.getElementsByTagName('Permission')[0].childNodes[0].nodeValue - self.assertEquals(name, 'FULL_CONTROL') - name = dom.getElementsByTagName('ID')[0].childNodes[0].nodeValue - self.assertEquals(name, owner) - - def test_bucket_acl_GET(self): - local_app = swift3.filter_factory({})(FakeAppBucket()) - bucket_name = 'junk' - req = Request.blank('/%s?acl' % bucket_name, - environ={'REQUEST_METHOD': 'GET'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, local_app.app.do_start_response) - self._check_acl('test:tester', resp) - - def _test_object_GETorHEAD(self, method): - local_app = swift3.filter_factory({})(FakeAppObject()) - req = Request.blank('/bucket/object', - environ={'REQUEST_METHOD': method}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, local_app.app.do_start_response) - self.assertEquals(local_app.app.response_args[0].split()[0], '200') - - headers = dict(local_app.app.response_args[1]) - for key, val in local_app.app.response_headers.iteritems(): - if key in ('Content-Length', 'Content-Type', 'Content-Encoding', - 'etag', 'last-modified'): - self.assertTrue(key in headers) - self.assertEquals(headers[key], val) - - elif key.startswith('x-object-meta-'): - self.assertTrue('x-amz-meta-' + key[14:] in headers) - self.assertEquals(headers['x-amz-meta-' + key[14:]], val) - - if method == 'GET': - self.assertEquals(''.join(resp), local_app.app.object_body) - - def test_object_HEAD(self): - self._test_object_GETorHEAD('HEAD') - - def test_object_GET_error(self): - code = self._test_method_error(FakeAppObject, 'GET', - '/bucket/object', 401) - self.assertEquals(code, 'AccessDenied') - code = self._test_method_error(FakeAppObject, 'GET', - '/bucket/object', 404) - self.assertEquals(code, 'NoSuchKey') - code = self._test_method_error(FakeAppObject, 'GET', - '/bucket/object', 0) - self.assertEquals(code, 'InvalidURI') - - def test_object_GET(self): - self._test_object_GETorHEAD('GET') - - def test_object_GET_Range(self): - local_app = swift3.filter_factory({})(FakeAppObject()) - req = Request.blank('/bucket/object', - environ={'REQUEST_METHOD': 'GET'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Range': 'bytes=0-3'}) - resp = local_app(req.environ, local_app.app.do_start_response) - self.assertEquals(local_app.app.response_args[0].split()[0], '206') - - headers = dict(local_app.app.response_args[1]) - self.assertTrue('Content-Range' in headers) - self.assertTrue(headers['Content-Range'].startswith('bytes 0-3')) - - def test_object_PUT_error(self): - code = self._test_method_error(FakeAppObject, 'PUT', - '/bucket/object', 401) - self.assertEquals(code, 'AccessDenied') - code = self._test_method_error(FakeAppObject, 'PUT', - '/bucket/object', 404) - self.assertEquals(code, 'NoSuchBucket') - code = self._test_method_error(FakeAppObject, 'PUT', - '/bucket/object', 0) - self.assertEquals(code, 'InvalidURI') - - def test_object_PUT(self): - local_app = swift3.filter_factory({})(FakeAppObject(201)) - req = Request.blank('/bucket/object', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'x-amz-storage-class': 'REDUCED_REDUNDANCY', - 'Content-MD5': 'Gyz1NfJ3Mcl0NDZFo5hTKA=='}) - req.date = datetime.now() - req.content_type = 'text/plain' - resp = local_app(req.environ, local_app.app.do_start_response) - self.assertEquals(local_app.app.response_args[0].split()[0], '200') - - headers = dict(local_app.app.response_args[1]) - self.assertEquals(headers['ETag'], - "\"%s\"" % local_app.app.response_headers['etag']) - - def test_object_PUT_headers(self): - class FakeApp(object): - def __call__(self, env, start_response): - self.req = Request(env) - start_response('200 OK', []) - return [] - app = FakeApp() - local_app = swift3.filter_factory({})(app) - req = Request.blank('/bucket/object', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'X-Amz-Storage-Class': 'REDUCED_REDUNDANCY', - 'X-Amz-Meta-Something': 'oh hai', - 'X-Amz-Copy-Source': '/some/source', - 'Content-MD5': 'ffoHqOWd280dyE1MT4KuoQ=='}) - req.date = datetime.now() - req.content_type = 'text/plain' - resp = local_app(req.environ, lambda *args: None) - self.assertEquals(app.req.headers['ETag'], - '7dfa07a8e59ddbcd1dc84d4c4f82aea1') - self.assertEquals(app.req.headers['X-Object-Meta-Something'], 'oh hai') - self.assertEquals(app.req.headers['X-Copy-From'], '/some/source') - - def test_object_DELETE_error(self): - code = self._test_method_error(FakeAppObject, 'DELETE', - '/bucket/object', 401) - self.assertEquals(code, 'AccessDenied') - code = self._test_method_error(FakeAppObject, 'DELETE', - '/bucket/object', 404) - self.assertEquals(code, 'NoSuchKey') - code = self._test_method_error(FakeAppObject, 'DELETE', - '/bucket/object', 0) - self.assertEquals(code, 'InvalidURI') - - def test_object_DELETE(self): - local_app = swift3.filter_factory({})(FakeAppObject(204)) - req = Request.blank('/bucket/object', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, local_app.app.do_start_response) - self.assertEquals(local_app.app.response_args[0].split()[0], '204') - - def test_object_acl_GET(self): - local_app = swift3.filter_factory({})(FakeAppObject()) - req = Request.blank('/bucket/object?acl', - environ={'REQUEST_METHOD': 'GET'}, - headers={'Authorization': 'AWS test:tester:hmac'}) - resp = local_app(req.environ, local_app.app.do_start_response) - self._check_acl('test:tester', resp) - - def test_canonical_string(self): - """ - The hashes here were generated by running the same requests against - boto.utils.canonical_string - """ - def verify(hash, path, headers): - req = Request.blank(path, headers=headers) - self.assertEquals(hash, - hashlib.md5(swift3.canonical_string(req)).hexdigest()) - - verify('6dd08c75e42190a1ce9468d1fd2eb787', '/bucket/object', - {'Content-Type': 'text/plain', 'X-Amz-Something': 'test', - 'Date': 'whatever'}) - - verify('c8447135da232ae7517328f3429df481', '/bucket/object', - {'Content-Type': 'text/plain', 'X-Amz-Something': 'test'}) - - verify('bf49304103a4de5c325dce6384f2a4a2', '/bucket/object', - {'content-type': 'text/plain'}) - - verify('be01bd15d8d47f9fe5e2d9248cc6f180', '/bucket/object', {}) - - verify('8d28cc4b8322211f6cc003256cd9439e', 'bucket/object', - {'Content-MD5': 'somestuff'}) - - verify('a822deb31213ad09af37b5a7fe59e55e', '/bucket/object?acl', {}) - - verify('cce5dd1016595cb706c93f28d3eaa18f', '/bucket/object', - {'Content-Type': 'text/plain', 'X-Amz-A': 'test', - 'X-Amz-Z': 'whatever', 'X-Amz-B': 'lalala', - 'X-Amz-Y': 'lalalalalalala'}) - - verify('7506d97002c7d2de922cc0ec34af8846', '/bucket/object', - {'Content-Type': None, 'X-Amz-Something': 'test'}) - - verify('28f76d6162444a193b612cd6cb20e0be', '/bucket/object', - {'Content-Type': None, - 'X-Amz-Date': 'Mon, 11 Jul 2011 10:52:57 +0000', - 'Date': 'Tue, 12 Jul 2011 10:52:57 +0000'}) - - verify('ed6971e3eca5af4ee361f05d7c272e49', '/bucket/object', - {'Content-Type': None, - 'Date': 'Tue, 12 Jul 2011 10:52:57 +0000'}) - - req1 = Request.blank('/', headers= - {'Content-Type': None, 'X-Amz-Something': 'test'}) - req2 = Request.blank('/', headers= - {'Content-Type': '', 'X-Amz-Something': 'test'}) - req3 = Request.blank('/', headers={'X-Amz-Something': 'test'}) - - self.assertEquals(swift3.canonical_string(req1), - swift3.canonical_string(req2)) - self.assertEquals(swift3.canonical_string(req2), - swift3.canonical_string(req3)) - - def test_signed_urls(self): - class FakeApp(object): - def __call__(self, env, start_response): - self.req = Request(env) - start_response('200 OK', []) - return [] - app = FakeApp() - local_app = swift3.filter_factory({})(app) - req = Request.blank('/bucket/object?Signature=X&Expires=Y&' - 'AWSAccessKeyId=Z', environ={'REQUEST_METHOD': 'GET'}) - req.date = datetime.now() - req.content_type = 'text/plain' - resp = local_app(req.environ, lambda *args: None) - self.assertEquals(app.req.headers['Authorization'], 'AWS Z:X') - self.assertEquals(app.req.headers['Date'], 'Y') - -if __name__ == '__main__': - unittest.main()