From 46d61a4dcd9a5d9157625c06d6fe7d916e80c3d2 Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Wed, 18 Feb 2015 11:59:31 +0530 Subject: [PATCH] Refactor server side copy as middleware Rewrite server side copy and 'object post as copy' feature as middleware to simplify the PUT method in the object controller code. COPY is no longer a verb implemented as public method in Proxy application. The server side copy middleware is inserted to the left of dlo, slo and versioned_writes middlewares in the proxy server pipeline. As a result, dlo and slo copy_hooks are no longer required. SLO manifests are now validated when copied so when copying a manifest to another account the referenced segments must be readable in that account for the manifest copy to succeed (previously this validation was not made, meaning the manifest was copied but could be unusable if the segments were not readable). With this change, there should be no change in functionality or existing behavior. This is asserted with (almost) no changes required to existing functional tests. Some notes (for operators): * Middleware required to be auto-inserted before slo and dlo and versioned_writes * Turning off server side copy is not configurable. * object_post_as_copy is no longer a configurable option of proxy server but of this middleware. However, for smooth upgrade, config option set in proxy server app is also read. DocImpact: Introducing server side copy as middleware Co-Authored-By: Alistair Coles Co-Authored-By: Thiago da Silva Change-Id: Ic96a92e938589a2f6add35a40741fd062f1c29eb Signed-off-by: Prashanth Pai Signed-off-by: Thiago da Silva --- doc/saio/swift/proxy-server.conf | 5 +- doc/source/logs.rst | 1 + doc/source/middleware.rst | 9 + etc/proxy-server.conf-sample | 20 +- setup.cfg | 1 + swift/common/constraints.py | 62 - swift/common/middleware/account_quotas.py | 22 +- swift/common/middleware/container_quotas.py | 36 +- swift/common/middleware/copy.py | 522 ++++++++ swift/common/middleware/dlo.py | 23 - swift/common/middleware/slo.py | 19 - swift/common/middleware/versioned_writes.py | 32 +- swift/common/swob.py | 5 + swift/common/wsgi.py | 2 +- swift/proxy/controllers/obj.py | 317 +---- swift/proxy/server.py | 15 +- test/functional/tests.py | 94 +- test/unit/common/middleware/helpers.py | 6 + .../common/middleware/test_account_quotas.py | 169 +-- test/unit/common/middleware/test_copy.py | 1183 +++++++++++++++++ test/unit/common/middleware/test_dlo.py | 101 -- test/unit/common/middleware/test_quotas.py | 323 ++--- test/unit/common/middleware/test_slo.py | 66 +- .../middleware/test_versioned_writes.py | 225 ++-- test/unit/common/test_constraints.py | 81 -- test/unit/common/test_swob.py | 13 +- test/unit/common/test_wsgi.py | 13 +- test/unit/proxy/controllers/test_obj.py | 209 +-- test/unit/proxy/test_server.py | 950 +------------ test/unit/proxy/test_sysmeta.py | 91 -- 30 files changed, 2301 insertions(+), 2314 deletions(-) create mode 100644 swift/common/middleware/copy.py create mode 100644 test/unit/common/middleware/test_copy.py diff --git a/doc/saio/swift/proxy-server.conf b/doc/saio/swift/proxy-server.conf index d9e5c95148..76b85d5818 100644 --- a/doc/saio/swift/proxy-server.conf +++ b/doc/saio/swift/proxy-server.conf @@ -9,7 +9,7 @@ eventlet_debug = true [pipeline:main] # Yes, proxy-logging appears twice. This is so that # middleware-originated requests get logged too. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain container_sync tempauth staticweb container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain container_sync tempauth staticweb copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server [filter:catch_errors] use = egg:swift#catch_errors @@ -68,6 +68,9 @@ use = egg:swift#gatekeeper use = egg:swift#versioned_writes allow_versioned_writes = true +[filter:copy] +use = egg:swift#copy + [app:proxy-server] use = egg:swift#proxy allow_account_management = true diff --git a/doc/source/logs.rst b/doc/source/logs.rst index 75b669f1a5..7e2c1dd94b 100644 --- a/doc/source/logs.rst +++ b/doc/source/logs.rst @@ -103,6 +103,7 @@ LE :ref:`list_endpoints` KS :ref:`keystoneauth` RL :ref:`ratelimit` VW :ref:`versioned_writes` +SSC :ref:`copy` ======================= ============================= diff --git a/doc/source/middleware.rst b/doc/source/middleware.rst index 3c17339b17..a078747204 100644 --- a/doc/source/middleware.rst +++ b/doc/source/middleware.rst @@ -187,6 +187,15 @@ Recon :members: :show-inheritance: +.. _copy: + +Server Side Copy +================ + +.. automodule:: swift.common.middleware.copy + :members: + :show-inheritance: + Static Large Objects ==================== diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index ba860b9b9d..b5cfbf873b 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -79,12 +79,12 @@ bind_port = 8080 [pipeline:main] # This sample pipeline uses tempauth and is used for SAIO dev work and # testing. See below for a pipeline using keystone. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server # The following pipeline shows keystone integration. Comment out the one # above and uncomment this one. Additional steps for integrating keystone are # covered further below in the filter sections for authtoken and keystoneauth. -#pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit authtoken keystoneauth container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server +#pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit authtoken keystoneauth copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server [app:proxy-server] use = egg:swift#proxy @@ -129,11 +129,6 @@ use = egg:swift#proxy # 'false' no one, even authorized, can. # allow_account_management = false # -# Set object_post_as_copy = false to turn on fast posts where only the metadata -# changes are stored anew and the original data file is kept in place. This -# makes for quicker posts. -# object_post_as_copy = true -# # If set to 'true' authorized accounts that do not yet exist within the Swift # cluster will be automatically created. # account_autocreate = false @@ -749,3 +744,14 @@ use = egg:swift#versioned_writes # in the container configuration file, which will be eventually # deprecated. See documentation for more details. # allow_versioned_writes = false + +# Note: Put after auth and before dlo and slo middlewares. +# If you don't put it in the pipeline, it will be inserted for you. +[filter:copy] +use = egg:swift#copy +# Set object_post_as_copy = false to turn on fast posts where only the metadata +# changes are stored anew and the original data file is kept in place. This +# makes for quicker posts. +# When object_post_as_copy is set to True, a POST request will be transformed +# into a COPY request where source and destination objects are the same. +# object_post_as_copy = true diff --git a/setup.cfg b/setup.cfg index 77c6824b44..098b6c64f7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -96,6 +96,7 @@ paste.filter_factory = container_sync = swift.common.middleware.container_sync:filter_factory xprofile = swift.common.middleware.xprofile:filter_factory versioned_writes = swift.common.middleware.versioned_writes:filter_factory + copy = swift.common.middleware.copy:filter_factory [build_sphinx] all_files = 1 diff --git a/swift/common/constraints.py b/swift/common/constraints.py index abfab4bb9e..787d2d91da 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -20,7 +20,6 @@ import time import six from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError from six.moves import urllib -from six.moves.urllib.parse import unquote from swift.common import utils, exceptions from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \ @@ -205,10 +204,6 @@ def check_object_creation(req, object_name): request=req, content_type='text/plain') - if 'X-Copy-From' in req.headers and req.content_length: - return HTTPBadRequest(body='Copy requests require a zero byte body', - request=req, content_type='text/plain') - if len(object_name) > MAX_OBJECT_NAME_LENGTH: return HTTPBadRequest(body='Object name length of %d longer than %d' % (len(object_name), MAX_OBJECT_NAME_LENGTH), @@ -359,63 +354,6 @@ def check_utf8(string): return False -def check_path_header(req, name, length, error_msg): - """ - Validate that the value of path-like header is - well formatted. We assume the caller ensures that - specific header is present in req.headers. - - :param req: HTTP request object - :param name: header name - :param length: length of path segment check - :param error_msg: error message for client - :returns: A tuple with path parts according to length - :raise: HTTPPreconditionFailed if header value - is not well formatted. - """ - src_header = unquote(req.headers.get(name)) - if not src_header.startswith('/'): - src_header = '/' + src_header - try: - return utils.split_path(src_header, length, length, True) - except ValueError: - raise HTTPPreconditionFailed( - request=req, - body=error_msg) - - -def check_copy_from_header(req): - """ - Validate that the value from x-copy-from header is - well formatted. We assume the caller ensures that - x-copy-from header is present in req.headers. - - :param req: HTTP request object - :returns: A tuple with container name and object name - :raise: HTTPPreconditionFailed if x-copy-from value - is not well formatted. - """ - return check_path_header(req, 'X-Copy-From', 2, - 'X-Copy-From header must be of the form ' - '/') - - -def check_destination_header(req): - """ - Validate that the value from destination header is - well formatted. We assume the caller ensures that - destination header is present in req.headers. - - :param req: HTTP request object - :returns: A tuple with container name and object name - :raise: HTTPPreconditionFailed if destination value - is not well formatted. - """ - return check_path_header(req, 'Destination', 2, - 'Destination header must be of the form ' - '/') - - def check_name_format(req, name, target_type): """ Validate that the header contains valid account or container name. diff --git a/swift/common/middleware/account_quotas.py b/swift/common/middleware/account_quotas.py index fcb55b5573..8811aad84c 100644 --- a/swift/common/middleware/account_quotas.py +++ b/swift/common/middleware/account_quotas.py @@ -52,11 +52,10 @@ Due to the eventual consistency further uploads might be possible until the account size has been updated. """ -from swift.common.constraints import check_copy_from_header from swift.common.swob import HTTPForbidden, HTTPBadRequest, \ HTTPRequestEntityTooLarge, wsgify from swift.common.utils import register_swift_info -from swift.proxy.controllers.base import get_account_info, get_object_info +from swift.proxy.controllers.base import get_account_info class AccountQuotaMiddleware(object): @@ -71,7 +70,7 @@ class AccountQuotaMiddleware(object): @wsgify def __call__(self, request): - if request.method not in ("POST", "PUT", "COPY"): + if request.method not in ("POST", "PUT"): return self.app try: @@ -106,15 +105,6 @@ class AccountQuotaMiddleware(object): if request.method == "POST" or not obj: return self.app - if request.method == 'COPY': - copy_from = container + '/' + obj - else: - if 'x-copy-from' in request.headers: - src_cont, src_obj = check_copy_from_header(request) - copy_from = "%s/%s" % (src_cont, src_obj) - else: - copy_from = None - content_length = (request.content_length or 0) account_info = get_account_info(request.environ, self.app) @@ -127,14 +117,6 @@ class AccountQuotaMiddleware(object): if quota < 0: return self.app - if copy_from: - path = '/' + ver + '/' + account + '/' + copy_from - object_info = get_object_info(request.environ, self.app, path) - if not object_info or not object_info['length']: - content_length = 0 - else: - content_length = int(object_info['length']) - new_size = int(account_info['bytes']) + content_length if quota < new_size: resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.') diff --git a/swift/common/middleware/container_quotas.py b/swift/common/middleware/container_quotas.py index 4feca69a7b..a78876aca5 100644 --- a/swift/common/middleware/container_quotas.py +++ b/swift/common/middleware/container_quotas.py @@ -51,13 +51,11 @@ For example:: [filter:container_quotas] use = egg:swift#container_quotas """ -from swift.common.constraints import check_copy_from_header, \ - check_account_format, check_destination_header from swift.common.http import is_success from swift.common.swob import HTTPRequestEntityTooLarge, HTTPBadRequest, \ wsgify from swift.common.utils import register_swift_info -from swift.proxy.controllers.base import get_container_info, get_object_info +from swift.proxy.controllers.base import get_container_info class ContainerQuotaMiddleware(object): @@ -91,25 +89,9 @@ class ContainerQuotaMiddleware(object): return HTTPBadRequest(body='Invalid count quota.') # check user uploads against quotas - elif obj and req.method in ('PUT', 'COPY'): - container_info = None - if req.method == 'PUT': - container_info = get_container_info( - req.environ, self.app, swift_source='CQ') - if req.method == 'COPY' and 'Destination' in req.headers: - dest_account = account - if 'Destination-Account' in req.headers: - dest_account = req.headers.get('Destination-Account') - dest_account = check_account_format(req, dest_account) - dest_container, dest_object = check_destination_header(req) - path_info = req.environ['PATH_INFO'] - req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( - version, dest_account, dest_container, dest_object) - try: - container_info = get_container_info( - req.environ, self.app, swift_source='CQ') - finally: - req.environ['PATH_INFO'] = path_info + elif obj and req.method in ('PUT'): + container_info = get_container_info( + req.environ, self.app, swift_source='CQ') if not container_info or not is_success(container_info['status']): # this will hopefully 404 later return self.app @@ -118,16 +100,6 @@ class ContainerQuotaMiddleware(object): 'bytes' in container_info and \ container_info['meta']['quota-bytes'].isdigit(): content_length = (req.content_length or 0) - if 'x-copy-from' in req.headers or req.method == 'COPY': - if 'x-copy-from' in req.headers: - container, obj = check_copy_from_header(req) - path = '/%s/%s/%s/%s' % (version, account, - container, obj) - object_info = get_object_info(req.environ, self.app, path) - if not object_info or not object_info['length']: - content_length = 0 - else: - content_length = int(object_info['length']) new_size = int(container_info['bytes']) + content_length if int(container_info['meta']['quota-bytes']) < new_size: return self.bad_response(req, container_info) diff --git a/swift/common/middleware/copy.py b/swift/common/middleware/copy.py new file mode 100644 index 0000000000..e895813e8d --- /dev/null +++ b/swift/common/middleware/copy.py @@ -0,0 +1,522 @@ +# Copyright (c) 2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Server side copy is a feature that enables users/clients to COPY objects +between accounts and containers without the need to download and then +re-upload objects, thus eliminating additional bandwidth consumption and +also saving time. This may be used when renaming/moving an object which +in Swift is a (COPY + DELETE) operation. + +The server side copy middleware should be inserted in the pipeline after auth +and before the quotas and large object middlewares. If it is not present in the +pipeline in the proxy-server configuration file, it will be inserted +automatically. There is no configurable option provided to turn off server +side copy. + +-------- +Metadata +-------- +* All metadata of source object is preserved during object copy. +* One can also provide additional metadata during PUT/COPY request. This will + over-write any existing conflicting keys. +* Server side copy can also be used to change content-type of an existing + object. + +----------- +Object Copy +----------- +* The destination container must exist before requesting copy of the object. +* When several replicas exist, the system copies from the most recent replica. + That is, the copy operation behaves as though the X-Newest header is in the + request. +* The request to copy an object should have no body (i.e. content-length of the + request must be zero). + +There are two ways in which an object can be copied: + +1. Send a PUT request to the new object (destination/target) with an additional + header named ``X-Copy-From`` specifying the source object + (in '/container/object' format). Example:: + + curl -i -X PUT http:///container1/destination_obj + -H 'X-Auth-Token: ' + -H 'X-Copy-From: /container2/source_obj' + -H 'Content-Length: 0' + +2. Send a COPY request with an existing object in URL with an additional header + named ``Destination`` specifying the destination/target object + (in '/container/object' format). Example:: + + curl -i -X COPY http:///container2/source_obj + -H 'X-Auth-Token: ' + -H 'Destination: /container1/destination_obj' + -H 'Content-Length: 0' + +Note that if the incoming request has some conditional headers (e.g. ``Range``, +``If-Match``), the *source* object will be evaluated for these headers (i.e. if +PUT with both ``X-Copy-From`` and ``Range``, Swift will make a partial copy to +the destination object). + +------------------------- +Cross Account Object Copy +------------------------- +Objects can also be copied from one account to another account if the user +has the necessary permissions (i.e. permission to read from container +in source account and permission to write to container in destination account). + +Similar to examples mentioned above, there are two ways to copy objects across +accounts: + +1. Like the example above, send PUT request to copy object but with an + additional header named ``X-Copy-From-Account`` specifying the source + account. Example:: + + curl -i -X PUT http://:/v1/AUTH_test1/container/destination_obj + -H 'X-Auth-Token: ' + -H 'X-Copy-From: /container/source_obj' + -H 'X-Copy-From-Account: AUTH_test2' + -H 'Content-Length: 0' + +2. Like the previous example, send a COPY request but with an additional header + named ``Destination-Account`` specifying the name of destination account. + Example:: + + curl -i -X COPY http://:/v1/AUTH_test2/container/source_obj + -H 'X-Auth-Token: ' + -H 'Destination: /container/destination_obj' + -H 'Destination-Account: AUTH_test1' + -H 'Content-Length: 0' + +------------------- +Large Object Copy +------------------- +The best option to copy a large option is to copy segments individually. +To copy the manifest object of a large object, add the query parameter to +the copy request:: + + ?multipart-manifest=get + +If a request is sent without the query parameter, an attempt will be made to +copy the whole object but will fail if the object size is +greater than 5GB. + +------------------- +Object Post as Copy +------------------- +Historically, this has been a feature (and a configurable option with default +set to True) in proxy server configuration. This has been moved to server side +copy middleware. + +When ``object_post_as_copy`` is set to ``true`` (default value), an incoming +POST request is morphed into a COPY request where source and destination +objects are same. + +This feature was necessary because of a previous behavior where POSTS would +update the metadata on the object but not on the container. As a result, +features like container sync would not work correctly. This is no longer the +case and the plan is to deprecate this option. It is being kept now for +backwards compatibility. At first chance, set ``object_post_as_copy`` to +``false``. +""" + +import os +from urllib import quote +from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from six.moves.urllib.parse import unquote + +from swift.common import utils +from swift.common.utils import get_logger, \ + config_true_value, FileLikeIter, read_conf_dir, close_if_possible +from swift.common.swob import Request, HTTPPreconditionFailed, \ + HTTPRequestEntityTooLarge, HTTPBadRequest +from swift.common.http import HTTP_MULTIPLE_CHOICES, HTTP_CREATED, \ + is_success +from swift.common.constraints import check_account_format, MAX_FILE_SIZE +from swift.common.request_helpers import copy_header_subset, remove_items, \ + is_sys_meta, is_sys_or_user_meta +from swift.common.wsgi import WSGIContext, make_subrequest + + +def _check_path_header(req, name, length, error_msg): + """ + Validate that the value of path-like header is + well formatted. We assume the caller ensures that + specific header is present in req.headers. + + :param req: HTTP request object + :param name: header name + :param length: length of path segment check + :param error_msg: error message for client + :returns: A tuple with path parts according to length + :raise: HTTPPreconditionFailed if header value + is not well formatted. + """ + src_header = unquote(req.headers.get(name)) + if not src_header.startswith('/'): + src_header = '/' + src_header + try: + return utils.split_path(src_header, length, length, True) + except ValueError: + raise HTTPPreconditionFailed( + request=req, + body=error_msg) + + +def _check_copy_from_header(req): + """ + Validate that the value from x-copy-from header is + well formatted. We assume the caller ensures that + x-copy-from header is present in req.headers. + + :param req: HTTP request object + :returns: A tuple with container name and object name + :raise: HTTPPreconditionFailed if x-copy-from value + is not well formatted. + """ + return _check_path_header(req, 'X-Copy-From', 2, + 'X-Copy-From header must be of the form ' + '/') + + +def _check_destination_header(req): + """ + Validate that the value from destination header is + well formatted. We assume the caller ensures that + destination header is present in req.headers. + + :param req: HTTP request object + :returns: A tuple with container name and object name + :raise: HTTPPreconditionFailed if destination value + is not well formatted. + """ + return _check_path_header(req, 'Destination', 2, + 'Destination header must be of the form ' + '/') + + +def _copy_headers_into(from_r, to_r): + """ + Will copy desired headers from from_r to to_r + :params from_r: a swob Request or Response + :params to_r: a swob Request or Response + """ + pass_headers = ['x-delete-at'] + for k, v in from_r.headers.items(): + if is_sys_or_user_meta('object', k) or k.lower() in pass_headers: + to_r.headers[k] = v + + +class ServerSideCopyWebContext(WSGIContext): + + def __init__(self, app, logger): + super(ServerSideCopyWebContext, self).__init__(app) + self.app = app + self.logger = logger + + def get_source_resp(self, req): + sub_req = make_subrequest( + req.environ, path=req.path_info, headers=req.headers, + swift_source='SSC') + return sub_req.get_response(self.app) + + def send_put_req(self, req, additional_resp_headers, start_response): + app_resp = self._app_call(req.environ) + self._adjust_put_response(req, additional_resp_headers) + start_response(self._response_status, + self._response_headers, + self._response_exc_info) + return app_resp + + def _adjust_put_response(self, req, additional_resp_headers): + if 'swift.post_as_copy' in req.environ: + # Older editions returned 202 Accepted on object POSTs, so we'll + # convert any 201 Created responses to that for compatibility with + # picky clients. + if self._get_status_int() == HTTP_CREATED: + self._response_status = '202 Accepted' + elif is_success(self._get_status_int()): + for header, value in additional_resp_headers.items(): + self._response_headers.append((header, value)) + + def handle_OPTIONS_request(self, req, start_response): + app_resp = self._app_call(req.environ) + if is_success(self._get_status_int()): + for i, (header, value) in enumerate(self._response_headers): + if header.lower() == 'allow' and 'COPY' not in value: + self._response_headers[i] = ('Allow', value + ', COPY') + if header.lower() == 'access-control-allow-methods' and \ + 'COPY' not in value: + self._response_headers[i] = \ + ('Access-Control-Allow-Methods', value + ', COPY') + start_response(self._response_status, + self._response_headers, + self._response_exc_info) + return app_resp + + +class ServerSideCopyMiddleware(object): + + def __init__(self, app, conf): + self.app = app + self.logger = get_logger(conf, log_route="copy") + # Read the old object_post_as_copy option from Proxy app just in case + # someone has set it to false (non default). This wouldn't cause + # problems during upgrade. + self._load_object_post_as_copy_conf(conf) + self.object_post_as_copy = \ + config_true_value(conf.get('object_post_as_copy', 'true')) + + def _load_object_post_as_copy_conf(self, conf): + if ('object_post_as_copy' in conf or '__file__' not in conf): + # Option is explicitly set in middleware conf. In that case, + # we assume operator knows what he's doing. + # This takes preference over the one set in proxy app + return + + cp = ConfigParser() + if os.path.isdir(conf['__file__']): + read_conf_dir(cp, conf['__file__']) + else: + cp.read(conf['__file__']) + + try: + pipe = cp.get("pipeline:main", "pipeline") + except (NoSectionError, NoOptionError): + return + + proxy_name = pipe.rsplit(None, 1)[-1] + proxy_section = "app:" + proxy_name + + try: + conf['object_post_as_copy'] = cp.get(proxy_section, + 'object_post_as_copy') + except (NoSectionError, NoOptionError): + pass + + def __call__(self, env, start_response): + req = Request(env) + try: + (version, account, container, obj) = req.split_path(4, 4, True) + except ValueError: + # If obj component is not present in req, do not proceed further. + return self.app(env, start_response) + + self.account_name = account + self.container_name = container + self.object_name = obj + + # Save off original request method (COPY/POST) in case it gets mutated + # into PUT during handling. This way logging can display the method + # the client actually sent. + req.environ['swift.orig_req_method'] = req.method + + if req.method == 'PUT' and req.headers.get('X-Copy-From'): + return self.handle_PUT(req, start_response) + elif req.method == 'COPY': + return self.handle_COPY(req, start_response) + elif req.method == 'POST' and self.object_post_as_copy: + return self.handle_object_post_as_copy(req, start_response) + elif req.method == 'OPTIONS': + # Does not interfere with OPTIONS response from (account,container) + # servers and /info response. + return self.handle_OPTIONS(req, start_response) + + return self.app(env, start_response) + + def handle_object_post_as_copy(self, req, start_response): + req.method = 'PUT' + req.path_info = '/v1/%s/%s/%s' % ( + self.account_name, self.container_name, self.object_name) + req.headers['Content-Length'] = 0 + req.headers.pop('Range', None) + req.headers['X-Copy-From'] = quote('/%s/%s' % (self.container_name, + self.object_name)) + req.environ['swift.post_as_copy'] = True + params = req.params + # for post-as-copy always copy the manifest itself if source is *LO + params['multipart-manifest'] = 'get' + req.params = params + return self.handle_PUT(req, start_response) + + def handle_COPY(self, req, start_response): + if not req.headers.get('Destination'): + return HTTPPreconditionFailed(request=req, + body='Destination header required' + )(req.environ, start_response) + dest_account = self.account_name + if 'Destination-Account' in req.headers: + dest_account = req.headers.get('Destination-Account') + dest_account = check_account_format(req, dest_account) + req.headers['X-Copy-From-Account'] = self.account_name + self.account_name = dest_account + del req.headers['Destination-Account'] + dest_container, dest_object = _check_destination_header(req) + source = '/%s/%s' % (self.container_name, self.object_name) + self.container_name = dest_container + self.object_name = dest_object + # re-write the existing request as a PUT instead of creating a new one + req.method = 'PUT' + # As this the path info is updated with destination container, + # the proxy server app will use the right object controller + # implementation corresponding to the container's policy type. + ver, _junk = req.split_path(1, 2, rest_with_last=True) + req.path_info = '/%s/%s/%s/%s' % \ + (ver, dest_account, dest_container, dest_object) + req.headers['Content-Length'] = 0 + req.headers['X-Copy-From'] = quote(source) + del req.headers['Destination'] + return self.handle_PUT(req, start_response) + + def _get_source_object(self, ssc_ctx, source_path, req): + source_req = req.copy_get() + + # make sure the source request uses it's container_info + source_req.headers.pop('X-Backend-Storage-Policy-Index', None) + source_req.path_info = quote(source_path) + source_req.headers['X-Newest'] = 'true' + if 'swift.post_as_copy' in req.environ: + # We're COPYing one object over itself because of a POST; rely on + # the PUT for write authorization, don't require read authorization + source_req.environ['swift.authorize'] = lambda req: None + source_req.environ['swift.authorize_override'] = True + + # in case we are copying an SLO manifest, set format=raw parameter + params = source_req.params + if params.get('multipart-manifest') == 'get': + params['format'] = 'raw' + source_req.params = params + + source_resp = ssc_ctx.get_source_resp(source_req) + + if source_resp.content_length is None: + # This indicates a transfer-encoding: chunked source object, + # which currently only happens because there are more than + # CONTAINER_LISTING_LIMIT segments in a segmented object. In + # this case, we're going to refuse to do the server-side copy. + return HTTPRequestEntityTooLarge(request=req) + + if source_resp.content_length > MAX_FILE_SIZE: + return HTTPRequestEntityTooLarge(request=req) + + return source_resp + + def _create_response_headers(self, source_path, source_resp, sink_req): + resp_headers = dict() + acct, path = source_path.split('/', 3)[2:4] + resp_headers['X-Copied-From-Account'] = quote(acct) + resp_headers['X-Copied-From'] = quote(path) + if 'last-modified' in source_resp.headers: + resp_headers['X-Copied-From-Last-Modified'] = \ + source_resp.headers['last-modified'] + # Existing sys and user meta of source object is added to response + # headers in addition to the new ones. + for k, v in sink_req.headers.items(): + if is_sys_or_user_meta('object', k) or k.lower() == 'x-delete-at': + resp_headers[k] = v + return resp_headers + + def handle_PUT(self, req, start_response): + if req.content_length: + return HTTPBadRequest(body='Copy requests require a zero byte ' + 'body', request=req, + content_type='text/plain')(req.environ, + start_response) + + # Form the path of source object to be fetched + ver, acct, _rest = req.split_path(2, 3, True) + src_account_name = req.headers.get('X-Copy-From-Account') + if src_account_name: + src_account_name = check_account_format(req, src_account_name) + else: + src_account_name = acct + src_container_name, src_obj_name = _check_copy_from_header(req) + source_path = '/%s/%s/%s/%s' % (ver, src_account_name, + src_container_name, src_obj_name) + + if req.environ.get('swift.orig_req_method', req.method) != 'POST': + self.logger.info("Copying object from %s to %s" % + (source_path, req.path)) + + # GET the source object, bail out on error + ssc_ctx = ServerSideCopyWebContext(self.app, self.logger) + source_resp = self._get_source_object(ssc_ctx, source_path, req) + if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: + close_if_possible(source_resp.app_iter) + return source_resp(source_resp.environ, start_response) + + # Create a new Request object based on the original req instance. + # This will preserve env and headers. + sink_req = Request.blank(req.path_info, + environ=req.environ, headers=req.headers) + + params = sink_req.params + if params.get('multipart-manifest') == 'get': + if 'X-Static-Large-Object' in source_resp.headers: + params['multipart-manifest'] = 'put' + if 'X-Object-Manifest' in source_resp.headers: + del params['multipart-manifest'] + sink_req.headers['X-Object-Manifest'] = \ + source_resp.headers['X-Object-Manifest'] + sink_req.params = params + + # Set data source, content length and etag for the PUT request + sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) + sink_req.content_length = source_resp.content_length + sink_req.etag = source_resp.etag + + # We no longer need these headers + sink_req.headers.pop('X-Copy-From', None) + sink_req.headers.pop('X-Copy-From-Account', None) + # If the copy request does not explicitly override content-type, + # use the one present in the source object. + if not req.headers.get('content-type'): + sink_req.headers['Content-Type'] = \ + source_resp.headers['Content-Type'] + + fresh_meta_flag = config_true_value( + sink_req.headers.get('x-fresh-metadata', 'false')) + + if fresh_meta_flag or 'swift.post_as_copy' in sink_req.environ: + # Post-as-copy: ignore new sysmeta, copy existing sysmeta + condition = lambda k: is_sys_meta('object', k) + remove_items(sink_req.headers, condition) + copy_header_subset(source_resp, sink_req, condition) + else: + # Copy/update existing sysmeta and user meta + _copy_headers_into(source_resp, sink_req) + # Copy/update new metadata provided in request if any + _copy_headers_into(req, sink_req) + + # Create response headers for PUT response + resp_headers = self._create_response_headers(source_path, + source_resp, sink_req) + + put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response) + close_if_possible(source_resp.app_iter) + return put_resp + + def handle_OPTIONS(self, req, start_response): + return ServerSideCopyWebContext(self.app, self.logger).\ + handle_OPTIONS_request(req, start_response) + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + + def copy_filter(app): + return ServerSideCopyMiddleware(app, conf) + + return copy_filter diff --git a/swift/common/middleware/dlo.py b/swift/common/middleware/dlo.py index 2fd37c3d29..1c27800eb2 100644 --- a/swift/common/middleware/dlo.py +++ b/swift/common/middleware/dlo.py @@ -405,11 +405,6 @@ class DynamicLargeObject(object): except ValueError: return self.app(env, start_response) - # install our COPY-callback hook - env['swift.copy_hook'] = self.copy_hook( - env.get('swift.copy_hook', - lambda src_req, src_resp, sink_req: src_resp)) - if ((req.method == 'GET' or req.method == 'HEAD') and req.params.get('multipart-manifest') != 'get'): return GetContext(self, self.logger).\ @@ -438,24 +433,6 @@ class DynamicLargeObject(object): body=('X-Object-Manifest must be in the ' 'format container/prefix')) - def copy_hook(self, inner_hook): - - def dlo_copy_hook(source_req, source_resp, sink_req): - x_o_m = source_resp.headers.get('X-Object-Manifest') - if x_o_m: - if source_req.params.get('multipart-manifest') == 'get': - # To copy the manifest, we let the copy proceed as normal, - # but ensure that X-Object-Manifest is set on the new - # object. - sink_req.headers['X-Object-Manifest'] = x_o_m - else: - ctx = GetContext(self, self.logger) - source_resp = ctx.get_or_head_response( - source_req, x_o_m, source_resp.headers.items()) - return inner_hook(source_req, source_resp, sink_req) - - return dlo_copy_hook - def filter_factory(global_conf, **local_conf): conf = global_conf.copy() diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index 0216264b99..b87c8f2984 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -798,20 +798,6 @@ class StaticLargeObject(object): """ return SloGetContext(self).handle_slo_get_or_head(req, start_response) - def copy_hook(self, inner_hook): - - def slo_hook(source_req, source_resp, sink_req): - x_slo = source_resp.headers.get('X-Static-Large-Object') - if (config_true_value(x_slo) - and source_req.params.get('multipart-manifest') != 'get' - and 'swift.post_as_copy' not in source_req.environ): - source_resp = SloGetContext(self).get_or_head_response( - source_req, source_resp.headers.items(), - source_resp.app_iter) - return inner_hook(source_req, source_resp, sink_req) - - return slo_hook - def handle_multipart_put(self, req, start_response): """ Will handle the PUT of a SLO manifest. @@ -1058,11 +1044,6 @@ class StaticLargeObject(object): except ValueError: return self.app(env, start_response) - # install our COPY-callback hook - env['swift.copy_hook'] = self.copy_hook( - env.get('swift.copy_hook', - lambda src_req, src_resp, sink_req: src_resp)) - try: if req.method == 'PUT' and \ req.params.get('multipart-manifest') == 'put': diff --git a/swift/common/middleware/versioned_writes.py b/swift/common/middleware/versioned_writes.py index 3cb0989bba..ae091cff20 100644 --- a/swift/common/middleware/versioned_writes.py +++ b/swift/common/middleware/versioned_writes.py @@ -127,9 +127,7 @@ from swift.common.request_helpers import get_sys_meta_prefix, \ from swift.common.wsgi import WSGIContext, make_pre_authed_request from swift.common.swob import ( Request, HTTPException, HTTPRequestEntityTooLarge) -from swift.common.constraints import ( - check_account_format, check_container_format, check_destination_header, - MAX_FILE_SIZE) +from swift.common.constraints import check_container_format, MAX_FILE_SIZE from swift.proxy.controllers.base import get_container_info from swift.common.http import ( is_success, is_client_error, HTTP_NOT_FOUND) @@ -493,24 +491,10 @@ class VersionedWritesMiddleware(object): account_name = unquote(account) container_name = unquote(container) object_name = unquote(obj) - container_info = None resp = None is_enabled = config_true_value(allow_versioned_writes) - if req.method in ('PUT', 'DELETE'): - container_info = get_container_info( - req.environ, self.app) - elif req.method == 'COPY' and 'Destination' in req.headers: - if 'Destination-Account' in req.headers: - account_name = req.headers.get('Destination-Account') - account_name = check_account_format(req, account_name) - container_name, object_name = check_destination_header(req) - req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( - api_version, account_name, container_name, object_name) - container_info = get_container_info( - req.environ, self.app) - - if not container_info: - return self.app + container_info = get_container_info( + req.environ, self.app) # To maintain backwards compatibility, container version # location could be stored as sysmeta or not, need to check both. @@ -530,7 +514,7 @@ class VersionedWritesMiddleware(object): if is_enabled and versions_cont: versions_cont = unquote(versions_cont).split('/')[0] vw_ctx = VersionedWritesContext(self.app, self.logger) - if req.method in ('PUT', 'COPY'): + if req.method == 'PUT': resp = vw_ctx.handle_obj_versions_put( req, versions_cont, api_version, account_name, object_name) @@ -545,10 +529,7 @@ class VersionedWritesMiddleware(object): return self.app def __call__(self, env, start_response): - # making a duplicate, because if this is a COPY request, we will - # modify the PATH_INFO to find out if the 'Destination' is in a - # versioned container - req = Request(env.copy()) + req = Request(env) try: (api_version, account, container, obj) = req.split_path(3, 4, True) except ValueError: @@ -576,7 +557,8 @@ class VersionedWritesMiddleware(object): allow_versioned_writes) except HTTPException as error_response: return error_response(env, start_response) - elif obj and req.method in ('PUT', 'COPY', 'DELETE'): + elif (obj and req.method in ('PUT', 'DELETE') and + not req.environ.get('swift.post_as_copy')): try: return self.object_request( req, api_version, account, container, obj, diff --git a/swift/common/swob.py b/swift/common/swob.py index 0954ef9d3c..f895c44f74 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -888,6 +888,11 @@ class Request(object): return self._params_cache str_params = params + @params.setter + def params(self, param_pairs): + self._params_cache = None + self.query_string = urllib.parse.urlencode(param_pairs) + @property def timestamp(self): """ diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 2c169eb2a6..534333999e 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -1100,7 +1100,7 @@ def make_env(env, method=None, path=None, agent='Swift', query_string=None, 'SERVER_PROTOCOL', 'swift.cache', 'swift.source', 'swift.trans_id', 'swift.authorize_override', 'swift.authorize', 'HTTP_X_USER_ID', 'HTTP_X_PROJECT_ID', - 'HTTP_REFERER'): + 'HTTP_REFERER', 'swift.orig_req_method', 'swift.log_info'): if name in env: newenv[name] = env[name] if method: diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 70400fc143..6f8559063a 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -25,7 +25,7 @@ # collected. We've seen objects hang around forever otherwise. import six -from six.moves.urllib.parse import unquote, quote +from six.moves.urllib.parse import unquote import collections import itertools @@ -49,9 +49,7 @@ from swift.common.utils import ( document_iters_to_http_response_body, parse_content_range, quorum_size, reiterate, close_if_possible) from swift.common.bufferedhttp import http_connect -from swift.common.constraints import check_metadata, check_object_creation, \ - check_copy_from_header, check_destination_header, \ - check_account_format +from swift.common.constraints import check_metadata, check_object_creation from swift.common import constraints from swift.common.exceptions import ChunkReadTimeout, \ ChunkWriteTimeout, ConnectionTimeout, ResponseTimeout, \ @@ -60,33 +58,19 @@ from swift.common.exceptions import ChunkReadTimeout, \ from swift.common.header_key_dict import HeaderKeyDict from swift.common.http import ( is_informational, is_success, is_client_error, is_server_error, - is_redirection, HTTP_CONTINUE, HTTP_CREATED, HTTP_MULTIPLE_CHOICES, - HTTP_INTERNAL_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE, - HTTP_INSUFFICIENT_STORAGE, HTTP_PRECONDITION_FAILED, HTTP_CONFLICT, - HTTP_UNPROCESSABLE_ENTITY, HTTP_REQUESTED_RANGE_NOT_SATISFIABLE) + is_redirection, HTTP_CONTINUE, HTTP_INTERNAL_SERVER_ERROR, + HTTP_SERVICE_UNAVAILABLE, HTTP_INSUFFICIENT_STORAGE, + HTTP_PRECONDITION_FAILED, HTTP_CONFLICT, HTTP_UNPROCESSABLE_ENTITY, + HTTP_REQUESTED_RANGE_NOT_SATISFIABLE) from swift.common.storage_policy import (POLICIES, REPL_POLICY, EC_POLICY, ECDriverError, PolicyError) from swift.proxy.controllers.base import Controller, delay_denial, \ cors_validation, ResumingGetter from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \ - HTTPServerError, HTTPServiceUnavailable, Request, \ - HTTPClientDisconnect, HTTPUnprocessableEntity, Response, HTTPException, \ + HTTPServerError, HTTPServiceUnavailable, HTTPClientDisconnect, \ + HTTPUnprocessableEntity, Response, HTTPException, \ HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError -from swift.common.request_helpers import is_sys_or_user_meta, is_sys_meta, \ - remove_items, copy_header_subset - - -def copy_headers_into(from_r, to_r): - """ - Will copy desired headers from from_r to to_r - :params from_r: a swob Request or Response - :params to_r: a swob Request or Response - """ - pass_headers = ['x-delete-at'] - for k, v in from_r.headers.items(): - if is_sys_or_user_meta('object', k) or k.lower() in pass_headers: - to_r.headers[k] = v def check_content_type(req): @@ -200,8 +184,7 @@ class BaseObjectController(Controller): self.account_name, self.container_name, self.object_name) node_iter = self.app.iter_nodes(obj_ring, partition) - resp = self._reroute(policy)._get_or_head_response( - req, node_iter, partition, policy) + resp = self._get_or_head_response(req, node_iter, partition, policy) if ';' in resp.headers.get('content-type', ''): resp.content_type = clean_content_type( @@ -227,55 +210,38 @@ class BaseObjectController(Controller): @delay_denial def POST(self, req): """HTTP POST request handler.""" - if self.app.object_post_as_copy: - req.method = 'PUT' - req.path_info = '/v1/%s/%s/%s' % ( - self.account_name, self.container_name, self.object_name) - req.headers['Content-Length'] = 0 - req.headers['X-Copy-From'] = quote('/%s/%s' % (self.container_name, - self.object_name)) - req.environ['swift.post_as_copy'] = True - req.environ['swift_versioned_copy'] = True - resp = self.PUT(req) - # Older editions returned 202 Accepted on object POSTs, so we'll - # convert any 201 Created responses to that for compatibility with - # picky clients. - if resp.status_int != HTTP_CREATED: - return resp - return HTTPAccepted(request=req) - else: - error_response = check_metadata(req, 'object') - if error_response: - return error_response - container_info = self.container_info( - self.account_name, self.container_name, req) - container_partition = container_info['partition'] - containers = container_info['nodes'] - req.acl = container_info['write_acl'] - if 'swift.authorize' in req.environ: - aresp = req.environ['swift.authorize'](req) - if aresp: - return aresp - if not containers: - return HTTPNotFound(request=req) + error_response = check_metadata(req, 'object') + if error_response: + return error_response + container_info = self.container_info( + self.account_name, self.container_name, req) + container_partition = container_info['partition'] + containers = container_info['nodes'] + req.acl = container_info['write_acl'] + if 'swift.authorize' in req.environ: + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp + if not containers: + return HTTPNotFound(request=req) - req, delete_at_container, delete_at_part, \ - delete_at_nodes = self._config_obj_expiration(req) + req, delete_at_container, delete_at_part, \ + delete_at_nodes = self._config_obj_expiration(req) - # pass the policy index to storage nodes via req header - policy_index = req.headers.get('X-Backend-Storage-Policy-Index', - container_info['storage_policy']) - obj_ring = self.app.get_object_ring(policy_index) - req.headers['X-Backend-Storage-Policy-Index'] = policy_index - partition, nodes = obj_ring.get_nodes( - self.account_name, self.container_name, self.object_name) + # pass the policy index to storage nodes via req header + policy_index = req.headers.get('X-Backend-Storage-Policy-Index', + container_info['storage_policy']) + obj_ring = self.app.get_object_ring(policy_index) + req.headers['X-Backend-Storage-Policy-Index'] = policy_index + partition, nodes = obj_ring.get_nodes( + self.account_name, self.container_name, self.object_name) - req.headers['X-Timestamp'] = Timestamp(time.time()).internal + req.headers['X-Timestamp'] = Timestamp(time.time()).internal - headers = self._backend_requests( - req, len(nodes), container_partition, containers, - delete_at_container, delete_at_part, delete_at_nodes) - return self._post_object(req, obj_ring, partition, headers) + headers = self._backend_requests( + req, len(nodes), container_partition, containers, + delete_at_container, delete_at_part, delete_at_nodes) + return self._post_object(req, obj_ring, partition, headers) def _backend_requests(self, req, n_outgoing, container_partition, containers, @@ -414,133 +380,8 @@ class BaseObjectController(Controller): return req, delete_at_container, delete_at_part, delete_at_nodes - def _handle_copy_request(self, req): - """ - This method handles copying objects based on values set in the headers - 'X-Copy-From' and 'X-Copy-From-Account' - - Note that if the incomming request has some conditional headers (e.g. - 'Range', 'If-Match'), *source* object will be evaluated for these - headers. i.e. if PUT with both 'X-Copy-From' and 'Range', Swift will - make a partial copy as a new object. - - This method was added as part of the refactoring of the PUT method and - the functionality is expected to be moved to middleware - """ - if req.environ.get('swift.orig_req_method', req.method) != 'POST': - req.environ.setdefault('swift.log_info', []).append( - 'x-copy-from:%s' % req.headers['X-Copy-From']) - ver, acct, _rest = req.split_path(2, 3, True) - src_account_name = req.headers.get('X-Copy-From-Account', None) - if src_account_name: - src_account_name = check_account_format(req, src_account_name) - else: - src_account_name = acct - src_container_name, src_obj_name = check_copy_from_header(req) - source_header = '/%s/%s/%s/%s' % ( - ver, src_account_name, src_container_name, src_obj_name) - source_req = req.copy_get() - - # make sure the source request uses it's container_info - source_req.headers.pop('X-Backend-Storage-Policy-Index', None) - source_req.path_info = source_header - source_req.headers['X-Newest'] = 'true' - if 'swift.post_as_copy' in req.environ: - # We're COPYing one object over itself because of a POST; rely on - # the PUT for write authorization, don't require read authorization - source_req.environ['swift.authorize'] = lambda req: None - source_req.environ['swift.authorize_override'] = True - - orig_obj_name = self.object_name - orig_container_name = self.container_name - orig_account_name = self.account_name - sink_req = Request.blank(req.path_info, - environ=req.environ, headers=req.headers) - - self.object_name = src_obj_name - self.container_name = src_container_name - self.account_name = src_account_name - - source_resp = self.GET(source_req) - - # This gives middlewares a way to change the source; for example, - # this lets you COPY a SLO manifest and have the new object be the - # concatenation of the segments (like what a GET request gives - # the client), not a copy of the manifest file. - hook = req.environ.get( - 'swift.copy_hook', - (lambda source_req, source_resp, sink_req: source_resp)) - source_resp = hook(source_req, source_resp, sink_req) - - # reset names - self.object_name = orig_obj_name - self.container_name = orig_container_name - self.account_name = orig_account_name - - if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: - # this is a bit of ugly code, but I'm willing to live with it - # until copy request handling moves to middleware - return source_resp, None, None, None - if source_resp.content_length is None: - # This indicates a transfer-encoding: chunked source object, - # which currently only happens because there are more than - # CONTAINER_LISTING_LIMIT segments in a segmented object. In - # this case, we're going to refuse to do the server-side copy. - raise HTTPRequestEntityTooLarge(request=req) - if source_resp.content_length > constraints.MAX_FILE_SIZE: - raise HTTPRequestEntityTooLarge(request=req) - - data_source = iter(source_resp.app_iter) - sink_req.content_length = source_resp.content_length - sink_req.etag = source_resp.etag - - # we no longer need the X-Copy-From header - del sink_req.headers['X-Copy-From'] - if 'X-Copy-From-Account' in sink_req.headers: - del sink_req.headers['X-Copy-From-Account'] - if not req.content_type_manually_set: - sink_req.headers['Content-Type'] = \ - source_resp.headers['Content-Type'] - - fresh_meta_flag = config_true_value( - sink_req.headers.get('x-fresh-metadata', 'false')) - - if fresh_meta_flag or 'swift.post_as_copy' in sink_req.environ: - # post-as-copy: ignore new sysmeta, copy existing sysmeta - condition = lambda k: is_sys_meta('object', k) - remove_items(sink_req.headers, condition) - copy_header_subset(source_resp, sink_req, condition) - else: - # copy/update existing sysmeta and user meta - copy_headers_into(source_resp, sink_req) - copy_headers_into(req, sink_req) - - # copy over x-static-large-object for POSTs and manifest copies - if 'X-Static-Large-Object' in source_resp.headers and \ - (req.params.get('multipart-manifest') == 'get' or - 'swift.post_as_copy' in req.environ): - sink_req.headers['X-Static-Large-Object'] = \ - source_resp.headers['X-Static-Large-Object'] - - req = sink_req - - def update_response(req, resp): - acct, path = source_resp.environ['PATH_INFO'].split('/', 3)[2:4] - resp.headers['X-Copied-From-Account'] = quote(acct) - resp.headers['X-Copied-From'] = quote(path) - if 'last-modified' in source_resp.headers: - resp.headers['X-Copied-From-Last-Modified'] = \ - source_resp.headers['last-modified'] - copy_headers_into(req, resp) - return resp - - # this is a bit of ugly code, but I'm willing to live with it - # until copy request handling moves to middleware - return None, req, data_source, update_response - def _update_content_type(self, req): # Sometimes the 'content-type' header exists, but is set to None. - req.content_type_manually_set = True detect_content_type = \ config_true_value(req.headers.get('x-detect-content-type')) if detect_content_type or not req.headers.get('content-type'): @@ -549,8 +390,6 @@ class BaseObjectController(Controller): 'application/octet-stream' if detect_content_type: req.headers.pop('x-detect-content-type') - else: - req.content_type_manually_set = False def _update_x_timestamp(self, req): # Used by container sync feature @@ -744,22 +583,13 @@ class BaseObjectController(Controller): self._update_x_timestamp(req) - # check if request is a COPY of an existing object - source_header = req.headers.get('X-Copy-From') - if source_header: - error_response, req, data_source, update_response = \ - self._handle_copy_request(req) - if error_response: - return error_response - else: - def reader(): - try: - return req.environ['wsgi.input'].read( - self.app.client_chunk_size) - except (ValueError, IOError) as e: - raise ChunkReadError(str(e)) - data_source = iter(reader, '') - update_response = lambda req, resp: resp + def reader(): + try: + return req.environ['wsgi.input'].read( + self.app.client_chunk_size) + except (ValueError, IOError) as e: + raise ChunkReadError(str(e)) + data_source = iter(reader, '') # check if object is set to be automatically deleted (i.e. expired) req, delete_at_container, delete_at_part, \ @@ -773,7 +603,7 @@ class BaseObjectController(Controller): # send object to storage nodes resp = self._store_object( req, data_source, nodes, partition, outgoing_headers) - return update_response(req, resp) + return resp @public @cors_validation @@ -817,63 +647,6 @@ class BaseObjectController(Controller): req, len(nodes), container_partition, containers) return self._delete_object(req, obj_ring, partition, headers) - def _reroute(self, policy): - """ - For COPY requests we need to make sure the controller instance the - request is routed through is the correct type for the policy. - """ - if not policy: - raise HTTPServiceUnavailable('Unknown Storage Policy') - if policy.policy_type != self.policy_type: - controller = self.app.obj_controller_router[policy]( - self.app, self.account_name, self.container_name, - self.object_name) - else: - controller = self - return controller - - @public - @cors_validation - @delay_denial - def COPY(self, req): - """HTTP COPY request handler.""" - if not req.headers.get('Destination'): - return HTTPPreconditionFailed(request=req, - body='Destination header required') - dest_account = self.account_name - if 'Destination-Account' in req.headers: - dest_account = req.headers.get('Destination-Account') - dest_account = check_account_format(req, dest_account) - req.headers['X-Copy-From-Account'] = self.account_name - self.account_name = dest_account - del req.headers['Destination-Account'] - dest_container, dest_object = check_destination_header(req) - - source = '/%s/%s' % (self.container_name, self.object_name) - self.container_name = dest_container - self.object_name = dest_object - # re-write the existing request as a PUT instead of creating a new one - # since this one is already attached to the posthooklogger - # TODO: Swift now has proxy-logging middleware instead of - # posthooklogger used in before. i.e. we don't have to - # keep the code depends on evnetlet.posthooks sequence, IMHO. - # However, creating a new sub request might - # cause the possibility to hide some bugs behindes the request - # so that we should discuss whichi is suitable (new-sub-request - # vs re-write-existing-request) for Swift. [kota_] - req.method = 'PUT' - req.path_info = '/v1/%s/%s/%s' % \ - (dest_account, dest_container, dest_object) - req.headers['Content-Length'] = 0 - req.headers['X-Copy-From'] = quote(source) - del req.headers['Destination'] - - container_info = self.container_info( - dest_account, dest_container, req) - dest_policy = POLICIES.get_by_index(container_info['storage_policy']) - - return self._reroute(dest_policy).PUT(req) - @ObjectControllerRouter.register(REPL_POLICY) class ReplicatedObjectController(BaseObjectController): diff --git a/swift/proxy/server.py b/swift/proxy/server.py index f8f4296a25..963bf34f0e 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -64,10 +64,14 @@ required_filters = [ if pipe.startswith('catch_errors') else [])}, {'name': 'dlo', 'after_fn': lambda _junk: [ - 'staticweb', 'tempauth', 'keystoneauth', + 'copy', 'staticweb', 'tempauth', 'keystoneauth', 'catch_errors', 'gatekeeper', 'proxy_logging']}, {'name': 'versioned_writes', 'after_fn': lambda _junk: [ - 'slo', 'dlo', 'staticweb', 'tempauth', 'keystoneauth', + 'slo', 'dlo', 'copy', 'staticweb', 'tempauth', + 'keystoneauth', 'catch_errors', 'gatekeeper', 'proxy_logging']}, + # Put copy before dlo, slo and versioned_writes + {'name': 'copy', 'after_fn': lambda _junk: [ + 'staticweb', 'tempauth', 'keystoneauth', 'catch_errors', 'gatekeeper', 'proxy_logging']}] @@ -107,8 +111,6 @@ class Application(object): int(conf.get('recheck_account_existence', 60)) self.allow_account_management = \ config_true_value(conf.get('allow_account_management', 'no')) - self.object_post_as_copy = \ - config_true_value(conf.get('object_post_as_copy', 'true')) self.container_ring = container_ring or Ring(swift_dir, ring_name='container') self.account_ring = account_ring or Ring(swift_dir, @@ -392,8 +394,7 @@ class Application(object): # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ['swift.authorize'](req) - if not resp and not req.headers.get('X-Copy-From-Account') \ - and not req.headers.get('Destination-Account'): + if not resp: # No resp means authorized, no delayed recheck required. old_authorize = req.environ['swift.authorize'] else: @@ -404,7 +405,7 @@ class Application(object): # Save off original request method (GET, POST, etc.) in case it # gets mutated during handling. This way logging can display the # method the client actually sent. - req.environ['swift.orig_req_method'] = req.method + req.environ.setdefault('swift.orig_req_method', req.method) try: if old_authorize: req.environ.pop('swift.authorize', None) diff --git a/test/functional/tests.py b/test/functional/tests.py index fc9e362f2a..e35e79706d 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -1306,12 +1306,10 @@ class TestFile(Base): acct, '%s%s' % (prefix, self.env.container), Utils.create_name())) - if acct == acct2: - # there is no such source container - # and foreign user can have no permission to read it - self.assert_status(403) - else: - self.assert_status(404) + # there is no such source container but user has + # permissions to do a GET (done internally via COPY) for + # objects in his own account. + self.assert_status(404) self.assertFalse(file_item.copy_account( acct, @@ -1325,12 +1323,10 @@ class TestFile(Base): acct, '%s%s' % (prefix, self.env.container), Utils.create_name())) - if acct == acct2: - # there is no such object - # and foreign user can have no permission to read it - self.assert_status(403) - else: - self.assert_status(404) + # there is no such source container but user has + # permissions to do a GET (done internally via COPY) for + # objects in his own account. + self.assert_status(404) self.assertFalse(file_item.copy_account( acct, @@ -2677,6 +2673,23 @@ class TestFileComparisonUTF8(Base2, TestFileComparison): class TestSloEnv(object): slo_enabled = None # tri-state: None initially, then True/False + @classmethod + def create_segments(cls, container): + seg_info = {} + for letter, size in (('a', 1024 * 1024), + ('b', 1024 * 1024), + ('c', 1024 * 1024), + ('d', 1024 * 1024), + ('e', 1)): + seg_name = "seg_%s" % letter + file_item = container.file(seg_name) + file_item.write(letter * size) + seg_info[seg_name] = { + 'size_bytes': size, + 'etag': file_item.md5, + 'path': '/%s/%s' % (container.name, seg_name)} + return seg_info + @classmethod def setUp(cls): cls.conn = Connection(tf.config) @@ -2711,19 +2724,7 @@ class TestSloEnv(object): if not cont.create(): raise ResponseError(cls.conn.response) - cls.seg_info = seg_info = {} - for letter, size in (('a', 1024 * 1024), - ('b', 1024 * 1024), - ('c', 1024 * 1024), - ('d', 1024 * 1024), - ('e', 1)): - seg_name = "seg_%s" % letter - file_item = cls.container.file(seg_name) - file_item.write(letter * size) - seg_info[seg_name] = { - 'size_bytes': size, - 'etag': file_item.md5, - 'path': '/%s/%s' % (cls.container.name, seg_name)} + cls.seg_info = seg_info = cls.create_segments(cls.container) file_item = cls.container.file("manifest-abcde") file_item.write( @@ -3125,8 +3126,9 @@ class TestSlo(Base): def test_slo_copy_the_manifest(self): file_item = self.env.container.file("manifest-abcde") - file_item.copy(self.env.container.name, "copied-abcde-manifest-only", - parms={'multipart-manifest': 'get'}) + self.assertTrue(file_item.copy(self.env.container.name, + "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'})) copied = self.env.container.file("copied-abcde-manifest-only") copied_contents = copied.read(parms={'multipart-manifest': 'get'}) @@ -3157,10 +3159,40 @@ class TestSlo(Base): self.assertTrue(dest_cont.create(hdrs={ 'X-Container-Write': self.env.conn.user_acl })) - file_item.copy_account(acct, - dest_cont, - "copied-abcde-manifest-only", - parms={'multipart-manifest': 'get'}) + + # manifest copy will fail because there is no read access to segments + # in destination account + file_item.copy_account( + acct, dest_cont, "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'}) + self.assertEqual(400, file_item.conn.response.status) + resp_body = file_item.conn.response.read() + self.assertEqual(5, resp_body.count('403 Forbidden'), + 'Unexpected response body %r' % resp_body) + + # create segments container in account2 with read access for account1 + segs_container = self.env.account2.container(self.env.container.name) + self.assertTrue(segs_container.create(hdrs={ + 'X-Container-Read': self.env.conn.user_acl + })) + + # manifest copy will still fail because there are no segments in + # destination account + file_item.copy_account( + acct, dest_cont, "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'}) + self.assertEqual(400, file_item.conn.response.status) + resp_body = file_item.conn.response.read() + self.assertEqual(5, resp_body.count('404 Not Found'), + 'Unexpected response body %r' % resp_body) + + # create segments in account2 container with same name as in account1, + # manifest copy now succeeds + self.env.create_segments(segs_container) + + self.assertTrue(file_item.copy_account( + acct, dest_cont, "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'})) copied = dest_cont.file("copied-abcde-manifest-only") copied_contents = copied.read(parms={'multipart-manifest': 'get'}) diff --git a/test/unit/common/middleware/helpers.py b/test/unit/common/middleware/helpers.py index 2432d0dc37..bcd3c4c2ec 100644 --- a/test/unit/common/middleware/helpers.py +++ b/test/unit/common/middleware/helpers.py @@ -20,6 +20,7 @@ from copy import deepcopy from hashlib import md5 from swift.common import swob from swift.common.header_key_dict import HeaderKeyDict +from swift.common.swob import HTTPNotImplemented from swift.common.utils import split_path from test.unit import FakeLogger, FakeRing @@ -43,6 +44,8 @@ class FakeSwift(object): """ A good-enough fake Swift proxy server to use in testing middleware. """ + ALLOWED_METHODS = [ + 'PUT', 'POST', 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'REPLICATE'] def __init__(self): self._calls = [] @@ -71,6 +74,9 @@ class FakeSwift(object): def __call__(self, env, start_response): method = env['REQUEST_METHOD'] + if method not in self.ALLOWED_METHODS: + raise HTTPNotImplemented() + path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) diff --git a/test/unit/common/middleware/test_account_quotas.py b/test/unit/common/middleware/test_account_quotas.py index 345e178cd1..b443b4a28d 100644 --- a/test/unit/common/middleware/test_account_quotas.py +++ b/test/unit/common/middleware/test_account_quotas.py @@ -13,9 +13,10 @@ import unittest -from swift.common.swob import Request, wsgify, HTTPForbidden +from swift.common.swob import Request, wsgify, HTTPForbidden, \ + HTTPException -from swift.common.middleware import account_quotas +from swift.common.middleware import account_quotas, copy from swift.proxy.controllers.base import _get_cache_key, \ headers_to_account_info, get_object_env_key, \ @@ -245,84 +246,6 @@ class TestAccountQuota(unittest.TestCase): res = req.get_response(app) self.assertEqual(res.status_int, 200) - def test_exceed_bytes_quota_copy_from(self): - headers = [('x-account-bytes-used', '500'), - ('x-account-meta-quota-bytes', '1000'), - ('content-length', '1000')] - app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) - cache = FakeCache(None) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - - def test_exceed_bytes_quota_copy_verb(self): - headers = [('x-account-bytes-used', '500'), - ('x-account-meta-quota-bytes', '1000'), - ('content-length', '1000')] - app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) - cache = FakeCache(None) - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - - def test_not_exceed_bytes_quota_copy_from(self): - headers = [('x-account-bytes-used', '0'), - ('x-account-meta-quota-bytes', '1000'), - ('content-length', '1000')] - app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) - cache = FakeCache(None) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_not_exceed_bytes_quota_copy_verb(self): - headers = [('x-account-bytes-used', '0'), - ('x-account-meta-quota-bytes', '1000'), - ('content-length', '1000')] - app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) - cache = FakeCache(None) - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_quota_copy_from_no_src(self): - headers = [('x-account-bytes-used', '0'), - ('x-account-meta-quota-bytes', '1000')] - app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) - cache = FakeCache(None) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o3'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_quota_copy_from_bad_src(self): - headers = [('x-account-bytes-used', '0'), - ('x-account-meta-quota-bytes', '1000')] - app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) - cache = FakeCache(None) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.cache': cache}, - headers={'x-copy-from': 'bad_path'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 412) - def test_exceed_bytes_quota_reseller(self): headers = [('x-account-bytes-used', '1000'), ('x-account-meta-quota-bytes', '0')] @@ -485,5 +408,91 @@ class TestAccountQuota(unittest.TestCase): self.assertEqual(res.status_int, 200) +class AccountQuotaCopyingTestCases(unittest.TestCase): + + def setUp(self): + self.app = FakeApp() + self.aq_filter = account_quotas.filter_factory({})(self.app) + self.copy_filter = copy.filter_factory({})(self.aq_filter) + + def test_exceed_bytes_quota_copy_from(self): + headers = [('x-account-bytes-used', '500'), + ('x-account-meta-quota-bytes', '1000'), + ('content-length', '1000')] + self.app.headers = headers + cache = FakeCache(None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_exceed_bytes_quota_copy_verb(self): + headers = [('x-account-bytes-used', '500'), + ('x-account-meta-quota-bytes', '1000'), + ('content-length', '1000')] + self.app.headers = headers + cache = FakeCache(None) + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.cache': cache}, + headers={'Destination': '/c/o'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_not_exceed_bytes_quota_copy_from(self): + headers = [('x-account-bytes-used', '0'), + ('x-account-meta-quota-bytes', '1000'), + ('content-length', '1000')] + self.app.headers = headers + cache = FakeCache(None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_not_exceed_bytes_quota_copy_verb(self): + headers = [('x-account-bytes-used', '0'), + ('x-account-meta-quota-bytes', '1000'), + ('content-length', '1000')] + self.app.headers = headers + cache = FakeCache(None) + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.cache': cache}, + headers={'Destination': '/c/o'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_quota_copy_from_no_src(self): + headers = [('x-account-bytes-used', '0'), + ('x-account-meta-quota-bytes', '1000')] + self.app.headers = headers + cache = FakeCache(None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o3'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_quota_copy_from_bad_src(self): + headers = [('x-account-bytes-used', '0'), + ('x-account-meta-quota-bytes', '1000')] + self.app.headers = headers + cache = FakeCache(None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': 'bad_path'}) + with self.assertRaises(HTTPException) as catcher: + req.get_response(self.copy_filter) + self.assertEqual(412, catcher.exception.status_int) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/test_copy.py b/test/unit/common/middleware/test_copy.py new file mode 100644 index 0000000000..190d7c9084 --- /dev/null +++ b/test/unit/common/middleware/test_copy.py @@ -0,0 +1,1183 @@ +#!/usr/bin/env python +# Copyright (c) 2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +import mock +import shutil +import tempfile +import unittest +from hashlib import md5 +from textwrap import dedent + +from swift.common import swob +from swift.common.middleware import copy +from swift.common.storage_policy import POLICIES +from swift.common.swob import Request, HTTPException +from test.unit import patch_policies, debug_logger, FakeMemcache, FakeRing +from test.unit.common.middleware.helpers import FakeSwift +from test.unit.proxy.controllers.test_obj import set_http_connect, \ + PatchedObjControllerApp + + +class TestCopyConstraints(unittest.TestCase): + def test_validate_copy_from(self): + req = Request.blank( + '/v/a/c/o', + headers={'x-copy-from': 'c/o2'}) + src_cont, src_obj = copy._check_copy_from_header(req) + self.assertEqual(src_cont, 'c') + self.assertEqual(src_obj, 'o2') + req = Request.blank( + '/v/a/c/o', + headers={'x-copy-from': 'c/subdir/o2'}) + src_cont, src_obj = copy._check_copy_from_header(req) + self.assertEqual(src_cont, 'c') + self.assertEqual(src_obj, 'subdir/o2') + req = Request.blank( + '/v/a/c/o', + headers={'x-copy-from': '/c/o2'}) + src_cont, src_obj = copy._check_copy_from_header(req) + self.assertEqual(src_cont, 'c') + self.assertEqual(src_obj, 'o2') + + def test_validate_bad_copy_from(self): + req = Request.blank( + '/v/a/c/o', + headers={'x-copy-from': 'bad_object'}) + self.assertRaises(HTTPException, + copy._check_copy_from_header, req) + + def test_validate_destination(self): + req = Request.blank( + '/v/a/c/o', + headers={'destination': 'c/o2'}) + src_cont, src_obj = copy._check_destination_header(req) + self.assertEqual(src_cont, 'c') + self.assertEqual(src_obj, 'o2') + req = Request.blank( + '/v/a/c/o', + headers={'destination': 'c/subdir/o2'}) + src_cont, src_obj = copy._check_destination_header(req) + self.assertEqual(src_cont, 'c') + self.assertEqual(src_obj, 'subdir/o2') + req = Request.blank( + '/v/a/c/o', + headers={'destination': '/c/o2'}) + src_cont, src_obj = copy._check_destination_header(req) + self.assertEqual(src_cont, 'c') + self.assertEqual(src_obj, 'o2') + + def test_validate_bad_destination(self): + req = Request.blank( + '/v/a/c/o', + headers={'destination': 'bad_object'}) + self.assertRaises(HTTPException, + copy._check_destination_header, req) + + +class TestServerSideCopyMiddleware(unittest.TestCase): + def setUp(self): + self.app = FakeSwift() + self.ssc = copy.filter_factory({ + 'object_post_as_copy': 'yes', + })(self.app) + self.ssc.logger = self.app.logger + + def call_app(self, req, app=None, expect_exception=False): + if app is None: + app = self.app + + self.authorized = [] + + def authorize(req): + self.authorized.append(req) + + if 'swift.authorize' not in req.environ: + req.environ['swift.authorize'] = authorize + + req.headers.setdefault("User-Agent", "Bruce Wayne") + + status = [None] + headers = [None] + + def start_response(s, h, ei=None): + status[0] = s + headers[0] = h + + body_iter = app(req.environ, start_response) + body = '' + caught_exc = None + try: + for chunk in body_iter: + body += chunk + except Exception as exc: + if expect_exception: + caught_exc = exc + else: + raise + + if expect_exception: + return status[0], headers[0], body, caught_exc + else: + return status[0], headers[0], body + + def call_ssc(self, req, **kwargs): + return self.call_app(req, app=self.ssc, **kwargs) + + def assertRequestEqual(self, req, other): + self.assertEqual(req.method, other.method) + self.assertEqual(req.path, other.path) + + def test_no_object_in_path_pass_through(self): + self.app.register('PUT', '/v1/a/c', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c', method='PUT') + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_object_delete_pass_through(self): + self.app.register('DELETE', '/v1/a/c/o', swob.HTTPOk, {}) + req = Request.blank('/v1/a/c/o', method='DELETE') + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_POST_as_COPY_simple(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPAccepted, {}) + req = Request.blank('/v1/a/c/o', method='POST') + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '202 Accepted') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_POST_as_COPY_201_return_202(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', method='POST') + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '202 Accepted') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_POST_delete_at(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPAccepted, {}) + t = str(int(time.time() + 100)) + req = Request.blank('/v1/a/c/o', method='POST', + headers={'Content-Type': 'foo/bar', + 'X-Delete-At': t}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '202 Accepted') + + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertTrue('X-Delete-At' in req_headers) + self.assertEqual(req_headers['X-Delete-At'], str(t)) + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_POST_as_COPY_static_large_object(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, + {'X-Static-Large-Object': True}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPAccepted, {}) + req = Request.blank('/v1/a/c/o', method='POST', + headers={}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '202 Accepted') + + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertNotIn('X-Static-Large-Object', req_headers) + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_basic_put_with_x_copy_from(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o2', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o2', self.authorized[1].path) + + def test_static_large_object(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, + {'X-Static-Large-Object': 'True'}, 'passed') + self.app.register('PUT', '/v1/a/c/o2?multipart-manifest=put', + swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o2?multipart-manifest=get', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o2?multipart-manifest=put', path) + self.assertNotIn('X-Static-Large-Object', req_headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o2', self.authorized[1].path) + + def test_basic_put_with_x_copy_from_across_container(self): + self.app.register('GET', '/v1/a/c1/o1', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c2/o2', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c1/o1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c1/o1') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c1/o1', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c2/o2', self.authorized[1].path) + + def test_basic_put_with_x_copy_from_across_container_and_account(self): + self.app.register('GET', '/v1/a1/c1/o1', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a2/c2/o2', swob.HTTPCreated, {}, + 'passed') + req = Request.blank('/v1/a2/c2/o2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c1/o1', + 'X-Copy-From-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c1/o1') in headers) + self.assertTrue(('X-Copied-From-Account', 'a1') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a1/c1/o1', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a2/c2/o2', self.authorized[1].path) + + def test_copy_non_zero_content_length(self): + req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '10', + 'X-Copy-From': 'c1/o1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '400 Bad Request') + + def test_copy_non_zero_content_length_with_account(self): + req = Request.blank('/v1/a2/c2/o2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '10', + 'X-Copy-From': 'c1/o1', + 'X-Copy-From-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '400 Bad Request') + + def test_copy_with_slashes_in_x_copy_from(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o/o2'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_copy_with_slashes_in_x_copy_from_and_account(self): + self.app.register('GET', '/v1/a1/c1/o/o1', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a2/c2/o2', swob.HTTPCreated, {}) + req = Request.blank('/v1/a2/c2/o2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c1/o/o1', + 'X-Copy-From-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c1/o/o1') in headers) + self.assertTrue(('X-Copied-From-Account', 'a1') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a1/c1/o/o1', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a2/c2/o2', self.authorized[1].path) + + def test_copy_with_spaces_in_x_copy_from(self): + self.app.register('GET', '/v1/a/c/o o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + # space in soure path + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o%20o2'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o o2', path) + self.assertTrue(('X-Copied-From', 'c/o%20o2') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o%20o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_copy_with_spaces_in_x_copy_from_and_account(self): + self.app.register('GET', '/v1/a/c/o o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + # space in soure path + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o%20o2', + 'X-Copy-From-Account': 'a'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o o2', path) + self.assertTrue(('X-Copied-From', 'c/o%20o2') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o%20o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_copy_with_leading_slash_in_x_copy_from(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + # repeat tests with leading / + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o', path) + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_copy_with_leading_slash_in_x_copy_from_and_account(self): + # repeat tests with leading / + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Copy-From-Account': 'a'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o', path) + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_copy_with_leading_slash_and_slashes_in_x_copy_from(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o/o2'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o/o2', path) + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_copy_with_leading_slash_and_slashes_in_x_copy_from_acct(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o/o2', + 'X-Copy-From-Account': 'a'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o/o2', path) + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_copy_with_no_object_in_x_copy_from(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c'}) + try: + status, headers, body = self.call_ssc(req) + except HTTPException as resp: + self.assertEqual("412 Precondition Failed", str(resp)) + else: + self.fail("Expecting HTTPException.") + + def test_copy_with_no_object_in_x_copy_from_and_account(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c', + 'X-Copy-From-Account': 'a'}) + try: + status, headers, body = self.call_ssc(req) + except HTTPException as resp: + self.assertEqual("412 Precondition Failed", str(resp)) + else: + self.fail("Expecting HTTPException.") + + def test_copy_with_bad_x_copy_from_account(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Copy-From-Account': '/i/am/bad'}) + try: + status, headers, body = self.call_ssc(req) + except HTTPException as resp: + self.assertEqual("412 Precondition Failed", str(resp)) + else: + self.fail("Expecting HTTPException.") + + def test_copy_server_error_reading_source(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPServiceUnavailable, {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '503 Service Unavailable') + + def test_copy_server_error_reading_source_and_account(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPServiceUnavailable, {}) + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Copy-From-Account': 'a'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '503 Service Unavailable') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_copy_not_found_reading_source(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPNotFound, {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '404 Not Found') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_copy_not_found_reading_source_and_account(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPNotFound, {}) + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Copy-From-Account': 'a'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '404 Not Found') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_copy_with_object_metadata(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Object-Meta-Ours': 'okay'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o', path) + self.assertEqual(req_headers['X-Object-Meta-Ours'], 'okay') + self.assertTrue(('X-Object-Meta-Ours', 'okay') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_copy_with_object_metadata_and_account(self): + self.app.register('GET', '/v1/a1/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Object-Meta-Ours': 'okay', + 'X-Copy-From-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o', path) + self.assertEqual(req_headers['X-Object-Meta-Ours'], 'okay') + self.assertTrue(('X-Object-Meta-Ours', 'okay') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a1/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_copy_source_larger_than_max_file_size(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, "largebody") + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + with mock.patch('swift.common.middleware.copy.' + 'MAX_FILE_SIZE', 1): + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '413 Request Entity Too Large') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_basic_COPY(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o-copy', swob.HTTPCreated, {}) + req = Request.blank( + '/v1/a/c/o', method='COPY', + headers={'Content-Length': 0, + 'Destination': 'c/o-copy'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o-copy', self.authorized[1].path) + + def test_COPY_no_destination_header(self): + req = Request.blank( + '/v1/a/c/o', method='COPY', headers={'Content-Length': 0}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '412 Precondition Failed') + self.assertEqual(len(self.authorized), 0) + + def test_basic_COPY_account(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o2', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c1/o2', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o', path) + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a1/c1/o2', path) + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o2', self.authorized[1].path) + + def test_COPY_across_containers(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c2/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c2/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c2/o', self.authorized[1].path) + + def test_COPY_source_with_slashes_in_name(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o', path) + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_COPY_account_source_with_slashes_in_name(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c1/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a1/c1/o', path) + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_COPY_destination_leading_slash(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_COPY_account_destination_leading_slash(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a1/c1/o', path) + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_COPY_source_with_slashes_destination_leading_slash(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o', path) + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_COPY_account_source_with_slashes_destination_leading_slash(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a1/c1/o', path) + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_COPY_no_object_in_destination(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c_o'}) + try: + status, headers, body = self.call_ssc(req) + except HTTPException as resp: + self.assertEqual("412 Precondition Failed", str(resp)) + else: + self.fail("Expecting HTTPException.") + + def test_COPY_account_no_object_in_destination(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c_o', + 'Destination-Account': 'a1'}) + try: + status, headers, body = self.call_ssc(req) + except HTTPException as resp: + self.assertEqual("412 Precondition Failed", str(resp)) + else: + self.fail("Expecting HTTPException.") + + def test_COPY_account_bad_destination_account(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o', + 'Destination-Account': '/i/am/bad'}) + try: + status, headers, body = self.call_ssc(req) + except HTTPException as resp: + self.assertEqual("412 Precondition Failed", str(resp)) + else: + self.fail("Expecting HTTPException.") + + def test_COPY_server_error_reading_source(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPServiceUnavailable, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '503 Service Unavailable') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_account_server_error_reading_source(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPServiceUnavailable, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '503 Service Unavailable') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_not_found_reading_source(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPNotFound, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '404 Not Found') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_account_not_found_reading_source(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPNotFound, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '404 Not Found') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_with_metadata(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, "passed") + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o', + 'X-Object-Meta-Ours': 'okay'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o', path) + self.assertEqual(req_headers['X-Object-Meta-Ours'], 'okay') + self.assertTrue(('X-Object-Meta-Ours', 'okay') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_COPY_account_with_metadata(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, "passed") + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'X-Object-Meta-Ours': 'okay', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a1/c1/o', path) + self.assertEqual(req_headers['X-Object-Meta-Ours'], 'okay') + self.assertTrue(('X-Object-Meta-Ours', 'okay') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_COPY_source_zero_content_length(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '413 Request Entity Too Large') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_source_larger_than_max_file_size(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, "largebody") + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + with mock.patch('swift.common.middleware.copy.' + 'MAX_FILE_SIZE', 1): + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '413 Request Entity Too Large') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_account_source_zero_content_length(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '413 Request Entity Too Large') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_account_source_larger_than_max_file_size(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, "largebody") + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + with mock.patch('swift.common.middleware.copy.' + 'MAX_FILE_SIZE', 1): + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '413 Request Entity Too Large') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_newest(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, + {'Last-Modified': '123'}, "passed") + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From-Last-Modified', '123') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_COPY_account_newest(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, + {'Last-Modified': '123'}, "passed") + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From-Last-Modified', '123') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_COPY_in_OPTIONS_response(self): + self.app.register('OPTIONS', '/v1/a/c/o', swob.HTTPOk, + {'Allow': 'GET, PUT'}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'OPTIONS'}, headers={}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '200 OK') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('OPTIONS', method) + self.assertEqual('/v1/a/c/o', path) + self.assertTrue(('Allow', 'GET, PUT, COPY') in headers) + self.assertEqual(len(self.authorized), 1) + self.assertEqual('OPTIONS', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_in_OPTIONS_response_CORS(self): + self.app.register('OPTIONS', '/v1/a/c/o', swob.HTTPOk, + {'Allow': 'GET, PUT', + 'Access-Control-Allow-Methods': 'GET, PUT'}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'OPTIONS'}, headers={}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '200 OK') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('OPTIONS', method) + self.assertEqual('/v1/a/c/o', path) + self.assertTrue(('Allow', 'GET, PUT, COPY') in headers) + self.assertTrue(('Access-Control-Allow-Methods', + 'GET, PUT, COPY') in headers) + self.assertEqual(len(self.authorized), 1) + self.assertEqual('OPTIONS', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + +class TestServerSideCopyConfiguration(unittest.TestCase): + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmpdir) + + def test_reading_proxy_conf_when_no_middleware_conf_present(self): + proxy_conf = dedent(""" + [DEFAULT] + bind_ip = 10.4.5.6 + + [pipeline:main] + pipeline = catch_errors copy ye-olde-proxy-server + + [filter:copy] + use = egg:swift#copy + + [app:ye-olde-proxy-server] + use = egg:swift#proxy + object_post_as_copy = no + """) + + conffile = tempfile.NamedTemporaryFile() + conffile.write(proxy_conf) + conffile.flush() + + ssc = copy.filter_factory({ + '__file__': conffile.name + })("no app here") + + self.assertEqual(ssc.object_post_as_copy, False) + + def test_middleware_conf_precedence(self): + proxy_conf = dedent(""" + [DEFAULT] + bind_ip = 10.4.5.6 + + [pipeline:main] + pipeline = catch_errors copy ye-olde-proxy-server + + [filter:copy] + use = egg:swift#copy + object_post_as_copy = no + + [app:ye-olde-proxy-server] + use = egg:swift#proxy + object_post_as_copy = yes + """) + + conffile = tempfile.NamedTemporaryFile() + conffile.write(proxy_conf) + conffile.flush() + + ssc = copy.filter_factory({ + 'object_post_as_copy': 'no', + '__file__': conffile.name + })("no app here") + + self.assertEqual(ssc.object_post_as_copy, False) + + +@patch_policies(with_ec_default=True) +class TestServerSideCopyMiddlewareWithEC(unittest.TestCase): + container_info = { + 'write_acl': None, + 'read_acl': None, + 'storage_policy': None, + 'sync_key': None, + 'versions': None, + } + + def setUp(self): + self.logger = debug_logger('proxy-server') + self.logger.thread_locals = ('txn1', '127.0.0.2') + self.app = PatchedObjControllerApp( + None, FakeMemcache(), account_ring=FakeRing(), + container_ring=FakeRing(), logger=self.logger) + self.ssc = copy.filter_factory({ + 'object_post_as_copy': 'yes', + })(self.app) + self.ssc.logger = self.app.logger + self.policy = POLICIES.default + self.app.container_info = dict(self.container_info) + + def test_COPY_with_ranges(self): + req = swob.Request.blank( + '/v1/a/c/o', method='COPY', + headers={'Destination': 'c1/o', + 'Range': 'bytes=5-10'}) + # turn a real body into fragments + segment_size = self.policy.ec_segment_size + real_body = ('asdf' * segment_size)[:-10] + + # split it up into chunks + chunks = [real_body[x:x + segment_size] + for x in range(0, len(real_body), segment_size)] + + # we need only first chunk to rebuild 5-10 range + fragments = self.policy.pyeclib_driver.encode(chunks[0]) + fragment_payloads = [] + fragment_payloads.append(fragments) + + node_fragments = zip(*fragment_payloads) + self.assertEqual(len(node_fragments), + self.policy.object_ring.replicas) # sanity + headers = {'X-Object-Sysmeta-Ec-Content-Length': str(len(real_body))} + responses = [(200, ''.join(node_fragments[i]), headers) + for i in range(POLICIES.default.ec_ndata)] + responses += [(201, '', {})] * self.policy.object_ring.replicas + status_codes, body_iter, headers = zip(*responses) + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*status_codes, body_iter=body_iter, + headers=headers, expect_headers=expect_headers): + resp = req.get_response(self.ssc) + self.assertEqual(resp.status_int, 201) + + def test_COPY_with_invalid_ranges(self): + # real body size is segment_size - 10 (just 1 segment) + segment_size = self.policy.ec_segment_size + real_body = ('a' * segment_size)[:-10] + + # range is out of real body but in segment size + self._test_invalid_ranges('COPY', real_body, + segment_size, '%s-' % (segment_size - 10)) + # range is out of both real body and segment size + self._test_invalid_ranges('COPY', real_body, + segment_size, '%s-' % (segment_size + 10)) + + def _test_invalid_ranges(self, method, real_body, segment_size, req_range): + # make a request with range starts from more than real size. + body_etag = md5(real_body).hexdigest() + req = swob.Request.blank( + '/v1/a/c/o', method=method, + headers={'Destination': 'c1/o', + 'Range': 'bytes=%s' % (req_range)}) + + fragments = self.policy.pyeclib_driver.encode(real_body) + fragment_payloads = [fragments] + + node_fragments = zip(*fragment_payloads) + self.assertEqual(len(node_fragments), + self.policy.object_ring.replicas) # sanity + headers = {'X-Object-Sysmeta-Ec-Content-Length': str(len(real_body)), + 'X-Object-Sysmeta-Ec-Etag': body_etag} + start = int(req_range.split('-')[0]) + self.assertTrue(start >= 0) # sanity + title, exp = swob.RESPONSE_REASONS[416] + range_not_satisfiable_body = \ + '

%s

%s

' % (title, exp) + if start >= segment_size: + responses = [(416, range_not_satisfiable_body, headers) + for i in range(POLICIES.default.ec_ndata)] + else: + responses = [(200, ''.join(node_fragments[i]), headers) + for i in range(POLICIES.default.ec_ndata)] + status_codes, body_iter, headers = zip(*responses) + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + # TODO possibly use FakeApp here + with set_http_connect(*status_codes, body_iter=body_iter, + headers=headers, expect_headers=expect_headers): + resp = req.get_response(self.ssc) + self.assertEqual(resp.status_int, 416) + self.assertEqual(resp.content_length, len(range_not_satisfiable_body)) + self.assertEqual(resp.body, range_not_satisfiable_body) + self.assertEqual(resp.etag, body_etag) + self.assertEqual(resp.headers['Accept-Ranges'], 'bytes') diff --git a/test/unit/common/middleware/test_dlo.py b/test/unit/common/middleware/test_dlo.py index 1374b403df..04fbc4e614 100644 --- a/test/unit/common/middleware/test_dlo.py +++ b/test/unit/common/middleware/test_dlo.py @@ -803,107 +803,6 @@ class TestDloGetManifest(DloTestCase): self.assertTrue(auth_got_called[0] > 1) -def fake_start_response(*args, **kwargs): - pass - - -class TestDloCopyHook(DloTestCase): - def setUp(self): - super(TestDloCopyHook, self).setUp() - - self.app.register( - 'GET', '/v1/AUTH_test/c/o1', swob.HTTPOk, - {'Content-Length': '10', 'Etag': 'o1-etag'}, - "aaaaaaaaaa") - self.app.register( - 'GET', '/v1/AUTH_test/c/o2', swob.HTTPOk, - {'Content-Length': '10', 'Etag': 'o2-etag'}, - "bbbbbbbbbb") - self.app.register( - 'GET', '/v1/AUTH_test/c/man', - swob.HTTPOk, {'X-Object-Manifest': 'c/o'}, - "manifest-contents") - - lm = '2013-11-22T02:42:13.781760' - ct = 'application/octet-stream' - segs = [{"hash": "o1-etag", "bytes": 10, "name": "o1", - "last_modified": lm, "content_type": ct}, - {"hash": "o2-etag", "bytes": 5, "name": "o2", - "last_modified": lm, "content_type": ct}] - - self.app.register( - 'GET', '/v1/AUTH_test/c?format=json&prefix=o', - swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, - json.dumps(segs)) - - copy_hook = [None] - - # slip this guy in there to pull out the hook - def extract_copy_hook(env, sr): - copy_hook[0] = env.get('swift.copy_hook') - return self.app(env, sr) - - self.dlo = dlo.filter_factory({})(extract_copy_hook) - - req = swob.Request.blank('/v1/AUTH_test/c/o1', - environ={'REQUEST_METHOD': 'GET'}) - self.dlo(req.environ, fake_start_response) - self.copy_hook = copy_hook[0] - - self.assertTrue(self.copy_hook is not None) # sanity check - - def test_copy_hook_passthrough(self): - source_req = swob.Request.blank( - '/v1/AUTH_test/c/man', - environ={'REQUEST_METHOD': 'GET'}) - sink_req = swob.Request.blank( - '/v1/AUTH_test/c/man', - environ={'REQUEST_METHOD': 'PUT'}) - source_resp = swob.Response(request=source_req, status=200) - - # no X-Object-Manifest header, so do nothing - modified_resp = self.copy_hook(source_req, source_resp, sink_req) - self.assertTrue(modified_resp is source_resp) - - def test_copy_hook_manifest(self): - source_req = swob.Request.blank( - '/v1/AUTH_test/c/man', - environ={'REQUEST_METHOD': 'GET'}) - sink_req = swob.Request.blank( - '/v1/AUTH_test/c/man', - environ={'REQUEST_METHOD': 'PUT'}) - source_resp = swob.Response( - request=source_req, status=200, - headers={"X-Object-Manifest": "c/o"}, - app_iter=["manifest"]) - - # it's a manifest, so copy the segments to make a normal object - modified_resp = self.copy_hook(source_req, source_resp, sink_req) - self.assertTrue(modified_resp is not source_resp) - self.assertEqual(modified_resp.etag, - hashlib.md5("o1-etago2-etag").hexdigest()) - self.assertEqual(sink_req.headers.get('X-Object-Manifest'), None) - - def test_copy_hook_manifest_with_multipart_manifest_get(self): - source_req = swob.Request.blank( - '/v1/AUTH_test/c/man', - environ={'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'multipart-manifest=get'}) - sink_req = swob.Request.blank( - '/v1/AUTH_test/c/man', - environ={'REQUEST_METHOD': 'PUT'}) - source_resp = swob.Response( - request=source_req, status=200, - headers={"X-Object-Manifest": "c/o"}, - app_iter=["manifest"]) - - # make sure the sink request (the backend PUT) gets X-Object-Manifest - # on it, but that's all - modified_resp = self.copy_hook(source_req, source_resp, sink_req) - self.assertTrue(modified_resp is source_resp) - self.assertEqual(sink_req.headers.get('X-Object-Manifest'), 'c/o') - - class TestDloConfiguration(unittest.TestCase): """ For backwards compatibility, we will read a couple of values out of the diff --git a/test/unit/common/middleware/test_quotas.py b/test/unit/common/middleware/test_quotas.py index b71b78ed83..f99b8df663 100644 --- a/test/unit/common/middleware/test_quotas.py +++ b/test/unit/common/middleware/test_quotas.py @@ -15,8 +15,9 @@ import unittest -from swift.common.swob import Request, HTTPUnauthorized -from swift.common.middleware import container_quotas +from swift.common.swob import Request, HTTPUnauthorized, HTTPOk, HTTPException +from swift.common.middleware import container_quotas, copy +from test.unit.common.middleware.helpers import FakeSwift class FakeCache(object): @@ -95,32 +96,6 @@ class TestContainerQuotas(unittest.TestCase): self.assertEqual(res.status_int, 413) self.assertEqual(res.body, 'Upload exceeds quota.') - def test_exceed_bytes_quota_copy_from(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}}) - - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - - def test_exceed_bytes_quota_copy_verb(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}}) - - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - def test_not_exceed_bytes_quota(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) @@ -131,60 +106,6 @@ class TestContainerQuotas(unittest.TestCase): res = req.get_response(app) self.assertEqual(res.status_int, 200) - def test_not_exceed_bytes_quota_copy_from(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_not_exceed_bytes_quota_copy_verb(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_bytes_quota_copy_from_no_src(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o3'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_bytes_quota_copy_from_bad_src(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.cache': cache}, - headers={'x-copy-from': 'bad_path'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 412) - - def test_bytes_quota_copy_verb_no_src(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) - req = Request.blank('/v1/a/c2/o3', - environ={'REQUEST_METHOD': 'COPY', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - def test_exceed_counts_quota(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) @@ -196,61 +117,6 @@ class TestContainerQuotas(unittest.TestCase): self.assertEqual(res.status_int, 413) self.assertEqual(res.body, 'Upload exceeds quota.') - def test_exceed_counts_quota_copy_from(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - - def test_exceed_counts_quota_copy_verb(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - - def test_exceed_counts_quota_copy_cross_account_verb(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'}, - 'status': 200, 'object_count': 1} - a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'}, - 'status': 200, 'object_count': 1} - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.container/a/c': a_c_cache, - 'swift.container/a2/c': a2_c_cache}, - headers={'Destination': '/c/o', - 'Destination-Account': 'a2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - - def test_exceed_counts_quota_copy_cross_account_PUT_verb(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'}, - 'status': 200, 'object_count': 1} - a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'}, - 'status': 200, 'object_count': 1} - req = Request.blank('/v1/a2/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.container/a/c': a_c_cache, - 'swift.container/a2/c': a2_c_cache}, - headers={'X-Copy-From': '/c2/o2', - 'X-Copy-From-Account': 'a'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - def test_not_exceed_counts_quota(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) @@ -261,26 +127,6 @@ class TestContainerQuotas(unittest.TestCase): res = req.get_response(app) self.assertEqual(res.status_int, 200) - def test_not_exceed_counts_quota_copy_from(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_not_exceed_counts_quota_copy_verb(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - def test_invalid_quotas(self): req = Request.blank( '/v1/a/c', @@ -346,5 +192,168 @@ class TestContainerQuotas(unittest.TestCase): res = req.get_response(app) self.assertEqual(res.status_int, 401) + +class ContainerQuotaCopyingTestCases(unittest.TestCase): + + def setUp(self): + self.app = FakeSwift() + self.cq_filter = container_quotas.filter_factory({})(self.app) + self.copy_filter = copy.filter_factory({})(self.cq_filter) + + def test_exceed_bytes_quota_copy_verb(self): + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}}) + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.cache': cache}, + headers={'Destination': '/c/o'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_not_exceed_bytes_quota_copy_verb(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + self.app.register( + 'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed') + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.cache': cache}, + headers={'Destination': '/c/o'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_exceed_counts_quota_copy_verb(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed') + cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.cache': cache}, + headers={'Destination': '/c/o'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_exceed_counts_quota_copy_cross_account_verb(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed') + a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'}, + 'status': 200, 'object_count': 1} + a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'}, + 'status': 200, 'object_count': 1} + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.container/a/c': a_c_cache, + 'swift.container/a2/c': a2_c_cache}, + headers={'Destination': '/c/o', + 'Destination-Account': 'a2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_exceed_counts_quota_copy_cross_account_PUT_verb(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed') + a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'}, + 'status': 200, 'object_count': 1} + a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'}, + 'status': 200, 'object_count': 1} + req = Request.blank('/v1/a2/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.container/a/c': a_c_cache, + 'swift.container/a2/c': a2_c_cache}, + headers={'X-Copy-From': '/c2/o2', + 'X-Copy-From-Account': 'a'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_exceed_bytes_quota_copy_from(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}}) + + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_not_exceed_bytes_quota_copy_from(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + self.app.register( + 'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed') + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_bytes_quota_copy_from_no_src(self): + self.app.register('GET', '/v1/a/c2/o3', HTTPOk, {}, 'passed') + self.app.register( + 'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed') + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o3'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_bytes_quota_copy_from_bad_src(self): + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': 'bad_path'}) + with self.assertRaises(HTTPException) as catcher: + req.get_response(self.copy_filter) + self.assertEqual(412, catcher.exception.status_int) + + def test_exceed_counts_quota_copy_from(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_not_exceed_counts_quota_copy_from(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + self.app.register( + 'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed') + cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_not_exceed_counts_quota_copy_verb(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + self.app.register( + 'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed') + cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.cache': cache}, + headers={'Destination': '/c/o'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 03f5c23213..79eaddcbf3 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -26,7 +26,7 @@ from swift.common import swob, utils from swift.common.exceptions import ListingIterError, SegmentError from swift.common.header_key_dict import HeaderKeyDict from swift.common.middleware import slo -from swift.common.swob import Request, Response, HTTPException +from swift.common.swob import Request, HTTPException from swift.common.utils import quote, closing_if_possible, close_if_possible from test.unit.common.middleware.helpers import FakeSwift @@ -2653,70 +2653,6 @@ class TestSloBulkLogger(unittest.TestCase): self.assertTrue(slo_mware.logger is slo_mware.bulk_deleter.logger) -class TestSloCopyHook(SloTestCase): - def setUp(self): - super(TestSloCopyHook, self).setUp() - - self.app.register( - 'GET', '/v1/AUTH_test/c/o', swob.HTTPOk, - {'Content-Length': '3', 'Etag': md5hex("obj")}, "obj") - self.app.register( - 'GET', '/v1/AUTH_test/c/man', - swob.HTTPOk, {'Content-Type': 'application/json', - 'X-Static-Large-Object': 'true'}, - json.dumps([{'name': '/c/o', 'hash': md5hex("obj"), - 'bytes': '3'}])) - self.app.register( - 'COPY', '/v1/AUTH_test/c/o', swob.HTTPCreated, {}) - - copy_hook = [None] - - # slip this guy in there to pull out the hook - def extract_copy_hook(env, sr): - if env['REQUEST_METHOD'] == 'COPY': - copy_hook[0] = env['swift.copy_hook'] - return self.app(env, sr) - - self.slo = slo.filter_factory({})(extract_copy_hook) - - req = Request.blank('/v1/AUTH_test/c/o', - environ={'REQUEST_METHOD': 'COPY'}) - self.slo(req.environ, fake_start_response) - self.copy_hook = copy_hook[0] - - self.assertTrue(self.copy_hook is not None) # sanity check - - def test_copy_hook_passthrough(self): - source_req = Request.blank( - '/v1/AUTH_test/c/o', - environ={'REQUEST_METHOD': 'GET'}) - sink_req = Request.blank( - '/v1/AUTH_test/c/o', - environ={'REQUEST_METHOD': 'PUT'}) - # no X-Static-Large-Object header, so do nothing - source_resp = Response(request=source_req, status=200) - - modified_resp = self.copy_hook(source_req, source_resp, sink_req) - self.assertTrue(modified_resp is source_resp) - - def test_copy_hook_manifest(self): - source_req = Request.blank( - '/v1/AUTH_test/c/o', - environ={'REQUEST_METHOD': 'GET'}) - sink_req = Request.blank( - '/v1/AUTH_test/c/o', - environ={'REQUEST_METHOD': 'PUT'}) - source_resp = Response(request=source_req, status=200, - headers={"X-Static-Large-Object": "true"}, - app_iter=[json.dumps([{'name': '/c/o', - 'hash': md5hex("obj"), - 'bytes': '3'}])]) - - modified_resp = self.copy_hook(source_req, source_resp, sink_req) - self.assertTrue(modified_resp is not source_resp) - self.assertEqual(modified_resp.etag, md5hex(md5hex("obj"))) - - class TestSwiftInfo(unittest.TestCase): def setUp(self): utils._swift_info = {} diff --git a/test/unit/common/middleware/test_versioned_writes.py b/test/unit/common/middleware/test_versioned_writes.py index c6da47fde8..27b8914555 100644 --- a/test/unit/common/middleware/test_versioned_writes.py +++ b/test/unit/common/middleware/test_versioned_writes.py @@ -19,7 +19,7 @@ import os import time import unittest from swift.common import swob -from swift.common.middleware import versioned_writes +from swift.common.middleware import versioned_writes, copy from swift.common.swob import Request from test.unit.common.middleware.helpers import FakeSwift @@ -259,6 +259,23 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase): self.assertEqual(len(self.authorized), 1) self.assertRequestEqual(req, self.authorized[0]) + def test_put_object_post_as_copy(self): + # PUTs due to a post-as-copy should NOT cause a versioning op + self.app.register( + 'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed') + + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100', + 'swift.post_as_copy': True}) + status, headers, body = self.call_vw(req) + self.assertEqual(status, '201 Created') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + self.assertEqual(1, self.app.call_count) + def test_put_first_object_success(self): self.app.register( 'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') @@ -333,7 +350,7 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase): def test_delete_object_no_versioning_with_container_config_true(self): # set False to versions_write obviously and expect no GET versioning - # container and PUT called (just delete object as normal) + # container and GET/PUT called (just delete object as normal) self.vw.conf = {'allow_versioned_writes': 'false'} self.app.register( 'DELETE', '/v1/a/c/o', swob.HTTPNoContent, {}, 'passed') @@ -351,25 +368,6 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase): self.assertTrue('GET' not in called_method) self.assertEqual(1, self.app.call_count) - def test_copy_object_no_versioning_with_container_config_true(self): - # set False to versions_write obviously and expect no extra - # COPY called (just copy object as normal) - self.vw.conf = {'allow_versioned_writes': 'false'} - self.app.register( - 'COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None) - cache = FakeCache({'versions': 'ver_cont'}) - req = Request.blank( - '/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache}) - status, headers, body = self.call_vw(req) - self.assertEqual(status, '201 Created') - self.assertEqual(len(self.authorized), 1) - self.assertRequestEqual(req, self.authorized[0]) - called_method = \ - [method for (method, path, rheaders) in self.app._calls] - self.assertTrue('COPY' in called_method) - self.assertEqual(called_method.count('COPY'), 1) - def test_new_version_success(self): self.app.register( 'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed') @@ -476,77 +474,6 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase): self.assertEqual('PUT', method) self.assertEqual('/v1/a/ver_cont/001o/0000000000.00000', path) - def test_copy_first_version(self): - self.app.register( - 'COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') - self.app.register( - 'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound, {}, None) - cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) - req = Request.blank( - '/v1/a/src_cont/src_obj', - environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, - 'CONTENT_LENGTH': '100'}, - headers={'Destination': 'tgt_cont/tgt_obj'}) - status, headers, body = self.call_vw(req) - self.assertEqual(status, '200 OK') - self.assertEqual(len(self.authorized), 1) - self.assertRequestEqual(req, self.authorized[0]) - self.assertEqual(2, self.app.call_count) - - def test_copy_new_version(self): - self.app.register( - 'COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') - self.app.register( - 'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk, - {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed') - self.app.register( - 'PUT', '/v1/a/ver_cont/007tgt_obj/0000000001.00000', swob.HTTPOk, - {}, None) - cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) - req = Request.blank( - '/v1/a/src_cont/src_obj', - environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, - 'CONTENT_LENGTH': '100'}, - headers={'Destination': 'tgt_cont/tgt_obj'}) - status, headers, body = self.call_vw(req) - self.assertEqual(status, '200 OK') - self.assertEqual(len(self.authorized), 1) - self.assertRequestEqual(req, self.authorized[0]) - self.assertEqual(3, self.app.call_count) - - def test_copy_new_version_different_account(self): - self.app.register( - 'COPY', '/v1/src_a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') - self.app.register( - 'GET', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk, - {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed') - self.app.register( - 'PUT', '/v1/tgt_a/ver_cont/007tgt_obj/0000000001.00000', - swob.HTTPOk, {}, None) - cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) - req = Request.blank( - '/v1/src_a/src_cont/src_obj', - environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, - 'CONTENT_LENGTH': '100'}, - headers={'Destination': 'tgt_cont/tgt_obj', - 'Destination-Account': 'tgt_a'}) - status, headers, body = self.call_vw(req) - self.assertEqual(status, '200 OK') - self.assertEqual(len(self.authorized), 1) - self.assertRequestEqual(req, self.authorized[0]) - self.assertEqual(3, self.app.call_count) - - def test_copy_new_version_bogus_account(self): - cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) - req = Request.blank( - '/v1/src_a/src_cont/src_obj', - environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, - 'CONTENT_LENGTH': '100'}, - headers={'Destination': 'tgt_cont/tgt_obj', - 'Destination-Account': '/im/on/a/boat'}) - status, headers, body = self.call_vw(req) - self.assertEqual(status, '412 Precondition Failed') - def test_delete_first_object_success(self): self.app.register( 'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') @@ -1057,3 +984,117 @@ class VersionedWritesOldContainersTestCase(VersionedWritesBaseTestCase): ('PUT', '/v1/a/c/o'), ('DELETE', '/v1/a/ver_cont/001o/2'), ]) + + +class VersionedWritesCopyingTestCase(VersionedWritesBaseTestCase): + # verify interaction of copy and versioned_writes middlewares + + def setUp(self): + self.app = FakeSwift() + conf = {'allow_versioned_writes': 'true'} + self.vw = versioned_writes.filter_factory(conf)(self.app) + self.filter = copy.filter_factory({})(self.vw) + + def call_filter(self, req, **kwargs): + return self.call_app(req, app=self.filter, **kwargs) + + def test_copy_first_version(self): + # no existing object to move to the versions container + self.app.register( + 'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound, {}, None) + self.app.register( + 'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') + self.app.register( + 'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed') + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/src_cont/src_obj', + environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}, + headers={'Destination': 'tgt_cont/tgt_obj'}) + status, headers, body = self.call_filter(req) + self.assertEqual(status, '201 Created') + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path) + # note the GET on tgt_cont/tgt_obj is pre-authed + self.assertEqual(3, self.app.call_count, self.app.calls) + + def test_copy_new_version(self): + # existing object should be moved to versions container + self.app.register( + 'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') + self.app.register( + 'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk, + {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed') + self.app.register( + 'PUT', '/v1/a/ver_cont/007tgt_obj/0000000001.00000', swob.HTTPOk, + {}, None) + self.app.register( + 'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed') + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/src_cont/src_obj', + environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}, + headers={'Destination': 'tgt_cont/tgt_obj'}) + status, headers, body = self.call_filter(req) + self.assertEqual(status, '201 Created') + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path) + self.assertEqual(4, self.app.call_count) + + def test_copy_new_version_different_account(self): + self.app.register( + 'GET', '/v1/src_a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') + self.app.register( + 'GET', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk, + {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed') + self.app.register( + 'PUT', '/v1/tgt_a/ver_cont/007tgt_obj/0000000001.00000', + swob.HTTPOk, {}, None) + self.app.register( + 'PUT', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, + 'passed') + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/src_a/src_cont/src_obj', + environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}, + headers={'Destination': 'tgt_cont/tgt_obj', + 'Destination-Account': 'tgt_a'}) + status, headers, body = self.call_filter(req) + self.assertEqual(status, '201 Created') + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/src_a/src_cont/src_obj', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/tgt_a/tgt_cont/tgt_obj', self.authorized[1].path) + self.assertEqual(4, self.app.call_count) + + def test_copy_object_no_versioning_with_container_config_true(self): + # set False to versions_write obviously and expect no extra + # COPY called (just copy object as normal) + self.vw.conf = {'allow_versioned_writes': 'false'} + self.app.register( + 'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') + self.app.register( + 'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed') + cache = FakeCache({'versions': 'ver_cont'}) + req = Request.blank( + '/v1/a/src_cont/src_obj', + environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache}, + headers={'Destination': '/tgt_cont/tgt_obj'}) + status, headers, body = self.call_filter(req) + self.assertEqual(status, '201 Created') + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path) + self.assertEqual(2, self.app.call_count) diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index 2f7fb85d9b..f9829d81d5 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -173,33 +173,6 @@ class TestConstraints(unittest.TestCase): '/', headers=headers), 'object_name').status_int, HTTP_NOT_IMPLEMENTED) - def test_check_object_creation_copy(self): - headers = {'Content-Length': '0', - 'X-Copy-From': 'c/o2', - 'Content-Type': 'text/plain'} - self.assertEqual(constraints.check_object_creation(Request.blank( - '/', headers=headers), 'object_name'), None) - - headers = {'Content-Length': '1', - 'X-Copy-From': 'c/o2', - 'Content-Type': 'text/plain'} - self.assertEqual(constraints.check_object_creation(Request.blank( - '/', headers=headers), 'object_name').status_int, - HTTP_BAD_REQUEST) - - headers = {'Transfer-Encoding': 'chunked', - 'X-Copy-From': 'c/o2', - 'Content-Type': 'text/plain'} - self.assertEqual(constraints.check_object_creation(Request.blank( - '/', headers=headers), 'object_name'), None) - - # a content-length header is always required - headers = {'X-Copy-From': 'c/o2', - 'Content-Type': 'text/plain'} - self.assertEqual(constraints.check_object_creation(Request.blank( - '/', headers=headers), 'object_name').status_int, - HTTP_LENGTH_REQUIRED) - def test_check_object_creation_name_length(self): headers = {'Transfer-Encoding': 'chunked', 'Content-Type': 'text/plain'} @@ -459,60 +432,6 @@ class TestConstraints(unittest.TestCase): self.assertTrue(c.MAX_HEADER_SIZE > c.MAX_META_NAME_LENGTH) self.assertTrue(c.MAX_HEADER_SIZE > c.MAX_META_VALUE_LENGTH) - def test_validate_copy_from(self): - req = Request.blank( - '/v/a/c/o', - headers={'x-copy-from': 'c/o2'}) - src_cont, src_obj = constraints.check_copy_from_header(req) - self.assertEqual(src_cont, 'c') - self.assertEqual(src_obj, 'o2') - req = Request.blank( - '/v/a/c/o', - headers={'x-copy-from': 'c/subdir/o2'}) - src_cont, src_obj = constraints.check_copy_from_header(req) - self.assertEqual(src_cont, 'c') - self.assertEqual(src_obj, 'subdir/o2') - req = Request.blank( - '/v/a/c/o', - headers={'x-copy-from': '/c/o2'}) - src_cont, src_obj = constraints.check_copy_from_header(req) - self.assertEqual(src_cont, 'c') - self.assertEqual(src_obj, 'o2') - - def test_validate_bad_copy_from(self): - req = Request.blank( - '/v/a/c/o', - headers={'x-copy-from': 'bad_object'}) - self.assertRaises(HTTPException, - constraints.check_copy_from_header, req) - - def test_validate_destination(self): - req = Request.blank( - '/v/a/c/o', - headers={'destination': 'c/o2'}) - src_cont, src_obj = constraints.check_destination_header(req) - self.assertEqual(src_cont, 'c') - self.assertEqual(src_obj, 'o2') - req = Request.blank( - '/v/a/c/o', - headers={'destination': 'c/subdir/o2'}) - src_cont, src_obj = constraints.check_destination_header(req) - self.assertEqual(src_cont, 'c') - self.assertEqual(src_obj, 'subdir/o2') - req = Request.blank( - '/v/a/c/o', - headers={'destination': '/c/o2'}) - src_cont, src_obj = constraints.check_destination_header(req) - self.assertEqual(src_cont, 'c') - self.assertEqual(src_obj, 'o2') - - def test_validate_bad_destination(self): - req = Request.blank( - '/v/a/c/o', - headers={'destination': 'bad_object'}) - self.assertRaises(HTTPException, - constraints.check_destination_header, req) - def test_check_account_format(self): req = Request.blank( '/v/a/c/o', diff --git a/test/unit/common/test_swob.py b/test/unit/common/test_swob.py index 4f8d8f7be9..f1a11e1fcb 100644 --- a/test/unit/common/test_swob.py +++ b/test/unit/common/test_swob.py @@ -431,9 +431,10 @@ class TestRequest(unittest.TestCase): def test_invalid_req_environ_property_args(self): # getter only property try: - swift.common.swob.Request.blank('/', params={'a': 'b'}) + swift.common.swob.Request.blank( + '/', host_url='http://example.com:8080/v1/a/c/o') except TypeError as e: - self.assertEqual("got unexpected keyword argument 'params'", + self.assertEqual("got unexpected keyword argument 'host_url'", str(e)) else: self.assertTrue(False, "invalid req_environ_property " @@ -525,6 +526,14 @@ class TestRequest(unittest.TestCase): self.assertEqual(req.params['a'], 'b') self.assertEqual(req.params['c'], 'd') + new_params = {'e': 'f', 'g': 'h'} + req.params = new_params + self.assertDictEqual(new_params, req.params) + + new_params = (('i', 'j'), ('k', 'l')) + req.params = new_params + self.assertDictEqual(dict(new_params), req.params) + def test_timestamp_missing(self): req = swift.common.swob.Request.blank('/') self.assertRaises(exceptions.InvalidTimestamp, diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index f39f215499..e09f339266 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -136,6 +136,11 @@ class TestWSGI(unittest.TestCase): expected = swift.common.middleware.gatekeeper.GatekeeperMiddleware self.assertTrue(isinstance(app, expected)) + app = app.app + expected = \ + swift.common.middleware.copy.ServerSideCopyMiddleware + self.assertIsInstance(app, expected) + app = app.app expected = swift.common.middleware.dlo.DynamicLargeObject self.assertTrue(isinstance(app, expected)) @@ -1437,6 +1442,7 @@ class TestPipelineModification(unittest.TestCase): self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.copy', 'swift.common.middleware.dlo', 'swift.common.middleware.versioned_writes', 'swift.proxy.server']) @@ -1468,6 +1474,7 @@ class TestPipelineModification(unittest.TestCase): self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.copy', 'swift.common.middleware.dlo', 'swift.common.middleware.versioned_writes', 'swift.common.middleware.healthcheck', @@ -1506,6 +1513,7 @@ class TestPipelineModification(unittest.TestCase): self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.copy', 'swift.common.middleware.slo', 'swift.common.middleware.dlo', 'swift.common.middleware.versioned_writes', @@ -1605,6 +1613,7 @@ class TestPipelineModification(unittest.TestCase): self.assertEqual(self.pipeline_modules(app), [ 'swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.copy', 'swift.common.middleware.dlo', 'swift.common.middleware.versioned_writes', 'swift.common.middleware.healthcheck', @@ -1619,6 +1628,7 @@ class TestPipelineModification(unittest.TestCase): 'swift.common.middleware.gatekeeper', 'swift.common.middleware.healthcheck', 'swift.common.middleware.catch_errors', + 'swift.common.middleware.copy', 'swift.common.middleware.dlo', 'swift.common.middleware.versioned_writes', 'swift.proxy.server']) @@ -1632,6 +1642,7 @@ class TestPipelineModification(unittest.TestCase): 'swift.common.middleware.healthcheck', 'swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.copy', 'swift.common.middleware.dlo', 'swift.common.middleware.versioned_writes', 'swift.proxy.server']) @@ -1666,7 +1677,7 @@ class TestPipelineModification(unittest.TestCase): tempdir, policy.ring_name + '.ring.gz') app = wsgi.loadapp(conf_path) - proxy_app = app.app.app.app.app.app + proxy_app = app.app.app.app.app.app.app self.assertEqual(proxy_app.account_ring.serialized_path, account_ring_path) self.assertEqual(proxy_app.container_ring.serialized_path, diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index d18ac4299b..95b92b298a 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -649,7 +649,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin, def test_PUT_error_during_transfer_data(self): class FakeReader(object): def read(self, size): - raise exceptions.ChunkReadError('exception message') + raise IOError('error message') req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT', body='test body') @@ -747,62 +747,6 @@ class TestReplicatedObjController(BaseObjectControllerMixin, resp = req.get_response(self.app) self.assertEqual(resp.status_int, 404) - def test_POST_as_COPY_simple(self): - req = swift.common.swob.Request.blank('/v1/a/c/o', method='POST') - get_resp = [200] * self.obj_ring.replicas + \ - [404] * self.obj_ring.max_more_nodes - put_resp = [201] * self.obj_ring.replicas - codes = get_resp + put_resp - with set_http_connect(*codes): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 202) - self.assertEqual(req.environ['QUERY_STRING'], '') - self.assertTrue('swift.post_as_copy' in req.environ) - - def test_POST_as_COPY_static_large_object(self): - req = swift.common.swob.Request.blank('/v1/a/c/o', method='POST') - get_resp = [200] * self.obj_ring.replicas + \ - [404] * self.obj_ring.max_more_nodes - put_resp = [201] * self.obj_ring.replicas - codes = get_resp + put_resp - slo_headers = \ - [{'X-Static-Large-Object': True}] * self.obj_ring.replicas - get_headers = slo_headers + [{}] * (len(codes) - len(slo_headers)) - headers = {'headers': get_headers} - with set_http_connect(*codes, **headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 202) - self.assertEqual(req.environ['QUERY_STRING'], '') - self.assertTrue('swift.post_as_copy' in req.environ) - - def test_POST_delete_at(self): - t = str(int(time.time() + 100)) - req = swob.Request.blank('/v1/a/c/o', method='POST', - headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - post_headers = [] - - def capture_headers(ip, port, device, part, method, path, headers, - **kwargs): - if method == 'POST': - post_headers.append(headers) - x_newest_responses = [200] * self.obj_ring.replicas + \ - [404] * self.obj_ring.max_more_nodes - post_resp = [200] * self.obj_ring.replicas - codes = x_newest_responses + post_resp - with set_http_connect(*codes, give_connect=capture_headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 200) - self.assertEqual(req.environ['QUERY_STRING'], '') # sanity - self.assertTrue('swift.post_as_copy' in req.environ) - - for given_headers in post_headers: - self.assertEqual(given_headers.get('X-Delete-At'), t) - self.assertTrue('X-Delete-At-Host' in given_headers) - self.assertTrue('X-Delete-At-Device' in given_headers) - self.assertTrue('X-Delete-At-Partition' in given_headers) - self.assertTrue('X-Delete-At-Container' in given_headers) - def test_PUT_delete_at(self): t = str(int(time.time() + 100)) req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', @@ -1000,43 +944,6 @@ class TestReplicatedObjController(BaseObjectControllerMixin, resp = req.get_response(self.app) self.assertEqual(resp.status_int, 202) - def test_COPY_simple(self): - req = swift.common.swob.Request.blank( - '/v1/a/c/o', method='COPY', - headers={'Content-Length': 0, - 'Destination': 'c/o-copy'}) - head_resp = [200] * self.obj_ring.replicas + \ - [404] * self.obj_ring.max_more_nodes - put_resp = [201] * self.obj_ring.replicas - codes = head_resp + put_resp - with set_http_connect(*codes): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 201) - - def test_PUT_log_info(self): - req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT') - req.headers['x-copy-from'] = 'some/where' - req.headers['Content-Length'] = 0 - # override FakeConn default resp headers to keep log_info clean - resp_headers = {'x-delete-at': None} - head_resp = [200] * self.obj_ring.replicas + \ - [404] * self.obj_ring.max_more_nodes - put_resp = [201] * self.obj_ring.replicas - codes = head_resp + put_resp - with set_http_connect(*codes, headers=resp_headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 201) - self.assertEqual( - req.environ.get('swift.log_info'), ['x-copy-from:some/where']) - # and then check that we don't do that for originating POSTs - req = swift.common.swob.Request.blank('/v1/a/c/o') - req.method = 'POST' - req.headers['x-copy-from'] = 'else/where' - with set_http_connect(*codes, headers=resp_headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 202) - self.assertEqual(req.environ.get('swift.log_info'), None) - @patch_policies( [StoragePolicy(0, '1-replica', True), @@ -1397,7 +1304,7 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): def test_PUT_ec_error_during_transfer_data(self): class FakeReader(object): def read(self, size): - raise exceptions.ChunkReadError('exception message') + raise IOError('error message') req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT', body='test body') @@ -1603,72 +1510,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 201) - def test_COPY_cross_policy_type_from_replicated(self): - self.app.per_container_info = { - 'c1': self.app.container_info.copy(), - 'c2': self.app.container_info.copy(), - } - # make c2 use replicated storage policy 1 - self.app.per_container_info['c2']['storage_policy'] = '1' - - # a put request with copy from source c2 - req = swift.common.swob.Request.blank('/v1/a/c1/o', method='PUT', - body='', headers={ - 'X-Copy-From': 'c2/o'}) - - # c2 get - codes = [200] * self.replicas(POLICIES[1]) - codes += [404] * POLICIES[1].object_ring.max_more_nodes - # c1 put - codes += [201] * self.replicas() - expect_headers = { - 'X-Obj-Metadata-Footer': 'yes', - 'X-Obj-Multiphase-Commit': 'yes' - } - with set_http_connect(*codes, expect_headers=expect_headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 201) - - def test_COPY_cross_policy_type_to_replicated(self): - self.app.per_container_info = { - 'c1': self.app.container_info.copy(), - 'c2': self.app.container_info.copy(), - } - # make c1 use replicated storage policy 1 - self.app.per_container_info['c1']['storage_policy'] = '1' - - # a put request with copy from source c2 - req = swift.common.swob.Request.blank('/v1/a/c1/o', method='PUT', - body='', headers={ - 'X-Copy-From': 'c2/o'}) - - # c2 get - codes = [404, 200] * self.policy.ec_ndata - headers = { - 'X-Object-Sysmeta-Ec-Content-Length': 0, - } - # c1 put - codes += [201] * self.replicas(POLICIES[1]) - with set_http_connect(*codes, headers=headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 201) - - def test_COPY_cross_policy_type_unknown(self): - self.app.per_container_info = { - 'c1': self.app.container_info.copy(), - 'c2': self.app.container_info.copy(), - } - # make c1 use some made up storage policy index - self.app.per_container_info['c1']['storage_policy'] = '13' - - # a COPY request of c2 with destination in c1 - req = swift.common.swob.Request.blank('/v1/a/c2/o', method='COPY', - body='', headers={ - 'Destination': 'c1/o'}) - with set_http_connect(): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 503) - def _make_ec_archive_bodies(self, test_body, policy=None): policy = policy or self.policy segment_size = policy.ec_segment_size @@ -2378,40 +2219,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 503) - def test_COPY_with_ranges(self): - req = swift.common.swob.Request.blank( - '/v1/a/c/o', method='COPY', - headers={'Destination': 'c1/o', - 'Range': 'bytes=5-10'}) - # turn a real body into fragments - segment_size = self.policy.ec_segment_size - real_body = ('asdf' * segment_size)[:-10] - - # split it up into chunks - chunks = [real_body[x:x + segment_size] - for x in range(0, len(real_body), segment_size)] - - # we need only first chunk to rebuild 5-10 range - fragments = self.policy.pyeclib_driver.encode(chunks[0]) - fragment_payloads = [] - fragment_payloads.append(fragments) - - node_fragments = zip(*fragment_payloads) - self.assertEqual(len(node_fragments), self.replicas()) # sanity - headers = {'X-Object-Sysmeta-Ec-Content-Length': str(len(real_body))} - responses = [(200, ''.join(node_fragments[i]), headers) - for i in range(POLICIES.default.ec_ndata)] - responses += [(201, '', {})] * self.obj_ring.replicas - status_codes, body_iter, headers = zip(*responses) - expect_headers = { - 'X-Obj-Metadata-Footer': 'yes', - 'X-Obj-Multiphase-Commit': 'yes' - } - with set_http_connect(*status_codes, body_iter=body_iter, - headers=headers, expect_headers=expect_headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 201) - def test_GET_with_invalid_ranges(self): # real body size is segment_size - 10 (just 1 segment) segment_size = self.policy.ec_segment_size @@ -2424,18 +2231,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): self._test_invalid_ranges('GET', real_body, segment_size, '%s-' % (segment_size + 10)) - def test_COPY_with_invalid_ranges(self): - # real body size is segment_size - 10 (just 1 segment) - segment_size = self.policy.ec_segment_size - real_body = ('a' * segment_size)[:-10] - - # range is out of real body but in segment size - self._test_invalid_ranges('COPY', real_body, - segment_size, '%s-' % (segment_size - 10)) - # range is out of both real body and segment size - self._test_invalid_ranges('COPY', real_body, - segment_size, '%s-' % (segment_size + 10)) - def _test_invalid_ranges(self, method, real_body, segment_size, req_range): # make a request with range starts from more than real size. body_etag = md5(real_body).hexdigest() diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 6e55e74fa2..1fc021a542 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -63,7 +63,8 @@ from swift.proxy.controllers.obj import ReplicatedObjectController from swift.account import server as account_server from swift.container import server as container_server from swift.obj import server as object_server -from swift.common.middleware import proxy_logging, versioned_writes +from swift.common.middleware import proxy_logging, versioned_writes, \ + copy from swift.common.middleware.acl import parse_acl, format_acl from swift.common.exceptions import ChunkReadTimeout, DiskFileNotExist, \ APIVersionError, ChunkWriteTimeout @@ -3017,7 +3018,6 @@ class TestObjectController(unittest.TestCase): @unpatch_policies def test_PUT_POST_last_modified(self): prolis = _test_sockets[0] - prosrv = _test_servers[0] def _do_HEAD(): # do a HEAD to get reported last modified time @@ -3078,9 +3078,7 @@ class TestObjectController(unittest.TestCase): _do_conditional_GET_checks(last_modified_put) - # now POST to the object using default object_post_as_copy setting - orig_post_as_copy = prosrv.object_post_as_copy - + # now POST to the object # last-modified rounded in sec so sleep a sec to increment sleep(1) @@ -3101,31 +3099,6 @@ class TestObjectController(unittest.TestCase): self.assertNotEqual(last_modified_put, last_modified_head) _do_conditional_GET_checks(last_modified_head) - # now POST using non-default object_post_as_copy setting - try: - # last-modified rounded in sec so sleep a sec to increment - last_modified_post = last_modified_head - sleep(1) - prosrv.object_post_as_copy = not orig_post_as_copy - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('POST /v1/a/c/o.last_modified HTTP/1.1\r\n' - 'Host: localhost\r\nConnection: close\r\n' - 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 202' - self.assertEqual(headers[:len(exp)], exp) - for line in headers.split('\r\n'): - self.assertFalse(line.startswith(lm_hdr)) - finally: - prosrv.object_post_as_copy = orig_post_as_copy - - # last modified time will have changed due to POST - last_modified_head = _do_HEAD() - self.assertNotEqual(last_modified_post, last_modified_head) - _do_conditional_GET_checks(last_modified_head) - def test_PUT_auto_content_type(self): with save_globals(): controller = ReplicatedObjectController( @@ -3412,59 +3385,6 @@ class TestObjectController(unittest.TestCase): } check_request(request, **expectations) - # and this time with post as copy - self.app.object_post_as_copy = True - self.app.memcache.store = {} - backend_requests = [] - req = Request.blank('/v1/a/c/o', {}, method='POST', - headers={'X-Object-Meta-Color': 'Blue', - 'X-Backend-Storage-Policy-Index': 0}) - with mocked_http_conn( - 200, 200, 200, 200, 200, 201, 201, 201, - headers=resp_headers, give_connect=capture_requests - ) as fake_conn: - resp = req.get_response(self.app) - self.assertRaises(StopIteration, fake_conn.code_iter.next) - self.assertEqual(resp.status_int, 202) - self.assertEqual(len(backend_requests), 8) - policy0 = {'X-Backend-Storage-Policy-Index': '0'} - policy1 = {'X-Backend-Storage-Policy-Index': '1'} - expected = [ - # account info - {'method': 'HEAD', 'path': '/0/a'}, - # container info - {'method': 'HEAD', 'path': '/0/a/c'}, - # x-newests - {'method': 'GET', 'path': '/0/a/c/o', 'headers': policy1}, - {'method': 'GET', 'path': '/0/a/c/o', 'headers': policy1}, - {'method': 'GET', 'path': '/0/a/c/o', 'headers': policy1}, - # new writes - {'method': 'PUT', 'path': '/0/a/c/o', 'headers': policy0}, - {'method': 'PUT', 'path': '/0/a/c/o', 'headers': policy0}, - {'method': 'PUT', 'path': '/0/a/c/o', 'headers': policy0}, - ] - for request, expectations in zip(backend_requests, expected): - check_request(request, **expectations) - - def test_POST_as_copy(self): - with save_globals(): - def test_status_map(statuses, expected): - set_http_connect(*statuses) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, - headers={'Content-Type': 'foo/bar'}) - self.app.update_request(req) - res = req.get_response(self.app) - expected = str(expected) - self.assertEqual(res.status[:len(expected)], expected) - test_status_map((200, 200, 200, 200, 200, 202, 202, 202), 202) - test_status_map((200, 200, 200, 200, 200, 202, 202, 500), 202) - test_status_map((200, 200, 200, 200, 200, 202, 500, 500), 503) - test_status_map((200, 200, 200, 200, 200, 202, 404, 500), 503) - test_status_map((200, 200, 200, 200, 200, 202, 404, 404), 404) - test_status_map((200, 200, 200, 200, 200, 404, 500, 500), 503) - test_status_map((200, 200, 200, 200, 200, 404, 404, 404), 404) - def test_DELETE(self): with save_globals(): def test_status_map(statuses, expected): @@ -3611,26 +3531,6 @@ class TestObjectController(unittest.TestCase): res = req.get_response(self.app) self.assertEqual(res.status_int, 400) - def test_POST_as_copy_meta_val_len(self): - with save_globals(): - limit = constraints.MAX_META_VALUE_LENGTH - set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) - # acct cont objc objc objc obj obj obj - req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, - headers={'Content-Type': 'foo/bar', - 'X-Object-Meta-Foo': 'x' * limit}) - self.app.update_request(req) - res = req.get_response(self.app) - self.assertEqual(res.status_int, 202) - set_http_connect(202, 202, 202) - req = Request.blank( - '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, - headers={'Content-Type': 'foo/bar', - 'X-Object-Meta-Foo': 'x' * (limit + 1)}) - self.app.update_request(req) - res = req.get_response(self.app) - self.assertEqual(res.status_int, 400) - def test_POST_meta_key_len(self): with save_globals(): limit = constraints.MAX_META_NAME_LENGTH @@ -3653,27 +3553,6 @@ class TestObjectController(unittest.TestCase): res = req.get_response(self.app) self.assertEqual(res.status_int, 400) - def test_POST_as_copy_meta_key_len(self): - with save_globals(): - limit = constraints.MAX_META_NAME_LENGTH - set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) - # acct cont objc objc objc obj obj obj - req = Request.blank( - '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, - headers={'Content-Type': 'foo/bar', - ('X-Object-Meta-' + 'x' * limit): 'x'}) - self.app.update_request(req) - res = req.get_response(self.app) - self.assertEqual(res.status_int, 202) - set_http_connect(202, 202, 202) - req = Request.blank( - '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, - headers={'Content-Type': 'foo/bar', - ('X-Object-Meta-' + 'x' * (limit + 1)): 'x'}) - self.app.update_request(req) - res = req.get_response(self.app) - self.assertEqual(res.status_int, 400) - def test_POST_meta_count(self): with save_globals(): limit = constraints.MAX_META_COUNT @@ -4419,25 +4298,6 @@ class TestObjectController(unittest.TestCase): resp = controller.POST(req) self.assertEqual(resp.status_int, 404) - def test_PUT_POST_as_copy_requires_container_exist(self): - with save_globals(): - self.app.memcache = FakeMemcacheReturnsNone() - controller = ReplicatedObjectController( - self.app, 'account', 'container', 'object') - set_http_connect(200, 404, 404, 404, 200, 200, 200) - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) - self.app.update_request(req) - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 404) - - set_http_connect(200, 404, 404, 404, 200, 200, 200, 200, 200, 200) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'POST'}, - headers={'Content-Type': 'text/plain'}) - self.app.update_request(req) - resp = controller.POST(req) - self.assertEqual(resp.status_int, 404) - def test_bad_metadata(self): with save_globals(): controller = ReplicatedObjectController( @@ -4554,755 +4414,6 @@ class TestObjectController(unittest.TestCase): raise self.fail('UN-USED STATUS CODES: %r' % unused_status_list) - def test_basic_put_with_x_copy_from(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - - def test_basic_put_with_x_copy_from_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acc1 con1 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_basic_put_with_x_copy_from_across_container(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c2/o'}) - status_list = (200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont conc objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c2/o') - - def test_basic_put_with_x_copy_from_across_container_and_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c2/o', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acc1 con1 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c2/o') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_copy_non_zero_content_length(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '5', - 'X-Copy-From': 'c/o'}) - status_list = (200, 200) - # acct cont - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 400) - - def test_copy_non_zero_content_length_with_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '5', - 'X-Copy-From': 'c/o', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200) - # acct cont - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 400) - - def test_copy_with_slashes_in_x_copy_from(self): - # extra source path parsing - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o/o2'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - - def test_copy_with_slashes_in_x_copy_from_and_account(self): - # extra source path parsing - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o/o2', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acc1 con1 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_copy_with_spaces_in_x_copy_from(self): - # space in soure path - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o%20o2'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o%20o2') - - def test_copy_with_spaces_in_x_copy_from_and_account(self): - # space in soure path - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o%20o2', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acc1 con1 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o%20o2') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_copy_with_leading_slash_in_x_copy_from(self): - # repeat tests with leading / - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - - def test_copy_with_leading_slash_in_x_copy_from_and_account(self): - # repeat tests with leading / - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acc1 con1 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_copy_with_leading_slash_and_slashes_in_x_copy_from(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o/o2'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - - def test_copy_with_leading_slash_and_slashes_in_x_copy_from_acct(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o/o2', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acc1 con1 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_copy_with_no_object_in_x_copy_from(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c'}) - status_list = (200, 200) - # acct cont - with self.controller_context(req, *status_list) as controller: - try: - controller.PUT(req) - except HTTPException as resp: - self.assertEqual(resp.status_int // 100, 4) # client error - else: - raise self.fail('Invalid X-Copy-From did not raise ' - 'client error') - - def test_copy_with_no_object_in_x_copy_from_and_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200) - # acct cont - with self.controller_context(req, *status_list) as controller: - try: - controller.PUT(req) - except HTTPException as resp: - self.assertEqual(resp.status_int // 100, 4) # client error - else: - raise self.fail('Invalid X-Copy-From did not raise ' - 'client error') - - def test_copy_server_error_reading_source(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - status_list = (200, 200, 503, 503, 503) - # acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 503) - - def test_copy_server_error_reading_source_and_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 503, 503, 503) - # acct cont acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 503) - - def test_copy_not_found_reading_source(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - # not found - status_list = (200, 200, 404, 404, 404) - # acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 404) - - def test_copy_not_found_reading_source_and_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Copy-From-Account': 'a'}) - # not found - status_list = (200, 200, 200, 200, 404, 404, 404) - # acct cont acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 404) - - def test_copy_with_some_missing_sources(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - status_list = (200, 200, 404, 404, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - - def test_copy_with_some_missing_sources_and_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 404, 404, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - - def test_copy_with_object_metadata(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Object-Meta-Ours': 'okay'}) - # test object metadata - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers.get('x-object-meta-test'), 'testing') - self.assertEqual(resp.headers.get('x-object-meta-ours'), 'okay') - self.assertEqual(resp.headers.get('x-delete-at'), '9876543210') - - def test_copy_with_object_metadata_and_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Object-Meta-Ours': 'okay', - 'X-Copy-From-Account': 'a'}) - # test object metadata - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers.get('x-object-meta-test'), 'testing') - self.assertEqual(resp.headers.get('x-object-meta-ours'), 'okay') - self.assertEqual(resp.headers.get('x-delete-at'), '9876543210') - - @_limit_max_file_size - def test_copy_source_larger_than_max_file_size(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - # copy-from object is too large to fit in target object - - class LargeResponseBody(object): - - def __len__(self): - return constraints.MAX_FILE_SIZE + 1 - - def __getitem__(self, key): - return '' - - copy_from_obj_body = LargeResponseBody() - status_list = (200, 200, 200, 200, 200) - # acct cont objc objc objc - kwargs = dict(body=copy_from_obj_body) - with self.controller_context(req, *status_list, - **kwargs) as controller: - self.app.update_request(req) - - self.app.memcache.store = {} - try: - resp = controller.PUT(req) - except HTTPException as resp: - pass - self.assertEqual(resp.status_int, 413) - - def test_basic_COPY(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c/o2'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - - def test_basic_COPY_account(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c1/o2', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_COPY_across_containers(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c2/o'}) - status_list = (200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont c2 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - - def test_COPY_source_with_slashes_in_name(self): - req = Request.blank('/v1/a/c/o/o2', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c/o'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - - def test_COPY_account_source_with_slashes_in_name(self): - req = Request.blank('/v1/a/c/o/o2', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c1/o', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_COPY_destination_leading_slash(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - - def test_COPY_account_destination_leading_slash(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_COPY_source_with_slashes_destination_leading_slash(self): - req = Request.blank('/v1/a/c/o/o2', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - - def test_COPY_account_source_with_slashes_destination_leading_slash(self): - req = Request.blank('/v1/a/c/o/o2', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_COPY_no_object_in_destination(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c_o'}) - status_list = [] # no requests needed - with self.controller_context(req, *status_list) as controller: - self.assertRaises(HTTPException, controller.COPY, req) - - def test_COPY_account_no_object_in_destination(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c_o', - 'Destination-Account': 'a1'}) - status_list = [] # no requests needed - with self.controller_context(req, *status_list) as controller: - self.assertRaises(HTTPException, controller.COPY, req) - - def test_COPY_server_error_reading_source(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - status_list = (200, 200, 503, 503, 503) - # acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 503) - - def test_COPY_account_server_error_reading_source(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 503, 503, 503) - # acct cont acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 503) - - def test_COPY_not_found_reading_source(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - status_list = (200, 200, 404, 404, 404) - # acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 404) - - def test_COPY_account_not_found_reading_source(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 404, 404, 404) - # acct cont acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 404) - - def test_COPY_with_some_missing_sources(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - status_list = (200, 200, 404, 404, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - - def test_COPY_account_with_some_missing_sources(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 404, 404, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - - def test_COPY_with_metadata(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o', - 'X-Object-Meta-Ours': 'okay'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers.get('x-object-meta-test'), - 'testing') - self.assertEqual(resp.headers.get('x-object-meta-ours'), 'okay') - self.assertEqual(resp.headers.get('x-delete-at'), '9876543210') - - def test_COPY_account_with_metadata(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'X-Object-Meta-Ours': 'okay', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers.get('x-object-meta-test'), - 'testing') - self.assertEqual(resp.headers.get('x-object-meta-ours'), 'okay') - self.assertEqual(resp.headers.get('x-delete-at'), '9876543210') - - @_limit_max_file_size - def test_COPY_source_larger_than_max_file_size(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - - class LargeResponseBody(object): - - def __len__(self): - return constraints.MAX_FILE_SIZE + 1 - - def __getitem__(self, key): - return '' - - copy_from_obj_body = LargeResponseBody() - status_list = (200, 200, 200, 200, 200) - # acct cont objc objc objc - kwargs = dict(body=copy_from_obj_body) - with self.controller_context(req, *status_list, - **kwargs) as controller: - try: - resp = controller.COPY(req) - except HTTPException as resp: - pass - self.assertEqual(resp.status_int, 413) - - @_limit_max_file_size - def test_COPY_account_source_larger_than_max_file_size(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - - class LargeResponseBody(object): - - def __len__(self): - return constraints.MAX_FILE_SIZE + 1 - - def __getitem__(self, key): - return '' - - copy_from_obj_body = LargeResponseBody() - status_list = (200, 200, 200, 200, 200) - # acct cont objc objc objc - kwargs = dict(body=copy_from_obj_body) - with self.controller_context(req, *status_list, - **kwargs) as controller: - try: - resp = controller.COPY(req) - except HTTPException as resp: - pass - self.assertEqual(resp.status_int, 413) - - def test_COPY_newest(self): - with save_globals(): - controller = ReplicatedObjectController( - self.app, 'a', 'c', 'o') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, - # act cont objc objc objc obj obj obj - timestamps=('1', '1', '1', '3', '2', '4', '4', - '4')) - self.app.memcache.store = {} - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from-last-modified'], - '3') - - def test_COPY_account_newest(self): - with save_globals(): - controller = ReplicatedObjectController( - self.app, 'a', 'c', 'o') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201, - # act cont acct cont objc objc objc obj obj obj - timestamps=('1', '1', '1', '1', '3', '2', '1', - '4', '4', '4')) - self.app.memcache.store = {} - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from-last-modified'], - '3') - - def test_COPY_delete_at(self): - with save_globals(): - backend_requests = [] - - def capture_requests(ipaddr, port, device, partition, method, path, - headers=None, query_string=None): - backend_requests.append((method, path, headers)) - - controller = ReplicatedObjectController( - self.app, 'a', 'c', 'o') - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, - give_connect=capture_requests) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - - self.app.update_request(req) - resp = controller.COPY(req) - self.assertEqual(201, resp.status_int) # sanity - for method, path, given_headers in backend_requests: - if method != 'PUT': - continue - self.assertEqual(given_headers.get('X-Delete-At'), - '9876543210') - self.assertTrue('X-Delete-At-Host' in given_headers) - self.assertTrue('X-Delete-At-Device' in given_headers) - self.assertTrue('X-Delete-At-Partition' in given_headers) - self.assertTrue('X-Delete-At-Container' in given_headers) - - def test_COPY_account_delete_at(self): - with save_globals(): - backend_requests = [] - - def capture_requests(ipaddr, port, device, partition, method, path, - headers=None, query_string=None): - backend_requests.append((method, path, headers)) - - controller = ReplicatedObjectController( - self.app, 'a', 'c', 'o') - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201, - give_connect=capture_requests) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - - self.app.update_request(req) - resp = controller.COPY(req) - self.assertEqual(201, resp.status_int) # sanity - for method, path, given_headers in backend_requests: - if method != 'PUT': - continue - self.assertEqual(given_headers.get('X-Delete-At'), - '9876543210') - self.assertTrue('X-Delete-At-Host' in given_headers) - self.assertTrue('X-Delete-At-Device' in given_headers) - self.assertTrue('X-Delete-At-Partition' in given_headers) - self.assertTrue('X-Delete-At-Container' in given_headers) - - def test_chunked_put(self): - - class ChunkedFile(object): - - def __init__(self, bytes): - self.bytes = bytes - self.read_bytes = 0 - - @property - def bytes_left(self): - return self.bytes - self.read_bytes - - def read(self, amt=None): - if self.read_bytes >= self.bytes: - raise StopIteration() - if not amt: - amt = self.bytes_left - data = 'a' * min(amt, self.bytes_left) - self.read_bytes += len(data) - return data - - with save_globals(): - set_http_connect(201, 201, 201, 201) - controller = ReplicatedObjectController( - self.app, 'account', 'container', 'object') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Transfer-Encoding': 'chunked', - 'Content-Type': 'foo/bar'}) - - req.body_file = ChunkedFile(10) - self.app.memcache.store = {} - self.app.update_request(req) - res = controller.PUT(req) - self.assertEqual(res.status_int // 100, 2) # success - - # test 413 entity to large - set_http_connect(201, 201, 201, 201) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Transfer-Encoding': 'chunked', - 'Content-Type': 'foo/bar'}) - req.body_file = ChunkedFile(11) - self.app.memcache.store = {} - self.app.update_request(req) - - with mock.patch('swift.common.constraints.MAX_FILE_SIZE', 10): - res = controller.PUT(req) - self.assertEqual(res.status_int, 413) - @unpatch_policies def test_chunked_put_bad_version(self): # Check bad version @@ -5720,24 +4831,6 @@ class TestObjectController(unittest.TestCase): controller.POST(req) self.assertTrue(called[0]) - def test_POST_as_copy_calls_authorize(self): - called = [False] - - def authorize(req): - called[0] = True - return HTTPUnauthorized(request=req) - with save_globals(): - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - controller = ReplicatedObjectController( - self.app, 'account', 'container', 'object') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'POST'}, - headers={'Content-Length': '5'}, body='12345') - req.environ['swift.authorize'] = authorize - self.app.update_request(req) - controller.POST(req) - self.assertTrue(called[0]) - def test_PUT_calls_authorize(self): called = [False] @@ -5755,24 +4848,6 @@ class TestObjectController(unittest.TestCase): controller.PUT(req) self.assertTrue(called[0]) - def test_COPY_calls_authorize(self): - called = [False] - - def authorize(req): - called[0] = True - return HTTPUnauthorized(request=req) - with save_globals(): - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - controller = ReplicatedObjectController( - self.app, 'account', 'container', 'object') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c/o'}) - req.environ['swift.authorize'] = authorize - self.app.update_request(req) - controller.COPY(req) - self.assertTrue(called[0]) - def test_POST_converts_delete_after_to_delete_at(self): with save_globals(): self.app.object_post_as_copy = False @@ -6021,12 +5096,12 @@ class TestObjectController(unittest.TestCase): self.assertEqual( 'https://foo.bar', resp.headers['access-control-allow-origin']) - for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split(): + for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) self.assertEqual( len(resp.headers['access-control-allow-methods'].split(', ')), - 7) + 6) self.assertEqual('999', resp.headers['access-control-max-age']) req = Request.blank( '/v1/a/c/o.jpg', @@ -6039,10 +5114,10 @@ class TestObjectController(unittest.TestCase): req.content_length = 0 resp = controller.OPTIONS(req) self.assertEqual(200, resp.status_int) - for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split(): + for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['Allow']) - self.assertEqual(len(resp.headers['Allow'].split(', ')), 7) + self.assertEqual(len(resp.headers['Allow'].split(', ')), 6) req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'}, @@ -6075,12 +5150,12 @@ class TestObjectController(unittest.TestCase): resp = controller.OPTIONS(req) self.assertEqual(200, resp.status_int) self.assertEqual('*', resp.headers['access-control-allow-origin']) - for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split(): + for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) self.assertEqual( len(resp.headers['access-control-allow-methods'].split(', ')), - 7) + 6) self.assertEqual('999', resp.headers['access-control-max-age']) def _get_CORS_response(self, container_cors, strict_mode, object_get=None): @@ -9232,9 +8307,10 @@ class TestSocketObjectVersions(unittest.TestCase): conf = {'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'allowed_headers': allowed_headers} prosrv = versioned_writes.VersionedWritesMiddleware( - proxy_logging.ProxyLoggingMiddleware( - _test_servers[0], conf, - logger=_test_servers[0].logger), + copy.ServerSideCopyMiddleware( + proxy_logging.ProxyLoggingMiddleware( + _test_servers[0], conf, + logger=_test_servers[0].logger), conf), {}) self.coro = spawn(wsgi.server, prolis, prosrv, NullLogger()) # replace global prosrv with one that's filtered with version diff --git a/test/unit/proxy/test_sysmeta.py b/test/unit/proxy/test_sysmeta.py index 3b3f8ddfd9..9548680791 100644 --- a/test/unit/proxy/test_sysmeta.py +++ b/test/unit/proxy/test_sysmeta.py @@ -283,94 +283,3 @@ class TestObjectSysmeta(unittest.TestCase): self._assertInHeaders(resp, self.changed_sysmeta_headers) self._assertInHeaders(resp, self.new_sysmeta_headers) self._assertNotInHeaders(resp, self.original_sysmeta_headers_2) - - def test_sysmeta_not_updated_by_POST(self): - self.app.object_post_as_copy = False - self._test_sysmeta_not_updated_by_POST() - - def test_sysmeta_not_updated_by_POST_as_copy(self): - self.app.object_post_as_copy = True - self._test_sysmeta_not_updated_by_POST() - - def test_sysmeta_updated_by_COPY(self): - # check sysmeta is updated by a COPY in same way as user meta - path = '/v1/a/c/o' - dest = '/c/o2' - env = {'REQUEST_METHOD': 'PUT'} - hdrs = dict(self.original_sysmeta_headers_1) - hdrs.update(self.original_sysmeta_headers_2) - hdrs.update(self.original_meta_headers_1) - hdrs.update(self.original_meta_headers_2) - req = Request.blank(path, environ=env, headers=hdrs, body='x') - resp = req.get_response(self.app) - self._assertStatus(resp, 201) - - env = {'REQUEST_METHOD': 'COPY'} - hdrs = dict(self.changed_sysmeta_headers) - hdrs.update(self.new_sysmeta_headers) - hdrs.update(self.changed_meta_headers) - hdrs.update(self.new_meta_headers) - hdrs.update(self.bad_headers) - hdrs.update({'Destination': dest}) - req = Request.blank(path, environ=env, headers=hdrs) - resp = req.get_response(self.app) - self._assertStatus(resp, 201) - self._assertInHeaders(resp, self.changed_sysmeta_headers) - self._assertInHeaders(resp, self.new_sysmeta_headers) - self._assertInHeaders(resp, self.original_sysmeta_headers_2) - self._assertInHeaders(resp, self.changed_meta_headers) - self._assertInHeaders(resp, self.new_meta_headers) - self._assertInHeaders(resp, self.original_meta_headers_2) - self._assertNotInHeaders(resp, self.bad_headers) - - req = Request.blank('/v1/a/c/o2', environ={}) - resp = req.get_response(self.app) - self._assertStatus(resp, 200) - self._assertInHeaders(resp, self.changed_sysmeta_headers) - self._assertInHeaders(resp, self.new_sysmeta_headers) - self._assertInHeaders(resp, self.original_sysmeta_headers_2) - self._assertInHeaders(resp, self.changed_meta_headers) - self._assertInHeaders(resp, self.new_meta_headers) - self._assertInHeaders(resp, self.original_meta_headers_2) - self._assertNotInHeaders(resp, self.bad_headers) - - def test_sysmeta_updated_by_COPY_from(self): - # check sysmeta is updated by a COPY in same way as user meta - path = '/v1/a/c/o' - env = {'REQUEST_METHOD': 'PUT'} - hdrs = dict(self.original_sysmeta_headers_1) - hdrs.update(self.original_sysmeta_headers_2) - hdrs.update(self.original_meta_headers_1) - hdrs.update(self.original_meta_headers_2) - req = Request.blank(path, environ=env, headers=hdrs, body='x') - resp = req.get_response(self.app) - self._assertStatus(resp, 201) - - env = {'REQUEST_METHOD': 'PUT'} - hdrs = dict(self.changed_sysmeta_headers) - hdrs.update(self.new_sysmeta_headers) - hdrs.update(self.changed_meta_headers) - hdrs.update(self.new_meta_headers) - hdrs.update(self.bad_headers) - hdrs.update({'X-Copy-From': '/c/o'}) - req = Request.blank('/v1/a/c/o2', environ=env, headers=hdrs, body='') - resp = req.get_response(self.app) - self._assertStatus(resp, 201) - self._assertInHeaders(resp, self.changed_sysmeta_headers) - self._assertInHeaders(resp, self.new_sysmeta_headers) - self._assertInHeaders(resp, self.original_sysmeta_headers_2) - self._assertInHeaders(resp, self.changed_meta_headers) - self._assertInHeaders(resp, self.new_meta_headers) - self._assertInHeaders(resp, self.original_meta_headers_2) - self._assertNotInHeaders(resp, self.bad_headers) - - req = Request.blank('/v1/a/c/o2', environ={}) - resp = req.get_response(self.app) - self._assertStatus(resp, 200) - self._assertInHeaders(resp, self.changed_sysmeta_headers) - self._assertInHeaders(resp, self.new_sysmeta_headers) - self._assertInHeaders(resp, self.original_sysmeta_headers_2) - self._assertInHeaders(resp, self.changed_meta_headers) - self._assertInHeaders(resp, self.new_meta_headers) - self._assertInHeaders(resp, self.original_meta_headers_2) - self._assertNotInHeaders(resp, self.bad_headers)