diff --git a/doc/saio/swift/proxy-server.conf b/doc/saio/swift/proxy-server.conf index d9e5c95148..76b85d5818 100644 --- a/doc/saio/swift/proxy-server.conf +++ b/doc/saio/swift/proxy-server.conf @@ -9,7 +9,7 @@ eventlet_debug = true [pipeline:main] # Yes, proxy-logging appears twice. This is so that # middleware-originated requests get logged too. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain container_sync tempauth staticweb container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain container_sync tempauth staticweb copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server [filter:catch_errors] use = egg:swift#catch_errors @@ -68,6 +68,9 @@ use = egg:swift#gatekeeper use = egg:swift#versioned_writes allow_versioned_writes = true +[filter:copy] +use = egg:swift#copy + [app:proxy-server] use = egg:swift#proxy allow_account_management = true diff --git a/doc/source/logs.rst b/doc/source/logs.rst index 75b669f1a5..7e2c1dd94b 100644 --- a/doc/source/logs.rst +++ b/doc/source/logs.rst @@ -103,6 +103,7 @@ LE :ref:`list_endpoints` KS :ref:`keystoneauth` RL :ref:`ratelimit` VW :ref:`versioned_writes` +SSC :ref:`copy` ======================= ============================= diff --git a/doc/source/middleware.rst b/doc/source/middleware.rst index 3c17339b17..a078747204 100644 --- a/doc/source/middleware.rst +++ b/doc/source/middleware.rst @@ -187,6 +187,15 @@ Recon :members: :show-inheritance: +.. _copy: + +Server Side Copy +================ + +.. automodule:: swift.common.middleware.copy + :members: + :show-inheritance: + Static Large Objects ==================== diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index ba860b9b9d..b5cfbf873b 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -79,12 +79,12 @@ bind_port = 8080 [pipeline:main] # This sample pipeline uses tempauth and is used for SAIO dev work and # testing. See below for a pipeline using keystone. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server # The following pipeline shows keystone integration. Comment out the one # above and uncomment this one. Additional steps for integrating keystone are # covered further below in the filter sections for authtoken and keystoneauth. -#pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit authtoken keystoneauth container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server +#pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit authtoken keystoneauth copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server [app:proxy-server] use = egg:swift#proxy @@ -129,11 +129,6 @@ use = egg:swift#proxy # 'false' no one, even authorized, can. # allow_account_management = false # -# Set object_post_as_copy = false to turn on fast posts where only the metadata -# changes are stored anew and the original data file is kept in place. This -# makes for quicker posts. -# object_post_as_copy = true -# # If set to 'true' authorized accounts that do not yet exist within the Swift # cluster will be automatically created. # account_autocreate = false @@ -749,3 +744,14 @@ use = egg:swift#versioned_writes # in the container configuration file, which will be eventually # deprecated. See documentation for more details. # allow_versioned_writes = false + +# Note: Put after auth and before dlo and slo middlewares. +# If you don't put it in the pipeline, it will be inserted for you. +[filter:copy] +use = egg:swift#copy +# Set object_post_as_copy = false to turn on fast posts where only the metadata +# changes are stored anew and the original data file is kept in place. This +# makes for quicker posts. +# When object_post_as_copy is set to True, a POST request will be transformed +# into a COPY request where source and destination objects are the same. +# object_post_as_copy = true diff --git a/setup.cfg b/setup.cfg index 77c6824b44..098b6c64f7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -96,6 +96,7 @@ paste.filter_factory = container_sync = swift.common.middleware.container_sync:filter_factory xprofile = swift.common.middleware.xprofile:filter_factory versioned_writes = swift.common.middleware.versioned_writes:filter_factory + copy = swift.common.middleware.copy:filter_factory [build_sphinx] all_files = 1 diff --git a/swift/common/constraints.py b/swift/common/constraints.py index abfab4bb9e..787d2d91da 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -20,7 +20,6 @@ import time import six from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError from six.moves import urllib -from six.moves.urllib.parse import unquote from swift.common import utils, exceptions from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \ @@ -205,10 +204,6 @@ def check_object_creation(req, object_name): request=req, content_type='text/plain') - if 'X-Copy-From' in req.headers and req.content_length: - return HTTPBadRequest(body='Copy requests require a zero byte body', - request=req, content_type='text/plain') - if len(object_name) > MAX_OBJECT_NAME_LENGTH: return HTTPBadRequest(body='Object name length of %d longer than %d' % (len(object_name), MAX_OBJECT_NAME_LENGTH), @@ -359,63 +354,6 @@ def check_utf8(string): return False -def check_path_header(req, name, length, error_msg): - """ - Validate that the value of path-like header is - well formatted. We assume the caller ensures that - specific header is present in req.headers. - - :param req: HTTP request object - :param name: header name - :param length: length of path segment check - :param error_msg: error message for client - :returns: A tuple with path parts according to length - :raise: HTTPPreconditionFailed if header value - is not well formatted. - """ - src_header = unquote(req.headers.get(name)) - if not src_header.startswith('/'): - src_header = '/' + src_header - try: - return utils.split_path(src_header, length, length, True) - except ValueError: - raise HTTPPreconditionFailed( - request=req, - body=error_msg) - - -def check_copy_from_header(req): - """ - Validate that the value from x-copy-from header is - well formatted. We assume the caller ensures that - x-copy-from header is present in req.headers. - - :param req: HTTP request object - :returns: A tuple with container name and object name - :raise: HTTPPreconditionFailed if x-copy-from value - is not well formatted. - """ - return check_path_header(req, 'X-Copy-From', 2, - 'X-Copy-From header must be of the form ' - '/') - - -def check_destination_header(req): - """ - Validate that the value from destination header is - well formatted. We assume the caller ensures that - destination header is present in req.headers. - - :param req: HTTP request object - :returns: A tuple with container name and object name - :raise: HTTPPreconditionFailed if destination value - is not well formatted. - """ - return check_path_header(req, 'Destination', 2, - 'Destination header must be of the form ' - '/') - - def check_name_format(req, name, target_type): """ Validate that the header contains valid account or container name. diff --git a/swift/common/middleware/account_quotas.py b/swift/common/middleware/account_quotas.py index fcb55b5573..8811aad84c 100644 --- a/swift/common/middleware/account_quotas.py +++ b/swift/common/middleware/account_quotas.py @@ -52,11 +52,10 @@ Due to the eventual consistency further uploads might be possible until the account size has been updated. """ -from swift.common.constraints import check_copy_from_header from swift.common.swob import HTTPForbidden, HTTPBadRequest, \ HTTPRequestEntityTooLarge, wsgify from swift.common.utils import register_swift_info -from swift.proxy.controllers.base import get_account_info, get_object_info +from swift.proxy.controllers.base import get_account_info class AccountQuotaMiddleware(object): @@ -71,7 +70,7 @@ class AccountQuotaMiddleware(object): @wsgify def __call__(self, request): - if request.method not in ("POST", "PUT", "COPY"): + if request.method not in ("POST", "PUT"): return self.app try: @@ -106,15 +105,6 @@ class AccountQuotaMiddleware(object): if request.method == "POST" or not obj: return self.app - if request.method == 'COPY': - copy_from = container + '/' + obj - else: - if 'x-copy-from' in request.headers: - src_cont, src_obj = check_copy_from_header(request) - copy_from = "%s/%s" % (src_cont, src_obj) - else: - copy_from = None - content_length = (request.content_length or 0) account_info = get_account_info(request.environ, self.app) @@ -127,14 +117,6 @@ class AccountQuotaMiddleware(object): if quota < 0: return self.app - if copy_from: - path = '/' + ver + '/' + account + '/' + copy_from - object_info = get_object_info(request.environ, self.app, path) - if not object_info or not object_info['length']: - content_length = 0 - else: - content_length = int(object_info['length']) - new_size = int(account_info['bytes']) + content_length if quota < new_size: resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.') diff --git a/swift/common/middleware/container_quotas.py b/swift/common/middleware/container_quotas.py index 4feca69a7b..a78876aca5 100644 --- a/swift/common/middleware/container_quotas.py +++ b/swift/common/middleware/container_quotas.py @@ -51,13 +51,11 @@ For example:: [filter:container_quotas] use = egg:swift#container_quotas """ -from swift.common.constraints import check_copy_from_header, \ - check_account_format, check_destination_header from swift.common.http import is_success from swift.common.swob import HTTPRequestEntityTooLarge, HTTPBadRequest, \ wsgify from swift.common.utils import register_swift_info -from swift.proxy.controllers.base import get_container_info, get_object_info +from swift.proxy.controllers.base import get_container_info class ContainerQuotaMiddleware(object): @@ -91,25 +89,9 @@ class ContainerQuotaMiddleware(object): return HTTPBadRequest(body='Invalid count quota.') # check user uploads against quotas - elif obj and req.method in ('PUT', 'COPY'): - container_info = None - if req.method == 'PUT': - container_info = get_container_info( - req.environ, self.app, swift_source='CQ') - if req.method == 'COPY' and 'Destination' in req.headers: - dest_account = account - if 'Destination-Account' in req.headers: - dest_account = req.headers.get('Destination-Account') - dest_account = check_account_format(req, dest_account) - dest_container, dest_object = check_destination_header(req) - path_info = req.environ['PATH_INFO'] - req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( - version, dest_account, dest_container, dest_object) - try: - container_info = get_container_info( - req.environ, self.app, swift_source='CQ') - finally: - req.environ['PATH_INFO'] = path_info + elif obj and req.method in ('PUT'): + container_info = get_container_info( + req.environ, self.app, swift_source='CQ') if not container_info or not is_success(container_info['status']): # this will hopefully 404 later return self.app @@ -118,16 +100,6 @@ class ContainerQuotaMiddleware(object): 'bytes' in container_info and \ container_info['meta']['quota-bytes'].isdigit(): content_length = (req.content_length or 0) - if 'x-copy-from' in req.headers or req.method == 'COPY': - if 'x-copy-from' in req.headers: - container, obj = check_copy_from_header(req) - path = '/%s/%s/%s/%s' % (version, account, - container, obj) - object_info = get_object_info(req.environ, self.app, path) - if not object_info or not object_info['length']: - content_length = 0 - else: - content_length = int(object_info['length']) new_size = int(container_info['bytes']) + content_length if int(container_info['meta']['quota-bytes']) < new_size: return self.bad_response(req, container_info) diff --git a/swift/common/middleware/copy.py b/swift/common/middleware/copy.py new file mode 100644 index 0000000000..e895813e8d --- /dev/null +++ b/swift/common/middleware/copy.py @@ -0,0 +1,522 @@ +# Copyright (c) 2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Server side copy is a feature that enables users/clients to COPY objects +between accounts and containers without the need to download and then +re-upload objects, thus eliminating additional bandwidth consumption and +also saving time. This may be used when renaming/moving an object which +in Swift is a (COPY + DELETE) operation. + +The server side copy middleware should be inserted in the pipeline after auth +and before the quotas and large object middlewares. If it is not present in the +pipeline in the proxy-server configuration file, it will be inserted +automatically. There is no configurable option provided to turn off server +side copy. + +-------- +Metadata +-------- +* All metadata of source object is preserved during object copy. +* One can also provide additional metadata during PUT/COPY request. This will + over-write any existing conflicting keys. +* Server side copy can also be used to change content-type of an existing + object. + +----------- +Object Copy +----------- +* The destination container must exist before requesting copy of the object. +* When several replicas exist, the system copies from the most recent replica. + That is, the copy operation behaves as though the X-Newest header is in the + request. +* The request to copy an object should have no body (i.e. content-length of the + request must be zero). + +There are two ways in which an object can be copied: + +1. Send a PUT request to the new object (destination/target) with an additional + header named ``X-Copy-From`` specifying the source object + (in '/container/object' format). Example:: + + curl -i -X PUT http:///container1/destination_obj + -H 'X-Auth-Token: ' + -H 'X-Copy-From: /container2/source_obj' + -H 'Content-Length: 0' + +2. Send a COPY request with an existing object in URL with an additional header + named ``Destination`` specifying the destination/target object + (in '/container/object' format). Example:: + + curl -i -X COPY http:///container2/source_obj + -H 'X-Auth-Token: ' + -H 'Destination: /container1/destination_obj' + -H 'Content-Length: 0' + +Note that if the incoming request has some conditional headers (e.g. ``Range``, +``If-Match``), the *source* object will be evaluated for these headers (i.e. if +PUT with both ``X-Copy-From`` and ``Range``, Swift will make a partial copy to +the destination object). + +------------------------- +Cross Account Object Copy +------------------------- +Objects can also be copied from one account to another account if the user +has the necessary permissions (i.e. permission to read from container +in source account and permission to write to container in destination account). + +Similar to examples mentioned above, there are two ways to copy objects across +accounts: + +1. Like the example above, send PUT request to copy object but with an + additional header named ``X-Copy-From-Account`` specifying the source + account. Example:: + + curl -i -X PUT http://:/v1/AUTH_test1/container/destination_obj + -H 'X-Auth-Token: ' + -H 'X-Copy-From: /container/source_obj' + -H 'X-Copy-From-Account: AUTH_test2' + -H 'Content-Length: 0' + +2. Like the previous example, send a COPY request but with an additional header + named ``Destination-Account`` specifying the name of destination account. + Example:: + + curl -i -X COPY http://:/v1/AUTH_test2/container/source_obj + -H 'X-Auth-Token: ' + -H 'Destination: /container/destination_obj' + -H 'Destination-Account: AUTH_test1' + -H 'Content-Length: 0' + +------------------- +Large Object Copy +------------------- +The best option to copy a large option is to copy segments individually. +To copy the manifest object of a large object, add the query parameter to +the copy request:: + + ?multipart-manifest=get + +If a request is sent without the query parameter, an attempt will be made to +copy the whole object but will fail if the object size is +greater than 5GB. + +------------------- +Object Post as Copy +------------------- +Historically, this has been a feature (and a configurable option with default +set to True) in proxy server configuration. This has been moved to server side +copy middleware. + +When ``object_post_as_copy`` is set to ``true`` (default value), an incoming +POST request is morphed into a COPY request where source and destination +objects are same. + +This feature was necessary because of a previous behavior where POSTS would +update the metadata on the object but not on the container. As a result, +features like container sync would not work correctly. This is no longer the +case and the plan is to deprecate this option. It is being kept now for +backwards compatibility. At first chance, set ``object_post_as_copy`` to +``false``. +""" + +import os +from urllib import quote +from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from six.moves.urllib.parse import unquote + +from swift.common import utils +from swift.common.utils import get_logger, \ + config_true_value, FileLikeIter, read_conf_dir, close_if_possible +from swift.common.swob import Request, HTTPPreconditionFailed, \ + HTTPRequestEntityTooLarge, HTTPBadRequest +from swift.common.http import HTTP_MULTIPLE_CHOICES, HTTP_CREATED, \ + is_success +from swift.common.constraints import check_account_format, MAX_FILE_SIZE +from swift.common.request_helpers import copy_header_subset, remove_items, \ + is_sys_meta, is_sys_or_user_meta +from swift.common.wsgi import WSGIContext, make_subrequest + + +def _check_path_header(req, name, length, error_msg): + """ + Validate that the value of path-like header is + well formatted. We assume the caller ensures that + specific header is present in req.headers. + + :param req: HTTP request object + :param name: header name + :param length: length of path segment check + :param error_msg: error message for client + :returns: A tuple with path parts according to length + :raise: HTTPPreconditionFailed if header value + is not well formatted. + """ + src_header = unquote(req.headers.get(name)) + if not src_header.startswith('/'): + src_header = '/' + src_header + try: + return utils.split_path(src_header, length, length, True) + except ValueError: + raise HTTPPreconditionFailed( + request=req, + body=error_msg) + + +def _check_copy_from_header(req): + """ + Validate that the value from x-copy-from header is + well formatted. We assume the caller ensures that + x-copy-from header is present in req.headers. + + :param req: HTTP request object + :returns: A tuple with container name and object name + :raise: HTTPPreconditionFailed if x-copy-from value + is not well formatted. + """ + return _check_path_header(req, 'X-Copy-From', 2, + 'X-Copy-From header must be of the form ' + '/') + + +def _check_destination_header(req): + """ + Validate that the value from destination header is + well formatted. We assume the caller ensures that + destination header is present in req.headers. + + :param req: HTTP request object + :returns: A tuple with container name and object name + :raise: HTTPPreconditionFailed if destination value + is not well formatted. + """ + return _check_path_header(req, 'Destination', 2, + 'Destination header must be of the form ' + '/') + + +def _copy_headers_into(from_r, to_r): + """ + Will copy desired headers from from_r to to_r + :params from_r: a swob Request or Response + :params to_r: a swob Request or Response + """ + pass_headers = ['x-delete-at'] + for k, v in from_r.headers.items(): + if is_sys_or_user_meta('object', k) or k.lower() in pass_headers: + to_r.headers[k] = v + + +class ServerSideCopyWebContext(WSGIContext): + + def __init__(self, app, logger): + super(ServerSideCopyWebContext, self).__init__(app) + self.app = app + self.logger = logger + + def get_source_resp(self, req): + sub_req = make_subrequest( + req.environ, path=req.path_info, headers=req.headers, + swift_source='SSC') + return sub_req.get_response(self.app) + + def send_put_req(self, req, additional_resp_headers, start_response): + app_resp = self._app_call(req.environ) + self._adjust_put_response(req, additional_resp_headers) + start_response(self._response_status, + self._response_headers, + self._response_exc_info) + return app_resp + + def _adjust_put_response(self, req, additional_resp_headers): + if 'swift.post_as_copy' in req.environ: + # Older editions returned 202 Accepted on object POSTs, so we'll + # convert any 201 Created responses to that for compatibility with + # picky clients. + if self._get_status_int() == HTTP_CREATED: + self._response_status = '202 Accepted' + elif is_success(self._get_status_int()): + for header, value in additional_resp_headers.items(): + self._response_headers.append((header, value)) + + def handle_OPTIONS_request(self, req, start_response): + app_resp = self._app_call(req.environ) + if is_success(self._get_status_int()): + for i, (header, value) in enumerate(self._response_headers): + if header.lower() == 'allow' and 'COPY' not in value: + self._response_headers[i] = ('Allow', value + ', COPY') + if header.lower() == 'access-control-allow-methods' and \ + 'COPY' not in value: + self._response_headers[i] = \ + ('Access-Control-Allow-Methods', value + ', COPY') + start_response(self._response_status, + self._response_headers, + self._response_exc_info) + return app_resp + + +class ServerSideCopyMiddleware(object): + + def __init__(self, app, conf): + self.app = app + self.logger = get_logger(conf, log_route="copy") + # Read the old object_post_as_copy option from Proxy app just in case + # someone has set it to false (non default). This wouldn't cause + # problems during upgrade. + self._load_object_post_as_copy_conf(conf) + self.object_post_as_copy = \ + config_true_value(conf.get('object_post_as_copy', 'true')) + + def _load_object_post_as_copy_conf(self, conf): + if ('object_post_as_copy' in conf or '__file__' not in conf): + # Option is explicitly set in middleware conf. In that case, + # we assume operator knows what he's doing. + # This takes preference over the one set in proxy app + return + + cp = ConfigParser() + if os.path.isdir(conf['__file__']): + read_conf_dir(cp, conf['__file__']) + else: + cp.read(conf['__file__']) + + try: + pipe = cp.get("pipeline:main", "pipeline") + except (NoSectionError, NoOptionError): + return + + proxy_name = pipe.rsplit(None, 1)[-1] + proxy_section = "app:" + proxy_name + + try: + conf['object_post_as_copy'] = cp.get(proxy_section, + 'object_post_as_copy') + except (NoSectionError, NoOptionError): + pass + + def __call__(self, env, start_response): + req = Request(env) + try: + (version, account, container, obj) = req.split_path(4, 4, True) + except ValueError: + # If obj component is not present in req, do not proceed further. + return self.app(env, start_response) + + self.account_name = account + self.container_name = container + self.object_name = obj + + # Save off original request method (COPY/POST) in case it gets mutated + # into PUT during handling. This way logging can display the method + # the client actually sent. + req.environ['swift.orig_req_method'] = req.method + + if req.method == 'PUT' and req.headers.get('X-Copy-From'): + return self.handle_PUT(req, start_response) + elif req.method == 'COPY': + return self.handle_COPY(req, start_response) + elif req.method == 'POST' and self.object_post_as_copy: + return self.handle_object_post_as_copy(req, start_response) + elif req.method == 'OPTIONS': + # Does not interfere with OPTIONS response from (account,container) + # servers and /info response. + return self.handle_OPTIONS(req, start_response) + + return self.app(env, start_response) + + def handle_object_post_as_copy(self, req, start_response): + req.method = 'PUT' + req.path_info = '/v1/%s/%s/%s' % ( + self.account_name, self.container_name, self.object_name) + req.headers['Content-Length'] = 0 + req.headers.pop('Range', None) + req.headers['X-Copy-From'] = quote('/%s/%s' % (self.container_name, + self.object_name)) + req.environ['swift.post_as_copy'] = True + params = req.params + # for post-as-copy always copy the manifest itself if source is *LO + params['multipart-manifest'] = 'get' + req.params = params + return self.handle_PUT(req, start_response) + + def handle_COPY(self, req, start_response): + if not req.headers.get('Destination'): + return HTTPPreconditionFailed(request=req, + body='Destination header required' + )(req.environ, start_response) + dest_account = self.account_name + if 'Destination-Account' in req.headers: + dest_account = req.headers.get('Destination-Account') + dest_account = check_account_format(req, dest_account) + req.headers['X-Copy-From-Account'] = self.account_name + self.account_name = dest_account + del req.headers['Destination-Account'] + dest_container, dest_object = _check_destination_header(req) + source = '/%s/%s' % (self.container_name, self.object_name) + self.container_name = dest_container + self.object_name = dest_object + # re-write the existing request as a PUT instead of creating a new one + req.method = 'PUT' + # As this the path info is updated with destination container, + # the proxy server app will use the right object controller + # implementation corresponding to the container's policy type. + ver, _junk = req.split_path(1, 2, rest_with_last=True) + req.path_info = '/%s/%s/%s/%s' % \ + (ver, dest_account, dest_container, dest_object) + req.headers['Content-Length'] = 0 + req.headers['X-Copy-From'] = quote(source) + del req.headers['Destination'] + return self.handle_PUT(req, start_response) + + def _get_source_object(self, ssc_ctx, source_path, req): + source_req = req.copy_get() + + # make sure the source request uses it's container_info + source_req.headers.pop('X-Backend-Storage-Policy-Index', None) + source_req.path_info = quote(source_path) + source_req.headers['X-Newest'] = 'true' + if 'swift.post_as_copy' in req.environ: + # We're COPYing one object over itself because of a POST; rely on + # the PUT for write authorization, don't require read authorization + source_req.environ['swift.authorize'] = lambda req: None + source_req.environ['swift.authorize_override'] = True + + # in case we are copying an SLO manifest, set format=raw parameter + params = source_req.params + if params.get('multipart-manifest') == 'get': + params['format'] = 'raw' + source_req.params = params + + source_resp = ssc_ctx.get_source_resp(source_req) + + if source_resp.content_length is None: + # This indicates a transfer-encoding: chunked source object, + # which currently only happens because there are more than + # CONTAINER_LISTING_LIMIT segments in a segmented object. In + # this case, we're going to refuse to do the server-side copy. + return HTTPRequestEntityTooLarge(request=req) + + if source_resp.content_length > MAX_FILE_SIZE: + return HTTPRequestEntityTooLarge(request=req) + + return source_resp + + def _create_response_headers(self, source_path, source_resp, sink_req): + resp_headers = dict() + acct, path = source_path.split('/', 3)[2:4] + resp_headers['X-Copied-From-Account'] = quote(acct) + resp_headers['X-Copied-From'] = quote(path) + if 'last-modified' in source_resp.headers: + resp_headers['X-Copied-From-Last-Modified'] = \ + source_resp.headers['last-modified'] + # Existing sys and user meta of source object is added to response + # headers in addition to the new ones. + for k, v in sink_req.headers.items(): + if is_sys_or_user_meta('object', k) or k.lower() == 'x-delete-at': + resp_headers[k] = v + return resp_headers + + def handle_PUT(self, req, start_response): + if req.content_length: + return HTTPBadRequest(body='Copy requests require a zero byte ' + 'body', request=req, + content_type='text/plain')(req.environ, + start_response) + + # Form the path of source object to be fetched + ver, acct, _rest = req.split_path(2, 3, True) + src_account_name = req.headers.get('X-Copy-From-Account') + if src_account_name: + src_account_name = check_account_format(req, src_account_name) + else: + src_account_name = acct + src_container_name, src_obj_name = _check_copy_from_header(req) + source_path = '/%s/%s/%s/%s' % (ver, src_account_name, + src_container_name, src_obj_name) + + if req.environ.get('swift.orig_req_method', req.method) != 'POST': + self.logger.info("Copying object from %s to %s" % + (source_path, req.path)) + + # GET the source object, bail out on error + ssc_ctx = ServerSideCopyWebContext(self.app, self.logger) + source_resp = self._get_source_object(ssc_ctx, source_path, req) + if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: + close_if_possible(source_resp.app_iter) + return source_resp(source_resp.environ, start_response) + + # Create a new Request object based on the original req instance. + # This will preserve env and headers. + sink_req = Request.blank(req.path_info, + environ=req.environ, headers=req.headers) + + params = sink_req.params + if params.get('multipart-manifest') == 'get': + if 'X-Static-Large-Object' in source_resp.headers: + params['multipart-manifest'] = 'put' + if 'X-Object-Manifest' in source_resp.headers: + del params['multipart-manifest'] + sink_req.headers['X-Object-Manifest'] = \ + source_resp.headers['X-Object-Manifest'] + sink_req.params = params + + # Set data source, content length and etag for the PUT request + sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) + sink_req.content_length = source_resp.content_length + sink_req.etag = source_resp.etag + + # We no longer need these headers + sink_req.headers.pop('X-Copy-From', None) + sink_req.headers.pop('X-Copy-From-Account', None) + # If the copy request does not explicitly override content-type, + # use the one present in the source object. + if not req.headers.get('content-type'): + sink_req.headers['Content-Type'] = \ + source_resp.headers['Content-Type'] + + fresh_meta_flag = config_true_value( + sink_req.headers.get('x-fresh-metadata', 'false')) + + if fresh_meta_flag or 'swift.post_as_copy' in sink_req.environ: + # Post-as-copy: ignore new sysmeta, copy existing sysmeta + condition = lambda k: is_sys_meta('object', k) + remove_items(sink_req.headers, condition) + copy_header_subset(source_resp, sink_req, condition) + else: + # Copy/update existing sysmeta and user meta + _copy_headers_into(source_resp, sink_req) + # Copy/update new metadata provided in request if any + _copy_headers_into(req, sink_req) + + # Create response headers for PUT response + resp_headers = self._create_response_headers(source_path, + source_resp, sink_req) + + put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response) + close_if_possible(source_resp.app_iter) + return put_resp + + def handle_OPTIONS(self, req, start_response): + return ServerSideCopyWebContext(self.app, self.logger).\ + handle_OPTIONS_request(req, start_response) + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + + def copy_filter(app): + return ServerSideCopyMiddleware(app, conf) + + return copy_filter diff --git a/swift/common/middleware/dlo.py b/swift/common/middleware/dlo.py index 2fd37c3d29..1c27800eb2 100644 --- a/swift/common/middleware/dlo.py +++ b/swift/common/middleware/dlo.py @@ -405,11 +405,6 @@ class DynamicLargeObject(object): except ValueError: return self.app(env, start_response) - # install our COPY-callback hook - env['swift.copy_hook'] = self.copy_hook( - env.get('swift.copy_hook', - lambda src_req, src_resp, sink_req: src_resp)) - if ((req.method == 'GET' or req.method == 'HEAD') and req.params.get('multipart-manifest') != 'get'): return GetContext(self, self.logger).\ @@ -438,24 +433,6 @@ class DynamicLargeObject(object): body=('X-Object-Manifest must be in the ' 'format container/prefix')) - def copy_hook(self, inner_hook): - - def dlo_copy_hook(source_req, source_resp, sink_req): - x_o_m = source_resp.headers.get('X-Object-Manifest') - if x_o_m: - if source_req.params.get('multipart-manifest') == 'get': - # To copy the manifest, we let the copy proceed as normal, - # but ensure that X-Object-Manifest is set on the new - # object. - sink_req.headers['X-Object-Manifest'] = x_o_m - else: - ctx = GetContext(self, self.logger) - source_resp = ctx.get_or_head_response( - source_req, x_o_m, source_resp.headers.items()) - return inner_hook(source_req, source_resp, sink_req) - - return dlo_copy_hook - def filter_factory(global_conf, **local_conf): conf = global_conf.copy() diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index 0216264b99..b87c8f2984 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -798,20 +798,6 @@ class StaticLargeObject(object): """ return SloGetContext(self).handle_slo_get_or_head(req, start_response) - def copy_hook(self, inner_hook): - - def slo_hook(source_req, source_resp, sink_req): - x_slo = source_resp.headers.get('X-Static-Large-Object') - if (config_true_value(x_slo) - and source_req.params.get('multipart-manifest') != 'get' - and 'swift.post_as_copy' not in source_req.environ): - source_resp = SloGetContext(self).get_or_head_response( - source_req, source_resp.headers.items(), - source_resp.app_iter) - return inner_hook(source_req, source_resp, sink_req) - - return slo_hook - def handle_multipart_put(self, req, start_response): """ Will handle the PUT of a SLO manifest. @@ -1058,11 +1044,6 @@ class StaticLargeObject(object): except ValueError: return self.app(env, start_response) - # install our COPY-callback hook - env['swift.copy_hook'] = self.copy_hook( - env.get('swift.copy_hook', - lambda src_req, src_resp, sink_req: src_resp)) - try: if req.method == 'PUT' and \ req.params.get('multipart-manifest') == 'put': diff --git a/swift/common/middleware/versioned_writes.py b/swift/common/middleware/versioned_writes.py index 3cb0989bba..ae091cff20 100644 --- a/swift/common/middleware/versioned_writes.py +++ b/swift/common/middleware/versioned_writes.py @@ -127,9 +127,7 @@ from swift.common.request_helpers import get_sys_meta_prefix, \ from swift.common.wsgi import WSGIContext, make_pre_authed_request from swift.common.swob import ( Request, HTTPException, HTTPRequestEntityTooLarge) -from swift.common.constraints import ( - check_account_format, check_container_format, check_destination_header, - MAX_FILE_SIZE) +from swift.common.constraints import check_container_format, MAX_FILE_SIZE from swift.proxy.controllers.base import get_container_info from swift.common.http import ( is_success, is_client_error, HTTP_NOT_FOUND) @@ -493,24 +491,10 @@ class VersionedWritesMiddleware(object): account_name = unquote(account) container_name = unquote(container) object_name = unquote(obj) - container_info = None resp = None is_enabled = config_true_value(allow_versioned_writes) - if req.method in ('PUT', 'DELETE'): - container_info = get_container_info( - req.environ, self.app) - elif req.method == 'COPY' and 'Destination' in req.headers: - if 'Destination-Account' in req.headers: - account_name = req.headers.get('Destination-Account') - account_name = check_account_format(req, account_name) - container_name, object_name = check_destination_header(req) - req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( - api_version, account_name, container_name, object_name) - container_info = get_container_info( - req.environ, self.app) - - if not container_info: - return self.app + container_info = get_container_info( + req.environ, self.app) # To maintain backwards compatibility, container version # location could be stored as sysmeta or not, need to check both. @@ -530,7 +514,7 @@ class VersionedWritesMiddleware(object): if is_enabled and versions_cont: versions_cont = unquote(versions_cont).split('/')[0] vw_ctx = VersionedWritesContext(self.app, self.logger) - if req.method in ('PUT', 'COPY'): + if req.method == 'PUT': resp = vw_ctx.handle_obj_versions_put( req, versions_cont, api_version, account_name, object_name) @@ -545,10 +529,7 @@ class VersionedWritesMiddleware(object): return self.app def __call__(self, env, start_response): - # making a duplicate, because if this is a COPY request, we will - # modify the PATH_INFO to find out if the 'Destination' is in a - # versioned container - req = Request(env.copy()) + req = Request(env) try: (api_version, account, container, obj) = req.split_path(3, 4, True) except ValueError: @@ -576,7 +557,8 @@ class VersionedWritesMiddleware(object): allow_versioned_writes) except HTTPException as error_response: return error_response(env, start_response) - elif obj and req.method in ('PUT', 'COPY', 'DELETE'): + elif (obj and req.method in ('PUT', 'DELETE') and + not req.environ.get('swift.post_as_copy')): try: return self.object_request( req, api_version, account, container, obj, diff --git a/swift/common/swob.py b/swift/common/swob.py index 0954ef9d3c..f895c44f74 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -888,6 +888,11 @@ class Request(object): return self._params_cache str_params = params + @params.setter + def params(self, param_pairs): + self._params_cache = None + self.query_string = urllib.parse.urlencode(param_pairs) + @property def timestamp(self): """ diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 2c169eb2a6..534333999e 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -1100,7 +1100,7 @@ def make_env(env, method=None, path=None, agent='Swift', query_string=None, 'SERVER_PROTOCOL', 'swift.cache', 'swift.source', 'swift.trans_id', 'swift.authorize_override', 'swift.authorize', 'HTTP_X_USER_ID', 'HTTP_X_PROJECT_ID', - 'HTTP_REFERER'): + 'HTTP_REFERER', 'swift.orig_req_method', 'swift.log_info'): if name in env: newenv[name] = env[name] if method: diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 70400fc143..6f8559063a 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -25,7 +25,7 @@ # collected. We've seen objects hang around forever otherwise. import six -from six.moves.urllib.parse import unquote, quote +from six.moves.urllib.parse import unquote import collections import itertools @@ -49,9 +49,7 @@ from swift.common.utils import ( document_iters_to_http_response_body, parse_content_range, quorum_size, reiterate, close_if_possible) from swift.common.bufferedhttp import http_connect -from swift.common.constraints import check_metadata, check_object_creation, \ - check_copy_from_header, check_destination_header, \ - check_account_format +from swift.common.constraints import check_metadata, check_object_creation from swift.common import constraints from swift.common.exceptions import ChunkReadTimeout, \ ChunkWriteTimeout, ConnectionTimeout, ResponseTimeout, \ @@ -60,33 +58,19 @@ from swift.common.exceptions import ChunkReadTimeout, \ from swift.common.header_key_dict import HeaderKeyDict from swift.common.http import ( is_informational, is_success, is_client_error, is_server_error, - is_redirection, HTTP_CONTINUE, HTTP_CREATED, HTTP_MULTIPLE_CHOICES, - HTTP_INTERNAL_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE, - HTTP_INSUFFICIENT_STORAGE, HTTP_PRECONDITION_FAILED, HTTP_CONFLICT, - HTTP_UNPROCESSABLE_ENTITY, HTTP_REQUESTED_RANGE_NOT_SATISFIABLE) + is_redirection, HTTP_CONTINUE, HTTP_INTERNAL_SERVER_ERROR, + HTTP_SERVICE_UNAVAILABLE, HTTP_INSUFFICIENT_STORAGE, + HTTP_PRECONDITION_FAILED, HTTP_CONFLICT, HTTP_UNPROCESSABLE_ENTITY, + HTTP_REQUESTED_RANGE_NOT_SATISFIABLE) from swift.common.storage_policy import (POLICIES, REPL_POLICY, EC_POLICY, ECDriverError, PolicyError) from swift.proxy.controllers.base import Controller, delay_denial, \ cors_validation, ResumingGetter from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \ - HTTPServerError, HTTPServiceUnavailable, Request, \ - HTTPClientDisconnect, HTTPUnprocessableEntity, Response, HTTPException, \ + HTTPServerError, HTTPServiceUnavailable, HTTPClientDisconnect, \ + HTTPUnprocessableEntity, Response, HTTPException, \ HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError -from swift.common.request_helpers import is_sys_or_user_meta, is_sys_meta, \ - remove_items, copy_header_subset - - -def copy_headers_into(from_r, to_r): - """ - Will copy desired headers from from_r to to_r - :params from_r: a swob Request or Response - :params to_r: a swob Request or Response - """ - pass_headers = ['x-delete-at'] - for k, v in from_r.headers.items(): - if is_sys_or_user_meta('object', k) or k.lower() in pass_headers: - to_r.headers[k] = v def check_content_type(req): @@ -200,8 +184,7 @@ class BaseObjectController(Controller): self.account_name, self.container_name, self.object_name) node_iter = self.app.iter_nodes(obj_ring, partition) - resp = self._reroute(policy)._get_or_head_response( - req, node_iter, partition, policy) + resp = self._get_or_head_response(req, node_iter, partition, policy) if ';' in resp.headers.get('content-type', ''): resp.content_type = clean_content_type( @@ -227,55 +210,38 @@ class BaseObjectController(Controller): @delay_denial def POST(self, req): """HTTP POST request handler.""" - if self.app.object_post_as_copy: - req.method = 'PUT' - req.path_info = '/v1/%s/%s/%s' % ( - self.account_name, self.container_name, self.object_name) - req.headers['Content-Length'] = 0 - req.headers['X-Copy-From'] = quote('/%s/%s' % (self.container_name, - self.object_name)) - req.environ['swift.post_as_copy'] = True - req.environ['swift_versioned_copy'] = True - resp = self.PUT(req) - # Older editions returned 202 Accepted on object POSTs, so we'll - # convert any 201 Created responses to that for compatibility with - # picky clients. - if resp.status_int != HTTP_CREATED: - return resp - return HTTPAccepted(request=req) - else: - error_response = check_metadata(req, 'object') - if error_response: - return error_response - container_info = self.container_info( - self.account_name, self.container_name, req) - container_partition = container_info['partition'] - containers = container_info['nodes'] - req.acl = container_info['write_acl'] - if 'swift.authorize' in req.environ: - aresp = req.environ['swift.authorize'](req) - if aresp: - return aresp - if not containers: - return HTTPNotFound(request=req) + error_response = check_metadata(req, 'object') + if error_response: + return error_response + container_info = self.container_info( + self.account_name, self.container_name, req) + container_partition = container_info['partition'] + containers = container_info['nodes'] + req.acl = container_info['write_acl'] + if 'swift.authorize' in req.environ: + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp + if not containers: + return HTTPNotFound(request=req) - req, delete_at_container, delete_at_part, \ - delete_at_nodes = self._config_obj_expiration(req) + req, delete_at_container, delete_at_part, \ + delete_at_nodes = self._config_obj_expiration(req) - # pass the policy index to storage nodes via req header - policy_index = req.headers.get('X-Backend-Storage-Policy-Index', - container_info['storage_policy']) - obj_ring = self.app.get_object_ring(policy_index) - req.headers['X-Backend-Storage-Policy-Index'] = policy_index - partition, nodes = obj_ring.get_nodes( - self.account_name, self.container_name, self.object_name) + # pass the policy index to storage nodes via req header + policy_index = req.headers.get('X-Backend-Storage-Policy-Index', + container_info['storage_policy']) + obj_ring = self.app.get_object_ring(policy_index) + req.headers['X-Backend-Storage-Policy-Index'] = policy_index + partition, nodes = obj_ring.get_nodes( + self.account_name, self.container_name, self.object_name) - req.headers['X-Timestamp'] = Timestamp(time.time()).internal + req.headers['X-Timestamp'] = Timestamp(time.time()).internal - headers = self._backend_requests( - req, len(nodes), container_partition, containers, - delete_at_container, delete_at_part, delete_at_nodes) - return self._post_object(req, obj_ring, partition, headers) + headers = self._backend_requests( + req, len(nodes), container_partition, containers, + delete_at_container, delete_at_part, delete_at_nodes) + return self._post_object(req, obj_ring, partition, headers) def _backend_requests(self, req, n_outgoing, container_partition, containers, @@ -414,133 +380,8 @@ class BaseObjectController(Controller): return req, delete_at_container, delete_at_part, delete_at_nodes - def _handle_copy_request(self, req): - """ - This method handles copying objects based on values set in the headers - 'X-Copy-From' and 'X-Copy-From-Account' - - Note that if the incomming request has some conditional headers (e.g. - 'Range', 'If-Match'), *source* object will be evaluated for these - headers. i.e. if PUT with both 'X-Copy-From' and 'Range', Swift will - make a partial copy as a new object. - - This method was added as part of the refactoring of the PUT method and - the functionality is expected to be moved to middleware - """ - if req.environ.get('swift.orig_req_method', req.method) != 'POST': - req.environ.setdefault('swift.log_info', []).append( - 'x-copy-from:%s' % req.headers['X-Copy-From']) - ver, acct, _rest = req.split_path(2, 3, True) - src_account_name = req.headers.get('X-Copy-From-Account', None) - if src_account_name: - src_account_name = check_account_format(req, src_account_name) - else: - src_account_name = acct - src_container_name, src_obj_name = check_copy_from_header(req) - source_header = '/%s/%s/%s/%s' % ( - ver, src_account_name, src_container_name, src_obj_name) - source_req = req.copy_get() - - # make sure the source request uses it's container_info - source_req.headers.pop('X-Backend-Storage-Policy-Index', None) - source_req.path_info = source_header - source_req.headers['X-Newest'] = 'true' - if 'swift.post_as_copy' in req.environ: - # We're COPYing one object over itself because of a POST; rely on - # the PUT for write authorization, don't require read authorization - source_req.environ['swift.authorize'] = lambda req: None - source_req.environ['swift.authorize_override'] = True - - orig_obj_name = self.object_name - orig_container_name = self.container_name - orig_account_name = self.account_name - sink_req = Request.blank(req.path_info, - environ=req.environ, headers=req.headers) - - self.object_name = src_obj_name - self.container_name = src_container_name - self.account_name = src_account_name - - source_resp = self.GET(source_req) - - # This gives middlewares a way to change the source; for example, - # this lets you COPY a SLO manifest and have the new object be the - # concatenation of the segments (like what a GET request gives - # the client), not a copy of the manifest file. - hook = req.environ.get( - 'swift.copy_hook', - (lambda source_req, source_resp, sink_req: source_resp)) - source_resp = hook(source_req, source_resp, sink_req) - - # reset names - self.object_name = orig_obj_name - self.container_name = orig_container_name - self.account_name = orig_account_name - - if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: - # this is a bit of ugly code, but I'm willing to live with it - # until copy request handling moves to middleware - return source_resp, None, None, None - if source_resp.content_length is None: - # This indicates a transfer-encoding: chunked source object, - # which currently only happens because there are more than - # CONTAINER_LISTING_LIMIT segments in a segmented object. In - # this case, we're going to refuse to do the server-side copy. - raise HTTPRequestEntityTooLarge(request=req) - if source_resp.content_length > constraints.MAX_FILE_SIZE: - raise HTTPRequestEntityTooLarge(request=req) - - data_source = iter(source_resp.app_iter) - sink_req.content_length = source_resp.content_length - sink_req.etag = source_resp.etag - - # we no longer need the X-Copy-From header - del sink_req.headers['X-Copy-From'] - if 'X-Copy-From-Account' in sink_req.headers: - del sink_req.headers['X-Copy-From-Account'] - if not req.content_type_manually_set: - sink_req.headers['Content-Type'] = \ - source_resp.headers['Content-Type'] - - fresh_meta_flag = config_true_value( - sink_req.headers.get('x-fresh-metadata', 'false')) - - if fresh_meta_flag or 'swift.post_as_copy' in sink_req.environ: - # post-as-copy: ignore new sysmeta, copy existing sysmeta - condition = lambda k: is_sys_meta('object', k) - remove_items(sink_req.headers, condition) - copy_header_subset(source_resp, sink_req, condition) - else: - # copy/update existing sysmeta and user meta - copy_headers_into(source_resp, sink_req) - copy_headers_into(req, sink_req) - - # copy over x-static-large-object for POSTs and manifest copies - if 'X-Static-Large-Object' in source_resp.headers and \ - (req.params.get('multipart-manifest') == 'get' or - 'swift.post_as_copy' in req.environ): - sink_req.headers['X-Static-Large-Object'] = \ - source_resp.headers['X-Static-Large-Object'] - - req = sink_req - - def update_response(req, resp): - acct, path = source_resp.environ['PATH_INFO'].split('/', 3)[2:4] - resp.headers['X-Copied-From-Account'] = quote(acct) - resp.headers['X-Copied-From'] = quote(path) - if 'last-modified' in source_resp.headers: - resp.headers['X-Copied-From-Last-Modified'] = \ - source_resp.headers['last-modified'] - copy_headers_into(req, resp) - return resp - - # this is a bit of ugly code, but I'm willing to live with it - # until copy request handling moves to middleware - return None, req, data_source, update_response - def _update_content_type(self, req): # Sometimes the 'content-type' header exists, but is set to None. - req.content_type_manually_set = True detect_content_type = \ config_true_value(req.headers.get('x-detect-content-type')) if detect_content_type or not req.headers.get('content-type'): @@ -549,8 +390,6 @@ class BaseObjectController(Controller): 'application/octet-stream' if detect_content_type: req.headers.pop('x-detect-content-type') - else: - req.content_type_manually_set = False def _update_x_timestamp(self, req): # Used by container sync feature @@ -744,22 +583,13 @@ class BaseObjectController(Controller): self._update_x_timestamp(req) - # check if request is a COPY of an existing object - source_header = req.headers.get('X-Copy-From') - if source_header: - error_response, req, data_source, update_response = \ - self._handle_copy_request(req) - if error_response: - return error_response - else: - def reader(): - try: - return req.environ['wsgi.input'].read( - self.app.client_chunk_size) - except (ValueError, IOError) as e: - raise ChunkReadError(str(e)) - data_source = iter(reader, '') - update_response = lambda req, resp: resp + def reader(): + try: + return req.environ['wsgi.input'].read( + self.app.client_chunk_size) + except (ValueError, IOError) as e: + raise ChunkReadError(str(e)) + data_source = iter(reader, '') # check if object is set to be automatically deleted (i.e. expired) req, delete_at_container, delete_at_part, \ @@ -773,7 +603,7 @@ class BaseObjectController(Controller): # send object to storage nodes resp = self._store_object( req, data_source, nodes, partition, outgoing_headers) - return update_response(req, resp) + return resp @public @cors_validation @@ -817,63 +647,6 @@ class BaseObjectController(Controller): req, len(nodes), container_partition, containers) return self._delete_object(req, obj_ring, partition, headers) - def _reroute(self, policy): - """ - For COPY requests we need to make sure the controller instance the - request is routed through is the correct type for the policy. - """ - if not policy: - raise HTTPServiceUnavailable('Unknown Storage Policy') - if policy.policy_type != self.policy_type: - controller = self.app.obj_controller_router[policy]( - self.app, self.account_name, self.container_name, - self.object_name) - else: - controller = self - return controller - - @public - @cors_validation - @delay_denial - def COPY(self, req): - """HTTP COPY request handler.""" - if not req.headers.get('Destination'): - return HTTPPreconditionFailed(request=req, - body='Destination header required') - dest_account = self.account_name - if 'Destination-Account' in req.headers: - dest_account = req.headers.get('Destination-Account') - dest_account = check_account_format(req, dest_account) - req.headers['X-Copy-From-Account'] = self.account_name - self.account_name = dest_account - del req.headers['Destination-Account'] - dest_container, dest_object = check_destination_header(req) - - source = '/%s/%s' % (self.container_name, self.object_name) - self.container_name = dest_container - self.object_name = dest_object - # re-write the existing request as a PUT instead of creating a new one - # since this one is already attached to the posthooklogger - # TODO: Swift now has proxy-logging middleware instead of - # posthooklogger used in before. i.e. we don't have to - # keep the code depends on evnetlet.posthooks sequence, IMHO. - # However, creating a new sub request might - # cause the possibility to hide some bugs behindes the request - # so that we should discuss whichi is suitable (new-sub-request - # vs re-write-existing-request) for Swift. [kota_] - req.method = 'PUT' - req.path_info = '/v1/%s/%s/%s' % \ - (dest_account, dest_container, dest_object) - req.headers['Content-Length'] = 0 - req.headers['X-Copy-From'] = quote(source) - del req.headers['Destination'] - - container_info = self.container_info( - dest_account, dest_container, req) - dest_policy = POLICIES.get_by_index(container_info['storage_policy']) - - return self._reroute(dest_policy).PUT(req) - @ObjectControllerRouter.register(REPL_POLICY) class ReplicatedObjectController(BaseObjectController): diff --git a/swift/proxy/server.py b/swift/proxy/server.py index f8f4296a25..963bf34f0e 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -64,10 +64,14 @@ required_filters = [ if pipe.startswith('catch_errors') else [])}, {'name': 'dlo', 'after_fn': lambda _junk: [ - 'staticweb', 'tempauth', 'keystoneauth', + 'copy', 'staticweb', 'tempauth', 'keystoneauth', 'catch_errors', 'gatekeeper', 'proxy_logging']}, {'name': 'versioned_writes', 'after_fn': lambda _junk: [ - 'slo', 'dlo', 'staticweb', 'tempauth', 'keystoneauth', + 'slo', 'dlo', 'copy', 'staticweb', 'tempauth', + 'keystoneauth', 'catch_errors', 'gatekeeper', 'proxy_logging']}, + # Put copy before dlo, slo and versioned_writes + {'name': 'copy', 'after_fn': lambda _junk: [ + 'staticweb', 'tempauth', 'keystoneauth', 'catch_errors', 'gatekeeper', 'proxy_logging']}] @@ -107,8 +111,6 @@ class Application(object): int(conf.get('recheck_account_existence', 60)) self.allow_account_management = \ config_true_value(conf.get('allow_account_management', 'no')) - self.object_post_as_copy = \ - config_true_value(conf.get('object_post_as_copy', 'true')) self.container_ring = container_ring or Ring(swift_dir, ring_name='container') self.account_ring = account_ring or Ring(swift_dir, @@ -392,8 +394,7 @@ class Application(object): # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ['swift.authorize'](req) - if not resp and not req.headers.get('X-Copy-From-Account') \ - and not req.headers.get('Destination-Account'): + if not resp: # No resp means authorized, no delayed recheck required. old_authorize = req.environ['swift.authorize'] else: @@ -404,7 +405,7 @@ class Application(object): # Save off original request method (GET, POST, etc.) in case it # gets mutated during handling. This way logging can display the # method the client actually sent. - req.environ['swift.orig_req_method'] = req.method + req.environ.setdefault('swift.orig_req_method', req.method) try: if old_authorize: req.environ.pop('swift.authorize', None) diff --git a/test/functional/tests.py b/test/functional/tests.py index fc9e362f2a..e35e79706d 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -1306,12 +1306,10 @@ class TestFile(Base): acct, '%s%s' % (prefix, self.env.container), Utils.create_name())) - if acct == acct2: - # there is no such source container - # and foreign user can have no permission to read it - self.assert_status(403) - else: - self.assert_status(404) + # there is no such source container but user has + # permissions to do a GET (done internally via COPY) for + # objects in his own account. + self.assert_status(404) self.assertFalse(file_item.copy_account( acct, @@ -1325,12 +1323,10 @@ class TestFile(Base): acct, '%s%s' % (prefix, self.env.container), Utils.create_name())) - if acct == acct2: - # there is no such object - # and foreign user can have no permission to read it - self.assert_status(403) - else: - self.assert_status(404) + # there is no such source container but user has + # permissions to do a GET (done internally via COPY) for + # objects in his own account. + self.assert_status(404) self.assertFalse(file_item.copy_account( acct, @@ -2677,6 +2673,23 @@ class TestFileComparisonUTF8(Base2, TestFileComparison): class TestSloEnv(object): slo_enabled = None # tri-state: None initially, then True/False + @classmethod + def create_segments(cls, container): + seg_info = {} + for letter, size in (('a', 1024 * 1024), + ('b', 1024 * 1024), + ('c', 1024 * 1024), + ('d', 1024 * 1024), + ('e', 1)): + seg_name = "seg_%s" % letter + file_item = container.file(seg_name) + file_item.write(letter * size) + seg_info[seg_name] = { + 'size_bytes': size, + 'etag': file_item.md5, + 'path': '/%s/%s' % (container.name, seg_name)} + return seg_info + @classmethod def setUp(cls): cls.conn = Connection(tf.config) @@ -2711,19 +2724,7 @@ class TestSloEnv(object): if not cont.create(): raise ResponseError(cls.conn.response) - cls.seg_info = seg_info = {} - for letter, size in (('a', 1024 * 1024), - ('b', 1024 * 1024), - ('c', 1024 * 1024), - ('d', 1024 * 1024), - ('e', 1)): - seg_name = "seg_%s" % letter - file_item = cls.container.file(seg_name) - file_item.write(letter * size) - seg_info[seg_name] = { - 'size_bytes': size, - 'etag': file_item.md5, - 'path': '/%s/%s' % (cls.container.name, seg_name)} + cls.seg_info = seg_info = cls.create_segments(cls.container) file_item = cls.container.file("manifest-abcde") file_item.write( @@ -3125,8 +3126,9 @@ class TestSlo(Base): def test_slo_copy_the_manifest(self): file_item = self.env.container.file("manifest-abcde") - file_item.copy(self.env.container.name, "copied-abcde-manifest-only", - parms={'multipart-manifest': 'get'}) + self.assertTrue(file_item.copy(self.env.container.name, + "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'})) copied = self.env.container.file("copied-abcde-manifest-only") copied_contents = copied.read(parms={'multipart-manifest': 'get'}) @@ -3157,10 +3159,40 @@ class TestSlo(Base): self.assertTrue(dest_cont.create(hdrs={ 'X-Container-Write': self.env.conn.user_acl })) - file_item.copy_account(acct, - dest_cont, - "copied-abcde-manifest-only", - parms={'multipart-manifest': 'get'}) + + # manifest copy will fail because there is no read access to segments + # in destination account + file_item.copy_account( + acct, dest_cont, "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'}) + self.assertEqual(400, file_item.conn.response.status) + resp_body = file_item.conn.response.read() + self.assertEqual(5, resp_body.count('403 Forbidden'), + 'Unexpected response body %r' % resp_body) + + # create segments container in account2 with read access for account1 + segs_container = self.env.account2.container(self.env.container.name) + self.assertTrue(segs_container.create(hdrs={ + 'X-Container-Read': self.env.conn.user_acl + })) + + # manifest copy will still fail because there are no segments in + # destination account + file_item.copy_account( + acct, dest_cont, "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'}) + self.assertEqual(400, file_item.conn.response.status) + resp_body = file_item.conn.response.read() + self.assertEqual(5, resp_body.count('404 Not Found'), + 'Unexpected response body %r' % resp_body) + + # create segments in account2 container with same name as in account1, + # manifest copy now succeeds + self.env.create_segments(segs_container) + + self.assertTrue(file_item.copy_account( + acct, dest_cont, "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'})) copied = dest_cont.file("copied-abcde-manifest-only") copied_contents = copied.read(parms={'multipart-manifest': 'get'}) diff --git a/test/unit/common/middleware/helpers.py b/test/unit/common/middleware/helpers.py index 2432d0dc37..bcd3c4c2ec 100644 --- a/test/unit/common/middleware/helpers.py +++ b/test/unit/common/middleware/helpers.py @@ -20,6 +20,7 @@ from copy import deepcopy from hashlib import md5 from swift.common import swob from swift.common.header_key_dict import HeaderKeyDict +from swift.common.swob import HTTPNotImplemented from swift.common.utils import split_path from test.unit import FakeLogger, FakeRing @@ -43,6 +44,8 @@ class FakeSwift(object): """ A good-enough fake Swift proxy server to use in testing middleware. """ + ALLOWED_METHODS = [ + 'PUT', 'POST', 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'REPLICATE'] def __init__(self): self._calls = [] @@ -71,6 +74,9 @@ class FakeSwift(object): def __call__(self, env, start_response): method = env['REQUEST_METHOD'] + if method not in self.ALLOWED_METHODS: + raise HTTPNotImplemented() + path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) diff --git a/test/unit/common/middleware/test_account_quotas.py b/test/unit/common/middleware/test_account_quotas.py index 345e178cd1..b443b4a28d 100644 --- a/test/unit/common/middleware/test_account_quotas.py +++ b/test/unit/common/middleware/test_account_quotas.py @@ -13,9 +13,10 @@ import unittest -from swift.common.swob import Request, wsgify, HTTPForbidden +from swift.common.swob import Request, wsgify, HTTPForbidden, \ + HTTPException -from swift.common.middleware import account_quotas +from swift.common.middleware import account_quotas, copy from swift.proxy.controllers.base import _get_cache_key, \ headers_to_account_info, get_object_env_key, \ @@ -245,84 +246,6 @@ class TestAccountQuota(unittest.TestCase): res = req.get_response(app) self.assertEqual(res.status_int, 200) - def test_exceed_bytes_quota_copy_from(self): - headers = [('x-account-bytes-used', '500'), - ('x-account-meta-quota-bytes', '1000'), - ('content-length', '1000')] - app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) - cache = FakeCache(None) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - - def test_exceed_bytes_quota_copy_verb(self): - headers = [('x-account-bytes-used', '500'), - ('x-account-meta-quota-bytes', '1000'), - ('content-length', '1000')] - app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) - cache = FakeCache(None) - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - - def test_not_exceed_bytes_quota_copy_from(self): - headers = [('x-account-bytes-used', '0'), - ('x-account-meta-quota-bytes', '1000'), - ('content-length', '1000')] - app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) - cache = FakeCache(None) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_not_exceed_bytes_quota_copy_verb(self): - headers = [('x-account-bytes-used', '0'), - ('x-account-meta-quota-bytes', '1000'), - ('content-length', '1000')] - app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) - cache = FakeCache(None) - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_quota_copy_from_no_src(self): - headers = [('x-account-bytes-used', '0'), - ('x-account-meta-quota-bytes', '1000')] - app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) - cache = FakeCache(None) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o3'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_quota_copy_from_bad_src(self): - headers = [('x-account-bytes-used', '0'), - ('x-account-meta-quota-bytes', '1000')] - app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) - cache = FakeCache(None) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.cache': cache}, - headers={'x-copy-from': 'bad_path'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 412) - def test_exceed_bytes_quota_reseller(self): headers = [('x-account-bytes-used', '1000'), ('x-account-meta-quota-bytes', '0')] @@ -485,5 +408,91 @@ class TestAccountQuota(unittest.TestCase): self.assertEqual(res.status_int, 200) +class AccountQuotaCopyingTestCases(unittest.TestCase): + + def setUp(self): + self.app = FakeApp() + self.aq_filter = account_quotas.filter_factory({})(self.app) + self.copy_filter = copy.filter_factory({})(self.aq_filter) + + def test_exceed_bytes_quota_copy_from(self): + headers = [('x-account-bytes-used', '500'), + ('x-account-meta-quota-bytes', '1000'), + ('content-length', '1000')] + self.app.headers = headers + cache = FakeCache(None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_exceed_bytes_quota_copy_verb(self): + headers = [('x-account-bytes-used', '500'), + ('x-account-meta-quota-bytes', '1000'), + ('content-length', '1000')] + self.app.headers = headers + cache = FakeCache(None) + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.cache': cache}, + headers={'Destination': '/c/o'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_not_exceed_bytes_quota_copy_from(self): + headers = [('x-account-bytes-used', '0'), + ('x-account-meta-quota-bytes', '1000'), + ('content-length', '1000')] + self.app.headers = headers + cache = FakeCache(None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_not_exceed_bytes_quota_copy_verb(self): + headers = [('x-account-bytes-used', '0'), + ('x-account-meta-quota-bytes', '1000'), + ('content-length', '1000')] + self.app.headers = headers + cache = FakeCache(None) + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.cache': cache}, + headers={'Destination': '/c/o'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_quota_copy_from_no_src(self): + headers = [('x-account-bytes-used', '0'), + ('x-account-meta-quota-bytes', '1000')] + self.app.headers = headers + cache = FakeCache(None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o3'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_quota_copy_from_bad_src(self): + headers = [('x-account-bytes-used', '0'), + ('x-account-meta-quota-bytes', '1000')] + self.app.headers = headers + cache = FakeCache(None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': 'bad_path'}) + with self.assertRaises(HTTPException) as catcher: + req.get_response(self.copy_filter) + self.assertEqual(412, catcher.exception.status_int) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/test_copy.py b/test/unit/common/middleware/test_copy.py new file mode 100644 index 0000000000..190d7c9084 --- /dev/null +++ b/test/unit/common/middleware/test_copy.py @@ -0,0 +1,1183 @@ +#!/usr/bin/env python +# Copyright (c) 2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +import mock +import shutil +import tempfile +import unittest +from hashlib import md5 +from textwrap import dedent + +from swift.common import swob +from swift.common.middleware import copy +from swift.common.storage_policy import POLICIES +from swift.common.swob import Request, HTTPException +from test.unit import patch_policies, debug_logger, FakeMemcache, FakeRing +from test.unit.common.middleware.helpers import FakeSwift +from test.unit.proxy.controllers.test_obj import set_http_connect, \ + PatchedObjControllerApp + + +class TestCopyConstraints(unittest.TestCase): + def test_validate_copy_from(self): + req = Request.blank( + '/v/a/c/o', + headers={'x-copy-from': 'c/o2'}) + src_cont, src_obj = copy._check_copy_from_header(req) + self.assertEqual(src_cont, 'c') + self.assertEqual(src_obj, 'o2') + req = Request.blank( + '/v/a/c/o', + headers={'x-copy-from': 'c/subdir/o2'}) + src_cont, src_obj = copy._check_copy_from_header(req) + self.assertEqual(src_cont, 'c') + self.assertEqual(src_obj, 'subdir/o2') + req = Request.blank( + '/v/a/c/o', + headers={'x-copy-from': '/c/o2'}) + src_cont, src_obj = copy._check_copy_from_header(req) + self.assertEqual(src_cont, 'c') + self.assertEqual(src_obj, 'o2') + + def test_validate_bad_copy_from(self): + req = Request.blank( + '/v/a/c/o', + headers={'x-copy-from': 'bad_object'}) + self.assertRaises(HTTPException, + copy._check_copy_from_header, req) + + def test_validate_destination(self): + req = Request.blank( + '/v/a/c/o', + headers={'destination': 'c/o2'}) + src_cont, src_obj = copy._check_destination_header(req) + self.assertEqual(src_cont, 'c') + self.assertEqual(src_obj, 'o2') + req = Request.blank( + '/v/a/c/o', + headers={'destination': 'c/subdir/o2'}) + src_cont, src_obj = copy._check_destination_header(req) + self.assertEqual(src_cont, 'c') + self.assertEqual(src_obj, 'subdir/o2') + req = Request.blank( + '/v/a/c/o', + headers={'destination': '/c/o2'}) + src_cont, src_obj = copy._check_destination_header(req) + self.assertEqual(src_cont, 'c') + self.assertEqual(src_obj, 'o2') + + def test_validate_bad_destination(self): + req = Request.blank( + '/v/a/c/o', + headers={'destination': 'bad_object'}) + self.assertRaises(HTTPException, + copy._check_destination_header, req) + + +class TestServerSideCopyMiddleware(unittest.TestCase): + def setUp(self): + self.app = FakeSwift() + self.ssc = copy.filter_factory({ + 'object_post_as_copy': 'yes', + })(self.app) + self.ssc.logger = self.app.logger + + def call_app(self, req, app=None, expect_exception=False): + if app is None: + app = self.app + + self.authorized = [] + + def authorize(req): + self.authorized.append(req) + + if 'swift.authorize' not in req.environ: + req.environ['swift.authorize'] = authorize + + req.headers.setdefault("User-Agent", "Bruce Wayne") + + status = [None] + headers = [None] + + def start_response(s, h, ei=None): + status[0] = s + headers[0] = h + + body_iter = app(req.environ, start_response) + body = '' + caught_exc = None + try: + for chunk in body_iter: + body += chunk + except Exception as exc: + if expect_exception: + caught_exc = exc + else: + raise + + if expect_exception: + return status[0], headers[0], body, caught_exc + else: + return status[0], headers[0], body + + def call_ssc(self, req, **kwargs): + return self.call_app(req, app=self.ssc, **kwargs) + + def assertRequestEqual(self, req, other): + self.assertEqual(req.method, other.method) + self.assertEqual(req.path, other.path) + + def test_no_object_in_path_pass_through(self): + self.app.register('PUT', '/v1/a/c', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c', method='PUT') + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_object_delete_pass_through(self): + self.app.register('DELETE', '/v1/a/c/o', swob.HTTPOk, {}) + req = Request.blank('/v1/a/c/o', method='DELETE') + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_POST_as_COPY_simple(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPAccepted, {}) + req = Request.blank('/v1/a/c/o', method='POST') + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '202 Accepted') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_POST_as_COPY_201_return_202(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', method='POST') + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '202 Accepted') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_POST_delete_at(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPAccepted, {}) + t = str(int(time.time() + 100)) + req = Request.blank('/v1/a/c/o', method='POST', + headers={'Content-Type': 'foo/bar', + 'X-Delete-At': t}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '202 Accepted') + + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertTrue('X-Delete-At' in req_headers) + self.assertEqual(req_headers['X-Delete-At'], str(t)) + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_POST_as_COPY_static_large_object(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, + {'X-Static-Large-Object': True}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPAccepted, {}) + req = Request.blank('/v1/a/c/o', method='POST', + headers={}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '202 Accepted') + + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertNotIn('X-Static-Large-Object', req_headers) + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + + def test_basic_put_with_x_copy_from(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o2', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o2', self.authorized[1].path) + + def test_static_large_object(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, + {'X-Static-Large-Object': 'True'}, 'passed') + self.app.register('PUT', '/v1/a/c/o2?multipart-manifest=put', + swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o2?multipart-manifest=get', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o2?multipart-manifest=put', path) + self.assertNotIn('X-Static-Large-Object', req_headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o2', self.authorized[1].path) + + def test_basic_put_with_x_copy_from_across_container(self): + self.app.register('GET', '/v1/a/c1/o1', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c2/o2', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c1/o1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c1/o1') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c1/o1', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c2/o2', self.authorized[1].path) + + def test_basic_put_with_x_copy_from_across_container_and_account(self): + self.app.register('GET', '/v1/a1/c1/o1', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a2/c2/o2', swob.HTTPCreated, {}, + 'passed') + req = Request.blank('/v1/a2/c2/o2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c1/o1', + 'X-Copy-From-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c1/o1') in headers) + self.assertTrue(('X-Copied-From-Account', 'a1') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a1/c1/o1', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a2/c2/o2', self.authorized[1].path) + + def test_copy_non_zero_content_length(self): + req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '10', + 'X-Copy-From': 'c1/o1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '400 Bad Request') + + def test_copy_non_zero_content_length_with_account(self): + req = Request.blank('/v1/a2/c2/o2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '10', + 'X-Copy-From': 'c1/o1', + 'X-Copy-From-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '400 Bad Request') + + def test_copy_with_slashes_in_x_copy_from(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o/o2'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_copy_with_slashes_in_x_copy_from_and_account(self): + self.app.register('GET', '/v1/a1/c1/o/o1', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a2/c2/o2', swob.HTTPCreated, {}) + req = Request.blank('/v1/a2/c2/o2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c1/o/o1', + 'X-Copy-From-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c1/o/o1') in headers) + self.assertTrue(('X-Copied-From-Account', 'a1') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a1/c1/o/o1', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a2/c2/o2', self.authorized[1].path) + + def test_copy_with_spaces_in_x_copy_from(self): + self.app.register('GET', '/v1/a/c/o o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + # space in soure path + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o%20o2'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o o2', path) + self.assertTrue(('X-Copied-From', 'c/o%20o2') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o%20o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_copy_with_spaces_in_x_copy_from_and_account(self): + self.app.register('GET', '/v1/a/c/o o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + # space in soure path + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o%20o2', + 'X-Copy-From-Account': 'a'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o o2', path) + self.assertTrue(('X-Copied-From', 'c/o%20o2') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o%20o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_copy_with_leading_slash_in_x_copy_from(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + # repeat tests with leading / + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o', path) + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_copy_with_leading_slash_in_x_copy_from_and_account(self): + # repeat tests with leading / + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Copy-From-Account': 'a'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o', path) + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_copy_with_leading_slash_and_slashes_in_x_copy_from(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o/o2'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o/o2', path) + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_copy_with_leading_slash_and_slashes_in_x_copy_from_acct(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o/o2', + 'X-Copy-From-Account': 'a'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o/o2', path) + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_copy_with_no_object_in_x_copy_from(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c'}) + try: + status, headers, body = self.call_ssc(req) + except HTTPException as resp: + self.assertEqual("412 Precondition Failed", str(resp)) + else: + self.fail("Expecting HTTPException.") + + def test_copy_with_no_object_in_x_copy_from_and_account(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c', + 'X-Copy-From-Account': 'a'}) + try: + status, headers, body = self.call_ssc(req) + except HTTPException as resp: + self.assertEqual("412 Precondition Failed", str(resp)) + else: + self.fail("Expecting HTTPException.") + + def test_copy_with_bad_x_copy_from_account(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Copy-From-Account': '/i/am/bad'}) + try: + status, headers, body = self.call_ssc(req) + except HTTPException as resp: + self.assertEqual("412 Precondition Failed", str(resp)) + else: + self.fail("Expecting HTTPException.") + + def test_copy_server_error_reading_source(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPServiceUnavailable, {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '503 Service Unavailable') + + def test_copy_server_error_reading_source_and_account(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPServiceUnavailable, {}) + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Copy-From-Account': 'a'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '503 Service Unavailable') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_copy_not_found_reading_source(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPNotFound, {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '404 Not Found') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_copy_not_found_reading_source_and_account(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPNotFound, {}) + req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Copy-From-Account': 'a'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '404 Not Found') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_copy_with_object_metadata(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Object-Meta-Ours': 'okay'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o', path) + self.assertEqual(req_headers['X-Object-Meta-Ours'], 'okay') + self.assertTrue(('X-Object-Meta-Ours', 'okay') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_copy_with_object_metadata_and_account(self): + self.app.register('GET', '/v1/a1/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Object-Meta-Ours': 'okay', + 'X-Copy-From-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o', path) + self.assertEqual(req_headers['X-Object-Meta-Ours'], 'okay') + self.assertTrue(('X-Object-Meta-Ours', 'okay') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a1/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_copy_source_larger_than_max_file_size(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, "largebody") + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + with mock.patch('swift.common.middleware.copy.' + 'MAX_FILE_SIZE', 1): + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '413 Request Entity Too Large') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_basic_COPY(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o-copy', swob.HTTPCreated, {}) + req = Request.blank( + '/v1/a/c/o', method='COPY', + headers={'Content-Length': 0, + 'Destination': 'c/o-copy'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o-copy', self.authorized[1].path) + + def test_COPY_no_destination_header(self): + req = Request.blank( + '/v1/a/c/o', method='COPY', headers={'Content-Length': 0}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '412 Precondition Failed') + self.assertEqual(len(self.authorized), 0) + + def test_basic_COPY_account(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o2', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c1/o2', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('GET', method) + self.assertEqual('/v1/a/c/o', path) + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a1/c1/o2', path) + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o2', self.authorized[1].path) + + def test_COPY_across_containers(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c2/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c2/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c2/o', self.authorized[1].path) + + def test_COPY_source_with_slashes_in_name(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o', path) + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_COPY_account_source_with_slashes_in_name(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c1/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a1/c1/o', path) + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_COPY_destination_leading_slash(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_COPY_account_destination_leading_slash(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a1/c1/o', path) + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_COPY_source_with_slashes_destination_leading_slash(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o', path) + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_COPY_account_source_with_slashes_destination_leading_slash(self): + self.app.register('GET', '/v1/a/c/o/o2', swob.HTTPOk, {}, 'passed') + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a1/c1/o', path) + self.assertTrue(('X-Copied-From', 'c/o/o2') in headers) + self.assertTrue(('X-Copied-From-Account', 'a') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o/o2', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_COPY_no_object_in_destination(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c_o'}) + try: + status, headers, body = self.call_ssc(req) + except HTTPException as resp: + self.assertEqual("412 Precondition Failed", str(resp)) + else: + self.fail("Expecting HTTPException.") + + def test_COPY_account_no_object_in_destination(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c_o', + 'Destination-Account': 'a1'}) + try: + status, headers, body = self.call_ssc(req) + except HTTPException as resp: + self.assertEqual("412 Precondition Failed", str(resp)) + else: + self.fail("Expecting HTTPException.") + + def test_COPY_account_bad_destination_account(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o', + 'Destination-Account': '/i/am/bad'}) + try: + status, headers, body = self.call_ssc(req) + except HTTPException as resp: + self.assertEqual("412 Precondition Failed", str(resp)) + else: + self.fail("Expecting HTTPException.") + + def test_COPY_server_error_reading_source(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPServiceUnavailable, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '503 Service Unavailable') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_account_server_error_reading_source(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPServiceUnavailable, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '503 Service Unavailable') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_not_found_reading_source(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPNotFound, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '404 Not Found') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_account_not_found_reading_source(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPNotFound, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '404 Not Found') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_with_metadata(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, "passed") + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o', + 'X-Object-Meta-Ours': 'okay'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o', path) + self.assertEqual(req_headers['X-Object-Meta-Ours'], 'okay') + self.assertTrue(('X-Object-Meta-Ours', 'okay') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_COPY_account_with_metadata(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, "passed") + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'X-Object-Meta-Ours': 'okay', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a1/c1/o', path) + self.assertEqual(req_headers['X-Object-Meta-Ours'], 'okay') + self.assertTrue(('X-Object-Meta-Ours', 'okay') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_COPY_source_zero_content_length(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '413 Request Entity Too Large') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_source_larger_than_max_file_size(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, "largebody") + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + with mock.patch('swift.common.middleware.copy.' + 'MAX_FILE_SIZE', 1): + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '413 Request Entity Too Large') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_account_source_zero_content_length(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '413 Request Entity Too Large') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_account_source_larger_than_max_file_size(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, "largebody") + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + with mock.patch('swift.common.middleware.copy.' + 'MAX_FILE_SIZE', 1): + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '413 Request Entity Too Large') + self.assertEqual(len(self.authorized), 1) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_newest(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, + {'Last-Modified': '123'}, "passed") + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From-Last-Modified', '123') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o', self.authorized[1].path) + + def test_COPY_account_newest(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, + {'Last-Modified': '123'}, "passed") + self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c1/o', + 'Destination-Account': 'a1'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From-Last-Modified', '123') in headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a1/c1/o', self.authorized[1].path) + + def test_COPY_in_OPTIONS_response(self): + self.app.register('OPTIONS', '/v1/a/c/o', swob.HTTPOk, + {'Allow': 'GET, PUT'}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'OPTIONS'}, headers={}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '200 OK') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('OPTIONS', method) + self.assertEqual('/v1/a/c/o', path) + self.assertTrue(('Allow', 'GET, PUT, COPY') in headers) + self.assertEqual(len(self.authorized), 1) + self.assertEqual('OPTIONS', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + def test_COPY_in_OPTIONS_response_CORS(self): + self.app.register('OPTIONS', '/v1/a/c/o', swob.HTTPOk, + {'Allow': 'GET, PUT', + 'Access-Control-Allow-Methods': 'GET, PUT'}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'OPTIONS'}, headers={}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '200 OK') + calls = self.app.calls_with_headers + method, path, req_headers = calls[0] + self.assertEqual('OPTIONS', method) + self.assertEqual('/v1/a/c/o', path) + self.assertTrue(('Allow', 'GET, PUT, COPY') in headers) + self.assertTrue(('Access-Control-Allow-Methods', + 'GET, PUT, COPY') in headers) + self.assertEqual(len(self.authorized), 1) + self.assertEqual('OPTIONS', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + + +class TestServerSideCopyConfiguration(unittest.TestCase): + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmpdir) + + def test_reading_proxy_conf_when_no_middleware_conf_present(self): + proxy_conf = dedent(""" + [DEFAULT] + bind_ip = 10.4.5.6 + + [pipeline:main] + pipeline = catch_errors copy ye-olde-proxy-server + + [filter:copy] + use = egg:swift#copy + + [app:ye-olde-proxy-server] + use = egg:swift#proxy + object_post_as_copy = no + """) + + conffile = tempfile.NamedTemporaryFile() + conffile.write(proxy_conf) + conffile.flush() + + ssc = copy.filter_factory({ + '__file__': conffile.name + })("no app here") + + self.assertEqual(ssc.object_post_as_copy, False) + + def test_middleware_conf_precedence(self): + proxy_conf = dedent(""" + [DEFAULT] + bind_ip = 10.4.5.6 + + [pipeline:main] + pipeline = catch_errors copy ye-olde-proxy-server + + [filter:copy] + use = egg:swift#copy + object_post_as_copy = no + + [app:ye-olde-proxy-server] + use = egg:swift#proxy + object_post_as_copy = yes + """) + + conffile = tempfile.NamedTemporaryFile() + conffile.write(proxy_conf) + conffile.flush() + + ssc = copy.filter_factory({ + 'object_post_as_copy': 'no', + '__file__': conffile.name + })("no app here") + + self.assertEqual(ssc.object_post_as_copy, False) + + +@patch_policies(with_ec_default=True) +class TestServerSideCopyMiddlewareWithEC(unittest.TestCase): + container_info = { + 'write_acl': None, + 'read_acl': None, + 'storage_policy': None, + 'sync_key': None, + 'versions': None, + } + + def setUp(self): + self.logger = debug_logger('proxy-server') + self.logger.thread_locals = ('txn1', '127.0.0.2') + self.app = PatchedObjControllerApp( + None, FakeMemcache(), account_ring=FakeRing(), + container_ring=FakeRing(), logger=self.logger) + self.ssc = copy.filter_factory({ + 'object_post_as_copy': 'yes', + })(self.app) + self.ssc.logger = self.app.logger + self.policy = POLICIES.default + self.app.container_info = dict(self.container_info) + + def test_COPY_with_ranges(self): + req = swob.Request.blank( + '/v1/a/c/o', method='COPY', + headers={'Destination': 'c1/o', + 'Range': 'bytes=5-10'}) + # turn a real body into fragments + segment_size = self.policy.ec_segment_size + real_body = ('asdf' * segment_size)[:-10] + + # split it up into chunks + chunks = [real_body[x:x + segment_size] + for x in range(0, len(real_body), segment_size)] + + # we need only first chunk to rebuild 5-10 range + fragments = self.policy.pyeclib_driver.encode(chunks[0]) + fragment_payloads = [] + fragment_payloads.append(fragments) + + node_fragments = zip(*fragment_payloads) + self.assertEqual(len(node_fragments), + self.policy.object_ring.replicas) # sanity + headers = {'X-Object-Sysmeta-Ec-Content-Length': str(len(real_body))} + responses = [(200, ''.join(node_fragments[i]), headers) + for i in range(POLICIES.default.ec_ndata)] + responses += [(201, '', {})] * self.policy.object_ring.replicas + status_codes, body_iter, headers = zip(*responses) + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*status_codes, body_iter=body_iter, + headers=headers, expect_headers=expect_headers): + resp = req.get_response(self.ssc) + self.assertEqual(resp.status_int, 201) + + def test_COPY_with_invalid_ranges(self): + # real body size is segment_size - 10 (just 1 segment) + segment_size = self.policy.ec_segment_size + real_body = ('a' * segment_size)[:-10] + + # range is out of real body but in segment size + self._test_invalid_ranges('COPY', real_body, + segment_size, '%s-' % (segment_size - 10)) + # range is out of both real body and segment size + self._test_invalid_ranges('COPY', real_body, + segment_size, '%s-' % (segment_size + 10)) + + def _test_invalid_ranges(self, method, real_body, segment_size, req_range): + # make a request with range starts from more than real size. + body_etag = md5(real_body).hexdigest() + req = swob.Request.blank( + '/v1/a/c/o', method=method, + headers={'Destination': 'c1/o', + 'Range': 'bytes=%s' % (req_range)}) + + fragments = self.policy.pyeclib_driver.encode(real_body) + fragment_payloads = [fragments] + + node_fragments = zip(*fragment_payloads) + self.assertEqual(len(node_fragments), + self.policy.object_ring.replicas) # sanity + headers = {'X-Object-Sysmeta-Ec-Content-Length': str(len(real_body)), + 'X-Object-Sysmeta-Ec-Etag': body_etag} + start = int(req_range.split('-')[0]) + self.assertTrue(start >= 0) # sanity + title, exp = swob.RESPONSE_REASONS[416] + range_not_satisfiable_body = \ + '

%s

%s

' % (title, exp) + if start >= segment_size: + responses = [(416, range_not_satisfiable_body, headers) + for i in range(POLICIES.default.ec_ndata)] + else: + responses = [(200, ''.join(node_fragments[i]), headers) + for i in range(POLICIES.default.ec_ndata)] + status_codes, body_iter, headers = zip(*responses) + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + # TODO possibly use FakeApp here + with set_http_connect(*status_codes, body_iter=body_iter, + headers=headers, expect_headers=expect_headers): + resp = req.get_response(self.ssc) + self.assertEqual(resp.status_int, 416) + self.assertEqual(resp.content_length, len(range_not_satisfiable_body)) + self.assertEqual(resp.body, range_not_satisfiable_body) + self.assertEqual(resp.etag, body_etag) + self.assertEqual(resp.headers['Accept-Ranges'], 'bytes') diff --git a/test/unit/common/middleware/test_dlo.py b/test/unit/common/middleware/test_dlo.py index 1374b403df..04fbc4e614 100644 --- a/test/unit/common/middleware/test_dlo.py +++ b/test/unit/common/middleware/test_dlo.py @@ -803,107 +803,6 @@ class TestDloGetManifest(DloTestCase): self.assertTrue(auth_got_called[0] > 1) -def fake_start_response(*args, **kwargs): - pass - - -class TestDloCopyHook(DloTestCase): - def setUp(self): - super(TestDloCopyHook, self).setUp() - - self.app.register( - 'GET', '/v1/AUTH_test/c/o1', swob.HTTPOk, - {'Content-Length': '10', 'Etag': 'o1-etag'}, - "aaaaaaaaaa") - self.app.register( - 'GET', '/v1/AUTH_test/c/o2', swob.HTTPOk, - {'Content-Length': '10', 'Etag': 'o2-etag'}, - "bbbbbbbbbb") - self.app.register( - 'GET', '/v1/AUTH_test/c/man', - swob.HTTPOk, {'X-Object-Manifest': 'c/o'}, - "manifest-contents") - - lm = '2013-11-22T02:42:13.781760' - ct = 'application/octet-stream' - segs = [{"hash": "o1-etag", "bytes": 10, "name": "o1", - "last_modified": lm, "content_type": ct}, - {"hash": "o2-etag", "bytes": 5, "name": "o2", - "last_modified": lm, "content_type": ct}] - - self.app.register( - 'GET', '/v1/AUTH_test/c?format=json&prefix=o', - swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, - json.dumps(segs)) - - copy_hook = [None] - - # slip this guy in there to pull out the hook - def extract_copy_hook(env, sr): - copy_hook[0] = env.get('swift.copy_hook') - return self.app(env, sr) - - self.dlo = dlo.filter_factory({})(extract_copy_hook) - - req = swob.Request.blank('/v1/AUTH_test/c/o1', - environ={'REQUEST_METHOD': 'GET'}) - self.dlo(req.environ, fake_start_response) - self.copy_hook = copy_hook[0] - - self.assertTrue(self.copy_hook is not None) # sanity check - - def test_copy_hook_passthrough(self): - source_req = swob.Request.blank( - '/v1/AUTH_test/c/man', - environ={'REQUEST_METHOD': 'GET'}) - sink_req = swob.Request.blank( - '/v1/AUTH_test/c/man', - environ={'REQUEST_METHOD': 'PUT'}) - source_resp = swob.Response(request=source_req, status=200) - - # no X-Object-Manifest header, so do nothing - modified_resp = self.copy_hook(source_req, source_resp, sink_req) - self.assertTrue(modified_resp is source_resp) - - def test_copy_hook_manifest(self): - source_req = swob.Request.blank( - '/v1/AUTH_test/c/man', - environ={'REQUEST_METHOD': 'GET'}) - sink_req = swob.Request.blank( - '/v1/AUTH_test/c/man', - environ={'REQUEST_METHOD': 'PUT'}) - source_resp = swob.Response( - request=source_req, status=200, - headers={"X-Object-Manifest": "c/o"}, - app_iter=["manifest"]) - - # it's a manifest, so copy the segments to make a normal object - modified_resp = self.copy_hook(source_req, source_resp, sink_req) - self.assertTrue(modified_resp is not source_resp) - self.assertEqual(modified_resp.etag, - hashlib.md5("o1-etago2-etag").hexdigest()) - self.assertEqual(sink_req.headers.get('X-Object-Manifest'), None) - - def test_copy_hook_manifest_with_multipart_manifest_get(self): - source_req = swob.Request.blank( - '/v1/AUTH_test/c/man', - environ={'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'multipart-manifest=get'}) - sink_req = swob.Request.blank( - '/v1/AUTH_test/c/man', - environ={'REQUEST_METHOD': 'PUT'}) - source_resp = swob.Response( - request=source_req, status=200, - headers={"X-Object-Manifest": "c/o"}, - app_iter=["manifest"]) - - # make sure the sink request (the backend PUT) gets X-Object-Manifest - # on it, but that's all - modified_resp = self.copy_hook(source_req, source_resp, sink_req) - self.assertTrue(modified_resp is source_resp) - self.assertEqual(sink_req.headers.get('X-Object-Manifest'), 'c/o') - - class TestDloConfiguration(unittest.TestCase): """ For backwards compatibility, we will read a couple of values out of the diff --git a/test/unit/common/middleware/test_quotas.py b/test/unit/common/middleware/test_quotas.py index b71b78ed83..f99b8df663 100644 --- a/test/unit/common/middleware/test_quotas.py +++ b/test/unit/common/middleware/test_quotas.py @@ -15,8 +15,9 @@ import unittest -from swift.common.swob import Request, HTTPUnauthorized -from swift.common.middleware import container_quotas +from swift.common.swob import Request, HTTPUnauthorized, HTTPOk, HTTPException +from swift.common.middleware import container_quotas, copy +from test.unit.common.middleware.helpers import FakeSwift class FakeCache(object): @@ -95,32 +96,6 @@ class TestContainerQuotas(unittest.TestCase): self.assertEqual(res.status_int, 413) self.assertEqual(res.body, 'Upload exceeds quota.') - def test_exceed_bytes_quota_copy_from(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}}) - - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - - def test_exceed_bytes_quota_copy_verb(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}}) - - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - def test_not_exceed_bytes_quota(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) @@ -131,60 +106,6 @@ class TestContainerQuotas(unittest.TestCase): res = req.get_response(app) self.assertEqual(res.status_int, 200) - def test_not_exceed_bytes_quota_copy_from(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_not_exceed_bytes_quota_copy_verb(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_bytes_quota_copy_from_no_src(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o3'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_bytes_quota_copy_from_bad_src(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.cache': cache}, - headers={'x-copy-from': 'bad_path'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 412) - - def test_bytes_quota_copy_verb_no_src(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) - req = Request.blank('/v1/a/c2/o3', - environ={'REQUEST_METHOD': 'COPY', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - def test_exceed_counts_quota(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) @@ -196,61 +117,6 @@ class TestContainerQuotas(unittest.TestCase): self.assertEqual(res.status_int, 413) self.assertEqual(res.body, 'Upload exceeds quota.') - def test_exceed_counts_quota_copy_from(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.object/a/c2/o2': {'length': 10}, - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - - def test_exceed_counts_quota_copy_verb(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - - def test_exceed_counts_quota_copy_cross_account_verb(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'}, - 'status': 200, 'object_count': 1} - a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'}, - 'status': 200, 'object_count': 1} - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.container/a/c': a_c_cache, - 'swift.container/a2/c': a2_c_cache}, - headers={'Destination': '/c/o', - 'Destination-Account': 'a2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - - def test_exceed_counts_quota_copy_cross_account_PUT_verb(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'}, - 'status': 200, 'object_count': 1} - a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'}, - 'status': 200, 'object_count': 1} - req = Request.blank('/v1/a2/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.container/a/c': a_c_cache, - 'swift.container/a2/c': a2_c_cache}, - headers={'X-Copy-From': '/c2/o2', - 'X-Copy-From-Account': 'a'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 413) - self.assertEqual(res.body, 'Upload exceeds quota.') - def test_not_exceed_counts_quota(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) @@ -261,26 +127,6 @@ class TestContainerQuotas(unittest.TestCase): res = req.get_response(app) self.assertEqual(res.status_int, 200) - def test_not_exceed_counts_quota_copy_from(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'swift.cache': cache}, - headers={'x-copy-from': '/c2/o2'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - - def test_not_exceed_counts_quota_copy_verb(self): - app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) - cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) - req = Request.blank('/v1/a/c2/o2', - environ={'REQUEST_METHOD': 'COPY', - 'swift.cache': cache}, - headers={'Destination': '/c/o'}) - res = req.get_response(app) - self.assertEqual(res.status_int, 200) - def test_invalid_quotas(self): req = Request.blank( '/v1/a/c', @@ -346,5 +192,168 @@ class TestContainerQuotas(unittest.TestCase): res = req.get_response(app) self.assertEqual(res.status_int, 401) + +class ContainerQuotaCopyingTestCases(unittest.TestCase): + + def setUp(self): + self.app = FakeSwift() + self.cq_filter = container_quotas.filter_factory({})(self.app) + self.copy_filter = copy.filter_factory({})(self.cq_filter) + + def test_exceed_bytes_quota_copy_verb(self): + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}}) + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.cache': cache}, + headers={'Destination': '/c/o'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_not_exceed_bytes_quota_copy_verb(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + self.app.register( + 'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed') + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.cache': cache}, + headers={'Destination': '/c/o'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_exceed_counts_quota_copy_verb(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed') + cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.cache': cache}, + headers={'Destination': '/c/o'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_exceed_counts_quota_copy_cross_account_verb(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed') + a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'}, + 'status': 200, 'object_count': 1} + a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'}, + 'status': 200, 'object_count': 1} + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.container/a/c': a_c_cache, + 'swift.container/a2/c': a2_c_cache}, + headers={'Destination': '/c/o', + 'Destination-Account': 'a2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_exceed_counts_quota_copy_cross_account_PUT_verb(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed') + a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'}, + 'status': 200, 'object_count': 1} + a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'}, + 'status': 200, 'object_count': 1} + req = Request.blank('/v1/a2/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.container/a/c': a_c_cache, + 'swift.container/a2/c': a2_c_cache}, + headers={'X-Copy-From': '/c2/o2', + 'X-Copy-From-Account': 'a'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_exceed_bytes_quota_copy_from(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}}) + + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_not_exceed_bytes_quota_copy_from(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + self.app.register( + 'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed') + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_bytes_quota_copy_from_no_src(self): + self.app.register('GET', '/v1/a/c2/o3', HTTPOk, {}, 'passed') + self.app.register( + 'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed') + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o3'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_bytes_quota_copy_from_bad_src(self): + cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': 'bad_path'}) + with self.assertRaises(HTTPException) as catcher: + req.get_response(self.copy_filter) + self.assertEqual(412, catcher.exception.status_int) + + def test_exceed_counts_quota_copy_from(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 413) + self.assertEqual(res.body, 'Upload exceeds quota.') + + def test_not_exceed_counts_quota_copy_from(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + self.app.register( + 'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed') + cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + + def test_not_exceed_counts_quota_copy_verb(self): + self.app.register('GET', '/v1/a/c2/o2', HTTPOk, + {'Content-Length': '10'}, 'passed') + self.app.register( + 'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed') + cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) + req = Request.blank('/v1/a/c2/o2', + environ={'REQUEST_METHOD': 'COPY', + 'swift.cache': cache}, + headers={'Destination': '/c/o'}) + res = req.get_response(self.copy_filter) + self.assertEqual(res.status_int, 200) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 03f5c23213..79eaddcbf3 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -26,7 +26,7 @@ from swift.common import swob, utils from swift.common.exceptions import ListingIterError, SegmentError from swift.common.header_key_dict import HeaderKeyDict from swift.common.middleware import slo -from swift.common.swob import Request, Response, HTTPException +from swift.common.swob import Request, HTTPException from swift.common.utils import quote, closing_if_possible, close_if_possible from test.unit.common.middleware.helpers import FakeSwift @@ -2653,70 +2653,6 @@ class TestSloBulkLogger(unittest.TestCase): self.assertTrue(slo_mware.logger is slo_mware.bulk_deleter.logger) -class TestSloCopyHook(SloTestCase): - def setUp(self): - super(TestSloCopyHook, self).setUp() - - self.app.register( - 'GET', '/v1/AUTH_test/c/o', swob.HTTPOk, - {'Content-Length': '3', 'Etag': md5hex("obj")}, "obj") - self.app.register( - 'GET', '/v1/AUTH_test/c/man', - swob.HTTPOk, {'Content-Type': 'application/json', - 'X-Static-Large-Object': 'true'}, - json.dumps([{'name': '/c/o', 'hash': md5hex("obj"), - 'bytes': '3'}])) - self.app.register( - 'COPY', '/v1/AUTH_test/c/o', swob.HTTPCreated, {}) - - copy_hook = [None] - - # slip this guy in there to pull out the hook - def extract_copy_hook(env, sr): - if env['REQUEST_METHOD'] == 'COPY': - copy_hook[0] = env['swift.copy_hook'] - return self.app(env, sr) - - self.slo = slo.filter_factory({})(extract_copy_hook) - - req = Request.blank('/v1/AUTH_test/c/o', - environ={'REQUEST_METHOD': 'COPY'}) - self.slo(req.environ, fake_start_response) - self.copy_hook = copy_hook[0] - - self.assertTrue(self.copy_hook is not None) # sanity check - - def test_copy_hook_passthrough(self): - source_req = Request.blank( - '/v1/AUTH_test/c/o', - environ={'REQUEST_METHOD': 'GET'}) - sink_req = Request.blank( - '/v1/AUTH_test/c/o', - environ={'REQUEST_METHOD': 'PUT'}) - # no X-Static-Large-Object header, so do nothing - source_resp = Response(request=source_req, status=200) - - modified_resp = self.copy_hook(source_req, source_resp, sink_req) - self.assertTrue(modified_resp is source_resp) - - def test_copy_hook_manifest(self): - source_req = Request.blank( - '/v1/AUTH_test/c/o', - environ={'REQUEST_METHOD': 'GET'}) - sink_req = Request.blank( - '/v1/AUTH_test/c/o', - environ={'REQUEST_METHOD': 'PUT'}) - source_resp = Response(request=source_req, status=200, - headers={"X-Static-Large-Object": "true"}, - app_iter=[json.dumps([{'name': '/c/o', - 'hash': md5hex("obj"), - 'bytes': '3'}])]) - - modified_resp = self.copy_hook(source_req, source_resp, sink_req) - self.assertTrue(modified_resp is not source_resp) - self.assertEqual(modified_resp.etag, md5hex(md5hex("obj"))) - - class TestSwiftInfo(unittest.TestCase): def setUp(self): utils._swift_info = {} diff --git a/test/unit/common/middleware/test_versioned_writes.py b/test/unit/common/middleware/test_versioned_writes.py index c6da47fde8..27b8914555 100644 --- a/test/unit/common/middleware/test_versioned_writes.py +++ b/test/unit/common/middleware/test_versioned_writes.py @@ -19,7 +19,7 @@ import os import time import unittest from swift.common import swob -from swift.common.middleware import versioned_writes +from swift.common.middleware import versioned_writes, copy from swift.common.swob import Request from test.unit.common.middleware.helpers import FakeSwift @@ -259,6 +259,23 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase): self.assertEqual(len(self.authorized), 1) self.assertRequestEqual(req, self.authorized[0]) + def test_put_object_post_as_copy(self): + # PUTs due to a post-as-copy should NOT cause a versioning op + self.app.register( + 'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed') + + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '100', + 'swift.post_as_copy': True}) + status, headers, body = self.call_vw(req) + self.assertEqual(status, '201 Created') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + self.assertEqual(1, self.app.call_count) + def test_put_first_object_success(self): self.app.register( 'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') @@ -333,7 +350,7 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase): def test_delete_object_no_versioning_with_container_config_true(self): # set False to versions_write obviously and expect no GET versioning - # container and PUT called (just delete object as normal) + # container and GET/PUT called (just delete object as normal) self.vw.conf = {'allow_versioned_writes': 'false'} self.app.register( 'DELETE', '/v1/a/c/o', swob.HTTPNoContent, {}, 'passed') @@ -351,25 +368,6 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase): self.assertTrue('GET' not in called_method) self.assertEqual(1, self.app.call_count) - def test_copy_object_no_versioning_with_container_config_true(self): - # set False to versions_write obviously and expect no extra - # COPY called (just copy object as normal) - self.vw.conf = {'allow_versioned_writes': 'false'} - self.app.register( - 'COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None) - cache = FakeCache({'versions': 'ver_cont'}) - req = Request.blank( - '/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache}) - status, headers, body = self.call_vw(req) - self.assertEqual(status, '201 Created') - self.assertEqual(len(self.authorized), 1) - self.assertRequestEqual(req, self.authorized[0]) - called_method = \ - [method for (method, path, rheaders) in self.app._calls] - self.assertTrue('COPY' in called_method) - self.assertEqual(called_method.count('COPY'), 1) - def test_new_version_success(self): self.app.register( 'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed') @@ -476,77 +474,6 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase): self.assertEqual('PUT', method) self.assertEqual('/v1/a/ver_cont/001o/0000000000.00000', path) - def test_copy_first_version(self): - self.app.register( - 'COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') - self.app.register( - 'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound, {}, None) - cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) - req = Request.blank( - '/v1/a/src_cont/src_obj', - environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, - 'CONTENT_LENGTH': '100'}, - headers={'Destination': 'tgt_cont/tgt_obj'}) - status, headers, body = self.call_vw(req) - self.assertEqual(status, '200 OK') - self.assertEqual(len(self.authorized), 1) - self.assertRequestEqual(req, self.authorized[0]) - self.assertEqual(2, self.app.call_count) - - def test_copy_new_version(self): - self.app.register( - 'COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') - self.app.register( - 'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk, - {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed') - self.app.register( - 'PUT', '/v1/a/ver_cont/007tgt_obj/0000000001.00000', swob.HTTPOk, - {}, None) - cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) - req = Request.blank( - '/v1/a/src_cont/src_obj', - environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, - 'CONTENT_LENGTH': '100'}, - headers={'Destination': 'tgt_cont/tgt_obj'}) - status, headers, body = self.call_vw(req) - self.assertEqual(status, '200 OK') - self.assertEqual(len(self.authorized), 1) - self.assertRequestEqual(req, self.authorized[0]) - self.assertEqual(3, self.app.call_count) - - def test_copy_new_version_different_account(self): - self.app.register( - 'COPY', '/v1/src_a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') - self.app.register( - 'GET', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk, - {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed') - self.app.register( - 'PUT', '/v1/tgt_a/ver_cont/007tgt_obj/0000000001.00000', - swob.HTTPOk, {}, None) - cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) - req = Request.blank( - '/v1/src_a/src_cont/src_obj', - environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, - 'CONTENT_LENGTH': '100'}, - headers={'Destination': 'tgt_cont/tgt_obj', - 'Destination-Account': 'tgt_a'}) - status, headers, body = self.call_vw(req) - self.assertEqual(status, '200 OK') - self.assertEqual(len(self.authorized), 1) - self.assertRequestEqual(req, self.authorized[0]) - self.assertEqual(3, self.app.call_count) - - def test_copy_new_version_bogus_account(self): - cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) - req = Request.blank( - '/v1/src_a/src_cont/src_obj', - environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, - 'CONTENT_LENGTH': '100'}, - headers={'Destination': 'tgt_cont/tgt_obj', - 'Destination-Account': '/im/on/a/boat'}) - status, headers, body = self.call_vw(req) - self.assertEqual(status, '412 Precondition Failed') - def test_delete_first_object_success(self): self.app.register( 'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') @@ -1057,3 +984,117 @@ class VersionedWritesOldContainersTestCase(VersionedWritesBaseTestCase): ('PUT', '/v1/a/c/o'), ('DELETE', '/v1/a/ver_cont/001o/2'), ]) + + +class VersionedWritesCopyingTestCase(VersionedWritesBaseTestCase): + # verify interaction of copy and versioned_writes middlewares + + def setUp(self): + self.app = FakeSwift() + conf = {'allow_versioned_writes': 'true'} + self.vw = versioned_writes.filter_factory(conf)(self.app) + self.filter = copy.filter_factory({})(self.vw) + + def call_filter(self, req, **kwargs): + return self.call_app(req, app=self.filter, **kwargs) + + def test_copy_first_version(self): + # no existing object to move to the versions container + self.app.register( + 'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound, {}, None) + self.app.register( + 'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') + self.app.register( + 'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed') + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/src_cont/src_obj', + environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}, + headers={'Destination': 'tgt_cont/tgt_obj'}) + status, headers, body = self.call_filter(req) + self.assertEqual(status, '201 Created') + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path) + # note the GET on tgt_cont/tgt_obj is pre-authed + self.assertEqual(3, self.app.call_count, self.app.calls) + + def test_copy_new_version(self): + # existing object should be moved to versions container + self.app.register( + 'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') + self.app.register( + 'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk, + {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed') + self.app.register( + 'PUT', '/v1/a/ver_cont/007tgt_obj/0000000001.00000', swob.HTTPOk, + {}, None) + self.app.register( + 'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed') + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/src_cont/src_obj', + environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}, + headers={'Destination': 'tgt_cont/tgt_obj'}) + status, headers, body = self.call_filter(req) + self.assertEqual(status, '201 Created') + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path) + self.assertEqual(4, self.app.call_count) + + def test_copy_new_version_different_account(self): + self.app.register( + 'GET', '/v1/src_a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') + self.app.register( + 'GET', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk, + {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed') + self.app.register( + 'PUT', '/v1/tgt_a/ver_cont/007tgt_obj/0000000001.00000', + swob.HTTPOk, {}, None) + self.app.register( + 'PUT', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, + 'passed') + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/src_a/src_cont/src_obj', + environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, + 'CONTENT_LENGTH': '100'}, + headers={'Destination': 'tgt_cont/tgt_obj', + 'Destination-Account': 'tgt_a'}) + status, headers, body = self.call_filter(req) + self.assertEqual(status, '201 Created') + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/src_a/src_cont/src_obj', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/tgt_a/tgt_cont/tgt_obj', self.authorized[1].path) + self.assertEqual(4, self.app.call_count) + + def test_copy_object_no_versioning_with_container_config_true(self): + # set False to versions_write obviously and expect no extra + # COPY called (just copy object as normal) + self.vw.conf = {'allow_versioned_writes': 'false'} + self.app.register( + 'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed') + self.app.register( + 'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed') + cache = FakeCache({'versions': 'ver_cont'}) + req = Request.blank( + '/v1/a/src_cont/src_obj', + environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache}, + headers={'Destination': '/tgt_cont/tgt_obj'}) + status, headers, body = self.call_filter(req) + self.assertEqual(status, '201 Created') + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path) + self.assertEqual(2, self.app.call_count) diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index 2f7fb85d9b..f9829d81d5 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -173,33 +173,6 @@ class TestConstraints(unittest.TestCase): '/', headers=headers), 'object_name').status_int, HTTP_NOT_IMPLEMENTED) - def test_check_object_creation_copy(self): - headers = {'Content-Length': '0', - 'X-Copy-From': 'c/o2', - 'Content-Type': 'text/plain'} - self.assertEqual(constraints.check_object_creation(Request.blank( - '/', headers=headers), 'object_name'), None) - - headers = {'Content-Length': '1', - 'X-Copy-From': 'c/o2', - 'Content-Type': 'text/plain'} - self.assertEqual(constraints.check_object_creation(Request.blank( - '/', headers=headers), 'object_name').status_int, - HTTP_BAD_REQUEST) - - headers = {'Transfer-Encoding': 'chunked', - 'X-Copy-From': 'c/o2', - 'Content-Type': 'text/plain'} - self.assertEqual(constraints.check_object_creation(Request.blank( - '/', headers=headers), 'object_name'), None) - - # a content-length header is always required - headers = {'X-Copy-From': 'c/o2', - 'Content-Type': 'text/plain'} - self.assertEqual(constraints.check_object_creation(Request.blank( - '/', headers=headers), 'object_name').status_int, - HTTP_LENGTH_REQUIRED) - def test_check_object_creation_name_length(self): headers = {'Transfer-Encoding': 'chunked', 'Content-Type': 'text/plain'} @@ -459,60 +432,6 @@ class TestConstraints(unittest.TestCase): self.assertTrue(c.MAX_HEADER_SIZE > c.MAX_META_NAME_LENGTH) self.assertTrue(c.MAX_HEADER_SIZE > c.MAX_META_VALUE_LENGTH) - def test_validate_copy_from(self): - req = Request.blank( - '/v/a/c/o', - headers={'x-copy-from': 'c/o2'}) - src_cont, src_obj = constraints.check_copy_from_header(req) - self.assertEqual(src_cont, 'c') - self.assertEqual(src_obj, 'o2') - req = Request.blank( - '/v/a/c/o', - headers={'x-copy-from': 'c/subdir/o2'}) - src_cont, src_obj = constraints.check_copy_from_header(req) - self.assertEqual(src_cont, 'c') - self.assertEqual(src_obj, 'subdir/o2') - req = Request.blank( - '/v/a/c/o', - headers={'x-copy-from': '/c/o2'}) - src_cont, src_obj = constraints.check_copy_from_header(req) - self.assertEqual(src_cont, 'c') - self.assertEqual(src_obj, 'o2') - - def test_validate_bad_copy_from(self): - req = Request.blank( - '/v/a/c/o', - headers={'x-copy-from': 'bad_object'}) - self.assertRaises(HTTPException, - constraints.check_copy_from_header, req) - - def test_validate_destination(self): - req = Request.blank( - '/v/a/c/o', - headers={'destination': 'c/o2'}) - src_cont, src_obj = constraints.check_destination_header(req) - self.assertEqual(src_cont, 'c') - self.assertEqual(src_obj, 'o2') - req = Request.blank( - '/v/a/c/o', - headers={'destination': 'c/subdir/o2'}) - src_cont, src_obj = constraints.check_destination_header(req) - self.assertEqual(src_cont, 'c') - self.assertEqual(src_obj, 'subdir/o2') - req = Request.blank( - '/v/a/c/o', - headers={'destination': '/c/o2'}) - src_cont, src_obj = constraints.check_destination_header(req) - self.assertEqual(src_cont, 'c') - self.assertEqual(src_obj, 'o2') - - def test_validate_bad_destination(self): - req = Request.blank( - '/v/a/c/o', - headers={'destination': 'bad_object'}) - self.assertRaises(HTTPException, - constraints.check_destination_header, req) - def test_check_account_format(self): req = Request.blank( '/v/a/c/o', diff --git a/test/unit/common/test_swob.py b/test/unit/common/test_swob.py index 4f8d8f7be9..f1a11e1fcb 100644 --- a/test/unit/common/test_swob.py +++ b/test/unit/common/test_swob.py @@ -431,9 +431,10 @@ class TestRequest(unittest.TestCase): def test_invalid_req_environ_property_args(self): # getter only property try: - swift.common.swob.Request.blank('/', params={'a': 'b'}) + swift.common.swob.Request.blank( + '/', host_url='http://example.com:8080/v1/a/c/o') except TypeError as e: - self.assertEqual("got unexpected keyword argument 'params'", + self.assertEqual("got unexpected keyword argument 'host_url'", str(e)) else: self.assertTrue(False, "invalid req_environ_property " @@ -525,6 +526,14 @@ class TestRequest(unittest.TestCase): self.assertEqual(req.params['a'], 'b') self.assertEqual(req.params['c'], 'd') + new_params = {'e': 'f', 'g': 'h'} + req.params = new_params + self.assertDictEqual(new_params, req.params) + + new_params = (('i', 'j'), ('k', 'l')) + req.params = new_params + self.assertDictEqual(dict(new_params), req.params) + def test_timestamp_missing(self): req = swift.common.swob.Request.blank('/') self.assertRaises(exceptions.InvalidTimestamp, diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index f39f215499..e09f339266 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -136,6 +136,11 @@ class TestWSGI(unittest.TestCase): expected = swift.common.middleware.gatekeeper.GatekeeperMiddleware self.assertTrue(isinstance(app, expected)) + app = app.app + expected = \ + swift.common.middleware.copy.ServerSideCopyMiddleware + self.assertIsInstance(app, expected) + app = app.app expected = swift.common.middleware.dlo.DynamicLargeObject self.assertTrue(isinstance(app, expected)) @@ -1437,6 +1442,7 @@ class TestPipelineModification(unittest.TestCase): self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.copy', 'swift.common.middleware.dlo', 'swift.common.middleware.versioned_writes', 'swift.proxy.server']) @@ -1468,6 +1474,7 @@ class TestPipelineModification(unittest.TestCase): self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.copy', 'swift.common.middleware.dlo', 'swift.common.middleware.versioned_writes', 'swift.common.middleware.healthcheck', @@ -1506,6 +1513,7 @@ class TestPipelineModification(unittest.TestCase): self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.copy', 'swift.common.middleware.slo', 'swift.common.middleware.dlo', 'swift.common.middleware.versioned_writes', @@ -1605,6 +1613,7 @@ class TestPipelineModification(unittest.TestCase): self.assertEqual(self.pipeline_modules(app), [ 'swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.copy', 'swift.common.middleware.dlo', 'swift.common.middleware.versioned_writes', 'swift.common.middleware.healthcheck', @@ -1619,6 +1628,7 @@ class TestPipelineModification(unittest.TestCase): 'swift.common.middleware.gatekeeper', 'swift.common.middleware.healthcheck', 'swift.common.middleware.catch_errors', + 'swift.common.middleware.copy', 'swift.common.middleware.dlo', 'swift.common.middleware.versioned_writes', 'swift.proxy.server']) @@ -1632,6 +1642,7 @@ class TestPipelineModification(unittest.TestCase): 'swift.common.middleware.healthcheck', 'swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.copy', 'swift.common.middleware.dlo', 'swift.common.middleware.versioned_writes', 'swift.proxy.server']) @@ -1666,7 +1677,7 @@ class TestPipelineModification(unittest.TestCase): tempdir, policy.ring_name + '.ring.gz') app = wsgi.loadapp(conf_path) - proxy_app = app.app.app.app.app.app + proxy_app = app.app.app.app.app.app.app self.assertEqual(proxy_app.account_ring.serialized_path, account_ring_path) self.assertEqual(proxy_app.container_ring.serialized_path, diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index d18ac4299b..95b92b298a 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -649,7 +649,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin, def test_PUT_error_during_transfer_data(self): class FakeReader(object): def read(self, size): - raise exceptions.ChunkReadError('exception message') + raise IOError('error message') req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT', body='test body') @@ -747,62 +747,6 @@ class TestReplicatedObjController(BaseObjectControllerMixin, resp = req.get_response(self.app) self.assertEqual(resp.status_int, 404) - def test_POST_as_COPY_simple(self): - req = swift.common.swob.Request.blank('/v1/a/c/o', method='POST') - get_resp = [200] * self.obj_ring.replicas + \ - [404] * self.obj_ring.max_more_nodes - put_resp = [201] * self.obj_ring.replicas - codes = get_resp + put_resp - with set_http_connect(*codes): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 202) - self.assertEqual(req.environ['QUERY_STRING'], '') - self.assertTrue('swift.post_as_copy' in req.environ) - - def test_POST_as_COPY_static_large_object(self): - req = swift.common.swob.Request.blank('/v1/a/c/o', method='POST') - get_resp = [200] * self.obj_ring.replicas + \ - [404] * self.obj_ring.max_more_nodes - put_resp = [201] * self.obj_ring.replicas - codes = get_resp + put_resp - slo_headers = \ - [{'X-Static-Large-Object': True}] * self.obj_ring.replicas - get_headers = slo_headers + [{}] * (len(codes) - len(slo_headers)) - headers = {'headers': get_headers} - with set_http_connect(*codes, **headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 202) - self.assertEqual(req.environ['QUERY_STRING'], '') - self.assertTrue('swift.post_as_copy' in req.environ) - - def test_POST_delete_at(self): - t = str(int(time.time() + 100)) - req = swob.Request.blank('/v1/a/c/o', method='POST', - headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - post_headers = [] - - def capture_headers(ip, port, device, part, method, path, headers, - **kwargs): - if method == 'POST': - post_headers.append(headers) - x_newest_responses = [200] * self.obj_ring.replicas + \ - [404] * self.obj_ring.max_more_nodes - post_resp = [200] * self.obj_ring.replicas - codes = x_newest_responses + post_resp - with set_http_connect(*codes, give_connect=capture_headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 200) - self.assertEqual(req.environ['QUERY_STRING'], '') # sanity - self.assertTrue('swift.post_as_copy' in req.environ) - - for given_headers in post_headers: - self.assertEqual(given_headers.get('X-Delete-At'), t) - self.assertTrue('X-Delete-At-Host' in given_headers) - self.assertTrue('X-Delete-At-Device' in given_headers) - self.assertTrue('X-Delete-At-Partition' in given_headers) - self.assertTrue('X-Delete-At-Container' in given_headers) - def test_PUT_delete_at(self): t = str(int(time.time() + 100)) req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', @@ -1000,43 +944,6 @@ class TestReplicatedObjController(BaseObjectControllerMixin, resp = req.get_response(self.app) self.assertEqual(resp.status_int, 202) - def test_COPY_simple(self): - req = swift.common.swob.Request.blank( - '/v1/a/c/o', method='COPY', - headers={'Content-Length': 0, - 'Destination': 'c/o-copy'}) - head_resp = [200] * self.obj_ring.replicas + \ - [404] * self.obj_ring.max_more_nodes - put_resp = [201] * self.obj_ring.replicas - codes = head_resp + put_resp - with set_http_connect(*codes): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 201) - - def test_PUT_log_info(self): - req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT') - req.headers['x-copy-from'] = 'some/where' - req.headers['Content-Length'] = 0 - # override FakeConn default resp headers to keep log_info clean - resp_headers = {'x-delete-at': None} - head_resp = [200] * self.obj_ring.replicas + \ - [404] * self.obj_ring.max_more_nodes - put_resp = [201] * self.obj_ring.replicas - codes = head_resp + put_resp - with set_http_connect(*codes, headers=resp_headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 201) - self.assertEqual( - req.environ.get('swift.log_info'), ['x-copy-from:some/where']) - # and then check that we don't do that for originating POSTs - req = swift.common.swob.Request.blank('/v1/a/c/o') - req.method = 'POST' - req.headers['x-copy-from'] = 'else/where' - with set_http_connect(*codes, headers=resp_headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 202) - self.assertEqual(req.environ.get('swift.log_info'), None) - @patch_policies( [StoragePolicy(0, '1-replica', True), @@ -1397,7 +1304,7 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): def test_PUT_ec_error_during_transfer_data(self): class FakeReader(object): def read(self, size): - raise exceptions.ChunkReadError('exception message') + raise IOError('error message') req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT', body='test body') @@ -1603,72 +1510,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 201) - def test_COPY_cross_policy_type_from_replicated(self): - self.app.per_container_info = { - 'c1': self.app.container_info.copy(), - 'c2': self.app.container_info.copy(), - } - # make c2 use replicated storage policy 1 - self.app.per_container_info['c2']['storage_policy'] = '1' - - # a put request with copy from source c2 - req = swift.common.swob.Request.blank('/v1/a/c1/o', method='PUT', - body='', headers={ - 'X-Copy-From': 'c2/o'}) - - # c2 get - codes = [200] * self.replicas(POLICIES[1]) - codes += [404] * POLICIES[1].object_ring.max_more_nodes - # c1 put - codes += [201] * self.replicas() - expect_headers = { - 'X-Obj-Metadata-Footer': 'yes', - 'X-Obj-Multiphase-Commit': 'yes' - } - with set_http_connect(*codes, expect_headers=expect_headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 201) - - def test_COPY_cross_policy_type_to_replicated(self): - self.app.per_container_info = { - 'c1': self.app.container_info.copy(), - 'c2': self.app.container_info.copy(), - } - # make c1 use replicated storage policy 1 - self.app.per_container_info['c1']['storage_policy'] = '1' - - # a put request with copy from source c2 - req = swift.common.swob.Request.blank('/v1/a/c1/o', method='PUT', - body='', headers={ - 'X-Copy-From': 'c2/o'}) - - # c2 get - codes = [404, 200] * self.policy.ec_ndata - headers = { - 'X-Object-Sysmeta-Ec-Content-Length': 0, - } - # c1 put - codes += [201] * self.replicas(POLICIES[1]) - with set_http_connect(*codes, headers=headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 201) - - def test_COPY_cross_policy_type_unknown(self): - self.app.per_container_info = { - 'c1': self.app.container_info.copy(), - 'c2': self.app.container_info.copy(), - } - # make c1 use some made up storage policy index - self.app.per_container_info['c1']['storage_policy'] = '13' - - # a COPY request of c2 with destination in c1 - req = swift.common.swob.Request.blank('/v1/a/c2/o', method='COPY', - body='', headers={ - 'Destination': 'c1/o'}) - with set_http_connect(): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 503) - def _make_ec_archive_bodies(self, test_body, policy=None): policy = policy or self.policy segment_size = policy.ec_segment_size @@ -2378,40 +2219,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 503) - def test_COPY_with_ranges(self): - req = swift.common.swob.Request.blank( - '/v1/a/c/o', method='COPY', - headers={'Destination': 'c1/o', - 'Range': 'bytes=5-10'}) - # turn a real body into fragments - segment_size = self.policy.ec_segment_size - real_body = ('asdf' * segment_size)[:-10] - - # split it up into chunks - chunks = [real_body[x:x + segment_size] - for x in range(0, len(real_body), segment_size)] - - # we need only first chunk to rebuild 5-10 range - fragments = self.policy.pyeclib_driver.encode(chunks[0]) - fragment_payloads = [] - fragment_payloads.append(fragments) - - node_fragments = zip(*fragment_payloads) - self.assertEqual(len(node_fragments), self.replicas()) # sanity - headers = {'X-Object-Sysmeta-Ec-Content-Length': str(len(real_body))} - responses = [(200, ''.join(node_fragments[i]), headers) - for i in range(POLICIES.default.ec_ndata)] - responses += [(201, '', {})] * self.obj_ring.replicas - status_codes, body_iter, headers = zip(*responses) - expect_headers = { - 'X-Obj-Metadata-Footer': 'yes', - 'X-Obj-Multiphase-Commit': 'yes' - } - with set_http_connect(*status_codes, body_iter=body_iter, - headers=headers, expect_headers=expect_headers): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 201) - def test_GET_with_invalid_ranges(self): # real body size is segment_size - 10 (just 1 segment) segment_size = self.policy.ec_segment_size @@ -2424,18 +2231,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): self._test_invalid_ranges('GET', real_body, segment_size, '%s-' % (segment_size + 10)) - def test_COPY_with_invalid_ranges(self): - # real body size is segment_size - 10 (just 1 segment) - segment_size = self.policy.ec_segment_size - real_body = ('a' * segment_size)[:-10] - - # range is out of real body but in segment size - self._test_invalid_ranges('COPY', real_body, - segment_size, '%s-' % (segment_size - 10)) - # range is out of both real body and segment size - self._test_invalid_ranges('COPY', real_body, - segment_size, '%s-' % (segment_size + 10)) - def _test_invalid_ranges(self, method, real_body, segment_size, req_range): # make a request with range starts from more than real size. body_etag = md5(real_body).hexdigest() diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 6e55e74fa2..1fc021a542 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -63,7 +63,8 @@ from swift.proxy.controllers.obj import ReplicatedObjectController from swift.account import server as account_server from swift.container import server as container_server from swift.obj import server as object_server -from swift.common.middleware import proxy_logging, versioned_writes +from swift.common.middleware import proxy_logging, versioned_writes, \ + copy from swift.common.middleware.acl import parse_acl, format_acl from swift.common.exceptions import ChunkReadTimeout, DiskFileNotExist, \ APIVersionError, ChunkWriteTimeout @@ -3017,7 +3018,6 @@ class TestObjectController(unittest.TestCase): @unpatch_policies def test_PUT_POST_last_modified(self): prolis = _test_sockets[0] - prosrv = _test_servers[0] def _do_HEAD(): # do a HEAD to get reported last modified time @@ -3078,9 +3078,7 @@ class TestObjectController(unittest.TestCase): _do_conditional_GET_checks(last_modified_put) - # now POST to the object using default object_post_as_copy setting - orig_post_as_copy = prosrv.object_post_as_copy - + # now POST to the object # last-modified rounded in sec so sleep a sec to increment sleep(1) @@ -3101,31 +3099,6 @@ class TestObjectController(unittest.TestCase): self.assertNotEqual(last_modified_put, last_modified_head) _do_conditional_GET_checks(last_modified_head) - # now POST using non-default object_post_as_copy setting - try: - # last-modified rounded in sec so sleep a sec to increment - last_modified_post = last_modified_head - sleep(1) - prosrv.object_post_as_copy = not orig_post_as_copy - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('POST /v1/a/c/o.last_modified HTTP/1.1\r\n' - 'Host: localhost\r\nConnection: close\r\n' - 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 202' - self.assertEqual(headers[:len(exp)], exp) - for line in headers.split('\r\n'): - self.assertFalse(line.startswith(lm_hdr)) - finally: - prosrv.object_post_as_copy = orig_post_as_copy - - # last modified time will have changed due to POST - last_modified_head = _do_HEAD() - self.assertNotEqual(last_modified_post, last_modified_head) - _do_conditional_GET_checks(last_modified_head) - def test_PUT_auto_content_type(self): with save_globals(): controller = ReplicatedObjectController( @@ -3412,59 +3385,6 @@ class TestObjectController(unittest.TestCase): } check_request(request, **expectations) - # and this time with post as copy - self.app.object_post_as_copy = True - self.app.memcache.store = {} - backend_requests = [] - req = Request.blank('/v1/a/c/o', {}, method='POST', - headers={'X-Object-Meta-Color': 'Blue', - 'X-Backend-Storage-Policy-Index': 0}) - with mocked_http_conn( - 200, 200, 200, 200, 200, 201, 201, 201, - headers=resp_headers, give_connect=capture_requests - ) as fake_conn: - resp = req.get_response(self.app) - self.assertRaises(StopIteration, fake_conn.code_iter.next) - self.assertEqual(resp.status_int, 202) - self.assertEqual(len(backend_requests), 8) - policy0 = {'X-Backend-Storage-Policy-Index': '0'} - policy1 = {'X-Backend-Storage-Policy-Index': '1'} - expected = [ - # account info - {'method': 'HEAD', 'path': '/0/a'}, - # container info - {'method': 'HEAD', 'path': '/0/a/c'}, - # x-newests - {'method': 'GET', 'path': '/0/a/c/o', 'headers': policy1}, - {'method': 'GET', 'path': '/0/a/c/o', 'headers': policy1}, - {'method': 'GET', 'path': '/0/a/c/o', 'headers': policy1}, - # new writes - {'method': 'PUT', 'path': '/0/a/c/o', 'headers': policy0}, - {'method': 'PUT', 'path': '/0/a/c/o', 'headers': policy0}, - {'method': 'PUT', 'path': '/0/a/c/o', 'headers': policy0}, - ] - for request, expectations in zip(backend_requests, expected): - check_request(request, **expectations) - - def test_POST_as_copy(self): - with save_globals(): - def test_status_map(statuses, expected): - set_http_connect(*statuses) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, - headers={'Content-Type': 'foo/bar'}) - self.app.update_request(req) - res = req.get_response(self.app) - expected = str(expected) - self.assertEqual(res.status[:len(expected)], expected) - test_status_map((200, 200, 200, 200, 200, 202, 202, 202), 202) - test_status_map((200, 200, 200, 200, 200, 202, 202, 500), 202) - test_status_map((200, 200, 200, 200, 200, 202, 500, 500), 503) - test_status_map((200, 200, 200, 200, 200, 202, 404, 500), 503) - test_status_map((200, 200, 200, 200, 200, 202, 404, 404), 404) - test_status_map((200, 200, 200, 200, 200, 404, 500, 500), 503) - test_status_map((200, 200, 200, 200, 200, 404, 404, 404), 404) - def test_DELETE(self): with save_globals(): def test_status_map(statuses, expected): @@ -3611,26 +3531,6 @@ class TestObjectController(unittest.TestCase): res = req.get_response(self.app) self.assertEqual(res.status_int, 400) - def test_POST_as_copy_meta_val_len(self): - with save_globals(): - limit = constraints.MAX_META_VALUE_LENGTH - set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) - # acct cont objc objc objc obj obj obj - req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, - headers={'Content-Type': 'foo/bar', - 'X-Object-Meta-Foo': 'x' * limit}) - self.app.update_request(req) - res = req.get_response(self.app) - self.assertEqual(res.status_int, 202) - set_http_connect(202, 202, 202) - req = Request.blank( - '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, - headers={'Content-Type': 'foo/bar', - 'X-Object-Meta-Foo': 'x' * (limit + 1)}) - self.app.update_request(req) - res = req.get_response(self.app) - self.assertEqual(res.status_int, 400) - def test_POST_meta_key_len(self): with save_globals(): limit = constraints.MAX_META_NAME_LENGTH @@ -3653,27 +3553,6 @@ class TestObjectController(unittest.TestCase): res = req.get_response(self.app) self.assertEqual(res.status_int, 400) - def test_POST_as_copy_meta_key_len(self): - with save_globals(): - limit = constraints.MAX_META_NAME_LENGTH - set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) - # acct cont objc objc objc obj obj obj - req = Request.blank( - '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, - headers={'Content-Type': 'foo/bar', - ('X-Object-Meta-' + 'x' * limit): 'x'}) - self.app.update_request(req) - res = req.get_response(self.app) - self.assertEqual(res.status_int, 202) - set_http_connect(202, 202, 202) - req = Request.blank( - '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, - headers={'Content-Type': 'foo/bar', - ('X-Object-Meta-' + 'x' * (limit + 1)): 'x'}) - self.app.update_request(req) - res = req.get_response(self.app) - self.assertEqual(res.status_int, 400) - def test_POST_meta_count(self): with save_globals(): limit = constraints.MAX_META_COUNT @@ -4419,25 +4298,6 @@ class TestObjectController(unittest.TestCase): resp = controller.POST(req) self.assertEqual(resp.status_int, 404) - def test_PUT_POST_as_copy_requires_container_exist(self): - with save_globals(): - self.app.memcache = FakeMemcacheReturnsNone() - controller = ReplicatedObjectController( - self.app, 'account', 'container', 'object') - set_http_connect(200, 404, 404, 404, 200, 200, 200) - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) - self.app.update_request(req) - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 404) - - set_http_connect(200, 404, 404, 404, 200, 200, 200, 200, 200, 200) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'POST'}, - headers={'Content-Type': 'text/plain'}) - self.app.update_request(req) - resp = controller.POST(req) - self.assertEqual(resp.status_int, 404) - def test_bad_metadata(self): with save_globals(): controller = ReplicatedObjectController( @@ -4554,755 +4414,6 @@ class TestObjectController(unittest.TestCase): raise self.fail('UN-USED STATUS CODES: %r' % unused_status_list) - def test_basic_put_with_x_copy_from(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - - def test_basic_put_with_x_copy_from_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acc1 con1 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_basic_put_with_x_copy_from_across_container(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c2/o'}) - status_list = (200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont conc objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c2/o') - - def test_basic_put_with_x_copy_from_across_container_and_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c2/o', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acc1 con1 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c2/o') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_copy_non_zero_content_length(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '5', - 'X-Copy-From': 'c/o'}) - status_list = (200, 200) - # acct cont - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 400) - - def test_copy_non_zero_content_length_with_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '5', - 'X-Copy-From': 'c/o', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200) - # acct cont - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 400) - - def test_copy_with_slashes_in_x_copy_from(self): - # extra source path parsing - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o/o2'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - - def test_copy_with_slashes_in_x_copy_from_and_account(self): - # extra source path parsing - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o/o2', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acc1 con1 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_copy_with_spaces_in_x_copy_from(self): - # space in soure path - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o%20o2'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o%20o2') - - def test_copy_with_spaces_in_x_copy_from_and_account(self): - # space in soure path - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o%20o2', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acc1 con1 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o%20o2') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_copy_with_leading_slash_in_x_copy_from(self): - # repeat tests with leading / - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - - def test_copy_with_leading_slash_in_x_copy_from_and_account(self): - # repeat tests with leading / - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acc1 con1 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_copy_with_leading_slash_and_slashes_in_x_copy_from(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o/o2'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - - def test_copy_with_leading_slash_and_slashes_in_x_copy_from_acct(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o/o2', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acc1 con1 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_copy_with_no_object_in_x_copy_from(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c'}) - status_list = (200, 200) - # acct cont - with self.controller_context(req, *status_list) as controller: - try: - controller.PUT(req) - except HTTPException as resp: - self.assertEqual(resp.status_int // 100, 4) # client error - else: - raise self.fail('Invalid X-Copy-From did not raise ' - 'client error') - - def test_copy_with_no_object_in_x_copy_from_and_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200) - # acct cont - with self.controller_context(req, *status_list) as controller: - try: - controller.PUT(req) - except HTTPException as resp: - self.assertEqual(resp.status_int // 100, 4) # client error - else: - raise self.fail('Invalid X-Copy-From did not raise ' - 'client error') - - def test_copy_server_error_reading_source(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - status_list = (200, 200, 503, 503, 503) - # acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 503) - - def test_copy_server_error_reading_source_and_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 503, 503, 503) - # acct cont acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 503) - - def test_copy_not_found_reading_source(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - # not found - status_list = (200, 200, 404, 404, 404) - # acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 404) - - def test_copy_not_found_reading_source_and_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Copy-From-Account': 'a'}) - # not found - status_list = (200, 200, 200, 200, 404, 404, 404) - # acct cont acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 404) - - def test_copy_with_some_missing_sources(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - status_list = (200, 200, 404, 404, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - - def test_copy_with_some_missing_sources_and_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Copy-From-Account': 'a'}) - status_list = (200, 200, 200, 200, 404, 404, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - - def test_copy_with_object_metadata(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Object-Meta-Ours': 'okay'}) - # test object metadata - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers.get('x-object-meta-test'), 'testing') - self.assertEqual(resp.headers.get('x-object-meta-ours'), 'okay') - self.assertEqual(resp.headers.get('x-delete-at'), '9876543210') - - def test_copy_with_object_metadata_and_account(self): - req = Request.blank('/v1/a1/c1/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Object-Meta-Ours': 'okay', - 'X-Copy-From-Account': 'a'}) - # test object metadata - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.PUT(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers.get('x-object-meta-test'), 'testing') - self.assertEqual(resp.headers.get('x-object-meta-ours'), 'okay') - self.assertEqual(resp.headers.get('x-delete-at'), '9876543210') - - @_limit_max_file_size - def test_copy_source_larger_than_max_file_size(self): - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - # copy-from object is too large to fit in target object - - class LargeResponseBody(object): - - def __len__(self): - return constraints.MAX_FILE_SIZE + 1 - - def __getitem__(self, key): - return '' - - copy_from_obj_body = LargeResponseBody() - status_list = (200, 200, 200, 200, 200) - # acct cont objc objc objc - kwargs = dict(body=copy_from_obj_body) - with self.controller_context(req, *status_list, - **kwargs) as controller: - self.app.update_request(req) - - self.app.memcache.store = {} - try: - resp = controller.PUT(req) - except HTTPException as resp: - pass - self.assertEqual(resp.status_int, 413) - - def test_basic_COPY(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c/o2'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - - def test_basic_COPY_account(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c1/o2', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_COPY_across_containers(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c2/o'}) - status_list = (200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont c2 objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - - def test_COPY_source_with_slashes_in_name(self): - req = Request.blank('/v1/a/c/o/o2', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c/o'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - - def test_COPY_account_source_with_slashes_in_name(self): - req = Request.blank('/v1/a/c/o/o2', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c1/o', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_COPY_destination_leading_slash(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - - def test_COPY_account_destination_leading_slash(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_COPY_source_with_slashes_destination_leading_slash(self): - req = Request.blank('/v1/a/c/o/o2', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - - def test_COPY_account_source_with_slashes_destination_leading_slash(self): - req = Request.blank('/v1/a/c/o/o2', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from'], 'c/o/o2') - self.assertEqual(resp.headers['x-copied-from-account'], 'a') - - def test_COPY_no_object_in_destination(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c_o'}) - status_list = [] # no requests needed - with self.controller_context(req, *status_list) as controller: - self.assertRaises(HTTPException, controller.COPY, req) - - def test_COPY_account_no_object_in_destination(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c_o', - 'Destination-Account': 'a1'}) - status_list = [] # no requests needed - with self.controller_context(req, *status_list) as controller: - self.assertRaises(HTTPException, controller.COPY, req) - - def test_COPY_server_error_reading_source(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - status_list = (200, 200, 503, 503, 503) - # acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 503) - - def test_COPY_account_server_error_reading_source(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 503, 503, 503) - # acct cont acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 503) - - def test_COPY_not_found_reading_source(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - status_list = (200, 200, 404, 404, 404) - # acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 404) - - def test_COPY_account_not_found_reading_source(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 404, 404, 404) - # acct cont acct cont objc objc objc - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 404) - - def test_COPY_with_some_missing_sources(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - status_list = (200, 200, 404, 404, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - - def test_COPY_account_with_some_missing_sources(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 404, 404, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - - def test_COPY_with_metadata(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o', - 'X-Object-Meta-Ours': 'okay'}) - status_list = (200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers.get('x-object-meta-test'), - 'testing') - self.assertEqual(resp.headers.get('x-object-meta-ours'), 'okay') - self.assertEqual(resp.headers.get('x-delete-at'), '9876543210') - - def test_COPY_account_with_metadata(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'X-Object-Meta-Ours': 'okay', - 'Destination-Account': 'a1'}) - status_list = (200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - with self.controller_context(req, *status_list) as controller: - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers.get('x-object-meta-test'), - 'testing') - self.assertEqual(resp.headers.get('x-object-meta-ours'), 'okay') - self.assertEqual(resp.headers.get('x-delete-at'), '9876543210') - - @_limit_max_file_size - def test_COPY_source_larger_than_max_file_size(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - - class LargeResponseBody(object): - - def __len__(self): - return constraints.MAX_FILE_SIZE + 1 - - def __getitem__(self, key): - return '' - - copy_from_obj_body = LargeResponseBody() - status_list = (200, 200, 200, 200, 200) - # acct cont objc objc objc - kwargs = dict(body=copy_from_obj_body) - with self.controller_context(req, *status_list, - **kwargs) as controller: - try: - resp = controller.COPY(req) - except HTTPException as resp: - pass - self.assertEqual(resp.status_int, 413) - - @_limit_max_file_size - def test_COPY_account_source_larger_than_max_file_size(self): - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - - class LargeResponseBody(object): - - def __len__(self): - return constraints.MAX_FILE_SIZE + 1 - - def __getitem__(self, key): - return '' - - copy_from_obj_body = LargeResponseBody() - status_list = (200, 200, 200, 200, 200) - # acct cont objc objc objc - kwargs = dict(body=copy_from_obj_body) - with self.controller_context(req, *status_list, - **kwargs) as controller: - try: - resp = controller.COPY(req) - except HTTPException as resp: - pass - self.assertEqual(resp.status_int, 413) - - def test_COPY_newest(self): - with save_globals(): - controller = ReplicatedObjectController( - self.app, 'a', 'c', 'o') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, - # act cont objc objc objc obj obj obj - timestamps=('1', '1', '1', '3', '2', '4', '4', - '4')) - self.app.memcache.store = {} - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from-last-modified'], - '3') - - def test_COPY_account_newest(self): - with save_globals(): - controller = ReplicatedObjectController( - self.app, 'a', 'c', 'o') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201, - # act cont acct cont objc objc objc obj obj obj - timestamps=('1', '1', '1', '1', '3', '2', '1', - '4', '4', '4')) - self.app.memcache.store = {} - resp = controller.COPY(req) - self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.headers['x-copied-from-last-modified'], - '3') - - def test_COPY_delete_at(self): - with save_globals(): - backend_requests = [] - - def capture_requests(ipaddr, port, device, partition, method, path, - headers=None, query_string=None): - backend_requests.append((method, path, headers)) - - controller = ReplicatedObjectController( - self.app, 'a', 'c', 'o') - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, - give_connect=capture_requests) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - - self.app.update_request(req) - resp = controller.COPY(req) - self.assertEqual(201, resp.status_int) # sanity - for method, path, given_headers in backend_requests: - if method != 'PUT': - continue - self.assertEqual(given_headers.get('X-Delete-At'), - '9876543210') - self.assertTrue('X-Delete-At-Host' in given_headers) - self.assertTrue('X-Delete-At-Device' in given_headers) - self.assertTrue('X-Delete-At-Partition' in given_headers) - self.assertTrue('X-Delete-At-Container' in given_headers) - - def test_COPY_account_delete_at(self): - with save_globals(): - backend_requests = [] - - def capture_requests(ipaddr, port, device, partition, method, path, - headers=None, query_string=None): - backend_requests.append((method, path, headers)) - - controller = ReplicatedObjectController( - self.app, 'a', 'c', 'o') - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201, - give_connect=capture_requests) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c1/o', - 'Destination-Account': 'a1'}) - - self.app.update_request(req) - resp = controller.COPY(req) - self.assertEqual(201, resp.status_int) # sanity - for method, path, given_headers in backend_requests: - if method != 'PUT': - continue - self.assertEqual(given_headers.get('X-Delete-At'), - '9876543210') - self.assertTrue('X-Delete-At-Host' in given_headers) - self.assertTrue('X-Delete-At-Device' in given_headers) - self.assertTrue('X-Delete-At-Partition' in given_headers) - self.assertTrue('X-Delete-At-Container' in given_headers) - - def test_chunked_put(self): - - class ChunkedFile(object): - - def __init__(self, bytes): - self.bytes = bytes - self.read_bytes = 0 - - @property - def bytes_left(self): - return self.bytes - self.read_bytes - - def read(self, amt=None): - if self.read_bytes >= self.bytes: - raise StopIteration() - if not amt: - amt = self.bytes_left - data = 'a' * min(amt, self.bytes_left) - self.read_bytes += len(data) - return data - - with save_globals(): - set_http_connect(201, 201, 201, 201) - controller = ReplicatedObjectController( - self.app, 'account', 'container', 'object') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Transfer-Encoding': 'chunked', - 'Content-Type': 'foo/bar'}) - - req.body_file = ChunkedFile(10) - self.app.memcache.store = {} - self.app.update_request(req) - res = controller.PUT(req) - self.assertEqual(res.status_int // 100, 2) # success - - # test 413 entity to large - set_http_connect(201, 201, 201, 201) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Transfer-Encoding': 'chunked', - 'Content-Type': 'foo/bar'}) - req.body_file = ChunkedFile(11) - self.app.memcache.store = {} - self.app.update_request(req) - - with mock.patch('swift.common.constraints.MAX_FILE_SIZE', 10): - res = controller.PUT(req) - self.assertEqual(res.status_int, 413) - @unpatch_policies def test_chunked_put_bad_version(self): # Check bad version @@ -5720,24 +4831,6 @@ class TestObjectController(unittest.TestCase): controller.POST(req) self.assertTrue(called[0]) - def test_POST_as_copy_calls_authorize(self): - called = [False] - - def authorize(req): - called[0] = True - return HTTPUnauthorized(request=req) - with save_globals(): - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - controller = ReplicatedObjectController( - self.app, 'account', 'container', 'object') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'POST'}, - headers={'Content-Length': '5'}, body='12345') - req.environ['swift.authorize'] = authorize - self.app.update_request(req) - controller.POST(req) - self.assertTrue(called[0]) - def test_PUT_calls_authorize(self): called = [False] @@ -5755,24 +4848,6 @@ class TestObjectController(unittest.TestCase): controller.PUT(req) self.assertTrue(called[0]) - def test_COPY_calls_authorize(self): - called = [False] - - def authorize(req): - called[0] = True - return HTTPUnauthorized(request=req) - with save_globals(): - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - controller = ReplicatedObjectController( - self.app, 'account', 'container', 'object') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c/o'}) - req.environ['swift.authorize'] = authorize - self.app.update_request(req) - controller.COPY(req) - self.assertTrue(called[0]) - def test_POST_converts_delete_after_to_delete_at(self): with save_globals(): self.app.object_post_as_copy = False @@ -6021,12 +5096,12 @@ class TestObjectController(unittest.TestCase): self.assertEqual( 'https://foo.bar', resp.headers['access-control-allow-origin']) - for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split(): + for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) self.assertEqual( len(resp.headers['access-control-allow-methods'].split(', ')), - 7) + 6) self.assertEqual('999', resp.headers['access-control-max-age']) req = Request.blank( '/v1/a/c/o.jpg', @@ -6039,10 +5114,10 @@ class TestObjectController(unittest.TestCase): req.content_length = 0 resp = controller.OPTIONS(req) self.assertEqual(200, resp.status_int) - for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split(): + for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['Allow']) - self.assertEqual(len(resp.headers['Allow'].split(', ')), 7) + self.assertEqual(len(resp.headers['Allow'].split(', ')), 6) req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'}, @@ -6075,12 +5150,12 @@ class TestObjectController(unittest.TestCase): resp = controller.OPTIONS(req) self.assertEqual(200, resp.status_int) self.assertEqual('*', resp.headers['access-control-allow-origin']) - for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split(): + for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) self.assertEqual( len(resp.headers['access-control-allow-methods'].split(', ')), - 7) + 6) self.assertEqual('999', resp.headers['access-control-max-age']) def _get_CORS_response(self, container_cors, strict_mode, object_get=None): @@ -9232,9 +8307,10 @@ class TestSocketObjectVersions(unittest.TestCase): conf = {'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'allowed_headers': allowed_headers} prosrv = versioned_writes.VersionedWritesMiddleware( - proxy_logging.ProxyLoggingMiddleware( - _test_servers[0], conf, - logger=_test_servers[0].logger), + copy.ServerSideCopyMiddleware( + proxy_logging.ProxyLoggingMiddleware( + _test_servers[0], conf, + logger=_test_servers[0].logger), conf), {}) self.coro = spawn(wsgi.server, prolis, prosrv, NullLogger()) # replace global prosrv with one that's filtered with version diff --git a/test/unit/proxy/test_sysmeta.py b/test/unit/proxy/test_sysmeta.py index 3b3f8ddfd9..9548680791 100644 --- a/test/unit/proxy/test_sysmeta.py +++ b/test/unit/proxy/test_sysmeta.py @@ -283,94 +283,3 @@ class TestObjectSysmeta(unittest.TestCase): self._assertInHeaders(resp, self.changed_sysmeta_headers) self._assertInHeaders(resp, self.new_sysmeta_headers) self._assertNotInHeaders(resp, self.original_sysmeta_headers_2) - - def test_sysmeta_not_updated_by_POST(self): - self.app.object_post_as_copy = False - self._test_sysmeta_not_updated_by_POST() - - def test_sysmeta_not_updated_by_POST_as_copy(self): - self.app.object_post_as_copy = True - self._test_sysmeta_not_updated_by_POST() - - def test_sysmeta_updated_by_COPY(self): - # check sysmeta is updated by a COPY in same way as user meta - path = '/v1/a/c/o' - dest = '/c/o2' - env = {'REQUEST_METHOD': 'PUT'} - hdrs = dict(self.original_sysmeta_headers_1) - hdrs.update(self.original_sysmeta_headers_2) - hdrs.update(self.original_meta_headers_1) - hdrs.update(self.original_meta_headers_2) - req = Request.blank(path, environ=env, headers=hdrs, body='x') - resp = req.get_response(self.app) - self._assertStatus(resp, 201) - - env = {'REQUEST_METHOD': 'COPY'} - hdrs = dict(self.changed_sysmeta_headers) - hdrs.update(self.new_sysmeta_headers) - hdrs.update(self.changed_meta_headers) - hdrs.update(self.new_meta_headers) - hdrs.update(self.bad_headers) - hdrs.update({'Destination': dest}) - req = Request.blank(path, environ=env, headers=hdrs) - resp = req.get_response(self.app) - self._assertStatus(resp, 201) - self._assertInHeaders(resp, self.changed_sysmeta_headers) - self._assertInHeaders(resp, self.new_sysmeta_headers) - self._assertInHeaders(resp, self.original_sysmeta_headers_2) - self._assertInHeaders(resp, self.changed_meta_headers) - self._assertInHeaders(resp, self.new_meta_headers) - self._assertInHeaders(resp, self.original_meta_headers_2) - self._assertNotInHeaders(resp, self.bad_headers) - - req = Request.blank('/v1/a/c/o2', environ={}) - resp = req.get_response(self.app) - self._assertStatus(resp, 200) - self._assertInHeaders(resp, self.changed_sysmeta_headers) - self._assertInHeaders(resp, self.new_sysmeta_headers) - self._assertInHeaders(resp, self.original_sysmeta_headers_2) - self._assertInHeaders(resp, self.changed_meta_headers) - self._assertInHeaders(resp, self.new_meta_headers) - self._assertInHeaders(resp, self.original_meta_headers_2) - self._assertNotInHeaders(resp, self.bad_headers) - - def test_sysmeta_updated_by_COPY_from(self): - # check sysmeta is updated by a COPY in same way as user meta - path = '/v1/a/c/o' - env = {'REQUEST_METHOD': 'PUT'} - hdrs = dict(self.original_sysmeta_headers_1) - hdrs.update(self.original_sysmeta_headers_2) - hdrs.update(self.original_meta_headers_1) - hdrs.update(self.original_meta_headers_2) - req = Request.blank(path, environ=env, headers=hdrs, body='x') - resp = req.get_response(self.app) - self._assertStatus(resp, 201) - - env = {'REQUEST_METHOD': 'PUT'} - hdrs = dict(self.changed_sysmeta_headers) - hdrs.update(self.new_sysmeta_headers) - hdrs.update(self.changed_meta_headers) - hdrs.update(self.new_meta_headers) - hdrs.update(self.bad_headers) - hdrs.update({'X-Copy-From': '/c/o'}) - req = Request.blank('/v1/a/c/o2', environ=env, headers=hdrs, body='') - resp = req.get_response(self.app) - self._assertStatus(resp, 201) - self._assertInHeaders(resp, self.changed_sysmeta_headers) - self._assertInHeaders(resp, self.new_sysmeta_headers) - self._assertInHeaders(resp, self.original_sysmeta_headers_2) - self._assertInHeaders(resp, self.changed_meta_headers) - self._assertInHeaders(resp, self.new_meta_headers) - self._assertInHeaders(resp, self.original_meta_headers_2) - self._assertNotInHeaders(resp, self.bad_headers) - - req = Request.blank('/v1/a/c/o2', environ={}) - resp = req.get_response(self.app) - self._assertStatus(resp, 200) - self._assertInHeaders(resp, self.changed_sysmeta_headers) - self._assertInHeaders(resp, self.new_sysmeta_headers) - self._assertInHeaders(resp, self.original_sysmeta_headers_2) - self._assertInHeaders(resp, self.changed_meta_headers) - self._assertInHeaders(resp, self.new_meta_headers) - self._assertInHeaders(resp, self.original_meta_headers_2) - self._assertNotInHeaders(resp, self.bad_headers)