diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml index c8dc8ae7d5..8ab65f58ea 100644 --- a/api-ref/source/parameters.yaml +++ b/api-ref/source/parameters.yaml @@ -164,7 +164,7 @@ ETag_obj_req: manifest objects, this value is the MD5 checksum of the concatenated string of ETag values for each of the segments in the manifest. You are strongly recommended to compute - the MD5 checksum value and include it in the request. This + the MD5 checksum value and include it in the request. This enables the Object Storage API to check the integrity of the upload. The value is not quoted. in: header @@ -850,6 +850,44 @@ X-Storage-Policy: in: header required: false type: string +X-Symlink-Target: + description: | + Set to specify that this is a symlink object. + The value is the relative path of the target object in the + format /. The target object does not need to + exist at the time of symlink creation. + You must UTF-8-encode and then URL-encode the names of the + container and object before you include them in this header. + in: header + required: false + type: string +X-Symlink-Target-Account: + description: | + Set to specify that this is a cross-account symlink to + an object in the account specified in the value. + The ``X-Symlink-Target`` must also be set for this to + be effective. + You must UTF-8-encode and then URL-encode the account name + before you include it in this header. + in: header + required: false + type: string +X-Symlink-Target-Account_resp: + description: | + If present, and ``X-Symlink-Target`` is present, then + this is a cross-account symlink to + an object in the account specified in the value. + in: header + required: false + type: string +X-Symlink-Target_resp: + description: | + If present, this is a symlink object. + The value is the relative path of the target object in the + format /. + in: header + required: false + type: string X-Timestamp: description: | The date and time in `UNIX Epoch time stamp @@ -1092,6 +1130,23 @@ swiftinfo_sig: in: query required: false type: string +symlink: + description: | + If you include the ``symlink=get`` query parameter + and the object is a symlink, then the response will include + data and metadata from the symlink itself rather than from the target. + in: query + required: false + type: string +symlink_copy: + description: | + If you include the ``symlink=get`` query parameter + and the object is a symlink, the target object + contents are not copied. Instead, the symlink is copied to + create a new symlink to the same target. + in: query + required: false + type: string temp_url_expires: description: | The date and time in `UNIX Epoch time stamp @@ -1180,5 +1235,12 @@ name_in_container_get: in: body required: true type: string +symlink_path: + description: | + This field exists only when the object is symlink. + This is the target path of the symlink object. + in: body + required: true + type: string diff --git a/api-ref/source/storage-container-services.inc b/api-ref/source/storage-container-services.inc index a66f85fe40..a1351275a0 100644 --- a/api-ref/source/storage-container-services.inc +++ b/api-ref/source/storage-container-services.inc @@ -96,6 +96,7 @@ Response Parameters - content_type: content_type - bytes: bytes_in_container_get - name: name_in_container_get + - symlink_path: symlink_path Response Example format=json diff --git a/api-ref/source/storage-object-services.inc b/api-ref/source/storage-object-services.inc index bb6833e31f..4c2b45b38f 100644 --- a/api-ref/source/storage-object-services.inc +++ b/api-ref/source/storage-object-services.inc @@ -111,6 +111,7 @@ Request - temp_url_expires: temp_url_expires - filename: filename - multipart-manifest: multipart-manifest_get + - symlink: symlink - Range: Range - If-Match: If-Match - If-None-Match: If-None-Match-get-request @@ -139,7 +140,8 @@ Response Parameters - X-Openstack-Request-Id: X-Openstack-Request-Id - Date: Date - X-Static-Large-Object: X-Static-Large-Object - + - X-Symlink-Target: X-Symlink-Target_resp + - X-Symlink-Target-Account: X-Symlink-Target-Account_resp Response Example @@ -263,6 +265,8 @@ Request - X-Object-Meta-name: X-Object-Meta-name - If-None-Match: If-None-Match-put-request - X-Trans-Id-Extra: X-Trans-Id-Extra + - X-Symlink-Target: X-Symlink-Target + - X-Symlink-Target-Account: X-Symlink-Target-Account Response Parameters @@ -321,6 +325,12 @@ The new object contains the same manifest as the original. The segment objects are not copied. Instead, both the original and new manifest objects share the same set of segment objects. +To copy a symlink either with a COPY or a PUT with the +``X-Copy-From`` request, include the ``symlink=get`` query string. +The new symlink will have the same target as the original. +The target object is not copied. Instead, both the original +and new symlinks point to the same target object. + All metadata is preserved during the object copy. If you specify metadata on the request to copy the object, either PUT or COPY , the metadata @@ -396,6 +406,7 @@ Request - container: container - object: object - multipart-manifest: multipart-manifest_copy + - symlink: symlink_copy - X-Auth-Token: X-Auth-Token - X-Service-Token: X-Service-Token - Destination: Destination @@ -445,6 +456,9 @@ manifest=delete`` query parameter. This operation deletes the segment objects and, if all deletions succeed, this operation deletes the manifest object. +A DELETE request made to a symlink path will delete the symlink +rather than the target object. + An alternative to using the DELETE operation is to use the POST operation with the ``bulk-delete`` query parameter. @@ -570,6 +584,7 @@ Request - temp_url_expires: temp_url_expires - filename: filename - multipart-manifest: multipart-manifest_head + - symlink: symlink - X-Newest: X-Newest - If-Match: If-Match - If-None-Match: If-None-Match-get-request @@ -597,7 +612,8 @@ Response Parameters - Date: Date - X-Static-Large-Object: X-Static-Large-Object - Content-Type: Content-Type_obj_resp - + - X-Symlink-Target: X-Symlink-Target_resp + - X-Symlink-Target-Account: X-Symlink-Target-Account_resp Response Example @@ -659,6 +675,15 @@ body. There are alternate uses of the POST operation as follows: can be used to upload an archive (tar file). The archive is then extracted to create objects. +A POST request must not include X-Symlink-Target header. If it does then a +400 status code is returned and the object metadata is not modified. + +When a POST request is sent to a symlink, the metadata will be applied to the +symlink, but the request will result in a ``307 Temporary Redirect`` response +to the client. The POST is never redirected to the target object, thus a +GET/HEAD request to the symlink without ``symlink=get`` will not return the +metadata that was sent as part of the POST request. + Example requests and responses: - Create object metadata: diff --git a/doc/saio/swift/proxy-server.conf b/doc/saio/swift/proxy-server.conf index 12b0386840..869eff871d 100644 --- a/doc/saio/swift/proxy-server.conf +++ b/doc/saio/swift/proxy-server.conf @@ -9,7 +9,7 @@ eventlet_debug = true [pipeline:main] # Yes, proxy-logging appears twice. This is so that # middleware-originated requests get logged too. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache listing_formats bulk tempurl ratelimit crossdomain container_sync tempauth staticweb copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache listing_formats bulk tempurl ratelimit crossdomain container_sync tempauth staticweb copy container-quotas account-quotas slo dlo versioned_writes symlink proxy-logging proxy-server [filter:catch_errors] use = egg:swift#catch_errors @@ -74,6 +74,9 @@ use = egg:swift#copy [filter:listing_formats] use = egg:swift#listing_formats +[filter:symlink] +use = egg:swift#symlink + [app:proxy-server] use = egg:swift#proxy allow_account_management = true diff --git a/doc/source/api/object_api_v1_overview.rst b/doc/source/api/object_api_v1_overview.rst index 7f8571bc87..099fbd1db0 100644 --- a/doc/source/api/object_api_v1_overview.rst +++ b/doc/source/api/object_api_v1_overview.rst @@ -87,7 +87,9 @@ The Object Storage system organizes data in a hierarchy, as follows: object. - Upload objects directly to the Object Storage system from a - browser by using form **POST** middleware + browser by using form **POST** middleware. + + - Create symbolic links to other objects. The account, container, and object hierarchy affects the way you interact with the Object Storage API. diff --git a/doc/source/cors.rst b/doc/source/cors.rst index e8b07819d8..89e1f715a3 100644 --- a/doc/source/cors.rst +++ b/doc/source/cors.rst @@ -49,6 +49,12 @@ returns the following values for this header, ``X-Object-Meta-*`` for objects) * headers listed in ``X-Container-Meta-Access-Control-Expose-Headers`` +.. note:: + An OPTIONS request to a symlink object will respond with the options for + the symlink only, the request will not be redirected to the target object. + Therefore, if the symlink's target object is in another container with + cors settings, the response will not reflect the settings. + ----------------- Sample Javascript diff --git a/doc/source/logs.rst b/doc/source/logs.rst index 7e2c1dd94b..f9a8ba2c62 100644 --- a/doc/source/logs.rst +++ b/doc/source/logs.rst @@ -104,6 +104,7 @@ KS :ref:`keystoneauth` RL :ref:`ratelimit` VW :ref:`versioned_writes` SSC :ref:`copy` +SYM :ref:`symlink` ======================= ============================= diff --git a/doc/source/middleware.rst b/doc/source/middleware.rst index 5a57f21a1c..faeb93b67f 100644 --- a/doc/source/middleware.rst +++ b/doc/source/middleware.rst @@ -244,6 +244,15 @@ StaticWeb :members: :show-inheritance: +.. _symlink: + +Symlink +======= + +.. automodule:: swift.common.middleware.symlink + :members: + :show-inheritance: + .. _common_tempauth: TempAuth diff --git a/doc/source/overview_container_sync.rst b/doc/source/overview_container_sync.rst index 45a30b42ba..a790e46d4b 100644 --- a/doc/source/overview_container_sync.rst +++ b/doc/source/overview_container_sync.rst @@ -36,9 +36,15 @@ synchronization key. .. note:: If you are using encryption middleware in the cluster from which objects - are being synced, then you should follow the instructions to configure + are being synced, then you should follow the instructions for :ref:`container_sync_client_config` to be compatible with encryption. +.. note:: + + If you are using symlink middleware in the cluster from which objects + are being synced, then you should follow the instructions for + :ref:`symlink_container_sync_client_config` to be compatible with symlinks. + -------------------------- Configuring Container Sync -------------------------- @@ -440,7 +446,7 @@ then a symlink to the container database is created in a sync-containers sub-directory on the same device. Similarly, when the container sync metadata keys are deleted, the container -server and container-replicator would take care of deleting the symlinks +server and container-replicator would take care of deleting the symlinks from ``sync-containers``. .. note:: diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index aaa5ab3725..47da8035fc 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -94,12 +94,12 @@ bind_port = 8080 [pipeline:main] # This sample pipeline uses tempauth and is used for SAIO dev work and # testing. See below for a pipeline using keystone. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache listing_formats container_sync bulk tempurl ratelimit tempauth copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache listing_formats container_sync bulk tempurl ratelimit tempauth copy container-quotas account-quotas slo dlo versioned_writes symlink proxy-logging proxy-server # The following pipeline shows keystone integration. Comment out the one # above and uncomment this one. Additional steps for integrating keystone are # covered further below in the filter sections for authtoken and keystoneauth. -#pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit authtoken keystoneauth copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server +#pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit authtoken keystoneauth copy container-quotas account-quotas slo dlo versioned_writes symlink proxy-logging proxy-server [app:proxy-server] use = egg:swift#proxy @@ -928,3 +928,12 @@ use = egg:swift#encryption # be automatically inserted for you. [filter:listing_formats] use = egg:swift#listing_formats + +# Note: Put after slo, dlo, versioned_writes, but before encryption in the +# pipeline. +[filter:symlink] +use = egg:swift#symlink +# Symlinks can point to other symlinks up to the symloop_max value. If the +# number of chained symlinks exceeds the limit ``symloop_max`` a 409 +# (HTTPConflict) error response will be produced. +# symloop_max = 2 diff --git a/setup.cfg b/setup.cfg index 190b59abb0..945de78c5f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -106,6 +106,7 @@ paste.filter_factory = encryption = swift.common.middleware.crypto:filter_factory kms_keymaster = swift.common.middleware.crypto.kms_keymaster:filter_factory listing_formats = swift.common.middleware.listing_formats:filter_factory + symlink = swift.common.middleware.symlink:filter_factory [build_sphinx] all_files = 1 diff --git a/swift/common/exceptions.py b/swift/common/exceptions.py index 8eebf99e6c..319c0f3757 100644 --- a/swift/common/exceptions.py +++ b/swift/common/exceptions.py @@ -199,6 +199,10 @@ class SegmentError(SwiftException): pass +class LinkIterError(SwiftException): + pass + + class ReplicationException(Exception): pass diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py index 462491f8c0..529005fe5a 100644 --- a/swift/common/internal_client.py +++ b/swift/common/internal_client.py @@ -157,7 +157,8 @@ class InternalClient(object): 'auto_create_account_prefix', default='.') def make_request( - self, method, path, headers, acceptable_statuses, body_file=None): + self, method, path, headers, acceptable_statuses, body_file=None, + params=None): """Makes a request to Swift with retries. :param method: HTTP method of request. @@ -166,6 +167,8 @@ class InternalClient(object): :param acceptable_statuses: List of acceptable statuses for request. :param body_file: Body file to be passed along with request, defaults to None. + :param params: A dict of params to be set in request query string, + defaults to None. :returns: Response object on success. @@ -185,6 +188,8 @@ class InternalClient(object): if hasattr(body_file, 'seek'): body_file.seek(0) req.body_file = body_file + if params: + req.params = params try: resp = req.get_response(self.app) if resp.status_int in acceptable_statuses or \ @@ -606,14 +611,30 @@ class InternalClient(object): headers=headers) def get_object(self, account, container, obj, headers, - acceptable_statuses=(2,)): + acceptable_statuses=(2,), params=None): """ - Returns a 3-tuple (status, headers, iterator of object body) + Gets an object. + + :param account: The object's account. + :param container: The object's container. + :param obj: The object name. + :param headers: Headers to send with request, defaults to empty dict. + :param acceptable_statuses: List of status for valid responses, + defaults to (2,). + :param params: A dict of params to be set in request query string, + defaults to None. + + :raises UnexpectedResponse: Exception raised when requests fail + to get a response with an acceptable status + :raises Exception: Exception is raised when code fails in an + unexpected way. + :returns: A 3-tuple (status, headers, iterator of object body) """ headers = headers or {} path = self.make_path(account, container, obj) - resp = self.make_request('GET', path, headers, acceptable_statuses) + resp = self.make_request( + 'GET', path, headers, acceptable_statuses, params=params) return (resp.status_int, resp.headers, resp.app_iter) def iter_object_lines( @@ -697,7 +718,7 @@ class InternalClient(object): :param account: The object's account. :param container: The object's container. :param obj: The object. - :param headers: Headers to send with request, defaults ot empty dict. + :param headers: Headers to send with request, defaults to empty dict. :raises UnexpectedResponse: Exception raised when requests fail to get a response with an acceptable status diff --git a/swift/common/middleware/copy.py b/swift/common/middleware/copy.py index c3fe0a20f1..162d8a3224 100644 --- a/swift/common/middleware/copy.py +++ b/swift/common/middleware/copy.py @@ -114,45 +114,20 @@ greater than 5GB. """ -from six.moves.urllib.parse import quote, unquote +from six.moves.urllib.parse import quote -from swift.common import utils -from swift.common.utils import get_logger, \ - config_true_value, FileLikeIter, close_if_possible +from swift.common.utils import get_logger, config_true_value, FileLikeIter, \ + close_if_possible from swift.common.swob import Request, HTTPPreconditionFailed, \ HTTPRequestEntityTooLarge, HTTPBadRequest, HTTPException from swift.common.http import HTTP_MULTIPLE_CHOICES, is_success, HTTP_OK from swift.common.constraints import check_account_format, MAX_FILE_SIZE from swift.common.request_helpers import copy_header_subset, remove_items, \ - is_sys_meta, is_sys_or_user_meta, is_object_transient_sysmeta + is_sys_meta, is_sys_or_user_meta, is_object_transient_sysmeta, \ + check_path_header from swift.common.wsgi import WSGIContext, make_subrequest, load_app_config -def _check_path_header(req, name, length, error_msg): - """ - Validate that the value of path-like header is - well formatted. We assume the caller ensures that - specific header is present in req.headers. - - :param req: HTTP request object - :param name: header name - :param length: length of path segment check - :param error_msg: error message for client - :returns: A tuple with path parts according to length - :raise HTTPPreconditionFailed: if header value - is not well formatted. - """ - src_header = unquote(req.headers.get(name)) - if not src_header.startswith('/'): - src_header = '/' + src_header - try: - return utils.split_path(src_header, length, length, True) - except ValueError: - raise HTTPPreconditionFailed( - request=req, - body=error_msg) - - def _check_copy_from_header(req): """ Validate that the value from x-copy-from header is @@ -164,9 +139,9 @@ def _check_copy_from_header(req): :raise HTTPPreconditionFailed: if x-copy-from value is not well formatted. """ - return _check_path_header(req, 'X-Copy-From', 2, - 'X-Copy-From header must be of the form ' - '/') + return check_path_header(req, 'X-Copy-From', 2, + 'X-Copy-From header must be of the form ' + '/') def _check_destination_header(req): @@ -180,9 +155,9 @@ def _check_destination_header(req): :raise HTTPPreconditionFailed: if destination value is not well formatted. """ - return _check_path_header(req, 'Destination', 2, - 'Destination header must be of the form ' - '/') + return check_path_header(req, 'Destination', 2, + 'Destination header must be of the form ' + '/') def _copy_headers(src, dest): diff --git a/swift/common/middleware/crypto/decrypter.py b/swift/common/middleware/crypto/decrypter.py index c8e78a59e4..abc2ef7d04 100644 --- a/swift/common/middleware/crypto/decrypter.py +++ b/swift/common/middleware/crypto/decrypter.py @@ -103,13 +103,14 @@ class BaseDecrypterContext(CryptoWSGIContext): to be decrypted but crypto meta was not found. """ - value, crypto_meta = extract_crypto_meta(value) + extracted_value, crypto_meta = extract_crypto_meta(value) if crypto_meta: self.crypto.check_crypto_meta(crypto_meta) - value = self.decrypt_value(value, key, crypto_meta) + value = self.decrypt_value(extracted_value, key, crypto_meta) elif required: raise EncryptionException( "Missing crypto meta in value %s" % value) + return value def decrypt_value(self, value, key, crypto_meta): diff --git a/swift/common/middleware/listing_formats.py b/swift/common/middleware/listing_formats.py index 436507aa89..8bc8d9f42c 100644 --- a/swift/common/middleware/listing_formats.py +++ b/swift/common/middleware/listing_formats.py @@ -92,6 +92,11 @@ def container_to_xml(listing, base_name): 'last_modified'): SubElement(sub, field).text = six.text_type( record.pop(field)) + + if 'symlink_path' in record: + SubElement(sub, 'symlink_path').text = six.text_type( + record.pop('symlink_path')) + return tostring(doc, encoding='UTF-8').replace( "", '', 1) diff --git a/swift/common/middleware/symlink.py b/swift/common/middleware/symlink.py new file mode 100644 index 0000000000..a4dea37c95 --- /dev/null +++ b/swift/common/middleware/symlink.py @@ -0,0 +1,570 @@ +# Copyright (c) 2010-2017 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Symlink Middleware + +Symlinks are objects stored in Swift that contains a reference to another +object (hereinafter, this is called "target object"). They are analogous to +symbolic links in Unix-like operating systems. The existence of a symlink +object does not affect the target object in any way. An important use case is +to use a path in one container to access an object in a different container, +with a different policy. This allows policy cost/performance tradeoffs to be +made on individual objects. + +Clients create a Swift symlink by performing a zero-length PUT request +with the header ``X-Symlink-Target: /``. For a cross-account +symlink, the header ``X-Symlink-Target-Account: `` must be included. +If omitted, it is inserted automatically with the account of the symlink +object in the PUT request process. + +Symlinks must be zero-byte objects. Attempting to PUT a symlink +with a non-empty request body will result in a 400-series error. Also, POST +with X-Symlink-Target header always results in a 400-series error. The target +object need not exist at symlink-creation time. It is suggested to set the +``Content-Type`` of symlink objects to a distinct value such as +``application/symlink``. + +A GET/HEAD request to a symlink will resolve in a request to the target +object referenced by the symlink's ``X-Symlink-Target-Account`` and +``X-Symlink-Target`` headers. The response of the GET/HEAD request will contain +a ``Content-Location`` header with the path location of the target object. A +GET/HEAD request to a symlink with the query parameter ``?symlink=get`` will +resolve in the request targeting the symlink itself. + +A symlink can point to another symlink. Chained symlinks will be traversed +until target is not a symlink. If the number of chained symlinks exceeds the +limit ``symloop_max`` an error response will be produced. The value of +``symloop_max`` can be defined in the symlink config section of +proxy-server.conf. If not specified, the default ``symloop_max`` value is 2. If +a value less than 1 is specified, the default value will be used. + +A HEAD/GET request to a symlink object behaves as a normal HEAD/GET request +to the target object. Therefore issuing a HEAD request to the symlink will +return the target metadata, and issuing a GET request to the symlink will +return the data and metadata of the target object. Only when a GET/HEAD +request sent to a symlink object with the ``?symlink=get`` query string +will return the symlink metadata with empty body. + +A POST request to a symlink will result in a 307 TemporaryRedirect response. +The response will contain a ``Location`` header with the path of the target +object as the value. The request is never redirected to the target object by +Swift. Nevertheless, the metadata in the POST request will be applied to the +symlink because object servers cannot know for sure if the current object is a +symlink or not in eventual consistency. + +A DELETE request to a symlink will delete the symlink itself. The target +object will not be deleted. + +A COPY request, or a PUT request with a ``X-Copy-From`` header, to a symlink +will copy the target object. The same request to a symlink with the query +parameter ``?symlink=get`` will copy the symlink itself. + +An OPTIONS request to a symlink will respond with the options for the symlink +only, the request will not be redirected to the target object. Please note that +if the symlink's target object is in another container with cors settings, the +response will not reflect the settings. + +Tempurls can be used to GET/HEAD symlink objects, but PUT is not allowed and +will result in a 400-series error. The GET/HEAD tempurls honor the scope of +the tempurl key. Container tempurl will only work on symlinks where the target +container is the same as the symlink. In case a symlink targets an object +in a different container, a GET/HEAD request will result in a 401 Unauthorized +error. The account level tempurl will allow cross container symlinks. + +If a symlink object is overwritten while it is in a versioned container, the +symlink object itself is versioned, not the referenced object. + +A GET request to a container which contains symlinks will respond with +additional information ``symlink_path`` for each symlink objects. +``symlink_path`` information are target path strings of the symlinks. Clients +can differentiate symlinks and other objects by this function. + +Errors + +* PUT with the header ``X-Symlink-Target`` with non-zero Content-Length + will produce a 400 BadRequest error. + +* POST with the header ``X-Symlink-Target`` will produce a + 400 BadRequest error. + +* GET/HEAD traversing more than ``symloop_max`` chained symlinks will + produce a 409 Conflict error. + +* POSTs will produce a 307 TemporaryRedirect error. + +---------- +Deployment +---------- + +Symlinks are enabled by adding the `symlink` middleware to the proxy server +WSGI pipeline and including a corresponding filter configuration section in the +`proxy-server.conf` file. The `symlink` middleware should be placed after +`slo`, `dlo` and `versioned_writes` middleware, but before `encryption` +middleware in the pipeline. See the `proxy-server.conf-sample` file for further +details. :ref:`Additional steps ` are +required if the container sync feature is being used. + +.. note:: + + Once you have deployed `symlink` middleware in your pipeline, you should + neither remove the `symlink` middleware nor downgrade swift to a version + earlier than symlinks being supported. Doing so may result in unexpected + container listing results in addition to symlink objects behaving like a + normal object. + +.. _symlink_container_sync_client_config: + +Container sync configuration +---------------------------- + +If container sync is being used then the `symlink` middleware +must be added to the container sync internal client pipeline. The following +configuration steps are required: + +#. Create a custom internal client configuration file for container sync (if + one is not already in use) based on the sample file + `internal-client.conf-sample`. For example, copy + `internal-client.conf-sample` to `/etc/swift/container-sync-client.conf`. +#. Modify this file to include the `symlink` middleware in the pipeline in + the same way as described above for the proxy server. +#. Modify the container-sync section of all container server config files to + point to this internal client config file using the + ``internal_client_conf_path`` option. For example:: + + internal_client_conf_path = /etc/swift/container-sync-client.conf + +.. note:: + + These container sync configuration steps will be necessary for container + sync probe tests to pass if the `symlink` middleware is included in the + proxy pipeline of a test cluster. +""" + +import json +import os +from cgi import parse_header +from six.moves.urllib.parse import unquote + +from swift.common.utils import get_logger, register_swift_info, split_path, \ + MD5_OF_EMPTY_STRING, closing_if_possible +from swift.common.constraints import check_account_format +from swift.common.wsgi import WSGIContext, make_subrequest +from swift.common.request_helpers import get_sys_meta_prefix, \ + check_path_header +from swift.common.swob import Request, HTTPBadRequest, HTTPTemporaryRedirect, \ + HTTPException, HTTPConflict, HTTPPreconditionFailed +from swift.common.http import is_success +from swift.common.exceptions import LinkIterError +from swift.common.header_key_dict import HeaderKeyDict + +DEFAULT_SYMLOOP_MAX = 2 +# Header values for symlink target path strings will be quoted values. +TGT_OBJ_SYMLINK_HDR = 'x-symlink-target' +TGT_ACCT_SYMLINK_HDR = 'x-symlink-target-account' +TGT_OBJ_SYSMETA_SYMLINK_HDR = get_sys_meta_prefix('object') + 'symlink-target' +TGT_ACCT_SYSMETA_SYMLINK_HDR = \ + get_sys_meta_prefix('object') + 'symlink-target-account' + + +def _check_symlink_header(req): + """ + Validate that the value from x-symlink-target header is + well formatted. We assume the caller ensures that + x-symlink-target header is present in req.headers. + + :param req: HTTP request object + :raise: HTTPPreconditionFailed if x-symlink-target value + is not well formatted. + """ + # N.B. check_path_header doesn't assert the leading slash and + # copy middleware may accpet the format. In the symlink, API + # says apparently to use "container/object" format so add the + # validation fist, here. + if unquote(req.headers[TGT_OBJ_SYMLINK_HDR]).startswith('/'): + raise HTTPPreconditionFailed( + body='X-Symlink-Target header must be of the ' + 'form /', + request=req, content_type='text/plain') + + # check container and object format + container, obj = check_path_header( + req, TGT_OBJ_SYMLINK_HDR, 2, + 'X-Symlink-Target header must be of the ' + 'form /') + + # Check account format if it exists + account = check_account_format( + req, unquote(req.headers[TGT_ACCT_SYMLINK_HDR])) \ + if TGT_ACCT_SYMLINK_HDR in req.headers else None + + # Extract request path + _junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True) + + if not account: + account = req_acc + + # Check if symlink targets the symlink itself or not + if (account, container, obj) == (req_acc, req_cont, req_obj): + raise HTTPBadRequest( + body='Symlink cannot target itself', + request=req, content_type='text/plain') + + +def symlink_usermeta_to_sysmeta(headers): + """ + Helper fucntion to translate from X-Symlink-Target and + X-Symlink-Target-Account to X-Object-Sysmeta-Symlink-Target + and X-Object-Sysmeta-Symlink-Target-Account. + + :param headers: request headers dict. Note that the headers dict + will be updated directly. + """ + # To preseve url-encoded value in the symlink header, use raw value + if TGT_OBJ_SYMLINK_HDR in headers: + headers[TGT_OBJ_SYSMETA_SYMLINK_HDR] = headers.pop( + TGT_OBJ_SYMLINK_HDR) + + if TGT_ACCT_SYMLINK_HDR in headers: + headers[TGT_ACCT_SYSMETA_SYMLINK_HDR] = headers.pop( + TGT_ACCT_SYMLINK_HDR) + + +def symlink_sysmeta_to_usermeta(headers): + """ + Helper fucntion to translate from X-Object-Sysmeta-Symlink-Target and + X-Object-Sysmeta-Symlink-Target-Account to X-Symlink-Target and + X-Sysmeta-Symlink-Target-Account + + :param headers: request headers dict. Note that the headers dict + will be updated directly. + """ + if TGT_OBJ_SYSMETA_SYMLINK_HDR in headers: + headers[TGT_OBJ_SYMLINK_HDR] = headers.pop( + TGT_OBJ_SYSMETA_SYMLINK_HDR) + + if TGT_ACCT_SYSMETA_SYMLINK_HDR in headers: + headers[TGT_ACCT_SYMLINK_HDR] = headers.pop( + TGT_ACCT_SYSMETA_SYMLINK_HDR) + + +class SymlinkContainerContext(WSGIContext): + def __init__(self, wsgi_app, logger): + super(SymlinkContainerContext, self).__init__(wsgi_app) + self.app = wsgi_app + self.logger = logger + + def handle_container(self, req, start_response): + """ + Handle container requests. + :return: Response Iterator after start_response called. + """ + app_resp = self._app_call(req.environ) + + if req.method == 'GET' and is_success(self._get_status_int()): + app_resp = self._process_json_resp(app_resp, req) + + start_response(self._response_status, self._response_headers, + self._response_exc_info) + + return app_resp + + def _process_json_resp(self, resp_iter, req): + """ + Iterate through json body looking for symlinks and modify its content + :return: modified json body + """ + with closing_if_possible(resp_iter): + resp_body = ''.join(resp_iter) + body_json = json.loads(resp_body) + swift_version, account, _junk = split_path(req.path, 2, 3, True) + new_body = json.dumps( + [self._extract_symlink_path_json(obj_dict, swift_version, account) + for obj_dict in body_json]) + self.update_content_length(len(new_body)) + return [new_body] + + def _extract_symlink_path_json(self, obj_dict, swift_version, account): + """ + Extract the symlink path from the hash value + :return: object dictionary with additional key:value pair if object + is a symlink. The new key is symlink_path. + """ + if 'hash' in obj_dict: + hash_value, meta = parse_header(obj_dict['hash']) + obj_dict['hash'] = hash_value + target = None + for key in meta: + if key == 'symlink_target': + target = meta[key] + elif key == 'symlink_target_account': + account = meta[key] + else: + # make sure to add all other (key, values) back in place + obj_dict['hash'] += '; %s=%s' % (key, meta[key]) + else: + if target: + obj_dict['symlink_path'] = os.path.join( + '/', swift_version, account, target) + + return obj_dict + + +class SymlinkObjectContext(WSGIContext): + + def __init__(self, wsgi_app, logger, symloop_max): + super(SymlinkObjectContext, self).__init__(wsgi_app) + self.app = wsgi_app + self.symloop_max = symloop_max + self.logger = logger + # N.B. _loop_count and _last_target_path are used to keep + # the statement in the _recursive_get. Hence it should not be touched + # from other resources. + self._loop_count = 0 + self._last_target_path = None + + def handle_get_head_symlink(self, req): + """ + Handle get/head request when client sent parameter ?symlink=get + + :param req: HTTP GET or HEAD object request with param ?symlink=get + :returns: Response Iterator + """ + + resp = self._app_call(req.environ) + response_header_dict = HeaderKeyDict(self._response_headers) + symlink_sysmeta_to_usermeta(response_header_dict) + self._response_headers = response_header_dict.items() + return resp + + def handle_get_head(self, req): + """ + Handle get/head request and in case the response is a symlink, + redirect request to target object. + + :param req: HTTP GET or HEAD object request + :returns: Response Iterator + """ + try: + return self._recursive_get_head(req) + except LinkIterError: + errmsg = 'Too many levels of symbolic links, ' \ + 'maximum allowed is %d' % self.symloop_max + raise HTTPConflict( + body=errmsg, request=req, content_type='text/plain') + + def _recursive_get_head(self, req): + resp = self._app_call(req.environ) + + def build_traversal_req(symlink_target): + """ + :returns: new request for target path if it's symlink otherwise + None + """ + version, account, _junk = split_path(req.path, 2, 3, True) + account = self._response_header_value( + TGT_ACCT_SYSMETA_SYMLINK_HDR) or account + target_path = os.path.join( + '/', version, account, + symlink_target.lstrip('/')) + self._last_target_path = target_path + new_req = make_subrequest( + req.environ, path=target_path, method=req.method, + headers=req.headers, swift_source='SYM') + new_req.headers.pop('X-Backend-Storage-Policy-Index', None) + return new_req + + symlink_target = self._response_header_value( + TGT_OBJ_SYSMETA_SYMLINK_HDR) + if symlink_target: + if self._loop_count >= self.symloop_max: + raise LinkIterError() + # format: /// + new_req = build_traversal_req(symlink_target) + self._loop_count += 1 + return self._recursive_get_head(new_req) + else: + if self._last_target_path: + # Content-Location will be applied only when one or more + # symlink recursion occurred. + # In this case, Content-Location is applied to show which + # object path caused the error response. + # To preserve '%2F'(= quote('/')) in X-Symlink-Target + # header value as it is, Content-Location value comes from + # TGT_OBJ_SYMLINK_HDR, not req.path + self._response_headers.extend( + [('Content-Location', self._last_target_path)]) + + return resp + + def handle_put(self, req): + """ + Handle put request when it contains X-Symlink-Target header. + + Symlink headers are validated and moved to sysmeta namespace. + :param req: HTTP PUT object request + :returns: Response Iterator + """ + if req.content_length != 0: + raise HTTPBadRequest( + body='Symlink requests require a zero byte body', + request=req, + content_type='text/plain') + + _check_symlink_header(req) + symlink_usermeta_to_sysmeta(req.headers) + # Store info in container update that this object is a symlink. + # We have a design decision to use etag space to store symlink info for + # object listing because it's immutable unless the object is + # overwritten. This may impact the downgrade scenario that the symlink + # info can be appreared as the suffix in the hash value of object + # listing result for clients. + # To create override etag easily, we have a contraint that the symlink + # must be 0 byte so we can add etag of the empty string + symlink info + # here, simply. Note that this override etag may be encrypted in the + # container db by encrypion middleware. + etag_override = [ + MD5_OF_EMPTY_STRING, + 'symlink_target=%s' % req.headers[TGT_OBJ_SYSMETA_SYMLINK_HDR] + ] + if TGT_ACCT_SYSMETA_SYMLINK_HDR in req.headers: + etag_override.append( + 'symlink_target_account=%s' % + req.headers[TGT_OBJ_SYSMETA_SYMLINK_HDR]) + req.headers['X-Object-Sysmeta-Container-Update-Override-Etag'] = \ + '; '.join(etag_override) + + return self._app_call(req.environ) + + def handle_post(self, req): + """ + Handle post request. If POSTing to a symlink, a HTTPTemporaryRedirect + error message is returned to client. + + Clients that POST to symlinks should understand that the POST is not + redirected to the target object like in a HEAD/GET request. POSTs to a + symlink will be handled just like a normal object by the object server. + It cannot reject it because it may not have symlink state when the POST + lands. The object server has no knowledge of what is a symlink object + is. On the other hand, on POST requests, the object server returns all + sysmeta of the object. This method uses that sysmeta to determine if + the stored object is a symlink or not. + + :param req: HTTP POST object request + :returns: HTTPTemporaryRedirect if POSTing to a symlink. + :returns: Response Iterator + """ + if TGT_OBJ_SYMLINK_HDR in req.headers: + raise HTTPBadRequest( + body='A PUT request is required to set a symlink target', + request=req, + content_type='text/plain') + + resp = self._app_call(req.environ) + if not is_success(self._get_status_int()): + return resp + + tgt_co = self._response_header_value(TGT_OBJ_SYSMETA_SYMLINK_HDR) + if tgt_co: + version, account, _junk = req.split_path(2, 3, True) + target_acc = self._response_header_value( + TGT_ACCT_SYSMETA_SYMLINK_HDR) or account + location_hdr = os.path.join( + '/', version, target_acc, tgt_co) + req.environ['swift.leave_relative_location'] = True + errmsg = 'The requested POST was applied to a symlink. POST ' +\ + 'directly to the target to apply requested metadata.' + raise HTTPTemporaryRedirect( + body=errmsg, headers={'location': location_hdr}) + else: + return resp + + def handle_object(self, req, start_response): + """ + Handle object requests + """ + if req.method in ('GET', 'HEAD'): + # if GET request came from versioned writes, then it should get + # the symlink only, not the referenced target + if req.params.get('symlink') == 'get' or \ + req.environ.get('swift.source') == 'VW': + resp = self.handle_get_head_symlink(req) + else: + resp = self.handle_get_head(req) + elif req.method == 'PUT' and (TGT_OBJ_SYMLINK_HDR in req.headers): + resp = self.handle_put(req) + elif req.method == 'POST': + resp = self.handle_post(req) + else: + # DELETE and OPTIONS reqs for a symlink and + # PUT reqs without X-Symlink-Target behave like any other object + resp = self._app_call(req.environ) + + start_response(self._response_status, self._response_headers, + self._response_exc_info) + + return resp + + +class SymlinkMiddleware(object): + """ + Middleware that implements symlinks. + + Symlinks are objects stored in Swift that contains a reference to another + object (i.e., the target object). An important use case is to use a path in + one container to access an object in a different container, with a + different policy. This allows policy cost/performance tradeoffs to be made + on individual objects. + """ + + def __init__(self, app, conf, symloop_max): + self.app = app + self.conf = conf + self.logger = get_logger(self.conf, log_route='symlink') + self.symloop_max = symloop_max + + def __call__(self, env, start_response): + req = Request(env) + try: + version, acc, cont, obj = req.split_path(3, 4, True) + except ValueError: + return self.app(env, start_response) + + try: + if obj: + # object context + context = SymlinkObjectContext(self.app, self.logger, + self.symloop_max) + return context.handle_object(req, start_response) + else: + # container context + context = SymlinkContainerContext(self.app, self.logger) + return context.handle_container(req, start_response) + except HTTPException as err_resp: + return err_resp(env, start_response) + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + + symloop_max = int(conf.get('symloop_max', DEFAULT_SYMLOOP_MAX)) + if symloop_max < 1: + symloop_max = int(DEFAULT_SYMLOOP_MAX) + register_swift_info('symlink', symloop_max=symloop_max) + + def symlink_mw(app): + return SymlinkMiddleware(app, conf, symloop_max) + return symlink_mw diff --git a/swift/common/middleware/tempurl.py b/swift/common/middleware/tempurl.py index f93950104d..3230b0680f 100644 --- a/swift/common/middleware/tempurl.py +++ b/swift/common/middleware/tempurl.py @@ -222,7 +222,7 @@ from swift.common.utils import split_path, get_valid_utf8_str, \ register_swift_info, get_hmac, streq_const_time, quote -DISALLOWED_INCOMING_HEADERS = 'x-object-manifest' +DISALLOWED_INCOMING_HEADERS = 'x-object-manifest x-symlink-target' #: Default headers to remove from incoming requests. Simply a whitespace #: delimited list of header names and names can optionally end with '*' to diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index c3de2f4fd2..1ac348dec5 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -34,7 +34,8 @@ from swift.common.storage_policy import POLICIES from swift.common.exceptions import ListingIterError, SegmentError from swift.common.http import is_success from swift.common.swob import HTTPBadRequest, \ - HTTPServiceUnavailable, Range, is_chunked, multi_range_iterator + HTTPServiceUnavailable, Range, is_chunked, multi_range_iterator, \ + HTTPPreconditionFailed from swift.common.utils import split_path, validate_device_partition, \ close_if_possible, maybe_multipart_byteranges_to_document_iters, \ multipart_byteranges_to_document_iters, parse_content_type, \ @@ -281,6 +282,31 @@ def copy_header_subset(from_r, to_r, condition): to_r.headers[k] = v +def check_path_header(req, name, length, error_msg): + """ + Validate that the value of path-like header is + well formatted. We assume the caller ensures that + specific header is present in req.headers. + + :param req: HTTP request object + :param name: header name + :param length: length of path segment check + :param error_msg: error message for client + :returns: A tuple with path parts according to length + :raise: HTTPPreconditionFailed if header value + is not well formatted. + """ + hdr = unquote(req.headers.get(name)) + if not hdr.startswith('/'): + hdr = '/' + hdr + try: + return split_path(hdr, length, length, True) + except ValueError: + raise HTTPPreconditionFailed( + request=req, + body=error_msg) + + class SegmentedIterable(object): """ Iterable that returns the object contents for a large object. diff --git a/swift/container/sync.py b/swift/container/sync.py index 3d48541ada..2d2d65d08a 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -568,7 +568,10 @@ class ContainerSync(Daemon): realm_key, ts_meta): return True exc = None - # look up for the newest one + # look up for the newest one; the symlink=get query-string has + # no effect unless symlinks are enabled in the internal client + # in which case it ensures that symlink objects retain their + # symlink property when sync'd. headers_out = {'X-Newest': True, 'X-Backend-Storage-Policy-Index': str(info['storage_policy_index'])} @@ -577,7 +580,8 @@ class ContainerSync(Daemon): self.swift.get_object(info['account'], info['container'], row['name'], headers=headers_out, - acceptable_statuses=(2, 4)) + acceptable_statuses=(2, 4), + params={'symlink': 'get'}) except (Exception, UnexpectedResponse, Timeout) as err: headers = {} diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index 609e7e43bd..4c1dcd8cd3 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -794,7 +794,8 @@ class File(Base): ['content_type', 'content-type'], ['last_modified', 'last-modified'], ['etag', 'etag']] - optional_fields = [['x_object_manifest', 'x-object-manifest']] + optional_fields = [['x_object_manifest', 'x-object-manifest'], + ['x_symlink_target', 'x-symlink-target']] header_fields = self.header_fields(fields, optional_fields=optional_fields) diff --git a/test/functional/test_symlink.py b/test/functional/test_symlink.py new file mode 100755 index 0000000000..1c566e3a70 --- /dev/null +++ b/test/functional/test_symlink.py @@ -0,0 +1,1839 @@ +#!/usr/bin/python +# Copyright (c) 2010-2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hmac +import unittest2 +import itertools +import hashlib +import time + +from six.moves import urllib +from uuid import uuid4 +from unittest2 import SkipTest + +from swift.common.utils import json, MD5_OF_EMPTY_STRING +from swift.common.middleware.slo import SloGetContext +from test.functional import check_response, retry, requires_acls, cluster_info +from test.functional.tests import Base, TestFileComparisonEnv, Utils, BaseEnv +from test.functional.test_slo import TestSloEnv +from test.functional.test_dlo import TestDloEnv +from test.functional.test_tempurl import TestContainerTempurlEnv, \ + TestTempurlEnv +from test.functional.swift_test_client import ResponseError +from test.functional.test_versioned_writes import TestObjectVersioningEnv +import test.functional as tf + +TARGET_BODY = 'target body' + + +def setUpModule(): + tf.setup_package() + if 'symlink' not in cluster_info: + raise SkipTest("Symlinks not enabled") + + +def tearDownModule(): + tf.teardown_package() + + +class TestSymlinkEnv(BaseEnv): + link_cont = uuid4().hex + tgt_cont = uuid4().hex + tgt_obj = uuid4().hex + + @classmethod + def setUp(cls): + if tf.skip or tf.skip2: + raise SkipTest + + cls._create_container(cls.tgt_cont) # use_account=1 + cls._create_container(cls.link_cont) # use_account=1 + + # container in account 2 + cls._create_container(cls.link_cont, use_account=2) + cls._create_tgt_object() + + @classmethod + def containers(cls): + return (cls.link_cont, cls.tgt_cont) + + @classmethod + def target_content_location(cls): + return '%s/%s' % (cls.tgt_cont, cls.tgt_obj) + + @classmethod + def _make_request(cls, url, token, parsed, conn, method, + container, obj='', headers=None, body='', + query_args=None): + headers = headers or {} + headers.update({'X-Auth-Token': token}) + path = '%s/%s/%s' % (parsed.path, container, obj) if obj \ + else '%s/%s' % (parsed.path, container) + if query_args: + path += '?%s' % query_args + conn.request(method, path, body, headers) + resp = check_response(conn) + # to read the buffer and keep it in the attribute, call resp.content + resp.content + return resp + + @classmethod + def _create_container(cls, name, headers=None, use_account=1): + headers = headers or {} + resp = retry(cls._make_request, method='PUT', container=name, + headers=headers, use_account=use_account) + if resp.status != 201: + raise ResponseError(resp) + return name + + @classmethod + def _create_tgt_object(cls): + resp = retry(cls._make_request, method='PUT', + container=cls.tgt_cont, obj=cls.tgt_obj, + body=TARGET_BODY) + if resp.status != 201: + raise ResponseError(resp) + + # sanity: successful put response has content-length 0 + cls.tgt_length = str(len(TARGET_BODY)) + cls.tgt_etag = resp.getheader('etag') + + resp = retry(cls._make_request, method='GET', + container=cls.tgt_cont, obj=cls.tgt_obj) + if resp.status != 200 and resp.content != TARGET_BODY: + raise ResponseError(resp) + + @classmethod + def tearDown(cls): + delete_containers = [ + (use_account, containers) for use_account, containers in + enumerate([cls.containers(), [cls.link_cont]], 1)] + # delete objects inside container + for use_account, containers in delete_containers: + for container in containers: + while True: + cont = container + resp = retry(cls._make_request, method='GET', + container=cont, query_args='format=json', + use_account=use_account) + if resp.status == 404: + break + if resp.status // 100 != 2: + raise ResponseError(resp) + objs = json.loads(resp.content) + if not objs: + break + for obj in objs: + resp = retry(cls._make_request, method='DELETE', + container=container, obj=obj['name'], + use_account=use_account) + if (resp.status != 204): + raise ResponseError(resp) + + # delete the containers + for use_account, containers in delete_containers: + for container in containers: + resp = retry(cls._make_request, method='DELETE', + container=container, + use_account=use_account) + if resp.status not in (204, 404): + raise ResponseError(resp) + + +class TestSymlink(Base): + env = TestSymlinkEnv + + @classmethod + def setUpClass(cls): + # To skip env setup for class setup, instead setUp the env for each + # test method + pass + + def setUp(self): + self.env.setUp() + + def object_name_generator(): + while True: + yield uuid4().hex + + self.obj_name_gen = object_name_generator() + + def tearDown(self): + self.env.tearDown() + + def _make_request(self, url, token, parsed, conn, method, + container, obj='', headers=None, body='', + query_args=None, allow_redirects=True): + headers = headers or {} + headers.update({'X-Auth-Token': token}) + path = '%s/%s/%s' % (parsed.path, container, obj) if obj \ + else '%s/%s' % (parsed.path, container) + if query_args: + path += '?%s' % query_args + conn.requests_args['allow_redirects'] = allow_redirects + conn.request(method, path, body, headers) + resp = check_response(conn) + # to read the buffer and keep it in the attribute, call resp.content + resp.content + return resp + + def _make_request_with_symlink_get(self, url, token, parsed, conn, method, + container, obj, headers=None, body=''): + resp = self._make_request( + url, token, parsed, conn, method, container, obj, headers, body, + query_args='symlink=get') + return resp + + def _test_put_symlink(self, link_cont, link_obj, tgt_cont, tgt_obj): + headers = {'X-Symlink-Target': '%s/%s' % (tgt_cont, tgt_obj)} + resp = retry(self._make_request, method='PUT', + container=link_cont, obj=link_obj, + headers=headers) + self.assertEqual(resp.status, 201) + + def _test_get_as_target_object( + self, link_cont, link_obj, expected_content_location, + use_account=1): + resp = retry( + self._make_request, method='GET', + container=link_cont, obj=link_obj, use_account=use_account) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.content, TARGET_BODY) + self.assertEqual(resp.getheader('content-length'), + str(self.env.tgt_length)) + self.assertEqual(resp.getheader('etag'), self.env.tgt_etag) + self.assertIn('Content-Location', resp.headers) + # TODO: content-location is a full path so it's better to assert + # with the value, instead of assertIn + self.assertIn(expected_content_location, + resp.getheader('content-location')) + return resp + + def _test_head_as_target_object(self, link_cont, link_obj, use_account=1): + resp = retry( + self._make_request, method='HEAD', + container=link_cont, obj=link_obj, use_account=use_account) + self.assertEqual(resp.status, 200) + + def _assertLinkObject(self, link_cont, link_obj, use_account=1): + # HEAD on link_obj itself + resp = retry( + self._make_request_with_symlink_get, method='HEAD', + container=link_cont, obj=link_obj, use_account=use_account) + self.assertEqual(resp.status, 200) + self.assertTrue(resp.getheader('x-symlink-target')) + + # GET on link_obj itself + resp = retry( + self._make_request_with_symlink_get, method='GET', + container=link_cont, obj=link_obj, use_account=use_account) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.content, '') + self.assertEqual(resp.getheader('content-length'), str(0)) + self.assertTrue(resp.getheader('x-symlink-target')) + + def _assertSymlink(self, link_cont, link_obj, + expected_content_location=None, use_account=1): + expected_content_location = \ + expected_content_location or self.env.target_content_location() + # sanity: HEAD/GET on link_obj + self._assertLinkObject(link_cont, link_obj, use_account) + + # HEAD target object via symlink + self._test_head_as_target_object( + link_cont=link_cont, link_obj=link_obj, use_account=use_account) + + # GET target object via symlink + self._test_get_as_target_object( + link_cont=link_cont, link_obj=link_obj, use_account=use_account, + expected_content_location=expected_content_location) + + def test_symlink_with_encoded_target_name(self): + # makes sure to test encoded characters as symlink target + target_obj = 'dealde%2Fl04 011e%204c8df/flash.png' + link_obj = uuid4().hex + + # Now let's write a new target object and symlink will be able to + # return object + resp = retry( + self._make_request, method='PUT', container=self.env.tgt_cont, + obj=target_obj, body=TARGET_BODY) + + self.assertEqual(resp.status, 201) + + # PUT symlink + self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj, + tgt_cont=self.env.tgt_cont, + tgt_obj=target_obj) + + self._assertSymlink( + self.env.link_cont, link_obj, + expected_content_location="%s/%s" % (self.env.tgt_cont, + target_obj)) + + def test_symlink_put_head_get(self): + link_obj = uuid4().hex + + # PUT link_obj + self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj, + tgt_cont=self.env.tgt_cont, + tgt_obj=self.env.tgt_obj) + + self._assertSymlink(self.env.link_cont, link_obj) + + def test_symlink_get_ranged(self): + link_obj = uuid4().hex + + # PUT symlink + self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj, + tgt_cont=self.env.tgt_cont, + tgt_obj=self.env.tgt_obj) + + headers = {'Range': 'bytes=7-10'} + resp = retry(self._make_request, method='GET', + container=self.env.link_cont, obj=link_obj, + headers=headers) + self.assertEqual(resp.status, 206) + self.assertEqual(resp.content, 'body') + + def test_create_symlink_before_target(self): + link_obj = uuid4().hex + target_obj = uuid4().hex + + # PUT link_obj before target object is written + # PUT, GET, HEAD (on symlink) should all work ok without target object + self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj, + tgt_cont=self.env.tgt_cont, tgt_obj=target_obj) + + # Try to GET target via symlink. + # 404 will be returned with Content-Location of target path. + resp = retry( + self._make_request, method='GET', + container=self.env.link_cont, obj=link_obj, use_account=1) + self.assertEqual(resp.status, 404) + self.assertIn('Content-Location', resp.headers) + expected_location_hdr = "%s/%s" % (self.env.tgt_cont, target_obj) + self.assertIn(expected_location_hdr, + resp.getheader('content-location')) + + # HEAD on target object via symlink should return a 404 since target + # object has not yet been written + resp = retry( + self._make_request, method='HEAD', + container=self.env.link_cont, obj=link_obj) + self.assertEqual(resp.status, 404) + + # GET on target object directly + resp = retry( + self._make_request, method='GET', + container=self.env.tgt_cont, obj=target_obj) + self.assertEqual(resp.status, 404) + + # Now let's write target object and symlink will be able to return + # object + resp = retry( + self._make_request, method='PUT', container=self.env.tgt_cont, + obj=target_obj, body=TARGET_BODY) + + self.assertEqual(resp.status, 201) + # successful put response has content-length 0 + target_length = str(len(TARGET_BODY)) + target_etag = resp.getheader('etag') + + # sanity: HEAD/GET on link_obj itself + self._assertLinkObject(self.env.link_cont, link_obj) + + # HEAD target object via symlink + self._test_head_as_target_object( + link_cont=self.env.link_cont, link_obj=link_obj) + + # GET target object via symlink + resp = retry(self._make_request, method='GET', + container=self.env.link_cont, obj=link_obj) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.content, TARGET_BODY) + self.assertEqual(resp.getheader('content-length'), str(target_length)) + self.assertEqual(resp.getheader('etag'), target_etag) + self.assertIn('Content-Location', resp.headers) + self.assertIn(expected_location_hdr, + resp.getheader('content-location')) + + def test_symlink_chain(self): + # Testing to symlink chain like symlink -> symlink -> target. + symloop_max = cluster_info['symlink']['symloop_max'] + + # create symlink chain in a container. To simplify, + # use target container for all objects (symlinks and target) here + previous = self.env.tgt_obj + container = self.env.tgt_cont + + for link_obj in itertools.islice(self.obj_name_gen, symloop_max): + # PUT link_obj point to tgt_obj + self._test_put_symlink( + link_cont=container, link_obj=link_obj, + tgt_cont=container, tgt_obj=previous) + + # set corrent link_obj to previous + previous = link_obj + + # the last link is valid for symloop_max constraint + max_chain_link = link_obj + self._assertSymlink(link_cont=container, link_obj=max_chain_link) + + # PUT a new link_obj points to the max_chain_link + # that will result in 409 error on the HEAD/GET. + too_many_chain_link = next(self.obj_name_gen) + self._test_put_symlink( + link_cont=container, link_obj=too_many_chain_link, + tgt_cont=container, tgt_obj=max_chain_link) + + # try to HEAD to target object via too_many_chain_link + resp = retry(self._make_request, method='HEAD', + container=container, + obj=too_many_chain_link) + self.assertEqual(resp.status, 409) + self.assertEqual(resp.content, '') + + # try to GET to target object via too_many_chain_link + resp = retry(self._make_request, method='GET', + container=container, + obj=too_many_chain_link) + self.assertEqual(resp.status, 409) + self.assertEqual( + resp.content, + 'Too many levels of symbolic links, maximum allowed is %d' % + symloop_max) + + # However, HEAD/GET to the (just) link is still ok + self._assertLinkObject(container, too_many_chain_link) + + def test_symlink_and_slo_manifest_chain(self): + if 'slo' not in cluster_info: + raise SkipTest + + symloop_max = cluster_info['symlink']['symloop_max'] + + # create symlink chain in a container. To simplify, + # use target container for all objects (symlinks and target) here + previous = self.env.tgt_obj + container = self.env.tgt_cont + + # make symlink and slo manifest chain + # e.g. slo -> symlink -> symlink -> slo -> symlink -> symlink -> target + for _ in range(SloGetContext.max_slo_recursion_depth or 1): + for link_obj in itertools.islice(self.obj_name_gen, symloop_max): + # PUT link_obj point to previous object + self._test_put_symlink( + link_cont=container, link_obj=link_obj, + tgt_cont=container, tgt_obj=previous) + + # next link will point to this link + previous = link_obj + else: + # PUT a manifest with single segment to the symlink + manifest_obj = next(self.obj_name_gen) + manifest = json.dumps( + [{'path': '/%s/%s' % (container, link_obj)}]) + resp = retry(self._make_request, method='PUT', + container=container, obj=manifest_obj, + body=manifest, + query_args='multipart-manifest=put') + self.assertEqual(resp.status, 201) # sanity + previous = manifest_obj + + # From the last manifest to the final target obj length is + # symloop_max * max_slo_recursion_depth + max_recursion_manifest = previous + + # Check GET to max_recursion_manifest returns valid target object + resp = retry( + self._make_request, method='GET', container=container, + obj=max_recursion_manifest) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.content, TARGET_BODY) + self.assertEqual(resp.getheader('content-length'), + str(self.env.tgt_length)) + # N.B. since the last manifest is slo so it will remove + # content-location info from the response header + self.assertNotIn('Content-Location', resp.headers) + + # sanity: one more link to the slo can work still + one_more_link = next(self.obj_name_gen) + self._test_put_symlink( + link_cont=container, link_obj=one_more_link, + tgt_cont=container, tgt_obj=max_recursion_manifest) + + resp = retry( + self._make_request, method='GET', container=container, + obj=one_more_link) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.content, TARGET_BODY) + self.assertEqual(resp.getheader('content-length'), + str(self.env.tgt_length)) + self.assertIn('Content-Location', resp.headers) + self.assertIn('%s/%s' % (container, max_recursion_manifest), + resp.getheader('content-location')) + + # PUT a new slo manifest point to the max_recursion_manifest + # Symlink and slo manifest chain from the new manifest to the final + # target has (max_slo_recursion_depth + 1) manifests. + too_many_recursion_manifest = next(self.obj_name_gen) + manifest = json.dumps( + [{'path': '/%s/%s' % (container, max_recursion_manifest)}]) + + resp = retry(self._make_request, method='PUT', + container=container, obj=too_many_recursion_manifest, + body=manifest, + query_args='multipart-manifest=put') + self.assertEqual(resp.status, 201) # sanity + + # Check GET to too_many_recursion_mani returns 409 error + resp = retry(self._make_request, method='GET', + container=container, obj=too_many_recursion_manifest) + self.assertEqual(resp.status, 409) + # N.B. This error message is from slo middleware that uses default. + self.assertEqual( + resp.content, + '

Conflict

There was a conflict when trying to' + ' complete your request.

') + + def test_symlink_put_missing_target_container(self): + link_obj = uuid4().hex + + # set only object, no container in the prefix + headers = {'X-Symlink-Target': self.env.tgt_obj} + resp = retry(self._make_request, method='PUT', + container=self.env.link_cont, obj=link_obj, + headers=headers) + self.assertEqual(resp.status, 412) + self.assertEqual(resp.content, + 'X-Symlink-Target header must be of the form' + ' /') + + def test_symlink_put_non_zero_length(self): + link_obj = uuid4().hex + headers = {'X-Symlink-Target': + '%s/%s' % (self.env.tgt_cont, self.env.tgt_obj)} + resp = retry( + self._make_request, method='PUT', container=self.env.link_cont, + obj=link_obj, body='non-zero-length', headers=headers) + + self.assertEqual(resp.status, 400) + self.assertEqual(resp.content, + 'Symlink requests require a zero byte body') + + def test_symlink_target_itself(self): + link_obj = uuid4().hex + headers = { + 'X-Symlink-Target': '%s/%s' % (self.env.link_cont, link_obj)} + resp = retry(self._make_request, method='PUT', + container=self.env.link_cont, obj=link_obj, + headers=headers) + self.assertEqual(resp.status, 400) + self.assertEqual(resp.content, 'Symlink cannot target itself') + + def test_symlink_target_each_other(self): + symloop_max = cluster_info['symlink']['symloop_max'] + + link_obj1 = uuid4().hex + link_obj2 = uuid4().hex + + # PUT two links which targets each other + self._test_put_symlink( + link_cont=self.env.link_cont, link_obj=link_obj1, + tgt_cont=self.env.link_cont, tgt_obj=link_obj2) + self._test_put_symlink( + link_cont=self.env.link_cont, link_obj=link_obj2, + tgt_cont=self.env.link_cont, tgt_obj=link_obj1) + + for obj in (link_obj1, link_obj2): + # sanity: HEAD/GET on the link itself is ok + self._assertLinkObject(self.env.link_cont, obj) + + for obj in (link_obj1, link_obj2): + resp = retry(self._make_request, method='HEAD', + container=self.env.link_cont, obj=obj) + self.assertEqual(resp.status, 409) + + resp = retry(self._make_request, method='GET', + container=self.env.link_cont, obj=obj) + self.assertEqual(resp.status, 409) + self.assertEqual( + resp.content, + 'Too many levels of symbolic links, maximum allowed is %d' % + symloop_max) + + def test_symlink_put_copy_from(self): + link_obj1 = uuid4().hex + link_obj2 = uuid4().hex + + self._test_put_symlink(link_cont=self.env.link_cont, + link_obj=link_obj1, + tgt_cont=self.env.tgt_cont, + tgt_obj=self.env.tgt_obj) + + copy_src = '%s/%s' % (self.env.link_cont, link_obj1) + + # copy symlink + headers = {'X-Copy-From': copy_src} + resp = retry(self._make_request_with_symlink_get, + method='PUT', + container=self.env.link_cont, obj=link_obj2, + headers=headers) + self.assertEqual(resp.status, 201) + + self._assertSymlink(link_cont=self.env.link_cont, link_obj=link_obj2) + + @requires_acls + def test_symlink_put_copy_from_cross_account(self): + link_obj1 = uuid4().hex + link_obj2 = uuid4().hex + + self._test_put_symlink(link_cont=self.env.link_cont, + link_obj=link_obj1, + tgt_cont=self.env.tgt_cont, + tgt_obj=self.env.tgt_obj) + + copy_src = '%s/%s' % (self.env.link_cont, link_obj1) + account_one = tf.parsed[0].path.split('/', 2)[2] + perm_two = tf.swift_test_perm[1] + + # add X-Content-Read to account 1 link_cont and tgt_cont + # permit account 2 to read account 1 link_cont to perform copy_src + # and tgt_cont so that link_obj2 can refer to tgt_object + # this ACL allows the copy to succeed + headers = {'X-Container-Read': perm_two} + resp = retry( + self._make_request, method='POST', + container=self.env.link_cont, headers=headers) + self.assertEqual(resp.status, 204) + + # this ACL allows link_obj in account 2 to target object in account 1 + resp = retry(self._make_request, method='POST', + container=self.env.tgt_cont, headers=headers) + self.assertEqual(resp.status, 204) + + # copy symlink itself to a different account w/o + # X-Symlink-Target-Account. This operation will result in copying + # symlink to the account 2 container that points to the + # container/object in the account 2. + # (the container/object is not prepared) + headers = {'X-Copy-From-Account': account_one, + 'X-Copy-From': copy_src} + resp = retry(self._make_request_with_symlink_get, method='PUT', + container=self.env.link_cont, obj=link_obj2, + headers=headers, use_account=2) + self.assertEqual(resp.status, 201) + + # sanity: HEAD/GET on link_obj itself + self._assertLinkObject(self.env.link_cont, link_obj2, use_account=2) + + # no target object in the account 2 + for method in ('HEAD', 'GET'): + resp = retry( + self._make_request, method=method, + container=self.env.link_cont, obj=link_obj2, use_account=2) + self.assertEqual(resp.status, 404) + self.assertIn('content-location', resp.headers) + self.assertIn(self.env.target_content_location(), + resp.getheader('content-location')) + + # copy symlink itself to a different account with target account + # the target path will be in account 1 + # the target path will have an object + headers = {'X-Symlink-target-Account': account_one, + 'X-Copy-From-Account': account_one, + 'X-Copy-From': copy_src} + resp = retry( + self._make_request_with_symlink_get, method='PUT', + container=self.env.link_cont, obj=link_obj2, + headers=headers, use_account=2) + self.assertEqual(resp.status, 201) + + self._assertSymlink(link_cont=self.env.link_cont, link_obj=link_obj2, + use_account=2) + + def test_symlink_copy_from_target(self): + link_obj1 = uuid4().hex + obj2 = uuid4().hex + + self._test_put_symlink(link_cont=self.env.link_cont, + link_obj=link_obj1, + tgt_cont=self.env.tgt_cont, + tgt_obj=self.env.tgt_obj) + + copy_src = '%s/%s' % (self.env.link_cont, link_obj1) + + # issuing a COPY request to a symlink w/o symlink=get, should copy + # the target object, not the symlink itself + headers = {'X-Copy-From': copy_src} + resp = retry(self._make_request, method='PUT', + container=self.env.tgt_cont, obj=obj2, + headers=headers) + self.assertEqual(resp.status, 201) + + # HEAD to the copied object + resp = retry(self._make_request, method='HEAD', + container=self.env.tgt_cont, obj=obj2) + self.assertEqual(200, resp.status) + self.assertNotIn('Content-Location', resp.headers) + # GET to the copied object + resp = retry(self._make_request, method='GET', + container=self.env.tgt_cont, obj=obj2) + # But... this is a raw object (not a symlink) + self.assertEqual(200, resp.status) + self.assertNotIn('Content-Location', resp.headers) + self.assertEqual(TARGET_BODY, resp.content) + + def test_symlink_copy(self): + link_obj1 = uuid4().hex + link_obj2 = uuid4().hex + + self._test_put_symlink(link_cont=self.env.link_cont, + link_obj=link_obj1, + tgt_cont=self.env.tgt_cont, + tgt_obj=self.env.tgt_obj) + + copy_dst = '%s/%s' % (self.env.link_cont, link_obj2) + + # copy symlink + headers = {'Destination': copy_dst} + resp = retry( + self._make_request_with_symlink_get, method='COPY', + container=self.env.link_cont, obj=link_obj1, headers=headers) + self.assertEqual(resp.status, 201) + + self._assertSymlink(link_cont=self.env.link_cont, link_obj=link_obj2) + + def test_symlink_copy_target(self): + link_obj1 = uuid4().hex + obj2 = uuid4().hex + + self._test_put_symlink(link_cont=self.env.link_cont, + link_obj=link_obj1, + tgt_cont=self.env.tgt_cont, + tgt_obj=self.env.tgt_obj) + + copy_dst = '%s/%s' % (self.env.tgt_cont, obj2) + + # copy target object + headers = {'Destination': copy_dst} + resp = retry(self._make_request, method='COPY', + container=self.env.link_cont, obj=link_obj1, + headers=headers) + self.assertEqual(resp.status, 201) + + # HEAD to target object via symlink + resp = retry(self._make_request, method='HEAD', + container=self.env.tgt_cont, obj=obj2) + self.assertEqual(resp.status, 200) + self.assertNotIn('Content-Location', resp.headers) + # GET to the copied object that should be a raw object (not symlink) + resp = retry(self._make_request, method='GET', + container=self.env.tgt_cont, obj=obj2) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.content, TARGET_BODY) + self.assertNotIn('Content-Location', resp.headers) + + def test_post_symlink(self): + link_obj = uuid4().hex + value1 = uuid4().hex + + self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj, + tgt_cont=self.env.tgt_cont, + tgt_obj=self.env.tgt_obj) + + # POSTing to a symlink is not allowed and should return a 307 + headers = {'X-Object-Meta-Alpha': 'apple'} + resp = retry( + self._make_request, method='POST', container=self.env.link_cont, + obj=link_obj, headers=headers, allow_redirects=False) + self.assertEqual(resp.status, 307) + # we are using account 0 in this test + expected_location_hdr = "%s/%s/%s" % ( + tf.parsed[0].path, self.env.tgt_cont, self.env.tgt_obj) + self.assertEqual(resp.getheader('Location'), expected_location_hdr) + + # Read header from symlink itself. The metadata is applied to symlink + resp = retry(self._make_request_with_symlink_get, method='GET', + container=self.env.link_cont, obj=link_obj) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.getheader('X-Object-Meta-Alpha'), 'apple') + + # Post the target object directly + headers = {'x-object-meta-test': value1} + resp = retry( + self._make_request, method='POST', container=self.env.tgt_cont, + obj=self.env.tgt_obj, headers=headers) + self.assertEqual(resp.status, 202) + resp = retry(self._make_request, method='GET', + container=self.env.tgt_cont, obj=self.env.tgt_obj) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.getheader('X-Object-Meta-Test'), value1) + + # Read header from target object via symlink, should exist now. + resp = retry( + self._make_request, method='GET', container=self.env.link_cont, + obj=link_obj) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.getheader('X-Object-Meta-Test'), value1) + # sanity: no X-Object-Meta-Alpha exists in the response header + self.assertNotIn('X-Object-Meta-Alpha', resp.headers) + + def test_post_with_symlink_header(self): + # POSTing to a symlink is not allowed and should return a 307 + # updating the symlink target with a POST should always fail + headers = {'X-Symlink-Target': 'container/new_target'} + resp = retry( + self._make_request, method='POST', container=self.env.tgt_cont, + obj=self.env.tgt_obj, headers=headers, allow_redirects=False) + self.assertEqual(resp.status, 400) + self.assertEqual(resp.content, + 'A PUT request is required to set a symlink target') + + def test_overwrite_symlink(self): + link_obj = uuid4().hex + new_tgt_obj = "new_target_object_name" + new_tgt = '%s/%s' % (self.env.tgt_cont, new_tgt_obj) + self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj, + tgt_cont=self.env.tgt_cont, + tgt_obj=self.env.tgt_obj) + + # sanity + self._assertSymlink(self.env.link_cont, link_obj) + + # Overwrite symlink with PUT + self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj, + tgt_cont=self.env.tgt_cont, + tgt_obj=new_tgt_obj) + + # head symlink to check X-Symlink-Target header + resp = retry(self._make_request_with_symlink_get, method='HEAD', + container=self.env.link_cont, obj=link_obj) + self.assertEqual(resp.status, 200) + # target should remain with old target + self.assertEqual(resp.getheader('X-Symlink-Target'), new_tgt) + + def test_delete_symlink(self): + link_obj = uuid4().hex + + self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj, + tgt_cont=self.env.tgt_cont, + tgt_obj=self.env.tgt_obj) + + resp = retry(self._make_request, method='DELETE', + container=self.env.link_cont, obj=link_obj) + self.assertEqual(resp.status, 204) + + # make sure target object was not deleted and is still reachable + resp = retry(self._make_request, method='GET', + container=self.env.tgt_cont, obj=self.env.tgt_obj) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.content, TARGET_BODY) + + @requires_acls + def test_symlink_put_target_account(self): + if tf.skip or tf.skip2: + raise SkipTest + link_obj = uuid4().hex + + account_one = tf.parsed[0].path.split('/', 2)[2] + + # create symlink in account 2 + # pointing to account 1 + headers = {'X-Symlink-Target-Account': account_one, + 'X-Symlink-Target': + '%s/%s' % (self.env.tgt_cont, self.env.tgt_obj)} + resp = retry(self._make_request, method='PUT', + container=self.env.link_cont, obj=link_obj, + headers=headers, use_account=2) + self.assertEqual(resp.status, 201) + perm_two = tf.swift_test_perm[1] + + # sanity test: + # it should be ok to get the symlink itself, but not the target object + # because the read acl has not been configured yet + self._assertLinkObject(self.env.link_cont, link_obj, use_account=2) + resp = retry( + self._make_request, method='GET', + container=self.env.link_cont, obj=link_obj, use_account=2) + + self.assertEqual(resp.status, 403) + + # add X-Content-Read to account 1 tgt_cont + # permit account 2 to read account 1 tgt_cont + # add acl to allow reading from source + headers = {'X-Container-Read': perm_two} + resp = retry(self._make_request, method='POST', + container=self.env.tgt_cont, headers=headers) + self.assertEqual(resp.status, 204) + + # GET on link_obj itself + self._assertLinkObject(self.env.link_cont, link_obj, use_account=2) + + # GET to target object via symlink + resp = self._test_get_as_target_object( + self.env.link_cont, link_obj, + expected_content_location=self.env.target_content_location(), + use_account=2) + self.assertIn(account_one, resp.getheader('content-location')) + + def test_symlink_object_listing(self): + link_obj = uuid4().hex + + self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj, + tgt_cont=self.env.tgt_cont, + tgt_obj=self.env.tgt_obj) + # sanity + self._assertSymlink(self.env.link_cont, link_obj) + resp = retry(self._make_request, method='GET', + container=self.env.link_cont, + query_args='format=json') + self.assertEqual(resp.status, 200) + object_list = json.loads(resp.content) + self.assertEqual(len(object_list), 1) + self.assertIn('symlink_path', object_list[0]) + self.assertIn(self.env.target_content_location(), + object_list[0]['symlink_path']) + + +class TestCrossPolicySymlinkEnv(TestSymlinkEnv): + multiple_policies_enabled = None + + @classmethod + def setUp(cls): + if tf.skip or tf.skip2: + raise SkipTest + + if cls.multiple_policies_enabled is None: + try: + cls.policies = tf.FunctionalStoragePolicyCollection.from_info() + except AssertionError: + pass + + if cls.policies and len(cls.policies) > 1: + cls.multiple_policies_enabled = True + else: + cls.multiple_policies_enabled = False + return + + link_policy = cls.policies.select() + tgt_policy = cls.policies.exclude(name=link_policy['name']).select() + link_header = {'X-Storage-Policy': link_policy['name']} + tgt_header = {'X-Storage-Policy': tgt_policy['name']} + + cls._create_container(cls.link_cont, headers=link_header) + cls._create_container(cls.tgt_cont, headers=tgt_header) + + # container in account 2 + cls._create_container(cls.link_cont, headers=link_header, + use_account=2) + cls._create_tgt_object() + + +class TestCrossPolicySymlink(TestSymlink): + env = TestCrossPolicySymlinkEnv + + def setUp(self): + super(TestCrossPolicySymlink, self).setUp() + if self.env.multiple_policies_enabled is False: + raise SkipTest('Cross policy test requires multiple policies') + elif self.env.multiple_policies_enabled is not True: + # just some sanity checking + raise Exception("Expected multiple_policies_enabled " + "to be True/False, got %r" % ( + self.env.multiple_policies_enabled,)) + + def tearDown(self): + self.env.tearDown() + + +class TestSymlinkSlo(Base): + """ + Just some sanity testing of SLO + symlinks. + It is basically a copy of SLO tests in test_slo, but the tested object is + a symlink to the manifest (instead of the manifest itself) + """ + env = TestSloEnv + + def setUp(self): + super(TestSymlinkSlo, self).setUp() + if self.env.slo_enabled is False: + raise SkipTest("SLO not enabled") + elif self.env.slo_enabled is not True: + # just some sanity checking + raise Exception( + "Expected slo_enabled to be True/False, got %r" % + (self.env.slo_enabled,)) + self.file_symlink = self.env.container.file(uuid4().hex) + + def test_symlink_target_slo_manifest(self): + self.file_symlink.write(hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + 'manifest-abcde')}) + file_contents = self.file_symlink.read() + self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents)) + self.assertEqual('a', file_contents[0]) + self.assertEqual('a', file_contents[1024 * 1024 - 1]) + self.assertEqual('b', file_contents[1024 * 1024]) + self.assertEqual('d', file_contents[-2]) + self.assertEqual('e', file_contents[-1]) + + def test_symlink_target_slo_nested_manifest(self): + self.file_symlink.write(hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + 'manifest-abcde-submanifest')}) + file_contents = self.file_symlink.read() + self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents)) + self.assertEqual('a', file_contents[0]) + self.assertEqual('a', file_contents[1024 * 1024 - 1]) + self.assertEqual('b', file_contents[1024 * 1024]) + self.assertEqual('d', file_contents[-2]) + self.assertEqual('e', file_contents[-1]) + + def test_slo_get_ranged_manifest(self): + self.file_symlink.write(hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + 'ranged-manifest')}) + grouped_file_contents = [ + (char, sum(1 for _char in grp)) + for char, grp in itertools.groupby(self.file_symlink.read())] + self.assertEqual([ + ('c', 1), + ('d', 1024 * 1024), + ('e', 1), + ('a', 512 * 1024), + ('b', 512 * 1024), + ('c', 1), + ('d', 1)], grouped_file_contents) + + def test_slo_ranged_get(self): + self.file_symlink.write(hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + 'manifest-abcde')}) + file_contents = self.file_symlink.read(size=1024 * 1024 + 2, + offset=1024 * 1024 - 1) + self.assertEqual('a', file_contents[0]) + self.assertEqual('b', file_contents[1]) + self.assertEqual('b', file_contents[-2]) + self.assertEqual('c', file_contents[-1]) + + +class TestSymlinkSloEnv(TestSloEnv): + + @classmethod + def create_links_to_segments(cls, container): + seg_info = {} + for letter in ('a', 'b'): + seg_name = "linkto_seg_%s" % letter + file_item = container.file(seg_name) + sym_hdr = {'X-Symlink-Target': '%s/seg_%s' % (container.name, + letter)} + file_item.write(hdrs=sym_hdr) + seg_info[seg_name] = { + 'path': '/%s/%s' % (container.name, seg_name)} + return seg_info + + @classmethod + def setUp(cls): + super(TestSymlinkSloEnv, cls).setUp() + + cls.link_seg_info = cls.create_links_to_segments(cls.container) + file_item = cls.container.file("manifest-linkto-ab") + file_item.write( + json.dumps([cls.link_seg_info['linkto_seg_a'], + cls.link_seg_info['linkto_seg_b']]), + parms={'multipart-manifest': 'put'}) + + +class TestSymlinkToSloSegments(Base): + """ + This test class will contain various tests where the segments of the SLO + manifest are symlinks to the actual segments. Again the tests are basicaly + a copy/paste of the tests in test_slo, only the manifest has been modified + to contain symlinks as the segments. + """ + env = TestSymlinkSloEnv + + def setUp(self): + super(TestSymlinkToSloSegments, self).setUp() + if self.env.slo_enabled is False: + raise SkipTest("SLO not enabled") + elif self.env.slo_enabled is not True: + # just some sanity checking + raise Exception( + "Expected slo_enabled to be True/False, got %r" % + (self.env.slo_enabled,)) + + def test_slo_get_simple_manifest_with_links(self): + file_item = self.env.container.file("manifest-linkto-ab") + file_contents = file_item.read() + self.assertEqual(2 * 1024 * 1024, len(file_contents)) + self.assertEqual('a', file_contents[0]) + self.assertEqual('a', file_contents[1024 * 1024 - 1]) + self.assertEqual('b', file_contents[1024 * 1024]) + + def test_slo_container_listing(self): + # the listing object size should equal the sum of the size of the + # segments, not the size of the manifest body + file_item = self.env.container.file(Utils.create_name()) + file_item.write( + json.dumps([self.env.link_seg_info['linkto_seg_a']]), + parms={'multipart-manifest': 'put'}) + + # The container listing has the etag of the actual manifest object + # contents which we get using multipart-manifest=get. Arguably this + # should be the etag that we get when NOT using multipart-manifest=get, + # to be consistent with size and content-type. But here we at least + # verify that it remains consistent when the object is updated with a + # POST. + file_item.initialize(parms={'multipart-manifest': 'get'}) + expected_etag = file_item.etag + + listing = self.env.container.files(parms={'format': 'json'}) + for f_dict in listing: + if f_dict['name'] == file_item.name: + self.assertEqual(1024 * 1024, f_dict['bytes']) + self.assertEqual('application/octet-stream', + f_dict['content_type']) + self.assertEqual(expected_etag, f_dict['hash']) + break + else: + self.fail('Failed to find manifest file in container listing') + + # now POST updated content-type file + file_item.content_type = 'image/jpeg' + file_item.sync_metadata({'X-Object-Meta-Test': 'blah'}) + file_item.initialize() + self.assertEqual('image/jpeg', file_item.content_type) # sanity + + # verify that the container listing is consistent with the file + listing = self.env.container.files(parms={'format': 'json'}) + for f_dict in listing: + if f_dict['name'] == file_item.name: + self.assertEqual(1024 * 1024, f_dict['bytes']) + self.assertEqual(file_item.content_type, + f_dict['content_type']) + self.assertEqual(expected_etag, f_dict['hash']) + break + else: + self.fail('Failed to find manifest file in container listing') + + # now POST with no change to content-type + file_item.sync_metadata({'X-Object-Meta-Test': 'blah'}, + cfg={'no_content_type': True}) + file_item.initialize() + self.assertEqual('image/jpeg', file_item.content_type) # sanity + + # verify that the container listing is consistent with the file + listing = self.env.container.files(parms={'format': 'json'}) + for f_dict in listing: + if f_dict['name'] == file_item.name: + self.assertEqual(1024 * 1024, f_dict['bytes']) + self.assertEqual(file_item.content_type, + f_dict['content_type']) + self.assertEqual(expected_etag, f_dict['hash']) + break + else: + self.fail('Failed to find manifest file in container listing') + + def test_slo_etag_is_hash_of_etags(self): + expected_hash = hashlib.md5() + expected_hash.update(hashlib.md5('a' * 1024 * 1024).hexdigest()) + expected_hash.update(hashlib.md5('b' * 1024 * 1024).hexdigest()) + expected_etag = expected_hash.hexdigest() + + file_item = self.env.container.file('manifest-linkto-ab') + self.assertEqual(expected_etag, file_item.info()['etag']) + + def test_slo_copy(self): + file_item = self.env.container.file("manifest-linkto-ab") + file_item.copy(self.env.container.name, "copied-abcde") + + copied = self.env.container.file("copied-abcde") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(2 * 1024 * 1024, len(copied_contents)) + + def test_slo_copy_the_manifest(self): + # first just perform some tests of the contents of the manifest itself + source = self.env.container.file("manifest-linkto-ab") + source_contents = source.read(parms={'multipart-manifest': 'get'}) + source_json = json.loads(source_contents) + source.initialize() + self.assertEqual('application/octet-stream', source.content_type) + source.initialize(parms={'multipart-manifest': 'get'}) + source_hash = hashlib.md5() + source_hash.update(source_contents) + self.assertEqual(source_hash.hexdigest(), source.etag) + + # now, copy the manifest + self.assertTrue(source.copy(self.env.container.name, + "copied-ab-manifest-only", + parms={'multipart-manifest': 'get'})) + + copied = self.env.container.file("copied-ab-manifest-only") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + try: + copied_json = json.loads(copied_contents) + except ValueError: + self.fail("COPY didn't copy the manifest (invalid json on GET)") + + # make sure content of copied manifest is the same as original man. + self.assertEqual(source_json, copied_json) + copied.initialize() + self.assertEqual('application/octet-stream', copied.content_type) + copied.initialize(parms={'multipart-manifest': 'get'}) + copied_hash = hashlib.md5() + copied_hash.update(copied_contents) + self.assertEqual(copied_hash.hexdigest(), copied.etag) + self.assertEqual(copied_hash.hexdigest(), source.etag) + + # verify the listing metadata + listing = self.env.container.files(parms={'format': 'json'}) + names = {} + for f_dict in listing: + if f_dict['name'] in ('manifest-linkto-ab', + 'copied-ab-manifest-only'): + names[f_dict['name']] = f_dict + + self.assertIn('manifest-linkto-ab', names) + actual = names['manifest-linkto-ab'] + self.assertEqual(2 * 1024 * 1024, actual['bytes']) + self.assertEqual('application/octet-stream', actual['content_type']) + self.assertEqual(source.etag, actual['hash']) + + self.assertIn('copied-ab-manifest-only', names) + actual = names['copied-ab-manifest-only'] + self.assertEqual(2 * 1024 * 1024, actual['bytes']) + self.assertEqual('application/octet-stream', actual['content_type']) + self.assertEqual(copied.etag, actual['hash']) + + +class TestSymlinkDlo(Base): + env = TestDloEnv + + def test_get_manifest(self): + link_obj = uuid4().hex + file_symlink = self.env.container.file(link_obj) + file_symlink.write(hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + 'man1')}) + + file_contents = file_symlink.read() + self.assertEqual( + file_contents, + "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee") + + link_obj = uuid4().hex + file_symlink = self.env.container.file(link_obj) + file_symlink.write(hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + 'man2')}) + file_contents = file_symlink.read() + self.assertEqual( + file_contents, + "AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE") + + link_obj = uuid4().hex + file_symlink = self.env.container.file(link_obj) + file_symlink.write(hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + 'manall')}) + file_contents = file_symlink.read() + self.assertEqual( + file_contents, + ("aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee" + + "AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE")) + + def test_get_manifest_document_itself(self): + link_obj = uuid4().hex + file_symlink = self.env.container.file(link_obj) + file_symlink.write(hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + 'man1')}) + file_contents = file_symlink.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(file_contents, "man1-contents") + self.assertEqual(file_symlink.info()['x_object_manifest'], + "%s/%s/seg_lower" % + (self.env.container.name, self.env.segment_prefix)) + + def test_get_range(self): + link_obj = uuid4().hex + "_symlink" + file_symlink = self.env.container.file(link_obj) + file_symlink.write(hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + 'man1')}) + file_contents = file_symlink.read(size=25, offset=8) + self.assertEqual(file_contents, "aabbbbbbbbbbccccccccccddd") + + file_contents = file_symlink.read(size=1, offset=47) + self.assertEqual(file_contents, "e") + + def test_get_range_out_of_range(self): + link_obj = uuid4().hex + file_symlink = self.env.container.file(link_obj) + file_symlink.write(hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + 'man1')}) + + self.assertRaises(ResponseError, file_symlink.read, size=7, offset=50) + self.assert_status(416) + + +class TestSymlinkTargetObjectComparisonEnv(TestFileComparisonEnv): + @classmethod + def setUp(cls): + super(TestSymlinkTargetObjectComparisonEnv, cls).setUp() + cls.parms = None + cls.expect_empty_etag = False + cls.expect_body = True + + +class TestSymlinkComparisonEnv(TestFileComparisonEnv): + @classmethod + def setUp(cls): + super(TestSymlinkComparisonEnv, cls).setUp() + cls.parms = {'symlink': 'get'} + cls.expect_empty_etag = True + cls.expect_body = False + + +class TestSymlinkTargetObjectComparison(Base): + env = TestSymlinkTargetObjectComparisonEnv + + def setUp(self): + super(TestSymlinkTargetObjectComparison, self).setUp() + for file_item in self.env.files: + link_obj = file_item.name + '_symlink' + file_symlink = self.env.container.file(link_obj) + file_symlink.write(hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + file_item.name)}) + + def testIfMatch(self): + for file_item in self.env.files: + link_obj = file_item.name + '_symlink' + file_symlink = self.env.container.file(link_obj) + + md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \ + file_item.md5 + hdrs = {'If-Match': md5} + body = file_symlink.read(hdrs=hdrs, parms=self.env.parms) + if self.env.expect_body: + self.assertTrue(body) + else: + self.assertEqual('', body) + self.assert_status(200) + self.assert_header('etag', md5) + + hdrs = {'If-Match': 'bogus'} + self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, + parms=self.env.parms) + self.assert_status(412) + self.assert_header('etag', md5) + + def testIfMatchMultipleEtags(self): + for file_item in self.env.files: + link_obj = file_item.name + '_symlink' + file_symlink = self.env.container.file(link_obj) + + md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \ + file_item.md5 + hdrs = {'If-Match': '"bogus1", "%s", "bogus2"' % md5} + body = file_symlink.read(hdrs=hdrs, parms=self.env.parms) + if self.env.expect_body: + self.assertTrue(body) + else: + self.assertEqual('', body) + self.assert_status(200) + self.assert_header('etag', md5) + + hdrs = {'If-Match': '"bogus1", "bogus2", "bogus3"'} + self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, + parms=self.env.parms) + self.assert_status(412) + self.assert_header('etag', md5) + + def testIfNoneMatch(self): + for file_item in self.env.files: + link_obj = file_item.name + '_symlink' + file_symlink = self.env.container.file(link_obj) + md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \ + file_item.md5 + + hdrs = {'If-None-Match': 'bogus'} + body = file_symlink.read(hdrs=hdrs, parms=self.env.parms) + if self.env.expect_body: + self.assertTrue(body) + else: + self.assertEqual('', body) + self.assert_status(200) + self.assert_header('etag', md5) + + hdrs = {'If-None-Match': md5} + self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, + parms=self.env.parms) + self.assert_status(304) + self.assert_header('etag', md5) + self.assert_header('accept-ranges', 'bytes') + + def testIfNoneMatchMultipleEtags(self): + for file_item in self.env.files: + link_obj = file_item.name + '_symlink' + file_symlink = self.env.container.file(link_obj) + md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \ + file_item.md5 + + hdrs = {'If-None-Match': '"bogus1", "bogus2", "bogus3"'} + body = file_symlink.read(hdrs=hdrs, parms=self.env.parms) + if self.env.expect_body: + self.assertTrue(body) + else: + self.assertEqual('', body) + self.assert_status(200) + self.assert_header('etag', md5) + + hdrs = {'If-None-Match': + '"bogus1", "bogus2", "%s"' % md5} + self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, + parms=self.env.parms) + self.assert_status(304) + self.assert_header('etag', md5) + self.assert_header('accept-ranges', 'bytes') + + def testIfModifiedSince(self): + for file_item in self.env.files: + link_obj = file_item.name + '_symlink' + file_symlink = self.env.container.file(link_obj) + md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \ + file_item.md5 + + hdrs = {'If-Modified-Since': self.env.time_old_f1} + body = file_symlink.read(hdrs=hdrs, parms=self.env.parms) + if self.env.expect_body: + self.assertTrue(body) + else: + self.assertEqual('', body) + self.assert_status(200) + self.assert_header('etag', md5) + self.assertTrue(file_symlink.info(hdrs=hdrs, parms=self.env.parms)) + + hdrs = {'If-Modified-Since': self.env.time_new} + self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, + parms=self.env.parms) + self.assert_status(304) + self.assert_header('etag', md5) + self.assert_header('accept-ranges', 'bytes') + self.assertRaises(ResponseError, file_symlink.info, hdrs=hdrs, + parms=self.env.parms) + self.assert_status(304) + self.assert_header('etag', md5) + self.assert_header('accept-ranges', 'bytes') + + def testIfUnmodifiedSince(self): + for file_item in self.env.files: + link_obj = file_item.name + '_symlink' + file_symlink = self.env.container.file(link_obj) + md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \ + file_item.md5 + + hdrs = {'If-Unmodified-Since': self.env.time_new} + body = file_symlink.read(hdrs=hdrs, parms=self.env.parms) + if self.env.expect_body: + self.assertTrue(body) + else: + self.assertEqual('', body) + self.assert_status(200) + self.assert_header('etag', md5) + self.assertTrue(file_symlink.info(hdrs=hdrs, parms=self.env.parms)) + + hdrs = {'If-Unmodified-Since': self.env.time_old_f2} + self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, + parms=self.env.parms) + self.assert_status(412) + self.assert_header('etag', md5) + self.assertRaises(ResponseError, file_symlink.info, hdrs=hdrs, + parms=self.env.parms) + self.assert_status(412) + self.assert_header('etag', md5) + + def testIfMatchAndUnmodified(self): + for file_item in self.env.files: + link_obj = file_item.name + '_symlink' + file_symlink = self.env.container.file(link_obj) + md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \ + file_item.md5 + + hdrs = {'If-Match': md5, + 'If-Unmodified-Since': self.env.time_new} + body = file_symlink.read(hdrs=hdrs, parms=self.env.parms) + if self.env.expect_body: + self.assertTrue(body) + else: + self.assertEqual('', body) + self.assert_status(200) + self.assert_header('etag', md5) + + hdrs = {'If-Match': 'bogus', + 'If-Unmodified-Since': self.env.time_new} + self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, + parms=self.env.parms) + self.assert_status(412) + self.assert_header('etag', md5) + + hdrs = {'If-Match': md5, + 'If-Unmodified-Since': self.env.time_old_f3} + self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, + parms=self.env.parms) + self.assert_status(412) + self.assert_header('etag', md5) + + def testLastModified(self): + file_item = self.env.container.file(Utils.create_name()) + file_item.content_type = Utils.create_name() + resp = file_item.write_random_return_resp(self.env.file_size) + put_last_modified = resp.getheader('last-modified') + md5 = file_item.md5 + + # create symlink + link_obj = file_item.name + '_symlink' + file_symlink = self.env.container.file(link_obj) + file_symlink.write(hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + file_item.name)}) + + info = file_symlink.info() + self.assertIn('last_modified', info) + last_modified = info['last_modified'] + self.assertEqual(put_last_modified, info['last_modified']) + + hdrs = {'If-Modified-Since': last_modified} + self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs) + self.assert_status(304) + self.assert_header('etag', md5) + self.assert_header('accept-ranges', 'bytes') + + hdrs = {'If-Unmodified-Since': last_modified} + self.assertTrue(file_symlink.read(hdrs=hdrs)) + + +class TestSymlinkComparison(TestSymlinkTargetObjectComparison): + env = TestSymlinkComparisonEnv + + def setUp(self): + super(TestSymlinkComparison, self).setUp() + + def testLastModified(self): + file_item = self.env.container.file(Utils.create_name()) + file_item.content_type = Utils.create_name() + resp = file_item.write_random_return_resp(self.env.file_size) + put_target_last_modified = resp.getheader('last-modified') + md5 = MD5_OF_EMPTY_STRING + + # get different last-modified between file and symlink + time.sleep(1) + + # create symlink + link_obj = file_item.name + '_symlink' + file_symlink = self.env.container.file(link_obj) + resp = file_symlink.write(return_resp=True, + hdrs={'X-Symlink-Target': + '%s/%s' % (self.env.container.name, + file_item.name)}) + put_sym_last_modified = resp.getheader('last-modified') + + info = file_symlink.info(parms=self.env.parms) + self.assertIn('last_modified', info) + last_modified = info['last_modified'] + self.assertEqual(put_sym_last_modified, info['last_modified']) + + hdrs = {'If-Modified-Since': put_target_last_modified} + body = file_symlink.read(hdrs=hdrs, parms=self.env.parms) + self.assertEqual('', body) + self.assert_status(200) + self.assert_header('etag', md5) + + hdrs = {'If-Modified-Since': last_modified} + self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, + parms=self.env.parms) + self.assert_status(304) + self.assert_header('etag', md5) + self.assert_header('accept-ranges', 'bytes') + + hdrs = {'If-Unmodified-Since': last_modified} + body = file_symlink.read(hdrs=hdrs, parms=self.env.parms) + self.assertEqual('', body) + self.assert_status(200) + self.assert_header('etag', md5) + + +class TestSymlinkAccountTempurl(Base): + env = TestTempurlEnv + + def setUp(self): + super(TestSymlinkAccountTempurl, self).setUp() + if self.env.tempurl_enabled is False: + raise SkipTest("TempURL not enabled") + elif self.env.tempurl_enabled is not True: + # just some sanity checking + raise Exception( + "Expected tempurl_enabled to be True/False, got %r" % + (self.env.tempurl_enabled,)) + + self.expires = int(time.time()) + 86400 + self.obj_tempurl_parms = self.tempurl_parms( + 'GET', self.expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + + def tempurl_parms(self, method, expires, path, key): + sig = hmac.new( + key, + '%s\n%s\n%s' % (method, expires, urllib.parse.unquote(path)), + hashlib.sha1).hexdigest() + return {'temp_url_sig': sig, 'temp_url_expires': str(expires)} + + def test_PUT_symlink(self): + new_sym = self.env.container.file(Utils.create_name()) + + # give out a signature which allows a PUT to new_obj + expires = int(time.time()) + 86400 + put_parms = self.tempurl_parms( + 'PUT', expires, self.env.conn.make_path(new_sym.path), + self.env.tempurl_key) + + # try to create symlink object + try: + new_sym.write( + '', {'x-symlink-target': 'cont/foo'}, parms=put_parms, + cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 400) + else: + self.fail('request did not error') + + def test_GET_symlink_inside_container(self): + tgt_obj = self.env.container.file(Utils.create_name()) + sym = self.env.container.file(Utils.create_name()) + tgt_obj.write("target object body") + sym.write( + '', + {'x-symlink-target': '%s/%s' % (self.env.container.name, tgt_obj)}) + + expires = int(time.time()) + 86400 + get_parms = self.tempurl_parms( + 'GET', expires, self.env.conn.make_path(sym.path), + self.env.tempurl_key) + + contents = sym.read(parms=get_parms, cfg={'no_auth_token': True}) + self.assert_status([200]) + self.assertEqual(contents, "target object body") + + def test_GET_symlink_outside_container(self): + tgt_obj = self.env.container.file(Utils.create_name()) + tgt_obj.write("target object body") + + container2 = self.env.account.container(Utils.create_name()) + container2.create() + + sym = container2.file(Utils.create_name()) + sym.write( + '', + {'x-symlink-target': '%s/%s' % (self.env.container.name, tgt_obj)}) + + expires = int(time.time()) + 86400 + get_parms = self.tempurl_parms( + 'GET', expires, self.env.conn.make_path(sym.path), + self.env.tempurl_key) + + # cross container tempurl works fine for account tempurl key + contents = sym.read(parms=get_parms, cfg={'no_auth_token': True}) + self.assert_status([200]) + self.assertEqual(contents, "target object body") + + +class TestSymlinkContainerTempurl(Base): + env = TestContainerTempurlEnv + + def setUp(self): + super(TestSymlinkContainerTempurl, self).setUp() + if self.env.tempurl_enabled is False: + raise SkipTest("TempURL not enabled") + elif self.env.tempurl_enabled is not True: + # just some sanity checking + raise Exception( + "Expected tempurl_enabled to be True/False, got %r" % + (self.env.tempurl_enabled,)) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + self.obj_tempurl_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + def tempurl_sig(self, method, expires, path, key): + return hmac.new( + key, + '%s\n%s\n%s' % (method, expires, urllib.parse.unquote(path)), + hashlib.sha1).hexdigest() + + def test_PUT_symlink(self): + new_sym = self.env.container.file(Utils.create_name()) + + # give out a signature which allows a PUT to new_obj + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'PUT', expires, self.env.conn.make_path(new_sym.path), + self.env.tempurl_key) + put_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + # try to create symlink object, should fail + try: + new_sym.write( + '', {'x-symlink-target': 'cont/foo'}, parms=put_parms, + cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 400) + else: + self.fail('request did not error') + + def test_GET_symlink_inside_container(self): + tgt_obj = self.env.container.file(Utils.create_name()) + sym = self.env.container.file(Utils.create_name()) + tgt_obj.write("target object body") + sym.write( + '', + {'x-symlink-target': '%s/%s' % (self.env.container.name, tgt_obj)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(sym.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = sym.read(parms=parms, cfg={'no_auth_token': True}) + self.assert_status([200]) + self.assertEqual(contents, "target object body") + + def test_GET_symlink_outside_container(self): + tgt_obj = self.env.container.file(Utils.create_name()) + tgt_obj.write("target object body") + + container2 = self.env.account.container(Utils.create_name()) + container2.create() + + sym = container2.file(Utils.create_name()) + sym.write( + '', + {'x-symlink-target': '%s/%s' % (self.env.container.name, tgt_obj)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(sym.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + # cross container tempurl does not work for container tempurl key + try: + sym.read(parms=parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 401) + else: + self.fail('request did not error') + try: + sym.info(parms=parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 401) + else: + self.fail('request did not error') + + +class TestSymlinkVersioning(Base): + env = TestObjectVersioningEnv + + def setUp(self): + super(TestSymlinkVersioning, self).setUp() + if self.env.versioning_enabled is False: + raise SkipTest("Object versioning not enabled") + elif self.env.versioning_enabled is not True: + # just some sanity checking + raise Exception( + "Expected versioning_enabled to be True/False, got %r" % + (self.env.versioning_enabled,)) + + def _tear_down_files(self): + try: + # only delete files and not containers + # as they were configured in self.env + self.env.versions_container.delete_files() + self.env.container.delete_files() + except ResponseError: + pass + + def tearDown(self): + super(TestSymlinkVersioning, self).tearDown() + self._tear_down_files() + + def test_overwriting(self): + container = self.env.container + versions_container = self.env.versions_container + + symlink_name = Utils.create_name() + tgt_a_name = Utils.create_name() + tgt_b_name = Utils.create_name() + + tgt_a = container.file(tgt_a_name) + tgt_a.write("aaaaa") + + tgt_b = container.file(tgt_b_name) + tgt_b.write("bbbbb") + + symlink_name = Utils.create_name() + sym_tgt_header = '%s/%s' % (container.name, tgt_a_name) + sym_headers_a = {'X-Symlink-Target': sym_tgt_header} + symlink = container.file(symlink_name) + symlink.write("", hdrs=sym_headers_a) + self.assertEqual("aaaaa", symlink.read()) + + sym_headers_b = {'X-Symlink-Target': '%s/%s' % (container.name, + tgt_b_name)} + symlink.write("", hdrs=sym_headers_b) + self.assertEqual("bbbbb", symlink.read()) + + # the old version got saved off + self.assertEqual(1, versions_container.info()['object_count']) + versioned_obj_name = versions_container.files()[0] + prev_version = versions_container.file(versioned_obj_name) + prev_version_info = prev_version.info(parms={'symlink': 'get'}) + self.assertEqual("aaaaa", prev_version.read()) + self.assertEqual(MD5_OF_EMPTY_STRING, prev_version_info['etag']) + self.assertEqual(sym_tgt_header, + prev_version_info['x_symlink_target']) + + # test delete + symlink.delete() + sym_info = symlink.info(parms={'symlink': 'get'}) + self.assertEqual("aaaaa", symlink.read()) + self.assertEqual(MD5_OF_EMPTY_STRING, sym_info['etag']) + self.assertEqual(sym_tgt_header, + sym_info['x_symlink_target']) + + +if __name__ == '__main__': + unittest2.main() diff --git a/test/probe/test_container_sync.py b/test/probe/test_container_sync.py index 8360922711..138c6b0531 100644 --- a/test/probe/test_container_sync.py +++ b/test/probe/test_container_sync.py @@ -25,14 +25,17 @@ from test.probe.brain import BrainSplitter from test.probe.common import ReplProbeTest, ENABLED_POLICIES -def get_current_realm_cluster(url): +def get_info(url): parts = urlparse(url) url = parts.scheme + '://' + parts.netloc + '/info' http_conn = client.http_connection(url) try: - info = client.get_capabilities(http_conn) + return client.get_capabilities(http_conn) except client.ClientException: raise unittest.SkipTest('Unable to retrieve cluster info') + + +def get_current_realm_cluster(info): try: realms = info['container_sync']['realms'] except KeyError: @@ -44,11 +47,12 @@ def get_current_realm_cluster(url): raise unittest.SkipTest('Unable find current realm cluster') -class TestContainerSync(ReplProbeTest): +class BaseTestContainerSync(ReplProbeTest): def setUp(self): - super(TestContainerSync, self).setUp() - self.realm, self.cluster = get_current_realm_cluster(self.url) + super(BaseTestContainerSync, self).setUp() + self.info = get_info(self.url) + self.realm, self.cluster = get_current_realm_cluster(self.info) def _setup_synced_containers( self, source_overrides=None, dest_overrides=None): @@ -92,6 +96,9 @@ class TestContainerSync(ReplProbeTest): return source['name'], dest['name'] + +class TestContainerSync(BaseTestContainerSync): + def test_sync(self): source_container, dest_container = self._setup_synced_containers() @@ -377,5 +384,184 @@ class TestContainerSync(ReplProbeTest): self.assertEqual(body, 'new-test-body') +class TestContainerSyncAndSymlink(BaseTestContainerSync): + + def setUp(self): + super(TestContainerSyncAndSymlink, self).setUp() + symlinks_enabled = self.info.get('symlink') or False + if not symlinks_enabled: + raise unittest.SkipTest("Symlinks not enabled") + + def test_sync_symlink(self): + # Verify that symlinks are sync'd as symlinks. + dest_account = self.account_2 + source_container, dest_container = self._setup_synced_containers( + dest_overrides=dest_account + ) + + # Create source and dest containers for target objects in separate + # accounts. + # These containers must have same name for the destination symlink + # to use the same target object. Initially the destination has no sync + # key so target will not sync. + tgt_container = 'targets-%s' % uuid.uuid4() + dest_tgt_info = dict(dest_account) + dest_tgt_info.update({'name': tgt_container, 'sync_key': None}) + self._setup_synced_containers( + source_overrides={'name': tgt_container, 'sync_key': 'tgt_key'}, + dest_overrides=dest_tgt_info) + + # upload a target to source + target_name = 'target-%s' % uuid.uuid4() + target_body = 'target body' + client.put_object( + self.url, self.token, tgt_container, target_name, + target_body) + + # Note that this tests when the target object is in the same account + target_path = '%s/%s' % (tgt_container, target_name) + symlink_name = 'symlink-%s' % uuid.uuid4() + put_headers = {'X-Symlink-Target': target_path} + + # upload the symlink + client.put_object( + self.url, self.token, source_container, symlink_name, + '', headers=put_headers) + + # verify object is a symlink + resp_headers, symlink_body = client.get_object( + self.url, self.token, source_container, symlink_name, + query_string='symlink=get') + self.assertEqual('', symlink_body) + self.assertIn('x-symlink-target', resp_headers) + + # verify symlink behavior + resp_headers, actual_target_body = client.get_object( + self.url, self.token, source_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + + # cycle container-sync + Manager(['container-sync']).once() + + # verify symlink was sync'd + resp_headers, dest_listing = client.get_container( + dest_account['url'], dest_account['token'], dest_container) + self.assertFalse(dest_listing[1:]) + self.assertEqual(symlink_name, dest_listing[0]['name']) + + # verify symlink remained only a symlink + resp_headers, symlink_body = client.get_object( + dest_account['url'], dest_account['token'], dest_container, + symlink_name, query_string='symlink=get') + self.assertEqual('', symlink_body) + self.assertIn('x-symlink-target', resp_headers) + + # attempt to GET the target object via symlink will fail because + # the target wasn't sync'd + with self.assertRaises(ClientException) as cm: + client.get_object(dest_account['url'], dest_account['token'], + dest_container, symlink_name) + self.assertEqual(404, cm.exception.http_status) + + # now set sync key on destination target container + client.put_container( + dest_account['url'], dest_account['token'], tgt_container, + headers={'X-Container-Sync-Key': 'tgt_key'}) + + # cycle container-sync + Manager(['container-sync']).once() + + # sanity: + resp_headers, body = client.get_object( + dest_account['url'], dest_account['token'], + tgt_container, target_name) + + # sanity check - verify symlink remained only a symlink + resp_headers, symlink_body = client.get_object( + dest_account['url'], dest_account['token'], dest_container, + symlink_name, query_string='symlink=get') + self.assertEqual('', symlink_body) + self.assertIn('x-symlink-target', resp_headers) + + # verify GET of target object via symlink now succeeds + resp_headers, actual_target_body = client.get_object( + dest_account['url'], dest_account['token'], dest_container, + symlink_name) + self.assertEqual(target_body, actual_target_body) + + def test_sync_cross_acc_symlink(self): + # Verify that cross-account symlinks are sync'd as cross-account + # symlinks. + source_container, dest_container = self._setup_synced_containers() + + # Sync'd symlinks will have the same target path "/a/c/o". + # So if we want to execute probe test with syncing targets, + # two swift clusters will be required. + # Therefore, for probe test in single cluster, target object is not + # sync'd in this test. + tgt_account = self.account_2 + tgt_container = 'targets-%s' % uuid.uuid4() + + tgt_container_headers = {'X-Container-Read': 'test:tester'} + if len(ENABLED_POLICIES) > 1: + tgt_policy = random.choice(ENABLED_POLICIES) + tgt_container_headers['X-Storage-Policy'] = tgt_policy.name + client.put_container(tgt_account['url'], tgt_account['token'], + tgt_container, headers=tgt_container_headers) + + # upload a target to source + target_name = 'target-%s' % uuid.uuid4() + target_body = 'target body' + client.put_object(tgt_account['url'], tgt_account['token'], + tgt_container, target_name, target_body) + + # Note that this tests when the target object is in a different account + target_path = '%s/%s' % (tgt_container, target_name) + symlink_name = 'symlink-%s' % uuid.uuid4() + put_headers = { + 'X-Symlink-Target': target_path, + 'X-Symlink-Target-Account': tgt_account['account']} + + # upload the symlink + client.put_object( + self.url, self.token, source_container, symlink_name, + '', headers=put_headers) + + # verify object is a cross-account symlink + resp_headers, symlink_body = client.get_object( + self.url, self.token, source_container, symlink_name, + query_string='symlink=get') + self.assertEqual('', symlink_body) + self.assertIn('x-symlink-target', resp_headers) + self.assertIn('x-symlink-target-account', resp_headers) + + # verify symlink behavior + resp_headers, actual_target_body = client.get_object( + self.url, self.token, source_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + + # cycle container-sync + Manager(['container-sync']).once() + + # verify symlink was sync'd + resp_headers, dest_listing = client.get_container( + self.url, self.token, dest_container) + self.assertFalse(dest_listing[1:]) + self.assertEqual(symlink_name, dest_listing[0]['name']) + + # verify symlink remained only a symlink + resp_headers, symlink_body = client.get_object( + self.url, self.token, dest_container, + symlink_name, query_string='symlink=get') + self.assertEqual('', symlink_body) + self.assertIn('x-symlink-target', resp_headers) + self.assertIn('x-symlink-target-account', resp_headers) + + # verify GET of target object via symlink now succeeds + resp_headers, actual_target_body = client.get_object( + self.url, self.token, dest_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + + if __name__ == "__main__": unittest.main() diff --git a/test/unit/common/middleware/crypto/test_decrypter.py b/test/unit/common/middleware/crypto/test_decrypter.py index 79f1b0384c..00cdbca168 100644 --- a/test/unit/common/middleware/crypto/test_decrypter.py +++ b/test/unit/common/middleware/crypto/test_decrypter.py @@ -19,6 +19,7 @@ import unittest import mock +from swift.common.utils import MD5_OF_EMPTY_STRING from swift.common.header_key_dict import HeaderKeyDict from swift.common.middleware.crypto import decrypter from swift.common.middleware.crypto.crypto_utils import CRYPTO_KEY_CALLBACK, \ @@ -960,6 +961,27 @@ class TestDecrypterContainerRequests(unittest.TestCase): self.assertIn("Cipher must be AES_CTR_256", self.decrypter.logger.get_lines_for_level('error')[0]) + def test_GET_container_json_not_encrypted_obj(self): + pt_etag = '%s; symlink_path=/a/c/o' % MD5_OF_EMPTY_STRING + + obj_dict = {"bytes": 0, + "last_modified": "2015-04-14T23:33:06.439040", + "hash": pt_etag, + "name": "symlink", + "content_type": 'application/symlink'} + + listing = [obj_dict] + fake_body = json.dumps(listing) + + resp = self._make_cont_get_req(fake_body, 'json') + + self.assertEqual('200 OK', resp.status) + body = resp.body + self.assertEqual(len(body), int(resp.headers['Content-Length'])) + body_json = json.loads(body) + self.assertEqual(1, len(body_json)) + self.assertEqual(pt_etag, body_json[0]['hash']) + class TestModuleMethods(unittest.TestCase): def test_purge_crypto_sysmeta_headers(self): diff --git a/test/unit/common/middleware/test_symlink.py b/test/unit/common/middleware/test_symlink.py new file mode 100644 index 0000000000..1640c3ec06 --- /dev/null +++ b/test/unit/common/middleware/test_symlink.py @@ -0,0 +1,835 @@ +#!/usr/bin/env python +# Copyright (c) 2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import json +import mock + +from six.moves.urllib.parse import quote +from swift.common import swob +from swift.common.middleware import symlink, copy, versioned_writes, \ + listing_formats +from swift.common.swob import Request +from swift.common.utils import MD5_OF_EMPTY_STRING +from test.unit.common.middleware.helpers import FakeSwift +from test.unit.common.middleware.test_versioned_writes import FakeCache + + +class TestSymlinkMiddlewareBase(unittest.TestCase): + def setUp(self): + self.app = FakeSwift() + self.sym = symlink.filter_factory({ + 'symloop_max': '2', + })(self.app) + self.sym.logger = self.app.logger + + def call_app(self, req, app=None, expect_exception=False): + if app is None: + app = self.app + + self.authorized = [] + + def authorize(req): + self.authorized.append(req) + + if 'swift.authorize' not in req.environ: + req.environ['swift.authorize'] = authorize + + status = [None] + headers = [None] + + def start_response(s, h, ei=None): + status[0] = s + headers[0] = h + + body_iter = app(req.environ, start_response) + body = '' + caught_exc = None + try: + for chunk in body_iter: + body += chunk + except Exception as exc: + if expect_exception: + caught_exc = exc + else: + raise + + if expect_exception: + return status[0], headers[0], body, caught_exc + else: + return status[0], headers[0], body + + def call_sym(self, req, **kwargs): + return self.call_app(req, app=self.sym, **kwargs) + + +class TestSymlinkMiddleware(TestSymlinkMiddlewareBase): + def test_symlink_simple_put(self): + self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/symlink', method='PUT', + headers={'X-Symlink-Target': 'c1/o'}, + body='') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '201 Created') + method, path, hdrs = self.app.calls_with_headers[0] + val = hdrs.get('X-Object-Sysmeta-Symlink-Target') + self.assertEqual(val, 'c1/o') + self.assertNotIn('X-Object-Sysmeta-Symlink-Target-Account', hdrs) + val = hdrs.get('X-Object-Sysmeta-Container-Update-Override-Etag') + self.assertEqual(val, '%s; symlink_target=c1/o' % MD5_OF_EMPTY_STRING) + + def test_symlink_put_different_account(self): + self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/symlink', method='PUT', + headers={'X-Symlink-Target': 'c1/o', + 'X-Symlink-Target-Account': 'a1'}, + body='') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '201 Created') + method, path, hdrs = self.app.calls_with_headers[0] + val = hdrs.get('X-Object-Sysmeta-Symlink-Target') + self.assertEqual(val, 'c1/o') + self.assertEqual(hdrs.get('X-Object-Sysmeta-Symlink-Target-Account'), + 'a1') + + def test_symlink_put_leading_slash(self): + self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/symlink', method='PUT', + headers={'X-Symlink-Target': '/c1/o'}, + body='') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '412 Precondition Failed') + self.assertEqual(body, "X-Symlink-Target header must be of " + "the form /") + + def test_symlink_put_non_zero_length(self): + req = Request.blank('/v1/a/c/symlink', method='PUT', body='req_body', + headers={'X-Symlink-Target': 'c1/o'}) + status, headers, body = self.call_sym(req) + self.assertEqual(status, '400 Bad Request') + self.assertEqual(body, 'Symlink requests require a zero byte body') + + def test_symlink_put_bad_object_header(self): + req = Request.blank('/v1/a/c/symlink', method='PUT', + headers={'X-Symlink-Target': 'o'}, + body='') + status, headers, body = self.call_sym(req) + self.assertEqual(status, "412 Precondition Failed") + self.assertEqual(body, "X-Symlink-Target header must be of " + "the form /") + + def test_symlink_put_bad_account_header(self): + req = Request.blank('/v1/a/c/symlink', method='PUT', + headers={'X-Symlink-Target': 'c1/o', + 'X-Symlink-Target-Account': 'a1/c1'}, + body='') + status, headers, body = self.call_sym(req) + self.assertEqual(status, "412 Precondition Failed") + self.assertEqual(body, "Account name cannot contain slashes") + + def test_get_symlink(self): + self.app.register('GET', '/v1/a/c/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o'}) + req = Request.blank('/v1/a/c/symlink?symlink=get', method='GET') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '200 OK') + self.assertIn(('X-Symlink-Target', 'c1/o'), headers) + self.assertNotIn('X-Symlink-Target-Account', dict(headers)) + + def test_get_symlink_with_account(self): + self.app.register('GET', '/v1/a/c/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o', + 'X-Object-Sysmeta-Symlink-Target-Account': 'a2'}) + req = Request.blank('/v1/a/c/symlink?symlink=get', method='GET') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '200 OK') + self.assertIn(('X-Symlink-Target', 'c1/o'), headers) + self.assertIn(('X-Symlink-Target-Account', 'a2'), headers) + + def test_get_symlink_not_found(self): + self.app.register('GET', '/v1/a/c/symlink', swob.HTTPNotFound, {}) + req = Request.blank('/v1/a/c/symlink', method='GET') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '404 Not Found') + self.assertNotIn('Content-Location', dict(headers)) + + def test_get_target_object(self): + self.app.register('GET', '/v1/a/c/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o', + 'X-Object-Sysmeta-Symlink-Target-Account': 'a2'}) + self.app.register('GET', '/v1/a2/c1/o', swob.HTTPOk, {}, 'resp_body') + req = Request.blank('/v1/a/c/symlink', method='GET') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '200 OK') + self.assertEqual(body, 'resp_body') + self.assertNotIn('X-Symlink-Target', dict(headers)) + self.assertNotIn('X-Symlink-Target-Account', dict(headers)) + self.assertIn(('Content-Location', '/v1/a2/c1/o'), headers) + + def test_get_target_object_not_found(self): + self.app.register('GET', '/v1/a/c/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o', + 'X-Object-Sysmeta-Symlink-Target-account': 'a2'}) + self.app.register('GET', '/v1/a2/c1/o', swob.HTTPNotFound, {}, '') + req = Request.blank('/v1/a/c/symlink', method='GET') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '404 Not Found') + self.assertEqual(body, '') + self.assertNotIn('X-Symlink-Target', dict(headers)) + self.assertNotIn('X-Symlink-Target-Account', dict(headers)) + self.assertIn(('Content-Location', '/v1/a2/c1/o'), headers) + + def test_get_target_object_range_not_satisfiable(self): + self.app.register('GET', '/v1/a/c/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o', + 'X-Object-Sysmeta-Symlink-Target-Account': 'a2'}) + self.app.register('GET', '/v1/a2/c1/o', + swob.HTTPRequestedRangeNotSatisfiable, {}, '') + req = Request.blank('/v1/a/c/symlink', method='GET', + headers={'Range': 'bytes=1-2'}) + status, headers, body = self.call_sym(req) + self.assertEqual(status, '416 Requested Range Not Satisfiable') + self.assertEqual( + body, '

Requested Range Not Satisfiable

' + '

The Range requested is not available.

') + self.assertNotIn('X-Symlink-Target', dict(headers)) + self.assertNotIn('X-Symlink-Target-Account', dict(headers)) + self.assertIn(('Content-Location', '/v1/a2/c1/o'), headers) + + def test_get_ec_symlink_range_unsatisfiable_can_redirect_to_target(self): + self.app.register('GET', '/v1/a/c/symlink', + swob.HTTPRequestedRangeNotSatisfiable, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o', + 'X-Object-Sysmeta-Symlink-Target-Account': 'a2'}) + self.app.register('GET', '/v1/a2/c1/o', swob.HTTPOk, + {'Content-Range': 'bytes 1-2/10'}, 'es') + req = Request.blank('/v1/a/c/symlink', method='GET', + headers={'Range': 'bytes=1-2'}) + status, headers, body = self.call_sym(req) + self.assertEqual(status, '200 OK') + self.assertEqual(body, 'es') + self.assertNotIn('X-Symlink-Target', dict(headers)) + self.assertNotIn('X-Symlink-Target-Account', dict(headers)) + self.assertIn(('Content-Location', '/v1/a2/c1/o'), headers) + self.assertIn(('Content-Range', 'bytes 1-2/10'), headers) + + def test_get_non_symlink(self): + # this is not symlink object + self.app.register('GET', '/v1/a/c/obj', swob.HTTPOk, {}, 'resp_body') + req = Request.blank('/v1/a/c/obj', method='GET') + + status, headers, body = self.call_sym(req) + self.assertEqual(status, '200 OK') + self.assertEqual(body, 'resp_body') + + # Assert special headers for symlink are not in response + self.assertNotIn('X-Symlink-Target', dict(headers)) + self.assertNotIn('X-Symlink-Target-Account', dict(headers)) + self.assertNotIn('Content-Location', dict(headers)) + + def test_head_symlink(self): + self.app.register('HEAD', '/v1/a/c/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o', + 'X-Object-Meta-Color': 'Red'}) + req = Request.blank('/v1/a/c/symlink?symlink=get', method='HEAD') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '200 OK') + self.assertIn(('X-Symlink-Target', 'c1/o'), headers) + self.assertNotIn('X-Symlink-Target-Account', dict(headers)) + self.assertIn(('X-Object-Meta-Color', 'Red'), headers) + + def test_head_symlink_with_account(self): + self.app.register('HEAD', '/v1/a/c/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o', + 'X-Object-Sysmeta-Symlink-Target-Account': 'a2', + 'X-Object-Meta-Color': 'Red'}) + req = Request.blank('/v1/a/c/symlink?symlink=get', method='HEAD') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '200 OK') + self.assertIn(('X-Symlink-Target', 'c1/o'), headers) + self.assertIn(('X-Symlink-Target-Account', 'a2'), headers) + self.assertIn(('X-Object-Meta-Color', 'Red'), headers) + + def test_head_target_object(self): + # this test is also validating that the symlink metadata is not + # returned, but the target object metadata does return + self.app.register('HEAD', '/v1/a/c/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o', + 'X-Object-Sysmeta-Symlink-Target-Account': 'a2', + 'X-Object-Meta-Color': 'Red'}) + self.app.register('HEAD', '/v1/a2/c1/o', swob.HTTPOk, + {'X-Object-Meta-Color': 'Green'}) + req = Request.blank('/v1/a/c/symlink', method='HEAD') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '200 OK') + self.assertNotIn('X-Symlink-Target', dict(headers)) + self.assertNotIn('X-Symlink-Target-Account', dict(headers)) + self.assertNotIn(('X-Object-Meta-Color', 'Red'), headers) + self.assertIn(('X-Object-Meta-Color', 'Green'), headers) + self.assertIn(('Content-Location', '/v1/a2/c1/o'), headers) + + def test_symlink_too_deep(self): + self.app.register('HEAD', '/v1/a/c/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c/sym1'}) + self.app.register('HEAD', '/v1/a/c/sym1', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c/sym2'}) + self.app.register('HEAD', '/v1/a/c/sym2', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c/o'}) + req = Request.blank('/v1/a/c/symlink', method='HEAD') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '409 Conflict') + + def test_symlink_change_symloopmax(self): + # similar test to test_symlink_too_deep, but now changed the limit to 3 + self.sym = symlink.filter_factory({ + 'symloop_max': '3', + })(self.app) + self.sym.logger = self.app.logger + self.app.register('HEAD', '/v1/a/c/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c/sym1'}) + self.app.register('HEAD', '/v1/a/c/sym1', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c/sym2'}) + self.app.register('HEAD', '/v1/a/c/sym2', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c/o', + 'X-Object-Meta-Color': 'Red'}) + self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk, + {'X-Object-Meta-Color': 'Green'}) + req = Request.blank('/v1/a/c/symlink', method='HEAD') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '200 OK') + + # assert that the correct metadata was returned + self.assertNotIn(('X-Object-Meta-Color', 'Red'), headers) + self.assertIn(('X-Object-Meta-Color', 'Green'), headers) + + def test_sym_to_sym_to_target(self): + # this test is also validating that the symlink metadata is not + # returned, but the target object metadata does return + self.app.register('HEAD', '/v1/a/c/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c/sym1', + 'X-Object-Meta-Color': 'Red'}) + self.app.register('HEAD', '/v1/a/c/sym1', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o', + 'X-Object-Meta-Color': 'Yellow'}) + self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPOk, + {'X-Object-Meta-Color': 'Green'}) + req = Request.blank('/v1/a/c/symlink', method='HEAD') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '200 OK') + self.assertNotIn(('X-Symlink-Target', 'c1/o'), headers) + self.assertNotIn(('X-Symlink-Target-Account', 'a2'), headers) + self.assertNotIn(('X-Object-Meta-Color', 'Red'), headers) + self.assertNotIn(('X-Object-Meta-Color', 'Yellow'), headers) + self.assertIn(('X-Object-Meta-Color', 'Green'), headers) + self.assertIn(('Content-Location', '/v1/a/c1/o'), headers) + + def test_symlink_post(self): + self.app.register('POST', '/v1/a/c/symlink', swob.HTTPAccepted, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o'}) + req = Request.blank('/v1/a/c/symlink', method='POST', + headers={'X-Object-Meta-Color': 'Red'}) + status, headers, body = self.call_sym(req) + self.assertEqual(status, '307 Temporary Redirect') + self.assertEqual(body, + 'The requested POST was applied to a symlink. POST ' + 'directly to the target to apply requested metadata.') + method, path, hdrs = self.app.calls_with_headers[0] + val = hdrs.get('X-Object-Meta-Color') + self.assertEqual(val, 'Red') + + def test_non_symlink_post(self): + self.app.register('POST', '/v1/a/c/o', swob.HTTPAccepted, {}) + req = Request.blank('/v1/a/c/o', method='POST', + headers={'X-Object-Meta-Color': 'Red'}) + status, headers, body = self.call_sym(req) + self.assertEqual(status, '202 Accepted') + + def test_set_symlink_POST_fail(self): + # Setting a link with a POST request is not allowed + req = Request.blank('/v1/a/c/o', method='POST', + headers={'X-Symlink-Target': 'c1/regular_obj'}) + status, headers, body = self.call_sym(req) + self.assertEqual(status, '400 Bad Request') + self.assertEqual(body, "A PUT request is required to set a symlink " + "target") + + def test_symlink_post_but_fail_at_server(self): + self.app.register('POST', '/v1/a/c/o', swob.HTTPNotFound, {}) + req = Request.blank('/v1/a/c/o', method='POST', + headers={'X-Object-Meta-Color': 'Red'}) + status, headers, body = self.call_sym(req) + self.assertEqual(status, '404 Not Found') + + def test_check_symlink_header(self): + def do_test(headers): + req = Request.blank('/v1/a/c/o', method='PUT', + headers=headers) + symlink._check_symlink_header(req) + + # normal cases + do_test({'X-Symlink-Target': 'c1/o1'}) + do_test({'X-Symlink-Target': 'c1/sub/o1'}) + do_test({'X-Symlink-Target': 'c1%2Fo1'}) + # specify account + do_test({'X-Symlink-Target': 'c1/o1', + 'X-Symlink-Target-Account': 'another'}) + # URL encoded is safe + do_test({'X-Symlink-Target': 'c1%2Fo1'}) + # URL encoded + multibytes is also safe + do_test( + {'X-Symlink-Target': + u'\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3'}) + target = u'\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3' + encoded_target = quote(target.encode('utf-8'), '') + do_test({'X-Symlink-Target': encoded_target}) + + do_test( + {'X-Symlink-Target': 'cont/obj', + 'X-Symlink-Target-Account': u'\u30b0\u30e9\u30d6\u30eb'}) + + def test_check_symlink_header_invalid_format(self): + def do_test(headers, status, err_msg): + req = Request.blank('/v1/a/c/o', method='PUT', + headers=headers) + with self.assertRaises(swob.HTTPException) as cm: + symlink._check_symlink_header(req) + + self.assertEqual(cm.exception.status, status) + self.assertEqual(cm.exception.body, err_msg) + + do_test({'X-Symlink-Target': '/c1/o1'}, + '412 Precondition Failed', + 'X-Symlink-Target header must be of the ' + 'form /') + do_test({'X-Symlink-Target': 'c1o1'}, + '412 Precondition Failed', + 'X-Symlink-Target header must be of the ' + 'form /') + do_test({'X-Symlink-Target': 'c1/o1', + 'X-Symlink-Target-Account': '/another'}, + '412 Precondition Failed', + 'Account name cannot contain slashes') + do_test({'X-Symlink-Target': 'c1/o1', + 'X-Symlink-Target-Account': 'an/other'}, + '412 Precondition Failed', + 'Account name cannot contain slashes') + # url encoded case + do_test({'X-Symlink-Target': '%2Fc1%2Fo1'}, + '412 Precondition Failed', + 'X-Symlink-Target header must be of the ' + 'form /') + do_test({'X-Symlink-Target': 'c1/o1', + 'X-Symlink-Target-Account': '%2Fanother'}, + '412 Precondition Failed', + 'Account name cannot contain slashes') + do_test({'X-Symlink-Target': 'c1/o1', + 'X-Symlink-Target-Account': 'an%2Fother'}, + '412 Precondition Failed', + 'Account name cannot contain slashes') + # with multi-bytes + do_test( + {'X-Symlink-Target': + u'/\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3'}, + '412 Precondition Failed', + 'X-Symlink-Target header must be of the ' + 'form /') + target = u'/\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3' + encoded_target = quote(target.encode('utf-8'), '') + do_test( + {'X-Symlink-Target': encoded_target}, + '412 Precondition Failed', + 'X-Symlink-Target header must be of the ' + 'form /') + account = u'\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3' + encoded_account = quote(account.encode('utf-8'), '') + do_test( + {'X-Symlink-Target': 'c/o', + 'X-Symlink-Target-Account': encoded_account}, + '412 Precondition Failed', + 'Account name cannot contain slashes') + + def test_check_symlink_header_points_to_itself(self): + req = Request.blank('/v1/a/c/o', method='PUT', + headers={'X-Symlink-Target': 'c/o'}) + with self.assertRaises(swob.HTTPException) as cm: + symlink._check_symlink_header(req) + self.assertEqual(cm.exception.status, '400 Bad Request') + self.assertEqual(cm.exception.body, 'Symlink cannot target itself') + + # Even if set account to itself, it will fail as well + req = Request.blank('/v1/a/c/o', method='PUT', + headers={'X-Symlink-Target': 'c/o', + 'X-Symlink-Target-Account': 'a'}) + with self.assertRaises(swob.HTTPException) as cm: + symlink._check_symlink_header(req) + self.assertEqual(cm.exception.status, '400 Bad Request') + self.assertEqual(cm.exception.body, 'Symlink cannot target itself') + + # sanity, the case to another account is safe + req = Request.blank('/v1/a/c/o', method='PUT', + headers={'X-Symlink-Target': 'c/o', + 'X-Symlink-Target-Account': 'a1'}) + symlink._check_symlink_header(req) + + def test_symloop_max_config(self): + self.app = FakeSwift() + # sanity + self.sym = symlink.filter_factory({ + 'symloop_max': '1', + })(self.app) + self.assertEqual(self.sym.symloop_max, 1) + # < 1 case will result in default + self.sym = symlink.filter_factory({ + 'symloop_max': '-1', + })(self.app) + self.assertEqual(self.sym.symloop_max, symlink.DEFAULT_SYMLOOP_MAX) + + +class SymlinkCopyingTestCase(TestSymlinkMiddlewareBase): + # verify interaction of copy and symlink middlewares + + def setUp(self): + self.app = FakeSwift() + conf = {'symloop_max': '2'} + self.sym = symlink.filter_factory(conf)(self.app) + self.sym.logger = self.app.logger + self.copy = copy.filter_factory({})(self.sym) + + def call_copy(self, req, **kwargs): + return self.call_app(req, app=self.copy, **kwargs) + + def test_copy_symlink_target(self): + self.app.register('GET', '/v1/a/src_cont/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o', + 'X-Object-Sysmeta-Symlink-Target-Account': 'a2'}) + self.app.register('GET', '/v1/a2/c1/o', swob.HTTPOk, {}, 'resp_body') + self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, + {}, 'resp_body') + req = Request.blank('/v1/a/src_cont/symlink', method='COPY', + headers={'Destination': 'tgt_cont/tgt_obj'}) + status, headers, body = self.call_copy(req) + self.assertEqual(status, '201 Created') + + def test_copy_symlink(self): + self.app.register('GET', '/v1/a/src_cont/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o', + 'X-Object-Sysmeta-Symlink-Target-Account': 'a2'}) + self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, + {'X-Symlink-Target': 'c1/o', + 'X-Symlink-Target-Account': 'a2'}) + req = Request.blank('/v1/a/src_cont/symlink?symlink=get', + method='COPY', + headers={'Destination': 'tgt_cont/tgt_obj'}) + status, headers, body = self.call_copy(req) + self.assertEqual(status, '201 Created') + method, path, hdrs = self.app.calls_with_headers[1] + val = hdrs.get('X-Object-Sysmeta-Symlink-Target') + self.assertEqual(val, 'c1/o') + self.assertEqual( + hdrs.get('X-Object-Sysmeta-Symlink-Target-Account'), 'a2') + + def test_copy_symlink_new_target(self): + self.app.register('GET', '/v1/a/src_cont/symlink', swob.HTTPOk, + {'X-Object-Sysmeta-Symlink-Target': 'c1/o', + 'X-Object-Sysmeta-Symlink-Target-Account': 'a2'}) + self.app.register('PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, + {'X-Symlink-Target': 'c1/o', + 'X-Symlink-Target-Account': 'a2'}) + req = Request.blank('/v1/a/src_cont/symlink?symlink=get', + method='COPY', + headers={'Destination': 'tgt_cont/tgt_obj', + 'X-Symlink-Target': 'new_cont/new_obj', + 'X-Symlink-Target-Account': 'new_acct'}) + status, headers, body = self.call_copy(req) + self.assertEqual(status, '201 Created') + method, path, hdrs = self.app.calls_with_headers[1] + self.assertEqual(method, 'PUT') + self.assertEqual(path, '/v1/a/tgt_cont/tgt_obj?symlink=get') + val = hdrs.get('X-Object-Sysmeta-Symlink-Target') + self.assertEqual(val, 'new_cont/new_obj') + self.assertEqual(hdrs.get('X-Object-Sysmeta-Symlink-Target-Account'), + 'new_acct') + + +class SymlinkVersioningTestCase(TestSymlinkMiddlewareBase): + # verify interaction of versioned_writes and symlink middlewares + + def setUp(self): + self.app = FakeSwift() + conf = {'symloop_max': '2'} + self.sym = symlink.filter_factory(conf)(self.app) + self.sym.logger = self.app.logger + vw_conf = {'allow_versioned_writes': 'true'} + self.vw = versioned_writes.filter_factory(vw_conf)(self.sym) + + def call_vw(self, req, **kwargs): + return self.call_app(req, app=self.vw, **kwargs) + + def assertRequestEqual(self, req, other): + self.assertEqual(req.method, other.method) + self.assertEqual(req.path, other.path) + + def test_new_symlink_version_success(self): + self.app.register( + 'PUT', '/v1/a/c/symlink', swob.HTTPCreated, + {'X-Symlink-Target': 'new_cont/new_tgt', + 'X-Symlink-Target-Account': 'a'}, None) + self.app.register( + 'GET', '/v1/a/c/symlink', swob.HTTPOk, + {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT', + 'X-Object-Sysmeta-Symlink-Target': 'old_cont/old_tgt', + 'X-Object-Sysmeta-Symlink-Target-Account': 'a'}, + '') + self.app.register( + 'PUT', '/v1/a/ver_cont/007symlink/0000000001.00000', + swob.HTTPCreated, + {'X-Symlink-Target': 'old_cont/old_tgt', + 'X-Symlink-Target-Account': 'a'}, None) + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/c/symlink', + headers={'X-Symlink-Target': 'new_cont/new_tgt'}, + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, + 'CONTENT_LENGTH': '0', + 'swift.trans_id': 'fake_trans_id'}) + status, headers, body = self.call_vw(req) + self.assertEqual(status, '201 Created') + # authorized twice now because versioned_writes now makes a check on + # PUT + self.assertEqual(len(self.authorized), 2) + self.assertRequestEqual(req, self.authorized[0]) + self.assertEqual(['VW', 'VW', None], self.app.swift_sources) + self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids)) + calls = self.app.calls_with_headers + method, path, req_headers = calls[2] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/symlink', path) + self.assertEqual( + 'new_cont/new_tgt', + req_headers['X-Object-Sysmeta-Symlink-Target']) + + def test_delete_latest_version_no_marker_success(self): + self.app.register( + 'GET', + '/v1/a/ver_cont?prefix=003sym/&marker=&reverse=on', + swob.HTTPOk, {}, + '[{"hash": "y", ' + '"last_modified": "2014-11-21T14:23:02.206740", ' + '"bytes": 0, ' + '"name": "003sym/2", ' + '"content_type": "text/plain"}, ' + '{"hash": "x", ' + '"last_modified": "2014-11-21T14:14:27.409100", ' + '"bytes": 0, ' + '"name": "003sym/1", ' + '"content_type": "text/plain"}]') + self.app.register( + 'GET', '/v1/a/ver_cont/003sym/2', swob.HTTPCreated, + {'content-length': '0', + 'X-Object-Sysmeta-Symlink-Target': 'c/tgt'}, None) + self.app.register( + 'PUT', '/v1/a/c/sym', swob.HTTPCreated, + {'X-Symlink-Target': 'c/tgt', 'X-Symlink-Target-Account': 'a'}, + None) + self.app.register( + 'DELETE', '/v1/a/ver_cont/003sym/2', swob.HTTPOk, + {}, None) + + cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}}) + req = Request.blank( + '/v1/a/c/sym', + headers={'X-If-Delete-At': 1}, + environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache, + 'CONTENT_LENGTH': '0', 'swift.trans_id': 'fake_trans_id'}) + status, headers, body = self.call_vw(req) + self.assertEqual(status, '200 OK') + self.assertEqual(len(self.authorized), 1) + self.assertRequestEqual(req, self.authorized[0]) + self.assertEqual(4, self.app.call_count) + self.assertEqual(['VW', 'VW', 'VW', 'VW'], self.app.swift_sources) + self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids)) + calls = self.app.calls_with_headers + method, path, req_headers = calls[2] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/sym', path) + self.assertEqual( + 'c/tgt', + req_headers['X-Object-Sysmeta-Symlink-Target']) + + +class TestSymlinkContainerContext(TestSymlinkMiddlewareBase): + + def setUp(self): + super(TestSymlinkContainerContext, self).setUp() + self.context = symlink.SymlinkContainerContext( + self.sym.app, self.sym.logger) + + def test_extract_symlink_path_json_simple_etag(self): + obj_dict = {"bytes": 6, + "last_modified": "1", + "hash": "etag", + "name": "obj", + "content_type": "application/octet-stream"} + obj_dict = self.context._extract_symlink_path_json( + obj_dict, 'v1', 'AUTH_a') + self.assertEqual(obj_dict['hash'], 'etag') + self.assertNotIn('symlink_path', obj_dict) + + def test_extract_symlink_path_json_symlink_path(self): + obj_dict = {"bytes": 6, + "last_modified": "1", + "hash": "etag; symlink_target=c/o", + "name": "obj", + "content_type": "application/octet-stream"} + obj_dict = self.context._extract_symlink_path_json( + obj_dict, 'v1', 'AUTH_a') + self.assertEqual(obj_dict['hash'], 'etag') + self.assertEqual(obj_dict['symlink_path'], '/v1/AUTH_a/c/o') + + def test_extract_symlink_path_json_symlink_path_and_account(self): + obj_dict = { + "bytes": 6, + "last_modified": "1", + "hash": "etag; symlink_target=c/o; symlink_target_account=AUTH_a2", + "name": "obj", + "content_type": "application/octet-stream"} + obj_dict = self.context._extract_symlink_path_json( + obj_dict, 'v1', 'AUTH_a') + self.assertEqual(obj_dict['hash'], 'etag') + self.assertEqual(obj_dict['symlink_path'], '/v1/AUTH_a2/c/o') + + def test_extract_symlink_path_json_extra_key(self): + obj_dict = {"bytes": 6, + "last_modified": "1", + "hash": "etag; symlink_target=c/o; extra_key=value", + "name": "obj", + "content_type": "application/octet-stream"} + obj_dict = self.context._extract_symlink_path_json( + obj_dict, 'v1', 'AUTH_a') + self.assertEqual(obj_dict['hash'], 'etag; extra_key=value') + self.assertEqual(obj_dict['symlink_path'], '/v1/AUTH_a/c/o') + + def test_get_container_simple(self): + self.app.register( + 'GET', + '/v1/a/c', + swob.HTTPOk, {}, + json.dumps( + [{"hash": "etag; symlink_target=c/o;", + "last_modified": "2014-11-21T14:23:02.206740", + "bytes": 0, + "name": "sym_obj", + "content_type": "text/plain"}, + {"hash": "etag2", + "last_modified": "2014-11-21T14:14:27.409100", + "bytes": 32, + "name": "normal_obj", + "content_type": "text/plain"}])) + req = Request.blank(path='/v1/a/c') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '200 OK') + obj_list = json.loads(body) + self.assertIn('symlink_path', obj_list[0]) + self.assertIn(obj_list[0]['symlink_path'], '/v1/a/c/o') + self.assertNotIn('symlink_path', obj_list[1]) + + def test_get_container_with_subdir(self): + self.app.register( + 'GET', + '/v1/a/c?delimiter=/', + swob.HTTPOk, {}, + json.dumps([{"subdir": "photos/"}])) + req = Request.blank(path='/v1/a/c?delimiter=/') + status, headers, body = self.call_sym(req) + self.assertEqual(status, '200 OK') + obj_list = json.loads(body) + self.assertEqual(len(obj_list), 1) + self.assertEqual(obj_list[0]['subdir'], 'photos/') + + def test_get_container_error_cases(self): + # No affect for error cases + for error in (swob.HTTPNotFound, swob.HTTPUnauthorized, + swob.HTTPServiceUnavailable, + swob.HTTPInternalServerError): + self.app.register('GET', '/v1/a/c', error, {}, '') + req = Request.blank(path='/v1/a/c') + status, headers, body = self.call_sym(req) + self.assertEqual(status, error().status) + + def test_no_affect_for_account_request(self): + with mock.patch.object(self.sym, 'app') as mock_app: + mock_app.return_value = 'ok' + req = Request.blank(path='/v1/a') + status, headers, body = self.call_sym(req) + self.assertEqual(body, 'ok') + + def test_get_container_simple_with_listing_format(self): + self.app.register( + 'GET', + '/v1/a/c?format=json', + swob.HTTPOk, {}, + json.dumps( + [{"hash": "etag; symlink_target=c/o;", + "last_modified": "2014-11-21T14:23:02.206740", + "bytes": 0, + "name": "sym_obj", + "content_type": "text/plain"}, + {"hash": "etag2", + "last_modified": "2014-11-21T14:14:27.409100", + "bytes": 32, + "name": "normal_obj", + "content_type": "text/plain"}])) + self.lf = listing_formats.filter_factory({})(self.sym) + req = Request.blank(path='/v1/a/c?format=json') + status, headers, body = self.call_app(req, app=self.lf) + self.assertEqual(status, '200 OK') + obj_list = json.loads(body) + self.assertIn('symlink_path', obj_list[0]) + self.assertIn(obj_list[0]['symlink_path'], '/v1/a/c/o') + self.assertNotIn('symlink_path', obj_list[1]) + + def test_get_container_simple_with_listing_format_xml(self): + self.app.register( + 'GET', + '/v1/a/c?format=json', + swob.HTTPOk, {'Content-Type': 'application/json'}, + json.dumps( + [{"hash": "etag; symlink_target=c/o;", + "last_modified": "2014-11-21T14:23:02.206740", + "bytes": 0, + "name": "sym_obj", + "content_type": "text/plain"}, + {"hash": "etag2", + "last_modified": "2014-11-21T14:14:27.409100", + "bytes": 32, + "name": "normal_obj", + "content_type": "text/plain"}])) + self.lf = listing_formats.filter_factory({})(self.sym) + req = Request.blank(path='/v1/a/c?format=xml') + status, headers, body = self.call_app(req, app=self.lf) + self.assertEqual(status, '200 OK') + self.assertEqual(body.split('\n'), [ + '', + 'sym_obj' + 'etag0' + 'text/plain' + '2014-11-21T14:23:02.206740' + '/v1/a/c/o' + '' + 'normal_objetag2' + '32text/plain' + '2014-11-21T14:14:27.409100' + '']) diff --git a/test/unit/common/middleware/test_tempurl.py b/test/unit/common/middleware/test_tempurl.py index 5e05e64355..a36ae7956b 100644 --- a/test/unit/common/middleware/test_tempurl.py +++ b/test/unit/common/middleware/test_tempurl.py @@ -857,18 +857,21 @@ class TestTempURL(unittest.TestCase): path = '/v1/a/c/o' key = 'abc' for method in ('PUT', 'POST'): - hmac_body = '%s\n%s\n%s' % (method, expires, path) - sig = hmac.new(key, hmac_body, sha1).hexdigest() - req = self._make_request( - path, method=method, keys=[key], - headers={'x-object-manifest': 'private/secret'}, - environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' - % (sig, expires)}) - resp = req.get_response(self.tempurl) - self.assertEqual(resp.status_int, 400) - self.assertTrue('header' in resp.body) - self.assertTrue('not allowed' in resp.body) - self.assertTrue('X-Object-Manifest' in resp.body) + for hdr, value in [('X-Object-Manifest', 'private/secret'), + ('X-Symlink-Target', 'cont/symlink')]: + hmac_body = '%s\n%s\n%s' % (method, expires, path) + sig = hmac.new(key, hmac_body, sha1).hexdigest() + req = self._make_request( + path, method=method, keys=[key], + headers={hdr: value}, + environ={'QUERY_STRING': + 'temp_url_sig=%s&temp_url_expires=%s' + % (sig, expires)}) + resp = req.get_response(self.tempurl) + self.assertEqual(resp.status_int, 400) + self.assertTrue('header' in resp.body) + self.assertTrue('not allowed' in resp.body) + self.assertTrue(hdr in resp.body) def test_removed_incoming_header(self): self.tempurl = tempurl.filter_factory({ diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index ef6002b715..e075d1bfc1 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -23,7 +23,7 @@ import os import six from six import StringIO from six.moves import range -from six.moves.urllib.parse import quote +from six.moves.urllib.parse import quote, parse_qsl from test.unit import FakeLogger from swift.common import exceptions, internal_client, swob from swift.common.header_key_dict import HeaderKeyDict @@ -316,6 +316,29 @@ class TestInternalClient(unittest.TestCase): client = InternalClient(self) client.make_request('GET', '/', {}, (200,)) + def test_make_request_sets_query_string(self): + captured_envs = [] + + class InternalClient(internal_client.InternalClient): + def __init__(self, test): + self.test = test + self.app = self.fake_app + self.user_agent = 'some_agent' + self.request_tries = 1 + + def fake_app(self, env, start_response): + captured_envs.append(env) + start_response('200 Ok', [('Content-Length', '0')]) + return [] + + client = InternalClient(self) + params = {'param1': 'p1', 'tasty': 'soup'} + client.make_request('GET', '/', {}, (200,), params=params) + actual_params = dict(parse_qsl(captured_envs[0]['QUERY_STRING'], + keep_blank_values=True, + strict_parsing=True)) + self.assertEqual(params, actual_params) + def test_make_request_retries(self): class InternalClient(internal_client.InternalClient): def __init__(self, test): @@ -1047,10 +1070,11 @@ class TestInternalClient(unittest.TestCase): client, app = get_client_app() headers = {'foo': 'bar'} body = 'some_object_body' + params = {'symlink': 'get'} app.register('GET', path_info, swob.HTTPOk, headers, body) req_headers = {'x-important-header': 'some_important_value'} status_int, resp_headers, obj_iter = client.get_object( - account, container, obj, req_headers) + account, container, obj, req_headers, params=params) self.assertEqual(status_int // 100, 2) for k, v in headers.items(): self.assertEqual(v, resp_headers[k]) @@ -1062,7 +1086,7 @@ class TestInternalClient(unittest.TestCase): 'user-agent': 'test', # from InternalClient.make_request }) self.assertEqual(app.calls_with_headers, [( - 'GET', path_info, HeaderKeyDict(req_headers))]) + 'GET', path_info + '?symlink=get', HeaderKeyDict(req_headers))]) def test_iter_object_lines(self): class InternalClient(internal_client.InternalClient): diff --git a/test/unit/container/test_sync.py b/test/unit/container/test_sync.py index f50f746ee8..5c4cc8670a 100644 --- a/test/unit/container/test_sync.py +++ b/test/unit/container/test_sync.py @@ -968,7 +968,9 @@ class TestContainerSync(unittest.TestCase): logger=self.logger) cs.http_proxies = ['http://proxy'] - def fake_get_object(acct, con, obj, headers, acceptable_statuses): + def fake_get_object(acct, con, obj, headers, acceptable_statuses, + params=None): + self.assertEqual({'symlink': 'get'}, params) self.assertEqual(headers['X-Backend-Storage-Policy-Index'], '0') return (200, @@ -1004,7 +1006,9 @@ class TestContainerSync(unittest.TestCase): expected_put_count += 1 self.assertEqual(cs.container_puts, expected_put_count) - def fake_get_object(acct, con, obj, headers, acceptable_statuses): + def fake_get_object(acct, con, obj, headers, acceptable_statuses, + params=None): + self.assertEqual({'symlink': 'get'}, params) self.assertEqual(headers['X-Newest'], True) self.assertEqual(headers['X-Backend-Storage-Policy-Index'], '0') @@ -1055,7 +1059,9 @@ class TestContainerSync(unittest.TestCase): expected_put_count += 1 self.assertEqual(cs.container_puts, expected_put_count) - def fake_get_object(acct, con, obj, headers, acceptable_statuses): + def fake_get_object(acct, con, obj, headers, acceptable_statuses, + params=None): + self.assertEqual({'symlink': 'get'}, params) self.assertEqual(headers['X-Newest'], True) self.assertEqual(headers['X-Backend-Storage-Policy-Index'], '0') @@ -1090,7 +1096,9 @@ class TestContainerSync(unittest.TestCase): exc = [] - def fake_get_object(acct, con, obj, headers, acceptable_statuses): + def fake_get_object(acct, con, obj, headers, acceptable_statuses, + params=None): + self.assertEqual({'symlink': 'get'}, params) self.assertEqual(headers['X-Newest'], True) self.assertEqual(headers['X-Backend-Storage-Policy-Index'], '0') @@ -1114,7 +1122,9 @@ class TestContainerSync(unittest.TestCase): exc = [] - def fake_get_object(acct, con, obj, headers, acceptable_statuses): + def fake_get_object(acct, con, obj, headers, acceptable_statuses, + params=None): + self.assertEqual({'symlink': 'get'}, params) self.assertEqual(headers['X-Newest'], True) self.assertEqual(headers['X-Backend-Storage-Policy-Index'], '0') @@ -1137,7 +1147,9 @@ class TestContainerSync(unittest.TestCase): self.assertEqual(len(exc), 1) self.assertEqual(str(exc[-1]), 'test client exception') - def fake_get_object(acct, con, obj, headers, acceptable_statuses): + def fake_get_object(acct, con, obj, headers, acceptable_statuses, + params=None): + self.assertEqual({'symlink': 'get'}, params) self.assertEqual(headers['X-Newest'], True) self.assertEqual(headers['X-Backend-Storage-Policy-Index'], '0')