From 11eb17d3b268258a1fa60957e33d5cbe8566db98 Mon Sep 17 00:00:00 2001 From: indianwhocodes Date: Tue, 21 Feb 2023 13:26:06 -0800 Subject: [PATCH] support x-open-expired header for expired objects If the global configuration option 'enable_open_expired' is set to true in the config, then the client will be able to make a request with the header 'x-open-expired' set to true in order to access an object that has expired, provided it is in its grace period. If this config flag is set to false, the client will not be able to access any expired objects, even with the header, which is the default behavior unless the flag is set. When a client sets a 'x-open-expired' header to a true value for a GET/HEAD/POST request the proxy will forward x-backend-open-expired to storage server. The storage server will allow clients that set x-backend-open-expired to open and read an object that has not yet been reaped by the object-expirer, even after the x-delete-at time has passed. The header is always ignored when used with temporary URLs. Co-Authored-By: Anish Kachinthaya Related-Change: I106103438c4162a561486ac73a09436e998ae1f0 Change-Id: Ibe7dde0e3bf587d77e14808b169c02f8fb3dddb3 --- doc/source/config/proxy_server_config.rst | 5 + doc/source/overview_expiring_objects.rst | 32 +++ etc/proxy-server.conf-sample | 10 +- swift/common/middleware/tempurl.py | 4 +- swift/common/request_helpers.py | 28 ++ swift/obj/server.py | 12 +- swift/proxy/controllers/obj.py | 7 +- swift/proxy/server.py | 3 + test/functional/test_object.py | 271 +++++++++++++++++++- test/probe/test_object_expirer.py | 202 ++++++++++++++- test/unit/common/middleware/test_tempurl.py | 38 ++- test/unit/common/test_request_helpers.py | 42 ++- test/unit/obj/test_server.py | 246 +++++++++++++++--- test/unit/proxy/controllers/test_obj.py | 197 ++++++++++++++ test/unit/proxy/test_server.py | 3 +- 15 files changed, 1038 insertions(+), 62 deletions(-) diff --git a/doc/source/config/proxy_server_config.rst b/doc/source/config/proxy_server_config.rst index d8f3e3eef3..d3d9d70c9c 100644 --- a/doc/source/config/proxy_server_config.rst +++ b/doc/source/config/proxy_server_config.rst @@ -386,4 +386,9 @@ write_affinity_handoff_delete_count auto The number of l (replicas - len(local_primary_nodes)). This option may be overridden in a per-policy configuration section. +allow_open_expired false If true (default is false), an object that + has expired but not yet been reaped can be + can be accessed by setting the + 'x-open-expired' header to true in + GET, HEAD, and POST requests. ============================================== =============== ===================================== diff --git a/doc/source/overview_expiring_objects.rst b/doc/source/overview_expiring_objects.rst index ef39d7ba74..b52c6e1a32 100644 --- a/doc/source/overview_expiring_objects.rst +++ b/doc/source/overview_expiring_objects.rst @@ -98,6 +98,38 @@ section in the ``object-server.conf``:: account if it exists. By default, no ``delay_reaping`` value is configured for any accounts or containers. +Accessing Objects After Expiration +---------------------------------- + +By default, objects that expire become inaccessible, even to the account owner. +The object may not have been deleted, but any GET/HEAD/POST client request for +the object will respond 404 Not Found after the ``x-delete-at`` timestamp +has passed. + +The ``swift-proxy-server`` offers the ability to globally configure a flag to +allow requests to access expired objects that have not yet been deleted. +When this flag is enabled, a user can make a GET, HEAD, or POST request with +the header ``x-open-expired`` set to true to access the expired object. + +The global configuration is an opt-in flag that can be set in the +``[proxy-server]`` section of the ``proxy-server.conf`` file. It is configured +with a single flag ``allow_open_expired`` set to true or false. By default, +this flag is set to false. + +Here is an example in the ``proxy-server`` section in ``proxy-server.conf``:: + + [proxy-server] + allow_open_expired = false + +To discover whether this flag is set, you can send a **GET** request to the +``/info`` :ref:`discoverability ` path. This will return +configuration data in JSON format where the value of ``allow_open_expired`` is +exposed. + +When using a temporary URL to access the object, this feature is not enabled. +This means that adding the header will not allow requests to temporary URLs +to access expired objects. + Upgrading impact: General Task Queue vs Legacy Queue ---------------------------------------------------- diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 51325fa764..9edb08d76f 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -339,6 +339,14 @@ use = egg:swift#proxy # the environment (default). For more information, see # https://bugs.launchpad.net/liberasurecode/+bug/1886088 # write_legacy_ec_crc = +# +# Setting 'allow_open_expired' to 'true' allows the 'x-open-expired' header +# to be used with HEAD, GET, or POST requests to access expired objects that +# have not yet been deleted from disk. This can be useful in conjunction with +# the object-expirer 'delay_reaping' feature. +# This flag is set to false by default, so it must be changed to access +# expired objects. +# allow_open_expired = false # Some proxy-server configuration options may be overridden on a per-policy # basis by including per-policy config section(s). The value of any option @@ -921,7 +929,7 @@ use = egg:swift#tempurl # list of header names and names can optionally end with '*' to indicate a # prefix match. incoming_allow_headers is a list of exceptions to these # removals. -# incoming_remove_headers = x-timestamp +# incoming_remove_headers = x-timestamp x-open-expired # # The headers allowed as exceptions to incoming_remove_headers. Simply a # whitespace delimited list of header names and names can optionally end with diff --git a/swift/common/middleware/tempurl.py b/swift/common/middleware/tempurl.py index 1f575066db..022f9e8f31 100644 --- a/swift/common/middleware/tempurl.py +++ b/swift/common/middleware/tempurl.py @@ -255,7 +255,7 @@ This middleware understands the following configuration settings: incoming requests. Names may optionally end with ``*`` to indicate a prefix match. ``incoming_allow_headers`` is a list of exceptions to these removals. - Default: ``x-timestamp`` + Default: ``x-timestamp x-open-expired`` ``incoming_allow_headers`` A whitespace-delimited list of the headers allowed as @@ -326,7 +326,7 @@ DISALLOWED_INCOMING_HEADERS = 'x-object-manifest x-symlink-target' #: delimited list of header names and names can optionally end with '*' to #: indicate a prefix match. DEFAULT_INCOMING_ALLOW_HEADERS is a list of #: exceptions to these removals. -DEFAULT_INCOMING_REMOVE_HEADERS = 'x-timestamp' +DEFAULT_INCOMING_REMOVE_HEADERS = 'x-timestamp x-open-expired' #: Default headers as exceptions to DEFAULT_INCOMING_REMOVE_HEADERS. Simply a #: whitespace delimited list of header names and names can optionally end with diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index e785941a23..f38048c3d1 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -993,3 +993,31 @@ def get_ip_port(node, headers): """ return select_ip_port( node, use_replication=is_use_replication_network(headers)) + + +def is_open_expired(app, req): + """ + Helper function to check if a request with the header 'x-open-expired' + can access an object that has not yet been reaped by the object-expirer + based on the allow_open_expired global config. + + :param app: the application instance + :param req: request object + """ + return (config_true_value(app.allow_open_expired) and + config_true_value(req.headers.get('x-open-expired'))) + + +def is_backend_open_expired(request): + """ + Helper function to check if a request has either the headers + 'x-backend-open-expired' or 'x-backend-replication' for the backend + to access expired objects. + + :param request: request object + """ + x_backend_open_expired = config_true_value(request.headers.get( + 'x-backend-open-expired', 'false')) + x_backend_replication = config_true_value(request.headers.get( + 'x-backend-replication', 'false')) + return x_backend_open_expired or x_backend_replication diff --git a/swift/obj/server.py b/swift/obj/server.py index 5853eac9a9..57651066ae 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -50,7 +50,8 @@ from swift.common.base_storage_server import BaseStorageServer from swift.common.header_key_dict import HeaderKeyDict from swift.common.request_helpers import get_name_and_placement, \ is_user_meta, is_sys_or_user_meta, is_object_transient_sysmeta, \ - resolve_etag_is_at_header, is_sys_meta, validate_internal_obj + resolve_etag_is_at_header, is_sys_meta, validate_internal_obj, \ + is_backend_open_expired from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \ HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \ @@ -635,8 +636,7 @@ class ObjectController(BaseStorageServer): try: disk_file = self.get_diskfile( device, partition, account, container, obj, - policy=policy, open_expired=config_true_value( - request.headers.get('x-backend-replication', 'false')), + policy=policy, open_expired=is_backend_open_expired(request), next_part_power=next_part_power) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) @@ -1074,8 +1074,7 @@ class ObjectController(BaseStorageServer): disk_file = self.get_diskfile( device, partition, account, container, obj, policy=policy, frag_prefs=frag_prefs, - open_expired=config_true_value( - request.headers.get('x-backend-replication', 'false'))) + open_expired=is_backend_open_expired(request)) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: @@ -1157,8 +1156,7 @@ class ObjectController(BaseStorageServer): disk_file = self.get_diskfile( device, partition, account, container, obj, policy=policy, frag_prefs=frag_prefs, - open_expired=config_true_value( - request.headers.get('x-backend-replication', 'false'))) + open_expired=is_backend_open_expired(request)) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index a2fdbb704e..fd79e0c970 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -77,7 +77,8 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError, \ normalize_etag, str_to_wsgi from swift.common.request_helpers import update_etag_is_at_header, \ - resolve_etag_is_at_header, validate_internal_obj, get_ip_port + resolve_etag_is_at_header, validate_internal_obj, get_ip_port, \ + is_open_expired def check_content_type(req): @@ -250,6 +251,8 @@ class BaseObjectController(Controller): policy = POLICIES.get_by_index(policy_index) obj_ring = self.app.get_object_ring(policy_index) req.headers['X-Backend-Storage-Policy-Index'] = policy_index + if is_open_expired(self.app, req): + req.headers['X-Backend-Open-Expired'] = 'true' if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: @@ -402,6 +405,8 @@ class BaseObjectController(Controller): container_partition, container_nodes, container_path = \ self._get_update_target(req, container_info) req.acl = container_info['write_acl'] + if is_open_expired(self.app, req): + req.headers['X-Backend-Open-Expired'] = 'true' if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 677dfa8976..3615290af9 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -286,6 +286,8 @@ class Application(object): if a.strip()] self.strict_cors_mode = config_true_value( conf.get('strict_cors_mode', 't')) + self.allow_open_expired = config_true_value( + conf.get('allow_open_expired', 'f')) self.node_timings = {} self.timing_expiry = int(conf.get('timing_expiry', 300)) value = conf.get('request_node_count', '2 * replicas') @@ -347,6 +349,7 @@ class Application(object): policies=POLICIES.get_policy_info(), allow_account_management=self.allow_account_management, account_autocreate=self.account_autocreate, + allow_open_expired=self.allow_open_expired, **constraints.EFFECTIVE_CONSTRAINTS) self.watchdog = Watchdog() self.watchdog.spawn() diff --git a/test/functional/test_object.py b/test/functional/test_object.py index 85b6894d78..24dd8d7291 100644 --- a/test/functional/test_object.py +++ b/test/functional/test_object.py @@ -26,10 +26,11 @@ from xml.dom import minidom import six from six.moves import range +from swift.common.header_key_dict import HeaderKeyDict from test.functional import check_response, retry, requires_acls, \ requires_policies, requires_bulk import test.functional as tf -from swift.common.utils import md5 +from swift.common.utils import md5, config_true_value def setUpModule(): @@ -465,6 +466,274 @@ class TestObject(unittest.TestCase): resp.read() self.assertEqual(resp.status, 201) + def test_open_expired_enabled(self): + allow_open_expired = config_true_value(tf.cluster_info['swift'].get( + 'allow_open_expired', 'false')) + + if not allow_open_expired: + raise SkipTest('allow_open_expired is disabled') + + def put(url, token, parsed, conn): + dt = datetime.datetime.now() + epoch = time.mktime(dt.timetuple()) + delete_time = str(int(epoch) + 2) + conn.request( + 'PUT', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': delete_time}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEqual(resp.status, 201) + + def get(url, token, parsed, conn, extra_headers=None): + headers = {'X-Auth-Token': token} + if extra_headers: + headers.update(extra_headers) + conn.request( + 'GET', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + headers) + return check_response(conn) + + def head(url, token, parsed, conn, extra_headers=None): + headers = {'X-Auth-Token': token} + if extra_headers: + headers.update(extra_headers) + conn.request( + 'HEAD', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + headers) + return check_response(conn) + + def post(url, token, parsed, conn, extra_headers=None): + dt = datetime.datetime.now() + epoch = time.mktime(dt.timetuple()) + delete_time = str(int(epoch) + 2) + headers = {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': delete_time + } + if extra_headers: + headers.update(extra_headers) + conn.request( + 'POST', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + headers) + return check_response(conn) + + resp = retry(get) + resp.read() + count = 0 + while resp.status == 200 and count < 10: + resp = retry(get) + resp.read() + count += 1 + time.sleep(1) + + # check to see object has expired + self.assertEqual(resp.status, 404) + + dt = datetime.datetime.now() + now = str(int(time.mktime(dt.timetuple()))) + resp = retry(get, extra_headers={'X-Open-Expired': True}) + resp.read() + headers = HeaderKeyDict(resp.getheaders()) + # read the expired object with magic x-open-expired header + self.assertEqual(resp.status, 200) + self.assertTrue(now > headers['X-Delete-At']) + + resp = retry(head, extra_headers={'X-Open-Expired': True}) + resp.read() + # head expired object with magic x-open-expired header + self.assertEqual(resp.status, 200) + + resp = retry(get) + resp.read() + # verify object is still expired + self.assertEqual(resp.status, 404) + + # verify object is still expired if x-open-expire is False + resp = retry(get, extra_headers={'X-Open-Expired': False}) + resp.read() + self.assertEqual(resp.status, 404) + + resp = retry(get, extra_headers={'X-Open-Expired': True}) + resp.read() + self.assertEqual(resp.status, 200) + headers = HeaderKeyDict(resp.getheaders()) + self.assertTrue(now > headers['X-Delete-At']) + + resp = retry(head, extra_headers={'X-Open-Expired': False}) + resp.read() + self.assertEqual(resp.status, 404) + + resp = retry(head, extra_headers={'X-Open-Expired': True}) + resp.read() + self.assertEqual(resp.status, 200) + headers = HeaderKeyDict(resp.getheaders()) + self.assertTrue(now > headers['X-Delete-At']) + + resp = retry(post, extra_headers={'X-Open-Expired': False}) + resp.read() + # verify object is not updated and remains deleted + self.assertEqual(resp.status, 404) + + # object got restored with magic x-open-expired header + resp = retry(post, extra_headers={'X-Open-Expired': True, + 'X-Object-Meta-Test': 'restored!'}) + resp.read() + self.assertEqual(resp.status, 202) + + # verify object could be restored and you can do normal GET + resp = retry(get) + resp.read() + self.assertEqual(resp.status, 200) + self.assertIn('X-Object-Meta-Test', resp.headers) + self.assertEqual(resp.headers['x-object-meta-test'], 'restored!') + + # verify object is restored and you can do normal HEAD + resp = retry(head) + resp.read() + self.assertEqual(resp.status, 200) + # verify object is updated with advanced delete time + self.assertIn('X-Delete-At', resp.headers) + + # To avoid an error when the object deletion in tearDown(), + # the object is added again. + resp = retry(put) + resp.read() + self.assertEqual(resp.status, 201) + + def test_allow_open_expired_disabled(self): + allow_open_expired = config_true_value(tf.cluster_info['swift'].get( + 'allow_open_expired', 'false')) + + if allow_open_expired: + raise SkipTest('allow_open_expired is enabled') + + def put(url, token, parsed, conn): + dt = datetime.datetime.now() + epoch = time.mktime(dt.timetuple()) + delete_time = str(int(epoch) + 2) + conn.request( + 'PUT', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': delete_time}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEqual(resp.status, 201) + + def get(url, token, parsed, conn, extra_headers=None): + headers = {'X-Auth-Token': token} + if extra_headers: + headers.update(extra_headers) + conn.request( + 'GET', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + headers) + return check_response(conn) + + def head(url, token, parsed, conn, extra_headers=None): + headers = {'X-Auth-Token': token} + if extra_headers: + headers.update(extra_headers) + conn.request( + 'HEAD', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + headers) + return check_response(conn) + + def post(url, token, parsed, conn, extra_headers=None): + dt = datetime.datetime.now() + epoch = time.mktime(dt.timetuple()) + delete_time = str(int(epoch) + 2) + headers = {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': delete_time + } + if extra_headers: + headers.update(extra_headers) + conn.request( + 'POST', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + headers) + return check_response(conn) + + resp = retry(get) + resp.read() + count = 0 + while resp.status == 200 and count < 10: + resp = retry(get) + resp.read() + count += 1 + time.sleep(1) + + # check to see object has expired + self.assertEqual(resp.status, 404) + + resp = retry(get, extra_headers={'X-Open-Expired': True}) + resp.read() + # read the expired object with magic x-open-expired header + self.assertEqual(resp.status, 404) + + resp = retry(head, extra_headers={'X-Open-Expired': True}) + resp.read() + # head expired object with magic x-open-expired header + self.assertEqual(resp.status, 404) + + resp = retry(get) + resp.read() + # verify object is still expired + self.assertEqual(resp.status, 404) + + # verify object is still expired if x-open-expire is False + resp = retry(get, extra_headers={'X-Open-Expired': False}) + resp.read() + self.assertEqual(resp.status, 404) + + resp = retry(get, extra_headers={'X-Open-Expired': True}) + resp.read() + self.assertEqual(resp.status, 404) + + resp = retry(head, extra_headers={'X-Open-Expired': False}) + resp.read() + self.assertEqual(resp.status, 404) + + resp = retry(head, extra_headers={'X-Open-Expired': True}) + resp.read() + self.assertEqual(resp.status, 404) + + resp = retry(post, extra_headers={'X-Open-Expired': False}) + resp.read() + # verify object is not updated and remains deleted + self.assertEqual(resp.status, 404) + + # object cannot be restored with magic x-open-expired header + resp = retry(post, extra_headers={'X-Open-Expired': True, + 'X-Object-Meta-Test': 'restored!'}) + resp.read() + self.assertEqual(resp.status, 404) + + # To avoid an error when the object deletion in tearDown(), + # the object is added again. + resp = retry(put) + resp.read() + self.assertEqual(resp.status, 201) + def test_non_integer_x_delete_after(self): def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, diff --git a/test/probe/test_object_expirer.py b/test/probe/test_object_expirer.py index e5afb989d8..b08f427ec9 100644 --- a/test/probe/test_object_expirer.py +++ b/test/probe/test_object_expirer.py @@ -17,15 +17,17 @@ import random import time import uuid import unittest +from io import BytesIO from swift.common.internal_client import InternalClient, UnexpectedResponse from swift.common.manager import Manager -from swift.common.utils import Timestamp +from swift.common.utils import Timestamp, config_true_value from test.probe.common import ReplProbeTest, ENABLED_POLICIES from test.probe.brain import BrainSplitter from swiftclient import client +from swiftclient.exceptions import ClientException class TestObjectExpirer(ReplProbeTest): @@ -272,6 +274,204 @@ class TestObjectExpirer(ReplProbeTest): self.assertIn('x-object-meta-expired', metadata) + def _setup_test_open_expired(self): + obj_brain = BrainSplitter(self.url, self.token, self.container_name, + self.object_name, 'object', self.policy) + + obj_brain.put_container() + + now = time.time() + delete_at = int(now + 2) + try: + path = self.client.make_path( + self.account, self.container_name, self.object_name) + self.client.make_request('PUT', path, { + 'X-Delete-At': str(delete_at), + 'X-Timestamp': Timestamp(now).normal, + 'Content-Length': '3', + 'X-Object-Meta-Test': 'foo', + }, (2,), BytesIO(b'foo')) + except UnexpectedResponse as e: + self.fail( + 'Expected 201 for PUT object but got %s' % e.resp.status) + + # sanity: check that the object was created + try: + resp = client.head_object(self.url, self.token, + self.container_name, self.object_name) + self.assertEqual('foo', resp.get('x-object-meta-test')) + except ClientException as e: + self.fail( + 'Expected 200 for HEAD object but got %s' % e.http_status) + + # make sure auto-created containers get in the account listing + Manager(['container-updater']).once() + + # sleep until after expired but not reaped + while time.time() <= delete_at: + time.sleep(0.1) + + # should get a 404, object is expired + with self.assertRaises(ClientException) as e: + client.head_object(self.url, self.token, + self.container_name, self.object_name) + self.assertEqual(e.exception.http_status, 404) + + def test_open_expired_enabled(self): + + # When the global configuration option allow_open_expired is set to + # true, the client should be able to access expired objects that have + # not yet been reaped using the x-open-expired flag. However, after + # they have been reaped, it should return 404. + + allow_open_expired = config_true_value( + self.cluster_info['swift'].get('allow_open_expired') + ) + + if not allow_open_expired: + raise unittest.SkipTest( + "allow_open_expired is disabled in this swift cluster") + + self._setup_test_open_expired() + + # since allow_open_expired is enabled, ensure object can be accessed + # with x-open-expired header + # HEAD request should succeed + try: + resp = client.head_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual('foo', resp.get('x-object-meta-test')) + except ClientException as e: + self.fail( + 'Expected 200 for HEAD object but got %s' % e.http_status) + + # GET request should succeed + try: + _, body = client.get_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual(body, b'foo') + except ClientException as e: + self.fail( + 'Expected 200 for GET object but got %s' % e.http_status) + + # POST request should succeed, update x-delete-at + now = time.time() + new_delete_at = int(now + 5) + try: + client.post_object(self.url, self.token, + self.container_name, self.object_name, + headers={ + 'X-Open-Expired': True, + 'X-Delete-At': str(new_delete_at), + 'X-Object-Meta-Test': 'bar' + }) + except ClientException as e: + self.fail( + 'Expected 200 for POST object but got %s' % e.http_status) + + # GET requests succeed again, even without the magic header + try: + _, body = client.get_object(self.url, self.token, + self.container_name, self.object_name) + self.assertEqual(body, b'foo') + except ClientException as e: + self.fail( + 'Expected 200 for GET object but got %s' % e.http_status) + + # make sure auto-created containers get in the account listing + Manager(['container-updater']).once() + + # run the expirer, but the object expiry time is now in the future + self.expirer.once() + try: + resp = client.head_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual('bar', resp.get('x-object-meta-test')) + except ClientException as e: + self.fail( + 'Expected 200 for HEAD object but got %s' % e.http_status) + + # wait for the object to expire + while time.time() <= new_delete_at: + time.sleep(0.1) + + # expirer runs to reap the object + self.expirer.once() + + # should get a 404 even with x-open-expired since object is reaped + with self.assertRaises(ClientException) as e: + client.head_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual(e.exception.http_status, 404) + + def test_open_expired_disabled(self): + + # When the global configuration option allow_open_expired is set to + # false or not configured, the client should not be able to access + # expired objects that have not yet been reaped using the + # x-open-expired flag. + + allow_open_expired = config_true_value( + self.cluster_info['swift'].get('allow_open_expired') + ) + + if allow_open_expired: + raise unittest.SkipTest( + "allow_open_expired is enabled in this swift cluster") + + self._setup_test_open_expired() + + # since allow_open_expired is disabled, should get 404 even + # with x-open-expired header + # HEAD request should fail + with self.assertRaises(ClientException) as e: + client.head_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual(e.exception.http_status, 404) + + # POST request should fail + with self.assertRaises(ClientException) as e: + client.post_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual(e.exception.http_status, 404) + + # GET request should fail + with self.assertRaises(ClientException) as e: + client.get_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual(e.exception.http_status, 404) + + # But with internal client, can GET with X-Backend-Open-Expired + # Since object still exists on disk + try: + object_metadata = self.client.get_object_metadata( + self.account, self.container_name, self.object_name, + acceptable_statuses=(2,), + headers={'X-Backend-Open-Expired': True}) + except UnexpectedResponse as e: + self.fail( + 'Expected 200 for GET object but got %s' % e.resp.status) + self.assertEqual('foo', object_metadata.get('x-object-meta-test')) + + # expirer runs to reap the object + self.expirer.once() + + # should get a 404 even with X-Backend-Open-Expired + # since object is reaped + with self.assertRaises(UnexpectedResponse) as e: + object_metadata = self.client.get_object_metadata( + self.account, self.container_name, self.object_name, + acceptable_statuses=(2,), + headers={'X-Backend-Open-Expired': True}) + self.assertEqual(e.exception.resp.status_int, 404) + def _test_expirer_delete_outdated_object_version(self, object_exists): # This test simulates a case where the expirer tries to delete # an outdated version of an object. diff --git a/test/unit/common/middleware/test_tempurl.py b/test/unit/common/middleware/test_tempurl.py index 6739a97837..2e2ff0727f 100644 --- a/test/unit/common/middleware/test_tempurl.py +++ b/test/unit/common/middleware/test_tempurl.py @@ -1040,9 +1040,14 @@ class TestTempURL(unittest.TestCase): self.assertIn(b'not allowed', resp.body) self.assertIn(hdr.encode('utf-8'), resp.body) - def test_removed_incoming_header(self): - self.tempurl = tempurl.filter_factory({ - 'incoming_remove_headers': 'x-remove-this'})(self.auth) + def test_removed_incoming_header_defaults(self): + self.tempurl = tempurl.filter_factory({})(self.auth) + + swift_info = registry.get_swift_info() + self.assertIn('tempurl', swift_info) + incoming_remove_headers = \ + swift_info['tempurl']['incoming_remove_headers'] + method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' @@ -1051,12 +1056,33 @@ class TestTempURL(unittest.TestCase): sig = hmac.new(key, hmac_body, hashlib.sha256).hexdigest() req = self._make_request( path, keys=[key], - headers={'x-remove-this': 'value'}, + headers={k: 'test_value' for k in incoming_remove_headers}, + environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( + sig, expires)}) + resp = req.get_response(self.tempurl) + self.assertEqual(resp.status_int, 404) + for incoming_remove_header in incoming_remove_headers: + self.assertNotIn(incoming_remove_header, self.app.request.headers) + + def test_removed_incoming_header(self): + self.tempurl = tempurl.filter_factory({ + 'incoming_remove_headers': 'x-remove-this' + })(self.auth) + method = 'GET' + expires = int(time() + 86400) + path = '/v1/a/c/o' + key = b'abc' + hmac_body = ('%s\n%i\n%s' % (method, expires, path)).encode('utf-8') + sig = hmac.new(key, hmac_body, hashlib.sha256).hexdigest() + req = self._make_request( + path, keys=[key], + headers={'x-remove-this': 'value', 'x-open-expired': 'true'}, environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEqual(resp.status_int, 404) self.assertNotIn('x-remove-this', self.app.request.headers) + self.assertIn('x-open-expired', self.app.request.headers) def test_removed_incoming_headers_match(self): self.tempurl = tempurl.filter_factory({ @@ -1669,7 +1695,7 @@ class TestSwiftInfo(unittest.TestCase): self.assertEqual(set(info['methods']), set(('GET', 'HEAD', 'PUT', 'POST', 'DELETE'))) self.assertEqual(set(info['incoming_remove_headers']), - set(('x-timestamp',))) + set(('x-timestamp', 'x-open-expired',))) self.assertEqual(set(info['incoming_allow_headers']), set()) self.assertEqual(set(info['outgoing_remove_headers']), set(('x-object-meta-*',))) @@ -1709,7 +1735,7 @@ class TestSwiftInfo(unittest.TestCase): self.assertEqual(set(info['methods']), set(('GET', 'HEAD', 'PUT', 'POST', 'DELETE'))) self.assertEqual(set(info['incoming_remove_headers']), - set(('x-timestamp',))) + set(('x-timestamp', 'x-open-expired',))) self.assertEqual(set(info['incoming_allow_headers']), set()) self.assertEqual(set(info['outgoing_remove_headers']), set(('x-object-meta-*',))) diff --git a/test/unit/common/test_request_helpers.py b/test/unit/common/test_request_helpers.py index 47ddc5daf9..767e2d11d5 100644 --- a/test/unit/common/test_request_helpers.py +++ b/test/unit/common/test_request_helpers.py @@ -14,7 +14,7 @@ # limitations under the License. """Tests for swift.common.request_helpers""" - +import argparse import unittest from swift.common.swob import Request, HTTPException, HeaderKeyDict, HTTPOk from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY @@ -473,6 +473,46 @@ class TestRequestHelpers(unittest.TestCase): self.assertEqual(str(ctx.exception), 'Invalid reserved name') + def test_is_open_expired(self): + app = argparse.Namespace(allow_open_expired=False) + req = Request.blank('/v1/a/c/o', headers={'X-Open-Expired': 'yes'}) + self.assertFalse(rh.is_open_expired(app, req)) + req = Request.blank('/v1/a/c/o', headers={'X-Open-Expired': 'no'}) + self.assertFalse(rh.is_open_expired(app, req)) + req = Request.blank('/v1/a/c/o', headers={}) + self.assertFalse(rh.is_open_expired(app, req)) + + app = argparse.Namespace(allow_open_expired=True) + req = Request.blank('/v1/a/c/o', headers={'X-Open-Expired': 'no'}) + self.assertFalse(rh.is_open_expired(app, req)) + req = Request.blank('/v1/a/c/o', headers={}) + self.assertFalse(rh.is_open_expired(app, req)) + + req = Request.blank('/v1/a/c/o', headers={'X-Open-Expired': 'yes'}) + self.assertTrue(rh.is_open_expired(app, req)) + + def test_is_backend_open_expired(self): + req = Request.blank('/v1/a/c/o', headers={ + 'X-Backend-Open-Expired': 'yes' + }) + self.assertTrue(rh.is_backend_open_expired(req)) + req = Request.blank('/v1/a/c/o', headers={ + 'X-Backend-Open-Expired': 'no' + }) + self.assertFalse(rh.is_backend_open_expired(req)) + + req = Request.blank('/v1/a/c/o', headers={ + 'X-Backend-Replication': 'yes' + }) + self.assertTrue(rh.is_backend_open_expired(req)) + req = Request.blank('/v1/a/c/o', headers={ + 'X-Backend-Replication': 'no' + }) + self.assertFalse(rh.is_backend_open_expired(req)) + + req = Request.blank('/v1/a/c/o', headers={}) + self.assertFalse(rh.is_backend_open_expired(req)) + class TestHTTPResponseToDocumentIters(unittest.TestCase): def test_200(self): diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 3c27c0b61a..3d019313c1 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -7009,19 +7009,24 @@ class TestObjectController(BaseTestCase): utils.Timestamp(now)) # ...unless X-Backend-Replication is sent - expected = { - 'GET': b'TEST', - 'HEAD': b'', - } - for meth, expected_body in expected.items(): - req = Request.blank( - '/sda1/p/a/c/o', method=meth, - headers={'X-Timestamp': - normalize_timestamp(delete_at_timestamp + 1), - 'X-Backend-Replication': 'True'}) - resp = req.get_response(self.object_controller) - self.assertEqual(resp.status_int, 200) - self.assertEqual(expected_body, resp.body) + req = Request.blank( + '/sda1/p/a/c/o', method='GET', + headers={'X-Timestamp': + normalize_timestamp(delete_at_timestamp + 1), + 'X-Backend-Replication': 'True'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(b'TEST', resp.body) + + # ...or x-backend-open-expired is sent + req = Request.blank( + '/sda1/p/a/c/o', method='GET', + headers={'X-Timestamp': + normalize_timestamp(delete_at_timestamp + 1), + 'x-backend-open-expired': 'True'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(b'TEST', resp.body) def test_HEAD_but_expired(self): # We have an object that expires in the future @@ -7061,7 +7066,27 @@ class TestObjectController(BaseTestCase): self.assertEqual(resp.headers['X-Backend-Timestamp'], utils.Timestamp(now)) + # It should be accessible with x-backend-open-expired + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'X-Timestamp': normalize_timestamp( + delete_at_timestamp + 2), 'x-backend-open-expired': 'true'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + + # It should be accessible with x-backend-replication + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'X-Timestamp': normalize_timestamp( + delete_at_timestamp + 2), 'x-backend-replication': 'true'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(b'', resp.body) + def test_POST_but_expired(self): + # We have an object that expires in the future now = time() delete_at_timestamp = int(now + 100) delete_at_container = str( @@ -7069,57 +7094,152 @@ class TestObjectController(BaseTestCase): self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) - # We recreate the test object every time to ensure a clean test; a - # POST may change attributes of the object, so it's not safe to - # re-use. - def recreate_test_object(when): - req = Request.blank( - '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(when), - 'X-Delete-At': str(delete_at_timestamp), - 'X-Delete-At-Container': delete_at_container, - 'Content-Length': '4', - 'Content-Type': 'application/octet-stream'}) - req.body = 'TEST' - resp = req.get_response(self.object_controller) - self.assertEqual(resp.status_int, 201) + # PUT the object + req = Request.blank( + '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(now), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, + 'Content-Length': '4', + 'Content-Type': 'application/octet-stream'}) + req.body = b'TEST' + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 201) - # You can POST to a not-yet-expired object - recreate_test_object(now) - the_time = now + 1 + # It's accessible since it expires in the future + the_time = now + 2 req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(the_time)}) + headers={'X-Timestamp': normalize_timestamp(the_time), + 'X-Delete-At': str(delete_at_timestamp)}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 202) - # You cannot POST to an expired object - now += 2 - recreate_test_object(now) + # It's not accessible now since it expires in the past the_time = delete_at_timestamp + 1 req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(the_time)}) + headers={'X-Timestamp': normalize_timestamp(the_time), + 'X-Delete-At': str(delete_at_timestamp + 100)}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 404) - # ...unless sending an x-backend-replication header...which lets you - # modify x-delete-at - now += 2 - recreate_test_object(now) + # It should be accessible with x-backend-open-expired + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'X-Timestamp': normalize_timestamp( + delete_at_timestamp + 2), 'x-backend-open-expired': 'true'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers.get('x-delete-at'), + str(delete_at_timestamp)) + + def test_POST_with_x_backend_open_expired(self): + now = time() + delete_at_timestamp = int(now + 100) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) + + # Create the object at x-delete-at + req = Request.blank( + '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(now), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, + 'Content-Length': '4', + 'Content-Type': 'application/octet-stream'}) + req.body = 'TEST' + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 201) + + # You can POST to an expired object with a much later x-delete-at + # with x-backend-open-expired the_time = delete_at_timestamp + 2 + new_delete_at_timestamp = int(delete_at_timestamp + 100) + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(the_time), + 'x-delete-at': str(new_delete_at_timestamp), + 'x-backend-open-expired': 'true'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 202) + + # Verify the later x-delete-at + the_time = delete_at_timestamp + 2 + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'X-Timestamp': normalize_timestamp(the_time), + 'x-backend-open-expired': 'false'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers.get('x-delete-at'), + str(new_delete_at_timestamp)) + + # Verify object has expired + # We have no x-delete-at in response + the_time = new_delete_at_timestamp + 1 + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'X-Timestamp': normalize_timestamp(the_time), + 'x-backend-open-expired': 'false'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 404) + self.assertIsNone(resp.headers.get('x-delete-at')) + + # But, it works with x-backend-open-expired set to true + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'X-Timestamp': normalize_timestamp(the_time), + 'x-backend-open-expired': 'true'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers.get('x-delete-at'), + str(new_delete_at_timestamp)) + + def test_POST_with_x_backend_replication(self): + now = time() + delete_at_timestamp = int(now + 100) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) + + # Create object with future x-delete-at + req = Request.blank( + '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(now), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, + 'Content-Length': '4', + 'Content-Type': 'application/octet-stream'}) + req.body = 'TEST' + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 201) + + # sending an x-backend-replication header lets you + # modify x-delete-at, even when object is expired + the_time = delete_at_timestamp + 2 + new_delete_at_timestamp = delete_at_timestamp + 100 req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(the_time), 'x-backend-replication': 'true', - 'x-delete-at': str(delete_at_timestamp + 100)}) + 'x-delete-at': str(new_delete_at_timestamp)}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 202) + # ...so the object becomes accessible again even without an - # x-backend-replication header + # x-backend-replication or x-backend-open-expired header the_time = delete_at_timestamp + 3 req = Request.blank( '/sda1/p/a/c/o', @@ -7129,6 +7249,50 @@ class TestObjectController(BaseTestCase): resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 202) + def test_POST_invalid_headers(self): + now = time() + delete_at_timestamp = int(now + 100) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) + + # Create the object at x-delete-at + req = Request.blank( + '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(now), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, + 'Content-Length': '4', + 'Content-Type': 'application/octet-stream'}) + req.body = 'TEST' + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 201) + + # You cannot send an x-delete-at that is in the past with a POST even + # when x-backend-open-expired is sent + the_time = delete_at_timestamp + 75 + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(the_time), + 'x-backend-open-expired': 'true', + 'x-delete-at': str(delete_at_timestamp - 50)}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 400) + + # Object server always ignores x-open-expired and + # only understands x-backend-open-expired on expired objects + the_time = delete_at_timestamp + 2 + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(the_time), + 'x-open-expired': 'true', + 'x-delete-at': str(delete_at_timestamp + 100)}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 404) + def test_DELETE_can_skip_updating_expirer_queue(self): policy = POLICIES.get_by_index(0) test_time = time() diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 34b29e9ea7..419651b13d 100644 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -817,6 +817,129 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin): self.assertEqual(resp.status_int, 400) self.assertEqual(b'X-Delete-At in past', resp.body) + def _test_x_open_expired(self, method, num_reqs, headers=None): + req = swift.common.swob.Request.blank( + '/v1/a/c/o', method=method, headers=headers) + codes = [404] * num_reqs + with mocked_http_conn(*codes) as fake_conn: + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 404) + return fake_conn.requests + + def test_x_open_expired_default_config(self): + for method, num_reqs in ( + ('GET', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('HEAD', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('POST', self.obj_ring.replicas)): + requests = self._test_x_open_expired(method, num_reqs) + for r in requests: + self.assertNotIn('X-Open-Expired', r['headers']) + self.assertNotIn('X-Backend-Open-Expired', r['headers']) + + requests = self._test_x_open_expired( + method, num_reqs, headers={'X-Open-Expired': 'true'}) + for r in requests: + self.assertEqual(r['headers']['X-Open-Expired'], 'true') + self.assertNotIn('X-Backend-Open-Expired', r['headers']) + + requests = self._test_x_open_expired( + method, num_reqs, headers={'X-Open-Expired': 'false'}) + for r in requests: + self.assertEqual(r['headers']['X-Open-Expired'], 'false') + self.assertNotIn('X-Backend-Open-Expired', r['headers']) + + def test_x_open_expired_custom_config(self): + # helper to check that PUT is not supported in all cases + def test_put_unsupported(): + req = swift.common.swob.Request.blank( + '/v1/a/c/o', method='PUT', headers={ + 'Content-Length': '0', + 'X-Open-Expired': 'true'}) + codes = [201] * self.obj_ring.replicas + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with mocked_http_conn( + *codes, expect_headers=expect_headers) as fake_conn: + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 201) + for r in fake_conn.requests: + self.assertEqual(r['headers']['X-Open-Expired'], 'true') + self.assertNotIn('X-Backend-Open-Expired', r['headers']) + + # Allow open expired + # Override app configuration + conf = {'allow_open_expired': 'true'} + # Create a new proxy instance for test with config + self.app = PatchedObjControllerApp( + conf, account_ring=FakeRing(), + container_ring=FakeRing(), logger=None) + # Use the same container info as the app used in other tests + self.app.container_info = dict(self.container_info) + self.obj_ring = self.app.get_object_ring(int(self.policy)) + + for method, num_reqs in ( + ('GET', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('HEAD', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('POST', self.obj_ring.replicas)): + requests = self._test_x_open_expired( + method, num_reqs, headers={'X-Open-Expired': 'true'}) + for r in requests: + # If the proxy server config is has allow_open_expired set + # to true, then we set x-backend-open-expired to true + self.assertEqual(r['headers']['X-Open-Expired'], 'true') + self.assertEqual(r['headers']['X-Backend-Open-Expired'], + 'true') + + for method, num_reqs in ( + ('GET', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('HEAD', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('POST', self.obj_ring.replicas)): + requests = self._test_x_open_expired( + method, num_reqs, headers={'X-Open-Expired': 'false'}) + for r in requests: + # If the proxy server config has allow_open_expired set + # to false, then we set x-backend-open-expired to false + self.assertEqual(r['headers']['X-Open-Expired'], 'false') + self.assertNotIn('X-Backend-Open-Expired', r['headers']) + + # we don't support x-open-expired on PUT when allow_open_expired + test_put_unsupported() + + # Disallow open expired + conf = {'allow_open_expired': 'false'} + # Create a new proxy instance for test with config + self.app = PatchedObjControllerApp( + conf, account_ring=FakeRing(), + container_ring=FakeRing(), logger=None) + # Use the same container info as the app used in other tests + self.app.container_info = dict(self.container_info) + self.obj_ring = self.app.get_object_ring(int(self.policy)) + + for method, num_reqs in ( + ('GET', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('HEAD', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('POST', self.obj_ring.replicas)): + # This case is different: we never add the 'X-Backend-Open-Expired' + # header if the proxy server config disables this feature + requests = self._test_x_open_expired( + method, num_reqs, headers={'X-Open-Expired': 'true'}) + for r in requests: + self.assertEqual(r['headers']['X-Open-Expired'], 'true') + self.assertNotIn('X-Backend-Open-Expired', r['headers']) + + # we don't support x-open-expired on PUT when not allow_open_expired + test_put_unsupported() + def test_HEAD_simple(self): req = swift.common.swob.Request.blank('/v1/a/c/o', method='HEAD') with set_http_connect(200): @@ -2280,6 +2403,80 @@ class TestReplicatedObjController(CommonObjectControllerMixin, self.assertIn('X-Delete-At-Partition', given_headers) self.assertIn('X-Delete-At-Container', given_headers) + def test_POST_delete_at_with_x_open_expired(self): + t_delete = str(int(time.time() + 30)) + + def capture_headers(ip, port, device, part, method, path, headers, + **kwargs): + if method == 'POST': + post_headers.append(headers) + + def do_post(extra_headers): + headers = {'Content-Type': 'foo/bar', + 'X-Delete-At': t_delete} + headers.update(extra_headers) + req_post = swob.Request.blank('/v1/a/c/o', method='POST', body=b'', + headers=headers) + + post_codes = [202] * self.obj_ring.replicas + with set_http_connect(*post_codes, give_connect=capture_headers): + resp = req_post.get_response(self.app) + self.assertEqual(resp.status_int, 202) + self.assertEqual(len(post_headers), self.obj_ring.replicas) + for given_headers in post_headers: + self.assertEqual(given_headers.get('X-Delete-At'), t_delete) + self.assertIn('X-Delete-At-Host', given_headers) + self.assertIn('X-Delete-At-Device', given_headers) + self.assertIn('X-Delete-At-Partition', given_headers) + self.assertIn('X-Delete-At-Container', given_headers) + + # Check when allow_open_expired config is set to true + conf = {'allow_open_expired': 'true'} + self.app = PatchedObjControllerApp( + conf, account_ring=FakeRing(), + container_ring=FakeRing(), logger=None) + self.app.container_info = dict(self.container_info) + self.obj_ring = self.app.get_object_ring(int(self.policy)) + + post_headers = [] + do_post({}) + for given_headers in post_headers: + self.assertNotIn('X-Backend-Open-Expired', given_headers) + + post_headers = [] + do_post({'X-Open-Expired': 'false'}) + for given_headers in post_headers: + self.assertNotIn('X-Backend-Open-Expired', given_headers) + + post_headers = [] + do_post({'X-Open-Expired': 'true'}) + for given_headers in post_headers: + self.assertEqual(given_headers.get('X-Backend-Open-Expired'), + 'true') + + # Check when allow_open_expired config is set to false + conf = {'allow_open_expired': 'false'} + self.app = PatchedObjControllerApp( + conf, account_ring=FakeRing(), + container_ring=FakeRing(), logger=None) + self.app.container_info = dict(self.container_info) + self.obj_ring = self.app.get_object_ring(int(self.policy)) + + post_headers = [] + do_post({}) + for given_headers in post_headers: + self.assertNotIn('X-Backend-Open-Expired', given_headers) + + post_headers = [] + do_post({'X-Open-Expired': 'false'}) + for given_headers in post_headers: + self.assertNotIn('X-Backend-Open-Expired', given_headers) + + post_headers = [] + do_post({'X-Open-Expired': 'true'}) + for given_headers in post_headers: + self.assertNotIn('X-Backend-Open-Expired', given_headers) + def test_PUT_converts_delete_after_to_delete_at(self): req = swob.Request.blank('/v1/a/c/o', method='PUT', body=b'', headers={'Content-Type': 'foo/bar', diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index d3fcb961fb..d51a1ff09a 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -12031,10 +12031,11 @@ class TestSwiftInfo(unittest.TestCase): constraints.MAX_OBJECT_NAME_LENGTH) self.assertIn('strict_cors_mode', si) self.assertFalse(si['allow_account_management']) + self.assertFalse(si['allow_open_expired']) self.assertFalse(si['account_autocreate']) # this next test is deliberately brittle in order to alert if # other items are added to swift info - self.assertEqual(len(si), 17) + self.assertEqual(len(si), 18) si = registry.get_swift_info()['swift'] # Tehse settings is by default excluded by disallowed_sections