diff --git a/doc/source/config/proxy_server_config.rst b/doc/source/config/proxy_server_config.rst index d8f3e3eef3..d3d9d70c9c 100644 --- a/doc/source/config/proxy_server_config.rst +++ b/doc/source/config/proxy_server_config.rst @@ -386,4 +386,9 @@ write_affinity_handoff_delete_count auto The number of l (replicas - len(local_primary_nodes)). This option may be overridden in a per-policy configuration section. +allow_open_expired false If true (default is false), an object that + has expired but not yet been reaped can be + can be accessed by setting the + 'x-open-expired' header to true in + GET, HEAD, and POST requests. ============================================== =============== ===================================== diff --git a/doc/source/overview_expiring_objects.rst b/doc/source/overview_expiring_objects.rst index ef39d7ba74..b52c6e1a32 100644 --- a/doc/source/overview_expiring_objects.rst +++ b/doc/source/overview_expiring_objects.rst @@ -98,6 +98,38 @@ section in the ``object-server.conf``:: account if it exists. By default, no ``delay_reaping`` value is configured for any accounts or containers. +Accessing Objects After Expiration +---------------------------------- + +By default, objects that expire become inaccessible, even to the account owner. +The object may not have been deleted, but any GET/HEAD/POST client request for +the object will respond 404 Not Found after the ``x-delete-at`` timestamp +has passed. + +The ``swift-proxy-server`` offers the ability to globally configure a flag to +allow requests to access expired objects that have not yet been deleted. +When this flag is enabled, a user can make a GET, HEAD, or POST request with +the header ``x-open-expired`` set to true to access the expired object. + +The global configuration is an opt-in flag that can be set in the +``[proxy-server]`` section of the ``proxy-server.conf`` file. It is configured +with a single flag ``allow_open_expired`` set to true or false. By default, +this flag is set to false. + +Here is an example in the ``proxy-server`` section in ``proxy-server.conf``:: + + [proxy-server] + allow_open_expired = false + +To discover whether this flag is set, you can send a **GET** request to the +``/info`` :ref:`discoverability ` path. This will return +configuration data in JSON format where the value of ``allow_open_expired`` is +exposed. + +When using a temporary URL to access the object, this feature is not enabled. +This means that adding the header will not allow requests to temporary URLs +to access expired objects. + Upgrading impact: General Task Queue vs Legacy Queue ---------------------------------------------------- diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 51325fa764..9edb08d76f 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -339,6 +339,14 @@ use = egg:swift#proxy # the environment (default). For more information, see # https://bugs.launchpad.net/liberasurecode/+bug/1886088 # write_legacy_ec_crc = +# +# Setting 'allow_open_expired' to 'true' allows the 'x-open-expired' header +# to be used with HEAD, GET, or POST requests to access expired objects that +# have not yet been deleted from disk. This can be useful in conjunction with +# the object-expirer 'delay_reaping' feature. +# This flag is set to false by default, so it must be changed to access +# expired objects. +# allow_open_expired = false # Some proxy-server configuration options may be overridden on a per-policy # basis by including per-policy config section(s). The value of any option @@ -921,7 +929,7 @@ use = egg:swift#tempurl # list of header names and names can optionally end with '*' to indicate a # prefix match. incoming_allow_headers is a list of exceptions to these # removals. -# incoming_remove_headers = x-timestamp +# incoming_remove_headers = x-timestamp x-open-expired # # The headers allowed as exceptions to incoming_remove_headers. Simply a # whitespace delimited list of header names and names can optionally end with diff --git a/swift/common/middleware/tempurl.py b/swift/common/middleware/tempurl.py index 1f575066db..022f9e8f31 100644 --- a/swift/common/middleware/tempurl.py +++ b/swift/common/middleware/tempurl.py @@ -255,7 +255,7 @@ This middleware understands the following configuration settings: incoming requests. Names may optionally end with ``*`` to indicate a prefix match. ``incoming_allow_headers`` is a list of exceptions to these removals. - Default: ``x-timestamp`` + Default: ``x-timestamp x-open-expired`` ``incoming_allow_headers`` A whitespace-delimited list of the headers allowed as @@ -326,7 +326,7 @@ DISALLOWED_INCOMING_HEADERS = 'x-object-manifest x-symlink-target' #: delimited list of header names and names can optionally end with '*' to #: indicate a prefix match. DEFAULT_INCOMING_ALLOW_HEADERS is a list of #: exceptions to these removals. -DEFAULT_INCOMING_REMOVE_HEADERS = 'x-timestamp' +DEFAULT_INCOMING_REMOVE_HEADERS = 'x-timestamp x-open-expired' #: Default headers as exceptions to DEFAULT_INCOMING_REMOVE_HEADERS. Simply a #: whitespace delimited list of header names and names can optionally end with diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index e785941a23..f38048c3d1 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -993,3 +993,31 @@ def get_ip_port(node, headers): """ return select_ip_port( node, use_replication=is_use_replication_network(headers)) + + +def is_open_expired(app, req): + """ + Helper function to check if a request with the header 'x-open-expired' + can access an object that has not yet been reaped by the object-expirer + based on the allow_open_expired global config. + + :param app: the application instance + :param req: request object + """ + return (config_true_value(app.allow_open_expired) and + config_true_value(req.headers.get('x-open-expired'))) + + +def is_backend_open_expired(request): + """ + Helper function to check if a request has either the headers + 'x-backend-open-expired' or 'x-backend-replication' for the backend + to access expired objects. + + :param request: request object + """ + x_backend_open_expired = config_true_value(request.headers.get( + 'x-backend-open-expired', 'false')) + x_backend_replication = config_true_value(request.headers.get( + 'x-backend-replication', 'false')) + return x_backend_open_expired or x_backend_replication diff --git a/swift/obj/server.py b/swift/obj/server.py index 5853eac9a9..57651066ae 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -50,7 +50,8 @@ from swift.common.base_storage_server import BaseStorageServer from swift.common.header_key_dict import HeaderKeyDict from swift.common.request_helpers import get_name_and_placement, \ is_user_meta, is_sys_or_user_meta, is_object_transient_sysmeta, \ - resolve_etag_is_at_header, is_sys_meta, validate_internal_obj + resolve_etag_is_at_header, is_sys_meta, validate_internal_obj, \ + is_backend_open_expired from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \ HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \ @@ -635,8 +636,7 @@ class ObjectController(BaseStorageServer): try: disk_file = self.get_diskfile( device, partition, account, container, obj, - policy=policy, open_expired=config_true_value( - request.headers.get('x-backend-replication', 'false')), + policy=policy, open_expired=is_backend_open_expired(request), next_part_power=next_part_power) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) @@ -1074,8 +1074,7 @@ class ObjectController(BaseStorageServer): disk_file = self.get_diskfile( device, partition, account, container, obj, policy=policy, frag_prefs=frag_prefs, - open_expired=config_true_value( - request.headers.get('x-backend-replication', 'false'))) + open_expired=is_backend_open_expired(request)) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: @@ -1157,8 +1156,7 @@ class ObjectController(BaseStorageServer): disk_file = self.get_diskfile( device, partition, account, container, obj, policy=policy, frag_prefs=frag_prefs, - open_expired=config_true_value( - request.headers.get('x-backend-replication', 'false'))) + open_expired=is_backend_open_expired(request)) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index a2fdbb704e..fd79e0c970 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -77,7 +77,8 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError, \ normalize_etag, str_to_wsgi from swift.common.request_helpers import update_etag_is_at_header, \ - resolve_etag_is_at_header, validate_internal_obj, get_ip_port + resolve_etag_is_at_header, validate_internal_obj, get_ip_port, \ + is_open_expired def check_content_type(req): @@ -250,6 +251,8 @@ class BaseObjectController(Controller): policy = POLICIES.get_by_index(policy_index) obj_ring = self.app.get_object_ring(policy_index) req.headers['X-Backend-Storage-Policy-Index'] = policy_index + if is_open_expired(self.app, req): + req.headers['X-Backend-Open-Expired'] = 'true' if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: @@ -402,6 +405,8 @@ class BaseObjectController(Controller): container_partition, container_nodes, container_path = \ self._get_update_target(req, container_info) req.acl = container_info['write_acl'] + if is_open_expired(self.app, req): + req.headers['X-Backend-Open-Expired'] = 'true' if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 677dfa8976..3615290af9 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -286,6 +286,8 @@ class Application(object): if a.strip()] self.strict_cors_mode = config_true_value( conf.get('strict_cors_mode', 't')) + self.allow_open_expired = config_true_value( + conf.get('allow_open_expired', 'f')) self.node_timings = {} self.timing_expiry = int(conf.get('timing_expiry', 300)) value = conf.get('request_node_count', '2 * replicas') @@ -347,6 +349,7 @@ class Application(object): policies=POLICIES.get_policy_info(), allow_account_management=self.allow_account_management, account_autocreate=self.account_autocreate, + allow_open_expired=self.allow_open_expired, **constraints.EFFECTIVE_CONSTRAINTS) self.watchdog = Watchdog() self.watchdog.spawn() diff --git a/test/functional/test_object.py b/test/functional/test_object.py index 85b6894d78..24dd8d7291 100644 --- a/test/functional/test_object.py +++ b/test/functional/test_object.py @@ -26,10 +26,11 @@ from xml.dom import minidom import six from six.moves import range +from swift.common.header_key_dict import HeaderKeyDict from test.functional import check_response, retry, requires_acls, \ requires_policies, requires_bulk import test.functional as tf -from swift.common.utils import md5 +from swift.common.utils import md5, config_true_value def setUpModule(): @@ -465,6 +466,274 @@ class TestObject(unittest.TestCase): resp.read() self.assertEqual(resp.status, 201) + def test_open_expired_enabled(self): + allow_open_expired = config_true_value(tf.cluster_info['swift'].get( + 'allow_open_expired', 'false')) + + if not allow_open_expired: + raise SkipTest('allow_open_expired is disabled') + + def put(url, token, parsed, conn): + dt = datetime.datetime.now() + epoch = time.mktime(dt.timetuple()) + delete_time = str(int(epoch) + 2) + conn.request( + 'PUT', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': delete_time}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEqual(resp.status, 201) + + def get(url, token, parsed, conn, extra_headers=None): + headers = {'X-Auth-Token': token} + if extra_headers: + headers.update(extra_headers) + conn.request( + 'GET', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + headers) + return check_response(conn) + + def head(url, token, parsed, conn, extra_headers=None): + headers = {'X-Auth-Token': token} + if extra_headers: + headers.update(extra_headers) + conn.request( + 'HEAD', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + headers) + return check_response(conn) + + def post(url, token, parsed, conn, extra_headers=None): + dt = datetime.datetime.now() + epoch = time.mktime(dt.timetuple()) + delete_time = str(int(epoch) + 2) + headers = {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': delete_time + } + if extra_headers: + headers.update(extra_headers) + conn.request( + 'POST', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + headers) + return check_response(conn) + + resp = retry(get) + resp.read() + count = 0 + while resp.status == 200 and count < 10: + resp = retry(get) + resp.read() + count += 1 + time.sleep(1) + + # check to see object has expired + self.assertEqual(resp.status, 404) + + dt = datetime.datetime.now() + now = str(int(time.mktime(dt.timetuple()))) + resp = retry(get, extra_headers={'X-Open-Expired': True}) + resp.read() + headers = HeaderKeyDict(resp.getheaders()) + # read the expired object with magic x-open-expired header + self.assertEqual(resp.status, 200) + self.assertTrue(now > headers['X-Delete-At']) + + resp = retry(head, extra_headers={'X-Open-Expired': True}) + resp.read() + # head expired object with magic x-open-expired header + self.assertEqual(resp.status, 200) + + resp = retry(get) + resp.read() + # verify object is still expired + self.assertEqual(resp.status, 404) + + # verify object is still expired if x-open-expire is False + resp = retry(get, extra_headers={'X-Open-Expired': False}) + resp.read() + self.assertEqual(resp.status, 404) + + resp = retry(get, extra_headers={'X-Open-Expired': True}) + resp.read() + self.assertEqual(resp.status, 200) + headers = HeaderKeyDict(resp.getheaders()) + self.assertTrue(now > headers['X-Delete-At']) + + resp = retry(head, extra_headers={'X-Open-Expired': False}) + resp.read() + self.assertEqual(resp.status, 404) + + resp = retry(head, extra_headers={'X-Open-Expired': True}) + resp.read() + self.assertEqual(resp.status, 200) + headers = HeaderKeyDict(resp.getheaders()) + self.assertTrue(now > headers['X-Delete-At']) + + resp = retry(post, extra_headers={'X-Open-Expired': False}) + resp.read() + # verify object is not updated and remains deleted + self.assertEqual(resp.status, 404) + + # object got restored with magic x-open-expired header + resp = retry(post, extra_headers={'X-Open-Expired': True, + 'X-Object-Meta-Test': 'restored!'}) + resp.read() + self.assertEqual(resp.status, 202) + + # verify object could be restored and you can do normal GET + resp = retry(get) + resp.read() + self.assertEqual(resp.status, 200) + self.assertIn('X-Object-Meta-Test', resp.headers) + self.assertEqual(resp.headers['x-object-meta-test'], 'restored!') + + # verify object is restored and you can do normal HEAD + resp = retry(head) + resp.read() + self.assertEqual(resp.status, 200) + # verify object is updated with advanced delete time + self.assertIn('X-Delete-At', resp.headers) + + # To avoid an error when the object deletion in tearDown(), + # the object is added again. + resp = retry(put) + resp.read() + self.assertEqual(resp.status, 201) + + def test_allow_open_expired_disabled(self): + allow_open_expired = config_true_value(tf.cluster_info['swift'].get( + 'allow_open_expired', 'false')) + + if allow_open_expired: + raise SkipTest('allow_open_expired is enabled') + + def put(url, token, parsed, conn): + dt = datetime.datetime.now() + epoch = time.mktime(dt.timetuple()) + delete_time = str(int(epoch) + 2) + conn.request( + 'PUT', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': delete_time}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEqual(resp.status, 201) + + def get(url, token, parsed, conn, extra_headers=None): + headers = {'X-Auth-Token': token} + if extra_headers: + headers.update(extra_headers) + conn.request( + 'GET', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + headers) + return check_response(conn) + + def head(url, token, parsed, conn, extra_headers=None): + headers = {'X-Auth-Token': token} + if extra_headers: + headers.update(extra_headers) + conn.request( + 'HEAD', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + headers) + return check_response(conn) + + def post(url, token, parsed, conn, extra_headers=None): + dt = datetime.datetime.now() + epoch = time.mktime(dt.timetuple()) + delete_time = str(int(epoch) + 2) + headers = {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': delete_time + } + if extra_headers: + headers.update(extra_headers) + conn.request( + 'POST', + '%s/%s/%s' % (parsed.path, self.container, 'x_delete_at'), + '', + headers) + return check_response(conn) + + resp = retry(get) + resp.read() + count = 0 + while resp.status == 200 and count < 10: + resp = retry(get) + resp.read() + count += 1 + time.sleep(1) + + # check to see object has expired + self.assertEqual(resp.status, 404) + + resp = retry(get, extra_headers={'X-Open-Expired': True}) + resp.read() + # read the expired object with magic x-open-expired header + self.assertEqual(resp.status, 404) + + resp = retry(head, extra_headers={'X-Open-Expired': True}) + resp.read() + # head expired object with magic x-open-expired header + self.assertEqual(resp.status, 404) + + resp = retry(get) + resp.read() + # verify object is still expired + self.assertEqual(resp.status, 404) + + # verify object is still expired if x-open-expire is False + resp = retry(get, extra_headers={'X-Open-Expired': False}) + resp.read() + self.assertEqual(resp.status, 404) + + resp = retry(get, extra_headers={'X-Open-Expired': True}) + resp.read() + self.assertEqual(resp.status, 404) + + resp = retry(head, extra_headers={'X-Open-Expired': False}) + resp.read() + self.assertEqual(resp.status, 404) + + resp = retry(head, extra_headers={'X-Open-Expired': True}) + resp.read() + self.assertEqual(resp.status, 404) + + resp = retry(post, extra_headers={'X-Open-Expired': False}) + resp.read() + # verify object is not updated and remains deleted + self.assertEqual(resp.status, 404) + + # object cannot be restored with magic x-open-expired header + resp = retry(post, extra_headers={'X-Open-Expired': True, + 'X-Object-Meta-Test': 'restored!'}) + resp.read() + self.assertEqual(resp.status, 404) + + # To avoid an error when the object deletion in tearDown(), + # the object is added again. + resp = retry(put) + resp.read() + self.assertEqual(resp.status, 201) + def test_non_integer_x_delete_after(self): def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, diff --git a/test/probe/test_object_expirer.py b/test/probe/test_object_expirer.py index e5afb989d8..b08f427ec9 100644 --- a/test/probe/test_object_expirer.py +++ b/test/probe/test_object_expirer.py @@ -17,15 +17,17 @@ import random import time import uuid import unittest +from io import BytesIO from swift.common.internal_client import InternalClient, UnexpectedResponse from swift.common.manager import Manager -from swift.common.utils import Timestamp +from swift.common.utils import Timestamp, config_true_value from test.probe.common import ReplProbeTest, ENABLED_POLICIES from test.probe.brain import BrainSplitter from swiftclient import client +from swiftclient.exceptions import ClientException class TestObjectExpirer(ReplProbeTest): @@ -272,6 +274,204 @@ class TestObjectExpirer(ReplProbeTest): self.assertIn('x-object-meta-expired', metadata) + def _setup_test_open_expired(self): + obj_brain = BrainSplitter(self.url, self.token, self.container_name, + self.object_name, 'object', self.policy) + + obj_brain.put_container() + + now = time.time() + delete_at = int(now + 2) + try: + path = self.client.make_path( + self.account, self.container_name, self.object_name) + self.client.make_request('PUT', path, { + 'X-Delete-At': str(delete_at), + 'X-Timestamp': Timestamp(now).normal, + 'Content-Length': '3', + 'X-Object-Meta-Test': 'foo', + }, (2,), BytesIO(b'foo')) + except UnexpectedResponse as e: + self.fail( + 'Expected 201 for PUT object but got %s' % e.resp.status) + + # sanity: check that the object was created + try: + resp = client.head_object(self.url, self.token, + self.container_name, self.object_name) + self.assertEqual('foo', resp.get('x-object-meta-test')) + except ClientException as e: + self.fail( + 'Expected 200 for HEAD object but got %s' % e.http_status) + + # make sure auto-created containers get in the account listing + Manager(['container-updater']).once() + + # sleep until after expired but not reaped + while time.time() <= delete_at: + time.sleep(0.1) + + # should get a 404, object is expired + with self.assertRaises(ClientException) as e: + client.head_object(self.url, self.token, + self.container_name, self.object_name) + self.assertEqual(e.exception.http_status, 404) + + def test_open_expired_enabled(self): + + # When the global configuration option allow_open_expired is set to + # true, the client should be able to access expired objects that have + # not yet been reaped using the x-open-expired flag. However, after + # they have been reaped, it should return 404. + + allow_open_expired = config_true_value( + self.cluster_info['swift'].get('allow_open_expired') + ) + + if not allow_open_expired: + raise unittest.SkipTest( + "allow_open_expired is disabled in this swift cluster") + + self._setup_test_open_expired() + + # since allow_open_expired is enabled, ensure object can be accessed + # with x-open-expired header + # HEAD request should succeed + try: + resp = client.head_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual('foo', resp.get('x-object-meta-test')) + except ClientException as e: + self.fail( + 'Expected 200 for HEAD object but got %s' % e.http_status) + + # GET request should succeed + try: + _, body = client.get_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual(body, b'foo') + except ClientException as e: + self.fail( + 'Expected 200 for GET object but got %s' % e.http_status) + + # POST request should succeed, update x-delete-at + now = time.time() + new_delete_at = int(now + 5) + try: + client.post_object(self.url, self.token, + self.container_name, self.object_name, + headers={ + 'X-Open-Expired': True, + 'X-Delete-At': str(new_delete_at), + 'X-Object-Meta-Test': 'bar' + }) + except ClientException as e: + self.fail( + 'Expected 200 for POST object but got %s' % e.http_status) + + # GET requests succeed again, even without the magic header + try: + _, body = client.get_object(self.url, self.token, + self.container_name, self.object_name) + self.assertEqual(body, b'foo') + except ClientException as e: + self.fail( + 'Expected 200 for GET object but got %s' % e.http_status) + + # make sure auto-created containers get in the account listing + Manager(['container-updater']).once() + + # run the expirer, but the object expiry time is now in the future + self.expirer.once() + try: + resp = client.head_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual('bar', resp.get('x-object-meta-test')) + except ClientException as e: + self.fail( + 'Expected 200 for HEAD object but got %s' % e.http_status) + + # wait for the object to expire + while time.time() <= new_delete_at: + time.sleep(0.1) + + # expirer runs to reap the object + self.expirer.once() + + # should get a 404 even with x-open-expired since object is reaped + with self.assertRaises(ClientException) as e: + client.head_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual(e.exception.http_status, 404) + + def test_open_expired_disabled(self): + + # When the global configuration option allow_open_expired is set to + # false or not configured, the client should not be able to access + # expired objects that have not yet been reaped using the + # x-open-expired flag. + + allow_open_expired = config_true_value( + self.cluster_info['swift'].get('allow_open_expired') + ) + + if allow_open_expired: + raise unittest.SkipTest( + "allow_open_expired is enabled in this swift cluster") + + self._setup_test_open_expired() + + # since allow_open_expired is disabled, should get 404 even + # with x-open-expired header + # HEAD request should fail + with self.assertRaises(ClientException) as e: + client.head_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual(e.exception.http_status, 404) + + # POST request should fail + with self.assertRaises(ClientException) as e: + client.post_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual(e.exception.http_status, 404) + + # GET request should fail + with self.assertRaises(ClientException) as e: + client.get_object(self.url, self.token, + self.container_name, self.object_name, + headers={'X-Open-Expired': True}) + self.assertEqual(e.exception.http_status, 404) + + # But with internal client, can GET with X-Backend-Open-Expired + # Since object still exists on disk + try: + object_metadata = self.client.get_object_metadata( + self.account, self.container_name, self.object_name, + acceptable_statuses=(2,), + headers={'X-Backend-Open-Expired': True}) + except UnexpectedResponse as e: + self.fail( + 'Expected 200 for GET object but got %s' % e.resp.status) + self.assertEqual('foo', object_metadata.get('x-object-meta-test')) + + # expirer runs to reap the object + self.expirer.once() + + # should get a 404 even with X-Backend-Open-Expired + # since object is reaped + with self.assertRaises(UnexpectedResponse) as e: + object_metadata = self.client.get_object_metadata( + self.account, self.container_name, self.object_name, + acceptable_statuses=(2,), + headers={'X-Backend-Open-Expired': True}) + self.assertEqual(e.exception.resp.status_int, 404) + def _test_expirer_delete_outdated_object_version(self, object_exists): # This test simulates a case where the expirer tries to delete # an outdated version of an object. diff --git a/test/unit/common/middleware/test_tempurl.py b/test/unit/common/middleware/test_tempurl.py index 6739a97837..2e2ff0727f 100644 --- a/test/unit/common/middleware/test_tempurl.py +++ b/test/unit/common/middleware/test_tempurl.py @@ -1040,9 +1040,14 @@ class TestTempURL(unittest.TestCase): self.assertIn(b'not allowed', resp.body) self.assertIn(hdr.encode('utf-8'), resp.body) - def test_removed_incoming_header(self): - self.tempurl = tempurl.filter_factory({ - 'incoming_remove_headers': 'x-remove-this'})(self.auth) + def test_removed_incoming_header_defaults(self): + self.tempurl = tempurl.filter_factory({})(self.auth) + + swift_info = registry.get_swift_info() + self.assertIn('tempurl', swift_info) + incoming_remove_headers = \ + swift_info['tempurl']['incoming_remove_headers'] + method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' @@ -1051,12 +1056,33 @@ class TestTempURL(unittest.TestCase): sig = hmac.new(key, hmac_body, hashlib.sha256).hexdigest() req = self._make_request( path, keys=[key], - headers={'x-remove-this': 'value'}, + headers={k: 'test_value' for k in incoming_remove_headers}, + environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( + sig, expires)}) + resp = req.get_response(self.tempurl) + self.assertEqual(resp.status_int, 404) + for incoming_remove_header in incoming_remove_headers: + self.assertNotIn(incoming_remove_header, self.app.request.headers) + + def test_removed_incoming_header(self): + self.tempurl = tempurl.filter_factory({ + 'incoming_remove_headers': 'x-remove-this' + })(self.auth) + method = 'GET' + expires = int(time() + 86400) + path = '/v1/a/c/o' + key = b'abc' + hmac_body = ('%s\n%i\n%s' % (method, expires, path)).encode('utf-8') + sig = hmac.new(key, hmac_body, hashlib.sha256).hexdigest() + req = self._make_request( + path, keys=[key], + headers={'x-remove-this': 'value', 'x-open-expired': 'true'}, environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEqual(resp.status_int, 404) self.assertNotIn('x-remove-this', self.app.request.headers) + self.assertIn('x-open-expired', self.app.request.headers) def test_removed_incoming_headers_match(self): self.tempurl = tempurl.filter_factory({ @@ -1669,7 +1695,7 @@ class TestSwiftInfo(unittest.TestCase): self.assertEqual(set(info['methods']), set(('GET', 'HEAD', 'PUT', 'POST', 'DELETE'))) self.assertEqual(set(info['incoming_remove_headers']), - set(('x-timestamp',))) + set(('x-timestamp', 'x-open-expired',))) self.assertEqual(set(info['incoming_allow_headers']), set()) self.assertEqual(set(info['outgoing_remove_headers']), set(('x-object-meta-*',))) @@ -1709,7 +1735,7 @@ class TestSwiftInfo(unittest.TestCase): self.assertEqual(set(info['methods']), set(('GET', 'HEAD', 'PUT', 'POST', 'DELETE'))) self.assertEqual(set(info['incoming_remove_headers']), - set(('x-timestamp',))) + set(('x-timestamp', 'x-open-expired',))) self.assertEqual(set(info['incoming_allow_headers']), set()) self.assertEqual(set(info['outgoing_remove_headers']), set(('x-object-meta-*',))) diff --git a/test/unit/common/test_request_helpers.py b/test/unit/common/test_request_helpers.py index 47ddc5daf9..767e2d11d5 100644 --- a/test/unit/common/test_request_helpers.py +++ b/test/unit/common/test_request_helpers.py @@ -14,7 +14,7 @@ # limitations under the License. """Tests for swift.common.request_helpers""" - +import argparse import unittest from swift.common.swob import Request, HTTPException, HeaderKeyDict, HTTPOk from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY @@ -473,6 +473,46 @@ class TestRequestHelpers(unittest.TestCase): self.assertEqual(str(ctx.exception), 'Invalid reserved name') + def test_is_open_expired(self): + app = argparse.Namespace(allow_open_expired=False) + req = Request.blank('/v1/a/c/o', headers={'X-Open-Expired': 'yes'}) + self.assertFalse(rh.is_open_expired(app, req)) + req = Request.blank('/v1/a/c/o', headers={'X-Open-Expired': 'no'}) + self.assertFalse(rh.is_open_expired(app, req)) + req = Request.blank('/v1/a/c/o', headers={}) + self.assertFalse(rh.is_open_expired(app, req)) + + app = argparse.Namespace(allow_open_expired=True) + req = Request.blank('/v1/a/c/o', headers={'X-Open-Expired': 'no'}) + self.assertFalse(rh.is_open_expired(app, req)) + req = Request.blank('/v1/a/c/o', headers={}) + self.assertFalse(rh.is_open_expired(app, req)) + + req = Request.blank('/v1/a/c/o', headers={'X-Open-Expired': 'yes'}) + self.assertTrue(rh.is_open_expired(app, req)) + + def test_is_backend_open_expired(self): + req = Request.blank('/v1/a/c/o', headers={ + 'X-Backend-Open-Expired': 'yes' + }) + self.assertTrue(rh.is_backend_open_expired(req)) + req = Request.blank('/v1/a/c/o', headers={ + 'X-Backend-Open-Expired': 'no' + }) + self.assertFalse(rh.is_backend_open_expired(req)) + + req = Request.blank('/v1/a/c/o', headers={ + 'X-Backend-Replication': 'yes' + }) + self.assertTrue(rh.is_backend_open_expired(req)) + req = Request.blank('/v1/a/c/o', headers={ + 'X-Backend-Replication': 'no' + }) + self.assertFalse(rh.is_backend_open_expired(req)) + + req = Request.blank('/v1/a/c/o', headers={}) + self.assertFalse(rh.is_backend_open_expired(req)) + class TestHTTPResponseToDocumentIters(unittest.TestCase): def test_200(self): diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 3c27c0b61a..3d019313c1 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -7009,19 +7009,24 @@ class TestObjectController(BaseTestCase): utils.Timestamp(now)) # ...unless X-Backend-Replication is sent - expected = { - 'GET': b'TEST', - 'HEAD': b'', - } - for meth, expected_body in expected.items(): - req = Request.blank( - '/sda1/p/a/c/o', method=meth, - headers={'X-Timestamp': - normalize_timestamp(delete_at_timestamp + 1), - 'X-Backend-Replication': 'True'}) - resp = req.get_response(self.object_controller) - self.assertEqual(resp.status_int, 200) - self.assertEqual(expected_body, resp.body) + req = Request.blank( + '/sda1/p/a/c/o', method='GET', + headers={'X-Timestamp': + normalize_timestamp(delete_at_timestamp + 1), + 'X-Backend-Replication': 'True'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(b'TEST', resp.body) + + # ...or x-backend-open-expired is sent + req = Request.blank( + '/sda1/p/a/c/o', method='GET', + headers={'X-Timestamp': + normalize_timestamp(delete_at_timestamp + 1), + 'x-backend-open-expired': 'True'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(b'TEST', resp.body) def test_HEAD_but_expired(self): # We have an object that expires in the future @@ -7061,7 +7066,27 @@ class TestObjectController(BaseTestCase): self.assertEqual(resp.headers['X-Backend-Timestamp'], utils.Timestamp(now)) + # It should be accessible with x-backend-open-expired + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'X-Timestamp': normalize_timestamp( + delete_at_timestamp + 2), 'x-backend-open-expired': 'true'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + + # It should be accessible with x-backend-replication + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'X-Timestamp': normalize_timestamp( + delete_at_timestamp + 2), 'x-backend-replication': 'true'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(b'', resp.body) + def test_POST_but_expired(self): + # We have an object that expires in the future now = time() delete_at_timestamp = int(now + 100) delete_at_container = str( @@ -7069,57 +7094,152 @@ class TestObjectController(BaseTestCase): self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) - # We recreate the test object every time to ensure a clean test; a - # POST may change attributes of the object, so it's not safe to - # re-use. - def recreate_test_object(when): - req = Request.blank( - '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(when), - 'X-Delete-At': str(delete_at_timestamp), - 'X-Delete-At-Container': delete_at_container, - 'Content-Length': '4', - 'Content-Type': 'application/octet-stream'}) - req.body = 'TEST' - resp = req.get_response(self.object_controller) - self.assertEqual(resp.status_int, 201) + # PUT the object + req = Request.blank( + '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(now), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, + 'Content-Length': '4', + 'Content-Type': 'application/octet-stream'}) + req.body = b'TEST' + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 201) - # You can POST to a not-yet-expired object - recreate_test_object(now) - the_time = now + 1 + # It's accessible since it expires in the future + the_time = now + 2 req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(the_time)}) + headers={'X-Timestamp': normalize_timestamp(the_time), + 'X-Delete-At': str(delete_at_timestamp)}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 202) - # You cannot POST to an expired object - now += 2 - recreate_test_object(now) + # It's not accessible now since it expires in the past the_time = delete_at_timestamp + 1 req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(the_time)}) + headers={'X-Timestamp': normalize_timestamp(the_time), + 'X-Delete-At': str(delete_at_timestamp + 100)}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 404) - # ...unless sending an x-backend-replication header...which lets you - # modify x-delete-at - now += 2 - recreate_test_object(now) + # It should be accessible with x-backend-open-expired + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'X-Timestamp': normalize_timestamp( + delete_at_timestamp + 2), 'x-backend-open-expired': 'true'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers.get('x-delete-at'), + str(delete_at_timestamp)) + + def test_POST_with_x_backend_open_expired(self): + now = time() + delete_at_timestamp = int(now + 100) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) + + # Create the object at x-delete-at + req = Request.blank( + '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(now), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, + 'Content-Length': '4', + 'Content-Type': 'application/octet-stream'}) + req.body = 'TEST' + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 201) + + # You can POST to an expired object with a much later x-delete-at + # with x-backend-open-expired the_time = delete_at_timestamp + 2 + new_delete_at_timestamp = int(delete_at_timestamp + 100) + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(the_time), + 'x-delete-at': str(new_delete_at_timestamp), + 'x-backend-open-expired': 'true'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 202) + + # Verify the later x-delete-at + the_time = delete_at_timestamp + 2 + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'X-Timestamp': normalize_timestamp(the_time), + 'x-backend-open-expired': 'false'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers.get('x-delete-at'), + str(new_delete_at_timestamp)) + + # Verify object has expired + # We have no x-delete-at in response + the_time = new_delete_at_timestamp + 1 + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'X-Timestamp': normalize_timestamp(the_time), + 'x-backend-open-expired': 'false'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 404) + self.assertIsNone(resp.headers.get('x-delete-at')) + + # But, it works with x-backend-open-expired set to true + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'X-Timestamp': normalize_timestamp(the_time), + 'x-backend-open-expired': 'true'}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers.get('x-delete-at'), + str(new_delete_at_timestamp)) + + def test_POST_with_x_backend_replication(self): + now = time() + delete_at_timestamp = int(now + 100) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) + + # Create object with future x-delete-at + req = Request.blank( + '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(now), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, + 'Content-Length': '4', + 'Content-Type': 'application/octet-stream'}) + req.body = 'TEST' + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 201) + + # sending an x-backend-replication header lets you + # modify x-delete-at, even when object is expired + the_time = delete_at_timestamp + 2 + new_delete_at_timestamp = delete_at_timestamp + 100 req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(the_time), 'x-backend-replication': 'true', - 'x-delete-at': str(delete_at_timestamp + 100)}) + 'x-delete-at': str(new_delete_at_timestamp)}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 202) + # ...so the object becomes accessible again even without an - # x-backend-replication header + # x-backend-replication or x-backend-open-expired header the_time = delete_at_timestamp + 3 req = Request.blank( '/sda1/p/a/c/o', @@ -7129,6 +7249,50 @@ class TestObjectController(BaseTestCase): resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 202) + def test_POST_invalid_headers(self): + now = time() + delete_at_timestamp = int(now + 100) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) + + # Create the object at x-delete-at + req = Request.blank( + '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(now), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, + 'Content-Length': '4', + 'Content-Type': 'application/octet-stream'}) + req.body = 'TEST' + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 201) + + # You cannot send an x-delete-at that is in the past with a POST even + # when x-backend-open-expired is sent + the_time = delete_at_timestamp + 75 + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(the_time), + 'x-backend-open-expired': 'true', + 'x-delete-at': str(delete_at_timestamp - 50)}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 400) + + # Object server always ignores x-open-expired and + # only understands x-backend-open-expired on expired objects + the_time = delete_at_timestamp + 2 + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(the_time), + 'x-open-expired': 'true', + 'x-delete-at': str(delete_at_timestamp + 100)}) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 404) + def test_DELETE_can_skip_updating_expirer_queue(self): policy = POLICIES.get_by_index(0) test_time = time() diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 34b29e9ea7..419651b13d 100644 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -817,6 +817,129 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin): self.assertEqual(resp.status_int, 400) self.assertEqual(b'X-Delete-At in past', resp.body) + def _test_x_open_expired(self, method, num_reqs, headers=None): + req = swift.common.swob.Request.blank( + '/v1/a/c/o', method=method, headers=headers) + codes = [404] * num_reqs + with mocked_http_conn(*codes) as fake_conn: + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 404) + return fake_conn.requests + + def test_x_open_expired_default_config(self): + for method, num_reqs in ( + ('GET', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('HEAD', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('POST', self.obj_ring.replicas)): + requests = self._test_x_open_expired(method, num_reqs) + for r in requests: + self.assertNotIn('X-Open-Expired', r['headers']) + self.assertNotIn('X-Backend-Open-Expired', r['headers']) + + requests = self._test_x_open_expired( + method, num_reqs, headers={'X-Open-Expired': 'true'}) + for r in requests: + self.assertEqual(r['headers']['X-Open-Expired'], 'true') + self.assertNotIn('X-Backend-Open-Expired', r['headers']) + + requests = self._test_x_open_expired( + method, num_reqs, headers={'X-Open-Expired': 'false'}) + for r in requests: + self.assertEqual(r['headers']['X-Open-Expired'], 'false') + self.assertNotIn('X-Backend-Open-Expired', r['headers']) + + def test_x_open_expired_custom_config(self): + # helper to check that PUT is not supported in all cases + def test_put_unsupported(): + req = swift.common.swob.Request.blank( + '/v1/a/c/o', method='PUT', headers={ + 'Content-Length': '0', + 'X-Open-Expired': 'true'}) + codes = [201] * self.obj_ring.replicas + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with mocked_http_conn( + *codes, expect_headers=expect_headers) as fake_conn: + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 201) + for r in fake_conn.requests: + self.assertEqual(r['headers']['X-Open-Expired'], 'true') + self.assertNotIn('X-Backend-Open-Expired', r['headers']) + + # Allow open expired + # Override app configuration + conf = {'allow_open_expired': 'true'} + # Create a new proxy instance for test with config + self.app = PatchedObjControllerApp( + conf, account_ring=FakeRing(), + container_ring=FakeRing(), logger=None) + # Use the same container info as the app used in other tests + self.app.container_info = dict(self.container_info) + self.obj_ring = self.app.get_object_ring(int(self.policy)) + + for method, num_reqs in ( + ('GET', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('HEAD', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('POST', self.obj_ring.replicas)): + requests = self._test_x_open_expired( + method, num_reqs, headers={'X-Open-Expired': 'true'}) + for r in requests: + # If the proxy server config is has allow_open_expired set + # to true, then we set x-backend-open-expired to true + self.assertEqual(r['headers']['X-Open-Expired'], 'true') + self.assertEqual(r['headers']['X-Backend-Open-Expired'], + 'true') + + for method, num_reqs in ( + ('GET', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('HEAD', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('POST', self.obj_ring.replicas)): + requests = self._test_x_open_expired( + method, num_reqs, headers={'X-Open-Expired': 'false'}) + for r in requests: + # If the proxy server config has allow_open_expired set + # to false, then we set x-backend-open-expired to false + self.assertEqual(r['headers']['X-Open-Expired'], 'false') + self.assertNotIn('X-Backend-Open-Expired', r['headers']) + + # we don't support x-open-expired on PUT when allow_open_expired + test_put_unsupported() + + # Disallow open expired + conf = {'allow_open_expired': 'false'} + # Create a new proxy instance for test with config + self.app = PatchedObjControllerApp( + conf, account_ring=FakeRing(), + container_ring=FakeRing(), logger=None) + # Use the same container info as the app used in other tests + self.app.container_info = dict(self.container_info) + self.obj_ring = self.app.get_object_ring(int(self.policy)) + + for method, num_reqs in ( + ('GET', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('HEAD', + self.obj_ring.replicas + self.obj_ring.max_more_nodes), + ('POST', self.obj_ring.replicas)): + # This case is different: we never add the 'X-Backend-Open-Expired' + # header if the proxy server config disables this feature + requests = self._test_x_open_expired( + method, num_reqs, headers={'X-Open-Expired': 'true'}) + for r in requests: + self.assertEqual(r['headers']['X-Open-Expired'], 'true') + self.assertNotIn('X-Backend-Open-Expired', r['headers']) + + # we don't support x-open-expired on PUT when not allow_open_expired + test_put_unsupported() + def test_HEAD_simple(self): req = swift.common.swob.Request.blank('/v1/a/c/o', method='HEAD') with set_http_connect(200): @@ -2280,6 +2403,80 @@ class TestReplicatedObjController(CommonObjectControllerMixin, self.assertIn('X-Delete-At-Partition', given_headers) self.assertIn('X-Delete-At-Container', given_headers) + def test_POST_delete_at_with_x_open_expired(self): + t_delete = str(int(time.time() + 30)) + + def capture_headers(ip, port, device, part, method, path, headers, + **kwargs): + if method == 'POST': + post_headers.append(headers) + + def do_post(extra_headers): + headers = {'Content-Type': 'foo/bar', + 'X-Delete-At': t_delete} + headers.update(extra_headers) + req_post = swob.Request.blank('/v1/a/c/o', method='POST', body=b'', + headers=headers) + + post_codes = [202] * self.obj_ring.replicas + with set_http_connect(*post_codes, give_connect=capture_headers): + resp = req_post.get_response(self.app) + self.assertEqual(resp.status_int, 202) + self.assertEqual(len(post_headers), self.obj_ring.replicas) + for given_headers in post_headers: + self.assertEqual(given_headers.get('X-Delete-At'), t_delete) + self.assertIn('X-Delete-At-Host', given_headers) + self.assertIn('X-Delete-At-Device', given_headers) + self.assertIn('X-Delete-At-Partition', given_headers) + self.assertIn('X-Delete-At-Container', given_headers) + + # Check when allow_open_expired config is set to true + conf = {'allow_open_expired': 'true'} + self.app = PatchedObjControllerApp( + conf, account_ring=FakeRing(), + container_ring=FakeRing(), logger=None) + self.app.container_info = dict(self.container_info) + self.obj_ring = self.app.get_object_ring(int(self.policy)) + + post_headers = [] + do_post({}) + for given_headers in post_headers: + self.assertNotIn('X-Backend-Open-Expired', given_headers) + + post_headers = [] + do_post({'X-Open-Expired': 'false'}) + for given_headers in post_headers: + self.assertNotIn('X-Backend-Open-Expired', given_headers) + + post_headers = [] + do_post({'X-Open-Expired': 'true'}) + for given_headers in post_headers: + self.assertEqual(given_headers.get('X-Backend-Open-Expired'), + 'true') + + # Check when allow_open_expired config is set to false + conf = {'allow_open_expired': 'false'} + self.app = PatchedObjControllerApp( + conf, account_ring=FakeRing(), + container_ring=FakeRing(), logger=None) + self.app.container_info = dict(self.container_info) + self.obj_ring = self.app.get_object_ring(int(self.policy)) + + post_headers = [] + do_post({}) + for given_headers in post_headers: + self.assertNotIn('X-Backend-Open-Expired', given_headers) + + post_headers = [] + do_post({'X-Open-Expired': 'false'}) + for given_headers in post_headers: + self.assertNotIn('X-Backend-Open-Expired', given_headers) + + post_headers = [] + do_post({'X-Open-Expired': 'true'}) + for given_headers in post_headers: + self.assertNotIn('X-Backend-Open-Expired', given_headers) + def test_PUT_converts_delete_after_to_delete_at(self): req = swob.Request.blank('/v1/a/c/o', method='PUT', body=b'', headers={'Content-Type': 'foo/bar', diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index d3fcb961fb..d51a1ff09a 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -12031,10 +12031,11 @@ class TestSwiftInfo(unittest.TestCase): constraints.MAX_OBJECT_NAME_LENGTH) self.assertIn('strict_cors_mode', si) self.assertFalse(si['allow_account_management']) + self.assertFalse(si['allow_open_expired']) self.assertFalse(si['account_autocreate']) # this next test is deliberately brittle in order to alert if # other items are added to swift info - self.assertEqual(len(si), 17) + self.assertEqual(len(si), 18) si = registry.get_swift_info()['swift'] # Tehse settings is by default excluded by disallowed_sections