diff --git a/doc/source/config/object_server_config.rst b/doc/source/config/object_server_config.rst index 7f5c4737ec..414be1dafb 100644 --- a/doc/source/config/object_server_config.rst +++ b/doc/source/config/object_server_config.rst @@ -203,6 +203,10 @@ keep_cache_size 5242880 Largest object size to buffer cache keep_cache_private false Allow non-public objects to stay in kernel's buffer cache +keep_cache_slo_manifest false Allow SLO object's manifest file to stay in + kernel's buffer cache if its size is under + keep_cache_size. This config will only matter + when 'keep_cache_private' is false. allowed_headers Content-Disposition, Comma separated list of headers Content-Encoding, that can be set in metadata on an object. X-Delete-At, This list is in addition to diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index 3355474417..ac7d497adb 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -143,6 +143,11 @@ use = egg:swift#object # if small enough # keep_cache_private = false # +# If true, SLO object's manifest file for GET requests may be kept in buffer cache +# if smaller than 'keep_cache_size'. And this config will only matter when +# 'keep_cache_private' is false. +# keep_cache_slo_manifest = false +# # on PUTs, sync data every n MB # mb_per_sync = 512 # diff --git a/swift/obj/server.py b/swift/obj/server.py index 0e47fd8188..bd2cee5793 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -150,6 +150,8 @@ class ObjectController(BaseStorageServer): self.slow = int(conf.get('slow', 0)) self.keep_cache_private = \ config_true_value(conf.get('keep_cache_private', 'false')) + self.keep_cache_slo_manifest = \ + config_true_value(conf.get('keep_cache_slo_manifest', 'false')) default_allowed_headers = ''' content-disposition, @@ -1098,9 +1100,19 @@ class ObjectController(BaseStorageServer): request.headers.pop('Range', None) obj_size = int(metadata['Content-Length']) file_x_ts = Timestamp(metadata['X-Timestamp']) - keep_cache = (self.keep_cache_private or - ('X-Auth-Token' not in request.headers and - 'X-Storage-Token' not in request.headers)) + keep_cache = ( + self.keep_cache_private + or ( + "X-Auth-Token" not in request.headers + and "X-Storage-Token" not in request.headers + ) + or ( + self.keep_cache_slo_manifest + and config_true_value( + metadata.get("X-Static-Large-Object") + ) + ) + ) conditional_etag = resolve_etag_is_at_header(request, metadata) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 1747a98145..519b34f394 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -2298,6 +2298,7 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertTrue(utils.config_true_value(True) is True) self.assertTrue(utils.config_true_value('foo') is False) self.assertTrue(utils.config_true_value(False) is False) + self.assertTrue(utils.config_true_value(None) is False) finally: utils.TRUE_VALUES = orig_trues diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index be7e2f2def..7aaf27ec2f 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -4437,6 +4437,183 @@ class TestObjectController(BaseTestCase): reader_mock.assert_called_with(keep_cache=False) self.assertEqual(resp.status_int, 200) + def test_GET_keep_cache_slo_manifest_no_config(self): + # Test swift.obj.server.ObjectController.GET that, when + # 'keep_cache_slo_manifest' is not configured and object + # metadata has "X-Static-Large-Object", then disk_file.reader + # will be called with keep_cache=False. + # Set up a new ObjectController with customized configurations. + conf = {'devices': self.testdir, 'mount_check': 'false', + 'container_update_timeout': 0.0, + 'keep_cache_private': 'false'} + obj_controller = object_server.ObjectController( + conf, logger=self.logger) + obj_controller.bytes_per_sync = 1 + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Type': 'application/x-test', + 'X-Static-Large-Object': 'True'}) + req.body = b'VERIFY' + resp = req.get_response(obj_controller) + self.assertEqual(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c/o', + headers={'Content-Type': 'application/x-test', + 'X-Auth-Token': '2340lsdfhhjl02lxfjj'}) + + reader_mock = mock.Mock(keep_cache=False) + with mock.patch('swift.obj.diskfile.BaseDiskFile.reader', reader_mock): + resp = req.get_response(obj_controller) + reader_mock.assert_called_with(keep_cache=False) + self.assertEqual(resp.status_int, 200) + etag = '"%s"' % md5(b'VERIFY', usedforsecurity=False).hexdigest() + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'application/x-test', + 'Content-Length': '6', + 'Etag': etag, + 'X-Static-Large-Object': 'True', + 'X-Backend-Timestamp': timestamp, + 'X-Timestamp': timestamp, + 'X-Backend-Data-Timestamp': timestamp, + 'X-Backend-Durable-Timestamp': timestamp, + 'Last-Modified': strftime( + '%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(timestamp)))), + }) + + def test_GET_keep_cache_slo_manifest_config_false(self): + # Test swift.obj.server.ObjectController.GET that, when + # 'keep_cache_slo_manifest' is configured False and object + # metadata has "X-Static-Large-Object", then disk_file.reader + # will be called with keep_cache=False. + # Set up a new ObjectController with customized configurations. + conf = {'devices': self.testdir, 'mount_check': 'false', + 'container_update_timeout': 0.0, + 'keep_cache_private': 'false', + 'keep_cache_slo_manifest': 'false'} + obj_controller = object_server.ObjectController( + conf, logger=self.logger) + obj_controller.bytes_per_sync = 1 + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Type': 'application/x-test', + 'X-Static-Large-Object': 'True'}) + req.body = b'VERIFY' + resp = req.get_response(obj_controller) + self.assertEqual(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c/o', + headers={'Content-Type': 'application/x-test', + 'X-Auth-Token': '2340lsdfhhjl02lxfjj'}) + + reader_mock = mock.Mock(keep_cache=False) + with mock.patch('swift.obj.diskfile.BaseDiskFile.reader', reader_mock): + resp = req.get_response(obj_controller) + reader_mock.assert_called_with(keep_cache=False) + self.assertEqual(resp.status_int, 200) + etag = '"%s"' % md5(b'VERIFY', usedforsecurity=False).hexdigest() + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'application/x-test', + 'Content-Length': '6', + 'Etag': etag, + 'X-Static-Large-Object': 'True', + 'X-Backend-Timestamp': timestamp, + 'X-Timestamp': timestamp, + 'X-Backend-Data-Timestamp': timestamp, + 'X-Backend-Durable-Timestamp': timestamp, + 'Last-Modified': strftime( + '%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(timestamp)))), + }) + + def test_GET_keep_cache_slo_manifest_config_true(self): + # Test swift.obj.server.ObjectController.GET that, when + # 'keep_cache_slo_manifest' is configured true and object + # metadata has "X-Static-Large-Object", then disk_file.reader + # will be called with keep_cache=True. + # Set up a new ObjectController with customized configurations. + conf = {'devices': self.testdir, 'mount_check': 'false', + 'container_update_timeout': 0.0, + 'keep_cache_private': 'false', + 'keep_cache_slo_manifest': 'true'} + obj_controller = object_server.ObjectController( + conf, logger=self.logger) + obj_controller.bytes_per_sync = 1 + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Type': 'application/x-test', + 'X-Static-Large-Object': 'True'}) + req.body = b'VERIFY' + resp = req.get_response(obj_controller) + self.assertEqual(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c/o', + headers={'Content-Type': 'application/x-test', + 'X-Auth-Token': '2340lsdfhhjl02lxfjj'}) + + reader_mock = mock.Mock(keep_cache=False) + with mock.patch('swift.obj.diskfile.BaseDiskFile.reader', reader_mock): + resp = req.get_response(obj_controller) + reader_mock.assert_called_with(keep_cache=True) + self.assertEqual(resp.status_int, 200) + etag = '"%s"' % md5(b'VERIFY', usedforsecurity=False).hexdigest() + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'application/x-test', + 'Content-Length': '6', + 'Etag': etag, + 'X-Static-Large-Object': 'True', + 'X-Backend-Timestamp': timestamp, + 'X-Timestamp': timestamp, + 'X-Backend-Data-Timestamp': timestamp, + 'X-Backend-Durable-Timestamp': timestamp, + 'Last-Modified': strftime( + '%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(timestamp)))), + }) + + def test_GET_keep_cache_slo_manifest_not_slo(self): + # Test swift.obj.server.ObjectController.GET that, when + # 'keep_cache_slo_manifest' is configured true and object + # metadata has NO "X-Static-Large-Object", then disk_file.reader + # will be called with keep_cache=False. + # Set up a new ObjectController with customized configurations. + conf = {'devices': self.testdir, 'mount_check': 'false', + 'container_update_timeout': 0.0, + 'keep_cache_private': 'false', + 'keep_cache_slo_manifest': 'true'} + obj_controller = object_server.ObjectController( + conf, logger=self.logger) + obj_controller.bytes_per_sync = 1 + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Type': 'application/x-test'}) + req.body = b'VERIFY' + resp = req.get_response(obj_controller) + self.assertEqual(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c/o', + headers={'Content-Type': 'application/x-test', + 'X-Auth-Token': '2340lsdfhhjl02lxfjj'}) + + reader_mock = mock.Mock(keep_cache=False) + with mock.patch('swift.obj.diskfile.BaseDiskFile.reader', reader_mock): + resp = req.get_response(obj_controller) + reader_mock.assert_called_with(keep_cache=False) + self.assertEqual(resp.status_int, 200) + etag = '"%s"' % md5(b'VERIFY', usedforsecurity=False).hexdigest() + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'application/x-test', + 'Content-Length': '6', + 'Etag': etag, + 'X-Backend-Timestamp': timestamp, + 'X-Timestamp': timestamp, + 'X-Backend-Data-Timestamp': timestamp, + 'X-Backend-Durable-Timestamp': timestamp, + 'Last-Modified': strftime( + '%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(timestamp)))), + }) + @mock.patch("time.time", mock_time) def test_DELETE(self): # Test swift.obj.server.ObjectController.DELETE