openstack/swift
Code Issues Proposed changes

Merge "Object-server: keep SLO manifest files in page cache."

Browse Source
This commit is contained in:
Zuul 2023-07-10 19:50:47 +00:00 committed by Gerrit Code Review
commit 7cc8a01729
5 changed files with 202 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/source/config/object_server_config.rst
View File

@ -203,6 +203,10 @@ keep_cache_size 5242880 Largest object size to
buffer cache
keep_cache_private false Allow non-public objects to stay
in kernel's buffer cache
keep_cache_slo_manifest false Allow SLO object's manifest file to stay in
kernel's buffer cache if its size is under
keep_cache_size. This config will only matter
when 'keep_cache_private' is false.
allowed_headers Content-Disposition, Comma separated list of headers
Content-Encoding, that can be set in metadata on an object.
X-Delete-At, This list is in addition to

5
etc/object-server.conf-sample
View File

@ -143,6 +143,11 @@ use = egg:swift#object
# if small enough
# keep_cache_private = false
#
# If true, SLO object's manifest file for GET requests may be kept in buffer cache
# if smaller than 'keep_cache_size'. And this config will only matter when
# 'keep_cache_private' is false.
# keep_cache_slo_manifest = false
#
# on PUTs, sync data every n MB
# mb_per_sync = 512
#

18
swift/obj/server.py
View File

@ -150,6 +150,8 @@ class ObjectController(BaseStorageServer):
self.slow = int(conf.get('slow', 0))
self.keep_cache_private = \
config_true_value(conf.get('keep_cache_private', 'false'))
self.keep_cache_slo_manifest = \
config_true_value(conf.get('keep_cache_slo_manifest', 'false'))
default_allowed_headers = '''
content-disposition,
@ -1098,9 +1100,19 @@ class ObjectController(BaseStorageServer):
request.headers.pop('Range', None)
obj_size = int(metadata['Content-Length'])
file_x_ts = Timestamp(metadata['X-Timestamp'])
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers))
keep_cache = (
self.keep_cache_private
or (
"X-Auth-Token" not in request.headers
and "X-Storage-Token" not in request.headers
)
or (
self.keep_cache_slo_manifest
and config_true_value(
metadata.get("X-Static-Large-Object")
)
)
)
conditional_etag = resolve_etag_is_at_header(request, metadata)
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache),

1
test/unit/common/test_utils.py
View File

@ -2298,6 +2298,7 @@ cluster_dfw1 = http://dfw1.host/v1/
self.assertTrue(utils.config_true_value(True) is True)
self.assertTrue(utils.config_true_value('foo') is False)
self.assertTrue(utils.config_true_value(False) is False)
self.assertTrue(utils.config_true_value(None) is False)
finally:
utils.TRUE_VALUES = orig_trues

177
test/unit/obj/test_server.py
View File

@ -4437,6 +4437,183 @@ class TestObjectController(BaseTestCase):
reader_mock.assert_called_with(keep_cache=False)
self.assertEqual(resp.status_int, 200)
def test_GET_keep_cache_slo_manifest_no_config(self):
# Test swift.obj.server.ObjectController.GET that, when
# 'keep_cache_slo_manifest' is not configured and object
# metadata has "X-Static-Large-Object", then disk_file.reader
# will be called with keep_cache=False.
# Set up a new ObjectController with customized configurations.
conf = {'devices': self.testdir, 'mount_check': 'false',
'container_update_timeout': 0.0,
'keep_cache_private': 'false'}
obj_controller = object_server.ObjectController(
conf, logger=self.logger)
obj_controller.bytes_per_sync = 1
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
'Content-Type': 'application/x-test',
'X-Static-Large-Object': 'True'})
req.body = b'VERIFY'
resp = req.get_response(obj_controller)
self.assertEqual(resp.status_int, 201)
req = Request.blank('/sda1/p/a/c/o',
headers={'Content-Type': 'application/x-test',
'X-Auth-Token': '2340lsdfhhjl02lxfjj'})
reader_mock = mock.Mock(keep_cache=False)
with mock.patch('swift.obj.diskfile.BaseDiskFile.reader', reader_mock):
resp = req.get_response(obj_controller)
reader_mock.assert_called_with(keep_cache=False)
self.assertEqual(resp.status_int, 200)
etag = '"%s"' % md5(b'VERIFY', usedforsecurity=False).hexdigest()
self.assertEqual(dict(resp.headers), {
'Content-Type': 'application/x-test',
'Content-Length': '6',
'Etag': etag,
'X-Static-Large-Object': 'True',
'X-Backend-Timestamp': timestamp,
'X-Timestamp': timestamp,
'X-Backend-Data-Timestamp': timestamp,
'X-Backend-Durable-Timestamp': timestamp,
'Last-Modified': strftime(
'%a, %d %b %Y %H:%M:%S GMT',
gmtime(math.ceil(float(timestamp)))),
})
def test_GET_keep_cache_slo_manifest_config_false(self):
# Test swift.obj.server.ObjectController.GET that, when
# 'keep_cache_slo_manifest' is configured False and object
# metadata has "X-Static-Large-Object", then disk_file.reader
# will be called with keep_cache=False.
# Set up a new ObjectController with customized configurations.
conf = {'devices': self.testdir, 'mount_check': 'false',
'container_update_timeout': 0.0,
'keep_cache_private': 'false',
'keep_cache_slo_manifest': 'false'}
obj_controller = object_server.ObjectController(
conf, logger=self.logger)
obj_controller.bytes_per_sync = 1
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
'Content-Type': 'application/x-test',
'X-Static-Large-Object': 'True'})
req.body = b'VERIFY'
resp = req.get_response(obj_controller)
self.assertEqual(resp.status_int, 201)
req = Request.blank('/sda1/p/a/c/o',
headers={'Content-Type': 'application/x-test',
'X-Auth-Token': '2340lsdfhhjl02lxfjj'})
reader_mock = mock.Mock(keep_cache=False)
with mock.patch('swift.obj.diskfile.BaseDiskFile.reader', reader_mock):
resp = req.get_response(obj_controller)
reader_mock.assert_called_with(keep_cache=False)
self.assertEqual(resp.status_int, 200)
etag = '"%s"' % md5(b'VERIFY', usedforsecurity=False).hexdigest()
self.assertEqual(dict(resp.headers), {
'Content-Type': 'application/x-test',
'Content-Length': '6',
'Etag': etag,
'X-Static-Large-Object': 'True',
'X-Backend-Timestamp': timestamp,
'X-Timestamp': timestamp,
'X-Backend-Data-Timestamp': timestamp,
'X-Backend-Durable-Timestamp': timestamp,
'Last-Modified': strftime(
'%a, %d %b %Y %H:%M:%S GMT',
gmtime(math.ceil(float(timestamp)))),
})
def test_GET_keep_cache_slo_manifest_config_true(self):
# Test swift.obj.server.ObjectController.GET that, when
# 'keep_cache_slo_manifest' is configured true and object
# metadata has "X-Static-Large-Object", then disk_file.reader
# will be called with keep_cache=True.
# Set up a new ObjectController with customized configurations.
conf = {'devices': self.testdir, 'mount_check': 'false',
'container_update_timeout': 0.0,
'keep_cache_private': 'false',
'keep_cache_slo_manifest': 'true'}
obj_controller = object_server.ObjectController(
conf, logger=self.logger)
obj_controller.bytes_per_sync = 1
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
'Content-Type': 'application/x-test',
'X-Static-Large-Object': 'True'})
req.body = b'VERIFY'
resp = req.get_response(obj_controller)
self.assertEqual(resp.status_int, 201)
req = Request.blank('/sda1/p/a/c/o',
headers={'Content-Type': 'application/x-test',
'X-Auth-Token': '2340lsdfhhjl02lxfjj'})
reader_mock = mock.Mock(keep_cache=False)
with mock.patch('swift.obj.diskfile.BaseDiskFile.reader', reader_mock):
resp = req.get_response(obj_controller)
reader_mock.assert_called_with(keep_cache=True)
self.assertEqual(resp.status_int, 200)
etag = '"%s"' % md5(b'VERIFY', usedforsecurity=False).hexdigest()
self.assertEqual(dict(resp.headers), {
'Content-Type': 'application/x-test',
'Content-Length': '6',
'Etag': etag,
'X-Static-Large-Object': 'True',
'X-Backend-Timestamp': timestamp,
'X-Timestamp': timestamp,
'X-Backend-Data-Timestamp': timestamp,
'X-Backend-Durable-Timestamp': timestamp,
'Last-Modified': strftime(
'%a, %d %b %Y %H:%M:%S GMT',
gmtime(math.ceil(float(timestamp)))),
})
def test_GET_keep_cache_slo_manifest_not_slo(self):
# Test swift.obj.server.ObjectController.GET that, when
# 'keep_cache_slo_manifest' is configured true and object
# metadata has NO "X-Static-Large-Object", then disk_file.reader
# will be called with keep_cache=False.
# Set up a new ObjectController with customized configurations.
conf = {'devices': self.testdir, 'mount_check': 'false',
'container_update_timeout': 0.0,
'keep_cache_private': 'false',
'keep_cache_slo_manifest': 'true'}
obj_controller = object_server.ObjectController(
conf, logger=self.logger)
obj_controller.bytes_per_sync = 1
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
'Content-Type': 'application/x-test'})
req.body = b'VERIFY'
resp = req.get_response(obj_controller)
self.assertEqual(resp.status_int, 201)
req = Request.blank('/sda1/p/a/c/o',
headers={'Content-Type': 'application/x-test',
'X-Auth-Token': '2340lsdfhhjl02lxfjj'})
reader_mock = mock.Mock(keep_cache=False)
with mock.patch('swift.obj.diskfile.BaseDiskFile.reader', reader_mock):
resp = req.get_response(obj_controller)
reader_mock.assert_called_with(keep_cache=False)
self.assertEqual(resp.status_int, 200)
etag = '"%s"' % md5(b'VERIFY', usedforsecurity=False).hexdigest()
self.assertEqual(dict(resp.headers), {
'Content-Type': 'application/x-test',
'Content-Length': '6',
'Etag': etag,
'X-Backend-Timestamp': timestamp,
'X-Timestamp': timestamp,
'X-Backend-Data-Timestamp': timestamp,
'X-Backend-Durable-Timestamp': timestamp,
'Last-Modified': strftime(
'%a, %d %b %Y %H:%M:%S GMT',
gmtime(math.ceil(float(timestamp)))),
})
@mock.patch("time.time", mock_time)
def test_DELETE(self):
# Test swift.obj.server.ObjectController.DELETE