Merge "Object-server: keep SLO manifest files in page cache."

doc/source/config/object_server_config.rst

@@ -203,6 +203,10 @@ keep_cache_size                    5242880                Largest object size to
                                                           buffer cache
 keep_cache_private                 false                  Allow non-public objects to stay
                                                           in kernel's buffer cache
+keep_cache_slo_manifest            false                  Allow SLO object's manifest file to stay in
+                                                          kernel's buffer cache if its size is under
+                                                          keep_cache_size. This config will only matter
+                                                          when 'keep_cache_private' is false.
 allowed_headers                    Content-Disposition,   Comma separated list of headers
                                    Content-Encoding,      that can be set in metadata on an object.
                                    X-Delete-At,           This list is in addition to

etc/object-server.conf-sample

@@ -143,6 +143,11 @@ use = egg:swift#object
 # if small enough
 # keep_cache_private = false
 #
+# If true, SLO object's manifest file for GET requests may be kept in buffer cache
+# if smaller than 'keep_cache_size'. And this config will only matter when
+# 'keep_cache_private' is false.
+# keep_cache_slo_manifest = false
+#
 # on PUTs, sync data every n MB
 # mb_per_sync = 512
 #

swift/obj/server.py

@@ -150,6 +150,8 @@ class ObjectController(BaseStorageServer):
         self.slow = int(conf.get('slow', 0))
         self.keep_cache_private = \
             config_true_value(conf.get('keep_cache_private', 'false'))
+        self.keep_cache_slo_manifest = \
+            config_true_value(conf.get('keep_cache_slo_manifest', 'false'))
 
         default_allowed_headers = '''
             content-disposition,
@@ -1098,9 +1100,19 @@ class ObjectController(BaseStorageServer):
                     request.headers.pop('Range', None)
                 obj_size = int(metadata['Content-Length'])
                 file_x_ts = Timestamp(metadata['X-Timestamp'])
-                keep_cache = (self.keep_cache_private or
+                keep_cache = (
-                              ('X-Auth-Token' not in request.headers and
+                    self.keep_cache_private
-                               'X-Storage-Token' not in request.headers))
+                    or (
+                        "X-Auth-Token" not in request.headers
+                        and "X-Storage-Token" not in request.headers
+                    )
+                    or (
+                        self.keep_cache_slo_manifest
+                        and config_true_value(
+                            metadata.get("X-Static-Large-Object")
+                        )
+                    )
+                )
                 conditional_etag = resolve_etag_is_at_header(request, metadata)
                 response = Response(
                     app_iter=disk_file.reader(keep_cache=keep_cache),

test/unit/common/test_utils.py

@@ -2298,6 +2298,7 @@ cluster_dfw1 = http://dfw1.host/v1/
             self.assertTrue(utils.config_true_value(True) is True)
             self.assertTrue(utils.config_true_value('foo') is False)
             self.assertTrue(utils.config_true_value(False) is False)
+            self.assertTrue(utils.config_true_value(None) is False)
         finally:
             utils.TRUE_VALUES = orig_trues
 

test/unit/obj/test_server.py

@@ -4437,6 +4437,183 @@ class TestObjectController(BaseTestCase):
             reader_mock.assert_called_with(keep_cache=False)
         self.assertEqual(resp.status_int, 200)
 
+    def test_GET_keep_cache_slo_manifest_no_config(self):
+        # Test swift.obj.server.ObjectController.GET that, when
+        # 'keep_cache_slo_manifest' is not configured and object
+        # metadata has "X-Static-Large-Object", then disk_file.reader
+        # will be called with keep_cache=False.
+        # Set up a new ObjectController with customized configurations.
+        conf = {'devices': self.testdir, 'mount_check': 'false',
+                'container_update_timeout': 0.0,
+                'keep_cache_private': 'false'}
+        obj_controller = object_server.ObjectController(
+            conf, logger=self.logger)
+        obj_controller.bytes_per_sync = 1
+        timestamp = normalize_timestamp(time())
+        req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': timestamp,
+                                     'Content-Type': 'application/x-test',
+                                     'X-Static-Large-Object': 'True'})
+        req.body = b'VERIFY'
+        resp = req.get_response(obj_controller)
+        self.assertEqual(resp.status_int, 201)
+        req = Request.blank('/sda1/p/a/c/o',
+                            headers={'Content-Type': 'application/x-test',
+                                     'X-Auth-Token': '2340lsdfhhjl02lxfjj'})
+
+        reader_mock = mock.Mock(keep_cache=False)
+        with mock.patch('swift.obj.diskfile.BaseDiskFile.reader', reader_mock):
+            resp = req.get_response(obj_controller)
+            reader_mock.assert_called_with(keep_cache=False)
+        self.assertEqual(resp.status_int, 200)
+        etag = '"%s"' % md5(b'VERIFY', usedforsecurity=False).hexdigest()
+        self.assertEqual(dict(resp.headers), {
+            'Content-Type': 'application/x-test',
+            'Content-Length': '6',
+            'Etag': etag,
+            'X-Static-Large-Object': 'True',
+            'X-Backend-Timestamp': timestamp,
+            'X-Timestamp': timestamp,
+            'X-Backend-Data-Timestamp': timestamp,
+            'X-Backend-Durable-Timestamp': timestamp,
+            'Last-Modified': strftime(
+                '%a, %d %b %Y %H:%M:%S GMT',
+                gmtime(math.ceil(float(timestamp)))),
+        })
+
+    def test_GET_keep_cache_slo_manifest_config_false(self):
+        # Test swift.obj.server.ObjectController.GET that, when
+        # 'keep_cache_slo_manifest' is configured False and object
+        # metadata has "X-Static-Large-Object", then disk_file.reader
+        # will be called with keep_cache=False.
+        # Set up a new ObjectController with customized configurations.
+        conf = {'devices': self.testdir, 'mount_check': 'false',
+                'container_update_timeout': 0.0,
+                'keep_cache_private': 'false',
+                'keep_cache_slo_manifest': 'false'}
+        obj_controller = object_server.ObjectController(
+            conf, logger=self.logger)
+        obj_controller.bytes_per_sync = 1
+        timestamp = normalize_timestamp(time())
+        req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': timestamp,
+                                     'Content-Type': 'application/x-test',
+                                     'X-Static-Large-Object': 'True'})
+        req.body = b'VERIFY'
+        resp = req.get_response(obj_controller)
+        self.assertEqual(resp.status_int, 201)
+        req = Request.blank('/sda1/p/a/c/o',
+                            headers={'Content-Type': 'application/x-test',
+                                     'X-Auth-Token': '2340lsdfhhjl02lxfjj'})
+
+        reader_mock = mock.Mock(keep_cache=False)
+        with mock.patch('swift.obj.diskfile.BaseDiskFile.reader', reader_mock):
+            resp = req.get_response(obj_controller)
+            reader_mock.assert_called_with(keep_cache=False)
+        self.assertEqual(resp.status_int, 200)
+        etag = '"%s"' % md5(b'VERIFY', usedforsecurity=False).hexdigest()
+        self.assertEqual(dict(resp.headers), {
+            'Content-Type': 'application/x-test',
+            'Content-Length': '6',
+            'Etag': etag,
+            'X-Static-Large-Object': 'True',
+            'X-Backend-Timestamp': timestamp,
+            'X-Timestamp': timestamp,
+            'X-Backend-Data-Timestamp': timestamp,
+            'X-Backend-Durable-Timestamp': timestamp,
+            'Last-Modified': strftime(
+                '%a, %d %b %Y %H:%M:%S GMT',
+                gmtime(math.ceil(float(timestamp)))),
+        })
+
+    def test_GET_keep_cache_slo_manifest_config_true(self):
+        # Test swift.obj.server.ObjectController.GET that, when
+        # 'keep_cache_slo_manifest' is configured true and object
+        # metadata has "X-Static-Large-Object", then disk_file.reader
+        # will be called with keep_cache=True.
+        # Set up a new ObjectController with customized configurations.
+        conf = {'devices': self.testdir, 'mount_check': 'false',
+                'container_update_timeout': 0.0,
+                'keep_cache_private': 'false',
+                'keep_cache_slo_manifest': 'true'}
+        obj_controller = object_server.ObjectController(
+            conf, logger=self.logger)
+        obj_controller.bytes_per_sync = 1
+        timestamp = normalize_timestamp(time())
+        req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': timestamp,
+                                     'Content-Type': 'application/x-test',
+                                     'X-Static-Large-Object': 'True'})
+        req.body = b'VERIFY'
+        resp = req.get_response(obj_controller)
+        self.assertEqual(resp.status_int, 201)
+        req = Request.blank('/sda1/p/a/c/o',
+                            headers={'Content-Type': 'application/x-test',
+                                     'X-Auth-Token': '2340lsdfhhjl02lxfjj'})
+
+        reader_mock = mock.Mock(keep_cache=False)
+        with mock.patch('swift.obj.diskfile.BaseDiskFile.reader', reader_mock):
+            resp = req.get_response(obj_controller)
+            reader_mock.assert_called_with(keep_cache=True)
+        self.assertEqual(resp.status_int, 200)
+        etag = '"%s"' % md5(b'VERIFY', usedforsecurity=False).hexdigest()
+        self.assertEqual(dict(resp.headers), {
+            'Content-Type': 'application/x-test',
+            'Content-Length': '6',
+            'Etag': etag,
+            'X-Static-Large-Object': 'True',
+            'X-Backend-Timestamp': timestamp,
+            'X-Timestamp': timestamp,
+            'X-Backend-Data-Timestamp': timestamp,
+            'X-Backend-Durable-Timestamp': timestamp,
+            'Last-Modified': strftime(
+                '%a, %d %b %Y %H:%M:%S GMT',
+                gmtime(math.ceil(float(timestamp)))),
+        })
+
+    def test_GET_keep_cache_slo_manifest_not_slo(self):
+        # Test swift.obj.server.ObjectController.GET that, when
+        # 'keep_cache_slo_manifest' is configured true and object
+        # metadata has NO "X-Static-Large-Object", then disk_file.reader
+        # will be called with keep_cache=False.
+        # Set up a new ObjectController with customized configurations.
+        conf = {'devices': self.testdir, 'mount_check': 'false',
+                'container_update_timeout': 0.0,
+                'keep_cache_private': 'false',
+                'keep_cache_slo_manifest': 'true'}
+        obj_controller = object_server.ObjectController(
+            conf, logger=self.logger)
+        obj_controller.bytes_per_sync = 1
+        timestamp = normalize_timestamp(time())
+        req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': timestamp,
+                                     'Content-Type': 'application/x-test'})
+        req.body = b'VERIFY'
+        resp = req.get_response(obj_controller)
+        self.assertEqual(resp.status_int, 201)
+        req = Request.blank('/sda1/p/a/c/o',
+                            headers={'Content-Type': 'application/x-test',
+                                     'X-Auth-Token': '2340lsdfhhjl02lxfjj'})
+
+        reader_mock = mock.Mock(keep_cache=False)
+        with mock.patch('swift.obj.diskfile.BaseDiskFile.reader', reader_mock):
+            resp = req.get_response(obj_controller)
+            reader_mock.assert_called_with(keep_cache=False)
+        self.assertEqual(resp.status_int, 200)
+        etag = '"%s"' % md5(b'VERIFY', usedforsecurity=False).hexdigest()
+        self.assertEqual(dict(resp.headers), {
+            'Content-Type': 'application/x-test',
+            'Content-Length': '6',
+            'Etag': etag,
+            'X-Backend-Timestamp': timestamp,
+            'X-Timestamp': timestamp,
+            'X-Backend-Data-Timestamp': timestamp,
+            'X-Backend-Durable-Timestamp': timestamp,
+            'Last-Modified': strftime(
+                '%a, %d %b %Y %H:%M:%S GMT',
+                gmtime(math.ceil(float(timestamp)))),
+        })
+
     @mock.patch("time.time", mock_time)
     def test_DELETE(self):
         # Test swift.obj.server.ObjectController.DELETE