From 5392a2057bdf7e91878023cfdb0b7193d677a5b2 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 8 Sep 2023 16:30:40 -0700 Subject: [PATCH] tests: Add test(s) for MPU part copy from range When using the copy-part API it is expected for s3api to write down an empty value for X-Object-Sysmeta-S3Api-Etag on segments. This was ostensibly to prevent writing down an unrelated S3Api-Etag when copying a part from another MPU the copy transfers object sysmeta. We should assume a S3Api-Etag w/o X-Static-Large-Object is non-sense, and SLO should forever expect empty values for it's sysmeta. Drive-By: consolidate handling of boto2 sigv4 skips Related-Bug: #2035158 Co-Authored-By: Clay Gerrard Change-Id: Ic6f04a5a6af8a3e65b226cff2ed6c9fce8ce1fa2 --- test/functional/s3api/__init__.py | 31 ++++ test/functional/s3api/test_multi_upload.py | 179 ++++++++++++++++----- test/functional/s3api/test_object.py | 56 ++----- test/unit/common/middleware/test_slo.py | 22 +++ 4 files changed, 204 insertions(+), 84 deletions(-) diff --git a/test/functional/s3api/__init__.py b/test/functional/s3api/__init__.py index 4993de61d8..a744e96e6d 100644 --- a/test/functional/s3api/__init__.py +++ b/test/functional/s3api/__init__.py @@ -19,6 +19,10 @@ from contextlib import contextmanager import logging from unittest import SkipTest +import os +import boto +from distutils.version import StrictVersion + import test.functional as tf from test.functional.s3api.s3_test_client import ( Connection, get_boto3_conn, tear_down_s3) @@ -104,3 +108,30 @@ class S3ApiBaseBoto3(S3ApiBase): def tearDown(self): tear_down_s3(self.conn) + + +def skip_boto2_sort_header_bug(m): + def wrapped(self, *args, **kwargs): + if (os.environ.get('S3_USE_SIGV4') == "True" and + StrictVersion(boto.__version__) < StrictVersion('3.0')): + # boto 2 doesn't sort headers properly; see + # https://github.com/boto/boto/pull/3032 + # or https://github.com/boto/boto/pull/3176 + # or https://github.com/boto/boto/pull/3751 + # or https://github.com/boto/boto/pull/3824 + self.skipTest('This stuff got the issue of boto<=2.x') + return m(self, *args, **kwargs) + return wrapped + + +class SigV4Mixin(object): + @classmethod + def setUpClass(cls): + os.environ['S3_USE_SIGV4'] = "True" + + @classmethod + def tearDownClass(cls): + del os.environ['S3_USE_SIGV4'] + + def setUp(self): + super(SigV4Mixin, self).setUp() diff --git a/test/functional/s3api/test_multi_upload.py b/test/functional/s3api/test_multi_upload.py index bdb17689be..82df2419b3 100644 --- a/test/functional/s3api/test_multi_upload.py +++ b/test/functional/s3api/test_multi_upload.py @@ -16,12 +16,6 @@ import base64 import binascii import unittest -import os -import boto - -# For an issue with venv and distutils, disable pylint message here -# pylint: disable-msg=E0611,F0401 -from distutils.version import StrictVersion import six from six.moves import urllib, zip, zip_longest @@ -33,7 +27,8 @@ from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX, mktime, \ S3Timestamp from swift.common.utils import md5 -from test.functional.s3api import S3ApiBase +from test.functional.s3api import S3ApiBase, SigV4Mixin, \ + skip_boto2_sort_header_bug from test.functional.s3api.s3_test_client import Connection from test.functional.s3api.utils import get_error_code, get_error_msg, \ calculate_md5 @@ -867,9 +862,8 @@ class TestS3ApiMultiUpload(S3ApiBase): query=query) self.assertEqual(status, 200) - def test_object_multi_upload_part_copy_range(self): - bucket = 'bucket' - keys = ['obj1'] + def _initiate_mpu_upload(self, bucket, key): + keys = [key] uploads = [] results_generator = self._initiate_multi_uploads_result_generator( @@ -893,10 +887,13 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertTrue((key, upload_id) not in uploads) uploads.append((key, upload_id)) - self.assertEqual(len(uploads), len(keys)) # sanity + # sanity, there's just one multi-part upload + self.assertEqual(1, len(uploads)) + self.assertEqual(1, len(keys)) + _, upload_id = uploads[0] + return upload_id - # Upload Part Copy Range - key, upload_id = uploads[0] + def _copy_part_from_new_src_range(self, bucket, key, upload_id): src_bucket = 'bucket2' src_obj = 'obj4' src_content = b'y' * (self.min_segment_size // 2) + b'z' * \ @@ -923,6 +920,7 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertEqual(headers['content-length'], str(len(body))) self.assertTrue('etag' not in headers) elem = fromstring(body, 'CopyPartResult') + etags = [elem.find('ETag').text] copy_resp_last_modified = elem.find('LastModified').text self.assertIsNotNone(copy_resp_last_modified) @@ -930,7 +928,6 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertEqual(resp_etag, etag) # Check last-modified timestamp - key, upload_id = uploads[0] query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('GET', bucket, key, query=query) @@ -942,8 +939,134 @@ class TestS3ApiMultiUpload(S3ApiBase): # There should be *exactly* one parts in the result self.assertEqual(listing_last_modified, [copy_resp_last_modified]) + # sanity, there's just one etag + self.assertEqual(1, len(etags)) + return etags[0] + + def _complete_mpu_upload(self, bucket, key, upload_id, etags): + # Complete Multipart Upload + query = 'uploadId=%s' % upload_id + xml = self._gen_comp_xml(etags) + status, headers, body = \ + self.conn.make_request('POST', bucket, key, body=xml, + query=query) + self.assertEqual(status, 200) + self.assertCommonResponseHeaders(headers) + self.assertTrue('content-type' in headers) + self.assertEqual(headers['content-type'], 'application/xml') + if 'content-length' in headers: + self.assertEqual(headers['content-length'], str(len(body))) + else: + self.assertIn('transfer-encoding', headers) + self.assertEqual(headers['transfer-encoding'], 'chunked') + lines = body.split(b'\n') + self.assertTrue(lines[0].startswith(b''), body) + elem = fromstring(body, 'CompleteMultipartUploadResult') + self.assertEqual( + '%s/%s/%s' % + (tf.config['s3_storage_url'].rstrip('/'), bucket, key), + elem.find('Location').text) + self.assertEqual(elem.find('Bucket').text, bucket) + self.assertEqual(elem.find('Key').text, key) + concatted_etags = b''.join( + etag.strip('"').encode('ascii') for etag in etags) + exp_etag = '"%s-%s"' % ( + md5(binascii.unhexlify(concatted_etags), + usedforsecurity=False).hexdigest(), len(etags)) + etag = elem.find('ETag').text + self.assertEqual(etag, exp_etag) + + @skip_boto2_sort_header_bug + def test_mpu_copy_part_from_range_then_complete(self): + bucket = 'mpu-copy-range' + key = 'obj-complete' + upload_id = self._initiate_mpu_upload(bucket, key) + etag = self._copy_part_from_new_src_range(bucket, key, upload_id) + self._complete_mpu_upload(bucket, key, upload_id, [etag]) + + @skip_boto2_sort_header_bug + def test_mpu_copy_part_from_range_then_abort(self): + bucket = 'mpu-copy-range' + key = 'obj-abort' + upload_id = self._initiate_mpu_upload(bucket, key) + self._copy_part_from_new_src_range(bucket, key, upload_id) + + # Abort Multipart Upload + query = 'uploadId=%s' % upload_id + status, headers, body = \ + self.conn.make_request('DELETE', bucket, key, query=query) + + # sanity checks + self.assertEqual(status, 204) + self.assertCommonResponseHeaders(headers) + self.assertTrue('content-type' in headers) + self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8') + self.assertTrue('content-length' in headers) + self.assertEqual(headers['content-length'], '0') + + def _copy_part_from_new_mpu_range(self, bucket, key, upload_id): + src_bucket = 'bucket2' + src_obj = 'mpu2' + src_upload_id = self._initiate_mpu_upload(src_bucket, src_obj) + # upload parts + etags = [] + for part_num in range(2): + # Upload Part + content = (chr(97 + part_num) * self.min_segment_size).encode() + etag = md5(content, usedforsecurity=False).hexdigest() + status, headers, body = \ + self._upload_part(src_bucket, src_obj, src_upload_id, + content, part_num=part_num + 1) + self.assertEqual(status, 200) + self.assertCommonResponseHeaders(headers, etag) + self.assertTrue('content-type' in headers) + self.assertEqual(headers['content-type'], + 'text/html; charset=UTF-8') + self.assertTrue('content-length' in headers) + self.assertEqual(headers['content-length'], '0') + self.assertEqual(headers['etag'], '"%s"' % etag) + etags.append(etag) + self._complete_mpu_upload(src_bucket, src_obj, src_upload_id, etags) + + # Upload Part Copy -- MPU as source + src_range = 'bytes=0-%d' % (self.min_segment_size - 1) + status, headers, body, resp_etag = \ + self._upload_part_copy(src_bucket, src_obj, bucket, + key, upload_id, part_num=1, + src_range=src_range) + self.assertEqual(status, 200) + self.assertCommonResponseHeaders(headers) + self.assertIn('content-type', headers) + self.assertEqual(headers['content-type'], 'application/xml') + self.assertIn('content-length', headers) + self.assertEqual(headers['content-length'], str(len(body))) + self.assertNotIn('etag', headers) + elem = fromstring(body, 'CopyPartResult') + + last_modified = elem.find('LastModified').text + self.assertIsNotNone(last_modified) + # use copied with src_range from src_obj?part-number=1 + self.assertEqual(resp_etag, etags[0]) + + return resp_etag + + @skip_boto2_sort_header_bug + def test_mpu_copy_part_from_mpu_part_number_then_complete(self): + bucket = 'mpu-copy-range' + key = 'obj-complete' + upload_id = self._initiate_mpu_upload(bucket, key) + etag = self._copy_part_from_new_mpu_range(bucket, key, upload_id) + self._complete_mpu_upload(bucket, key, upload_id, [etag]) + + @skip_boto2_sort_header_bug + def test_mpu_copy_part_from_mpu_part_number_then_abort(self): + bucket = 'mpu-copy-range' + key = 'obj-abort' + upload_id = self._initiate_mpu_upload(bucket, key) + self._copy_part_from_new_mpu_range(bucket, key, upload_id) + # Abort Multipart Upload - key, upload_id = uploads[0] query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('DELETE', bucket, key, query=query) @@ -1079,28 +1202,6 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], '0') - -class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload): - @classmethod - def setUpClass(cls): - os.environ['S3_USE_SIGV4'] = "True" - - @classmethod - def tearDownClass(cls): - del os.environ['S3_USE_SIGV4'] - - def setUp(self): - super(TestS3ApiMultiUploadSigV4, self).setUp() - - def test_object_multi_upload_part_copy_range(self): - if StrictVersion(boto.__version__) < StrictVersion('3.0'): - # boto 2 doesn't sort headers properly; see - # https://github.com/boto/boto/pull/3032 - # or https://github.com/boto/boto/pull/3176 - # or https://github.com/boto/boto/pull/3751 - # or https://github.com/boto/boto/pull/3824 - self.skipTest('This stuff got the issue of boto<=2.x') - def test_delete_bucket_multi_upload_object_exisiting(self): bucket = 'bucket' keys = ['obj1'] @@ -1178,5 +1279,9 @@ class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload): self.assertEqual(status, 204) # sanity +class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload, SigV4Mixin): + pass + + if __name__ == '__main__': unittest.main() diff --git a/test/functional/s3api/test_object.py b/test/functional/s3api/test_object.py index ca4e692dbd..b6fee2d723 100644 --- a/test/functional/s3api/test_object.py +++ b/test/functional/s3api/test_object.py @@ -15,12 +15,6 @@ # limitations under the License. import unittest -import os -import boto - -# For an issue with venv and distutils, disable pylint message here -# pylint: disable-msg=E0611,F0401 -from distutils.version import StrictVersion import calendar import email.parser @@ -35,7 +29,8 @@ from swift.common.middleware.s3api.etree import fromstring from swift.common.middleware.s3api.utils import S3Timestamp from swift.common.utils import md5, quote -from test.functional.s3api import S3ApiBase +from test.functional.s3api import S3ApiBase, SigV4Mixin, \ + skip_boto2_sort_header_bug from test.functional.s3api.s3_test_client import Connection from test.functional.s3api.utils import get_error_code, calculate_md5, \ get_error_msg @@ -368,6 +363,7 @@ class TestS3ApiObject(S3ApiBase): self.assertCommonResponseHeaders(headers) self._assertObjectEtag(self.bucket, obj, etag) + @skip_boto2_sort_header_bug def test_put_object_metadata(self): self._test_put_object_headers({ 'X-Amz-Meta-Bar': 'foo', @@ -586,6 +582,7 @@ class TestS3ApiObject(S3ApiBase): self.conn.make_request('PUT', dst_bucket, dst_obj, headers) self.assertEqual(status, 400) + @skip_boto2_sort_header_bug def test_put_object_copy_source_if_modified_since(self): obj = 'object' dst_bucket = 'dst-bucket' @@ -606,6 +603,7 @@ class TestS3ApiObject(S3ApiBase): self.assertCommonResponseHeaders(headers) self._assertObjectEtag(self.bucket, obj, etag) + @skip_boto2_sort_header_bug def test_put_object_copy_source_if_unmodified_since(self): obj = 'object' dst_bucket = 'dst-bucket' @@ -626,6 +624,7 @@ class TestS3ApiObject(S3ApiBase): self.assertCommonResponseHeaders(headers) self._assertObjectEtag(self.bucket, obj, etag) + @skip_boto2_sort_header_bug def test_put_object_copy_source_if_match(self): obj = 'object' dst_bucket = 'dst-bucket' @@ -645,6 +644,7 @@ class TestS3ApiObject(S3ApiBase): self.assertCommonResponseHeaders(headers) self._assertObjectEtag(self.bucket, obj, etag) + @skip_boto2_sort_header_bug def test_put_object_copy_source_if_none_match(self): obj = 'object' dst_bucket = 'dst-bucket' @@ -954,46 +954,8 @@ class TestS3ApiObject(S3ApiBase): self.assertCommonResponseHeaders(headers) -class TestS3ApiObjectSigV4(TestS3ApiObject): - @classmethod - def setUpClass(cls): - os.environ['S3_USE_SIGV4'] = "True" - - @classmethod - def tearDownClass(cls): - del os.environ['S3_USE_SIGV4'] - - def setUp(self): - super(TestS3ApiObjectSigV4, self).setUp() - - @unittest.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'), - 'This stuff got the signing issue of boto<=2.x') - def test_put_object_metadata(self): - super(TestS3ApiObjectSigV4, self).test_put_object_metadata() - - @unittest.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'), - 'This stuff got the signing issue of boto<=2.x') - def test_put_object_copy_source_if_modified_since(self): - super(TestS3ApiObjectSigV4, self).\ - test_put_object_copy_source_if_modified_since() - - @unittest.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'), - 'This stuff got the signing issue of boto<=2.x') - def test_put_object_copy_source_if_unmodified_since(self): - super(TestS3ApiObjectSigV4, self).\ - test_put_object_copy_source_if_unmodified_since() - - @unittest.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'), - 'This stuff got the signing issue of boto<=2.x') - def test_put_object_copy_source_if_match(self): - super(TestS3ApiObjectSigV4, - self).test_put_object_copy_source_if_match() - - @unittest.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'), - 'This stuff got the signing issue of boto<=2.x') - def test_put_object_copy_source_if_none_match(self): - super(TestS3ApiObjectSigV4, - self).test_put_object_copy_source_if_none_match() +class TestS3ApiObjectSigV4(TestS3ApiObject, SigV4Mixin): + pass if __name__ == '__main__': diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 0c0c13a8c5..20390ebed6 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -2377,6 +2377,28 @@ class TestSloGetOldManifests(SloTestCase): self.assertEqual(status, '200 OK') self.assertEqual(body, b'aaaaa') + def test_get_invalid_sysmeta_passthrough(self): + # in an attempt to workaround lp bug#2035158 s3api used to set some + # invalid slo/s3api sysmeta, we will always have some data stored with + # empty values for these headers, but they're not SLOs and are missing + # the X-Static-Large-Object marker sysmeta (thank goodness!) + headers = { + } + self.app.register( + 'GET', '/v1/AUTH_test/bucket+segments/obj/uload-id/1', + swob.HTTPOk, { + 'X-Object-Sysmeta-S3Api-Acl': "{'some': 'json'}", + 'X-Object-Sysmeta-S3Api-Etag': '', + 'X-Object-Sysmeta-Slo-Etag': '', + 'X-Object-Sysmeta-Slo-Size': '', + 'X-Object-Sysmeta-Swift3-Etag': '', + }, "any seg created with copy-part") + req = Request.blank('/v1/AUTH_test/bucket+segments/obj/uload-id/1') + status, headers, body = self.call_slo(req) + + self.assertEqual(status, '200 OK') + self.assertEqual(body, b"any seg created with copy-part") + def test_get_manifest(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc',