Merge "tests: Add test(s) for MPU part copy from range"

test/functional/s3api/__init__.py

@@ -19,6 +19,10 @@ from contextlib import contextmanager
 import logging
 from unittest import SkipTest
 
+import os
+import boto
+from distutils.version import StrictVersion
+
 import test.functional as tf
 from test.functional.s3api.s3_test_client import (
     Connection, get_boto3_conn, tear_down_s3)
@@ -104,3 +108,30 @@ class S3ApiBaseBoto3(S3ApiBase):
 
     def tearDown(self):
         tear_down_s3(self.conn)
+
+
+def skip_boto2_sort_header_bug(m):
+    def wrapped(self, *args, **kwargs):
+        if (os.environ.get('S3_USE_SIGV4') == "True" and
+                StrictVersion(boto.__version__) < StrictVersion('3.0')):
+            # boto 2 doesn't sort headers properly; see
+            # https://github.com/boto/boto/pull/3032
+            # or https://github.com/boto/boto/pull/3176
+            # or https://github.com/boto/boto/pull/3751
+            # or https://github.com/boto/boto/pull/3824
+            self.skipTest('This stuff got the issue of boto<=2.x')
+        return m(self, *args, **kwargs)
+    return wrapped
+
+
+class SigV4Mixin(object):
+    @classmethod
+    def setUpClass(cls):
+        os.environ['S3_USE_SIGV4'] = "True"
+
+    @classmethod
+    def tearDownClass(cls):
+        del os.environ['S3_USE_SIGV4']
+
+    def setUp(self):
+        super(SigV4Mixin, self).setUp()

test/functional/s3api/test_multi_upload.py

@@ -16,12 +16,6 @@
 import base64
 import binascii
 import unittest
-import os
-import boto
-
-# For an issue with venv and distutils, disable pylint message here
-# pylint: disable-msg=E0611,F0401
-from distutils.version import StrictVersion
 
 import six
 from six.moves import urllib, zip, zip_longest
@@ -33,7 +27,8 @@ from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX, mktime, \
     S3Timestamp
 from swift.common.utils import md5
 
-from test.functional.s3api import S3ApiBase
+from test.functional.s3api import S3ApiBase, SigV4Mixin, \
+    skip_boto2_sort_header_bug
 from test.functional.s3api.s3_test_client import Connection
 from test.functional.s3api.utils import get_error_code, get_error_msg, \
     calculate_md5
@@ -867,9 +862,8 @@ class TestS3ApiMultiUpload(S3ApiBase):
                                    query=query)
         self.assertEqual(status, 200)
 
-    def test_object_multi_upload_part_copy_range(self):
+    def _initiate_mpu_upload(self, bucket, key):
-        bucket = 'bucket'
+        keys = [key]
-        keys = ['obj1']
         uploads = []
 
         results_generator = self._initiate_multi_uploads_result_generator(
@@ -893,10 +887,13 @@ class TestS3ApiMultiUpload(S3ApiBase):
             self.assertTrue((key, upload_id) not in uploads)
             uploads.append((key, upload_id))
 
-        self.assertEqual(len(uploads), len(keys))  # sanity
+        # sanity, there's just one multi-part upload
+        self.assertEqual(1, len(uploads))
+        self.assertEqual(1, len(keys))
+        _, upload_id = uploads[0]
+        return upload_id
 
-        # Upload Part Copy Range
+    def _copy_part_from_new_src_range(self, bucket, key, upload_id):
-        key, upload_id = uploads[0]
         src_bucket = 'bucket2'
         src_obj = 'obj4'
         src_content = b'y' * (self.min_segment_size // 2) + b'z' * \
@@ -923,6 +920,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
         self.assertEqual(headers['content-length'], str(len(body)))
         self.assertTrue('etag' not in headers)
         elem = fromstring(body, 'CopyPartResult')
+        etags = [elem.find('ETag').text]
 
         copy_resp_last_modified = elem.find('LastModified').text
         self.assertIsNotNone(copy_resp_last_modified)
@@ -930,7 +928,6 @@ class TestS3ApiMultiUpload(S3ApiBase):
         self.assertEqual(resp_etag, etag)
 
         # Check last-modified timestamp
-        key, upload_id = uploads[0]
         query = 'uploadId=%s' % upload_id
         status, headers, body = \
             self.conn.make_request('GET', bucket, key, query=query)
@@ -942,8 +939,134 @@ class TestS3ApiMultiUpload(S3ApiBase):
         # There should be *exactly* one parts in the result
         self.assertEqual(listing_last_modified, [copy_resp_last_modified])
 
+        # sanity, there's just one etag
+        self.assertEqual(1, len(etags))
+        return etags[0]
+
+    def _complete_mpu_upload(self, bucket, key, upload_id, etags):
+        # Complete Multipart Upload
+        query = 'uploadId=%s' % upload_id
+        xml = self._gen_comp_xml(etags)
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, body=xml,
+                                   query=query)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-type' in headers)
+        self.assertEqual(headers['content-type'], 'application/xml')
+        if 'content-length' in headers:
+            self.assertEqual(headers['content-length'], str(len(body)))
+        else:
+            self.assertIn('transfer-encoding', headers)
+            self.assertEqual(headers['transfer-encoding'], 'chunked')
+        lines = body.split(b'\n')
+        self.assertTrue(lines[0].startswith(b'<?xml'), body)
+        self.assertTrue(lines[0].endswith(b'?>'), body)
+        elem = fromstring(body, 'CompleteMultipartUploadResult')
+        self.assertEqual(
+            '%s/%s/%s' %
+            (tf.config['s3_storage_url'].rstrip('/'), bucket, key),
+            elem.find('Location').text)
+        self.assertEqual(elem.find('Bucket').text, bucket)
+        self.assertEqual(elem.find('Key').text, key)
+        concatted_etags = b''.join(
+            etag.strip('"').encode('ascii') for etag in etags)
+        exp_etag = '"%s-%s"' % (
+            md5(binascii.unhexlify(concatted_etags),
+                usedforsecurity=False).hexdigest(), len(etags))
+        etag = elem.find('ETag').text
+        self.assertEqual(etag, exp_etag)
+
+    @skip_boto2_sort_header_bug
+    def test_mpu_copy_part_from_range_then_complete(self):
+        bucket = 'mpu-copy-range'
+        key = 'obj-complete'
+        upload_id = self._initiate_mpu_upload(bucket, key)
+        etag = self._copy_part_from_new_src_range(bucket, key, upload_id)
+        self._complete_mpu_upload(bucket, key, upload_id, [etag])
+
+    @skip_boto2_sort_header_bug
+    def test_mpu_copy_part_from_range_then_abort(self):
+        bucket = 'mpu-copy-range'
+        key = 'obj-abort'
+        upload_id = self._initiate_mpu_upload(bucket, key)
+        self._copy_part_from_new_src_range(bucket, key, upload_id)
+
+        # Abort Multipart Upload
+        query = 'uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('DELETE', bucket, key, query=query)
+
+        # sanity checks
+        self.assertEqual(status, 204)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-type' in headers)
+        self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8')
+        self.assertTrue('content-length' in headers)
+        self.assertEqual(headers['content-length'], '0')
+
+    def _copy_part_from_new_mpu_range(self, bucket, key, upload_id):
+        src_bucket = 'bucket2'
+        src_obj = 'mpu2'
+        src_upload_id = self._initiate_mpu_upload(src_bucket, src_obj)
+        # upload parts
+        etags = []
+        for part_num in range(2):
+            # Upload Part
+            content = (chr(97 + part_num) * self.min_segment_size).encode()
+            etag = md5(content, usedforsecurity=False).hexdigest()
+            status, headers, body = \
+                self._upload_part(src_bucket, src_obj, src_upload_id,
+                                  content, part_num=part_num + 1)
+            self.assertEqual(status, 200)
+            self.assertCommonResponseHeaders(headers, etag)
+            self.assertTrue('content-type' in headers)
+            self.assertEqual(headers['content-type'],
+                             'text/html; charset=UTF-8')
+            self.assertTrue('content-length' in headers)
+            self.assertEqual(headers['content-length'], '0')
+            self.assertEqual(headers['etag'], '"%s"' % etag)
+            etags.append(etag)
+        self._complete_mpu_upload(src_bucket, src_obj, src_upload_id, etags)
+
+        # Upload Part Copy -- MPU as source
+        src_range = 'bytes=0-%d' % (self.min_segment_size - 1)
+        status, headers, body, resp_etag = \
+            self._upload_part_copy(src_bucket, src_obj, bucket,
+                                   key, upload_id, part_num=1,
+                                   src_range=src_range)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertIn('content-type', headers)
+        self.assertEqual(headers['content-type'], 'application/xml')
+        self.assertIn('content-length', headers)
+        self.assertEqual(headers['content-length'], str(len(body)))
+        self.assertNotIn('etag', headers)
+        elem = fromstring(body, 'CopyPartResult')
+
+        last_modified = elem.find('LastModified').text
+        self.assertIsNotNone(last_modified)
+        # use copied with src_range from src_obj?part-number=1
+        self.assertEqual(resp_etag, etags[0])
+
+        return resp_etag
+
+    @skip_boto2_sort_header_bug
+    def test_mpu_copy_part_from_mpu_part_number_then_complete(self):
+        bucket = 'mpu-copy-range'
+        key = 'obj-complete'
+        upload_id = self._initiate_mpu_upload(bucket, key)
+        etag = self._copy_part_from_new_mpu_range(bucket, key, upload_id)
+        self._complete_mpu_upload(bucket, key, upload_id, [etag])
+
+    @skip_boto2_sort_header_bug
+    def test_mpu_copy_part_from_mpu_part_number_then_abort(self):
+        bucket = 'mpu-copy-range'
+        key = 'obj-abort'
+        upload_id = self._initiate_mpu_upload(bucket, key)
+        self._copy_part_from_new_mpu_range(bucket, key, upload_id)
+
         # Abort Multipart Upload
-        key, upload_id = uploads[0]
         query = 'uploadId=%s' % upload_id
         status, headers, body = \
             self.conn.make_request('DELETE', bucket, key, query=query)
@@ -1079,28 +1202,6 @@ class TestS3ApiMultiUpload(S3ApiBase):
         self.assertTrue('content-length' in headers)
         self.assertEqual(headers['content-length'], '0')
 
-
-class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload):
-    @classmethod
-    def setUpClass(cls):
-        os.environ['S3_USE_SIGV4'] = "True"
-
-    @classmethod
-    def tearDownClass(cls):
-        del os.environ['S3_USE_SIGV4']
-
-    def setUp(self):
-        super(TestS3ApiMultiUploadSigV4, self).setUp()
-
-    def test_object_multi_upload_part_copy_range(self):
-        if StrictVersion(boto.__version__) < StrictVersion('3.0'):
-            # boto 2 doesn't sort headers properly; see
-            # https://github.com/boto/boto/pull/3032
-            # or https://github.com/boto/boto/pull/3176
-            # or https://github.com/boto/boto/pull/3751
-            # or https://github.com/boto/boto/pull/3824
-            self.skipTest('This stuff got the issue of boto<=2.x')
-
     def test_delete_bucket_multi_upload_object_exisiting(self):
         bucket = 'bucket'
         keys = ['obj1']
@@ -1178,5 +1279,9 @@ class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload):
         self.assertEqual(status, 204)  # sanity
 
 
+class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload, SigV4Mixin):
+    pass
+
+
 if __name__ == '__main__':
     unittest.main()

test/functional/s3api/test_object.py

@@ -15,12 +15,6 @@
 # limitations under the License.
 
 import unittest
-import os
-import boto
-
-# For an issue with venv and distutils, disable pylint message here
-# pylint: disable-msg=E0611,F0401
-from distutils.version import StrictVersion
 
 import calendar
 import email.parser
@@ -35,7 +29,8 @@ from swift.common.middleware.s3api.etree import fromstring
 from swift.common.middleware.s3api.utils import S3Timestamp
 from swift.common.utils import md5, quote
 
-from test.functional.s3api import S3ApiBase
+from test.functional.s3api import S3ApiBase, SigV4Mixin, \
+    skip_boto2_sort_header_bug
 from test.functional.s3api.s3_test_client import Connection
 from test.functional.s3api.utils import get_error_code, calculate_md5, \
     get_error_msg
@@ -368,6 +363,7 @@ class TestS3ApiObject(S3ApiBase):
         self.assertCommonResponseHeaders(headers)
         self._assertObjectEtag(self.bucket, obj, etag)
 
+    @skip_boto2_sort_header_bug
     def test_put_object_metadata(self):
         self._test_put_object_headers({
             'X-Amz-Meta-Bar': 'foo',
@@ -586,6 +582,7 @@ class TestS3ApiObject(S3ApiBase):
             self.conn.make_request('PUT', dst_bucket, dst_obj, headers)
         self.assertEqual(status, 400)
 
+    @skip_boto2_sort_header_bug
     def test_put_object_copy_source_if_modified_since(self):
         obj = 'object'
         dst_bucket = 'dst-bucket'
@@ -606,6 +603,7 @@ class TestS3ApiObject(S3ApiBase):
         self.assertCommonResponseHeaders(headers)
         self._assertObjectEtag(self.bucket, obj, etag)
 
+    @skip_boto2_sort_header_bug
     def test_put_object_copy_source_if_unmodified_since(self):
         obj = 'object'
         dst_bucket = 'dst-bucket'
@@ -626,6 +624,7 @@ class TestS3ApiObject(S3ApiBase):
         self.assertCommonResponseHeaders(headers)
         self._assertObjectEtag(self.bucket, obj, etag)
 
+    @skip_boto2_sort_header_bug
     def test_put_object_copy_source_if_match(self):
         obj = 'object'
         dst_bucket = 'dst-bucket'
@@ -645,6 +644,7 @@ class TestS3ApiObject(S3ApiBase):
         self.assertCommonResponseHeaders(headers)
         self._assertObjectEtag(self.bucket, obj, etag)
 
+    @skip_boto2_sort_header_bug
     def test_put_object_copy_source_if_none_match(self):
         obj = 'object'
         dst_bucket = 'dst-bucket'
@@ -954,46 +954,8 @@ class TestS3ApiObject(S3ApiBase):
         self.assertCommonResponseHeaders(headers)
 
 
-class TestS3ApiObjectSigV4(TestS3ApiObject):
+class TestS3ApiObjectSigV4(TestS3ApiObject, SigV4Mixin):
-    @classmethod
+    pass
-    def setUpClass(cls):
-        os.environ['S3_USE_SIGV4'] = "True"
-
-    @classmethod
-    def tearDownClass(cls):
-        del os.environ['S3_USE_SIGV4']
-
-    def setUp(self):
-        super(TestS3ApiObjectSigV4, self).setUp()
-
-    @unittest.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'),
-                     'This stuff got the signing issue of boto<=2.x')
-    def test_put_object_metadata(self):
-        super(TestS3ApiObjectSigV4, self).test_put_object_metadata()
-
-    @unittest.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'),
-                     'This stuff got the signing issue of boto<=2.x')
-    def test_put_object_copy_source_if_modified_since(self):
-        super(TestS3ApiObjectSigV4, self).\
-            test_put_object_copy_source_if_modified_since()
-
-    @unittest.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'),
-                     'This stuff got the signing issue of boto<=2.x')
-    def test_put_object_copy_source_if_unmodified_since(self):
-        super(TestS3ApiObjectSigV4, self).\
-            test_put_object_copy_source_if_unmodified_since()
-
-    @unittest.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'),
-                     'This stuff got the signing issue of boto<=2.x')
-    def test_put_object_copy_source_if_match(self):
-        super(TestS3ApiObjectSigV4,
-              self).test_put_object_copy_source_if_match()
-
-    @unittest.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'),
-                     'This stuff got the signing issue of boto<=2.x')
-    def test_put_object_copy_source_if_none_match(self):
-        super(TestS3ApiObjectSigV4,
-              self).test_put_object_copy_source_if_none_match()
 
 
 if __name__ == '__main__':

test/unit/common/middleware/test_slo.py

@@ -2377,6 +2377,28 @@ class TestSloGetOldManifests(SloTestCase):
         self.assertEqual(status, '200 OK')
         self.assertEqual(body, b'aaaaa')
 
+    def test_get_invalid_sysmeta_passthrough(self):
+        # in an attempt to workaround lp bug#2035158 s3api used to set some
+        # invalid slo/s3api sysmeta, we will always have some data stored with
+        # empty values for these headers, but they're not SLOs and are missing
+        # the X-Static-Large-Object marker sysmeta (thank goodness!)
+        headers = {
+        }
+        self.app.register(
+            'GET', '/v1/AUTH_test/bucket+segments/obj/uload-id/1',
+            swob.HTTPOk, {
+                'X-Object-Sysmeta-S3Api-Acl': "{'some': 'json'}",
+                'X-Object-Sysmeta-S3Api-Etag': '',
+                'X-Object-Sysmeta-Slo-Etag': '',
+                'X-Object-Sysmeta-Slo-Size': '',
+                'X-Object-Sysmeta-Swift3-Etag': '',
+            }, "any seg created with copy-part")
+        req = Request.blank('/v1/AUTH_test/bucket+segments/obj/uload-id/1')
+        status, headers, body = self.call_slo(req)
+
+        self.assertEqual(status, '200 OK')
+        self.assertEqual(body, b"any seg created with copy-part")
+
     def test_get_manifest(self):
         req = Request.blank(
             '/v1/AUTH_test/gettest/manifest-bc',