py3: mostly port s3 func tests

test_bucket.py is proving somewhat problematic.

Change-Id: I5b337ef66a23fc989762801dd6a5ba1ed903f57b

.zuul.yaml

@@ -132,6 +132,18 @@
     vars:
       tox_envlist: func-domain-remap-staticweb-py3
 
+- job:
+    name: swift-tox-func-s3api-py37
+    parent: swift-tox-func-py37
+    description: |
+      Run functional tests for swift under cPython version 3.7.
+
+      Uses tox with the ``func-s3api`` environment.
+      It sets TMPDIR to an XFS mount point created via
+      tools/test-setup.sh.
+    vars:
+      tox_envlist: func-s3api-py3
+
 - job:
     name: swift-tox-func-centos-7
     parent: swift-tox-func
@@ -480,6 +492,11 @@
               - ^(api-ref|doc|releasenotes)/.*$
               - ^test/probe/.*$
               - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
+        - swift-tox-func-s3api-py37:
+            irrelevant-files:
+              - ^(api-ref|doc|releasenotes)/.*$
+              - ^test/probe/.*$
+              - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
 
         # Other tests
         - swift-tox-func-s3api-ceph-s3tests-tempauth:
@@ -555,6 +572,7 @@
         - swift-tox-func-encryption
         - swift-tox-func-domain-remap-staticweb-py37
         - swift-tox-func-ec-py37
+        - swift-tox-func-s3api-py37
         - swift-probetests-centos-7:
             irrelevant-files:
               - ^(api-ref|releasenotes)/.*$

test/functional/s3api/s3_test_client.py

@@ -81,7 +81,7 @@ class Connection(object):
                     break
 
                 for bucket in buckets:
-                    if not isinstance(bucket.name, six.binary_type):
+                    if six.PY2 and not isinstance(bucket.name, bytes):
                         bucket.name = bucket.name.encode('utf-8')
 
                     try:
@@ -103,7 +103,7 @@ class Connection(object):
             exceptions.insert(0, 'Too many errors to continue:')
             raise Exception('\n========\n'.join(exceptions))
 
-    def make_request(self, method, bucket='', obj='', headers=None, body='',
+    def make_request(self, method, bucket='', obj='', headers=None, body=b'',
                      query=None):
         """
         Wrapper method of S3Connection.make_request.
@@ -123,7 +123,9 @@ class Connection(object):
                                    query_args=query, sender=None,
                                    override_num_retries=RETRY_COUNT,
                                    retry_handler=None)
-        return response.status, dict(response.getheaders()), response.read()
+        return (response.status,
+                {h.lower(): v for h, v in response.getheaders()},
+                response.read())
 
     def generate_url_and_headers(self, method, bucket='', obj='',
                                  expires_in=3600):

test/functional/s3api/test_acl.py

@@ -40,7 +40,8 @@ class TestS3Acl(S3ApiBase):
             raise tf.SkipTest(
                 'TestS3Acl requires s3_access_key3 and s3_secret_key3 '
                 'configured for reduced-access user')
-        self.conn.make_request('PUT', self.bucket)
+        status, headers, body = self.conn.make_request('PUT', self.bucket)
+        self.assertEqual(status, 200, body)
         access_key3 = tf.config['s3_access_key3']
         secret_key3 = tf.config['s3_secret_key3']
         self.conn3 = Connection(access_key3, secret_key3, access_key3)

test/functional/s3api/test_multi_upload.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import base64
+import binascii
 import unittest2
 import os
 import boto
@@ -23,7 +24,7 @@ import boto
 from distutils.version import StrictVersion
 
 from hashlib import md5
-from itertools import izip, izip_longest
+from six.moves import zip, zip_longest
 
 import test.functional as tf
 from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \
@@ -67,7 +68,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
             headers = [None] * len(keys)
         self.conn.make_request('PUT', bucket)
         query = 'uploads'
-        for key, key_headers in izip_longest(keys, headers):
+        for key, key_headers in zip_longest(keys, headers):
             for i in range(trials):
                 status, resp_headers, body = \
                     self.conn.make_request('POST', bucket, key,
@@ -76,7 +77,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
 
     def _upload_part(self, bucket, key, upload_id, content=None, part_num=1):
         query = 'partNumber=%s&uploadId=%s' % (part_num, upload_id)
-        content = content if content else 'a' * self.min_segment_size
+        content = content if content else b'a' * self.min_segment_size
         status, headers, body = \
             self.conn.make_request('PUT', bucket, key, body=content,
                                    query=query)
@@ -108,8 +109,9 @@ class TestS3ApiMultiUpload(S3ApiBase):
     def test_object_multi_upload(self):
         bucket = 'bucket'
         keys = ['obj1', 'obj2', 'obj3']
+        bad_content_md5 = base64.b64encode(b'a' * 16).strip().decode('ascii')
         headers = [None,
-                   {'Content-MD5': base64.b64encode('a' * 16).strip()},
+                   {'Content-MD5': bad_content_md5},
                    {'Etag': 'nonsense'}]
         uploads = []
 
@@ -118,20 +120,20 @@ class TestS3ApiMultiUpload(S3ApiBase):
 
         # Initiate Multipart Upload
         for expected_key, (status, headers, body) in \
-                izip(keys, results_generator):
+                zip(keys, results_generator):
-            self.assertEqual(status, 200)
+            self.assertEqual(status, 200, body)
             self.assertCommonResponseHeaders(headers)
-            self.assertTrue('content-type' in headers)
+            self.assertIn('content-type', headers)
             self.assertEqual(headers['content-type'], 'application/xml')
-            self.assertTrue('content-length' in headers)
+            self.assertIn('content-length', headers)
             self.assertEqual(headers['content-length'], str(len(body)))
             elem = fromstring(body, 'InitiateMultipartUploadResult')
             self.assertEqual(elem.find('Bucket').text, bucket)
             key = elem.find('Key').text
             self.assertEqual(expected_key, key)
             upload_id = elem.find('UploadId').text
-            self.assertTrue(upload_id is not None)
+            self.assertIsNotNone(upload_id)
-            self.assertTrue((key, upload_id) not in uploads)
+            self.assertNotIn((key, upload_id), uploads)
             uploads.append((key, upload_id))
 
         self.assertEqual(len(uploads), len(keys))  # sanity
@@ -157,7 +159,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
         self.assertEqual(elem.find('IsTruncated').text, 'false')
         self.assertEqual(len(elem.findall('Upload')), 3)
         for (expected_key, expected_upload_id), u in \
-                izip(uploads, elem.findall('Upload')):
+                zip(uploads, elem.findall('Upload')):
             key = u.find('Key').text
             upload_id = u.find('UploadId').text
             self.assertEqual(expected_key, key)
@@ -174,7 +176,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
 
         # Upload Part
         key, upload_id = uploads[0]
-        content = 'a' * self.min_segment_size
+        content = b'a' * self.min_segment_size
         etag = md5(content).hexdigest()
         status, headers, body = \
             self._upload_part(bucket, key, upload_id, content)
@@ -190,7 +192,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
         key, upload_id = uploads[1]
         src_bucket = 'bucket2'
         src_obj = 'obj3'
-        src_content = 'b' * self.min_segment_size
+        src_content = b'b' * self.min_segment_size
         etag = md5(src_content).hexdigest()
 
         # prepare src obj
@@ -266,7 +268,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
         # etags will be used to generate xml for Complete Multipart Upload
         etags = []
         for (expected_etag, expected_date), p in \
-                izip(expected_parts_list, elem.findall('Part')):
+                zip(expected_parts_list, elem.findall('Part')):
             last_modified = p.find('LastModified').text
             self.assertTrue(last_modified is not None)
             # TODO: sanity check
@@ -295,9 +297,9 @@ class TestS3ApiMultiUpload(S3ApiBase):
         else:
             self.assertIn('transfer-encoding', headers)
             self.assertEqual(headers['transfer-encoding'], 'chunked')
-        lines = body.split('\n')
+        lines = body.split(b'\n')
-        self.assertTrue(lines[0].startswith('<?xml'), body)
+        self.assertTrue(lines[0].startswith(b'<?xml'), body)
-        self.assertTrue(lines[0].endswith('?>'), body)
+        self.assertTrue(lines[0].endswith(b'?>'), body)
         elem = fromstring(body, 'CompleteMultipartUploadResult')
         # TODO: use tf.config value
         self.assertEqual(
@@ -305,9 +307,10 @@ class TestS3ApiMultiUpload(S3ApiBase):
             elem.find('Location').text)
         self.assertEqual(elem.find('Bucket').text, bucket)
         self.assertEqual(elem.find('Key').text, key)
-        concatted_etags = ''.join(etag.strip('"') for etag in etags)
+        concatted_etags = b''.join(
+            etag.strip('"').encode('ascii') for etag in etags)
         exp_etag = '"%s-%s"' % (
-            md5(concatted_etags.decode('hex')).hexdigest(), len(etags))
+            md5(binascii.unhexlify(concatted_etags)).hexdigest(), len(etags))
         etag = elem.find('ETag').text
         self.assertEqual(etag, exp_etag)
 
@@ -332,7 +335,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
         last_modified = elem.find('LastModified').text
         self.assertIsNotNone(last_modified)
 
-        exp_content = 'a' * self.min_segment_size
+        exp_content = b'a' * self.min_segment_size
         etag = md5(exp_content).hexdigest()
         self.assertEqual(resp_etag, etag)
 
@@ -723,7 +726,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
             query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
             status, headers, body = \
                 self.conn.make_request('PUT', bucket, key, query=query,
-                                       body='A' * body_size[i])
+                                       body=b'A' * body_size[i])
             etags.append(headers['etag'])
             xml = self._gen_comp_xml(etags)
 
@@ -747,7 +750,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
             query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
             status, headers, body = \
                 self.conn.make_request('PUT', bucket, key, query=query,
-                                       body='A' * body_size[i])
+                                       body=b'A' * body_size[i])
             etags.append(headers['etag'])
             xml = self._gen_comp_xml(etags)
 
@@ -770,9 +773,9 @@ class TestS3ApiMultiUpload(S3ApiBase):
         etags = []
         for i in range(1, 4):
             query = 'partNumber=%s&uploadId=%s' % (2 * i - 1, upload_id)
-            status, headers, body = \
+            status, headers, body = self.conn.make_request(
-                self.conn.make_request('PUT', bucket, key,
+                'PUT', bucket, key, body=b'A' * 1024 * 1024 * 5,
-                                       body='A' * 1024 * 1024 * 5, query=query)
+                query=query)
             etags.append(headers['etag'])
         query = 'uploadId=%s' % upload_id
         xml = self._gen_comp_xml(etags[:-1], step=2)
@@ -791,7 +794,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
 
         # Initiate Multipart Upload
         for expected_key, (status, headers, body) in \
-                izip(keys, results_generator):
+                zip(keys, results_generator):
             self.assertEqual(status, 200)
             self.assertCommonResponseHeaders(headers)
             self.assertTrue('content-type' in headers)
@@ -813,7 +816,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
         key, upload_id = uploads[0]
         src_bucket = 'bucket2'
         src_obj = 'obj4'
-        src_content = 'y' * (self.min_segment_size / 2) + 'z' * \
+        src_content = b'y' * (self.min_segment_size // 2) + b'z' * \
             self.min_segment_size
         src_range = 'bytes=0-%d' % (self.min_segment_size - 1)
         etag = md5(src_content[:self.min_segment_size]).hexdigest()
@@ -901,7 +904,7 @@ class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload):
 
         # Initiate Multipart Upload
         for expected_key, (status, _, body) in \
-                izip(keys, results_generator):
+                zip(keys, results_generator):
             self.assertEqual(status, 200)  # sanity
             elem = fromstring(body, 'InitiateMultipartUploadResult')
             key = elem.find('Key').text
@@ -915,7 +918,7 @@ class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload):
 
         # Upload Part
         key, upload_id = uploads[0]
-        content = 'a' * self.min_segment_size
+        content = b'a' * self.min_segment_size
         status, headers, body = \
             self._upload_part(bucket, key, upload_id, content)
         self.assertEqual(status, 200)

test/functional/s3api/test_object.py

@@ -25,7 +25,8 @@ import email.parser
 from email.utils import formatdate, parsedate
 from time import mktime
 from hashlib import md5
-from urllib import quote
+import six
+from six.moves.urllib.parse import quote
 
 import test.functional as tf
 
@@ -59,7 +60,7 @@ class TestS3ApiObject(S3ApiBase):
 
     def test_object(self):
         obj = 'object name with %-sign'
-        content = 'abc123'
+        content = b'abc123'
         etag = md5(content).hexdigest()
 
         # PUT Object
@@ -219,19 +220,19 @@ class TestS3ApiObject(S3ApiBase):
         status, headers, body = \
             auth_error_conn.make_request('HEAD', self.bucket, obj)
         self.assertEqual(status, 403)
-        self.assertEqual(body, '')  # sanity
+        self.assertEqual(body, b'')  # sanity
         self.assertEqual(headers['content-type'], 'application/xml')
 
         status, headers, body = \
             self.conn.make_request('HEAD', self.bucket, 'invalid')
         self.assertEqual(status, 404)
-        self.assertEqual(body, '')  # sanity
+        self.assertEqual(body, b'')  # sanity
         self.assertEqual(headers['content-type'], 'application/xml')
 
         status, headers, body = \
             self.conn.make_request('HEAD', 'invalid', obj)
         self.assertEqual(status, 404)
-        self.assertEqual(body, '')  # sanity
+        self.assertEqual(body, b'')  # sanity
         self.assertEqual(headers['content-type'], 'application/xml')
 
     def test_delete_object_error(self):
@@ -265,7 +266,7 @@ class TestS3ApiObject(S3ApiBase):
 
     def test_put_object_content_md5(self):
         obj = 'object'
-        content = 'abcdefghij'
+        content = b'abcdefghij'
         etag = md5(content).hexdigest()
         headers = {'Content-MD5': calculate_md5(content)}
         status, headers, body = \
@@ -276,7 +277,7 @@ class TestS3ApiObject(S3ApiBase):
 
     def test_put_object_content_type(self):
         obj = 'object'
-        content = 'abcdefghij'
+        content = b'abcdefghij'
         etag = md5(content).hexdigest()
         headers = {'Content-Type': 'text/plain'}
         status, headers, body = \
@@ -290,7 +291,7 @@ class TestS3ApiObject(S3ApiBase):
 
     def test_put_object_conditional_requests(self):
         obj = 'object'
-        content = 'abcdefghij'
+        content = b'abcdefghij'
         headers = {'If-None-Match': '*'}
         status, headers, body = \
             self.conn.make_request('PUT', self.bucket, obj, headers, content)
@@ -318,7 +319,7 @@ class TestS3ApiObject(S3ApiBase):
 
     def test_put_object_expect(self):
         obj = 'object'
-        content = 'abcdefghij'
+        content = b'abcdefghij'
         etag = md5(content).hexdigest()
         headers = {'Expect': '100-continue'}
         status, headers, body = \
@@ -331,7 +332,7 @@ class TestS3ApiObject(S3ApiBase):
         if expected_headers is None:
             expected_headers = req_headers
         obj = 'object'
-        content = 'abcdefghij'
+        content = b'abcdefghij'
         etag = md5(content).hexdigest()
         status, headers, body = \
             self.conn.make_request('PUT', self.bucket, obj,
@@ -387,7 +388,7 @@ class TestS3ApiObject(S3ApiBase):
 
     def test_put_object_storage_class(self):
         obj = 'object'
-        content = 'abcdefghij'
+        content = b'abcdefghij'
         etag = md5(content).hexdigest()
         headers = {'X-Amz-Storage-Class': 'STANDARD'}
         status, headers, body = \
@@ -399,7 +400,7 @@ class TestS3ApiObject(S3ApiBase):
     def test_put_object_copy_source_params(self):
         obj = 'object'
         src_headers = {'X-Amz-Meta-Test': 'src'}
-        src_body = 'some content'
+        src_body = b'some content'
         dst_bucket = 'dst-bucket'
         dst_obj = 'dst_object'
         self.conn.make_request('PUT', self.bucket, obj, src_headers, src_body)
@@ -433,7 +434,7 @@ class TestS3ApiObject(S3ApiBase):
 
     def test_put_object_copy_source(self):
         obj = 'object'
-        content = 'abcdefghij'
+        content = b'abcdefghij'
         etag = md5(content).hexdigest()
         self.conn.make_request('PUT', self.bucket, obj, body=content)
 
@@ -648,7 +649,7 @@ class TestS3ApiObject(S3ApiBase):
 
     def test_get_object_range(self):
         obj = 'object'
-        content = 'abcdefghij'
+        content = b'abcdefghij'
         headers = {'x-amz-meta-test': 'swift'}
         self.conn.make_request(
             'PUT', self.bucket, obj, headers=headers, body=content)
@@ -662,7 +663,7 @@ class TestS3ApiObject(S3ApiBase):
         self.assertEqual(headers['content-length'], '5')
         self.assertTrue('x-amz-meta-test' in headers)
         self.assertEqual('swift', headers['x-amz-meta-test'])
-        self.assertEqual(body, 'bcdef')
+        self.assertEqual(body, b'bcdef')
 
         headers = {'Range': 'bytes=5-'}
         status, headers, body = \
@@ -673,7 +674,7 @@ class TestS3ApiObject(S3ApiBase):
         self.assertEqual(headers['content-length'], '5')
         self.assertTrue('x-amz-meta-test' in headers)
         self.assertEqual('swift', headers['x-amz-meta-test'])
-        self.assertEqual(body, 'fghij')
+        self.assertEqual(body, b'fghij')
 
         headers = {'Range': 'bytes=-5'}
         status, headers, body = \
@@ -684,7 +685,7 @@ class TestS3ApiObject(S3ApiBase):
         self.assertEqual(headers['content-length'], '5')
         self.assertTrue('x-amz-meta-test' in headers)
         self.assertEqual('swift', headers['x-amz-meta-test'])
-        self.assertEqual(body, 'fghij')
+        self.assertEqual(body, b'fghij')
 
         ranges = ['1-2', '4-5']
 
@@ -693,9 +694,9 @@ class TestS3ApiObject(S3ApiBase):
             self.conn.make_request('GET', self.bucket, obj, headers=headers)
         self.assertEqual(status, 206)
         self.assertCommonResponseHeaders(headers)
-        self.assertTrue('content-length' in headers)
+        self.assertIn('content-length', headers)
 
-        self.assertTrue('content-type' in headers)  # sanity
+        self.assertIn('content-type', headers)  # sanity
         content_type, boundary = headers['content-type'].split(';')
 
         self.assertEqual('multipart/byteranges', content_type)
@@ -704,10 +705,13 @@ class TestS3ApiObject(S3ApiBase):
 
         # TODO: Using swift.common.utils.multipart_byteranges_to_document_iters
         #       could be easy enough.
+        if six.PY2:
             parser = email.parser.FeedParser()
+        else:
+            parser = email.parser.BytesFeedParser()
         parser.feed(
-            "Content-Type: multipart/byterange; boundary=%s\r\n\r\n" %
+            b"Content-Type: multipart/byterange; boundary=%s\r\n\r\n" %
-            boundary_str)
+            boundary_str.encode('ascii'))
         parser.feed(body)
         message = parser.close()
 
@@ -727,7 +731,7 @@ class TestS3ApiObject(S3ApiBase):
             self.assertEqual(
                 expected_range, part.get('Content-Range'))
             # rest
-            payload = part.get_payload().strip()
+            payload = part.get_payload(decode=True).strip()
             self.assertEqual(content[start:end + 1], payload)
 
     def test_get_object_if_modified_since(self):
@@ -783,7 +787,7 @@ class TestS3ApiObject(S3ApiBase):
 
     def test_head_object_range(self):
         obj = 'object'
-        content = 'abcdefghij'
+        content = b'abcdefghij'
         self.conn.make_request('PUT', self.bucket, obj, body=content)
 
         headers = {'Range': 'bytes=1-5'}

test/functional/s3api/test_presigned.py

@@ -190,7 +190,7 @@ class TestS3ApiPresignedUrls(S3ApiBase):
         # PUT empty object
         put_url, headers = self.conn.generate_url_and_headers(
             'PUT', bucket, obj)
-        resp = requests.put(put_url, data='', headers=headers)
+        resp = requests.put(put_url, data=b'', headers=headers)
         self.assertEqual(resp.status_code, 200,
                          'Got %d %s' % (resp.status_code, resp.content))
         # GET empty object
@@ -199,10 +199,10 @@ class TestS3ApiPresignedUrls(S3ApiBase):
         resp = requests.get(get_url, headers=headers)
         self.assertEqual(resp.status_code, 200,
                          'Got %d %s' % (resp.status_code, resp.content))
-        self.assertEqual(resp.content, '')
+        self.assertEqual(resp.content, b'')
 
         # PUT over object
-        resp = requests.put(put_url, data='foobar', headers=headers)
+        resp = requests.put(put_url, data=b'foobar', headers=headers)
         self.assertEqual(resp.status_code, 200,
                          'Got %d %s' % (resp.status_code, resp.content))
 
@@ -210,7 +210,7 @@ class TestS3ApiPresignedUrls(S3ApiBase):
         resp = requests.get(get_url, headers=headers)
         self.assertEqual(resp.status_code, 200,
                          'Got %d %s' % (resp.status_code, resp.content))
-        self.assertEqual(resp.content, 'foobar')
+        self.assertEqual(resp.content, b'foobar')
 
         # DELETE Object
         delete_url, headers = self.conn.generate_url_and_headers(

test/functional/s3api/test_service.py

@@ -80,8 +80,8 @@ class TestS3ApiService(S3ApiBase):
             'GET', headers={'Date': '', 'x-amz-date': ''})
         self.assertEqual(status, 403)
         self.assertEqual(get_error_code(body), 'AccessDenied')
-        self.assertIn('AWS authentication requires a valid Date '
+        self.assertIn(b'AWS authentication requires a valid Date '
-                      'or x-amz-date header', body)
+                      b'or x-amz-date header', body)
 
 
 class TestS3ApiServiceSigV4(TestS3ApiService):

test/functional/s3api/utils.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from base64 import b64encode
 from hashlib import md5
 from swift.common.middleware.s3api.etree import fromstring
 
@@ -28,4 +29,4 @@ def get_error_msg(body):
 
 
 def calculate_md5(body):
-    return md5(body).digest().encode('base64').strip()
+    return b64encode(md5(body).digest()).strip().decode('ascii')

tox.ini

@@ -48,6 +48,12 @@ commands = ./.functests {posargs}
 basepython = python3
 commands =
   nosetests {posargs: \
+    test/functional/s3api/test_acl.py \
+    test/functional/s3api/test_multi_delete.py \
+    test/functional/s3api/test_multi_upload.py \
+    test/functional/s3api/test_object.py \
+    test/functional/s3api/test_presigned.py \
+    test/functional/s3api/test_service.py \
     test/functional/test_access_control.py \
     test/functional/test_domain_remap.py \
     test/functional/test_object.py \
@@ -62,6 +68,12 @@ commands = {[testenv:func-py3]commands}
 setenv = SWIFT_TEST_IN_PROCESS=1
          SWIFT_TEST_IN_PROCESS_CONF_LOADER=ec
 
+[testenv:func-s3api-py3]
+basepython = python3
+commands = {[testenv:func-py3]commands}
+setenv = SWIFT_TEST_IN_PROCESS=1
+         SWIFT_TEST_IN_PROCESS_CONF_LOADER=s3api
+
 [testenv:func-encryption-py3]
 basepython = python3
 commands = {[testenv:func-py3]commands}