From 2926fc96a36a573380eca18d60440a2337442015 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 22 Mar 2019 14:48:10 -0700 Subject: [PATCH] py3: Bring functional/test_object.py under test; add func-ec-py37 job Change-Id: I2929ce1f8e08ae5b4059605d09189c8521852423 --- .zuul.yaml | 38 +++++++++++-- test/functional/test_object.py | 100 +++++++++++++++++++-------------- tox.ini | 7 +++ 3 files changed, 98 insertions(+), 47 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 18dca3cac8..14e713e1b2 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -96,6 +96,18 @@ bindep_profile: test py37 python_version: 3.7 +- job: + name: swift-tox-func-ec-py37 + parent: swift-tox-func-py37 + description: | + Run functional tests for swift under cPython version 3.7. + + Uses tox with the ``func-ec-py3`` environment. + It sets TMPDIR to an XFS mount point created via + tools/test-setup.sh. + vars: + tox_envlist: func-ec-py3 + - job: name: swift-tox-func-domain-remap-staticweb-py37 parent: swift-tox-func-py37 @@ -393,6 +405,8 @@ irrelevant-files: - ^(api-ref|doc|releasenotes)/.*$ - ^test/(functional|probe)/.*$ + + # Unit tests - swift-tox-py27: irrelevant-files: - ^(api-ref|doc|releasenotes)/.*$ @@ -406,16 +420,13 @@ irrelevant-files: - ^(api-ref|doc|releasenotes)/.*$ - ^test/(functional|probe)/.*$ + + # Functional tests - swift-tox-func: irrelevant-files: - ^(api-ref|doc|releasenotes)/.*$ - ^test/probe/.*$ - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$ - - swift-tox-func-py37: - irrelevant-files: - - ^(api-ref|doc|releasenotes)/.*$ - - ^test/probe/.*$ - - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$ - swift-tox-func-encryption: irrelevant-files: - ^(api-ref|doc|releasenotes)/.*$ @@ -436,11 +447,25 @@ - ^(api-ref|doc|releasenotes)/.*$ - ^test/probe/.*$ - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$ + + # py3 functional tests + - swift-tox-func-py37: + irrelevant-files: + - ^(api-ref|doc|releasenotes)/.*$ + - ^test/probe/.*$ + - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$ - swift-tox-func-domain-remap-staticweb-py37: irrelevant-files: - ^(api-ref|doc|releasenotes)/.*$ - ^test/probe/.*$ - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$ + - swift-tox-func-ec-py37: + irrelevant-files: + - ^(api-ref|doc|releasenotes)/.*$ + - ^test/probe/.*$ + - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$ + + # Other tests - swift-tox-func-s3api-ceph-s3tests-tempauth: irrelevant-files: - ^(api-ref|releasenotes)/.*$ @@ -496,12 +521,13 @@ - swift-tox-py27 - swift-tox-py37 - swift-tox-func - - swift-tox-func-py37 - swift-tox-func-encryption - swift-tox-func-domain-remap-staticweb - swift-tox-func-ec - swift-tox-func-s3api + - swift-tox-func-py37 - swift-tox-func-domain-remap-staticweb-py37 + - swift-tox-func-ec-py37 - swift-probetests-centos-7: irrelevant-files: - ^(api-ref|releasenotes)/.*$ diff --git a/test/functional/test_object.py b/test/functional/test_object.py index 03854481c1..befa21308f 100644 --- a/test/functional/test_object.py +++ b/test/functional/test_object.py @@ -22,6 +22,7 @@ from uuid import uuid4 import time from xml.dom import minidom +import six from six.moves import range from test.functional import check_response, retry, requires_acls, \ @@ -105,8 +106,11 @@ class TestObject(unittest2.TestCase): # delete an object def delete(url, token, parsed, conn, container, obj): - path = '/'.join([parsed.path, container, - obj['name'].encode('utf8')]) + if six.PY2: + obj_name = obj['name'].encode('utf8') + else: + obj_name = obj['name'] + path = '/'.join([parsed.path, container, obj_name]) conn.request('DELETE', path, '', {'X-Auth-Token': token}) return check_response(conn) @@ -176,7 +180,7 @@ class TestObject(unittest2.TestCase): resp.read() self.assertEqual(resp.status, 201) resp = retry(get) - self.assertEqual('', resp.read()) + self.assertEqual(b'', resp.read()) self.assertEqual(resp.status, 200) self.assertEqual(metadata(resp), {}) # empty post @@ -184,7 +188,7 @@ class TestObject(unittest2.TestCase): resp.read() self.assertEqual(resp.status, 202) resp = retry(get) - self.assertEqual('', resp.read()) + self.assertEqual(b'', resp.read()) self.assertEqual(resp.status, 200) self.assertEqual(metadata(resp), {}) @@ -197,7 +201,7 @@ class TestObject(unittest2.TestCase): resp.read() self.assertEqual(resp.status, 201) resp = retry(get) - self.assertEqual('', resp.read()) + self.assertEqual(b'', resp.read()) self.assertEqual(resp.status, 200) self.assertEqual(metadata(resp), { 'X-Object-Meta-Color': 'blUe', @@ -209,7 +213,7 @@ class TestObject(unittest2.TestCase): resp.read() self.assertEqual(resp.status, 202) resp = retry(get) - self.assertEqual('', resp.read()) + self.assertEqual(b'', resp.read()) self.assertEqual(resp.status, 200) self.assertEqual(metadata(resp), { 'X-Object-Meta-Color': 'oraNge' @@ -225,7 +229,7 @@ class TestObject(unittest2.TestCase): resp.read() self.assertEqual(resp.status, 201) resp = retry(get) - self.assertEqual('', resp.read()) + self.assertEqual(b'', resp.read()) self.assertEqual(resp.status, 200) self.assertEqual(metadata(resp), { 'X-Object-Meta-Color': 'Red', @@ -241,7 +245,7 @@ class TestObject(unittest2.TestCase): resp.read() self.assertEqual(resp.status, 202) resp = retry(get) - self.assertEqual('', resp.read()) + self.assertEqual(b'', resp.read()) self.assertEqual(resp.status, 200) self.assertEqual(metadata(resp), { 'X-Object-Meta-Food': 'Burger', @@ -256,7 +260,7 @@ class TestObject(unittest2.TestCase): resp.read() self.assertEqual(resp.status, 201) resp = retry(get) - self.assertEqual('', resp.read()) + self.assertEqual(b'', resp.read()) self.assertEqual(resp.status, 200) self.assertEqual(metadata(resp), { 'X-Object-Meta-Foo': 'B\xc3\xa2r', @@ -269,7 +273,7 @@ class TestObject(unittest2.TestCase): resp.read() self.assertEqual(resp.status, 202) resp = retry(get) - self.assertEqual('', resp.read()) + self.assertEqual(b'', resp.read()) self.assertEqual(resp.status, 200) self.assertEqual(metadata(resp), { 'X-Object-Meta-Foo': 'B\xc3\xa5z', @@ -341,7 +345,7 @@ class TestObject(unittest2.TestCase): 'X-Timestamp should be a UNIX timestamp float value', body) else: self.assertEqual(resp.status, 201) - self.assertEqual(body, '') + self.assertEqual(body, b'') resp = retry(head) resp.read() self.assertGreater(float(resp.headers['x-timestamp']), ts_before) @@ -374,7 +378,7 @@ class TestObject(unittest2.TestCase): 'X-Timestamp should be a UNIX timestamp float value', body) else: self.assertEqual(resp.status, 201) - self.assertEqual(body, '') + self.assertEqual(body, b'') resp = retry(head) resp.read() self.assertGreater(float(resp.headers['x-timestamp']), ts_before) @@ -470,7 +474,7 @@ class TestObject(unittest2.TestCase): resp = retry(put) body = resp.read() self.assertEqual(resp.status, 400) - self.assertEqual(body, 'Non-integer X-Delete-After') + self.assertEqual(body, b'Non-integer X-Delete-After') def test_non_integer_x_delete_at(self): def put(url, token, parsed, conn): @@ -483,7 +487,7 @@ class TestObject(unittest2.TestCase): resp = retry(put) body = resp.read() self.assertEqual(resp.status, 400) - self.assertEqual(body, 'Non-integer X-Delete-At') + self.assertEqual(body, b'Non-integer X-Delete-At') def test_x_delete_at_in_the_past(self): def put(url, token, parsed, conn): @@ -496,7 +500,7 @@ class TestObject(unittest2.TestCase): resp = retry(put) body = resp.read() self.assertEqual(resp.status, 400) - self.assertEqual(body, 'X-Delete-At in past') + self.assertEqual(body, b'X-Delete-At in past') def test_copy_object(self): if tf.skip: @@ -514,7 +518,7 @@ class TestObject(unittest2.TestCase): resp = retry(get_source) source_contents = resp.read() self.assertEqual(resp.status, 200) - self.assertEqual(source_contents, 'test') + self.assertEqual(source_contents, b'test') # copy source to dest with X-Copy-From def put(url, token, parsed, conn): @@ -605,7 +609,7 @@ class TestObject(unittest2.TestCase): resp = retry(get_source) source_contents = resp.read() self.assertEqual(resp.status, 200) - self.assertEqual(source_contents, 'test') + self.assertEqual(source_contents, b'test') acct = tf.parsed[0].path.split('/', 2)[2] @@ -964,14 +968,16 @@ class TestObject(unittest2.TestCase): # can list objects resp = retry(get_listing, use_account=3) listing = resp.read() + if not six.PY2: + listing = listing.decode('utf8') self.assertEqual(resp.status, 200) - self.assertIn(self.obj, listing) + self.assertIn(self.obj, listing.split('\n')) # can get object resp = retry(get, self.obj, use_account=3) body = resp.read() self.assertEqual(resp.status, 200) - self.assertEqual(body, 'test') + self.assertEqual(body, b'test') # can not put an object obj_name = str(uuid4()) @@ -987,9 +993,11 @@ class TestObject(unittest2.TestCase): # sanity with account1 resp = retry(get_listing, use_account=3) listing = resp.read() + if not six.PY2: + listing = listing.decode('utf8') self.assertEqual(resp.status, 200) - self.assertNotIn(obj_name, listing) - self.assertIn(self.obj, listing) + self.assertNotIn(obj_name, listing.split('\n')) + self.assertIn(self.obj, listing.split('\n')) @requires_acls def test_read_write(self): @@ -1045,14 +1053,16 @@ class TestObject(unittest2.TestCase): # can list objects resp = retry(get_listing, use_account=3) listing = resp.read() + if not six.PY2: + listing = listing.decode('utf8') self.assertEqual(resp.status, 200) - self.assertIn(self.obj, listing) + self.assertIn(self.obj, listing.split('\n')) # can get object resp = retry(get, self.obj, use_account=3) body = resp.read() self.assertEqual(resp.status, 200) - self.assertEqual(body, 'test') + self.assertEqual(body, b'test') # can put an object obj_name = str(uuid4()) @@ -1068,9 +1078,11 @@ class TestObject(unittest2.TestCase): # sanity with account1 resp = retry(get_listing, use_account=3) listing = resp.read() + if not six.PY2: + listing = listing.decode('utf8') self.assertEqual(resp.status, 200) - self.assertIn(obj_name, listing) - self.assertNotIn(self.obj, listing) + self.assertIn(obj_name, listing.split('\n')) + self.assertNotIn(self.obj, listing.split('\n')) @requires_acls def test_admin(self): @@ -1126,14 +1138,16 @@ class TestObject(unittest2.TestCase): # can list objects resp = retry(get_listing, use_account=3) listing = resp.read() + if not six.PY2: + listing = listing.decode('utf8') self.assertEqual(resp.status, 200) - self.assertIn(self.obj, listing) + self.assertIn(self.obj, listing.split('\n')) # can get object resp = retry(get, self.obj, use_account=3) body = resp.read() self.assertEqual(resp.status, 200) - self.assertEqual(body, 'test') + self.assertEqual(body, b'test') # can put an object obj_name = str(uuid4()) @@ -1149,17 +1163,19 @@ class TestObject(unittest2.TestCase): # sanity with account1 resp = retry(get_listing, use_account=3) listing = resp.read() + if not six.PY2: + listing = listing.decode('utf8') self.assertEqual(resp.status, 200) - self.assertIn(obj_name, listing) + self.assertIn(obj_name, listing.split('\n')) self.assertNotIn(self.obj, listing) def test_manifest(self): if tf.skip: raise SkipTest # Data for the object segments - segments1 = ['one', 'two', 'three', 'four', 'five'] - segments2 = ['six', 'seven', 'eight'] - segments3 = ['nine', 'ten', 'eleven'] + segments1 = [b'one', b'two', b'three', b'four', b'five'] + segments2 = [b'six', b'seven', b'eight'] + segments3 = [b'nine', b'ten', b'eleven'] # Upload the first set of segments def put(url, token, parsed, conn, objnum): @@ -1190,7 +1206,7 @@ class TestObject(unittest2.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) - self.assertEqual(resp.read(), ''.join(segments1)) + self.assertEqual(resp.read(), b''.join(segments1)) self.assertEqual(resp.status, 200) self.assertEqual(resp.getheader('content-type'), 'text/jibberish') @@ -1201,7 +1217,7 @@ class TestObject(unittest2.TestCase): 'X-Auth-Token': token, 'Range': 'bytes=3-'}) return check_response(conn) resp = retry(get) - self.assertEqual(resp.read(), ''.join(segments1[1:])) + self.assertEqual(resp.read(), b''.join(segments1[1:])) self.assertEqual(resp.status, 206) # Get with a range in the middle of the second segment @@ -1211,7 +1227,7 @@ class TestObject(unittest2.TestCase): 'X-Auth-Token': token, 'Range': 'bytes=5-'}) return check_response(conn) resp = retry(get) - self.assertEqual(resp.read(), ''.join(segments1)[5:]) + self.assertEqual(resp.read(), b''.join(segments1)[5:]) self.assertEqual(resp.status, 206) # Get with a full start and stop range @@ -1221,7 +1237,7 @@ class TestObject(unittest2.TestCase): 'X-Auth-Token': token, 'Range': 'bytes=5-10'}) return check_response(conn) resp = retry(get) - self.assertEqual(resp.read(), ''.join(segments1)[5:11]) + self.assertEqual(resp.read(), b''.join(segments1)[5:11]) self.assertEqual(resp.status, 206) # Upload the second set of segments @@ -1241,7 +1257,7 @@ class TestObject(unittest2.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) - self.assertEqual(resp.read(), ''.join(segments1)) + self.assertEqual(resp.read(), b''.join(segments1)) self.assertEqual(resp.status, 200) # Update the manifest @@ -1262,7 +1278,7 @@ class TestObject(unittest2.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) - self.assertEqual(resp.read(), ''.join(segments2)) + self.assertEqual(resp.read(), b''.join(segments2)) self.assertEqual(resp.status, 200) if not tf.skip3: @@ -1292,7 +1308,7 @@ class TestObject(unittest2.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get, use_account=3) - self.assertEqual(resp.read(), ''.join(segments2)) + self.assertEqual(resp.read(), b''.join(segments2)) self.assertEqual(resp.status, 200) # Create another container for the third set of segments @@ -1335,7 +1351,7 @@ class TestObject(unittest2.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) - self.assertEqual(resp.read(), ''.join(segments3)) + self.assertEqual(resp.read(), b''.join(segments3)) self.assertEqual(resp.status, 200) if not tf.skip3: @@ -1368,7 +1384,7 @@ class TestObject(unittest2.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get, use_account=3) - self.assertEqual(resp.read(), ''.join(segments3)) + self.assertEqual(resp.read(), b''.join(segments3)) self.assertEqual(resp.status, 200) # Delete the manifest @@ -1480,7 +1496,7 @@ class TestObject(unittest2.TestCase): if (tf.web_front_end == 'apache2'): self.assertEqual(resp.status, 404) else: - self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL') + self.assertEqual(resp.read(), b'Invalid UTF8 or contains NULL') self.assertEqual(resp.status, 412) def test_cors(self): @@ -1645,6 +1661,8 @@ class TestObject(unittest2.TestCase): for c, o, body in validate_requests: resp = retry(get_obj, c, o) self.assertEqual(resp.status, 200) + if not six.PY2: + body = body.encode('utf8') self.assertEqual(body, resp.read()) @requires_bulk diff --git a/tox.ini b/tox.ini index e4e8efa2fb..2e5f607259 100644 --- a/tox.ini +++ b/tox.ini @@ -49,17 +49,24 @@ basepython = python3 commands = nosetests {posargs: \ test/functional/test_domain_remap.py \ + test/functional/test_object.py \ test/functional/test_staticweb.py \ test/functional/test_symlink.py \ test/functional/test_tempurl.py \ test/functional/tests.py} +[testenv:func-ec-py3] +basepython = python3 +commands = {[testenv:func-py3]commands} +setenv = SWIFT_TEST_IN_PROCESS=1 + SWIFT_TEST_IN_PROCESS_CONF_LOADER=ec [testenv:func-domain-remap-staticweb-py3] basepython = python3 commands = {[testenv:func-py3]commands} setenv = SWIFT_TEST_IN_PROCESS=1 SWIFT_TEST_IN_PROCESS_CONF_LOADER=domain_remap_staticweb + [testenv:func-encryption] commands = ./.functests {posargs} setenv = SWIFT_TEST_IN_PROCESS=1