From 99947150dd923ce19112a8f8c35c41f5a1271d72 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 19 Nov 2019 21:25:45 -0800 Subject: [PATCH] func tests: work with etag-quoter on by default Also, run the in-process encryption func tests like that. Change-Id: I984ab8d1304d23b89589973950b10dda4aea0db3 --- test/functional/__init__.py | 4 + test/functional/test_object.py | 9 ++- test/functional/test_object_versioning.py | 19 +++-- test/functional/test_slo.py | 31 +++++--- test/functional/test_symlink.py | 79 ++++++++++++------- test/functional/test_versioned_writes.py | 11 ++- test/functional/tests.py | 75 +++++++++++------- test/probe/test_object_async_update.py | 7 +- .../probe/test_object_metadata_replication.py | 3 +- 9 files changed, 152 insertions(+), 86 deletions(-) diff --git a/test/functional/__init__.py b/test/functional/__init__.py index 4edb721d3e..fa824c0596 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -322,12 +322,16 @@ def _load_encryption(proxy_conf_file, swift_conf_file, **kwargs): pipeline = pipeline.replace( "proxy-logging proxy-server", "keymaster encryption proxy-logging proxy-server") + pipeline = pipeline.replace( + "cache listing_formats", + "cache etag-quoter listing_formats") conf.set(section, 'pipeline', pipeline) root_secret = base64.b64encode(os.urandom(32)) if not six.PY2: root_secret = root_secret.decode('ascii') conf.set('filter:keymaster', 'encryption_root_secret', root_secret) conf.set('filter:versioned_writes', 'allow_object_versioning', 'true') + conf.set('filter:etag-quoter', 'enable_by_default', 'true') except NoSectionError as err: msg = 'Error problem with proxy conf file %s: %s' % \ (proxy_conf_file, err) diff --git a/test/functional/test_object.py b/test/functional/test_object.py index dbc72acef0..6145d4a98e 100644 --- a/test/functional/test_object.py +++ b/test/functional/test_object.py @@ -1726,7 +1726,7 @@ class TestObject(unittest.TestCase): if 'etag_quoter' not in tf.cluster_info: raise SkipTest("etag-quoter middleware is not enabled") - def do_head(expect_quoted=False): + def do_head(expect_quoted=None): def head(url, token, parsed, conn): conn.request('HEAD', '%s/%s/%s' % ( parsed.path, self.container, self.obj), '', @@ -1736,6 +1736,11 @@ class TestObject(unittest.TestCase): resp = retry(head) resp.read() self.assertEqual(resp.status, 200) + + if expect_quoted is None: + expect_quoted = tf.cluster_info.get('etag_quoter', {}).get( + 'enable_by_default', False) + expected_etag = hashlib.md5(b'test').hexdigest() if expect_quoted: expected_etag = '"%s"' % expected_etag @@ -1771,7 +1776,7 @@ class TestObject(unittest.TestCase): post_container('') do_head(expect_quoted=True) post_container('f') - do_head() + do_head(expect_quoted=False) finally: # Don't leave a dirty account post_account('') diff --git a/test/functional/test_object_versioning.py b/test/functional/test_object_versioning.py index eccb1e2d06..d7db187c08 100644 --- a/test/functional/test_object_versioning.py +++ b/test/functional/test_object_versioning.py @@ -26,6 +26,7 @@ from six.moves.urllib.parse import quote, unquote import test.functional as tf +from swift.common.swob import normalize_etag from swift.common.utils import MD5_OF_EMPTY_STRING, config_true_value from swift.common.middleware.versioned_writes.object_versioning import \ DELETE_MARKER_CONTENT_TYPE @@ -337,7 +338,7 @@ class TestObjectVersioning(TestObjectVersioningBase): obj = self.env.unversioned_container.file(oname) resp = obj.write(body, return_resp=True) etag = resp.getheader('etag') - self.assertEqual(md5(body).hexdigest(), etag) + self.assertEqual(md5(body).hexdigest(), normalize_etag(etag)) # un-versioned object is cool with with if-match self.assertEqual(body, obj.read(hdrs={'if-match': etag})) @@ -350,7 +351,11 @@ class TestObjectVersioning(TestObjectVersioningBase): self.assertEqual(resp.getheader('etag'), etag) # versioned object is too with with if-match - self.assertEqual(body, v_obj.read(hdrs={'if-match': etag})) + self.assertEqual(body, v_obj.read(hdrs={ + 'if-match': normalize_etag(etag)})) + # works quoted, too + self.assertEqual(body, v_obj.read(hdrs={ + 'if-match': '"%s"' % normalize_etag(etag)})) with self.assertRaises(ResponseError) as cm: v_obj.read(hdrs={'if-match': 'not-the-etag'}) self.assertEqual(412, cm.exception.status) @@ -989,13 +994,13 @@ class TestObjectVersioning(TestObjectVersioningBase): 'Content-Type': 'text/jibberish32' }, return_resp=True) v1_version_id = resp.getheader('x-object-version-id') - v1_etag = resp.getheader('etag') + v1_etag = normalize_etag(resp.getheader('etag')) resp = obj.write(b'version2', hdrs={ 'Content-Type': 'text/jibberish33' }, return_resp=True) v2_version_id = resp.getheader('x-object-version-id') - v2_etag = resp.getheader('etag') + v2_etag = normalize_etag(resp.getheader('etag')) # sanity self.assertEqual(b'version2', obj.read()) @@ -1062,7 +1067,7 @@ class TestObjectVersioning(TestObjectVersioningBase): self.assertEqual(b'version1', obj.read()) obj_info = obj.info() self.assertEqual('text/jibberish32', obj_info['content_type']) - self.assertEqual(v1_etag, obj_info['etag']) + self.assertEqual(v1_etag, normalize_etag(obj_info['etag'])) def test_delete_with_version_api_old_object(self): versioned_obj_name = Utils.create_name() @@ -2378,7 +2383,7 @@ class TestSloWithVersioning(TestObjectVersioningBase): expected = { 'bytes': file_info['content_length'], 'content_type': 'application/octet-stream', - 'hash': manifest_info['etag'], + 'hash': normalize_etag(manifest_info['etag']), 'name': 'my-slo-manifest', 'slo_etag': file_info['etag'], 'version_symlink': True, @@ -2410,7 +2415,7 @@ class TestSloWithVersioning(TestObjectVersioningBase): expected = { 'bytes': file_info['content_length'], 'content_type': 'application/octet-stream', - 'hash': manifest_info['etag'], + 'hash': normalize_etag(manifest_info['etag']), 'name': 'my-slo-manifest', 'slo_etag': file_info['etag'], 'version_symlink': True, diff --git a/test/functional/test_slo.py b/test/functional/test_slo.py index c055f7bbd3..8003a2d70a 100644 --- a/test/functional/test_slo.py +++ b/test/functional/test_slo.py @@ -23,6 +23,8 @@ from copy import deepcopy import six +from swift.common.swob import normalize_etag + import test.functional as tf from test.functional import cluster_info, SkipTest from test.functional.tests import Utils, Base, Base2, BaseEnv @@ -299,8 +301,14 @@ class TestSlo(Base): # a POST. file_item.initialize(parms={'multipart-manifest': 'get'}) manifest_etag = file_item.etag - self.assertFalse(manifest_etag.startswith('"')) - self.assertFalse(manifest_etag.endswith('"')) + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + self.assertTrue(manifest_etag.startswith('"')) + self.assertTrue(manifest_etag.endswith('"')) + # ...but in the listing, it'll be stripped + manifest_etag = manifest_etag[1:-1] + else: + self.assertFalse(manifest_etag.startswith('"')) + self.assertFalse(manifest_etag.endswith('"')) file_item.initialize() slo_etag = file_item.etag @@ -715,6 +723,8 @@ class TestSlo(Base): source_contents = source.read(parms={'multipart-manifest': 'get'}) source_json = json.loads(source_contents) manifest_etag = hashlib.md5(source_contents).hexdigest() + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + manifest_etag = '"%s"' % manifest_etag self.assertEqual(manifest_etag, source.etag) source.initialize() @@ -752,14 +762,14 @@ class TestSlo(Base): actual = names['manifest-abcde'] self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes']) self.assertEqual('application/octet-stream', actual['content_type']) - self.assertEqual(manifest_etag, actual['hash']) + self.assertEqual(normalize_etag(manifest_etag), actual['hash']) self.assertEqual(slo_etag, actual['slo_etag']) self.assertIn('copied-abcde-manifest-only', names) actual = names['copied-abcde-manifest-only'] self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes']) self.assertEqual('application/octet-stream', actual['content_type']) - self.assertEqual(manifest_etag, actual['hash']) + self.assertEqual(normalize_etag(manifest_etag), actual['hash']) self.assertEqual(slo_etag, actual['slo_etag']) # Test copy manifest including data segments @@ -789,6 +799,8 @@ class TestSlo(Base): source_contents = source.read(parms={'multipart-manifest': 'get'}) source_json = json.loads(source_contents) manifest_etag = hashlib.md5(source_contents).hexdigest() + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + manifest_etag = '"%s"' % manifest_etag self.assertEqual(manifest_etag, source.etag) source.initialize() @@ -831,14 +843,14 @@ class TestSlo(Base): self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes']) self.assertEqual('application/octet-stream', actual['content_type']) # the container listing should have the etag of the manifest contents - self.assertEqual(manifest_etag, actual['hash']) + self.assertEqual(normalize_etag(manifest_etag), actual['hash']) self.assertEqual(slo_etag, actual['slo_etag']) self.assertIn('copied-abcde-manifest-only', names) actual = names['copied-abcde-manifest-only'] self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes']) self.assertEqual('image/jpeg', actual['content_type']) - self.assertEqual(manifest_etag, actual['hash']) + self.assertEqual(normalize_etag(manifest_etag), actual['hash']) self.assertEqual(slo_etag, actual['slo_etag']) def test_slo_copy_the_manifest_account(self): @@ -1098,12 +1110,7 @@ class TestSlo(Base): manifest = self.env.container.file("manifest-db") got_body = manifest.read(parms={'multipart-manifest': 'get', 'format': 'raw'}) - body_md5 = hashlib.md5(got_body).hexdigest() - headers = dict( - (h.lower(), v) - for h, v in manifest.conn.response.getheaders()) - self.assertIn('etag', headers) - self.assertEqual(headers['etag'], body_md5) + self.assert_etag(hashlib.md5(got_body).hexdigest()) # raw format should have the actual manifest object content-type self.assertEqual('application/octet-stream', manifest.content_type) diff --git a/test/functional/test_symlink.py b/test/functional/test_symlink.py index 5cd66d510c..1b6ec820f9 100755 --- a/test/functional/test_symlink.py +++ b/test/functional/test_symlink.py @@ -25,6 +25,7 @@ from six.moves import urllib from uuid import uuid4 from swift.common.http import is_success +from swift.common.swob import normalize_etag from swift.common.utils import json, MD5_OF_EMPTY_STRING from swift.common.middleware.slo import SloGetContext from test.functional import check_response, retry, requires_acls, \ @@ -1135,7 +1136,7 @@ class TestSymlink(Base): etag=self.env.tgt_etag) # overwrite tgt object - old_tgt_etag = self.env.tgt_etag + old_tgt_etag = normalize_etag(self.env.tgt_etag) self.env._create_tgt_object(body='updated target body') # sanity @@ -1380,7 +1381,7 @@ class TestSymlink(Base): object_list[0]['symlink_path']) obj_info = object_list[0] self.assertIn('symlink_etag', obj_info) - self.assertEqual(self.env.tgt_etag, + self.assertEqual(normalize_etag(self.env.tgt_etag), obj_info['symlink_etag']) self.assertEqual(int(self.env.tgt_length), obj_info['symlink_bytes']) @@ -1550,7 +1551,7 @@ class TestSymlinkSlo(Base): 'symlink_path': '/v1/%s/%s/manifest-abcde' % ( self.account_name, self.env.container2.name), 'symlink_bytes': 4 * 2 ** 20 + 1, - 'symlink_etag': manifest_etag, + 'symlink_etag': normalize_etag(manifest_etag), }) def test_static_link_target_slo_manifest_wrong_etag(self): @@ -1740,7 +1741,11 @@ class TestSymlinkToSloSegments(Base): self.assertEqual(1024 * 1024, f_dict['bytes']) self.assertEqual('application/octet-stream', f_dict['content_type']) - self.assertEqual(manifest_etag, f_dict['hash']) + if tf.cluster_info.get('etag_quoter', {}).get( + 'enable_by_default'): + self.assertEqual(manifest_etag, '"%s"' % f_dict['hash']) + else: + self.assertEqual(manifest_etag, f_dict['hash']) self.assertEqual(slo_etag, f_dict['slo_etag']) break else: @@ -1759,7 +1764,11 @@ class TestSymlinkToSloSegments(Base): self.assertEqual(1024 * 1024, f_dict['bytes']) self.assertEqual(file_item.content_type, f_dict['content_type']) - self.assertEqual(manifest_etag, f_dict['hash']) + if tf.cluster_info.get('etag_quoter', {}).get( + 'enable_by_default'): + self.assertEqual(manifest_etag, '"%s"' % f_dict['hash']) + else: + self.assertEqual(manifest_etag, f_dict['hash']) self.assertEqual(slo_etag, f_dict['slo_etag']) break else: @@ -1778,7 +1787,11 @@ class TestSymlinkToSloSegments(Base): self.assertEqual(1024 * 1024, f_dict['bytes']) self.assertEqual(file_item.content_type, f_dict['content_type']) - self.assertEqual(manifest_etag, f_dict['hash']) + if tf.cluster_info.get('etag_quoter', {}).get( + 'enable_by_default'): + self.assertEqual(manifest_etag, '"%s"' % f_dict['hash']) + else: + self.assertEqual(manifest_etag, f_dict['hash']) self.assertEqual(slo_etag, f_dict['slo_etag']) break else: @@ -1811,6 +1824,8 @@ class TestSymlinkToSloSegments(Base): source_contents = source.read(parms={'multipart-manifest': 'get'}) source_json = json.loads(source_contents) manifest_etag = hashlib.md5(source_contents).hexdigest() + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + manifest_etag = '"%s"' % manifest_etag source.initialize() slo_etag = source.etag @@ -1857,14 +1872,20 @@ class TestSymlinkToSloSegments(Base): actual = names['manifest-linkto-ab'] self.assertEqual(2 * 1024 * 1024, actual['bytes']) self.assertEqual('application/octet-stream', actual['content_type']) - self.assertEqual(manifest_etag, actual['hash']) + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + self.assertEqual(manifest_etag, '"%s"' % actual['hash']) + else: + self.assertEqual(manifest_etag, actual['hash']) self.assertEqual(slo_etag, actual['slo_etag']) self.assertIn('copied-ab-manifest-only', names) actual = names['copied-ab-manifest-only'] self.assertEqual(2 * 1024 * 1024, actual['bytes']) self.assertEqual('application/octet-stream', actual['content_type']) - self.assertEqual(manifest_etag, actual['hash']) + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + self.assertEqual(manifest_etag, '"%s"' % actual['hash']) + else: + self.assertEqual(manifest_etag, actual['hash']) self.assertEqual(slo_etag, actual['slo_etag']) @@ -2000,13 +2021,13 @@ class TestSymlinkTargetObjectComparison(Base): else: self.assertEqual(b'', body) self.assert_status(200) - self.assert_header('etag', md5) + self.assert_etag(md5) hdrs = {'If-Match': 'bogus'} self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, parms=self.env.parms) self.assert_status(412) - self.assert_header('etag', md5) + self.assert_etag(md5) def testIfMatchMultipleEtags(self): for file_item in self.env.files: @@ -2022,13 +2043,13 @@ class TestSymlinkTargetObjectComparison(Base): else: self.assertEqual(b'', body) self.assert_status(200) - self.assert_header('etag', md5) + self.assert_etag(md5) hdrs = {'If-Match': '"bogus1", "bogus2", "bogus3"'} self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, parms=self.env.parms) self.assert_status(412) - self.assert_header('etag', md5) + self.assert_etag(md5) def testIfNoneMatch(self): for file_item in self.env.files: @@ -2044,13 +2065,13 @@ class TestSymlinkTargetObjectComparison(Base): else: self.assertEqual(b'', body) self.assert_status(200) - self.assert_header('etag', md5) + self.assert_etag(md5) hdrs = {'If-None-Match': md5} self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, parms=self.env.parms) self.assert_status(304) - self.assert_header('etag', md5) + self.assert_etag(md5) self.assert_header('accept-ranges', 'bytes') def testIfNoneMatchMultipleEtags(self): @@ -2067,14 +2088,14 @@ class TestSymlinkTargetObjectComparison(Base): else: self.assertEqual(b'', body) self.assert_status(200) - self.assert_header('etag', md5) + self.assert_etag(md5) hdrs = {'If-None-Match': '"bogus1", "bogus2", "%s"' % md5} self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, parms=self.env.parms) self.assert_status(304) - self.assert_header('etag', md5) + self.assert_etag(md5) self.assert_header('accept-ranges', 'bytes') def testIfModifiedSince(self): @@ -2091,19 +2112,19 @@ class TestSymlinkTargetObjectComparison(Base): else: self.assertEqual(b'', body) self.assert_status(200) - self.assert_header('etag', md5) + self.assert_etag(md5) self.assertTrue(file_symlink.info(hdrs=hdrs, parms=self.env.parms)) hdrs = {'If-Modified-Since': self.env.time_new} self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, parms=self.env.parms) self.assert_status(304) - self.assert_header('etag', md5) + self.assert_etag(md5) self.assert_header('accept-ranges', 'bytes') self.assertRaises(ResponseError, file_symlink.info, hdrs=hdrs, parms=self.env.parms) self.assert_status(304) - self.assert_header('etag', md5) + self.assert_etag(md5) self.assert_header('accept-ranges', 'bytes') def testIfUnmodifiedSince(self): @@ -2120,18 +2141,18 @@ class TestSymlinkTargetObjectComparison(Base): else: self.assertEqual(b'', body) self.assert_status(200) - self.assert_header('etag', md5) + self.assert_etag(md5) self.assertTrue(file_symlink.info(hdrs=hdrs, parms=self.env.parms)) hdrs = {'If-Unmodified-Since': self.env.time_old_f2} self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, parms=self.env.parms) self.assert_status(412) - self.assert_header('etag', md5) + self.assert_etag(md5) self.assertRaises(ResponseError, file_symlink.info, hdrs=hdrs, parms=self.env.parms) self.assert_status(412) - self.assert_header('etag', md5) + self.assert_etag(md5) def testIfMatchAndUnmodified(self): for file_item in self.env.files: @@ -2148,21 +2169,21 @@ class TestSymlinkTargetObjectComparison(Base): else: self.assertEqual(b'', body) self.assert_status(200) - self.assert_header('etag', md5) + self.assert_etag(md5) hdrs = {'If-Match': 'bogus', 'If-Unmodified-Since': self.env.time_new} self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, parms=self.env.parms) self.assert_status(412) - self.assert_header('etag', md5) + self.assert_etag(md5) hdrs = {'If-Match': md5, 'If-Unmodified-Since': self.env.time_old_f3} self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, parms=self.env.parms) self.assert_status(412) - self.assert_header('etag', md5) + self.assert_etag(md5) def testLastModified(self): file_item = self.env.container.file(Utils.create_name()) @@ -2186,7 +2207,7 @@ class TestSymlinkTargetObjectComparison(Base): hdrs = {'If-Modified-Since': last_modified} self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs) self.assert_status(304) - self.assert_header('etag', md5) + self.assert_etag(md5) self.assert_header('accept-ranges', 'bytes') hdrs = {'If-Unmodified-Since': last_modified} @@ -2227,20 +2248,20 @@ class TestSymlinkComparison(TestSymlinkTargetObjectComparison): body = file_symlink.read(hdrs=hdrs, parms=self.env.parms) self.assertEqual(b'', body) self.assert_status(200) - self.assert_header('etag', md5) + self.assert_etag(md5) hdrs = {'If-Modified-Since': last_modified} self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs, parms=self.env.parms) self.assert_status(304) - self.assert_header('etag', md5) + self.assert_etag(md5) self.assert_header('accept-ranges', 'bytes') hdrs = {'If-Unmodified-Since': last_modified} body = file_symlink.read(hdrs=hdrs, parms=self.env.parms) self.assertEqual(b'', body) self.assert_status(200) - self.assert_header('etag', md5) + self.assert_etag(md5) class TestSymlinkAccountTempurl(Base): diff --git a/test/functional/test_versioned_writes.py b/test/functional/test_versioned_writes.py index d58da88e62..7521825f2c 100644 --- a/test/functional/test_versioned_writes.py +++ b/test/functional/test_versioned_writes.py @@ -684,7 +684,11 @@ class TestObjectVersioning(Base): prev_version = versions_container.file(versioned_obj_name) prev_version_info = prev_version.info(parms={'symlink': 'get'}) self.assertEqual(b"aaaaa", prev_version.read()) - self.assertEqual(MD5_OF_EMPTY_STRING, prev_version_info['etag']) + symlink_etag = prev_version_info['etag'] + if symlink_etag.startswith('"') and symlink_etag.endswith('"') and \ + symlink_etag[1:-1]: + symlink_etag = symlink_etag[1:-1] + self.assertEqual(MD5_OF_EMPTY_STRING, symlink_etag) self.assertEqual(sym_tgt_header, prev_version_info['x_symlink_target']) return symlink, tgt_a @@ -698,7 +702,10 @@ class TestObjectVersioning(Base): symlink.delete() sym_info = symlink.info(parms={'symlink': 'get'}) self.assertEqual(b"aaaaa", symlink.read()) - self.assertEqual(MD5_OF_EMPTY_STRING, sym_info['etag']) + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + self.assertEqual('"%s"' % MD5_OF_EMPTY_STRING, sym_info['etag']) + else: + self.assertEqual(MD5_OF_EMPTY_STRING, sym_info['etag']) self.assertEqual( quote(unquote('%s/%s' % (self.env.container.name, target.name))), sym_info['x_symlink_target']) diff --git a/test/functional/tests.py b/test/functional/tests.py index dc149cffa1..51b4c663f8 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -27,6 +27,7 @@ import uuid from copy import deepcopy import eventlet from swift.common.http import is_success, is_client_error +from swift.common.swob import normalize_etag from email.utils import parsedate if six.PY2: @@ -131,6 +132,13 @@ class Base(unittest.TestCase): 'Expected header name %r not found in response.' % header_name) self.assertEqual(expected_value, actual_value) + def assert_etag(self, unquoted_value): + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + expected = '"%s"' % unquoted_value + else: + expected = unquoted_value + self.assert_header('etag', expected) + class Base2(object): @classmethod @@ -874,7 +882,11 @@ class TestContainer(Base): for actual in file_list: name = actual['name'] self.assertIn(name, expected) - self.assertEqual(expected[name]['etag'], actual['hash']) + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + self.assertEqual(expected[name]['etag'], + '"%s"' % actual['hash']) + else: + self.assertEqual(expected[name]['etag'], actual['hash']) self.assertEqual( expected[name]['content_type'], actual['content_type']) self.assertEqual( @@ -1365,6 +1377,8 @@ class TestFile(Base): 'x-delete-at': mock.ANY, 'x-trans-id': mock.ANY, 'x-openstack-request-id': mock.ANY} + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + expected_headers['etag'] = '"%s"' % expected_headers['etag'] unexpected_headers = ['connection', 'x-delete-after'] do_test(put_headers, {}, expected_headers, unexpected_headers) @@ -1420,7 +1434,7 @@ class TestFile(Base): self.fail('Failed to find %s in listing' % dest_filename) self.assertEqual(file_item.size, obj['bytes']) - self.assertEqual(file_item.etag, obj['hash']) + self.assertEqual(normalize_etag(file_item.etag), obj['hash']) self.assertEqual(file_item.content_type, obj['content_type']) file_copy = cont.file(dest_filename) @@ -1470,7 +1484,7 @@ class TestFile(Base): self.fail('Failed to find %s in listing' % dest_filename) self.assertEqual(file_item.size, obj['bytes']) - self.assertEqual(file_item.etag, obj['hash']) + self.assertEqual(normalize_etag(file_item.etag), obj['hash']) self.assertEqual( 'application/test-changed', obj['content_type']) @@ -1505,7 +1519,7 @@ class TestFile(Base): self.fail('Failed to find %s in listing' % dest_filename) self.assertEqual(file_item.size, obj['bytes']) - self.assertEqual(file_item.etag, obj['hash']) + self.assertEqual(normalize_etag(file_item.etag), obj['hash']) self.assertEqual( 'application/test-updated', obj['content_type']) @@ -2088,7 +2102,7 @@ class TestFile(Base): self.assertEqual(file_item.read(hdrs=hdrs), data[-i:]) self.assert_header('content-range', 'bytes %d-%d/%d' % ( file_length - i, file_length - 1, file_length)) - self.assert_header('etag', file_item.md5) + self.assert_etag(file_item.md5) self.assert_header('accept-ranges', 'bytes') range_string = 'bytes=%d-' % (i) @@ -2102,7 +2116,7 @@ class TestFile(Base): self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(416) self.assert_header('content-range', 'bytes */%d' % file_length) - self.assert_header('etag', file_item.md5) + self.assert_etag(file_item.md5) self.assert_header('accept-ranges', 'bytes') range_string = 'bytes=%d-%d' % (file_length - 1000, file_length + 2000) @@ -2416,14 +2430,16 @@ class TestFile(Base): file_item.content_type = content_type file_item.write_random(self.env.file_size) - md5 = file_item.md5 + expected_etag = file_item.md5 + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + expected_etag = '"%s"' % expected_etag file_item = self.env.container.file(file_name) info = file_item.info() self.assert_status(200) self.assertEqual(info['content_length'], self.env.file_size) - self.assertEqual(info['etag'], md5) + self.assertEqual(info['etag'], expected_etag) self.assertEqual(info['content_type'], content_type) self.assertIn('last_modified', info) @@ -2612,14 +2628,7 @@ class TestFile(Base): file_item = self.env.container.file(Utils.create_name()) data = io.BytesIO(file_item.write_random(512)) - etag = File.compute_md5sum(data) - - headers = dict((h.lower(), v) - for h, v in self.env.conn.response.getheaders()) - self.assertIn('etag', headers.keys()) - - header_etag = headers['etag'].strip('"') - self.assertEqual(etag, header_etag) + self.assert_etag(File.compute_md5sum(data)) def testChunkedPut(self): if (tf.web_front_end == 'apache2'): @@ -2645,7 +2654,7 @@ class TestFile(Base): self.assertEqual(data, file_item.read()) info = file_item.info() - self.assertEqual(etag, info['etag']) + self.assertEqual(normalize_etag(info['etag']), etag) def test_POST(self): # verify consistency between object and container listing metadata @@ -2670,7 +2679,10 @@ class TestFile(Base): self.fail('Failed to find file %r in listing' % file_name) self.assertEqual(1024, f_dict['bytes']) self.assertEqual('text/foobar', f_dict['content_type']) - self.assertEqual(etag, f_dict['hash']) + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + self.assertEqual(etag, '"%s"' % f_dict['hash']) + else: + self.assertEqual(etag, f_dict['hash']) put_last_modified = f_dict['last_modified'] # now POST updated content-type to each file @@ -2697,7 +2709,10 @@ class TestFile(Base): self.assertEqual(1024, f_dict['bytes']) self.assertEqual('image/foobarbaz', f_dict['content_type']) self.assertLess(put_last_modified, f_dict['last_modified']) - self.assertEqual(etag, f_dict['hash']) + if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'): + self.assertEqual(etag, '"%s"' % f_dict['hash']) + else: + self.assertEqual(etag, f_dict['hash']) class TestFileUTF8(Base2, TestFile): @@ -2742,7 +2757,7 @@ class TestFileComparison(Base): hdrs = {'If-Match': 'bogus'} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(412) - self.assert_header('etag', file_item.md5) + self.assert_etag(file_item.md5) def testIfMatchMultipleEtags(self): for file_item in self.env.files: @@ -2752,7 +2767,7 @@ class TestFileComparison(Base): hdrs = {'If-Match': '"bogus1", "bogus2", "bogus3"'} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(412) - self.assert_header('etag', file_item.md5) + self.assert_etag(file_item.md5) def testIfNoneMatch(self): for file_item in self.env.files: @@ -2762,7 +2777,7 @@ class TestFileComparison(Base): hdrs = {'If-None-Match': file_item.md5} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(304) - self.assert_header('etag', file_item.md5) + self.assert_etag(file_item.md5) self.assert_header('accept-ranges', 'bytes') def testIfNoneMatchMultipleEtags(self): @@ -2774,7 +2789,7 @@ class TestFileComparison(Base): '"bogus1", "bogus2", "%s"' % file_item.md5} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(304) - self.assert_header('etag', file_item.md5) + self.assert_etag(file_item.md5) self.assert_header('accept-ranges', 'bytes') def testIfModifiedSince(self): @@ -2786,11 +2801,11 @@ class TestFileComparison(Base): hdrs = {'If-Modified-Since': self.env.time_new} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(304) - self.assert_header('etag', file_item.md5) + self.assert_etag(file_item.md5) self.assert_header('accept-ranges', 'bytes') self.assertRaises(ResponseError, file_item.info, hdrs=hdrs) self.assert_status(304) - self.assert_header('etag', file_item.md5) + self.assert_etag(file_item.md5) self.assert_header('accept-ranges', 'bytes') def testIfUnmodifiedSince(self): @@ -2802,10 +2817,10 @@ class TestFileComparison(Base): hdrs = {'If-Unmodified-Since': self.env.time_old_f2} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(412) - self.assert_header('etag', file_item.md5) + self.assert_etag(file_item.md5) self.assertRaises(ResponseError, file_item.info, hdrs=hdrs) self.assert_status(412) - self.assert_header('etag', file_item.md5) + self.assert_etag(file_item.md5) def testIfMatchAndUnmodified(self): for file_item in self.env.files: @@ -2817,13 +2832,13 @@ class TestFileComparison(Base): 'If-Unmodified-Since': self.env.time_new} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(412) - self.assert_header('etag', file_item.md5) + self.assert_etag(file_item.md5) hdrs = {'If-Match': file_item.md5, 'If-Unmodified-Since': self.env.time_old_f3} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(412) - self.assert_header('etag', file_item.md5) + self.assert_etag(file_item.md5) def testLastModified(self): file_name = Utils.create_name() @@ -2844,7 +2859,7 @@ class TestFileComparison(Base): hdrs = {'If-Modified-Since': last_modified} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(304) - self.assert_header('etag', etag) + self.assert_etag(etag) self.assert_header('accept-ranges', 'bytes') hdrs = {'If-Unmodified-Since': last_modified} diff --git a/test/probe/test_object_async_update.py b/test/probe/test_object_async_update.py index 44a25eb148..00aefcccbf 100644 --- a/test/probe/test_object_async_update.py +++ b/test/probe/test_object_async_update.py @@ -23,6 +23,7 @@ from swiftclient.exceptions import ClientException from swift.common import direct_client from swift.common.manager import Manager +from swift.common.swob import normalize_etag from test.probe.common import kill_nonprimary_server, \ kill_server, ReplProbeTest, start_server, ECProbeTest @@ -210,7 +211,7 @@ class TestUpdateOverridesEC(ECProbeTest): self.assertEqual(1, len(listing)) self.assertEqual('o1', listing[0]['name']) self.assertEqual(len(content), listing[0]['bytes']) - self.assertEqual(meta['etag'], listing[0]['hash']) + self.assertEqual(normalize_etag(meta['etag']), listing[0]['hash']) self.assertEqual('test/ctype', listing[0]['content_type']) def test_update_during_POST_only(self): @@ -261,7 +262,7 @@ class TestUpdateOverridesEC(ECProbeTest): self.assertEqual(1, len(listing)) self.assertEqual('o1', listing[0]['name']) self.assertEqual(len(content), listing[0]['bytes']) - self.assertEqual(meta['etag'], listing[0]['hash']) + self.assertEqual(normalize_etag(meta['etag']), listing[0]['hash']) self.assertEqual('test/ctype', listing[0]['content_type']) # Run the object-updaters to send the async pending from the PUT @@ -328,7 +329,7 @@ class TestUpdateOverridesEC(ECProbeTest): self.assertEqual(1, len(listing)) self.assertEqual('o1', listing[0]['name']) self.assertEqual(len(content), listing[0]['bytes']) - self.assertEqual(meta['etag'], listing[0]['hash']) + self.assertEqual(normalize_etag(meta['etag']), listing[0]['hash']) self.assertEqual('test/ctype', listing[0]['content_type']) diff --git a/test/probe/test_object_metadata_replication.py b/test/probe/test_object_metadata_replication.py index c5cb93f98d..4b5b0d4484 100644 --- a/test/probe/test_object_metadata_replication.py +++ b/test/probe/test_object_metadata_replication.py @@ -22,6 +22,7 @@ import uuid from swift.common.direct_client import direct_get_suffix_hashes from swift.common.exceptions import DiskFileDeleted from swift.common.internal_client import UnexpectedResponse +from swift.common.swob import normalize_etag from swift.container.backend import ContainerBroker from swift.common import utils from swiftclient import client @@ -129,7 +130,7 @@ class Test(ReplProbeTest): def _assert_object_metadata_matches_listing(self, listing, metadata): self.assertEqual(listing['bytes'], int(metadata['content-length'])) - self.assertEqual(listing['hash'], metadata['etag']) + self.assertEqual(listing['hash'], normalize_etag(metadata['etag'])) self.assertEqual(listing['content_type'], metadata['content-type']) modified = Timestamp(metadata['x-timestamp']).isoformat self.assertEqual(listing['last_modified'], modified)