diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index 77dc0c1c0b..a1c105b180 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -482,8 +482,11 @@ class Account(Base): fields = [['object_count', 'x-account-object-count'], ['container_count', 'x-account-container-count'], ['bytes_used', 'x-account-bytes-used']] + optional_fields = [ + ['temp-url-key', 'x-account-meta-temp-url-key'], + ['temp-url-key-2', 'x-account-meta-temp-url-key-2']] - return self.header_fields(fields) + return self.header_fields(fields, optional_fields=optional_fields) @property def path(self): diff --git a/test/functional/test_account.py b/test/functional/test_account.py index 57bbe6b815..cc781cc3a1 100644 --- a/test/functional/test_account.py +++ b/test/functional/test_account.py @@ -38,6 +38,54 @@ def tearDownModule(): class TestAccount(unittest2.TestCase): + existing_metadata = None + + @classmethod + def get_meta(cls): + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(head) + resp.read() + return dict((k, v) for k, v in resp.getheaders() if + k.lower().startswith('x-account-meta')) + + @classmethod + def clear_meta(cls, remove_metadata_keys): + def post(url, token, parsed, conn, hdr_keys): + headers = {'X-Auth-Token': token} + headers.update((k, '') for k in hdr_keys) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + + for i in range(0, len(remove_metadata_keys), 90): + batch = remove_metadata_keys[i:i + 90] + resp = retry(post, batch) + resp.read() + + @classmethod + def set_meta(cls, metadata): + def post(url, token, parsed, conn, meta_hdrs): + headers = {'X-Auth-Token': token} + headers.update(meta_hdrs) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + + if not metadata: + return + resp = retry(post, metadata) + resp.read() + + @classmethod + def setUpClass(cls): + # remove and stash any existing account user metadata before tests + cls.existing_metadata = cls.get_meta() + cls.clear_meta(cls.existing_metadata.keys()) + + @classmethod + def tearDownClass(cls): + # replace any stashed account user metadata + cls.set_meta(cls.existing_metadata) def setUp(self): self.max_meta_count = load_constraint('max_meta_count') @@ -45,35 +93,10 @@ class TestAccount(unittest2.TestCase): self.max_meta_overall_size = load_constraint('max_meta_overall_size') self.max_meta_value_length = load_constraint('max_meta_value_length') - def head(url, token, parsed, conn): - conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) - return check_response(conn) - resp = retry(head) - self.existing_metadata = set([ - k for k, v in resp.getheaders() if - k.lower().startswith('x-account-meta')]) - def tearDown(self): - def head(url, token, parsed, conn): - conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) - return check_response(conn) - resp = retry(head) - resp.read() - new_metadata = set( - [k for k, v in resp.getheaders() if - k.lower().startswith('x-account-meta')]) - - def clear_meta(url, token, parsed, conn, remove_metadata_keys): - headers = {'X-Auth-Token': token} - headers.update((k, '') for k in remove_metadata_keys) - conn.request('POST', parsed.path, '', headers) - return check_response(conn) - extra_metadata = list(self.existing_metadata ^ new_metadata) - for i in range(0, len(extra_metadata), 90): - batch = extra_metadata[i:i + 90] - resp = retry(clear_meta, batch) - resp.read() - self.assertEqual(resp.status // 100, 2) + # clean up any account user metadata created by test + new_metadata = self.get_meta().keys() + self.clear_meta(new_metadata) def test_metadata(self): if tf.skip: @@ -794,11 +817,6 @@ class TestAccount(unittest2.TestCase): conn.request('POST', parsed.path, '', headers) return check_response(conn) - # TODO: Find the test that adds these and remove them. - headers = {'x-remove-account-meta-temp-url-key': 'remove', - 'x-remove-account-meta-temp-url-key-2': 'remove'} - resp = retry(post, headers) - headers = {} for x in range(self.max_meta_count): headers['X-Account-Meta-%d' % x] = 'v' diff --git a/test/functional/test_dlo.py b/test/functional/test_dlo.py index a8159eaf14..076e8d7e28 100644 --- a/test/functional/test_dlo.py +++ b/test/functional/test_dlo.py @@ -15,9 +15,8 @@ # limitations under the License. import test.functional as tf -from test.functional.tests import Utils, Base, Base2 -from test.functional.swift_test_client import Account, Connection, \ - ResponseError +from test.functional.tests import Utils, Base, Base2, BaseEnv +from test.functional.swift_test_client import Connection, ResponseError def setUpModule(): @@ -28,22 +27,16 @@ def tearDownModule(): tf.teardown_package() -class TestDloEnv(object): +class TestDloEnv(BaseEnv): @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - + super(TestDloEnv, cls).setUp() config2 = tf.config.copy() config2['username'] = tf.config['username3'] config2['password'] = tf.config['password3'] cls.conn2 = Connection(config2) cls.conn2.authenticate() - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - cls.account.delete_containers() - cls.container = cls.account.container(Utils.create_name()) cls.container2 = cls.account.container(Utils.create_name()) @@ -92,7 +85,6 @@ class TestDloEnv(object): class TestDlo(Base): env = TestDloEnv - set_up = False def test_get_manifest(self): file_item = self.env.container.file('man1') @@ -394,4 +386,4 @@ class TestDlo(Base): class TestDloUTF8(Base2, TestDlo): - set_up = False + pass diff --git a/test/functional/test_slo.py b/test/functional/test_slo.py index 9e73020912..e1c6f3d806 100644 --- a/test/functional/test_slo.py +++ b/test/functional/test_slo.py @@ -23,9 +23,8 @@ from unittest2 import SkipTest import test.functional as tf from test.functional import cluster_info -from test.functional.tests import Utils, Base, Base2 -from test.functional.swift_test_client import Account, Connection, \ - ResponseError +from test.functional.tests import Utils, Base, Base2, BaseEnv +from test.functional.swift_test_client import Connection, ResponseError def setUpModule(): @@ -36,7 +35,7 @@ def tearDownModule(): tf.teardown_package() -class TestSloEnv(object): +class TestSloEnv(BaseEnv): slo_enabled = None # tri-state: None initially, then True/False @classmethod @@ -58,8 +57,13 @@ class TestSloEnv(object): @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() + if cls.slo_enabled is None: + cls.slo_enabled = 'slo' in cluster_info + if not cls.slo_enabled: + return + + super(TestSloEnv, cls).setUp() + config2 = deepcopy(tf.config) config2['account'] = tf.config['account2'] config2['username'] = tf.config['username2'] @@ -74,15 +78,6 @@ class TestSloEnv(object): cls.conn3 = Connection(config3) cls.conn3.authenticate() - if cls.slo_enabled is None: - cls.slo_enabled = 'slo' in cluster_info - if not cls.slo_enabled: - return - - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - cls.account.delete_containers() - cls.container = cls.account.container(Utils.create_name()) cls.container2 = cls.account.container(Utils.create_name()) @@ -214,7 +209,6 @@ class TestSloEnv(object): class TestSlo(Base): env = TestSloEnv - set_up = False def setUp(self): super(TestSlo, self).setUp() @@ -963,4 +957,4 @@ class TestSlo(Base): class TestSloUTF8(Base2, TestSlo): - set_up = False + pass diff --git a/test/functional/test_tempurl.py b/test/functional/test_tempurl.py index 08b74b6c4c..581fb1e8fd 100644 --- a/test/functional/test_tempurl.py +++ b/test/functional/test_tempurl.py @@ -24,7 +24,7 @@ from unittest2 import SkipTest import test.functional as tf from test.functional import cluster_info -from test.functional.tests import Utils, Base, Base2 +from test.functional.tests import Utils, Base, Base2, BaseEnv from test.functional import requires_acls from test.functional.swift_test_client import Account, Connection, \ ResponseError @@ -38,25 +38,38 @@ def tearDownModule(): tf.teardown_package() -class TestTempurlEnv(object): +class TestTempurlBaseEnv(BaseEnv): + original_account_meta = None + + @classmethod + def setUp(cls): + super(TestTempurlBaseEnv, cls).setUp() + cls.original_account_meta = cls.account.info() + + @classmethod + def tearDown(cls): + if cls.original_account_meta: + # restore any tempurl keys that the tests may have overwritten + cls.account.update_metadata( + dict((k, cls.original_account_meta.get(k, '')) + for k in ('temp-url-key', 'temp-url-key-2',))) + + +class TestTempurlEnv(TestTempurlBaseEnv): tempurl_enabled = None # tri-state: None initially, then True/False @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - if cls.tempurl_enabled is None: cls.tempurl_enabled = 'tempurl' in cluster_info if not cls.tempurl_enabled: return + super(TestTempurlEnv, cls).setUp() + cls.tempurl_key = Utils.create_name() cls.tempurl_key2 = Utils.create_name() - cls.account = Account( - cls.conn, tf.config.get('account', tf.config['username'])) - cls.account.delete_containers() cls.account.update_metadata({ 'temp-url-key': cls.tempurl_key, 'temp-url-key-2': cls.tempurl_key2 @@ -74,7 +87,6 @@ class TestTempurlEnv(object): class TestTempurl(Base): env = TestTempurlEnv - set_up = False def setUp(self): super(TestTempurl, self).setUp() @@ -376,29 +388,24 @@ class TestTempURLPrefix(TestTempurl): class TestTempurlUTF8(Base2, TestTempurl): - set_up = False + pass -class TestContainerTempurlEnv(object): +class TestContainerTempurlEnv(BaseEnv): tempurl_enabled = None # tri-state: None initially, then True/False @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - if cls.tempurl_enabled is None: cls.tempurl_enabled = 'tempurl' in cluster_info if not cls.tempurl_enabled: return + super(TestContainerTempurlEnv, cls).setUp() + cls.tempurl_key = Utils.create_name() cls.tempurl_key2 = Utils.create_name() - cls.account = Account( - cls.conn, tf.config.get('account', tf.config['username'])) - cls.account.delete_containers() - # creating another account and connection # for ACL tests config2 = deepcopy(tf.config) @@ -426,7 +433,6 @@ class TestContainerTempurlEnv(object): class TestContainerTempurl(Base): env = TestContainerTempurlEnv - set_up = False def setUp(self): super(TestContainerTempurl, self).setUp() @@ -647,25 +653,20 @@ class TestContainerTempurl(Base): class TestContainerTempurlUTF8(Base2, TestContainerTempurl): - set_up = False + pass -class TestSloTempurlEnv(object): +class TestSloTempurlEnv(TestTempurlBaseEnv): enabled = None # tri-state: None initially, then True/False @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - + super(TestSloTempurlEnv, cls).setUp() if cls.enabled is None: cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info cls.tempurl_key = Utils.create_name() - cls.account = Account( - cls.conn, tf.config.get('account', tf.config['username'])) - cls.account.delete_containers() cls.account.update_metadata({'temp-url-key': cls.tempurl_key}) cls.manifest_container = cls.account.container(Utils.create_name()) @@ -698,7 +699,6 @@ class TestSloTempurlEnv(object): class TestSloTempurl(Base): env = TestSloTempurlEnv - set_up = False def setUp(self): super(TestSloTempurl, self).setUp() @@ -734,4 +734,4 @@ class TestSloTempurl(Base): class TestSloTempurlUTF8(Base2, TestSloTempurl): - set_up = False + pass diff --git a/test/functional/test_versioned_writes.py b/test/functional/test_versioned_writes.py index 03c627a949..b198ad0eff 100644 --- a/test/functional/test_versioned_writes.py +++ b/test/functional/test_versioned_writes.py @@ -16,11 +16,12 @@ import json import time +import unittest2 from unittest2 import SkipTest import test.functional as tf from copy import deepcopy -from test.functional.tests import Base, Base2, Utils +from test.functional.tests import Base, Base2, BaseEnv, Utils from test.functional import cluster_info from test.functional.swift_test_client import Account, Connection, \ ResponseError @@ -34,18 +35,14 @@ def tearDownModule(): tf.teardown_package() -class TestObjectVersioningEnv(object): +class TestObjectVersioningEnv(BaseEnv): versioning_enabled = None # tri-state: None initially, then True/False location_header_key = 'X-Versions-Location' + account2 = None @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.storage_url, cls.storage_token = cls.conn.authenticate() - - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - + super(TestObjectVersioningEnv, cls).setUp() # Second connection for ACL tests config2 = deepcopy(tf.config) config2['account'] = tf.config['account2'] @@ -96,22 +93,23 @@ class TestObjectVersioningEnv(object): @classmethod def tearDown(cls): - cls.account.delete_containers() - cls.account2.delete_containers() + if cls.account: + cls.account.delete_containers() + if cls.account2: + cls.account2.delete_containers() -class TestCrossPolicyObjectVersioningEnv(object): +class TestCrossPolicyObjectVersioningEnv(BaseEnv): # tri-state: None initially, then True/False versioning_enabled = None multiple_policies_enabled = None policies = None location_header_key = 'X-Versions-Location' + account2 = None @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - + super(TestCrossPolicyObjectVersioningEnv, cls).setUp() if cls.multiple_policies_enabled is None: try: cls.policies = tf.FunctionalStoragePolicyCollection.from_info() @@ -131,9 +129,6 @@ class TestCrossPolicyObjectVersioningEnv(object): policy = cls.policies.select() version_policy = cls.policies.exclude(name=policy['name']).select() - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - # Second connection for ACL tests config2 = deepcopy(tf.config) config2['account'] = tf.config['account2'] @@ -185,8 +180,10 @@ class TestCrossPolicyObjectVersioningEnv(object): @classmethod def tearDown(cls): - cls.account.delete_containers() - cls.account2.delete_containers() + if cls.account: + cls.account.delete_containers() + if cls.account2: + cls.account2.delete_containers() class TestObjectVersioningHistoryModeEnv(TestObjectVersioningEnv): @@ -195,7 +192,6 @@ class TestObjectVersioningHistoryModeEnv(TestObjectVersioningEnv): class TestObjectVersioning(Base): env = TestObjectVersioningEnv - set_up = False def setUp(self): super(TestObjectVersioning, self).setUp() @@ -487,7 +483,6 @@ class TestObjectVersioning(Base): class TestObjectVersioningUTF8(Base2, TestObjectVersioning): - set_up = False def tearDown(self): self._tear_down_files() @@ -496,7 +491,6 @@ class TestObjectVersioningUTF8(Base2, TestObjectVersioning): class TestCrossPolicyObjectVersioning(TestObjectVersioning): env = TestCrossPolicyObjectVersioningEnv - set_up = False def setUp(self): super(TestCrossPolicyObjectVersioning, self).setUp() @@ -511,7 +505,6 @@ class TestCrossPolicyObjectVersioning(TestObjectVersioning): class TestObjectVersioningHistoryMode(TestObjectVersioning): env = TestObjectVersioningHistoryModeEnv - set_up = False # those override tests includes assertions for delete versioned objects # behaviors different from default object versioning using @@ -695,7 +688,7 @@ class TestObjectVersioningHistoryMode(TestObjectVersioning): self.assertEqual(expected, prev_version.read()) -class TestSloWithVersioning(Base): +class TestSloWithVersioning(unittest2.TestCase): def setUp(self): if 'slo' not in cluster_info: diff --git a/test/functional/tests.py b/test/functional/tests.py index e5bde960eb..a27fec9818 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -67,12 +67,33 @@ class Utils(object): create_name = create_ascii_name +class BaseEnv(object): + account = conn = None + + @classmethod + def setUp(cls): + cls.conn = Connection(tf.config) + cls.conn.authenticate() + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) + cls.account.delete_containers() + + @classmethod + def tearDown(cls): + pass + + class Base(unittest2.TestCase): - def setUp(self): - cls = type(self) - if not cls.set_up: - cls.env.setUp() - cls.set_up = True + # subclasses may override env class + env = BaseEnv + + @classmethod + def setUpClass(cls): + cls.env.setUp() + + @classmethod + def tearDownClass(cls): + cls.env.tearDown() def assert_body(self, body): response_body = self.env.conn.response.read() @@ -105,15 +126,10 @@ class Base2(object): Utils.create_name = Utils.create_ascii_name -class TestAccountEnv(object): +class TestAccountEnv(BaseEnv): @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - cls.account.delete_containers() - + super(TestAccountEnv, cls).setUp() cls.containers = [] for i in range(10): cont = cls.account.container(Utils.create_name()) @@ -125,16 +141,14 @@ class TestAccountEnv(object): class TestAccountDev(Base): env = TestAccountEnv - set_up = False class TestAccountDevUTF8(Base2, TestAccountDev): - set_up = False + pass class TestAccount(Base): env = TestAccountEnv - set_up = False def testNoAuthToken(self): self.assertRaises(ResponseError, self.env.account.info, @@ -352,23 +366,10 @@ class TestAccount(Base): class TestAccountUTF8(Base2, TestAccount): - set_up = False - - -class TestAccountNoContainersEnv(object): - @classmethod - def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - cls.account.delete_containers() + pass class TestAccountNoContainers(Base): - env = TestAccountNoContainersEnv - set_up = False - def testGetRequest(self): for format_type in [None, 'json', 'xml']: self.assertFalse(self.env.account.containers( @@ -381,18 +382,13 @@ class TestAccountNoContainers(Base): class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers): - set_up = False + pass -class TestAccountSortingEnv(object): +class TestAccountSortingEnv(BaseEnv): @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - cls.account.delete_containers() - + super(TestAccountSortingEnv, cls).setUp() postfix = Utils.create_name() cls.cont_items = ('a1', 'a2', 'A3', 'b1', 'B2', 'a10', 'b10', 'zz') cls.cont_items = ['%s%s' % (x, postfix) for x in cls.cont_items] @@ -405,7 +401,6 @@ class TestAccountSortingEnv(object): class TestAccountSorting(Base): env = TestAccountSortingEnv - set_up = False def testAccountContainerListSorting(self): # name (byte order) sorting. @@ -470,15 +465,10 @@ class TestAccountSorting(Base): self.assertEqual([], cont_listing) -class TestContainerEnv(object): +class TestContainerEnv(BaseEnv): @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - cls.account.delete_containers() - + super(TestContainerEnv, cls).setUp() cls.container = cls.account.container(Utils.create_name()) if not cls.container.create(): raise ResponseError(cls.conn.response) @@ -494,16 +484,14 @@ class TestContainerEnv(object): class TestContainerDev(Base): env = TestContainerEnv - set_up = False class TestContainerDevUTF8(Base2, TestContainerDev): - set_up = False + pass class TestContainer(Base): env = TestContainerEnv - set_up = False def testContainerNameLimit(self): limit = load_constraint('max_container_name_length') @@ -889,18 +877,13 @@ class TestContainer(Base): class TestContainerUTF8(Base2, TestContainer): - set_up = False + pass -class TestContainerSortingEnv(object): +class TestContainerSortingEnv(BaseEnv): @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - cls.account.delete_containers() - + super(TestContainerSortingEnv, cls).setUp() cls.container = cls.account.container(Utils.create_name()) if not cls.container.create(): raise ResponseError(cls.conn.response) @@ -916,7 +899,6 @@ class TestContainerSortingEnv(object): class TestContainerSorting(Base): env = TestContainerSortingEnv - set_up = False def testContainerFileListSortingReversed(self): file_list = list(sorted(self.env.file_items)) @@ -1004,15 +986,10 @@ class TestContainerSorting(Base): self.assertEqual(file_list, cont_files) -class TestContainerPathsEnv(object): +class TestContainerPathsEnv(BaseEnv): @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - cls.account.delete_containers() - + super(TestContainerPathsEnv, cls).setUp() cls.file_size = 8 cls.container = cls.account.container(Utils.create_name()) @@ -1082,7 +1059,6 @@ class TestContainerPathsEnv(object): class TestContainerPaths(Base): env = TestContainerPathsEnv - set_up = False def testTraverseContainer(self): found_files = [] @@ -1183,13 +1159,10 @@ class TestContainerPaths(Base): ['dir1/subdir with spaces/file B']) -class TestFileEnv(object): +class TestFileEnv(BaseEnv): @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) + super(TestFileEnv, cls).setUp() # creating another account and connection # for account to account copy tests config2 = deepcopy(tf.config) @@ -1199,9 +1172,6 @@ class TestFileEnv(object): cls.conn2 = Connection(config2) cls.conn2.authenticate() - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - cls.account.delete_containers() cls.account2 = cls.conn2.get_account() cls.account2.delete_containers() @@ -1223,16 +1193,14 @@ class TestFileEnv(object): class TestFileDev(Base): env = TestFileEnv - set_up = False class TestFileDevUTF8(Base2, TestFileDev): - set_up = False + pass class TestFile(Base): env = TestFileEnv - set_up = False def testCopy(self): # makes sure to test encoded characters @@ -2505,18 +2473,13 @@ class TestFile(Base): class TestFileUTF8(Base2, TestFile): - set_up = False + pass -class TestFileComparisonEnv(object): +class TestFileComparisonEnv(BaseEnv): @classmethod def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - cls.account.delete_containers() - + super(TestFileComparisonEnv, cls).setUp() cls.container = cls.account.container(Utils.create_name()) if not cls.container.create(): @@ -2542,7 +2505,6 @@ class TestFileComparisonEnv(object): class TestFileComparison(Base): env = TestFileComparisonEnv - set_up = False def testIfMatch(self): for file_item in self.env.files: @@ -2662,7 +2624,7 @@ class TestFileComparison(Base): class TestFileComparisonUTF8(Base2, TestFileComparison): - set_up = False + pass class TestServiceToken(unittest2.TestCase):