diff --git a/test/probe/common.py b/test/probe/common.py index 3c9192f615..846d9ca66d 100644 --- a/test/probe/common.py +++ b/test/probe/common.py @@ -19,12 +19,14 @@ from subprocess import Popen, PIPE import sys from time import sleep, time from collections import defaultdict +from nose import SkipTest from swiftclient import get_auth, head_account from swift.common.ring import Ring from swift.common.utils import readconf from swift.common.manager import Manager +from swift.common.storage_policy import POLICIES from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC @@ -136,13 +138,16 @@ def kill_nonprimary_server(primary_nodes, port2server, pids): return port -def get_ring(server, force_validate=None): - ring = Ring('/etc/swift/%s.ring.gz' % server) +def get_ring(ring_name, server=None, force_validate=None): + if not server: + server = ring_name + ring = Ring('/etc/swift', ring_name=ring_name) if not VALIDATE_RSYNC and not force_validate: return ring # easy sanity checks - assert 3 == ring.replica_count, '%s has %s replicas instead of 3' % ( - ring.serialized_path, ring.replica_count) + if ring.replica_count != 3: + print 'WARNING: %s has %s replicas instead of 3' % ( + ring.serialized_path, ring.replica_count) assert 4 == len(ring.devs), '%s has %s devices instead of 4' % ( ring.serialized_path, len(ring.devs)) # map server to config by port @@ -197,7 +202,8 @@ def reset_environment(): try: account_ring = get_ring('account') container_ring = get_ring('container') - object_ring = get_ring('object') + policy = POLICIES.default + object_ring = get_ring(policy.ring_name, 'object') Manager(['main']).start(wait=False) port2server = {} for server, port in [('account', 6002), ('container', 6001), @@ -218,15 +224,14 @@ def reset_environment(): try: raise except AssertionError as e: - print >>sys.stderr, 'ERROR: %s' % e - os._exit(1) + raise SkipTest(e) finally: try: kill_servers(port2server, pids) except Exception: pass - return pids, port2server, account_ring, container_ring, object_ring, url, \ - token, account, config_dict + return pids, port2server, account_ring, container_ring, object_ring, \ + policy, url, token, account, config_dict def get_to_final_state(): @@ -242,6 +247,15 @@ def get_to_final_state(): if __name__ == "__main__": - for server in ('account', 'container', 'object'): - get_ring(server, force_validate=True) + for server in ('account', 'container'): + try: + get_ring(server, force_validate=True) + except AssertionError as err: + sys.exit('%s ERROR: %s' % (server, err)) print '%s OK' % server + for policy in POLICIES: + try: + get_ring(policy.ring_name, server='object', force_validate=True) + except AssertionError as err: + sys.exit('object ERROR (%s): %s' % (policy.name, err)) + print 'object OK (%s)' % policy.name diff --git a/test/probe/test_account_failures.py b/test/probe/test_account_failures.py index aa7713f14e..04b998ac0d 100755 --- a/test/probe/test_account_failures.py +++ b/test/probe/test_account_failures.py @@ -28,7 +28,7 @@ class TestAccountFailures(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, - self.object_ring, self.url, self.token, + self.object_ring, self.policy, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): diff --git a/test/probe/test_account_get_fake_responses_match.py b/test/probe/test_account_get_fake_responses_match.py index 5f83fa4b41..1400d6109c 100755 --- a/test/probe/test_account_get_fake_responses_match.py +++ b/test/probe/test_account_get_fake_responses_match.py @@ -27,7 +27,7 @@ class TestAccountGetFakeResponsesMatch(unittest.TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, - self.object_ring, self.url, self.token, + self.object_ring, self.policy, self.url, self.token, self.account, self.configs) = reset_environment() self.url, self.token = get_auth( 'http://127.0.0.1:8080/auth/v1.0', 'admin:admin', 'admin') diff --git a/test/probe/test_container_failures.py b/test/probe/test_container_failures.py index f42dd3cee8..146d0fd46a 100755 --- a/test/probe/test_container_failures.py +++ b/test/probe/test_container_failures.py @@ -44,7 +44,7 @@ class TestContainerFailures(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, - self.object_ring, self.url, self.token, + self.object_ring, self.policy, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): diff --git a/test/probe/test_container_merge_policy_index.py b/test/probe/test_container_merge_policy_index.py index 3f26f1dac1..1090a2452a 100644 --- a/test/probe/test_container_merge_policy_index.py +++ b/test/probe/test_container_merge_policy_index.py @@ -179,7 +179,7 @@ class TestContainerMergePolicyIndex(unittest.TestCase): if len(POLICIES) < 2: raise SkipTest() (self.pids, self.port2server, self.account_ring, self.container_ring, - self.object_ring, self.url, self.token, + self.object_ring, self.policy, self.url, self.token, self.account, self.configs) = reset_environment() self.container_name = 'container-%s' % uuid.uuid4() self.object_name = 'object-%s' % uuid.uuid4() diff --git a/test/probe/test_empty_device_handoff.py b/test/probe/test_empty_device_handoff.py index 2841bcd425..f154d1ad48 100755 --- a/test/probe/test_empty_device_handoff.py +++ b/test/probe/test_empty_device_handoff.py @@ -24,6 +24,8 @@ from uuid import uuid4 from swiftclient import client from swift.common import direct_client +from swift.common.storage_policy import POLICY_INDEX +from swift.obj.diskfile import get_data_dir from swift.common.exceptions import ClientException from test.probe.common import kill_server, kill_servers, reset_environment,\ start_server @@ -35,7 +37,7 @@ class TestEmptyDevice(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, - self.object_ring, self.url, self.token, + self.object_ring, self.policy, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): @@ -52,7 +54,7 @@ class TestEmptyDevice(TestCase): def test_main(self): # Create container # Kill one container/obj primary server - # Delete the "objects" directory on the primary server + # Delete the default data directory for objects on the primary server # Create container/obj (goes to two primary servers and one handoff) # Kill other two container/obj primary servers # Indirectly through proxy assert we can get container/obj @@ -76,7 +78,8 @@ class TestEmptyDevice(TestCase): self.account, container, obj) onode = onodes[0] kill_server(onode['port'], self.port2server, self.pids) - obj_dir = '%s/objects' % self._get_objects_dir(onode) + obj_dir = '%s/%s' % (self._get_objects_dir(onode), + get_data_dir(self.policy.idx)) shutil.rmtree(obj_dir, True) self.assertFalse(os.path.exists(obj_dir)) client.put_object(self.url, self.token, container, obj, 'VERIFY') @@ -98,7 +101,8 @@ class TestEmptyDevice(TestCase): # let's directly verify it. another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object( - another_onode, opart, self.account, container, obj)[-1] + another_onode, opart, self.account, container, obj, + headers={POLICY_INDEX: self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) @@ -128,8 +132,9 @@ class TestEmptyDevice(TestCase): self.assertFalse(os.path.exists(obj_dir)) exc = None try: - direct_client.direct_get_object(onode, opart, self.account, - container, obj) + direct_client.direct_get_object( + onode, opart, self.account, container, obj, headers={ + POLICY_INDEX: self.policy.idx}) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) @@ -150,15 +155,17 @@ class TestEmptyDevice(TestCase): another_num = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_num) - odata = direct_client.direct_get_object(onode, opart, self.account, - container, obj)[-1] + odata = direct_client.direct_get_object( + onode, opart, self.account, container, obj, headers={ + POLICY_INDEX: self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) exc = None try: - direct_client.direct_get_object(another_onode, opart, self.account, - container, obj) + direct_client.direct_get_object( + another_onode, opart, self.account, container, obj, headers={ + POLICY_INDEX: self.policy.idx}) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) diff --git a/test/probe/test_object_async_update.py b/test/probe/test_object_async_update.py index e48dd91ad5..ed2bcdf0be 100755 --- a/test/probe/test_object_async_update.py +++ b/test/probe/test_object_async_update.py @@ -29,7 +29,7 @@ class TestObjectAsyncUpdate(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, - self.object_ring, self.url, self.token, + self.object_ring, self.policy, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): diff --git a/test/probe/test_object_failures.py b/test/probe/test_object_failures.py index f8fc119fa4..1ea0fc25eb 100755 --- a/test/probe/test_object_failures.py +++ b/test/probe/test_object_failures.py @@ -23,9 +23,10 @@ from uuid import uuid4 from swiftclient import client from swift.common import direct_client +from swift.common.storage_policy import POLICY_INDEX from swift.common.exceptions import ClientException from swift.common.utils import hash_path, readconf -from swift.obj.diskfile import write_metadata, read_metadata +from swift.obj.diskfile import write_metadata, read_metadata, get_data_dir from test.probe.common import kill_servers, reset_environment @@ -53,7 +54,7 @@ class TestObjectFailures(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, - self.object_ring, self.url, self.token, + self.object_ring, self.policy, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): @@ -72,9 +73,9 @@ class TestObjectFailures(TestCase): hash_str = hash_path(self.account, container, obj) obj_server_conf = readconf(self.configs['object-server'][node_id]) devices = obj_server_conf['app:object-server']['devices'] - obj_dir = '%s/%s/objects/%s/%s/%s/' % (devices, - device, opart, - hash_str[-3:], hash_str) + obj_dir = '%s/%s/%s/%s/%s/%s/' % (devices, device, + get_data_dir(self.policy.idx), + opart, hash_str[-3:], hash_str) data_file = get_data_file_path(obj_dir) return onode, opart, data_file @@ -88,11 +89,13 @@ class TestObjectFailures(TestCase): write_metadata(data_file, metadata) odata = direct_client.direct_get_object( - onode, opart, self.account, container, obj)[-1] + onode, opart, self.account, container, obj, headers={ + POLICY_INDEX: self.policy.idx})[-1] self.assertEquals(odata, 'VERIFY') try: - direct_client.direct_get_object(onode, opart, self.account, - container, obj) + direct_client.direct_get_object( + onode, opart, self.account, container, obj, headers={ + POLICY_INDEX: self.policy.idx}) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404) @@ -106,16 +109,21 @@ class TestObjectFailures(TestCase): metadata = read_metadata(data_file) metadata['ETag'] = 'badetag' write_metadata(data_file, metadata) + base_headers = {POLICY_INDEX: self.policy.idx} for header, result in [({'Range': 'bytes=0-2'}, 'RAN'), ({'Range': 'bytes=1-11'}, 'ANGE'), ({'Range': 'bytes=0-11'}, 'RANGE')]: + req_headers = base_headers.copy() + req_headers.update(header) odata = direct_client.direct_get_object( - onode, opart, self.account, container, obj, headers=header)[-1] + onode, opart, self.account, container, obj, + headers=req_headers)[-1] self.assertEquals(odata, result) try: - direct_client.direct_get_object(onode, opart, self.account, - container, obj) + direct_client.direct_get_object( + onode, opart, self.account, container, obj, headers={ + POLICY_INDEX: self.policy.idx}) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404) @@ -130,9 +138,9 @@ class TestObjectFailures(TestCase): with open(data_file, 'w') as fpointer: write_metadata(fpointer, metadata) try: - direct_client.direct_get_object(onode, opart, self.account, - container, obj, conn_timeout=1, - response_timeout=1) + direct_client.direct_get_object( + onode, opart, self.account, container, obj, conn_timeout=1, + response_timeout=1, headers={POLICY_INDEX: self.policy.idx}) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404) @@ -147,9 +155,9 @@ class TestObjectFailures(TestCase): with open(data_file, 'w') as fpointer: write_metadata(fpointer, metadata) try: - direct_client.direct_head_object(onode, opart, self.account, - container, obj, conn_timeout=1, - response_timeout=1) + direct_client.direct_head_object( + onode, opart, self.account, container, obj, conn_timeout=1, + response_timeout=1, headers={POLICY_INDEX: self.policy.idx}) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404) @@ -164,10 +172,12 @@ class TestObjectFailures(TestCase): with open(data_file, 'w') as fpointer: write_metadata(fpointer, metadata) try: + headers = {'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two', + POLICY_INDEX: self.policy.idx} direct_client.direct_post_object( onode, opart, self.account, container, obj, - {'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'}, + headers=headers, conn_timeout=1, response_timeout=1) raise Exception("Did not quarantine object") diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py index 565b2ff7cc..12ade94e1a 100755 --- a/test/probe/test_object_handoff.py +++ b/test/probe/test_object_handoff.py @@ -20,6 +20,7 @@ from uuid import uuid4 from swiftclient import client from swift.common import direct_client +from swift.common.storage_policy import POLICY_INDEX from swift.common.exceptions import ClientException from swift.common.manager import Manager from test.probe.common import kill_server, kill_servers, reset_environment, \ @@ -30,7 +31,7 @@ class TestObjectHandoff(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, - self.object_ring, self.url, self.token, + self.object_ring, self.policy, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): @@ -90,7 +91,8 @@ class TestObjectHandoff(TestCase): # directly verify it. another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object( - another_onode, opart, self.account, container, obj)[-1] + another_onode, opart, self.account, container, obj, headers={ + POLICY_INDEX: self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) @@ -109,8 +111,9 @@ class TestObjectHandoff(TestCase): start_server(onode['port'], self.port2server, self.pids) exc = None try: - direct_client.direct_get_object(onode, opart, self.account, - container, obj) + direct_client.direct_get_object( + onode, opart, self.account, container, obj, headers={ + POLICY_INDEX: self.policy.idx}) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) @@ -128,21 +131,31 @@ class TestObjectHandoff(TestCase): another_port_num = another_onode['port'] another_num = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_num) - odata = direct_client.direct_get_object(onode, opart, self.account, - container, obj)[-1] + odata = direct_client.direct_get_object( + onode, opart, self.account, container, obj, headers={ + POLICY_INDEX: self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) exc = None try: - direct_client.direct_get_object(another_onode, opart, self.account, - container, obj) + direct_client.direct_get_object( + another_onode, opart, self.account, container, obj, headers={ + POLICY_INDEX: self.policy.idx}) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) kill_server(onode['port'], self.port2server, self.pids) - client.delete_object(self.url, self.token, container, obj) + try: + client.delete_object(self.url, self.token, container, obj) + except client.ClientException as err: + if self.object_ring.replica_count > 2: + raise + # Object DELETE returning 503 for (404, 204) + # remove this with fix for + # https://bugs.launchpad.net/swift/+bug/1318375 + self.assertEqual(503, err.http_status) exc = None try: client.head_object(self.url, self.token, container, obj) @@ -162,8 +175,9 @@ class TestObjectHandoff(TestCase): 'Container server %s:%s still knew about object' % (cnode['ip'], cnode['port'])) start_server(onode['port'], self.port2server, self.pids) - direct_client.direct_get_object(onode, opart, self.account, container, - obj) + direct_client.direct_get_object( + onode, opart, self.account, container, obj, headers={ + POLICY_INDEX: self.policy.idx}) # Run the extra server last so it'll remove its extra partition for node in onodes: try: @@ -176,8 +190,9 @@ class TestObjectHandoff(TestCase): Manager(['object-replicator']).once(number=another_node_id) exc = None try: - direct_client.direct_get_object(another_onode, opart, self.account, - container, obj) + direct_client.direct_get_object( + another_onode, opart, self.account, container, obj, headers={ + POLICY_INDEX: self.policy.idx}) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) diff --git a/test/probe/test_replication_servers_working.py b/test/probe/test_replication_servers_working.py index 07416bb9cd..f08c115633 100644 --- a/test/probe/test_replication_servers_working.py +++ b/test/probe/test_replication_servers_working.py @@ -21,6 +21,8 @@ import time import shutil from swiftclient import client +from swift.common.storage_policy import POLICIES +from swift.obj.diskfile import get_data_dir from test.probe.common import kill_servers, reset_environment from swift.common.utils import readconf @@ -80,7 +82,7 @@ class TestReplicatorFunctions(TestCase): Reset all environment and start all servers. """ (self.pids, self.port2server, self.account_ring, self.container_ring, - self.object_ring, self.url, self.token, + self.object_ring, self.policy, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): @@ -100,6 +102,7 @@ class TestReplicatorFunctions(TestCase): # Delete file "hashes.pkl". # Check, that all files were replicated. path_list = [] + data_dir = get_data_dir(POLICIES.default.idx) # Figure out where the devices are for node_id in range(1, 5): conf = readconf(self.configs['object-server'][node_id]) @@ -124,7 +127,10 @@ class TestReplicatorFunctions(TestCase): for files in files_list[num]: if not files.endswith('.pending'): test_node_files_list.append(files) - test_node_dir_list = dir_list[num] + test_node_dir_list = [] + for d in dir_list[num]: + if not d.startswith('tmp'): + test_node_dir_list.append(d) # Run all replicators try: Manager(['object-replicator', 'container-replicator', @@ -155,24 +161,24 @@ class TestReplicatorFunctions(TestCase): time.sleep(1) # Check behavior by deleting hashes.pkl file - for directory in os.listdir(os.path.join(test_node, 'objects')): + for directory in os.listdir(os.path.join(test_node, data_dir)): for input_dir in os.listdir(os.path.join( - test_node, 'objects', directory)): + test_node, data_dir, directory)): if os.path.isdir(os.path.join( - test_node, 'objects', directory, input_dir)): + test_node, data_dir, directory, input_dir)): shutil.rmtree(os.path.join( - test_node, 'objects', directory, input_dir)) + test_node, data_dir, directory, input_dir)) # We will keep trying these tests until they pass for up to 60s begin = time.time() while True: try: for directory in os.listdir(os.path.join( - test_node, 'objects')): + test_node, data_dir)): for input_dir in os.listdir(os.path.join( - test_node, 'objects', directory)): + test_node, data_dir, directory)): self.assertFalse(os.path.isdir( - os.path.join(test_node, 'objects', + os.path.join(test_node, data_dir, directory, '/', input_dir))) break except Exception: @@ -180,9 +186,9 @@ class TestReplicatorFunctions(TestCase): raise time.sleep(1) - for directory in os.listdir(os.path.join(test_node, 'objects')): + for directory in os.listdir(os.path.join(test_node, data_dir)): os.remove(os.path.join( - test_node, 'objects', directory, 'hashes.pkl')) + test_node, data_dir, directory, 'hashes.pkl')) # We will keep trying these tests until they pass for up to 60s begin = time.time()