openstack/swift
Code Issues Proposed changes

Merge "Updated probe tests"

Browse Source
This commit is contained in:
Jenkins 2012-08-21 20:53:42 +00:00 committed by Gerrit Code Review
commit a0a8b484cc
7 changed files with 496 additions and 792 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

198
test/probe/common.py
View File

@ -13,22 +13,112 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from httplib import HTTPConnection
from os import kill from os import kill
from signal import SIGTERM from signal import SIGTERM
from subprocess import call, Popen from subprocess import call, Popen
from time import sleep from time import sleep, time
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swiftclient import get_auth, head_account from swiftclient import get_auth, head_account
from swift.common.ring import Ring from swift.common.ring import Ring
def kill_pids(pids): def start_server(port, port2server, pids, check=True):
for pid in pids.values(): server = port2server[port]
if server[:-1] in ('account', 'container', 'object'):
pids[server] = Popen([
'swift-%s-server' % server[:-1],
'/etc/swift/%s-server/%s.conf' % (server[:-1], server[-1])]).pid
if check:
return check_server(port, port2server, pids)
else:
pids[server] = Popen(['swift-%s-server' % server,
'/etc/swift/%s-server.conf' % server]).pid
if check:
return check_server(port, port2server, pids)
return None
def check_server(port, port2server, pids):
server = port2server[port]
if server[:-1] in ('account', 'container', 'object'):
path = '/connect/1/2'
if server[:-1] == 'container':
path += '/3'
elif server[:-1] == 'object':
path += '/3/4'
try_until = time() + 30
while True:
try:
conn = HTTPConnection('127.0.0.1', port)
conn.request('GET', path)
resp = conn.getresponse()
if resp.status != 404:
raise Exception(
'Unexpected status %s' % resp.status)
break
except Exception, err:
if time() > try_until:
print err
print 'Giving up on %s:%s after 30 seconds.' % (
server, port)
raise err
sleep(0.1)
else:
try_until = time() + 30
while True:
try:
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
except Exception, err:
if time() > try_until:
print err
print 'Giving up on proxy:8080 after 30 seconds.'
raise err
sleep(0.1)
return None
def kill_server(port, port2server, pids):
try:
kill(pids[port2server[port]], SIGTERM)
except Exception, err:
print err
try_until = time() + 30
while True:
try: try:
kill(pid, SIGTERM) conn = HTTPConnection('127.0.0.1', port)
except Exception: conn.request('GET', '/')
pass conn.getresponse()
except Exception, err:
break
if time() > try_until:
raise Exception(
'Still answering on port %s after 30 seconds' % port)
sleep(0.1)
def kill_servers(port2server, pids):
for port in port2server:
kill_server(port, port2server, pids)
def kill_nonprimary_server(primary_nodes, port2server, pids):
primary_ports = [n['port'] for n in primary_nodes]
for port, server in port2server.iteritems():
if port in primary_ports:
server_type = server[:-1]
break
else:
raise Exception('Cannot figure out server type for %r' % primary_nodes)
for port, server in list(port2server.iteritems()):
if server[:-1] == server_type and port not in primary_ports:
kill_server(port, port2server, pids)
return port
def reset_environment(): def reset_environment():
@ -36,68 +126,58 @@ def reset_environment():
pids = {} pids = {}
try: try:
port2server = {} port2server = {}
for s, p in (('account', 6002), ('container', 6001), ('object', 6000)): for server, port in [('account', 6002), ('container', 6001),
for n in xrange(1, 5): ('object', 6000)]:
pids['%s%d' % (s, n)] = \ for number in xrange(1, 5):
Popen(['swift-%s-server' % s, port2server[port + (number * 10)] = '%s%d' % (server, number)
'/etc/swift/%s-server/%d.conf' % (s, n)]).pid for port in port2server:
port2server[p + (n * 10)] = '%s%d' % (s, n) start_server(port, port2server, pids, check=False)
pids['proxy'] = Popen(['swift-proxy-server', for port in port2server:
'/etc/swift/proxy-server.conf']).pid check_server(port, port2server, pids)
port2server[8080] = 'proxy'
url, token, account = start_server(8080, port2server, pids)
account_ring = Ring('/etc/swift/account.ring.gz') account_ring = Ring('/etc/swift/account.ring.gz')
container_ring = Ring('/etc/swift/container.ring.gz') container_ring = Ring('/etc/swift/container.ring.gz')
object_ring = Ring('/etc/swift/object.ring.gz') object_ring = Ring('/etc/swift/object.ring.gz')
attempt = 0 except BaseException:
while True: try:
attempt += 1 raise
finally:
try: try:
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', kill_servers(port2server, pids)
'test:tester', 'testing') except Exception:
account = url.split('/')[-1] pass
head_account(url, token)
break
except Exception, err:
if attempt > 9:
print err
print 'Giving up after %s retries.' % attempt
raise err
print err
print 'Retrying in 2 seconds...'
sleep(2)
except BaseException, err:
kill_pids(pids)
raise err
return pids, port2server, account_ring, container_ring, object_ring, url, \ return pids, port2server, account_ring, container_ring, object_ring, url, \
token, account token, account
def get_to_final_state(): def get_to_final_state():
ps = [] processes = []
for job in ('account-replicator', 'container-replicator', for job in ('account-replicator', 'container-replicator',
'object-replicator'): 'object-replicator'):
for n in xrange(1, 5): for number in xrange(1, 5):
ps.append(Popen(['swift-%s' % job, processes.append(Popen([
'/etc/swift/%s-server/%d.conf' % 'swift-%s' % job,
(job.split('-')[0], n), '/etc/swift/%s-server/%d.conf' % (job.split('-')[0], number),
'once'])) 'once']))
for p in ps: for process in processes:
p.wait() process.wait()
ps = [] processes = []
for job in ('container-updater', 'object-updater'): for job in ('container-updater', 'object-updater'):
for n in xrange(1, 5): for number in xrange(1, 5):
ps.append(Popen(['swift-%s' % job, processes.append(Popen([
'/etc/swift/%s-server/%d.conf' % 'swift-%s' % job,
(job.split('-')[0], n), '/etc/swift/%s-server/%d.conf' % (job.split('-')[0], number),
'once'])) 'once']))
for p in ps: for process in processes:
p.wait() process.wait()
ps = [] processes = []
for job in ('account-replicator', 'container-replicator', for job in ('account-replicator', 'container-replicator',
'object-replicator'): 'object-replicator'):
for n in xrange(1, 5): for number in xrange(1, 5):
ps.append(Popen(['swift-%s' % job, processes.append(Popen([
'/etc/swift/%s-server/%d.conf' % 'swift-%s' % job,
(job.split('-')[0], n), '/etc/swift/%s-server/%d.conf' % (job.split('-')[0], number),
'once'])) 'once']))
for p in ps: for process in processes:
p.wait() process.wait()

153
test/probe/test_account_failures.py
View File

@ -14,28 +14,46 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import unittest
from os import kill
from signal import SIGTERM
from subprocess import Popen from subprocess import Popen
from time import sleep from unittest import main, TestCase
from swiftclient import client from swiftclient import client
from swift.common import direct_client from swift.common import direct_client
from test.probe.common import get_to_final_state, kill_pids, reset_environment from test.probe.common import get_to_final_state, kill_nonprimary_server, \
kill_server, kill_servers, reset_environment, start_server
class TestAccountFailures(unittest.TestCase): class TestAccountFailures(TestCase):
def setUp(self): def setUp(self):
self.pids, self.port2server, self.account_ring, self.container_ring, \ (self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token, self.account = \ self.object_ring, self.url, self.token,
reset_environment() self.account) = reset_environment()
def tearDown(self): def tearDown(self):
kill_pids(self.pids) kill_servers(self.port2server, self.pids)
def test_main(self): def test_main(self):
# Create container1 and container2
# Assert account level sees them
# Create container2/object1
# Assert account level doesn't see it yet
# Get to final state
# Assert account level now sees the container2/object1
# Kill account servers excepting two of the primaries
# Delete container1
# Assert account level knows container1 is gone but doesn't know about
# container2/object2 yet
# Put container2/object2
# Run container updaters
# Assert account level now knows about container2/object2
# Restart other primary account server
# Assert that server doesn't know about container1's deletion or the
# new container2/object2 yet
# Get to final state
# Assert that server is now up to date
container1 = 'container1' container1 = 'container1'
client.put_container(self.url, self.token, container1) client.put_container(self.url, self.token, container1)
container2 = 'container2' container2 = 'container2'
@ -46,15 +64,15 @@ class TestAccountFailures(unittest.TestCase):
self.assertEquals(headers['x-account-bytes-used'], '0') self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False found1 = False
found2 = False found2 = False
for c in containers: for container in containers:
if c['name'] == container1: if container['name'] == container1:
found1 = True found1 = True
self.assertEquals(c['count'], 0) self.assertEquals(container['count'], 0)
self.assertEquals(c['bytes'], 0) self.assertEquals(container['bytes'], 0)
elif c['name'] == container2: elif container['name'] == container2:
found2 = True found2 = True
self.assertEquals(c['count'], 0) self.assertEquals(container['count'], 0)
self.assertEquals(c['bytes'], 0) self.assertEquals(container['bytes'], 0)
self.assert_(found1) self.assert_(found1)
self.assert_(found2) self.assert_(found2)
@ -65,15 +83,15 @@ class TestAccountFailures(unittest.TestCase):
self.assertEquals(headers['x-account-bytes-used'], '0') self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False found1 = False
found2 = False found2 = False
for c in containers: for container in containers:
if c['name'] == container1: if container['name'] == container1:
found1 = True found1 = True
self.assertEquals(c['count'], 0) self.assertEquals(container['count'], 0)
self.assertEquals(c['bytes'], 0) self.assertEquals(container['bytes'], 0)
elif c['name'] == container2: elif container['name'] == container2:
found2 = True found2 = True
self.assertEquals(c['count'], 0) self.assertEquals(container['count'], 0)
self.assertEquals(c['bytes'], 0) self.assertEquals(container['bytes'], 0)
self.assert_(found1) self.assert_(found1)
self.assert_(found2) self.assert_(found2)
@ -84,20 +102,21 @@ class TestAccountFailures(unittest.TestCase):
self.assertEquals(headers['x-account-bytes-used'], '4') self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False found1 = False
found2 = False found2 = False
for c in containers: for container in containers:
if c['name'] == container1: if container['name'] == container1:
found1 = True found1 = True
self.assertEquals(c['count'], 0) self.assertEquals(container['count'], 0)
self.assertEquals(c['bytes'], 0) self.assertEquals(container['bytes'], 0)
elif c['name'] == container2: elif container['name'] == container2:
found2 = True found2 = True
self.assertEquals(c['count'], 1) self.assertEquals(container['count'], 1)
self.assertEquals(c['bytes'], 4) self.assertEquals(container['bytes'], 4)
self.assert_(found1) self.assert_(found1)
self.assert_(found2) self.assert_(found2)
apart, anodes = self.account_ring.get_nodes(self.account) apart, anodes = self.account_ring.get_nodes(self.account)
kill(self.pids[self.port2server[anodes[0]['port']]], SIGTERM) kill_nonprimary_server(anodes, self.port2server, self.pids)
kill_server(anodes[0]['port'], self.port2server, self.pids)
client.delete_container(self.url, self.token, container1) client.delete_container(self.url, self.token, container1)
client.put_object(self.url, self.token, container2, 'object2', '12345') client.put_object(self.url, self.token, container2, 'object2', '12345')
@ -107,46 +126,42 @@ class TestAccountFailures(unittest.TestCase):
self.assertEquals(headers['x-account-bytes-used'], '4') self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False found1 = False
found2 = False found2 = False
for c in containers: for container in containers:
if c['name'] == container1: if container['name'] == container1:
found1 = True found1 = True
elif c['name'] == container2: elif container['name'] == container2:
found2 = True found2 = True
self.assertEquals(c['count'], 1) self.assertEquals(container['count'], 1)
self.assertEquals(c['bytes'], 4) self.assertEquals(container['bytes'], 4)
self.assert_(not found1) self.assert_(not found1)
self.assert_(found2) self.assert_(found2)
ps = [] processes = []
for n in xrange(1, 5): for node in xrange(1, 5):
ps.append(Popen(['swift-container-updater', processes.append(Popen([
'/etc/swift/container-server/%d.conf' % n, 'swift-container-updater',
'once'])) '/etc/swift/container-server/%d.conf' % node,
for p in ps: 'once']))
p.wait() for process in processes:
process.wait()
headers, containers = client.get_account(self.url, self.token) headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9') self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False found1 = False
found2 = False found2 = False
for c in containers: for container in containers:
if c['name'] == container1: if container['name'] == container1:
found1 = True found1 = True
elif c['name'] == container2: elif container['name'] == container2:
found2 = True found2 = True
self.assertEquals(c['count'], 2) self.assertEquals(container['count'], 2)
self.assertEquals(c['bytes'], 9) self.assertEquals(container['bytes'], 9)
self.assert_(not found1) self.assert_(not found1)
self.assert_(found2) self.assert_(found2)
self.pids[self.port2server[anodes[0]['port']]] = \ start_server(anodes[0]['port'], self.port2server, self.pids)
Popen(['swift-account-server',
'/etc/swift/account-server/%d.conf' %
((anodes[0]['port'] - 6002) / 10)]).pid
sleep(2)
# This is the earlier counts and bytes because the first node doesn't
# have the newest udpates yet.
headers, containers = \ headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account) direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-container-count'], '2')
@ -154,17 +169,13 @@ class TestAccountFailures(unittest.TestCase):
self.assertEquals(headers['x-account-bytes-used'], '4') self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False found1 = False
found2 = False found2 = False
for c in containers: for container in containers:
if c['name'] == container1: if container['name'] == container1:
found1 = True found1 = True
elif c['name'] == container2: elif container['name'] == container2:
found2 = True found2 = True
# This is the earlier count and bytes because the first node self.assertEquals(container['count'], 1)
# doesn't have the newest udpates yet. self.assertEquals(container['bytes'], 4)
self.assertEquals(c['count'], 1)
self.assertEquals(c['bytes'], 4)
# This okay because the first node hasn't got the update that
# container1 was deleted yet.
self.assert_(found1) self.assert_(found1)
self.assert_(found2) self.assert_(found2)
@ -176,16 +187,16 @@ class TestAccountFailures(unittest.TestCase):
self.assertEquals(headers['x-account-bytes-used'], '9') self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False found1 = False
found2 = False found2 = False
for c in containers: for container in containers:
if c['name'] == container1: if container['name'] == container1:
found1 = True found1 = True
elif c['name'] == container2: elif container['name'] == container2:
found2 = True found2 = True
self.assertEquals(c['count'], 2) self.assertEquals(container['count'], 2)
self.assertEquals(c['bytes'], 9) self.assertEquals(container['bytes'], 9)
self.assert_(not found1) self.assert_(not found1)
self.assert_(found2) self.assert_(found2)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() main()

414
test/probe/test_container_failures.py
View File

@ -14,325 +14,103 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import unittest from os import listdir
import os from os.path import join as path_join
from os import kill from unittest import main, TestCase
from signal import SIGTERM
from subprocess import Popen
from time import sleep
from uuid import uuid4 from uuid import uuid4
import eventlet
import sqlite3
from eventlet import GreenPool, Timeout
from sqlite3 import connect
from swiftclient import client from swiftclient import client
from swift.common import direct_client from swift.common import direct_client
from swift.common.utils import hash_path, readconf from swift.common.utils import hash_path, readconf
from test.probe.common import get_to_final_state, kill_nonprimary_server, \
from test.probe.common import get_to_final_state, kill_pids, reset_environment kill_server, kill_servers, reset_environment, start_server
class TestContainerFailures(unittest.TestCase): def get_db_file_path(obj_dir):
files = sorted(listdir(obj_dir), reverse=True)
for filename in files:
if filename.endswith('db'):
return path_join(obj_dir, filename)
class TestContainerFailures(TestCase):
def setUp(self): def setUp(self):
self.pids, self.port2server, self.account_ring, self.container_ring, \ (self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token, self.account = \ self.object_ring, self.url, self.token,
reset_environment() self.account) = reset_environment()
def tearDown(self): def tearDown(self):
kill_pids(self.pids) kill_servers(self.port2server, self.pids)
def test_first_node_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
self.pids[self.port2server[cnodes[0]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[0]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [o['name'] for o in
direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# Unfortunately, the following might pass or fail, depending on the
# position of the account server associated with the first container
# server we had killed. If the associated happens to be the first
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
# First node still doesn't know object1 was deleted yet; this is okay.
self.assert_(object1 in [o['name'] for o in
direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
def test_one_node_fails(self):
# Create container1
# Kill container1 servers excepting two of the primaries
# Delete container1
# Restart other container1 primary server
# Create container1/object1 (allowed because at least server thinks the
# container exists)
# Get to a final state
# Assert all container1 servers indicate container1 is alive and
# well with object1
# Assert account level also indicates container1 is alive and
# well with object1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
client.delete_container(self.url, self.token, container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
client.put_object(self.url, self.token, container1, 'object1', '123')
get_to_final_state() get_to_final_state()
# Our container delete never "finalized" because we started using it for cnode in cnodes:
# before the delete settled. self.assertEquals(
self.assert_(container in [c['name'] for c in [o['name'] for o in direct_client.direct_get_container(
client.get_account(self.url, self.token)[1]]) cnode, cpart, self.account, container1)[1]],
# And, so our object2 should still exist and object1's delete should ['object1'])
# have finalized. headers, containers = client.get_account(self.url, self.token)
self.assert_(object1 not in [o['name'] for o in self.assertEquals(headers['x-account-container-count'], '1')
client.get_container(self.url, self.token, container)[1]]) self.assertEquals(headers['x-account-object-count'], '1')
self.assert_(object2 in [o['name'] for o in self.assertEquals(headers['x-account-bytes-used'], '3')
client.get_container(self.url, self.token, container)[1]])
def test_second_node_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
kill(self.pids[self.port2server[cnodes[1]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
self.pids[self.port2server[cnodes[1]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[1]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
# Unfortunately, the following might pass or fail, depending on the
# position of the account server associated with the first container
# server we had killed. If the associated happens to be the first
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
self.assert_(object1 not in [o['name'] for o in
direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
def test_two_nodes_fail(self):
# Create container1
# Kill container1 servers excepting one of the primaries
# Delete container1 directly to the one primary still up
# Restart other container1 servers
# Get to a final state
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
# Assert account level also indicates container1 is gone
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
kill_server(cnodes[1]['port'], self.port2server, self.pids)
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
start_server(cnodes[1]['port'], self.port2server, self.pids)
start_server(cnp_port, self.port2server, self.pids)
get_to_final_state() get_to_final_state()
# Our container delete never "finalized" because we started using it for cnode in cnodes:
# before the delete settled. exc = None
self.assert_(container in [c['name'] for c in try:
client.get_account(self.url, self.token)[1]]) direct_client.direct_get_container(cnode, cpart, self.account,
# And, so our object2 should still exist and object1's delete should container1)
# have finalized. except client.ClientException, err:
self.assert_(object1 not in [o['name'] for o in exc = err
client.get_container(self.url, self.token, container)[1]]) self.assertEquals(exc.http_status, 404)
self.assert_(object2 in [o['name'] for o in headers, containers = client.get_account(self.url, self.token)
client.get_container(self.url, self.token, container)[1]]) self.assertEquals(headers['x-account-container-count'], '0')
self.assertEquals(headers['x-account-object-count'], '0')
def test_first_two_nodes_fail(self): self.assertEquals(headers['x-account-bytes-used'], '0')
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
for x in xrange(2):
kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
for x in xrange(2):
self.pids[self.port2server[cnodes[x]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[x]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [o['name'] for o in
direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
# the update that the object was deleted yet), the whole must 503
# (until every is synced up, then the delete would work).
exc = None
try:
client.delete_container(self.url, self.token, container)
except client.ClientException, err:
exc = err
self.assert_(exc)
self.assert_(exc.http_status, 503)
# Unfortunately, the following might pass or fail, depending on the
# position of the account server associated with the first container
# server we had killed. If the associated happens to be the first
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
# First node still doesn't know object1 was deleted yet; this is okay.
self.assert_(object1 in [o['name'] for o in
direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
def test_last_two_nodes_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
for x in (1, 2):
kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
for x in (1, 2):
self.pids[self.port2server[cnodes[x]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[x]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
# the update that the object was deleted yet), the whole must 503
# (until every is synced up, then the delete would work).
exc = None
try:
client.delete_container(self.url, self.token, container)
except client.ClientException, err:
exc = err
self.assert_(exc)
self.assert_(exc.http_status, 503)
# Unfortunately, the following might pass or fail, depending on the
# position of the account server associated with the first container
# server we had killed. If the associated happens to be the first
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
self.assert_(object1 not in [o['name'] for o in
direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
def _get_db_file_path(self, obj_dir):
files = sorted(os.listdir(obj_dir), reverse=True)
for file in files:
if file.endswith('db'):
return os.path.join(obj_dir, file)
def _get_container_db_files(self, container): def _get_container_db_files(self, container):
opart, onodes = self.container_ring.get_nodes(self.account, container) opart, onodes = self.container_ring.get_nodes(self.account, container)
@ -343,12 +121,12 @@ class TestContainerFailures(unittest.TestCase):
device = onode['device'] device = onode['device']
hash_str = hash_path(self.account, container) hash_str = hash_path(self.account, container)
server_conf = readconf('/etc/swift/container-server/%s.conf' % server_conf = readconf('/etc/swift/container-server/%s.conf' %
node_id) node_id)
devices = server_conf['app:container-server']['devices'] devices = server_conf['app:container-server']['devices']
obj_dir = '%s/%s/containers/%s/%s/%s/' % (devices, obj_dir = '%s/%s/containers/%s/%s/%s/' % (devices,
device, opart, device, opart,
hash_str[-3:], hash_str) hash_str[-3:], hash_str)
db_files.append(self._get_db_file_path(obj_dir)) db_files.append(get_db_file_path(obj_dir))
return db_files return db_files
@ -360,29 +138,31 @@ class TestContainerFailures(unittest.TestCase):
db_files = self._get_container_db_files(container) db_files = self._get_container_db_files(container)
db_conns = [] db_conns = []
for i in range(num_locks): for i in range(num_locks):
db_conn = sqlite3.connect(db_files[i]) db_conn = connect(db_files[i])
db_conn.execute('begin exclusive transaction') db_conn.execute('begin exclusive transaction')
db_conns.append(db_conn) db_conns.append(db_conn)
if catch_503: if catch_503:
exc = None
try: try:
client.delete_container(self.url, self.token, container) client.delete_container(self.url, self.token, container)
except client.ClientException, e: except client.ClientException, err:
self.assertEquals(e.http_status, 503) exc = err
self.assertEquals(exc.http_status, 503)
else: else:
client.delete_container(self.url, self.token, container) client.delete_container(self.url, self.token, container)
pool = eventlet.GreenPool() pool = GreenPool()
try: try:
with eventlet.Timeout(15): with Timeout(15):
p = pool.spawn(run_test, 1, False) pool.spawn(run_test, 1, False)
r = pool.spawn(run_test, 2, True) pool.spawn(run_test, 2, True)
q = pool.spawn(run_test, 3, True) pool.spawn(run_test, 3, True)
pool.waitall() pool.waitall()
except eventlet.Timeout, e: except Timeout, err:
raise Exception( raise Exception(
"The server did not return a 503 on container db locks, " "The server did not return a 503 on container db locks, "
"it just hangs: %s" % e) "it just hangs: %s" % err)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() main()

61
test/probe/test_object_async_update.py
View File

@ -14,56 +14,57 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import unittest
from os import kill
from signal import SIGTERM
from subprocess import Popen from subprocess import Popen
from time import sleep from unittest import main, TestCase
from uuid import uuid4 from uuid import uuid4
from swiftclient import client from swiftclient import client
from swift.common import direct_client from swift.common import direct_client
from test.probe.common import kill_nonprimary_server, kill_server, \
from test.probe.common import kill_pids, reset_environment kill_servers, reset_environment, start_server
class TestObjectAsyncUpdate(unittest.TestCase): class TestObjectAsyncUpdate(TestCase):
def setUp(self): def setUp(self):
self.pids, self.port2server, self.account_ring, self.container_ring, \ (self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token, self.account = \ self.object_ring, self.url, self.token,
reset_environment() self.account) = reset_environment()
def tearDown(self): def tearDown(self):
kill_pids(self.pids) kill_servers(self.port2server, self.pids)
def test_main(self): def test_main(self):
# Create container
# Kill container servers excepting two of the primaries
# Create container/obj
# Restart other primary server
# Assert it does not know about container/obj
# Run the object-updaters
# Assert the other primary server now knows about container/obj
container = 'container-%s' % uuid4() container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container) client.put_container(self.url, self.token, container)
apart, anodes = self.account_ring.get_nodes(self.account)
anode = anodes[0]
cpart, cnodes = self.container_ring.get_nodes(self.account, container) cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0] cnode = cnodes[0]
kill(self.pids[self.port2server[cnode['port']]], SIGTERM) kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnode['port'], self.port2server, self.pids)
obj = 'object-%s' % uuid4() obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '') client.put_object(self.url, self.token, container, obj, '')
self.pids[self.port2server[cnode['port']]] = \ start_server(cnode['port'], self.port2server, self.pids)
Popen(['swift-container-server', self.assert_(not direct_client.direct_get_container(
'/etc/swift/container-server/%d.conf' % cnode, cpart, self.account, container)[1])
((cnode['port'] - 6001) / 10)]).pid processes = []
sleep(2) for node in xrange(1, 5):
self.assert_(not direct_client.direct_get_container(cnode, cpart, processes.append(Popen(['swift-object-updater',
self.account, container)[1]) '/etc/swift/object-server/%d.conf' % node,
ps = [] 'once']))
for n in xrange(1, 5): for process in processes:
ps.append(Popen(['swift-object-updater', process.wait()
'/etc/swift/object-server/%d.conf' % n, 'once'])) objs = [o['name'] for o in direct_client.direct_get_container(
for p in ps: cnode, cpart, self.account, container)[1]]
p.wait()
objs = [o['name'] for o in direct_client.direct_get_container(cnode,
cpart, self.account, container)[1]]
self.assert_(obj in objs) self.assert_(obj in objs)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() main()

144
test/probe/test_object_failures.py
View File

@ -14,31 +14,34 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import unittest from os import listdir, unlink
import os from os.path import join as path_join
from unittest import main, TestCase
from uuid import uuid4 from uuid import uuid4
from swiftclient import client from swiftclient import client
from swift.common import direct_client from swift.common import direct_client
from swift.common.utils import hash_path, readconf from swift.common.utils import hash_path, readconf
from swift.obj.server import write_metadata, read_metadata from swift.obj.server import write_metadata, read_metadata
from test.probe.common import kill_pids, reset_environment from test.probe.common import kill_servers, reset_environment
class TestObjectFailures(unittest.TestCase): def get_data_file_path(obj_dir):
files = sorted(listdir(obj_dir), reverse=True)
for filename in files:
return path_join(obj_dir, filename)
class TestObjectFailures(TestCase):
def setUp(self): def setUp(self):
self.pids, self.port2server, self.account_ring, self.container_ring, \ (self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token, self.account = \ self.object_ring, self.url, self.token,
reset_environment() self.account) = reset_environment()
def tearDown(self): def tearDown(self):
kill_pids(self.pids) kill_servers(self.port2server, self.pids)
def _get_data_file_path(self, obj_dir):
files = sorted(os.listdir(obj_dir), reverse=True)
for file in files:
return os.path.join(obj_dir, file)
def _setup_data_file(self, container, obj, data): def _setup_data_file(self, container, obj, data):
client.put_container(self.url, self.token, container) client.put_container(self.url, self.token, container)
@ -57,7 +60,7 @@ class TestObjectFailures(unittest.TestCase):
obj_dir = '%s/%s/objects/%s/%s/%s/' % (devices, obj_dir = '%s/%s/objects/%s/%s/%s/' % (devices,
device, opart, device, opart,
hash_str[-3:], hash_str) hash_str[-3:], hash_str)
data_file = self._get_data_file_path(obj_dir) data_file = get_data_file_path(obj_dir)
return onode, opart, data_file return onode, opart, data_file
def run_quarantine(self): def run_quarantine(self):
@ -65,106 +68,102 @@ class TestObjectFailures(unittest.TestCase):
obj = 'object-%s' % uuid4() obj = 'object-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, onode, opart, data_file = self._setup_data_file(container, obj,
'VERIFY') 'VERIFY')
with open(data_file) as fp: with open(data_file) as fpointer:
metadata = read_metadata(fp) metadata = read_metadata(fpointer)
metadata['ETag'] = 'badetag' metadata['ETag'] = 'badetag'
with open(data_file) as fp: with open(data_file) as fpointer:
write_metadata(fp, metadata) write_metadata(fpointer, metadata)
odata = direct_client.direct_get_object(onode, opart, odata = direct_client.direct_get_object(
self.account, container, obj)[-1] onode, opart, self.account, container, obj)[-1]
self.assertEquals(odata, 'VERIFY') self.assertEquals(odata, 'VERIFY')
try: try:
resp = direct_client.direct_get_object(onode, opart, self.account, direct_client.direct_get_object(onode, opart, self.account,
container, obj) container, obj)
raise "Did not quarantine object" raise Exception("Did not quarantine object")
except client.ClientException, e: except client.ClientException, err:
self.assertEquals(e.http_status, 404) self.assertEquals(err.http_status, 404)
def run_quarantine_range_etag(self): def run_quarantine_range_etag(self):
container = 'container-range-%s' % uuid4() container = 'container-range-%s' % uuid4()
obj = 'object-range-%s' % uuid4() obj = 'object-range-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, onode, opart, data_file = self._setup_data_file(container, obj,
'RANGE') 'RANGE')
with open(data_file) as fp: with open(data_file) as fpointer:
metadata = read_metadata(fp) metadata = read_metadata(fpointer)
metadata['ETag'] = 'badetag' metadata['ETag'] = 'badetag'
with open(data_file) as fp: with open(data_file) as fpointer:
write_metadata(fp, metadata) write_metadata(fpointer, metadata)
for header, result in [({'Range': 'bytes=0-2'}, 'RAN'), for header, result in [({'Range': 'bytes=0-2'}, 'RAN'),
({'Range': 'bytes=1-11'}, 'ANGE'), ({'Range': 'bytes=1-11'}, 'ANGE'),
({'Range': 'bytes=0-11'}, 'RANGE')]: ({'Range': 'bytes=0-11'}, 'RANGE')]:
odata = direct_client.direct_get_object(onode, opart, odata = direct_client.direct_get_object(
self.account, container, obj, onode, opart, self.account, container, obj, headers=header)[-1]
headers=header)[-1]
self.assertEquals(odata, result) self.assertEquals(odata, result)
try: try:
resp = direct_client.direct_get_object(onode, opart, self.account, direct_client.direct_get_object(onode, opart, self.account,
container, obj) container, obj)
raise "Did not quarantine object" raise Exception("Did not quarantine object")
except client.ClientException, e: except client.ClientException, err:
self.assertEquals(e.http_status, 404) self.assertEquals(err.http_status, 404)
def run_quarantine_zero_byte_get(self): def run_quarantine_zero_byte_get(self):
container = 'container-zbyte-%s' % uuid4() container = 'container-zbyte-%s' % uuid4()
obj = 'object-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') onode, opart, data_file = self._setup_data_file(container, obj, 'DATA')
with open(data_file) as fp: with open(data_file) as fpointer:
metadata = read_metadata(fp) metadata = read_metadata(fpointer)
os.unlink(data_file) unlink(data_file)
with open(data_file, 'w') as fp: with open(data_file, 'w') as fpointer:
write_metadata(fp, metadata) write_metadata(fpointer, metadata)
try: try:
resp = direct_client.direct_get_object(onode, opart, self.account, direct_client.direct_get_object(onode, opart, self.account,
container, obj, container, obj, conn_timeout=1,
conn_timeout=1, response_timeout=1)
response_timeout=1) raise Exception("Did not quarantine object")
raise "Did not quarantine object" except client.ClientException, err:
except client.ClientException, e: self.assertEquals(err.http_status, 404)
self.assertEquals(e.http_status, 404)
def run_quarantine_zero_byte_head(self): def run_quarantine_zero_byte_head(self):
container = 'container-zbyte-%s' % uuid4() container = 'container-zbyte-%s' % uuid4()
obj = 'object-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') onode, opart, data_file = self._setup_data_file(container, obj, 'DATA')
with open(data_file) as fp: with open(data_file) as fpointer:
metadata = read_metadata(fp) metadata = read_metadata(fpointer)
os.unlink(data_file) unlink(data_file)
with open(data_file, 'w') as fp: with open(data_file, 'w') as fpointer:
write_metadata(fp, metadata) write_metadata(fpointer, metadata)
try: try:
resp = direct_client.direct_head_object(onode, opart, self.account, direct_client.direct_head_object(onode, opart, self.account,
container, obj, container, obj, conn_timeout=1,
conn_timeout=1, response_timeout=1)
response_timeout=1) raise Exception("Did not quarantine object")
raise "Did not quarantine object" except client.ClientException, err:
except client.ClientException, e: self.assertEquals(err.http_status, 404)
self.assertEquals(e.http_status, 404)
def run_quarantine_zero_byte_post(self): def run_quarantine_zero_byte_post(self):
container = 'container-zbyte-%s' % uuid4() container = 'container-zbyte-%s' % uuid4()
obj = 'object-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') onode, opart, data_file = self._setup_data_file(container, obj, 'DATA')
with open(data_file) as fp: with open(data_file) as fpointer:
metadata = read_metadata(fp) metadata = read_metadata(fpointer)
os.unlink(data_file) unlink(data_file)
with open(data_file, 'w') as fp: with open(data_file, 'w') as fpointer:
write_metadata(fp, metadata) write_metadata(fpointer, metadata)
try: try:
resp = direct_client.direct_post_object( direct_client.direct_post_object(
onode, opart, self.account, onode, opart, self.account,
container, obj, container, obj,
{'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'}, {'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'},
conn_timeout=1, conn_timeout=1,
response_timeout=1) response_timeout=1)
raise "Did not quarantine object" raise Exception("Did not quarantine object")
except client.ClientException, e: except client.ClientException, err:
self.assertEquals(e.http_status, 404) self.assertEquals(err.http_status, 404)
def test_runner(self): def test_runner(self):
self.run_quarantine() self.run_quarantine()
@ -173,5 +172,6 @@ class TestObjectFailures(unittest.TestCase):
self.run_quarantine_zero_byte_head() self.run_quarantine_zero_byte_head()
self.run_quarantine_zero_byte_post() self.run_quarantine_zero_byte_post()
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() main()

188
test/probe/test_object_handoff.py
View File

@ -14,33 +14,55 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import unittest
from os import kill
from signal import SIGTERM
from subprocess import call, Popen from subprocess import call, Popen
from time import sleep from unittest import main, TestCase
from uuid import uuid4 from uuid import uuid4
from swiftclient import client from swiftclient import client
from swift.common import direct_client from swift.common import direct_client
from test.probe.common import kill_server, kill_servers, reset_environment, \
from test.probe.common import kill_pids, reset_environment start_server
class TestObjectHandoff(unittest.TestCase): class TestObjectHandoff(TestCase):
def setUp(self): def setUp(self):
self.pids, self.port2server, self.account_ring, self.container_ring, \ (self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token, self.account = \ self.object_ring, self.url, self.token,
reset_environment() self.account) = reset_environment()
def tearDown(self): def tearDown(self):
kill_pids(self.pids) kill_servers(self.port2server, self.pids)
def test_main(self): def test_main(self):
# Create container
# Kill one container/obj primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
# Restart those other two container/obj primary servers
# Directly to handoff server assert we can get container/obj
# Assert container listing (via proxy and directly) has container/obj
# Bring the first container/obj primary server back up
# Assert that it doesn't have container/obj yet
# Run object replication, ensuring we run the handoff node last so it
# should remove its extra handoff partition
# Assert the first container/obj primary server now has container/obj
# Assert the handoff server no longer has container/obj
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
# Delete container/obj
# Assert we can't head container/obj
# Assert container/obj is not in the container listing, both indirectly
# and directly
# Restart the first container/obj primary server again
# Assert it still has container/obj
# Run object replication, ensuring we run the handoff node last so it
# should remove its extra handoff partition
# Assert primary node no longer has container/obj
container = 'container-%s' % uuid4() container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container) client.put_container(self.url, self.token, container)
apart, anodes = self.account_ring.get_nodes(self.account)
cpart, cnodes = self.container_ring.get_nodes(self.account, container) cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0] cnode = cnodes[0]
@ -48,7 +70,7 @@ class TestObjectHandoff(unittest.TestCase):
opart, onodes = self.object_ring.get_nodes( opart, onodes = self.object_ring.get_nodes(
self.account, container, obj) self.account, container, obj)
onode = onodes[0] onode = onodes[0]
kill(self.pids[self.port2server[onode['port']]], SIGTERM) kill_server(onode['port'], self.port2server, self.pids)
client.put_object(self.url, self.token, container, obj, 'VERIFY') client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1] odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY': if odata != 'VERIFY':
@ -56,22 +78,18 @@ class TestObjectHandoff(unittest.TestCase):
'returned: %s' % repr(odata)) 'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works # Kill all primaries to ensure GET handoff works
for node in onodes[1:]: for node in onodes[1:]:
kill(self.pids[self.port2server[node['port']]], SIGTERM) kill_server(node['port'], self.port2server, self.pids)
odata = client.get_object(self.url, self.token, container, obj)[-1] odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY': if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it ' raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata)) 'returned: %s' % repr(odata))
for node in onodes[1:]: for node in onodes[1:]:
self.pids[self.port2server[node['port']]] = Popen([ start_server(node['port'], self.port2server, self.pids)
'swift-object-server',
'/etc/swift/object-server/%d.conf' %
((node['port'] - 6000) / 10)]).pid
sleep(2)
# We've indirectly verified the handoff node has the object, but let's # We've indirectly verified the handoff node has the object, but let's
# directly verify it. # directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next() another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(another_onode, opart, odata = direct_client.direct_get_object(
self.account, container, obj)[-1] another_onode, opart, self.account, container, obj)[-1]
if odata != 'VERIFY': if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead ' raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata)) 'it returned: %s' % repr(odata))
@ -81,33 +99,28 @@ class TestObjectHandoff(unittest.TestCase):
raise Exception('Container listing did not know about object') raise Exception('Container listing did not know about object')
for cnode in cnodes: for cnode in cnodes:
objs = [o['name'] for o in objs = [o['name'] for o in
direct_client.direct_get_container(cnode, cpart, direct_client.direct_get_container(
self.account, container)[1]] cnode, cpart, self.account, container)[1]]
if obj not in objs: if obj not in objs:
raise Exception( raise Exception(
'Container server %s:%s did not know about object' % 'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port'])) (cnode['ip'], cnode['port']))
self.pids[self.port2server[onode['port']]] = Popen([ start_server(onode['port'], self.port2server, self.pids)
'swift-object-server', exc = None
'/etc/swift/object-server/%d.conf' %
((onode['port'] - 6000) / 10)]).pid
sleep(2)
exc = False
try: try:
direct_client.direct_get_object(onode, opart, self.account, direct_client.direct_get_object(onode, opart, self.account,
container, obj) container, obj)
except Exception: except direct_client.ClientException, err:
exc = True exc = err
if not exc: self.assertEquals(exc.http_status, 404)
raise Exception('Previously downed object server had test object')
# Run the extra server last so it'll remove its extra partition # Run the extra server last so it'll remove its extra partition
ps = [] processes = []
for n in onodes: for node in onodes:
ps.append(Popen(['swift-object-replicator', processes.append(Popen(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' % '/etc/swift/object-server/%d.conf' %
((n['port'] - 6000) / 10), 'once'])) ((node['port'] - 6000) / 10), 'once']))
for p in ps: for process in processes:
p.wait() process.wait()
call(['swift-object-replicator', call(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' % '/etc/swift/object-server/%d.conf' %
((another_onode['port'] - 6000) / 10), 'once']) ((another_onode['port'] - 6000) / 10), 'once'])
@ -116,68 +129,22 @@ class TestObjectHandoff(unittest.TestCase):
if odata != 'VERIFY': if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead ' raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata)) 'it returned: %s' % repr(odata))
exc = False exc = None
try: try:
direct_client.direct_get_object(another_onode, opart, self.account, direct_client.direct_get_object(another_onode, opart, self.account,
container, obj) container, obj)
except Exception: except direct_client.ClientException, err:
exc = True exc = err
if not exc: self.assertEquals(exc.http_status, 404)
raise Exception('Handoff object server still had test object')
# Because POST has changed to a COPY by default, POSTs will succeed on all up kill_server(onode['port'], self.port2server, self.pids)
# nodes now if at least one up node has the object.
# kill(self.pids[self.port2server[onode['port']]], SIGTERM)
# client.post_object(self.url, self.token, container, obj,
# headers={'x-object-meta-probe': 'value'})
# oheaders = client.head_object(self.url, self.token, container, obj)
# if oheaders.get('x-object-meta-probe') != 'value':
# raise Exception('Metadata incorrect, was %s' % repr(oheaders))
# exc = False
# try:
# direct_client.direct_get_object(another_onode, opart, self.account,
# container, obj)
# except Exception:
# exc = True
# if not exc:
# raise Exception('Handoff server claimed it had the object when '
# 'it should not have it')
# self.pids[self.port2server[onode['port']]] = Popen([
# 'swift-object-server',
# '/etc/swift/object-server/%d.conf' %
# ((onode['port'] - 6000) / 10)]).pid
# sleep(2)
# oheaders = direct_client.direct_get_object(onode, opart, self.account,
# container, obj)[0]
# if oheaders.get('x-object-meta-probe') == 'value':
# raise Exception('Previously downed object server had the new '
# 'metadata when it should not have it')
# # Run the extra server last so it'll remove its extra partition
# ps = []
# for n in onodes:
# ps.append(Popen(['swift-object-replicator',
# '/etc/swift/object-server/%d.conf' %
# ((n['port'] - 6000) / 10), 'once']))
# for p in ps:
# p.wait()
# call(['swift-object-replicator',
# '/etc/swift/object-server/%d.conf' %
# ((another_onode['port'] - 6000) / 10), 'once'])
# oheaders = direct_client.direct_get_object(onode, opart, self.account,
# container, obj)[0]
# if oheaders.get('x-object-meta-probe') != 'value':
# raise Exception(
# 'Previously downed object server did not have the new metadata')
kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, obj) client.delete_object(self.url, self.token, container, obj)
exc = False exc = None
try: try:
client.head_object(self.url, self.token, container, obj) client.head_object(self.url, self.token, container, obj)
except Exception: except direct_client.ClientException, err:
exc = True exc = err
if not exc: self.assertEquals(exc.http_status, 404)
raise Exception('Regular object HEAD was still successful')
objs = [o['name'] for o in objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]] client.get_container(self.url, self.token, container)[1]]
if obj in objs: if obj in objs:
@ -190,33 +157,28 @@ class TestObjectHandoff(unittest.TestCase):
raise Exception( raise Exception(
'Container server %s:%s still knew about object' % 'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port'])) (cnode['ip'], cnode['port']))
self.pids[self.port2server[onode['port']]] = Popen([ start_server(onode['port'], self.port2server, self.pids)
'swift-object-server',
'/etc/swift/object-server/%d.conf' %
((onode['port'] - 6000) / 10)]).pid
sleep(2)
direct_client.direct_get_object(onode, opart, self.account, container, direct_client.direct_get_object(onode, opart, self.account, container,
obj) obj)
# Run the extra server last so it'll remove its extra partition # Run the extra server last so it'll remove its extra partition
ps = [] processes = []
for n in onodes: for node in onodes:
ps.append(Popen(['swift-object-replicator', processes.append(Popen(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' % '/etc/swift/object-server/%d.conf' %
((n['port'] - 6000) / 10), 'once'])) ((node['port'] - 6000) / 10), 'once']))
for p in ps: for process in processes:
p.wait() process.wait()
call(['swift-object-replicator', call(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' % '/etc/swift/object-server/%d.conf' %
((another_onode['port'] - 6000) / 10), 'once']) ((another_onode['port'] - 6000) / 10), 'once'])
exc = False exc = None
try: try:
direct_client.direct_get_object(another_onode, opart, self.account, direct_client.direct_get_object(another_onode, opart, self.account,
container, obj) container, obj)
except Exception: except direct_client.ClientException, err:
exc = True exc = err
if not exc: self.assertEquals(exc.http_status, 404)
raise Exception('Handoff object server still had the object')
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() main()

130
test/probe/test_running_with_each_type_down.py
View File

@ -1,130 +0,0 @@
#!/usr/bin/python -u
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from os import kill
from signal import SIGTERM
from subprocess import Popen
from time import sleep
from swiftclient import client
from test.probe.common import get_to_final_state, kill_pids, reset_environment
class TestRunningWithEachTypeDown(unittest.TestCase):
def setUp(self):
self.pids, self.port2server, self.account_ring, self.container_ring, \
self.object_ring, self.url, self.token, self.account = \
reset_environment()
def tearDown(self):
kill_pids(self.pids)
def test_main(self):
# TODO: This test "randomly" pass or doesn't pass; need to find out why
return
apart, anodes = self.account_ring.get_nodes(self.account)
kill(self.pids[self.port2server[anodes[0]['port']]], SIGTERM)
cpart, cnodes = \
self.container_ring.get_nodes(self.account, 'container1')
kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM)
opart, onodes = \
self.object_ring.get_nodes(self.account, 'container1', 'object1')
kill(self.pids[self.port2server[onodes[0]['port']]], SIGTERM)
try:
client.put_container(self.url, self.token, 'container1')
except client.ClientException, err:
# This might 503 if one of the up container nodes tries to update
# the down account node. It'll still be saved on one node, but we
# can't assure the user.
pass
client.put_object(self.url, self.token, 'container1', 'object1', '1234')
get_to_final_state()
headers, containers = client.head_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
for container in containers:
if container['name'] == 'container1':
found1 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
found1 = False
for obj in client.get_container(self.url, self.token, 'container1')[1]:
if obj['name'] == 'object1':
found1 = True
self.assertEquals(obj['bytes'], 4)
self.assert_(found1)
self.pids[self.port2server[anodes[0]['port']]] = \
Popen(['swift-account-server',
'/etc/swift/account-server/%d.conf' %
((anodes[0]['port'] - 6002) / 10)]).pid
self.pids[self.port2server[cnodes[0]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[0]['port'] - 6001) / 10)]).pid
self.pids[self.port2server[onodes[0]['port']]] = \
Popen(['swift-object-server',
'/etc/swift/object-server/%d.conf' %
((onodes[0]['port'] - 6000) / 10)]).pid
sleep(2)
headers, containers = client.head_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
for container in containers:
if container['name'] == 'container1':
found1 = True
# The account node was previously down.
self.assert_(not found1)
found1 = False
for obj in client.get_container(self.url, self.token, 'container1')[1]:
if obj['name'] == 'object1':
found1 = True
self.assertEquals(obj['bytes'], 4)
# The first container node 404s, but the proxy will try the next node
# and succeed.
self.assert_(found1)
get_to_final_state()
headers, containers = client.head_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
for container in containers:
if container['name'] == 'container1':
found1 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
found1 = False
for obj in client.get_container(self.url, self.token, 'container1')[1]:
if obj['name'] == 'object1':
found1 = True
self.assertEquals(obj['bytes'], 4)
self.assert_(found1)
if __name__ == '__main__':
unittest.main()