Merge "Fixed Bug 1187200"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
5e6ce5d5b5
@ -28,6 +28,7 @@ from eventlet.green import subprocess
|
||||
import simplejson
|
||||
|
||||
import swift.common.db
|
||||
from swift.common.direct_client import quote
|
||||
from swift.common.utils import get_logger, whataremyips, storage_directory, \
|
||||
renamer, mkdirs, lock_parent_directory, config_true_value, \
|
||||
unlink_older_than, dump_recon_cache, rsync_ip
|
||||
@ -408,12 +409,26 @@ class Replicator(Daemon):
|
||||
self.logger.debug(_('Replicating db %s'), object_file)
|
||||
self.stats['attempted'] += 1
|
||||
self.logger.increment('attempts')
|
||||
shouldbehere = True
|
||||
try:
|
||||
broker = self.brokerclass(object_file, pending_timeout=30)
|
||||
broker.reclaim(time.time() - self.reclaim_age,
|
||||
time.time() - (self.reclaim_age * 2))
|
||||
info = broker.get_replication_info()
|
||||
full_info = broker.get_info()
|
||||
bpart = self.ring.get_part(
|
||||
full_info['account'], full_info.get('container'))
|
||||
if bpart != int(partition):
|
||||
partition = bpart
|
||||
# Important to set this false here since the later check only
|
||||
# checks if it's on the proper device, not partition.
|
||||
shouldbehere = False
|
||||
name = '/' + quote(full_info['account'])
|
||||
if 'container' in full_info:
|
||||
name += '/' + quote(full_info['container'])
|
||||
self.logger.error(
|
||||
'Found %s for %s when it should be on partition %s; will '
|
||||
'replicate out and remove.' % (object_file, name, bpart))
|
||||
except (Exception, Timeout), e:
|
||||
if 'no such table' in str(e):
|
||||
self.logger.error(_('Quarantining DB %s'), object_file)
|
||||
@ -444,7 +459,8 @@ class Replicator(Daemon):
|
||||
return
|
||||
responses = []
|
||||
nodes = self.ring.get_part_nodes(int(partition))
|
||||
shouldbehere = bool([n for n in nodes if n['id'] == node_id])
|
||||
if shouldbehere:
|
||||
shouldbehere = bool([n for n in nodes if n['id'] == node_id])
|
||||
# See Footnote [1] for an explanation of the repl_nodes assignment.
|
||||
i = 0
|
||||
while i < len(nodes) and nodes[i]['id'] != node_id:
|
||||
|
||||
@ -623,6 +623,17 @@ class ObjectController(object):
|
||||
'x-trans-id': headers_in.get('x-trans-id', '-'),
|
||||
'referer': request.as_referer()})
|
||||
if op != 'DELETE':
|
||||
delete_at_container = headers_in.get('X-Delete-At-Container', None)
|
||||
if not delete_at_container:
|
||||
self.logger.warning(
|
||||
'X-Delete-At-Container header must be specified for '
|
||||
'expiring objects background %s to work properly. Making '
|
||||
'best guess as to the container name for now.' % op)
|
||||
# TODO(gholt): In a future release, change the above warning to
|
||||
# a raised exception and remove the guess code below.
|
||||
delete_at_container = str(
|
||||
delete_at / self.expiring_objects_container_divisor *
|
||||
self.expiring_objects_container_divisor)
|
||||
partition = headers_in.get('X-Delete-At-Partition', None)
|
||||
hosts = headers_in.get('X-Delete-At-Host', '')
|
||||
contdevices = headers_in.get('X-Delete-At-Device', '')
|
||||
@ -635,12 +646,21 @@ class ObjectController(object):
|
||||
headers_out['x-size'] = '0'
|
||||
headers_out['x-content-type'] = 'text/plain'
|
||||
headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'
|
||||
else:
|
||||
# DELETEs of old expiration data have no way of knowing what the
|
||||
# old X-Delete-At-Container was at the time of the initial setting
|
||||
# of the data, so a best guess is made here.
|
||||
# Worst case is a DELETE is issued now for something that doesn't
|
||||
# exist there and the original data is left where it is, where
|
||||
# it will be ignored when the expirer eventually tries to issue the
|
||||
# object DELETE later since the X-Delete-At value won't match up.
|
||||
delete_at_container = str(
|
||||
delete_at / self.expiring_objects_container_divisor *
|
||||
self.expiring_objects_container_divisor)
|
||||
|
||||
for host, contdevice in updates:
|
||||
self.async_update(
|
||||
op, self.expiring_objects_account,
|
||||
str(delete_at / self.expiring_objects_container_divisor *
|
||||
self.expiring_objects_container_divisor),
|
||||
op, self.expiring_objects_account, delete_at_container,
|
||||
'%s-%s/%s/%s' % (delete_at, account, container, obj),
|
||||
host, partition, contdevice, headers_out, objdevice)
|
||||
|
||||
|
||||
@ -597,14 +597,14 @@ class ObjectController(Controller):
|
||||
self.app.container_ring.get_nodes(
|
||||
self.app.expiring_objects_account, delete_at_container)
|
||||
else:
|
||||
delete_at_part = delete_at_nodes = None
|
||||
delete_at_container = delete_at_part = delete_at_nodes = None
|
||||
partition, nodes = self.app.object_ring.get_nodes(
|
||||
self.account_name, self.container_name, self.object_name)
|
||||
req.headers['X-Timestamp'] = normalize_timestamp(time.time())
|
||||
|
||||
headers = self._backend_requests(
|
||||
req, len(nodes), container_partition, containers,
|
||||
delete_at_part, delete_at_nodes)
|
||||
delete_at_container, delete_at_part, delete_at_nodes)
|
||||
|
||||
resp = self.make_requests(req, self.app.object_ring, partition,
|
||||
'POST', req.path_info, headers)
|
||||
@ -612,7 +612,8 @@ class ObjectController(Controller):
|
||||
|
||||
def _backend_requests(self, req, n_outgoing,
|
||||
container_partition, containers,
|
||||
delete_at_partition=None, delete_at_nodes=None):
|
||||
delete_at_container=None, delete_at_partition=None,
|
||||
delete_at_nodes=None):
|
||||
headers = [self.generate_request_headers(req, additional=req.headers)
|
||||
for _junk in range(n_outgoing)]
|
||||
|
||||
@ -633,6 +634,7 @@ class ObjectController(Controller):
|
||||
for i, node in enumerate(delete_at_nodes or []):
|
||||
i = i % len(headers)
|
||||
|
||||
headers[i]['X-Delete-At-Container'] = delete_at_container
|
||||
headers[i]['X-Delete-At-Partition'] = delete_at_partition
|
||||
headers[i]['X-Delete-At-Host'] = csv_append(
|
||||
headers[i].get('X-Delete-At-Host'),
|
||||
@ -872,7 +874,7 @@ class ObjectController(Controller):
|
||||
self.app.container_ring.get_nodes(
|
||||
self.app.expiring_objects_account, delete_at_container)
|
||||
else:
|
||||
delete_at_part = delete_at_nodes = None
|
||||
delete_at_container = delete_at_part = delete_at_nodes = None
|
||||
|
||||
node_iter = GreenthreadSafeIterator(
|
||||
self.iter_nodes(self.app.object_ring, partition))
|
||||
@ -882,7 +884,7 @@ class ObjectController(Controller):
|
||||
|
||||
outgoing_headers = self._backend_requests(
|
||||
req, len(nodes), container_partition, containers,
|
||||
delete_at_part, delete_at_nodes)
|
||||
delete_at_container, delete_at_part, delete_at_nodes)
|
||||
|
||||
for nheaders in outgoing_headers:
|
||||
# RFC2616:8.2.3 disallows 100-continue without a body
|
||||
|
||||
@ -30,6 +30,10 @@ from swift.container import server as container_server
|
||||
from test.unit import FakeLogger
|
||||
|
||||
|
||||
TEST_ACCOUNT_NAME = 'a c t'
|
||||
TEST_CONTAINER_NAME = 'c o n'
|
||||
|
||||
|
||||
def teardown_module():
|
||||
"clean up my monkey patching"
|
||||
reload(db_replicator)
|
||||
@ -47,6 +51,9 @@ class FakeRing:
|
||||
def __init__(self, path, reload_time=15, ring_name=None):
|
||||
pass
|
||||
|
||||
def get_part(self, account, container=None, obj=None):
|
||||
return 0
|
||||
|
||||
def get_part_nodes(self, part):
|
||||
return []
|
||||
|
||||
@ -72,6 +79,9 @@ class FakeRingWithNodes:
|
||||
def __init__(self, path, reload_time=15, ring_name=None):
|
||||
pass
|
||||
|
||||
def get_part(self, account, container=None, obj=None):
|
||||
return 0
|
||||
|
||||
def get_part_nodes(self, part):
|
||||
return self.devs[:3]
|
||||
|
||||
@ -139,6 +149,7 @@ class FakeBroker:
|
||||
get_repl_missing_table = False
|
||||
stub_replication_info = None
|
||||
db_type = 'container'
|
||||
info = {'account': TEST_ACCOUNT_NAME, 'container': TEST_CONTAINER_NAME}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.locked = False
|
||||
@ -178,7 +189,12 @@ class FakeBroker:
|
||||
pass
|
||||
|
||||
def get_info(self):
|
||||
pass
|
||||
return self.info
|
||||
|
||||
|
||||
class FakeAccountBroker(FakeBroker):
|
||||
db_type = 'account'
|
||||
info = {'account': TEST_ACCOUNT_NAME}
|
||||
|
||||
|
||||
class TestReplicator(db_replicator.Replicator):
|
||||
@ -453,6 +469,40 @@ class TestDBReplicator(unittest.TestCase):
|
||||
replicator._replicate_object('0', '/path/to/file', 'node_id')
|
||||
self.assertEquals(['/path/to/file'], self.delete_db_calls)
|
||||
|
||||
def test_replicate_account_out_of_place(self):
|
||||
replicator = TestReplicator({})
|
||||
replicator.ring = FakeRingWithNodes().Ring('path')
|
||||
replicator.brokerclass = FakeAccountBroker
|
||||
replicator._repl_to_node = lambda *args: True
|
||||
replicator.delete_db = self.stub_delete_db
|
||||
replicator.logger = FakeLogger()
|
||||
# Correct node_id, wrong part
|
||||
part = replicator.ring.get_part(TEST_ACCOUNT_NAME) + 1
|
||||
node_id = replicator.ring.get_part_nodes(part)[0]['id']
|
||||
replicator._replicate_object(str(part), '/path/to/file', node_id)
|
||||
self.assertEqual(['/path/to/file'], self.delete_db_calls)
|
||||
self.assertEqual(
|
||||
replicator.logger.log_dict['error'],
|
||||
[(('Found /path/to/file for /a%20c%20t when it should be on '
|
||||
'partition 0; will replicate out and remove.',), {})])
|
||||
|
||||
def test_replicate_container_out_of_place(self):
|
||||
replicator = TestReplicator({})
|
||||
replicator.ring = FakeRingWithNodes().Ring('path')
|
||||
replicator._repl_to_node = lambda *args: True
|
||||
replicator.delete_db = self.stub_delete_db
|
||||
replicator.logger = FakeLogger()
|
||||
# Correct node_id, wrong part
|
||||
part = replicator.ring.get_part(
|
||||
TEST_ACCOUNT_NAME, TEST_CONTAINER_NAME) + 1
|
||||
node_id = replicator.ring.get_part_nodes(part)[0]['id']
|
||||
replicator._replicate_object(str(part), '/path/to/file', node_id)
|
||||
self.assertEqual(['/path/to/file'], self.delete_db_calls)
|
||||
self.assertEqual(
|
||||
replicator.logger.log_dict['error'],
|
||||
[(('Found /path/to/file for /a%20c%20t/c%20o%20n when it should '
|
||||
'be on partition 0; will replicate out and remove.',), {})])
|
||||
|
||||
def test_delete_db(self):
|
||||
db_replicator.lock_parent_directory = lock_parent_directory
|
||||
replicator = TestReplicator({})
|
||||
|
||||
@ -1771,6 +1771,7 @@ class TestObjectController(unittest.TestCase):
|
||||
'X-Container-Host': '1.2.3.4:5',
|
||||
'X-Container-Device': 'sdb1',
|
||||
'X-Delete-At': 9999999999,
|
||||
'X-Delete-At-Container': '9999999960',
|
||||
'X-Delete-At-Host': "10.1.1.1:6001,10.2.2.2:6002",
|
||||
'X-Delete-At-Partition': '6237',
|
||||
'X-Delete-At-Device': 'sdp,sdq'})
|
||||
@ -2047,7 +2048,9 @@ class TestObjectController(unittest.TestCase):
|
||||
'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'},
|
||||
'sda1'])
|
||||
|
||||
def test_delete_at_update_put(self):
|
||||
def test_delete_at_update_on_put(self):
|
||||
# Test how delete_at_update works when issued a delete for old
|
||||
# expiration info after a new put with no new expiration info.
|
||||
given_args = []
|
||||
|
||||
def fake_async_update(*args):
|
||||
@ -2058,17 +2061,17 @@ class TestObjectController(unittest.TestCase):
|
||||
environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': 1,
|
||||
'X-Trans-Id': '123'})
|
||||
self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o',
|
||||
self.object_controller.delete_at_update('DELETE', 2, 'a', 'c', 'o',
|
||||
req, 'sda1')
|
||||
self.assertEquals(given_args, ['PUT', '.expiring_objects', '0',
|
||||
self.assertEquals(given_args, ['DELETE', '.expiring_objects', '0',
|
||||
'2-a/c/o', None, None, None,
|
||||
HeaderKeyDict({'x-size': '0',
|
||||
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
|
||||
'x-content-type': 'text/plain', 'x-timestamp': '1',
|
||||
HeaderKeyDict({'x-timestamp': '1',
|
||||
'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'}),
|
||||
'sda1'])
|
||||
|
||||
def test_delete_at_negative(self):
|
||||
# Test how delete_at_update works when issued a delete for old
|
||||
# expiration info after a new put with no new expiration info.
|
||||
# Test negative is reset to 0
|
||||
given_args = []
|
||||
|
||||
@ -2081,16 +2084,16 @@ class TestObjectController(unittest.TestCase):
|
||||
headers={'X-Timestamp': 1,
|
||||
'X-Trans-Id': '1234'})
|
||||
self.object_controller.delete_at_update(
|
||||
'PUT', -2, 'a', 'c', 'o', req, 'sda1')
|
||||
'DELETE', -2, 'a', 'c', 'o', req, 'sda1')
|
||||
self.assertEquals(given_args, [
|
||||
'PUT', '.expiring_objects', '0', '0-a/c/o', None, None, None,
|
||||
HeaderKeyDict({'x-size': '0',
|
||||
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
|
||||
'x-content-type': 'text/plain', 'x-timestamp': '1',
|
||||
'DELETE', '.expiring_objects', '0', '0-a/c/o', None, None, None,
|
||||
HeaderKeyDict({'x-timestamp': '1',
|
||||
'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}),
|
||||
'sda1'])
|
||||
|
||||
def test_delete_at_cap(self):
|
||||
# Test how delete_at_update works when issued a delete for old
|
||||
# expiration info after a new put with no new expiration info.
|
||||
# Test past cap is reset to cap
|
||||
given_args = []
|
||||
|
||||
@ -2103,17 +2106,18 @@ class TestObjectController(unittest.TestCase):
|
||||
headers={'X-Timestamp': 1,
|
||||
'X-Trans-Id': '1234'})
|
||||
self.object_controller.delete_at_update(
|
||||
'PUT', 12345678901, 'a', 'c', 'o', req, 'sda1')
|
||||
'DELETE', 12345678901, 'a', 'c', 'o', req, 'sda1')
|
||||
self.assertEquals(given_args, [
|
||||
'PUT', '.expiring_objects', '9999936000', '9999999999-a/c/o', None,
|
||||
None, None,
|
||||
HeaderKeyDict({'x-size': '0',
|
||||
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
|
||||
'x-content-type': 'text/plain', 'x-timestamp': '1',
|
||||
'DELETE', '.expiring_objects', '9999936000', '9999999999-a/c/o',
|
||||
None, None, None,
|
||||
HeaderKeyDict({'x-timestamp': '1',
|
||||
'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}),
|
||||
'sda1'])
|
||||
|
||||
def test_delete_at_update_put_with_info(self):
|
||||
# Keep next test,
|
||||
# test_delete_at_update_put_with_info_but_missing_container, in sync
|
||||
# with this one but just missing the X-Delete-At-Container header.
|
||||
given_args = []
|
||||
|
||||
def fake_async_update(*args):
|
||||
@ -2124,6 +2128,7 @@ class TestObjectController(unittest.TestCase):
|
||||
environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': 1,
|
||||
'X-Trans-Id': '1234',
|
||||
'X-Delete-At-Container': '0',
|
||||
'X-Delete-At-Host': '127.0.0.1:1234',
|
||||
'X-Delete-At-Partition': '3',
|
||||
'X-Delete-At-Device': 'sdc1'})
|
||||
@ -2137,6 +2142,31 @@ class TestObjectController(unittest.TestCase):
|
||||
'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}),
|
||||
'sda1'])
|
||||
|
||||
def test_delete_at_update_put_with_info_but_missing_container(self):
|
||||
# Same as previous test, test_delete_at_update_put_with_info, but just
|
||||
# missing the X-Delete-At-Container header.
|
||||
given_args = []
|
||||
|
||||
def fake_async_update(*args):
|
||||
given_args.extend(args)
|
||||
|
||||
self.object_controller.async_update = fake_async_update
|
||||
self.object_controller.logger = FakeLogger()
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': 1,
|
||||
'X-Trans-Id': '1234',
|
||||
'X-Delete-At-Host': '127.0.0.1:1234',
|
||||
'X-Delete-At-Partition': '3',
|
||||
'X-Delete-At-Device': 'sdc1'})
|
||||
self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o',
|
||||
req, 'sda1')
|
||||
self.assertEquals(
|
||||
self.object_controller.logger.log_dict['warning'],
|
||||
[(('X-Delete-At-Container header must be specified for expiring '
|
||||
'objects background PUT to work properly. Making best guess as '
|
||||
'to the container name for now.',), {})])
|
||||
|
||||
def test_delete_at_update_delete(self):
|
||||
given_args = []
|
||||
|
||||
@ -2269,9 +2299,15 @@ class TestObjectController(unittest.TestCase):
|
||||
|
||||
def test_GET_but_expired(self):
|
||||
test_time = time() + 10000
|
||||
delete_at_timestamp = int(test_time + 100)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(test_time - 2000),
|
||||
'X-Delete-At': str(int(test_time + 100)),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
req.body = 'TEST'
|
||||
@ -2287,10 +2323,16 @@ class TestObjectController(unittest.TestCase):
|
||||
try:
|
||||
t = time()
|
||||
object_server.time.time = lambda: t
|
||||
delete_at_timestamp = int(t + 1)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank('/sda1/p/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(test_time - 1000),
|
||||
'X-Delete-At': str(int(t + 1)),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
req.body = 'TEST'
|
||||
@ -2318,9 +2360,15 @@ class TestObjectController(unittest.TestCase):
|
||||
|
||||
def test_HEAD_but_expired(self):
|
||||
test_time = time() + 10000
|
||||
delete_at_timestamp = int(test_time + 100)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(test_time - 2000),
|
||||
'X-Delete-At': str(int(test_time + 100)),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
req.body = 'TEST'
|
||||
@ -2336,11 +2384,17 @@ class TestObjectController(unittest.TestCase):
|
||||
orig_time = object_server.time.time
|
||||
try:
|
||||
t = time()
|
||||
delete_at_timestamp = int(t + 1)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
object_server.time.time = lambda: t
|
||||
req = Request.blank('/sda1/p/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(test_time - 1000),
|
||||
'X-Delete-At': str(int(t + 1)),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
req.body = 'TEST'
|
||||
@ -2368,9 +2422,15 @@ class TestObjectController(unittest.TestCase):
|
||||
|
||||
def test_POST_but_expired(self):
|
||||
test_time = time() + 10000
|
||||
delete_at_timestamp = int(test_time + 100)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(test_time - 2000),
|
||||
'X-Delete-At': str(int(test_time + 100)),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
req.body = 'TEST'
|
||||
@ -2383,9 +2443,15 @@ class TestObjectController(unittest.TestCase):
|
||||
resp = self.object_controller.POST(req)
|
||||
self.assertEquals(resp.status_int, 202)
|
||||
|
||||
delete_at_timestamp = int(time() + 1)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(test_time - 1000),
|
||||
'X-Delete-At': str(int(time() + 1)),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
req.body = 'TEST'
|
||||
@ -2406,9 +2472,15 @@ class TestObjectController(unittest.TestCase):
|
||||
|
||||
def test_DELETE_but_expired(self):
|
||||
test_time = time() + 10000
|
||||
delete_at_timestamp = int(test_time + 100)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(test_time - 2000),
|
||||
'X-Delete-At': str(int(test_time + 100)),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
req.body = 'TEST'
|
||||
@ -2443,9 +2515,15 @@ class TestObjectController(unittest.TestCase):
|
||||
resp = self.object_controller.DELETE(req)
|
||||
self.assertEquals(resp.status_int, 204)
|
||||
|
||||
delete_at_timestamp = int(test_time - 1)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(test_time - 97),
|
||||
'X-Delete-At': str(int(test_time - 1)),
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
req.body = 'TEST'
|
||||
@ -2465,10 +2543,15 @@ class TestObjectController(unittest.TestCase):
|
||||
resp = self.object_controller.DELETE(req)
|
||||
self.assertEquals(resp.status_int, 204)
|
||||
|
||||
delete_at_timestamp = str(int(test_time - 1))
|
||||
delete_at_timestamp = int(test_time - 1)
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': normalize_timestamp(test_time - 94),
|
||||
'X-Delete-At': delete_at_timestamp,
|
||||
'X-Delete-At': str(delete_at_timestamp),
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream'})
|
||||
req.body = 'TEST'
|
||||
@ -2498,12 +2581,17 @@ class TestObjectController(unittest.TestCase):
|
||||
self.object_controller.delete_at_update = fake_delete_at_update
|
||||
|
||||
timestamp1 = normalize_timestamp(time())
|
||||
delete_at_timestamp1 = str(int(time() + 1000))
|
||||
delete_at_timestamp1 = int(time() + 1000)
|
||||
delete_at_container1 = str(
|
||||
delete_at_timestamp1 /
|
||||
self.object_controller.expiring_objects_container_divisor *
|
||||
self.object_controller.expiring_objects_container_divisor)
|
||||
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'X-Timestamp': timestamp1,
|
||||
'Content-Length': '4',
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'X-Delete-At': delete_at_timestamp1})
|
||||
'X-Delete-At': str(delete_at_timestamp1),
|
||||
'X-Delete-At-Container': delete_at_container1})
|
||||
req.body = 'TEST'
|
||||
resp = self.object_controller.PUT(req)
|
||||
self.assertEquals(resp.status_int, 201)
|
||||
|
||||
@ -58,6 +58,7 @@ from swift.common.swob import Request, Response, HTTPNotFound, \
|
||||
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
|
||||
|
||||
|
||||
STATIC_TIME = time.time()
|
||||
_request_instances = 0
|
||||
|
||||
|
||||
@ -2844,6 +2845,7 @@ class TestObjectController(unittest.TestCase):
|
||||
self.assertTrue('X-Delete-At-Host' in given_headers)
|
||||
self.assertTrue('X-Delete-At-Device' in given_headers)
|
||||
self.assertTrue('X-Delete-At-Partition' in given_headers)
|
||||
self.assertTrue('X-Delete-At-Container' in given_headers)
|
||||
|
||||
def test_chunked_put(self):
|
||||
|
||||
@ -4102,6 +4104,7 @@ class TestObjectController(unittest.TestCase):
|
||||
self.assertTrue('X-Delete-At-Host' in given_headers)
|
||||
self.assertTrue('X-Delete-At-Device' in given_headers)
|
||||
self.assertTrue('X-Delete-At-Partition' in given_headers)
|
||||
self.assertTrue('X-Delete-At-Container' in given_headers)
|
||||
|
||||
t = str(int(time.time() + 100)) + '.1'
|
||||
req = Request.blank('/a/c/o', {},
|
||||
@ -4197,6 +4200,7 @@ class TestObjectController(unittest.TestCase):
|
||||
self.assertTrue('X-Delete-At-Host' in given_headers)
|
||||
self.assertTrue('X-Delete-At-Device' in given_headers)
|
||||
self.assertTrue('X-Delete-At-Partition' in given_headers)
|
||||
self.assertTrue('X-Delete-At-Container' in given_headers)
|
||||
|
||||
t = str(int(time.time() + 100)) + '.1'
|
||||
req = Request.blank('/a/c/o', {},
|
||||
@ -4537,54 +4541,72 @@ class TestObjectController(unittest.TestCase):
|
||||
'X-Container-Partition': '1',
|
||||
'X-Container-Device': 'sdc'}])
|
||||
|
||||
@mock.patch('time.time', new=lambda: STATIC_TIME)
|
||||
def test_PUT_x_delete_at_with_fewer_container_replicas(self):
|
||||
self.app.container_ring.set_replicas(2)
|
||||
|
||||
delete_at_timestamp = int(time.time()) + 100000
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.app.expiring_objects_container_divisor *
|
||||
self.app.expiring_objects_container_divisor)
|
||||
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'Content-Type': 'application/stuff',
|
||||
'Content-Length': '0',
|
||||
'X-Delete-At': int(time.time()) + 100000})
|
||||
'X-Delete-At': str(delete_at_timestamp)})
|
||||
controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
|
||||
seen_headers = self._gather_x_container_headers(
|
||||
controller.PUT, req,
|
||||
200, 200, 201, 201, 201, # HEAD HEAD PUT PUT PUT
|
||||
header_list=('X-Delete-At-Host', 'X-Delete-At-Device',
|
||||
'X-Delete-At-Partition'))
|
||||
'X-Delete-At-Partition', 'X-Delete-At-Container'))
|
||||
|
||||
self.assertEqual(seen_headers, [
|
||||
{'X-Delete-At-Host': '10.0.0.0:1000',
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'X-Delete-At-Partition': '1',
|
||||
'X-Delete-At-Device': 'sda'},
|
||||
{'X-Delete-At-Host': '10.0.0.1:1001',
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'X-Delete-At-Partition': '1',
|
||||
'X-Delete-At-Device': 'sdb'},
|
||||
{'X-Delete-At-Host': None,
|
||||
'X-Delete-At-Container': None,
|
||||
'X-Delete-At-Partition': None,
|
||||
'X-Delete-At-Device': None}])
|
||||
|
||||
@mock.patch('time.time', new=lambda: STATIC_TIME)
|
||||
def test_PUT_x_delete_at_with_more_container_replicas(self):
|
||||
self.app.container_ring.set_replicas(4)
|
||||
self.app.expiring_objects_account = 'expires'
|
||||
self.app.expiring_objects_container_divisor = 60
|
||||
|
||||
delete_at_timestamp = int(time.time()) + 100000
|
||||
delete_at_container = str(
|
||||
delete_at_timestamp /
|
||||
self.app.expiring_objects_container_divisor *
|
||||
self.app.expiring_objects_container_divisor)
|
||||
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'Content-Type': 'application/stuff',
|
||||
'Content-Length': 0,
|
||||
'X-Delete-At': int(time.time()) + 100000})
|
||||
'X-Delete-At': str(delete_at_timestamp)})
|
||||
controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
|
||||
seen_headers = self._gather_x_container_headers(
|
||||
controller.PUT, req,
|
||||
200, 200, 201, 201, 201, # HEAD HEAD PUT PUT PUT
|
||||
header_list=('X-Delete-At-Host', 'X-Delete-At-Device',
|
||||
'X-Delete-At-Partition'))
|
||||
'X-Delete-At-Partition', 'X-Delete-At-Container'))
|
||||
self.assertEqual(seen_headers, [
|
||||
{'X-Delete-At-Host': '10.0.0.0:1000,10.0.0.3:1003',
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'X-Delete-At-Partition': '1',
|
||||
'X-Delete-At-Device': 'sda,sdd'},
|
||||
{'X-Delete-At-Host': '10.0.0.1:1001',
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'X-Delete-At-Partition': '1',
|
||||
'X-Delete-At-Device': 'sdb'},
|
||||
{'X-Delete-At-Host': '10.0.0.2:1002',
|
||||
'X-Delete-At-Container': delete_at_container,
|
||||
'X-Delete-At-Partition': '1',
|
||||
'X-Delete-At-Device': 'sdc'}])
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user