6140190791
Apparently FakeSwift has always been persnickety about the order of query params of registered responses and the app making the calls. Since query params can often be converted to dictionaries the order of the encoded params should be able to change without effecting the apps dispatching of registered responses. Change-Id: Ied68c9334201a7663e9c85f3bdaa5b0643d4b6db
1781 lines
79 KiB
Python
1781 lines
79 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import json
|
|
import numbers
|
|
|
|
import mock
|
|
import operator
|
|
import time
|
|
import unittest
|
|
import socket
|
|
import os
|
|
import errno
|
|
import itertools
|
|
import random
|
|
|
|
from collections import defaultdict
|
|
from datetime import datetime
|
|
import six
|
|
from six.moves import urllib
|
|
from swift.container import reconciler
|
|
from swift.container.server import gen_resp_headers
|
|
from swift.common.direct_client import ClientException
|
|
from swift.common import swob
|
|
from swift.common.header_key_dict import HeaderKeyDict
|
|
from swift.common.utils import split_path, Timestamp, encode_timestamps
|
|
|
|
from test.unit import debug_logger, FakeRing, fake_http_connect
|
|
from test.unit.common.middleware import helpers
|
|
|
|
|
|
def timestamp_to_last_modified(timestamp):
|
|
return datetime.utcfromtimestamp(
|
|
float(Timestamp(timestamp))).strftime('%Y-%m-%dT%H:%M:%S.%f')
|
|
|
|
|
|
def container_resp_headers(**kwargs):
|
|
return HeaderKeyDict(gen_resp_headers(kwargs))
|
|
|
|
|
|
class FakeStoragePolicySwift(object):
|
|
|
|
def __init__(self):
|
|
self.storage_policy = defaultdict(helpers.FakeSwift)
|
|
self._mock_oldest_spi_map = {}
|
|
|
|
def __getattribute__(self, name):
|
|
try:
|
|
return object.__getattribute__(self, name)
|
|
except AttributeError:
|
|
return getattr(self.storage_policy[None], name)
|
|
|
|
def __call__(self, env, start_response):
|
|
method = env['REQUEST_METHOD']
|
|
path = env['PATH_INFO']
|
|
_, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4,
|
|
rest_with_last=True)
|
|
if not obj:
|
|
policy_index = None
|
|
else:
|
|
policy_index = self._mock_oldest_spi_map.get(cont, 0)
|
|
# allow backend policy override
|
|
if 'HTTP_X_BACKEND_STORAGE_POLICY_INDEX' in env:
|
|
policy_index = int(env['HTTP_X_BACKEND_STORAGE_POLICY_INDEX'])
|
|
|
|
try:
|
|
return self.storage_policy[policy_index].__call__(
|
|
env, start_response)
|
|
except KeyError:
|
|
pass
|
|
|
|
if method == 'PUT':
|
|
resp_class = swob.HTTPCreated
|
|
else:
|
|
resp_class = swob.HTTPNotFound
|
|
self.storage_policy[policy_index].register(
|
|
method, path, resp_class, {}, '')
|
|
|
|
return self.storage_policy[policy_index].__call__(
|
|
env, start_response)
|
|
|
|
|
|
class FakeInternalClient(reconciler.InternalClient):
|
|
def __init__(self, listings):
|
|
self.app = FakeStoragePolicySwift()
|
|
self.user_agent = 'fake-internal-client'
|
|
self.request_tries = 1
|
|
self.parse(listings)
|
|
|
|
def parse(self, listings):
|
|
self.accounts = defaultdict(lambda: defaultdict(list))
|
|
for item, timestamp in listings.items():
|
|
# XXX this interface is stupid
|
|
if isinstance(timestamp, tuple):
|
|
timestamp, content_type = timestamp
|
|
else:
|
|
timestamp, content_type = timestamp, 'application/x-put'
|
|
storage_policy_index, path = item
|
|
if six.PY2 and isinstance(path, six.text_type):
|
|
path = path.encode('utf-8')
|
|
account, container_name, obj_name = split_path(
|
|
path, 0, 3, rest_with_last=True)
|
|
self.accounts[account][container_name].append(
|
|
(obj_name, storage_policy_index, timestamp, content_type))
|
|
for account_name, containers in self.accounts.items():
|
|
for con in containers:
|
|
self.accounts[account_name][con].sort(key=lambda t: t[0])
|
|
for account, containers in self.accounts.items():
|
|
account_listing_data = []
|
|
account_path = '/v1/%s' % account
|
|
for container, objects in containers.items():
|
|
container_path = account_path + '/' + container
|
|
container_listing_data = []
|
|
for entry in objects:
|
|
(obj_name, storage_policy_index,
|
|
timestamp, content_type) = entry
|
|
if storage_policy_index is None and not obj_name:
|
|
# empty container
|
|
continue
|
|
obj_path = swob.str_to_wsgi(
|
|
container_path + '/' + obj_name)
|
|
ts = Timestamp(timestamp)
|
|
headers = {'X-Timestamp': ts.normal,
|
|
'X-Backend-Timestamp': ts.internal}
|
|
# register object response
|
|
self.app.storage_policy[storage_policy_index].register(
|
|
'GET', obj_path, swob.HTTPOk, headers)
|
|
self.app.storage_policy[storage_policy_index].register(
|
|
'DELETE', obj_path, swob.HTTPNoContent, {})
|
|
# container listing entry
|
|
last_modified = timestamp_to_last_modified(timestamp)
|
|
# some tests setup mock listings using floats, some use
|
|
# strings, so normalize here
|
|
if isinstance(timestamp, numbers.Number):
|
|
timestamp = '%f' % timestamp
|
|
if six.PY2:
|
|
obj_name = obj_name.decode('utf-8')
|
|
timestamp = timestamp.decode('utf-8')
|
|
obj_data = {
|
|
'bytes': 0,
|
|
# listing data is unicode
|
|
'name': obj_name,
|
|
'last_modified': last_modified,
|
|
'hash': timestamp,
|
|
'content_type': content_type,
|
|
}
|
|
container_listing_data.append(obj_data)
|
|
container_listing_data.sort(key=operator.itemgetter('name'))
|
|
# register container listing response
|
|
container_headers = {}
|
|
container_qry_string = helpers.normalize_query_string(
|
|
'?format=json&marker=&end_marker=&prefix=')
|
|
self.app.register('GET', container_path + container_qry_string,
|
|
swob.HTTPOk, container_headers,
|
|
json.dumps(container_listing_data))
|
|
if container_listing_data:
|
|
obj_name = container_listing_data[-1]['name']
|
|
# client should quote and encode marker
|
|
end_qry_string = helpers.normalize_query_string(
|
|
'?format=json&marker=%s&end_marker=&prefix=' % (
|
|
urllib.parse.quote(obj_name.encode('utf-8'))))
|
|
self.app.register('GET', container_path + end_qry_string,
|
|
swob.HTTPOk, container_headers,
|
|
json.dumps([]))
|
|
self.app.register('DELETE', container_path,
|
|
swob.HTTPConflict, {}, '')
|
|
# simple account listing entry
|
|
container_data = {'name': container}
|
|
account_listing_data.append(container_data)
|
|
# register account response
|
|
account_listing_data.sort(key=operator.itemgetter('name'))
|
|
account_headers = {}
|
|
account_qry_string = '?format=json&marker=&end_marker=&prefix='
|
|
self.app.register('GET', account_path + account_qry_string,
|
|
swob.HTTPOk, account_headers,
|
|
json.dumps(account_listing_data))
|
|
end_qry_string = '?format=json&marker=%s&end_marker=&prefix=' % (
|
|
urllib.parse.quote(account_listing_data[-1]['name']))
|
|
self.app.register('GET', account_path + end_qry_string,
|
|
swob.HTTPOk, account_headers,
|
|
json.dumps([]))
|
|
|
|
|
|
class TestReconcilerUtils(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.fake_ring = FakeRing()
|
|
reconciler.direct_get_container_policy_index.reset()
|
|
|
|
def test_parse_raw_obj(self):
|
|
got = reconciler.parse_raw_obj({
|
|
'name': "2:/AUTH_bob/con/obj",
|
|
'hash': Timestamp(2017551.49350).internal,
|
|
'last_modified': timestamp_to_last_modified(2017551.49352),
|
|
'content_type': 'application/x-delete',
|
|
})
|
|
self.assertEqual(got['q_policy_index'], 2)
|
|
self.assertEqual(got['account'], 'AUTH_bob')
|
|
self.assertEqual(got['container'], 'con')
|
|
self.assertEqual(got['obj'], 'obj')
|
|
self.assertEqual(got['q_ts'], 2017551.49350)
|
|
self.assertEqual(got['q_record'], 2017551.49352)
|
|
self.assertEqual(got['q_op'], 'DELETE')
|
|
|
|
got = reconciler.parse_raw_obj({
|
|
'name': "1:/AUTH_bob/con/obj",
|
|
'hash': Timestamp(1234.20190).internal,
|
|
'last_modified': timestamp_to_last_modified(1234.20192),
|
|
'content_type': 'application/x-put',
|
|
})
|
|
self.assertEqual(got['q_policy_index'], 1)
|
|
self.assertEqual(got['account'], 'AUTH_bob')
|
|
self.assertEqual(got['container'], 'con')
|
|
self.assertEqual(got['obj'], 'obj')
|
|
self.assertEqual(got['q_ts'], 1234.20190)
|
|
self.assertEqual(got['q_record'], 1234.20192)
|
|
self.assertEqual(got['q_op'], 'PUT')
|
|
|
|
# the 'hash' field in object listing has the raw 'created_at' value
|
|
# which could be a composite of timestamps
|
|
timestamp_str = encode_timestamps(Timestamp(1234.20190),
|
|
Timestamp(1245.20190),
|
|
Timestamp(1256.20190),
|
|
explicit=True)
|
|
got = reconciler.parse_raw_obj({
|
|
'name': "1:/AUTH_bob/con/obj",
|
|
'hash': timestamp_str,
|
|
'last_modified': timestamp_to_last_modified(1234.20192),
|
|
'content_type': 'application/x-put',
|
|
})
|
|
self.assertEqual(got['q_policy_index'], 1)
|
|
self.assertEqual(got['account'], 'AUTH_bob')
|
|
self.assertEqual(got['container'], 'con')
|
|
self.assertEqual(got['obj'], 'obj')
|
|
self.assertEqual(got['q_ts'], 1234.20190)
|
|
self.assertEqual(got['q_record'], 1234.20192)
|
|
self.assertEqual(got['q_op'], 'PUT')
|
|
|
|
# negative test
|
|
obj_info = {
|
|
'name': "1:/AUTH_bob/con/obj",
|
|
'hash': Timestamp(1234.20190).internal,
|
|
'last_modified': timestamp_to_last_modified(1234.20192),
|
|
}
|
|
self.assertRaises(ValueError, reconciler.parse_raw_obj, obj_info)
|
|
obj_info['content_type'] = 'foo'
|
|
self.assertRaises(ValueError, reconciler.parse_raw_obj, obj_info)
|
|
obj_info['content_type'] = 'appliation/x-post'
|
|
self.assertRaises(ValueError, reconciler.parse_raw_obj, obj_info)
|
|
self.assertRaises(ValueError, reconciler.parse_raw_obj,
|
|
{'name': 'bogus'})
|
|
self.assertRaises(ValueError, reconciler.parse_raw_obj,
|
|
{'name': '-1:/AUTH_test/container'})
|
|
self.assertRaises(ValueError, reconciler.parse_raw_obj,
|
|
{'name': 'asdf:/AUTH_test/c/obj'})
|
|
self.assertRaises(KeyError, reconciler.parse_raw_obj,
|
|
{'name': '0:/AUTH_test/c/obj',
|
|
'content_type': 'application/x-put'})
|
|
|
|
def test_get_container_policy_index(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=1,
|
|
),
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
]
|
|
for permutation in itertools.permutations((0, 1, 2)):
|
|
reconciler.direct_get_container_policy_index.reset()
|
|
resp_headers = [stub_resp_headers[i] for i in permutation]
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
test_values = [(info['x-storage-policy-index'],
|
|
info['x-backend-status-changed-at']) for
|
|
info in resp_headers]
|
|
self.assertEqual(oldest_spi, 0,
|
|
"oldest policy index wrong "
|
|
"for permutation %r" % test_values)
|
|
|
|
def test_get_container_policy_index_with_error(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
container_resp_headers(
|
|
status_change_at=next(ts),
|
|
storage_policy_index=2,
|
|
),
|
|
container_resp_headers(
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=1,
|
|
),
|
|
# old timestamp, but 500 should be ignored...
|
|
ClientException(
|
|
'Container Server blew up',
|
|
http_status=500, http_reason='Server Error',
|
|
http_headers=container_resp_headers(
|
|
status_changed_at=Timestamp(0).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 2)
|
|
|
|
def test_get_container_policy_index_with_socket_error(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=1,
|
|
),
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 1)
|
|
|
|
def test_get_container_policy_index_with_too_many_errors(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)),
|
|
ClientException(
|
|
'Container Server blew up',
|
|
http_status=500, http_reason='Server Error',
|
|
http_headers=container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=1,
|
|
),
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertIsNone(oldest_spi)
|
|
|
|
def test_get_container_policy_index_for_deleted(self):
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
headers = container_resp_headers(
|
|
status_changed_at=Timestamp.now().internal,
|
|
storage_policy_index=1,
|
|
)
|
|
stub_resp_headers = [
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=headers,
|
|
),
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=headers,
|
|
),
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=headers,
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 1)
|
|
|
|
def test_get_container_policy_index_for_recently_deleted(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=container_resp_headers(
|
|
put_timestamp=next(ts),
|
|
delete_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=0,
|
|
),
|
|
),
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=container_resp_headers(
|
|
put_timestamp=next(ts),
|
|
delete_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=1,
|
|
),
|
|
),
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=container_resp_headers(
|
|
put_timestamp=next(ts),
|
|
delete_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=2,
|
|
),
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 2)
|
|
|
|
def test_get_container_policy_index_for_recently_recreated(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
# old put, no recreate
|
|
container_resp_headers(
|
|
delete_timestamp=0,
|
|
put_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=0,
|
|
),
|
|
# recently deleted
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=container_resp_headers(
|
|
put_timestamp=next(ts),
|
|
delete_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=1,
|
|
),
|
|
),
|
|
# recently recreated
|
|
container_resp_headers(
|
|
delete_timestamp=next(ts),
|
|
put_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=2,
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 2)
|
|
|
|
def test_get_container_policy_index_for_recently_split_brain(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
# oldest put
|
|
container_resp_headers(
|
|
delete_timestamp=0,
|
|
put_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=0,
|
|
),
|
|
# old recreate
|
|
container_resp_headers(
|
|
delete_timestamp=next(ts),
|
|
put_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=1,
|
|
),
|
|
# recently put
|
|
container_resp_headers(
|
|
delete_timestamp=0,
|
|
put_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=2,
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 1)
|
|
|
|
def test_get_container_policy_index_cache(self):
|
|
now = time.time()
|
|
ts = itertools.count(int(now))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=1,
|
|
),
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 0)
|
|
# re-mock with errors
|
|
stub_resp_headers = [
|
|
socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)),
|
|
socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)),
|
|
socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)),
|
|
]
|
|
with mock.patch('time.time', new=lambda: now):
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
# still cached
|
|
self.assertEqual(oldest_spi, 0)
|
|
# propel time forward
|
|
the_future = now + 31
|
|
with mock.patch('time.time', new=lambda: the_future):
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
# expired
|
|
self.assertIsNone(oldest_spi)
|
|
|
|
def test_direct_delete_container_entry(self):
|
|
mock_path = 'swift.common.direct_client.http_connect'
|
|
connect_args = []
|
|
|
|
def test_connect(ipaddr, port, device, partition, method, path,
|
|
headers=None, query_string=None):
|
|
connect_args.append({
|
|
'ipaddr': ipaddr, 'port': port, 'device': device,
|
|
'partition': partition, 'method': method, 'path': path,
|
|
'headers': headers, 'query_string': query_string})
|
|
|
|
x_timestamp = Timestamp.now()
|
|
headers = {'x-timestamp': x_timestamp.internal}
|
|
fake_hc = fake_http_connect(200, 200, 200, give_connect=test_connect)
|
|
with mock.patch(mock_path, fake_hc):
|
|
reconciler.direct_delete_container_entry(
|
|
self.fake_ring, 'a', 'c', 'o', headers=headers)
|
|
|
|
self.assertEqual(len(connect_args), 3)
|
|
for args in connect_args:
|
|
self.assertEqual(args['method'], 'DELETE')
|
|
self.assertEqual(args['path'], '/a/c/o')
|
|
self.assertEqual(args['headers'].get('x-timestamp'),
|
|
headers['x-timestamp'])
|
|
|
|
def test_direct_delete_container_entry_with_errors(self):
|
|
# setup mock direct_delete
|
|
mock_path = \
|
|
'swift.container.reconciler.direct_delete_container_object'
|
|
stub_resp = [
|
|
None,
|
|
socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)),
|
|
ClientException(
|
|
'Container Server blew up',
|
|
'10.0.0.12', 6201, 'sdj', 404, 'Not Found'
|
|
),
|
|
]
|
|
mock_direct_delete = mock.MagicMock()
|
|
mock_direct_delete.side_effect = stub_resp
|
|
|
|
with mock.patch(mock_path, mock_direct_delete), \
|
|
mock.patch('eventlet.greenpool.DEBUG', False):
|
|
rv = reconciler.direct_delete_container_entry(
|
|
self.fake_ring, 'a', 'c', 'o')
|
|
self.assertIsNone(rv)
|
|
self.assertEqual(len(mock_direct_delete.mock_calls), 3)
|
|
|
|
def test_add_to_reconciler_queue(self):
|
|
mock_path = 'swift.common.direct_client.http_connect'
|
|
connect_args = []
|
|
|
|
def test_connect(ipaddr, port, device, partition, method, path,
|
|
headers=None, query_string=None):
|
|
connect_args.append({
|
|
'ipaddr': ipaddr, 'port': port, 'device': device,
|
|
'partition': partition, 'method': method, 'path': path,
|
|
'headers': headers, 'query_string': query_string})
|
|
|
|
fake_hc = fake_http_connect(200, 200, 200, give_connect=test_connect)
|
|
with mock.patch(mock_path, fake_hc):
|
|
ret = reconciler.add_to_reconciler_queue(
|
|
self.fake_ring, 'a', 'c', 'o', 17, 5948918.63946, 'DELETE')
|
|
|
|
self.assertTrue(ret)
|
|
self.assertEqual(ret, str(int(5948918.63946 // 3600 * 3600)))
|
|
self.assertEqual(len(connect_args), 3)
|
|
|
|
required_headers = ('x-content-type', 'x-etag')
|
|
|
|
for args in connect_args:
|
|
self.assertEqual(args['headers']['X-Timestamp'], '5948918.63946')
|
|
self.assertEqual(args['path'],
|
|
'/.misplaced_objects/5947200/17:/a/c/o')
|
|
self.assertEqual(args['headers']['X-Content-Type'],
|
|
'application/x-delete')
|
|
for header in required_headers:
|
|
self.assertTrue(header in args['headers'],
|
|
'%r was missing request headers %r' % (
|
|
header, args['headers']))
|
|
|
|
def test_add_to_reconciler_queue_force(self):
|
|
mock_path = 'swift.common.direct_client.http_connect'
|
|
connect_args = []
|
|
|
|
def test_connect(ipaddr, port, device, partition, method, path,
|
|
headers=None, query_string=None):
|
|
connect_args.append({
|
|
'ipaddr': ipaddr, 'port': port, 'device': device,
|
|
'partition': partition, 'method': method, 'path': path,
|
|
'headers': headers, 'query_string': query_string})
|
|
|
|
fake_hc = fake_http_connect(200, 200, 200, give_connect=test_connect)
|
|
now = time.time()
|
|
with mock.patch(mock_path, fake_hc), \
|
|
mock.patch('swift.container.reconciler.time.time',
|
|
lambda: now):
|
|
ret = reconciler.add_to_reconciler_queue(
|
|
self.fake_ring, 'a', 'c', 'o', 17, 5948918.63946, 'PUT',
|
|
force=True)
|
|
|
|
self.assertTrue(ret)
|
|
self.assertEqual(ret, str(int(5948918.63946 // 3600 * 3600)))
|
|
self.assertEqual(len(connect_args), 3)
|
|
|
|
required_headers = ('x-size', 'x-content-type')
|
|
|
|
for args in connect_args:
|
|
self.assertEqual(args['headers']['X-Timestamp'],
|
|
Timestamp(now).internal)
|
|
self.assertEqual(args['headers']['X-Etag'], '5948918.63946')
|
|
self.assertEqual(args['path'],
|
|
'/.misplaced_objects/5947200/17:/a/c/o')
|
|
for header in required_headers:
|
|
self.assertTrue(header in args['headers'],
|
|
'%r was missing request headers %r' % (
|
|
header, args['headers']))
|
|
|
|
def test_add_to_reconciler_queue_fails(self):
|
|
mock_path = 'swift.common.direct_client.http_connect'
|
|
|
|
fake_connects = [fake_http_connect(200),
|
|
fake_http_connect(200, raise_timeout_exc=True),
|
|
fake_http_connect(507)]
|
|
|
|
def fake_hc(*a, **kw):
|
|
return fake_connects.pop()(*a, **kw)
|
|
|
|
with mock.patch(mock_path, fake_hc):
|
|
ret = reconciler.add_to_reconciler_queue(
|
|
self.fake_ring, 'a', 'c', 'o', 17, 5948918.63946, 'PUT')
|
|
self.assertFalse(ret)
|
|
|
|
def test_add_to_reconciler_queue_socket_error(self):
|
|
mock_path = 'swift.common.direct_client.http_connect'
|
|
|
|
exc = socket.error(errno.ECONNREFUSED,
|
|
os.strerror(errno.ECONNREFUSED))
|
|
fake_connects = [fake_http_connect(200),
|
|
fake_http_connect(200, raise_timeout_exc=True),
|
|
fake_http_connect(500, raise_exc=exc)]
|
|
|
|
def fake_hc(*a, **kw):
|
|
return fake_connects.pop()(*a, **kw)
|
|
|
|
with mock.patch(mock_path, fake_hc):
|
|
ret = reconciler.add_to_reconciler_queue(
|
|
self.fake_ring, 'a', 'c', 'o', 17, 5948918.63946, 'DELETE')
|
|
self.assertFalse(ret)
|
|
|
|
|
|
def listing_qs(marker):
|
|
return helpers.normalize_query_string(
|
|
"?format=json&marker=%s&end_marker=&prefix=" %
|
|
urllib.parse.quote(marker.encode('utf-8')))
|
|
|
|
|
|
class TestReconciler(unittest.TestCase):
|
|
|
|
maxDiff = None
|
|
|
|
def setUp(self):
|
|
self.logger = debug_logger()
|
|
conf = {}
|
|
with mock.patch('swift.container.reconciler.InternalClient'):
|
|
self.reconciler = reconciler.ContainerReconciler(conf)
|
|
self.reconciler.logger = self.logger
|
|
self.start_interval = int(time.time() // 3600 * 3600)
|
|
self.current_container_path = '/v1/.misplaced_objects/%d' % (
|
|
self.start_interval) + listing_qs('')
|
|
|
|
def _mock_listing(self, objects):
|
|
self.reconciler.swift = FakeInternalClient(objects)
|
|
self.fake_swift = self.reconciler.swift.app
|
|
|
|
def _mock_oldest_spi(self, container_oldest_spi_map):
|
|
self.fake_swift._mock_oldest_spi_map = container_oldest_spi_map
|
|
|
|
def _run_once(self):
|
|
"""
|
|
Helper method to run the reconciler once with appropriate direct-client
|
|
mocks in place.
|
|
|
|
Returns the list of direct-deleted container entries in the format
|
|
[(acc1, con1, obj1), ...]
|
|
"""
|
|
|
|
def mock_oldest_spi(ring, account, container_name):
|
|
return self.fake_swift._mock_oldest_spi_map.get(container_name, 0)
|
|
|
|
items = {
|
|
'direct_get_container_policy_index': mock_oldest_spi,
|
|
'direct_delete_container_entry': mock.DEFAULT,
|
|
}
|
|
|
|
mock_time_iter = itertools.count(self.start_interval)
|
|
with mock.patch.multiple(reconciler, **items) as mocks:
|
|
self.mock_delete_container_entry = \
|
|
mocks['direct_delete_container_entry']
|
|
with mock.patch('time.time', lambda: next(mock_time_iter)):
|
|
self.reconciler.run_once()
|
|
|
|
return [c[1][1:4] for c in
|
|
mocks['direct_delete_container_entry'].mock_calls]
|
|
|
|
def test_invalid_queue_name(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/bogus"): 3618.84187,
|
|
})
|
|
deleted_container_entries = self._run_once()
|
|
# we try to find something useful
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('bogus'))])
|
|
# but only get the bogus record
|
|
self.assertEqual(self.reconciler.stats['invalid_record'], 1)
|
|
# and just leave it on the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 0)
|
|
self.assertFalse(deleted_container_entries)
|
|
|
|
def test_invalid_queue_name_marches_onward(self):
|
|
# there's something useful there on the queue
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/00000bogus"): 3600.0000,
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3618.84187,
|
|
(1, "/AUTH_bob/c/o1"): 3618.84187,
|
|
})
|
|
self._mock_oldest_spi({'c': 1}) # already in the right spot!
|
|
deleted_container_entries = self._run_once()
|
|
# we get all the queue entries we can
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# and one is garbage
|
|
self.assertEqual(self.reconciler.stats['invalid_record'], 1)
|
|
# but the other is workable
|
|
self.assertEqual(self.reconciler.stats['noop_object'], 1)
|
|
# so pop the queue for that one
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_queue_name_with_policy_index_delimiter_in_name(self):
|
|
q_path = '.misplaced_objects/3600'
|
|
obj_path = "AUTH_bob/c:sneaky/o1:sneaky"
|
|
# there's something useful there on the queue
|
|
self._mock_listing({
|
|
(None, "/%s/1:/%s" % (q_path, obj_path)): 3618.84187,
|
|
(1, '/%s' % obj_path): 3618.84187,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
# we find the misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/%s' % obj_path))])
|
|
# move it
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/%s' % obj_path),
|
|
('DELETE', '/v1/%s' % obj_path)])
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/%s' % obj_path),
|
|
('PUT', '/v1/%s' % obj_path)])
|
|
# clean up the source
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# we DELETE the object from the wrong place with source_ts + offset 1
|
|
# timestamp to make sure the change takes effect
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=1).internal)
|
|
# and pop the queue for that one
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries, [(
|
|
'.misplaced_objects', '3600', '1:/%s' % obj_path)])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_unable_to_direct_get_oldest_storage_policy(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3618.84187,
|
|
})
|
|
# the reconciler gets "None" if we can't quorum the container
|
|
self._mock_oldest_spi({'c': None})
|
|
deleted_container_entries = self._run_once()
|
|
# we look for misplaced objects
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# but can't really say where to go looking
|
|
self.assertEqual(self.reconciler.stats['unavailable_container'], 1)
|
|
# we don't clean up anything
|
|
self.assertEqual(self.reconciler.stats['cleanup_object'], 0)
|
|
# and we definitely should not pop_queue
|
|
self.assertFalse(deleted_container_entries)
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_object_move(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3618.84187,
|
|
(1, "/AUTH_bob/c/o1"): 3618.84187,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# moves it
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1'),
|
|
('DELETE', '/v1/AUTH_bob/c/o1')])
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'),
|
|
('PUT', '/v1/AUTH_bob/c/o1')])
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
# we PUT the object in the right place with q_ts + offset 2
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=2))
|
|
# cleans up the old
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# we DELETE the object from the wrong place with source_ts + offset 1
|
|
# timestamp to make sure the change takes effect
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=1))
|
|
# and when we're done, we pop the entry from the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_the_other_direction(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/0:/AUTH_bob/c/o1"): 3618.84187,
|
|
(0, "/AUTH_bob/c/o1"): 3618.84187,
|
|
})
|
|
self._mock_oldest_spi({'c': 1})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('0:/AUTH_bob/c/o1'))])
|
|
# moves it
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1'), # 2
|
|
('DELETE', '/v1/AUTH_bob/c/o1')]) # 4
|
|
delete_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'), # 1
|
|
('PUT', '/v1/AUTH_bob/c/o1')]) # 3
|
|
put_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
# we PUT the object in the right place with q_ts + offset 2
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=2).internal)
|
|
# cleans up the old
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# we DELETE the object from the wrong place with source_ts + offset 1
|
|
# timestamp to make sure the change takes effect
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=1).internal)
|
|
# and when we're done, we pop the entry from the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '0:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_with_unicode_and_spaces(self):
|
|
# the "name" in listings and the unicode string passed to all
|
|
# functions where we call them with (account, container, obj)
|
|
obj_name = u"AUTH_bob/c \u062a/o1 \u062a"
|
|
# anytime we talk about a call made to swift for a path
|
|
if six.PY2:
|
|
obj_path = obj_name.encode('utf-8')
|
|
else:
|
|
obj_path = obj_name.encode('utf-8').decode('latin-1')
|
|
# this mock expects unquoted unicode because it handles container
|
|
# listings as well as paths
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/%s" % obj_name): 3618.84187,
|
|
(1, "/%s" % obj_name): 3618.84187,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
# listing_qs encodes and quotes - so give it name
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/%s' % obj_name))])
|
|
# moves it
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
# these calls are to the real path
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/%s' % obj_path), # 2
|
|
('DELETE', '/v1/%s' % obj_path)]) # 4
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/%s' % obj_path), # 1
|
|
('PUT', '/v1/%s' % obj_path)]) # 3
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
# we PUT the object in the right place with q_ts + offset 2
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=2).internal)
|
|
# cleans up the old
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# we DELETE the object from the wrong place with source_ts + offset 1
|
|
# timestamp to make sure the change takes effect
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=1).internal)
|
|
self.assertEqual(
|
|
delete_headers.get('X-Backend-Storage-Policy-Index'), '1')
|
|
# and when we're done, we pop the entry from the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
# this mock received the name, it's encoded down in buffered_http
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/%s' % obj_name)])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_delete(self):
|
|
q_ts = time.time()
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): (
|
|
Timestamp(q_ts).internal, 'application/x-delete'),
|
|
# object exists in "correct" storage policy - slightly older
|
|
(0, "/AUTH_bob/c/o1"): Timestamp(q_ts - 1).internal,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
# the tombstone exists in the enqueued storage policy
|
|
self.fake_swift.storage_policy[1].register(
|
|
'GET', '/v1/AUTH_bob/c/o1', swob.HTTPNotFound,
|
|
{'X-Backend-Timestamp': Timestamp(q_ts).internal})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# delete it
|
|
self.assertEqual(self.reconciler.stats['delete_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['delete_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1'),
|
|
('DELETE', '/v1/AUTH_bob/c/o1')])
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'),
|
|
('DELETE', '/v1/AUTH_bob/c/o1')])
|
|
reconcile_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
# we DELETE the object in the right place with q_ts + offset 2
|
|
self.assertEqual(reconcile_headers.get('X-Timestamp'),
|
|
Timestamp(q_ts, offset=2).internal)
|
|
# cleans up the old
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# we DELETE the object from the wrong place with source_ts + offset 1
|
|
# timestamp to make sure the change takes effect
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(q_ts, offset=1))
|
|
# and when we're done, we pop the entry from the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_enqueued_for_the_correct_dest_noop(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3618.84187,
|
|
(1, "/AUTH_bob/c/o1"): 3618.84187,
|
|
})
|
|
self._mock_oldest_spi({'c': 1}) # already in the right spot!
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# nothing to see here
|
|
self.assertEqual(self.reconciler.stats['noop_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# so we just pop the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_src_object_newer_than_queue_entry(self):
|
|
# setup the cluster
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3600.123456,
|
|
(1, '/AUTH_bob/c/o1'): 3600.234567, # slightly newer
|
|
})
|
|
self._mock_oldest_spi({'c': 0}) # destination
|
|
# turn the crank
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# proceed with the move
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1'), # 2
|
|
('DELETE', '/v1/AUTH_bob/c/o1')]) # 4
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'), # 1
|
|
('PUT', '/v1/AUTH_bob/c/o1')]) # 3
|
|
# .. with source timestamp + offset 2
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(3600.234567, offset=2))
|
|
# src object is cleaned up
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# ... with q_ts + offset 1
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3600.123456, offset=1))
|
|
# and queue is popped
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_src_object_older_than_queue_entry(self):
|
|
# should be some sort of retry case
|
|
q_ts = time.time()
|
|
container = str(int(q_ts // 3600 * 3600))
|
|
q_path = '.misplaced_objects/%s' % container
|
|
self._mock_listing({
|
|
(None, "/%s/1:/AUTH_bob/c/o1" % q_path): q_ts,
|
|
(1, '/AUTH_bob/c/o1'): q_ts - 1, # slightly older
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', '/v1/%s' % q_path + listing_qs('')),
|
|
('GET', '/v1/%s' % q_path +
|
|
listing_qs('1:/AUTH_bob/c/o1')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(container))])
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1')])
|
|
# but no object copy is attempted
|
|
self.assertEqual(self.reconciler.stats['unavailable_source'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 0)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1')])
|
|
# src object is un-modified
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
|
|
# queue is un-changed, we'll have to retry
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 0)
|
|
self.assertEqual(deleted_container_entries, [])
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_src_object_unavailable_with_slightly_newer_tombstone(self):
|
|
# should be some sort of retry case
|
|
q_ts = float(Timestamp.now())
|
|
container = str(int(q_ts // 3600 * 3600))
|
|
q_path = '.misplaced_objects/%s' % container
|
|
self._mock_listing({
|
|
(None, "/%s/1:/AUTH_bob/c/o1" % q_path): q_ts,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
self.fake_swift.storage_policy[1].register(
|
|
'GET', '/v1/AUTH_bob/c/o1', swob.HTTPNotFound,
|
|
{'X-Backend-Timestamp': Timestamp(q_ts, offset=2).internal})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', '/v1/%s' % q_path + listing_qs('')),
|
|
('GET', '/v1/%s' % q_path +
|
|
listing_qs('1:/AUTH_bob/c/o1')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(container))])
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1')])
|
|
# but no object copy is attempted
|
|
self.assertEqual(self.reconciler.stats['unavailable_source'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 0)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1')])
|
|
# src object is un-modified
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
|
|
# queue is un-changed, we'll have to retry
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 0)
|
|
self.assertEqual(deleted_container_entries, [])
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_src_object_unavailable_server_error(self):
|
|
# should be some sort of retry case
|
|
q_ts = float(Timestamp.now())
|
|
container = str(int(q_ts // 3600 * 3600))
|
|
q_path = '.misplaced_objects/%s' % container
|
|
self._mock_listing({
|
|
(None, "/%s/1:/AUTH_bob/c/o1" % q_path): q_ts,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
self.fake_swift.storage_policy[1].register(
|
|
'GET', '/v1/AUTH_bob/c/o1', swob.HTTPServiceUnavailable, {})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', '/v1/%s' % q_path + listing_qs('')),
|
|
('GET', '/v1/%s' % q_path +
|
|
listing_qs('1:/AUTH_bob/c/o1')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(container))])
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1')])
|
|
# but no object copy is attempted
|
|
self.assertEqual(self.reconciler.stats['unavailable_source'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 0)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1')])
|
|
# src object is un-modified
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
|
|
# queue is un-changed, we'll have to retry
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 0)
|
|
self.assertEqual(deleted_container_entries, [])
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_object_move_fails_cleanup(self):
|
|
# setup the cluster
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3600.123456,
|
|
(1, '/AUTH_bob/c/o1'): 3600.123457, # slightly newer
|
|
})
|
|
self._mock_oldest_spi({'c': 0}) # destination
|
|
|
|
# make the DELETE blow up
|
|
self.fake_swift.storage_policy[1].register(
|
|
'DELETE', '/v1/AUTH_bob/c/o1', swob.HTTPServiceUnavailable, {})
|
|
# turn the crank
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# proceed with the move
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1'), # 2
|
|
('DELETE', '/v1/AUTH_bob/c/o1')]) # 4
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'), # 1
|
|
('PUT', '/v1/AUTH_bob/c/o1')]) # 3
|
|
# .. with source timestamp + offset 2
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(3600.123457, offset=2))
|
|
# we try to cleanup
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
# ... with q_ts + offset 1
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3600.12346, offset=1))
|
|
# but cleanup fails!
|
|
self.assertEqual(self.reconciler.stats['cleanup_failed'], 1)
|
|
# so the queue is not popped
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 0)
|
|
self.assertEqual(deleted_container_entries, [])
|
|
# and we'll have to retry
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_object_move_src_object_is_forever_gone(self):
|
|
# oh boy, hate to be here - this is an oldy
|
|
q_ts = self.start_interval - self.reconciler.reclaim_age - 1
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): q_ts,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1')])
|
|
# but it's gone :\
|
|
self.assertEqual(self.reconciler.stats['lost_source'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1')])
|
|
# gah, look, even if it was out there somewhere - we've been at this
|
|
# two weeks and haven't found it. We can't just keep looking forever,
|
|
# so... we're done
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
# dunno if this is helpful, but FWIW we don't throw tombstones?
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
|
|
self.assertEqual(self.reconciler.stats['success'], 1) # lol
|
|
|
|
def test_object_move_dest_already_moved(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3679.2019,
|
|
(1, "/AUTH_bob/c/o1"): 3679.2019,
|
|
(0, "/AUTH_bob/c/o1"): 3679.2019,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# we look for misplaced objects
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# but we found it already in the right place!
|
|
self.assertEqual(self.reconciler.stats['found_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1')])
|
|
# so no attempt to read the source is made, but we do cleanup
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('DELETE', '/v1/AUTH_bob/c/o1')])
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[0]
|
|
# rather we just clean up the dark matter
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3679.2019, offset=1))
|
|
# and wipe our hands of it
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_dest_object_newer_than_queue_entry(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3679.2019,
|
|
(1, "/AUTH_bob/c/o1"): 3679.2019,
|
|
(0, "/AUTH_bob/c/o1"): 3679.2019 + 1, # slightly newer
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# we look for misplaced objects...
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# but we found it already in the right place!
|
|
self.assertEqual(self.reconciler.stats['found_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1')])
|
|
# so not attempt to read is made, but we do cleanup
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 0)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('DELETE', '/v1/AUTH_bob/c/o1')])
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[0]
|
|
# rather we just clean up the dark matter
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3679.2019, offset=1))
|
|
# and since we cleaned up the old object, so this counts as done
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_dest_object_older_than_queue_entry(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/36000/1:/AUTH_bob/c/o1"): 36123.38393,
|
|
(1, "/AUTH_bob/c/o1"): 36123.38393,
|
|
(0, "/AUTH_bob/c/o1"): 36123.38393 - 1, # slightly older
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# we found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('36000')),
|
|
('GET', '/v1/.misplaced_objects/36000' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/36000' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# and since our version is *newer*, we overwrite
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1'), # 2
|
|
('DELETE', '/v1/AUTH_bob/c/o1')]) # 4
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'), # 1
|
|
('PUT', '/v1/AUTH_bob/c/o1')]) # 3
|
|
# ... with a q_ts + offset 2
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(36123.38393, offset=2))
|
|
# then clean the dark matter
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# ... with a q_ts + offset 1
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(36123.38393, offset=1))
|
|
|
|
# and pop the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '36000', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_put_fails(self):
|
|
# setup the cluster
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/36000/1:/AUTH_bob/c/o1"): 36123.383925,
|
|
(1, "/AUTH_bob/c/o1"): 36123.383925,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
|
|
# make the put to dest fail!
|
|
self.fake_swift.storage_policy[0].register(
|
|
'PUT', '/v1/AUTH_bob/c/o1', swob.HTTPServiceUnavailable, {})
|
|
|
|
# turn the crank
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# we find a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('36000')),
|
|
('GET', '/v1/.misplaced_objects/36000' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/36000' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# and try to move it, but it fails
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1')]) # 2
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'), # 1
|
|
('PUT', '/v1/AUTH_bob/c/o1')]) # 3
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
# ...with q_ts + offset 2 (20-microseconds)
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(36123.383925, offset=2))
|
|
# but it failed
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 0)
|
|
self.assertEqual(self.reconciler.stats['copy_failed'], 1)
|
|
# ... so we don't clean up the source
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
|
|
# and we don't pop the queue
|
|
self.assertEqual(deleted_container_entries, [])
|
|
self.assertEqual(self.reconciler.stats['unhandled_errors'], 0)
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_object_move_put_blows_up_crazy_town(self):
|
|
# setup the cluster
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/36000/1:/AUTH_bob/c/o1"): 36123.383925,
|
|
(1, "/AUTH_bob/c/o1"): 36123.383925,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
|
|
# make the put to dest blow up crazy town
|
|
def blow_up(*args, **kwargs):
|
|
raise Exception('kaboom!')
|
|
|
|
self.fake_swift.storage_policy[0].register(
|
|
'PUT', '/v1/AUTH_bob/c/o1', blow_up, {})
|
|
|
|
# turn the crank
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# we find a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('36000')),
|
|
('GET', '/v1/.misplaced_objects/36000' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/36000' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# and attempt to move it
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1')]) # 2
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'), # 1
|
|
('PUT', '/v1/AUTH_bob/c/o1')]) # 3
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
# ...with q_ts + offset 2 (20-microseconds)
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(36123.383925, offset=2))
|
|
# but it blows up hard
|
|
self.assertEqual(self.reconciler.stats['unhandled_error'], 1)
|
|
# so we don't cleanup
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
|
|
# and we don't pop the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 0)
|
|
self.assertEqual(deleted_container_entries, [])
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_object_move_no_such_object_no_tombstone_recent(self):
|
|
q_ts = float(Timestamp.now())
|
|
container = str(int(q_ts // 3600 * 3600))
|
|
q_path = '.misplaced_objects/%s' % container
|
|
|
|
self._mock_listing({
|
|
(None, "/%s/1:/AUTH_jeb/c/o1" % q_path): q_ts
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
|
|
deleted_container_entries = self._run_once()
|
|
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', '/v1/.misplaced_objects/%s' % container + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/%s' % container +
|
|
listing_qs('1:/AUTH_jeb/c/o1')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(container))])
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_jeb/c/o1')],
|
|
)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_jeb/c/o1')],
|
|
)
|
|
# the queue entry is recent enough that there could easily be
|
|
# tombstones on offline nodes or something, so we'll just leave it
|
|
# here and try again later
|
|
self.assertEqual(deleted_container_entries, [])
|
|
|
|
def test_object_move_no_such_object_no_tombstone_ancient(self):
|
|
queue_ts = float(Timestamp.now()) - \
|
|
self.reconciler.reclaim_age * 1.1
|
|
container = str(int(queue_ts // 3600 * 3600))
|
|
|
|
self._mock_listing({
|
|
(
|
|
None, "/.misplaced_objects/%s/1:/AUTH_jeb/c/o1" % container
|
|
): queue_ts
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
|
|
deleted_container_entries = self._run_once()
|
|
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(container)),
|
|
('GET', '/v1/.misplaced_objects/%s' % container + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/%s' % container +
|
|
listing_qs('1:/AUTH_jeb/c/o1'))])
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_jeb/c/o1')],
|
|
)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_jeb/c/o1')],
|
|
)
|
|
|
|
# the queue entry is old enough that the tombstones, if any, have
|
|
# probably been reaped, so we'll just give up
|
|
self.assertEqual(
|
|
deleted_container_entries,
|
|
[('.misplaced_objects', container, '1:/AUTH_jeb/c/o1')])
|
|
|
|
def test_delete_old_empty_queue_containers(self):
|
|
ts = time.time() - self.reconciler.reclaim_age * 1.1
|
|
container = str(int(ts // 3600 * 3600))
|
|
older_ts = ts - 3600
|
|
older_container = str(int(older_ts // 3600 * 3600))
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/%s/" % container): 0,
|
|
(None, "/.misplaced_objects/%s/something" % older_container): 0,
|
|
})
|
|
deleted_container_entries = self._run_once()
|
|
self.assertEqual(deleted_container_entries, [])
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(container)),
|
|
('GET', '/v1/.misplaced_objects/%s' % container + listing_qs('')),
|
|
('DELETE', '/v1/.misplaced_objects/%s' % container),
|
|
('GET', '/v1/.misplaced_objects/%s' % older_container +
|
|
listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/%s' % older_container +
|
|
listing_qs('something'))])
|
|
self.assertEqual(self.reconciler.stats['invalid_record'], 1)
|
|
|
|
def test_iter_over_old_containers_in_reverse(self):
|
|
step = reconciler.MISPLACED_OBJECTS_CONTAINER_DIVISOR
|
|
now = self.start_interval
|
|
containers = []
|
|
for i in range(10):
|
|
container_ts = int(now - step * i)
|
|
container_name = str(container_ts // 3600 * 3600)
|
|
containers.append(container_name)
|
|
# add some old containers too
|
|
now -= self.reconciler.reclaim_age
|
|
old_containers = []
|
|
for i in range(10):
|
|
container_ts = int(now - step * i)
|
|
container_name = str(container_ts // 3600 * 3600)
|
|
old_containers.append(container_name)
|
|
containers.sort()
|
|
old_containers.sort()
|
|
all_containers = old_containers + containers
|
|
self._mock_listing(dict((
|
|
(None, "/.misplaced_objects/%s/" % container), 0
|
|
) for container in all_containers))
|
|
deleted_container_entries = self._run_once()
|
|
self.assertEqual(deleted_container_entries, [])
|
|
last_container = all_containers[-1]
|
|
account_listing_calls = [
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(last_container)),
|
|
]
|
|
new_container_calls = [
|
|
('GET', '/v1/.misplaced_objects/%s' % container +
|
|
listing_qs('')) for container in reversed(containers)
|
|
][1:] # current_container get's skipped the second time around...
|
|
old_container_listings = [
|
|
('GET', '/v1/.misplaced_objects/%s' % container +
|
|
listing_qs('')) for container in reversed(old_containers)
|
|
]
|
|
old_container_deletes = [
|
|
('DELETE', '/v1/.misplaced_objects/%s' % container)
|
|
for container in reversed(old_containers)
|
|
]
|
|
old_container_calls = list(itertools.chain(*zip(
|
|
old_container_listings, old_container_deletes)))
|
|
self.assertEqual(self.fake_swift.calls,
|
|
[('GET', self.current_container_path)] +
|
|
account_listing_calls + new_container_calls +
|
|
old_container_calls)
|
|
|
|
def test_error_in_iter_containers(self):
|
|
self._mock_listing({})
|
|
|
|
# make the listing return an error
|
|
self.fake_swift.storage_policy[None].register(
|
|
'GET', '/v1/.misplaced_objects' + listing_qs(''),
|
|
swob.HTTPServiceUnavailable, {})
|
|
|
|
self._run_once()
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(''))])
|
|
self.assertEqual(self.reconciler.stats, {})
|
|
errors = self.reconciler.logger.get_lines_for_level('error')
|
|
self.assertEqual(errors, [
|
|
'Error listing containers in account '
|
|
'.misplaced_objects (Unexpected response: '
|
|
'503 Service Unavailable)'])
|
|
|
|
def test_unhandled_exception_in_reconcile(self):
|
|
self._mock_listing({})
|
|
|
|
# make the listing blow up
|
|
def blow_up(*args, **kwargs):
|
|
raise Exception('kaboom!')
|
|
|
|
self.fake_swift.storage_policy[None].register(
|
|
'GET', '/v1/.misplaced_objects' + listing_qs(''),
|
|
blow_up, {})
|
|
self._run_once()
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(''))])
|
|
self.assertEqual(self.reconciler.stats, {})
|
|
errors = self.reconciler.logger.get_lines_for_level('error')
|
|
self.assertEqual(errors,
|
|
['Unhandled Exception trying to reconcile: '])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|