e78377624a
Add a new config option to SLO, allow_async_delete, to allow operators to opt-in to this new behavior. If their expirer queues get out of hand, they can always turn it back off. If the option is disabled, handle the delete inline; this matches the behavior of old Swift. Only allow an async delete if all segments are in the same container and none are nested SLOs, that way we only have two auth checks to make. Have s3api try to use this new mode if the data seems to have been uploaded via S3 (since it should be safe to assume that the above criteria are met). Drive-by: Allow the expirer queue and swift-container-deleter to use high-precision timestamps. Change-Id: I0bbe1ccd06776ef3e23438b40d8fb9a7c2de8921
279 lines
10 KiB
Python
279 lines
10 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
# use this file except in compliance with the License. You may obtain a copy
|
|
# of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import collections
|
|
import itertools
|
|
import json
|
|
import mock
|
|
import six
|
|
import unittest
|
|
|
|
from swift.cli import container_deleter
|
|
from swift.common import internal_client
|
|
from swift.common import swob
|
|
from swift.common import utils
|
|
|
|
AppCall = collections.namedtuple('AppCall', [
|
|
'method', 'path', 'query', 'headers', 'body'])
|
|
|
|
|
|
class FakeInternalClient(internal_client.InternalClient):
|
|
def __init__(self, responses):
|
|
self.resp_iter = iter(responses)
|
|
self.calls = []
|
|
|
|
def make_request(self, method, path, headers, acceptable_statuses,
|
|
body_file=None, params=None):
|
|
if body_file is None:
|
|
body = None
|
|
else:
|
|
body = body_file.read()
|
|
path, _, query = path.partition('?')
|
|
self.calls.append(AppCall(method, path, query, headers, body))
|
|
resp = next(self.resp_iter)
|
|
if isinstance(resp, Exception):
|
|
raise resp
|
|
return resp
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
unused_responses = [r for r in self.resp_iter]
|
|
if unused_responses:
|
|
raise Exception('Unused responses: %r' % unused_responses)
|
|
|
|
|
|
class TestContainerDeleter(unittest.TestCase):
|
|
def setUp(self):
|
|
patcher = mock.patch.object(container_deleter.time, 'time',
|
|
side_effect=itertools.count())
|
|
patcher.__enter__()
|
|
self.addCleanup(patcher.__exit__)
|
|
|
|
patcher = mock.patch.object(container_deleter, 'OBJECTS_PER_UPDATE', 5)
|
|
patcher.__enter__()
|
|
self.addCleanup(patcher.__exit__)
|
|
|
|
def test_make_delete_jobs(self):
|
|
ts = '1558463777.42739'
|
|
self.assertEqual(
|
|
container_deleter.make_delete_jobs(
|
|
'acct', 'cont', ['obj1', 'obj2'],
|
|
utils.Timestamp(ts)),
|
|
[{'name': ts + '-acct/cont/obj1',
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'},
|
|
{'name': ts + '-acct/cont/obj2',
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'}])
|
|
|
|
def test_make_delete_jobs_native_utf8(self):
|
|
ts = '1558463777.42739'
|
|
uacct = acct = u'acct-\U0001f334'
|
|
ucont = cont = u'cont-\N{SNOWMAN}'
|
|
uobj1 = obj1 = u'obj-\N{GREEK CAPITAL LETTER ALPHA}'
|
|
uobj2 = obj2 = u'/obj-\N{GREEK CAPITAL LETTER OMEGA}'
|
|
if six.PY2:
|
|
acct = acct.encode('utf8')
|
|
cont = cont.encode('utf8')
|
|
obj1 = obj1.encode('utf8')
|
|
obj2 = obj2.encode('utf8')
|
|
self.assertEqual(
|
|
container_deleter.make_delete_jobs(
|
|
acct, cont, [obj1, obj2], utils.Timestamp(ts)),
|
|
[{'name': u'%s-%s/%s/%s' % (ts, uacct, ucont, uobj1),
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'},
|
|
{'name': u'%s-%s/%s/%s' % (ts, uacct, ucont, uobj2),
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'}])
|
|
|
|
def test_make_delete_jobs_unicode_utf8(self):
|
|
ts = '1558463777.42739'
|
|
acct = u'acct-\U0001f334'
|
|
cont = u'cont-\N{SNOWMAN}'
|
|
obj1 = u'obj-\N{GREEK CAPITAL LETTER ALPHA}'
|
|
obj2 = u'obj-\N{GREEK CAPITAL LETTER OMEGA}'
|
|
self.assertEqual(
|
|
container_deleter.make_delete_jobs(
|
|
acct, cont, [obj1, obj2], utils.Timestamp(ts)),
|
|
[{'name': u'%s-%s/%s/%s' % (ts, acct, cont, obj1),
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'},
|
|
{'name': u'%s-%s/%s/%s' % (ts, acct, cont, obj2),
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'}])
|
|
|
|
def test_mark_for_deletion_empty_no_yield(self):
|
|
with FakeInternalClient([
|
|
swob.Response(json.dumps([
|
|
])),
|
|
]) as swift:
|
|
self.assertEqual(container_deleter.mark_for_deletion(
|
|
swift,
|
|
'account',
|
|
'container',
|
|
'marker',
|
|
'end',
|
|
'prefix',
|
|
timestamp=None,
|
|
yield_time=None,
|
|
), 0)
|
|
self.assertEqual(swift.calls, [
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=marker&end_marker=end&prefix=prefix',
|
|
{}, None),
|
|
])
|
|
|
|
def test_mark_for_deletion_empty_with_yield(self):
|
|
with FakeInternalClient([
|
|
swob.Response(json.dumps([
|
|
])),
|
|
]) as swift:
|
|
self.assertEqual(list(container_deleter.mark_for_deletion(
|
|
swift,
|
|
'account',
|
|
'container',
|
|
'marker',
|
|
'end',
|
|
'prefix',
|
|
timestamp=None,
|
|
yield_time=0.5,
|
|
)), [(0, None)])
|
|
self.assertEqual(swift.calls, [
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=marker&end_marker=end&prefix=prefix',
|
|
{}, None),
|
|
])
|
|
|
|
def test_mark_for_deletion_one_update_no_yield(self):
|
|
ts = '1558463777.42739'
|
|
with FakeInternalClient([
|
|
swob.Response(json.dumps([
|
|
{'name': '/obj1'},
|
|
{'name': 'obj2'},
|
|
{'name': 'obj3'},
|
|
])),
|
|
swob.Response(json.dumps([
|
|
])),
|
|
swob.Response(status=202),
|
|
]) as swift:
|
|
self.assertEqual(container_deleter.mark_for_deletion(
|
|
swift,
|
|
'account',
|
|
'container',
|
|
'',
|
|
'',
|
|
'',
|
|
timestamp=utils.Timestamp(ts),
|
|
yield_time=None,
|
|
), 3)
|
|
self.assertEqual(swift.calls, [
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=&end_marker=&prefix=', {}, None),
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=obj3&end_marker=&prefix=', {}, None),
|
|
('UPDATE', '/v1/.expiring_objects/' + ts.split('.')[0], '', {
|
|
'X-Backend-Allow-Private-Methods': 'True',
|
|
'X-Backend-Storage-Policy-Index': '0',
|
|
'X-Timestamp': ts}, mock.ANY),
|
|
])
|
|
self.assertEqual(
|
|
json.loads(swift.calls[-1].body),
|
|
container_deleter.make_delete_jobs(
|
|
'account', 'container', ['/obj1', 'obj2', 'obj3'],
|
|
utils.Timestamp(ts)
|
|
)
|
|
)
|
|
|
|
def test_mark_for_deletion_two_updates_with_yield(self):
|
|
ts = '1558463777.42739'
|
|
with FakeInternalClient([
|
|
swob.Response(json.dumps([
|
|
{'name': 'obj1'},
|
|
{'name': 'obj2'},
|
|
{'name': 'obj3'},
|
|
{'name': u'obj4-\N{SNOWMAN}'},
|
|
{'name': 'obj5'},
|
|
{'name': 'obj6'},
|
|
])),
|
|
swob.Response(status=202),
|
|
swob.Response(json.dumps([
|
|
])),
|
|
swob.Response(status=202),
|
|
]) as swift:
|
|
self.assertEqual(list(container_deleter.mark_for_deletion(
|
|
swift,
|
|
'account',
|
|
'container',
|
|
'',
|
|
'end',
|
|
'pre',
|
|
timestamp=utils.Timestamp(ts),
|
|
yield_time=0,
|
|
)), [(5, 'obj5'), (6, 'obj6'), (6, None)])
|
|
self.assertEqual(swift.calls, [
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=&end_marker=end&prefix=pre', {}, None),
|
|
('UPDATE', '/v1/.expiring_objects/' + ts.split('.')[0], '', {
|
|
'X-Backend-Allow-Private-Methods': 'True',
|
|
'X-Backend-Storage-Policy-Index': '0',
|
|
'X-Timestamp': ts}, mock.ANY),
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=obj6&end_marker=end&prefix=pre',
|
|
{}, None),
|
|
('UPDATE', '/v1/.expiring_objects/' + ts.split('.')[0], '', {
|
|
'X-Backend-Allow-Private-Methods': 'True',
|
|
'X-Backend-Storage-Policy-Index': '0',
|
|
'X-Timestamp': ts}, mock.ANY),
|
|
])
|
|
self.assertEqual(
|
|
json.loads(swift.calls[-3].body),
|
|
container_deleter.make_delete_jobs(
|
|
'account', 'container',
|
|
['obj1', 'obj2', 'obj3', u'obj4-\N{SNOWMAN}', 'obj5'],
|
|
utils.Timestamp(ts)
|
|
)
|
|
)
|
|
self.assertEqual(
|
|
json.loads(swift.calls[-1].body),
|
|
container_deleter.make_delete_jobs(
|
|
'account', 'container', ['obj6'],
|
|
utils.Timestamp(ts)
|
|
)
|
|
)
|