035d91dce5
Modify the 'log_name' option in the InternalClient wsgi config for the following services: container-sharder, container-reconciler, container-deleter, container-sync and object-expirer. Previously the 'log_name' value for all internal client instances sharing a single internal-client.conf file took the value configured in the conf file, or would default to 'swift'. This resulted in no distinction between logs from each internal client, and no association with the service using a particular internal client. With this change the 'log_name' value will typically be <log_route>-ic where <log_route> is the service's conf file section name. For example, 'container-sharder-ic'. Note: any 'log_name' value configured in an internal client conf file will now be ignored for these services unless the option key is preceded by 'set'. Note: by default, the logger's StatdsClient uses the log_name as its tail_prefix when composing metrics' names. However, the proxy-logging middleware overrides the tail_prefix with the hard-coded value 'proxy-server'. This change to log_name therefore does not change the statsd metric names emitted by the internal client's proxy-logging. This patch does not change the logging of the services themselves, just their internal clients. Change-Id: I844381fb9e1f3462043d27eb93e3fa188b206d05 Related-Change: Ida39ec7eb02a93cf4b2aa68fc07b7f0ae27b5439
289 lines
11 KiB
Python
289 lines
11 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
# use this file except in compliance with the License. You may obtain a copy
|
|
# of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import collections
|
|
import itertools
|
|
import json
|
|
import mock
|
|
import six
|
|
import unittest
|
|
|
|
from swift.cli import container_deleter
|
|
from swift.common import internal_client
|
|
from swift.common import swob
|
|
from swift.common import utils
|
|
|
|
AppCall = collections.namedtuple('AppCall', [
|
|
'method', 'path', 'query', 'headers', 'body'])
|
|
|
|
|
|
class FakeInternalClient(internal_client.InternalClient):
|
|
def __init__(self, responses):
|
|
self.resp_iter = iter(responses)
|
|
self.calls = []
|
|
|
|
def make_request(self, method, path, headers, acceptable_statuses,
|
|
body_file=None, params=None):
|
|
if body_file is None:
|
|
body = None
|
|
else:
|
|
body = body_file.read()
|
|
path, _, query = path.partition('?')
|
|
self.calls.append(AppCall(method, path, query, headers, body))
|
|
resp = next(self.resp_iter)
|
|
if isinstance(resp, Exception):
|
|
raise resp
|
|
return resp
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
unused_responses = [r for r in self.resp_iter]
|
|
if unused_responses:
|
|
raise Exception('Unused responses: %r' % unused_responses)
|
|
|
|
|
|
class TestContainerDeleter(unittest.TestCase):
|
|
def setUp(self):
|
|
patcher = mock.patch.object(container_deleter.time, 'time',
|
|
side_effect=itertools.count())
|
|
patcher.__enter__()
|
|
self.addCleanup(patcher.__exit__, None, None, None)
|
|
|
|
patcher = mock.patch.object(container_deleter, 'OBJECTS_PER_UPDATE', 5)
|
|
patcher.__enter__()
|
|
self.addCleanup(patcher.__exit__, None, None, None)
|
|
|
|
def test_make_delete_jobs(self):
|
|
ts = '1558463777.42739'
|
|
self.assertEqual(
|
|
container_deleter.make_delete_jobs(
|
|
'acct', 'cont', ['obj1', 'obj2'],
|
|
utils.Timestamp(ts)),
|
|
[{'name': ts + '-acct/cont/obj1',
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'},
|
|
{'name': ts + '-acct/cont/obj2',
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'}])
|
|
|
|
def test_make_delete_jobs_native_utf8(self):
|
|
ts = '1558463777.42739'
|
|
uacct = acct = u'acct-\U0001f334'
|
|
ucont = cont = u'cont-\N{SNOWMAN}'
|
|
uobj1 = obj1 = u'obj-\N{GREEK CAPITAL LETTER ALPHA}'
|
|
uobj2 = obj2 = u'/obj-\N{GREEK CAPITAL LETTER OMEGA}'
|
|
if six.PY2:
|
|
acct = acct.encode('utf8')
|
|
cont = cont.encode('utf8')
|
|
obj1 = obj1.encode('utf8')
|
|
obj2 = obj2.encode('utf8')
|
|
self.assertEqual(
|
|
container_deleter.make_delete_jobs(
|
|
acct, cont, [obj1, obj2], utils.Timestamp(ts)),
|
|
[{'name': u'%s-%s/%s/%s' % (ts, uacct, ucont, uobj1),
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'},
|
|
{'name': u'%s-%s/%s/%s' % (ts, uacct, ucont, uobj2),
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'}])
|
|
|
|
def test_make_delete_jobs_unicode_utf8(self):
|
|
ts = '1558463777.42739'
|
|
acct = u'acct-\U0001f334'
|
|
cont = u'cont-\N{SNOWMAN}'
|
|
obj1 = u'obj-\N{GREEK CAPITAL LETTER ALPHA}'
|
|
obj2 = u'obj-\N{GREEK CAPITAL LETTER OMEGA}'
|
|
self.assertEqual(
|
|
container_deleter.make_delete_jobs(
|
|
acct, cont, [obj1, obj2], utils.Timestamp(ts)),
|
|
[{'name': u'%s-%s/%s/%s' % (ts, acct, cont, obj1),
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'},
|
|
{'name': u'%s-%s/%s/%s' % (ts, acct, cont, obj2),
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'}])
|
|
|
|
def test_mark_for_deletion_empty_no_yield(self):
|
|
with FakeInternalClient([
|
|
swob.Response(json.dumps([
|
|
])),
|
|
]) as swift:
|
|
self.assertEqual(container_deleter.mark_for_deletion(
|
|
swift,
|
|
'account',
|
|
'container',
|
|
'marker',
|
|
'end',
|
|
'prefix',
|
|
timestamp=None,
|
|
yield_time=None,
|
|
), 0)
|
|
self.assertEqual(swift.calls, [
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=marker&end_marker=end&prefix=prefix',
|
|
{}, None),
|
|
])
|
|
|
|
def test_mark_for_deletion_empty_with_yield(self):
|
|
with FakeInternalClient([
|
|
swob.Response(json.dumps([
|
|
])),
|
|
]) as swift:
|
|
self.assertEqual(list(container_deleter.mark_for_deletion(
|
|
swift,
|
|
'account',
|
|
'container',
|
|
'marker',
|
|
'end',
|
|
'prefix',
|
|
timestamp=None,
|
|
yield_time=0.5,
|
|
)), [(0, None)])
|
|
self.assertEqual(swift.calls, [
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=marker&end_marker=end&prefix=prefix',
|
|
{}, None),
|
|
])
|
|
|
|
def test_mark_for_deletion_one_update_no_yield(self):
|
|
ts = '1558463777.42739'
|
|
with FakeInternalClient([
|
|
swob.Response(json.dumps([
|
|
{'name': '/obj1'},
|
|
{'name': 'obj2'},
|
|
{'name': 'obj3'},
|
|
])),
|
|
swob.Response(json.dumps([
|
|
])),
|
|
swob.Response(status=202),
|
|
]) as swift:
|
|
self.assertEqual(container_deleter.mark_for_deletion(
|
|
swift,
|
|
'account',
|
|
'container',
|
|
'',
|
|
'',
|
|
'',
|
|
timestamp=utils.Timestamp(ts),
|
|
yield_time=None,
|
|
), 3)
|
|
self.assertEqual(swift.calls, [
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=&end_marker=&prefix=', {}, None),
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=obj3&end_marker=&prefix=', {}, None),
|
|
('UPDATE', '/v1/.expiring_objects/' + ts.split('.')[0], '', {
|
|
'X-Backend-Allow-Private-Methods': 'True',
|
|
'X-Backend-Storage-Policy-Index': '0',
|
|
'X-Timestamp': ts}, mock.ANY),
|
|
])
|
|
self.assertEqual(
|
|
json.loads(swift.calls[-1].body),
|
|
container_deleter.make_delete_jobs(
|
|
'account', 'container', ['/obj1', 'obj2', 'obj3'],
|
|
utils.Timestamp(ts)
|
|
)
|
|
)
|
|
|
|
def test_mark_for_deletion_two_updates_with_yield(self):
|
|
ts = '1558463777.42739'
|
|
with FakeInternalClient([
|
|
swob.Response(json.dumps([
|
|
{'name': 'obj1'},
|
|
{'name': 'obj2'},
|
|
{'name': 'obj3'},
|
|
{'name': u'obj4-\N{SNOWMAN}'},
|
|
{'name': 'obj5'},
|
|
{'name': 'obj6'},
|
|
])),
|
|
swob.Response(status=202),
|
|
swob.Response(json.dumps([
|
|
])),
|
|
swob.Response(status=202),
|
|
]) as swift:
|
|
self.assertEqual(list(container_deleter.mark_for_deletion(
|
|
swift,
|
|
'account',
|
|
'container',
|
|
'',
|
|
'end',
|
|
'pre',
|
|
timestamp=utils.Timestamp(ts),
|
|
yield_time=0,
|
|
)), [(5, 'obj5'), (6, 'obj6'), (6, None)])
|
|
self.assertEqual(swift.calls, [
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=&end_marker=end&prefix=pre', {}, None),
|
|
('UPDATE', '/v1/.expiring_objects/' + ts.split('.')[0], '', {
|
|
'X-Backend-Allow-Private-Methods': 'True',
|
|
'X-Backend-Storage-Policy-Index': '0',
|
|
'X-Timestamp': ts}, mock.ANY),
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=obj6&end_marker=end&prefix=pre',
|
|
{}, None),
|
|
('UPDATE', '/v1/.expiring_objects/' + ts.split('.')[0], '', {
|
|
'X-Backend-Allow-Private-Methods': 'True',
|
|
'X-Backend-Storage-Policy-Index': '0',
|
|
'X-Timestamp': ts}, mock.ANY),
|
|
])
|
|
self.assertEqual(
|
|
json.loads(swift.calls[-3].body),
|
|
container_deleter.make_delete_jobs(
|
|
'account', 'container',
|
|
['obj1', 'obj2', 'obj3', u'obj4-\N{SNOWMAN}', 'obj5'],
|
|
utils.Timestamp(ts)
|
|
)
|
|
)
|
|
self.assertEqual(
|
|
json.loads(swift.calls[-1].body),
|
|
container_deleter.make_delete_jobs(
|
|
'account', 'container', ['obj6'],
|
|
utils.Timestamp(ts)
|
|
)
|
|
)
|
|
|
|
def test_init_internal_client_log_name(self):
|
|
with mock.patch(
|
|
'swift.cli.container_deleter.InternalClient') \
|
|
as mock_ic:
|
|
container_deleter.main(['a', 'c', '--request-tries', '2'])
|
|
mock_ic.assert_called_once_with(
|
|
'/etc/swift/internal-client.conf',
|
|
'Swift Container Deleter', 2,
|
|
global_conf={'log_name': 'container-deleter-ic'})
|