openstack/swift
Code Issues Proposed changes

Allow InternalClient to container/object listing with prefix

Browse Source 
This patch adds 'prefix' argument to iter_containers/iter_objects
method of InternalClient.
This change will be used in general task queue feature [1].

[1]: https://review.openstack.org/#/c/517389/

Change-Id: I8c2067c07fe35681fdc9403da771f451c21136d3
This commit is contained in:
Kazuhiro MIYAHARA 2017-12-25 07:09:49 +00:00
parent 26822232b3
commit 1449532fb8
3 changed files with 79 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
swift/common/internal_client.py
View File

@ -241,7 +241,7 @@ class InternalClient(object):
return metadata
def _iter_items(
self, path, marker='', end_marker='',
self, path, marker='', end_marker='', prefix='',
acceptable_statuses=(2, HTTP_NOT_FOUND)):
"""
Returns an iterator of items from a json listing. Assumes listing has
@ -251,6 +251,7 @@ class InternalClient(object):
:param marker: Prefix of first desired item, defaults to ''.
:param end_marker: Last item returned will be 'less' than this,
defaults to ''.
:param prefix: Prefix of items
:param acceptable_statuses: List of status for valid responses,
defaults to (2, HTTP_NOT_FOUND).
@ -262,8 +263,8 @@ class InternalClient(object):
while True:
resp = self.make_request(
'GET', '%s?format=json&marker=%s&end_marker=%s' %
(path, quote(marker), quote(end_marker)),
'GET', '%s?format=json&marker=%s&end_marker=%s&prefix=%s' %
(path, quote(marker), quote(end_marker), quote(prefix)),
{}, acceptable_statuses)
if not resp.status_int == 200:
if resp.status_int >= HTTP_MULTIPLE_CHOICES:
@ -331,7 +332,7 @@ class InternalClient(object):
# account methods
def iter_containers(
self, account, marker='', end_marker='',
self, account, marker='', end_marker='', prefix='',
acceptable_statuses=(2, HTTP_NOT_FOUND)):
"""
Returns an iterator of containers dicts from an account.
@ -340,6 +341,7 @@ class InternalClient(object):
:param marker: Prefix of first desired item, defaults to ''.
:param end_marker: Last item returned will be 'less' than this,
defaults to ''.
:param prefix: Prefix of containers
:param acceptable_statuses: List of status for valid responses,
defaults to (2, HTTP_NOT_FOUND).
@ -350,7 +352,8 @@ class InternalClient(object):
"""
path = self.make_path(account)
return self._iter_items(path, marker, end_marker, acceptable_statuses)
return self._iter_items(path, marker, end_marker, prefix,
acceptable_statuses)
def get_account_info(
self, account, acceptable_statuses=(2, HTTP_NOT_FOUND)):
@ -508,7 +511,7 @@ class InternalClient(object):
return self._get_metadata(path, metadata_prefix, acceptable_statuses)
def iter_objects(
self, account, container, marker='', end_marker='',
self, account, container, marker='', end_marker='', prefix='',
acceptable_statuses=(2, HTTP_NOT_FOUND)):
"""
Returns an iterator of object dicts from a container.
@ -518,6 +521,7 @@ class InternalClient(object):
:param marker: Prefix of first desired item, defaults to ''.
:param end_marker: Last item returned will be 'less' than this,
defaults to ''.
:param prefix: Prefix of objects
:param acceptable_statuses: List of status for valid responses,
defaults to (2, HTTP_NOT_FOUND).
@ -528,7 +532,8 @@ class InternalClient(object):
"""
path = self.make_path(account, container)
return self._iter_items(path, marker, end_marker, acceptable_statuses)
return self._iter_items(path, marker, end_marker, prefix,
acceptable_statuses)
def set_container_metadata(
self, account, container, metadata, metadata_prefix='',

68
test/unit/common/test_internal_client.py
View File

@ -136,19 +136,23 @@ class SetMetadataInternalClient(internal_client.InternalClient):
class IterInternalClient(internal_client.InternalClient):
def __init__(
self, test, path, marker, end_marker, acceptable_statuses, items):
self, test, path, marker, end_marker, prefix, acceptable_statuses,
items):
self.test = test
self.path = path
self.marker = marker
self.end_marker = end_marker
self.prefix = prefix
self.acceptable_statuses = acceptable_statuses
self.items = items
def _iter_items(
self, path, marker='', end_marker='', acceptable_statuses=None):
self, path, marker='', end_marker='', prefix='',
acceptable_statuses=None):
self.test.assertEqual(self.path, path)
self.test.assertEqual(self.marker, marker)
self.test.assertEqual(self.end_marker, end_marker)
self.test.assertEqual(self.prefix, prefix)
self.test.assertEqual(self.acceptable_statuses, acceptable_statuses)
for item in self.items:
yield item
@ -667,9 +671,9 @@ class TestInternalClient(unittest.TestCase):
return self.responses.pop(0)
paths = [
'/?format=json&marker=start&end_marker=end',
'/?format=json&marker=one%C3%A9&end_marker=end',
'/?format=json&marker=two&end_marker=end',
'/?format=json&marker=start&end_marker=end&prefix=',
'/?format=json&marker=one%C3%A9&end_marker=end&prefix=',
'/?format=json&marker=two&end_marker=end&prefix=',
]
responses = [
@ -685,6 +689,49 @@ class TestInternalClient(unittest.TestCase):
self.assertEqual('one\xc3\xa9 two'.split(), items)
def test_iter_items_with_markers_and_prefix(self):
class Response(object):
def __init__(self, status_int, body):
self.status_int = status_int
self.body = body
class InternalClient(internal_client.InternalClient):
def __init__(self, test, paths, responses):
self.test = test
self.paths = paths
self.responses = responses
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
exp_path = self.paths.pop(0)
self.test.assertEqual(exp_path, path)
return self.responses.pop(0)
paths = [
'/?format=json&marker=prefixed_start&end_marker=prefixed_end'
'&prefix=prefixed_',
'/?format=json&marker=prefixed_one%C3%A9&end_marker=prefixed_end'
'&prefix=prefixed_',
'/?format=json&marker=prefixed_two&end_marker=prefixed_end'
'&prefix=prefixed_',
]
responses = [
Response(200, json.dumps([{'name': 'prefixed_one\xc3\xa9'}, ])),
Response(200, json.dumps([{'name': 'prefixed_two'}, ])),
Response(204, ''),
]
items = []
client = InternalClient(self, paths, responses)
for item in client._iter_items('/', marker='prefixed_start',
end_marker='prefixed_end',
prefix='prefixed_'):
items.append(item['name'].encode('utf8'))
self.assertEqual('prefixed_one\xc3\xa9 prefixed_two'.split(), items)
def test_iter_item_read_response_if_status_is_acceptable(self):
class Response(object):
def __init__(self, status_int, body, app_iter):
@ -779,12 +826,13 @@ class TestInternalClient(unittest.TestCase):
items = '0 1 2'.split()
marker = 'some_marker'
end_marker = 'some_end_marker'
prefix = 'some_prefix'
acceptable_statuses = 'some_status_list'
client = IterInternalClient(
self, path, marker, end_marker, acceptable_statuses, items)
self, path, marker, end_marker, prefix, acceptable_statuses, items)
ret_items = []
for container in client.iter_containers(
account, marker, end_marker,
account, marker, end_marker, prefix,
acceptable_statuses=acceptable_statuses):
ret_items.append(container)
self.assertEqual(items, ret_items)
@ -987,13 +1035,15 @@ class TestInternalClient(unittest.TestCase):
path = make_path(account, container)
marker = 'some_maker'
end_marker = 'some_end_marker'
prefix = 'some_prefix'
acceptable_statuses = 'some_status_list'
items = '0 1 2'.split()
client = IterInternalClient(
self, path, marker, end_marker, acceptable_statuses, items)
self, path, marker, end_marker, prefix, acceptable_statuses, items)
ret_items = []
for obj in client.iter_objects(
account, container, marker, end_marker, acceptable_statuses):
account, container, marker, end_marker, prefix,
acceptable_statuses):
ret_items.append(obj)
self.assertEqual(items, ret_items)

14
test/unit/container/test_reconciler.py
View File

@ -151,15 +151,17 @@ class FakeInternalClient(reconciler.InternalClient):
container_listing_data.sort(key=operator.itemgetter('name'))
# register container listing response
container_headers = {}
container_qry_string = '?format=json&marker=&end_marker='
container_qry_string = \
'?format=json&marker=&end_marker=&prefix='
self.app.register('GET', container_path + container_qry_string,
swob.HTTPOk, container_headers,
json.dumps(container_listing_data))
if container_listing_data:
obj_name = container_listing_data[-1]['name']
# client should quote and encode marker
end_qry_string = '?format=json&marker=%s&end_marker=' % (
urllib.parse.quote(obj_name.encode('utf-8')))
end_qry_string = \
'?format=json&marker=%s&end_marker=&prefix=' % (
urllib.parse.quote(obj_name.encode('utf-8')))
self.app.register('GET', container_path + end_qry_string,
swob.HTTPOk, container_headers,
json.dumps([]))
@ -171,11 +173,11 @@ class FakeInternalClient(reconciler.InternalClient):
# register account response
account_listing_data.sort(key=operator.itemgetter('name'))
account_headers = {}
account_qry_string = '?format=json&marker=&end_marker='
account_qry_string = '?format=json&marker=&end_marker=&prefix='
self.app.register('GET', account_path + account_qry_string,
swob.HTTPOk, account_headers,
json.dumps(account_listing_data))
end_qry_string = '?format=json&marker=%s&end_marker=' % (
end_qry_string = '?format=json&marker=%s&end_marker=&prefix=' % (
urllib.parse.quote(account_listing_data[-1]['name']))
self.app.register('GET', account_path + end_qry_string,
swob.HTTPOk, account_headers,
@ -704,7 +706,7 @@ class TestReconcilerUtils(unittest.TestCase):
def listing_qs(marker):
return "?format=json&marker=%s&end_marker=" % \
return "?format=json&marker=%s&end_marker=&prefix=" % \
urllib.parse.quote(marker.encode('utf-8'))