openstack/swift
Code Issues Proposed changes

Merge "Allow InternalClient to container/object listing with prefix"

Browse Source
This commit is contained in:
Zuul 2018-01-09 11:59:41 +00:00 committed by Gerrit Code Review
commit 3acf292699
3 changed files with 79 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
swift/common/internal_client.py
View File

@ -241,7 +241,7 @@ class InternalClient(object):
return metadata return metadata
def _iter_items( def _iter_items(
self, path, marker='', end_marker='', self, path, marker='', end_marker='', prefix='',
acceptable_statuses=(2, HTTP_NOT_FOUND)): acceptable_statuses=(2, HTTP_NOT_FOUND)):
""" """
Returns an iterator of items from a json listing. Assumes listing has Returns an iterator of items from a json listing. Assumes listing has
@ -251,6 +251,7 @@ class InternalClient(object):
:param marker: Prefix of first desired item, defaults to ''. :param marker: Prefix of first desired item, defaults to ''.
:param end_marker: Last item returned will be 'less' than this, :param end_marker: Last item returned will be 'less' than this,
defaults to ''. defaults to ''.
:param prefix: Prefix of items
:param acceptable_statuses: List of status for valid responses, :param acceptable_statuses: List of status for valid responses,
defaults to (2, HTTP_NOT_FOUND). defaults to (2, HTTP_NOT_FOUND).
@ -262,8 +263,8 @@ class InternalClient(object):
while True: while True:
resp = self.make_request( resp = self.make_request(
'GET', '%s?format=json&marker=%s&end_marker=%s' % 'GET', '%s?format=json&marker=%s&end_marker=%s&prefix=%s' %
(path, quote(marker), quote(end_marker)), (path, quote(marker), quote(end_marker), quote(prefix)),
{}, acceptable_statuses) {}, acceptable_statuses)
if not resp.status_int == 200: if not resp.status_int == 200:
if resp.status_int >= HTTP_MULTIPLE_CHOICES: if resp.status_int >= HTTP_MULTIPLE_CHOICES:
@ -331,7 +332,7 @@ class InternalClient(object):
# account methods # account methods
def iter_containers( def iter_containers(
self, account, marker='', end_marker='', self, account, marker='', end_marker='', prefix='',
acceptable_statuses=(2, HTTP_NOT_FOUND)): acceptable_statuses=(2, HTTP_NOT_FOUND)):
""" """
Returns an iterator of containers dicts from an account. Returns an iterator of containers dicts from an account.
@ -340,6 +341,7 @@ class InternalClient(object):
:param marker: Prefix of first desired item, defaults to ''. :param marker: Prefix of first desired item, defaults to ''.
:param end_marker: Last item returned will be 'less' than this, :param end_marker: Last item returned will be 'less' than this,
defaults to ''. defaults to ''.
:param prefix: Prefix of containers
:param acceptable_statuses: List of status for valid responses, :param acceptable_statuses: List of status for valid responses,
defaults to (2, HTTP_NOT_FOUND). defaults to (2, HTTP_NOT_FOUND).
@ -350,7 +352,8 @@ class InternalClient(object):
""" """
path = self.make_path(account) path = self.make_path(account)
return self._iter_items(path, marker, end_marker, acceptable_statuses) return self._iter_items(path, marker, end_marker, prefix,
acceptable_statuses)
def get_account_info( def get_account_info(
self, account, acceptable_statuses=(2, HTTP_NOT_FOUND)): self, account, acceptable_statuses=(2, HTTP_NOT_FOUND)):
@ -508,7 +511,7 @@ class InternalClient(object):
return self._get_metadata(path, metadata_prefix, acceptable_statuses) return self._get_metadata(path, metadata_prefix, acceptable_statuses)
def iter_objects( def iter_objects(
self, account, container, marker='', end_marker='', self, account, container, marker='', end_marker='', prefix='',
acceptable_statuses=(2, HTTP_NOT_FOUND)): acceptable_statuses=(2, HTTP_NOT_FOUND)):
""" """
Returns an iterator of object dicts from a container. Returns an iterator of object dicts from a container.
@ -518,6 +521,7 @@ class InternalClient(object):
:param marker: Prefix of first desired item, defaults to ''. :param marker: Prefix of first desired item, defaults to ''.
:param end_marker: Last item returned will be 'less' than this, :param end_marker: Last item returned will be 'less' than this,
defaults to ''. defaults to ''.
:param prefix: Prefix of objects
:param acceptable_statuses: List of status for valid responses, :param acceptable_statuses: List of status for valid responses,
defaults to (2, HTTP_NOT_FOUND). defaults to (2, HTTP_NOT_FOUND).
@ -528,7 +532,8 @@ class InternalClient(object):
""" """
path = self.make_path(account, container) path = self.make_path(account, container)
return self._iter_items(path, marker, end_marker, acceptable_statuses) return self._iter_items(path, marker, end_marker, prefix,
acceptable_statuses)
def set_container_metadata( def set_container_metadata(
self, account, container, metadata, metadata_prefix='', self, account, container, metadata, metadata_prefix='',

68
test/unit/common/test_internal_client.py
View File

@ -136,19 +136,23 @@ class SetMetadataInternalClient(internal_client.InternalClient):
class IterInternalClient(internal_client.InternalClient): class IterInternalClient(internal_client.InternalClient):
def __init__( def __init__(
self, test, path, marker, end_marker, acceptable_statuses, items): self, test, path, marker, end_marker, prefix, acceptable_statuses,
items):
self.test = test self.test = test
self.path = path self.path = path
self.marker = marker self.marker = marker
self.end_marker = end_marker self.end_marker = end_marker
self.prefix = prefix
self.acceptable_statuses = acceptable_statuses self.acceptable_statuses = acceptable_statuses
self.items = items self.items = items
def _iter_items( def _iter_items(
self, path, marker='', end_marker='', acceptable_statuses=None): self, path, marker='', end_marker='', prefix='',
acceptable_statuses=None):
self.test.assertEqual(self.path, path) self.test.assertEqual(self.path, path)
self.test.assertEqual(self.marker, marker) self.test.assertEqual(self.marker, marker)
self.test.assertEqual(self.end_marker, end_marker) self.test.assertEqual(self.end_marker, end_marker)
self.test.assertEqual(self.prefix, prefix)
self.test.assertEqual(self.acceptable_statuses, acceptable_statuses) self.test.assertEqual(self.acceptable_statuses, acceptable_statuses)
for item in self.items: for item in self.items:
yield item yield item
@ -667,9 +671,9 @@ class TestInternalClient(unittest.TestCase):
return self.responses.pop(0) return self.responses.pop(0)
paths = [ paths = [
'/?format=json&marker=start&end_marker=end', '/?format=json&marker=start&end_marker=end&prefix=',
'/?format=json&marker=one%C3%A9&end_marker=end', '/?format=json&marker=one%C3%A9&end_marker=end&prefix=',
'/?format=json&marker=two&end_marker=end', '/?format=json&marker=two&end_marker=end&prefix=',
] ]
responses = [ responses = [
@ -685,6 +689,49 @@ class TestInternalClient(unittest.TestCase):
self.assertEqual('one\xc3\xa9 two'.split(), items) self.assertEqual('one\xc3\xa9 two'.split(), items)
def test_iter_items_with_markers_and_prefix(self):
class Response(object):
def __init__(self, status_int, body):
self.status_int = status_int
self.body = body
class InternalClient(internal_client.InternalClient):
def __init__(self, test, paths, responses):
self.test = test
self.paths = paths
self.responses = responses
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
exp_path = self.paths.pop(0)
self.test.assertEqual(exp_path, path)
return self.responses.pop(0)
paths = [
'/?format=json&marker=prefixed_start&end_marker=prefixed_end'
'&prefix=prefixed_',
'/?format=json&marker=prefixed_one%C3%A9&end_marker=prefixed_end'
'&prefix=prefixed_',
'/?format=json&marker=prefixed_two&end_marker=prefixed_end'
'&prefix=prefixed_',
]
responses = [
Response(200, json.dumps([{'name': 'prefixed_one\xc3\xa9'}, ])),
Response(200, json.dumps([{'name': 'prefixed_two'}, ])),
Response(204, ''),
]
items = []
client = InternalClient(self, paths, responses)
for item in client._iter_items('/', marker='prefixed_start',
end_marker='prefixed_end',
prefix='prefixed_'):
items.append(item['name'].encode('utf8'))
self.assertEqual('prefixed_one\xc3\xa9 prefixed_two'.split(), items)
def test_iter_item_read_response_if_status_is_acceptable(self): def test_iter_item_read_response_if_status_is_acceptable(self):
class Response(object): class Response(object):
def __init__(self, status_int, body, app_iter): def __init__(self, status_int, body, app_iter):
@ -779,12 +826,13 @@ class TestInternalClient(unittest.TestCase):
items = '0 1 2'.split() items = '0 1 2'.split()
marker = 'some_marker' marker = 'some_marker'
end_marker = 'some_end_marker' end_marker = 'some_end_marker'
prefix = 'some_prefix'
acceptable_statuses = 'some_status_list' acceptable_statuses = 'some_status_list'
client = IterInternalClient( client = IterInternalClient(
self, path, marker, end_marker, acceptable_statuses, items) self, path, marker, end_marker, prefix, acceptable_statuses, items)
ret_items = [] ret_items = []
for container in client.iter_containers( for container in client.iter_containers(
account, marker, end_marker, account, marker, end_marker, prefix,
acceptable_statuses=acceptable_statuses): acceptable_statuses=acceptable_statuses):
ret_items.append(container) ret_items.append(container)
self.assertEqual(items, ret_items) self.assertEqual(items, ret_items)
@ -987,13 +1035,15 @@ class TestInternalClient(unittest.TestCase):
path = make_path(account, container) path = make_path(account, container)
marker = 'some_maker' marker = 'some_maker'
end_marker = 'some_end_marker' end_marker = 'some_end_marker'
prefix = 'some_prefix'
acceptable_statuses = 'some_status_list' acceptable_statuses = 'some_status_list'
items = '0 1 2'.split() items = '0 1 2'.split()
client = IterInternalClient( client = IterInternalClient(
self, path, marker, end_marker, acceptable_statuses, items) self, path, marker, end_marker, prefix, acceptable_statuses, items)
ret_items = [] ret_items = []
for obj in client.iter_objects( for obj in client.iter_objects(
account, container, marker, end_marker, acceptable_statuses): account, container, marker, end_marker, prefix,
acceptable_statuses):
ret_items.append(obj) ret_items.append(obj)
self.assertEqual(items, ret_items) self.assertEqual(items, ret_items)

12
test/unit/container/test_reconciler.py
View File

@ -151,14 +151,16 @@ class FakeInternalClient(reconciler.InternalClient):
container_listing_data.sort(key=operator.itemgetter('name')) container_listing_data.sort(key=operator.itemgetter('name'))
# register container listing response # register container listing response
container_headers = {} container_headers = {}
container_qry_string = '?format=json&marker=&end_marker=' container_qry_string = \
'?format=json&marker=&end_marker=&prefix='
self.app.register('GET', container_path + container_qry_string, self.app.register('GET', container_path + container_qry_string,
swob.HTTPOk, container_headers, swob.HTTPOk, container_headers,
json.dumps(container_listing_data)) json.dumps(container_listing_data))
if container_listing_data: if container_listing_data:
obj_name = container_listing_data[-1]['name'] obj_name = container_listing_data[-1]['name']
# client should quote and encode marker # client should quote and encode marker
end_qry_string = '?format=json&marker=%s&end_marker=' % ( end_qry_string = \
'?format=json&marker=%s&end_marker=&prefix=' % (
urllib.parse.quote(obj_name.encode('utf-8'))) urllib.parse.quote(obj_name.encode('utf-8')))
self.app.register('GET', container_path + end_qry_string, self.app.register('GET', container_path + end_qry_string,
swob.HTTPOk, container_headers, swob.HTTPOk, container_headers,
@ -171,11 +173,11 @@ class FakeInternalClient(reconciler.InternalClient):
# register account response # register account response
account_listing_data.sort(key=operator.itemgetter('name')) account_listing_data.sort(key=operator.itemgetter('name'))
account_headers = {} account_headers = {}
account_qry_string = '?format=json&marker=&end_marker=' account_qry_string = '?format=json&marker=&end_marker=&prefix='
self.app.register('GET', account_path + account_qry_string, self.app.register('GET', account_path + account_qry_string,
swob.HTTPOk, account_headers, swob.HTTPOk, account_headers,
json.dumps(account_listing_data)) json.dumps(account_listing_data))
end_qry_string = '?format=json&marker=%s&end_marker=' % ( end_qry_string = '?format=json&marker=%s&end_marker=&prefix=' % (
urllib.parse.quote(account_listing_data[-1]['name'])) urllib.parse.quote(account_listing_data[-1]['name']))
self.app.register('GET', account_path + end_qry_string, self.app.register('GET', account_path + end_qry_string,
swob.HTTPOk, account_headers, swob.HTTPOk, account_headers,
@ -704,7 +706,7 @@ class TestReconcilerUtils(unittest.TestCase):
def listing_qs(marker): def listing_qs(marker):
return "?format=json&marker=%s&end_marker=" % \ return "?format=json&marker=%s&end_marker=&prefix=" % \
urllib.parse.quote(marker.encode('utf-8')) urllib.parse.quote(marker.encode('utf-8'))