added end_marker query parameter for container and object listings

This commit is contained in:
John Dickinson 2010-11-18 21:42:49 +00:00 committed by Tarmac
commit 90934c91a4
8 changed files with 89 additions and 59 deletions

View File

@ -224,7 +224,8 @@ class AccountReaper(Daemon):
marker = ''
while True:
containers = \
list(broker.list_containers_iter(1000, marker, None, None))
list(broker.list_containers_iter(1000, marker, None, None,
None))
if not containers:
break
try:

View File

@ -200,6 +200,7 @@ class AccountController(object):
return HTTPPreconditionFailed(request=req,
body='Maximum limit is %d' % ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
query_format = get_param(req, 'format')
except UnicodeDecodeError, err:
return HTTPBadRequest(body='parameters not utf8',
@ -210,8 +211,8 @@ class AccountController(object):
['text/plain', 'application/json',
'application/xml', 'text/xml'],
default_match='text/plain')
account_list = broker.list_containers_iter(limit, marker, prefix,
delimiter)
account_list = broker.list_containers_iter(limit, marker, end_marker,
prefix, delimiter)
if out_content_type == 'application/json':
json_pattern = ['"name":%s', '"count":%s', '"bytes":%s']
json_pattern = '{' + ','.join(json_pattern) + '}'

View File

@ -940,8 +940,8 @@ class ContainerBroker(DatabaseBroker):
rv.append(row['name'])
return list(set(rv))
def list_objects_iter(self, limit, marker, prefix, delimiter, path=None,
format=None):
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
path=None, format=None):
"""
Get a list of objects sorted by name starting at marker onward, up
to limit entries. Entries will begin with the prefix and will not
@ -949,6 +949,7 @@ class ContainerBroker(DatabaseBroker):
:param limit: maximum number of entries to get
:param marker: marker query
:param end_marker: end marker query
:param prefix: prefix query
:param delimeter: delimeter for query
:param path: if defined, will set the prefix and delimter based on
@ -977,6 +978,9 @@ class ContainerBroker(DatabaseBroker):
query = '''SELECT name, created_at, size, content_type, etag
FROM object WHERE'''
query_args = []
if end_marker:
query += ' name <= ? AND'
query_args.append(end_marker)
if marker and marker >= prefix:
query += ' name > ? AND'
query_args.append(marker)
@ -1440,7 +1444,8 @@ class AccountBroker(DatabaseBroker):
rv.append(row['name'])
return list(set(rv))
def list_containers_iter(self, limit, marker, prefix, delimiter):
def list_containers_iter(self, limit, marker, end_marker, prefix,
delimiter):
"""
Get a list of containerss sorted by name starting at marker onward, up
to limit entries. Entries will begin with the prefix and will not
@ -1448,6 +1453,7 @@ class AccountBroker(DatabaseBroker):
:param limit: maximum number of entries to get
:param marker: marker query
:param end_marker: end marker query
:param prefix: prefix query
:param delimeter: delimeter for query
@ -1469,6 +1475,9 @@ class AccountBroker(DatabaseBroker):
FROM container
WHERE deleted = 0 AND """
query_args = []
if end_marker:
query += ' name <= ? AND'
query_args.append(end_marker)
if marker and marker >= prefix:
query += ' name > ? AND'
query_args.append(marker)

View File

@ -156,14 +156,16 @@ class InternalProxy(object):
tries += 1
return 200 <= resp.status_int < 300
def get_container_list(self, account, container, marker=None, limit=None,
prefix=None, delimiter=None, full_listing=True):
def get_container_list(self, account, container, marker=None,
end_marker=None, limit=None, prefix=None,
delimiter=None, full_listing=True):
"""
Get container listing.
:param account: account name for the container
:param container: container name to get the listing of
:param marker: marker query
:param end_marker: end marker query
:param limit: limit to query
:param prefix: prefix query
:param delimeter: delimeter for query
@ -173,7 +175,7 @@ class InternalProxy(object):
if full_listing:
rv = []
listing = self.get_container_list(account, container, marker,
limit, prefix, delimiter, full_listing=False)
end_marker, limit, prefix, delimiter, full_listing=False)
while listing:
rv.extend(listing)
if not delimiter:
@ -181,12 +183,14 @@ class InternalProxy(object):
else:
marker = listing[-1].get('name', listing[-1].get('subdir'))
listing = self.get_container_list(account, container, marker,
limit, prefix, delimiter, full_listing=False)
end_marker, limit, prefix, delimiter, full_listing=False)
return rv
path = '/v1/%s/%s' % (account, container)
qs = 'format=json'
if marker:
qs += '&marker=%s' % quote(marker)
if end_marker:
qs += '&end_marker=%s' % quote(end_marker)
if limit:
qs += '&limit=%d' % limit
if prefix:

View File

@ -267,6 +267,7 @@ class ContainerController(object):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
@ -284,8 +285,8 @@ class ContainerController(object):
['text/plain', 'application/json',
'application/xml', 'text/xml'],
default_match='text/plain')
container_list = broker.list_objects_iter(limit, marker, prefix,
delimiter, path)
container_list = broker.list_objects_iter(limit, marker, end_marker,
prefix, delimiter, path)
if out_content_type == 'application/json':
json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s',
'"content_type":%s, "last_modified":"%s"']

View File

@ -158,15 +158,14 @@ class LogProcessor(object):
container_listing = self.internal_proxy.get_container_list(
swift_account,
container_name,
marker=search_key)
marker=search_key,
end_marker=end_key)
results = []
if container_listing is not None:
if listing_filter is None:
listing_filter = set()
for item in container_listing:
name = item['name']
if end_key and name > end_key:
break
if name not in listing_filter:
results.append(name)
return results

View File

@ -978,52 +978,57 @@ class TestContainerBroker(unittest.TestCase):
normalize_timestamp(time()), 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
listing = broker.list_objects_iter(100, '', None, '')
listing = broker.list_objects_iter(100, '', None, None, '')
self.assertEquals(len(listing), 100)
self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0099')
listing = broker.list_objects_iter(100, '0/0099', None, '')
listing = broker.list_objects_iter(100, '', '0/0050', None, '')
self.assertEquals(len(listing), 51)
self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0050')
listing = broker.list_objects_iter(100, '0/0099', None, None, '')
self.assertEquals(len(listing), 100)
self.assertEquals(listing[0][0], '0/0100')
self.assertEquals(listing[-1][0], '1/0074')
listing = broker.list_objects_iter(55, '1/0074', None, '')
listing = broker.list_objects_iter(55, '1/0074', None, None, '')
self.assertEquals(len(listing), 55)
self.assertEquals(listing[0][0], '1/0075')
self.assertEquals(listing[-1][0], '2/0004')
listing = broker.list_objects_iter(10, '', '0/01', '')
listing = broker.list_objects_iter(10, '', None, '0/01', '')
self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0/0100')
self.assertEquals(listing[-1][0], '0/0109')
listing = broker.list_objects_iter(10, '', '0/', '/')
listing = broker.list_objects_iter(10, '', None, '0/', '/')
self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0009')
listing = broker.list_objects_iter(10, '', '', '/')
listing = broker.list_objects_iter(10, '', None, '', '/')
self.assertEquals(len(listing), 4)
self.assertEquals([row[0] for row in listing],
['0/', '1/', '2/', '3/'])
listing = broker.list_objects_iter(10, '2', None, '/')
listing = broker.list_objects_iter(10, '2', None, None, '/')
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['2/', '3/'])
listing = broker.list_objects_iter(10, '2/', None, '/')
listing = broker.list_objects_iter(10, '2/',None, None, '/')
self.assertEquals(len(listing), 1)
self.assertEquals([row[0] for row in listing], ['3/'])
listing = broker.list_objects_iter(10, '2/0050', '2/', '/')
listing = broker.list_objects_iter(10, '2/0050', None, '2/', '/')
self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '2/0051')
self.assertEquals(listing[1][0], '2/0051/')
self.assertEquals(listing[2][0], '2/0052')
self.assertEquals(listing[-1][0], '2/0059')
listing = broker.list_objects_iter(10, '3/0045', '3/', '/')
listing = broker.list_objects_iter(10, '3/0045', None, '3/', '/')
self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing],
['3/0045/', '3/0046', '3/0046/', '3/0047',
@ -1032,33 +1037,34 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('3/0049/', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
listing = broker.list_objects_iter(10, '3/0048', None, None)
listing = broker.list_objects_iter(10, '3/0048', None, None, None)
self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing],
['3/0048/0049', '3/0049', '3/0049/',
'3/0049/0049', '3/0050', '3/0050/0049', '3/0051', '3/0051/0049',
'3/0052', '3/0052/0049'])
listing = broker.list_objects_iter(10, '3/0048', '3/', '/')
listing = broker.list_objects_iter(10, '3/0048', None, '3/', '/')
self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing],
['3/0048/', '3/0049', '3/0049/', '3/0050',
'3/0050/', '3/0051', '3/0051/', '3/0052', '3/0052/', '3/0053'])
listing = broker.list_objects_iter(10, None, '3/0049/', '/')
listing = broker.list_objects_iter(10, None, None, '3/0049/', '/')
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing],
['3/0049/', '3/0049/0049'])
listing = broker.list_objects_iter(10, None, None, None, '3/0049')
listing = broker.list_objects_iter(10, None, None, None, None,
'3/0049')
self.assertEquals(len(listing), 1)
self.assertEquals([row[0] for row in listing], ['3/0049/0049'])
listing = broker.list_objects_iter(2, None, '3/', '/')
listing = broker.list_objects_iter(2, None, None, '3/', '/')
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['3/0000', '3/0000/'])
listing = broker.list_objects_iter(2, None, None, None, '3')
listing = broker.list_objects_iter(2, None, None, None, None, '3')
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['3/0000', '3/0001'])
@ -1082,11 +1088,11 @@ class TestContainerBroker(unittest.TestCase):
#def list_objects_iter(self, limit, marker, prefix, delimiter, path=None,
# format=None):
listing = broker.list_objects_iter(100, None, '/pets/f', '/')
listing = broker.list_objects_iter(100, None, None, '/pets/f', '/')
self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt'])
listing = broker.list_objects_iter(100, None, '/pets/fish', '/')
listing = broker.list_objects_iter(100, None, None, '/pets/fish', '/')
self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt'])
listing = broker.list_objects_iter(100, None, '/pets/fish/', '/')
listing = broker.list_objects_iter(100, None, None, '/pets/fish/', '/')
self.assertEquals([row[0] for row in listing], ['/pets/fish/a', '/pets/fish/b'])
def test_double_check_trailing_delimiter(self):
@ -1114,19 +1120,19 @@ class TestContainerBroker(unittest.TestCase):
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('c', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
listing = broker.list_objects_iter(15, None, None, None)
listing = broker.list_objects_iter(15, None, None, None, None)
self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing],
['a', 'a/', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b', 'b/a', 'b/b', 'c'])
listing = broker.list_objects_iter(15, None, '', '/')
listing = broker.list_objects_iter(15, None, None, '', '/')
self.assertEquals(len(listing), 5)
self.assertEquals([row[0] for row in listing],
['a', 'a/', 'b', 'b/', 'c'])
listing = broker.list_objects_iter(15, None, 'a/', '/')
listing = broker.list_objects_iter(15, None, None, 'a/', '/')
self.assertEquals(len(listing), 4)
self.assertEquals([row[0] for row in listing],
['a/', 'a/a', 'a/a/', 'a/b'])
listing = broker.list_objects_iter(15, None, 'b/', '/')
listing = broker.list_objects_iter(15, None, None, 'b/', '/')
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['b/a', 'b/b'])
@ -1646,57 +1652,62 @@ class TestAccountBroker(unittest.TestCase):
broker.put_container('3/%04d/0049' % cont,
normalize_timestamp(time()), 0, 0, 0)
listing = broker.list_containers_iter(100, '', None, '')
listing = broker.list_containers_iter(100, '', None, None, '')
self.assertEquals(len(listing), 100)
self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0099')
listing = broker.list_containers_iter(100, '0/0099', None, '')
listing = broker.list_containers_iter(100, '', '0/0050', None, '')
self.assertEquals(len(listing), 51)
self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0050')
listing = broker.list_containers_iter(100, '0/0099', None, None, '')
self.assertEquals(len(listing), 100)
self.assertEquals(listing[0][0], '0/0100')
self.assertEquals(listing[-1][0], '1/0074')
listing = broker.list_containers_iter(55, '1/0074', None, '')
listing = broker.list_containers_iter(55, '1/0074', None, None, '')
self.assertEquals(len(listing), 55)
self.assertEquals(listing[0][0], '1/0075')
self.assertEquals(listing[-1][0], '2/0004')
listing = broker.list_containers_iter(10, '', '0/01', '')
listing = broker.list_containers_iter(10, '', None, '0/01', '')
self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0/0100')
self.assertEquals(listing[-1][0], '0/0109')
listing = broker.list_containers_iter(10, '', '0/01', '/')
listing = broker.list_containers_iter(10, '', None, '0/01', '/')
self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0/0100')
self.assertEquals(listing[-1][0], '0/0109')
listing = broker.list_containers_iter(10, '', '0/', '/')
listing = broker.list_containers_iter(10, '', None, '0/', '/')
self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0009')
listing = broker.list_containers_iter(10, '', '', '/')
listing = broker.list_containers_iter(10, '', None, '', '/')
self.assertEquals(len(listing), 4)
self.assertEquals([row[0] for row in listing],
['0/', '1/', '2/', '3/'])
listing = broker.list_containers_iter(10, '2/', None, '/')
listing = broker.list_containers_iter(10, '2/', None, None, '/')
self.assertEquals(len(listing), 1)
self.assertEquals([row[0] for row in listing], ['3/'])
listing = broker.list_containers_iter(10, '', '2', '/')
listing = broker.list_containers_iter(10, '', None, '2', '/')
self.assertEquals(len(listing), 1)
self.assertEquals([row[0] for row in listing], ['2/'])
listing = broker.list_containers_iter(10, '2/0050', '2/', '/')
listing = broker.list_containers_iter(10, '2/0050', None, '2/', '/')
self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '2/0051')
self.assertEquals(listing[1][0], '2/0051/')
self.assertEquals(listing[2][0], '2/0052')
self.assertEquals(listing[-1][0], '2/0059')
listing = broker.list_containers_iter(10, '3/0045', '3/', '/')
listing = broker.list_containers_iter(10, '3/0045', None, '3/', '/')
self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing],
['3/0045/', '3/0046', '3/0046/', '3/0047',
@ -1704,21 +1715,21 @@ class TestAccountBroker(unittest.TestCase):
'3/0049/', '3/0050'])
broker.put_container('3/0049/', normalize_timestamp(time()), 0, 0, 0)
listing = broker.list_containers_iter(10, '3/0048', None, None)
listing = broker.list_containers_iter(10, '3/0048', None, None, None)
self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing],
['3/0048/0049', '3/0049', '3/0049/', '3/0049/0049',
'3/0050', '3/0050/0049', '3/0051', '3/0051/0049',
'3/0052', '3/0052/0049'])
listing = broker.list_containers_iter(10, '3/0048', '3/', '/')
listing = broker.list_containers_iter(10, '3/0048', None, '3/', '/')
self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing],
['3/0048/', '3/0049', '3/0049/', '3/0050',
'3/0050/', '3/0051', '3/0051/', '3/0052',
'3/0052/', '3/0053'])
listing = broker.list_containers_iter(10, None, '3/0049/', '/')
listing = broker.list_containers_iter(10, None, None, '3/0049/', '/')
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing],
['3/0049/', '3/0049/0049'])
@ -1738,20 +1749,20 @@ class TestAccountBroker(unittest.TestCase):
broker.put_container('b/a', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('b/b', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('c', normalize_timestamp(time()), 0, 0, 0)
listing = broker.list_containers_iter(15, None, None, None)
listing = broker.list_containers_iter(15, None, None, None, None)
self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing],
['a', 'a/', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b',
'b/a', 'b/b', 'c'])
listing = broker.list_containers_iter(15, None, '', '/')
listing = broker.list_containers_iter(15, None, None, '', '/')
self.assertEquals(len(listing), 5)
self.assertEquals([row[0] for row in listing],
['a', 'a/', 'b', 'b/', 'c'])
listing = broker.list_containers_iter(15, None, 'a/', '/')
listing = broker.list_containers_iter(15, None, None, 'a/', '/')
self.assertEquals(len(listing), 4)
self.assertEquals([row[0] for row in listing],
['a/', 'a/a', 'a/a/', 'a/b'])
listing = broker.list_containers_iter(15, None, 'b/', '/')
listing = broker.list_containers_iter(15, None, None, 'b/', '/')
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['b/a', 'b/b'])

View File

@ -46,12 +46,16 @@ class DumbLogger(object):
pass
class DumbInternalProxy(object):
def get_container_list(self, account, container, marker=None):
def get_container_list(self, account, container, marker=None,
end_marker=None):
n = '2010/03/14/13/obj1'
if marker is None or n > marker:
if end_marker and n <= end_marker:
return [{'name': n}]
elif end_marker:
return []
return [{'name': n}]
else:
return []
return []
def get_object(self, account, container, object_name):
code = 200