openstack/swift
Code Issues Proposed changes

added end_marker query parameter for container and object listings

Browse Source
This commit is contained in:
John Dickinson 2010-11-18 21:42:49 +00:00 committed by Tarmac
commit 90934c91a4
8 changed files with 89 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
swift/account/reaper.py
View File

@ -224,7 +224,8 @@ class AccountReaper(Daemon):
marker = '' marker = ''
while True: while True:
containers = \ containers = \
list(broker.list_containers_iter(1000, marker, None, None)) list(broker.list_containers_iter(1000, marker, None, None,
None))
if not containers: if not containers:
break break
try: try:

5
swift/account/server.py
View File

@ -200,6 +200,7 @@ class AccountController(object):
return HTTPPreconditionFailed(request=req, return HTTPPreconditionFailed(request=req,
body='Maximum limit is %d' % ACCOUNT_LISTING_LIMIT) body='Maximum limit is %d' % ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '') marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
query_format = get_param(req, 'format') query_format = get_param(req, 'format')
except UnicodeDecodeError, err: except UnicodeDecodeError, err:
return HTTPBadRequest(body='parameters not utf8', return HTTPBadRequest(body='parameters not utf8',
@ -210,8 +211,8 @@ class AccountController(object):
['text/plain', 'application/json', ['text/plain', 'application/json',
'application/xml', 'text/xml'], 'application/xml', 'text/xml'],
default_match='text/plain') default_match='text/plain')
account_list = broker.list_containers_iter(limit, marker, prefix, account_list = broker.list_containers_iter(limit, marker, end_marker,
delimiter) prefix, delimiter)
if out_content_type == 'application/json': if out_content_type == 'application/json':
json_pattern = ['"name":%s', '"count":%s', '"bytes":%s'] json_pattern = ['"name":%s', '"count":%s', '"bytes":%s']
json_pattern = '{' + ','.join(json_pattern) + '}' json_pattern = '{' + ','.join(json_pattern) + '}'

15
swift/common/db.py
View File

@ -940,8 +940,8 @@ class ContainerBroker(DatabaseBroker):
rv.append(row['name']) rv.append(row['name'])
return list(set(rv)) return list(set(rv))
def list_objects_iter(self, limit, marker, prefix, delimiter, path=None, def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
format=None): path=None, format=None):
""" """
Get a list of objects sorted by name starting at marker onward, up Get a list of objects sorted by name starting at marker onward, up
to limit entries. Entries will begin with the prefix and will not to limit entries. Entries will begin with the prefix and will not
@ -949,6 +949,7 @@ class ContainerBroker(DatabaseBroker):
:param limit: maximum number of entries to get :param limit: maximum number of entries to get
:param marker: marker query :param marker: marker query
:param end_marker: end marker query
:param prefix: prefix query :param prefix: prefix query
:param delimeter: delimeter for query :param delimeter: delimeter for query
:param path: if defined, will set the prefix and delimter based on :param path: if defined, will set the prefix and delimter based on
@ -977,6 +978,9 @@ class ContainerBroker(DatabaseBroker):
query = '''SELECT name, created_at, size, content_type, etag query = '''SELECT name, created_at, size, content_type, etag
FROM object WHERE''' FROM object WHERE'''
query_args = [] query_args = []
if end_marker:
query += ' name <= ? AND'
query_args.append(end_marker)
if marker and marker >= prefix: if marker and marker >= prefix:
query += ' name > ? AND' query += ' name > ? AND'
query_args.append(marker) query_args.append(marker)
@ -1440,7 +1444,8 @@ class AccountBroker(DatabaseBroker):
rv.append(row['name']) rv.append(row['name'])
return list(set(rv)) return list(set(rv))
def list_containers_iter(self, limit, marker, prefix, delimiter): def list_containers_iter(self, limit, marker, end_marker, prefix,
delimiter):
""" """
Get a list of containerss sorted by name starting at marker onward, up Get a list of containerss sorted by name starting at marker onward, up
to limit entries. Entries will begin with the prefix and will not to limit entries. Entries will begin with the prefix and will not
@ -1448,6 +1453,7 @@ class AccountBroker(DatabaseBroker):
:param limit: maximum number of entries to get :param limit: maximum number of entries to get
:param marker: marker query :param marker: marker query
:param end_marker: end marker query
:param prefix: prefix query :param prefix: prefix query
:param delimeter: delimeter for query :param delimeter: delimeter for query
@ -1469,6 +1475,9 @@ class AccountBroker(DatabaseBroker):
FROM container FROM container
WHERE deleted = 0 AND """ WHERE deleted = 0 AND """
query_args = [] query_args = []
if end_marker:
query += ' name <= ? AND'
query_args.append(end_marker)
if marker and marker >= prefix: if marker and marker >= prefix:
query += ' name > ? AND' query += ' name > ? AND'
query_args.append(marker) query_args.append(marker)

12
swift/common/internal_proxy.py
View File

@ -156,14 +156,16 @@ class InternalProxy(object):
tries += 1 tries += 1
return 200 <= resp.status_int < 300 return 200 <= resp.status_int < 300
def get_container_list(self, account, container, marker=None, limit=None, def get_container_list(self, account, container, marker=None,
prefix=None, delimiter=None, full_listing=True): end_marker=None, limit=None, prefix=None,
delimiter=None, full_listing=True):
""" """
Get container listing. Get container listing.
:param account: account name for the container :param account: account name for the container
:param container: container name to get the listing of :param container: container name to get the listing of
:param marker: marker query :param marker: marker query
:param end_marker: end marker query
:param limit: limit to query :param limit: limit to query
:param prefix: prefix query :param prefix: prefix query
:param delimeter: delimeter for query :param delimeter: delimeter for query
@ -173,7 +175,7 @@ class InternalProxy(object):
if full_listing: if full_listing:
rv = [] rv = []
listing = self.get_container_list(account, container, marker, listing = self.get_container_list(account, container, marker,
limit, prefix, delimiter, full_listing=False) end_marker, limit, prefix, delimiter, full_listing=False)
while listing: while listing:
rv.extend(listing) rv.extend(listing)
if not delimiter: if not delimiter:
@ -181,12 +183,14 @@ class InternalProxy(object):
else: else:
marker = listing[-1].get('name', listing[-1].get('subdir')) marker = listing[-1].get('name', listing[-1].get('subdir'))
listing = self.get_container_list(account, container, marker, listing = self.get_container_list(account, container, marker,
limit, prefix, delimiter, full_listing=False) end_marker, limit, prefix, delimiter, full_listing=False)
return rv return rv
path = '/v1/%s/%s' % (account, container) path = '/v1/%s/%s' % (account, container)
qs = 'format=json' qs = 'format=json'
if marker: if marker:
qs += '&marker=%s' % quote(marker) qs += '&marker=%s' % quote(marker)
if end_marker:
qs += '&end_marker=%s' % quote(end_marker)
if limit: if limit:
qs += '&limit=%d' % limit qs += '&limit=%d' % limit
if prefix: if prefix:

5
swift/container/server.py
View File

@ -267,6 +267,7 @@ class ContainerController(object):
# delimiters can be made more flexible later # delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter') return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '') marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = CONTAINER_LISTING_LIMIT limit = CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit') given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit(): if given_limit and given_limit.isdigit():
@ -284,8 +285,8 @@ class ContainerController(object):
['text/plain', 'application/json', ['text/plain', 'application/json',
'application/xml', 'text/xml'], 'application/xml', 'text/xml'],
default_match='text/plain') default_match='text/plain')
container_list = broker.list_objects_iter(limit, marker, prefix, container_list = broker.list_objects_iter(limit, marker, end_marker,
delimiter, path) prefix, delimiter, path)
if out_content_type == 'application/json': if out_content_type == 'application/json':
json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s', json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s',
'"content_type":%s, "last_modified":"%s"'] '"content_type":%s, "last_modified":"%s"']

5
swift/stats/log_processor.py
View File

@ -158,15 +158,14 @@ class LogProcessor(object):
container_listing = self.internal_proxy.get_container_list( container_listing = self.internal_proxy.get_container_list(
swift_account, swift_account,
container_name, container_name,
marker=search_key) marker=search_key,
end_marker=end_key)
results = [] results = []
if container_listing is not None: if container_listing is not None:
if listing_filter is None: if listing_filter is None:
listing_filter = set() listing_filter = set()
for item in container_listing: for item in container_listing:
name = item['name'] name = item['name']
if end_key and name > end_key:
break
if name not in listing_filter: if name not in listing_filter:
results.append(name) results.append(name)
return results return results

93
test/unit/common/test_db.py
View File

@ -978,52 +978,57 @@ class TestContainerBroker(unittest.TestCase):
normalize_timestamp(time()), 0, 'text/plain', normalize_timestamp(time()), 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e') 'd41d8cd98f00b204e9800998ecf8427e')
listing = broker.list_objects_iter(100, '', None, '') listing = broker.list_objects_iter(100, '', None, None, '')
self.assertEquals(len(listing), 100) self.assertEquals(len(listing), 100)
self.assertEquals(listing[0][0], '0/0000') self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0099') self.assertEquals(listing[-1][0], '0/0099')
listing = broker.list_objects_iter(100, '0/0099', None, '') listing = broker.list_objects_iter(100, '', '0/0050', None, '')
self.assertEquals(len(listing), 51)
self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0050')
listing = broker.list_objects_iter(100, '0/0099', None, None, '')
self.assertEquals(len(listing), 100) self.assertEquals(len(listing), 100)
self.assertEquals(listing[0][0], '0/0100') self.assertEquals(listing[0][0], '0/0100')
self.assertEquals(listing[-1][0], '1/0074') self.assertEquals(listing[-1][0], '1/0074')
listing = broker.list_objects_iter(55, '1/0074', None, '') listing = broker.list_objects_iter(55, '1/0074', None, None, '')
self.assertEquals(len(listing), 55) self.assertEquals(len(listing), 55)
self.assertEquals(listing[0][0], '1/0075') self.assertEquals(listing[0][0], '1/0075')
self.assertEquals(listing[-1][0], '2/0004') self.assertEquals(listing[-1][0], '2/0004')
listing = broker.list_objects_iter(10, '', '0/01', '') listing = broker.list_objects_iter(10, '', None, '0/01', '')
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0/0100') self.assertEquals(listing[0][0], '0/0100')
self.assertEquals(listing[-1][0], '0/0109') self.assertEquals(listing[-1][0], '0/0109')
listing = broker.list_objects_iter(10, '', '0/', '/') listing = broker.list_objects_iter(10, '', None, '0/', '/')
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0/0000') self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0009') self.assertEquals(listing[-1][0], '0/0009')
listing = broker.list_objects_iter(10, '', '', '/') listing = broker.list_objects_iter(10, '', None, '', '/')
self.assertEquals(len(listing), 4) self.assertEquals(len(listing), 4)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['0/', '1/', '2/', '3/']) ['0/', '1/', '2/', '3/'])
listing = broker.list_objects_iter(10, '2', None, '/') listing = broker.list_objects_iter(10, '2', None, None, '/')
self.assertEquals(len(listing), 2) self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['2/', '3/']) self.assertEquals([row[0] for row in listing], ['2/', '3/'])
listing = broker.list_objects_iter(10, '2/', None, '/') listing = broker.list_objects_iter(10, '2/',None, None, '/')
self.assertEquals(len(listing), 1) self.assertEquals(len(listing), 1)
self.assertEquals([row[0] for row in listing], ['3/']) self.assertEquals([row[0] for row in listing], ['3/'])
listing = broker.list_objects_iter(10, '2/0050', '2/', '/') listing = broker.list_objects_iter(10, '2/0050', None, '2/', '/')
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '2/0051') self.assertEquals(listing[0][0], '2/0051')
self.assertEquals(listing[1][0], '2/0051/') self.assertEquals(listing[1][0], '2/0051/')
self.assertEquals(listing[2][0], '2/0052') self.assertEquals(listing[2][0], '2/0052')
self.assertEquals(listing[-1][0], '2/0059') self.assertEquals(listing[-1][0], '2/0059')
listing = broker.list_objects_iter(10, '3/0045', '3/', '/') listing = broker.list_objects_iter(10, '3/0045', None, '3/', '/')
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['3/0045/', '3/0046', '3/0046/', '3/0047', ['3/0045/', '3/0046', '3/0046/', '3/0047',
@ -1032,33 +1037,34 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('3/0049/', normalize_timestamp(time()), 0, broker.put_object('3/0049/', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
listing = broker.list_objects_iter(10, '3/0048', None, None) listing = broker.list_objects_iter(10, '3/0048', None, None, None)
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['3/0048/0049', '3/0049', '3/0049/', ['3/0048/0049', '3/0049', '3/0049/',
'3/0049/0049', '3/0050', '3/0050/0049', '3/0051', '3/0051/0049', '3/0049/0049', '3/0050', '3/0050/0049', '3/0051', '3/0051/0049',
'3/0052', '3/0052/0049']) '3/0052', '3/0052/0049'])
listing = broker.list_objects_iter(10, '3/0048', '3/', '/') listing = broker.list_objects_iter(10, '3/0048', None, '3/', '/')
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['3/0048/', '3/0049', '3/0049/', '3/0050', ['3/0048/', '3/0049', '3/0049/', '3/0050',
'3/0050/', '3/0051', '3/0051/', '3/0052', '3/0052/', '3/0053']) '3/0050/', '3/0051', '3/0051/', '3/0052', '3/0052/', '3/0053'])
listing = broker.list_objects_iter(10, None, '3/0049/', '/') listing = broker.list_objects_iter(10, None, None, '3/0049/', '/')
self.assertEquals(len(listing), 2) self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['3/0049/', '3/0049/0049']) ['3/0049/', '3/0049/0049'])
listing = broker.list_objects_iter(10, None, None, None, '3/0049') listing = broker.list_objects_iter(10, None, None, None, None,
'3/0049')
self.assertEquals(len(listing), 1) self.assertEquals(len(listing), 1)
self.assertEquals([row[0] for row in listing], ['3/0049/0049']) self.assertEquals([row[0] for row in listing], ['3/0049/0049'])
listing = broker.list_objects_iter(2, None, '3/', '/') listing = broker.list_objects_iter(2, None, None, '3/', '/')
self.assertEquals(len(listing), 2) self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['3/0000', '3/0000/']) self.assertEquals([row[0] for row in listing], ['3/0000', '3/0000/'])
listing = broker.list_objects_iter(2, None, None, None, '3') listing = broker.list_objects_iter(2, None, None, None, None, '3')
self.assertEquals(len(listing), 2) self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['3/0000', '3/0001']) self.assertEquals([row[0] for row in listing], ['3/0000', '3/0001'])
@ -1082,11 +1088,11 @@ class TestContainerBroker(unittest.TestCase):
#def list_objects_iter(self, limit, marker, prefix, delimiter, path=None, #def list_objects_iter(self, limit, marker, prefix, delimiter, path=None,
# format=None): # format=None):
listing = broker.list_objects_iter(100, None, '/pets/f', '/') listing = broker.list_objects_iter(100, None, None, '/pets/f', '/')
self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt']) self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt'])
listing = broker.list_objects_iter(100, None, '/pets/fish', '/') listing = broker.list_objects_iter(100, None, None, '/pets/fish', '/')
self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt']) self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt'])
listing = broker.list_objects_iter(100, None, '/pets/fish/', '/') listing = broker.list_objects_iter(100, None, None, '/pets/fish/', '/')
self.assertEquals([row[0] for row in listing], ['/pets/fish/a', '/pets/fish/b']) self.assertEquals([row[0] for row in listing], ['/pets/fish/a', '/pets/fish/b'])
def test_double_check_trailing_delimiter(self): def test_double_check_trailing_delimiter(self):
@ -1114,19 +1120,19 @@ class TestContainerBroker(unittest.TestCase):
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('c', normalize_timestamp(time()), 0, broker.put_object('c', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
listing = broker.list_objects_iter(15, None, None, None) listing = broker.list_objects_iter(15, None, None, None, None)
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['a', 'a/', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b', 'b/a', 'b/b', 'c']) ['a', 'a/', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b', 'b/a', 'b/b', 'c'])
listing = broker.list_objects_iter(15, None, '', '/') listing = broker.list_objects_iter(15, None, None, '', '/')
self.assertEquals(len(listing), 5) self.assertEquals(len(listing), 5)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['a', 'a/', 'b', 'b/', 'c']) ['a', 'a/', 'b', 'b/', 'c'])
listing = broker.list_objects_iter(15, None, 'a/', '/') listing = broker.list_objects_iter(15, None, None, 'a/', '/')
self.assertEquals(len(listing), 4) self.assertEquals(len(listing), 4)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['a/', 'a/a', 'a/a/', 'a/b']) ['a/', 'a/a', 'a/a/', 'a/b'])
listing = broker.list_objects_iter(15, None, 'b/', '/') listing = broker.list_objects_iter(15, None, None, 'b/', '/')
self.assertEquals(len(listing), 2) self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['b/a', 'b/b']) self.assertEquals([row[0] for row in listing], ['b/a', 'b/b'])
@ -1646,57 +1652,62 @@ class TestAccountBroker(unittest.TestCase):
broker.put_container('3/%04d/0049' % cont, broker.put_container('3/%04d/0049' % cont,
normalize_timestamp(time()), 0, 0, 0) normalize_timestamp(time()), 0, 0, 0)
listing = broker.list_containers_iter(100, '', None, '') listing = broker.list_containers_iter(100, '', None, None, '')
self.assertEquals(len(listing), 100) self.assertEquals(len(listing), 100)
self.assertEquals(listing[0][0], '0/0000') self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0099') self.assertEquals(listing[-1][0], '0/0099')
listing = broker.list_containers_iter(100, '0/0099', None, '') listing = broker.list_containers_iter(100, '', '0/0050', None, '')
self.assertEquals(len(listing), 51)
self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0050')
listing = broker.list_containers_iter(100, '0/0099', None, None, '')
self.assertEquals(len(listing), 100) self.assertEquals(len(listing), 100)
self.assertEquals(listing[0][0], '0/0100') self.assertEquals(listing[0][0], '0/0100')
self.assertEquals(listing[-1][0], '1/0074') self.assertEquals(listing[-1][0], '1/0074')
listing = broker.list_containers_iter(55, '1/0074', None, '') listing = broker.list_containers_iter(55, '1/0074', None, None, '')
self.assertEquals(len(listing), 55) self.assertEquals(len(listing), 55)
self.assertEquals(listing[0][0], '1/0075') self.assertEquals(listing[0][0], '1/0075')
self.assertEquals(listing[-1][0], '2/0004') self.assertEquals(listing[-1][0], '2/0004')
listing = broker.list_containers_iter(10, '', '0/01', '') listing = broker.list_containers_iter(10, '', None, '0/01', '')
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0/0100') self.assertEquals(listing[0][0], '0/0100')
self.assertEquals(listing[-1][0], '0/0109') self.assertEquals(listing[-1][0], '0/0109')
listing = broker.list_containers_iter(10, '', '0/01', '/') listing = broker.list_containers_iter(10, '', None, '0/01', '/')
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0/0100') self.assertEquals(listing[0][0], '0/0100')
self.assertEquals(listing[-1][0], '0/0109') self.assertEquals(listing[-1][0], '0/0109')
listing = broker.list_containers_iter(10, '', '0/', '/') listing = broker.list_containers_iter(10, '', None, '0/', '/')
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0/0000') self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0009') self.assertEquals(listing[-1][0], '0/0009')
listing = broker.list_containers_iter(10, '', '', '/') listing = broker.list_containers_iter(10, '', None, '', '/')
self.assertEquals(len(listing), 4) self.assertEquals(len(listing), 4)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['0/', '1/', '2/', '3/']) ['0/', '1/', '2/', '3/'])
listing = broker.list_containers_iter(10, '2/', None, '/') listing = broker.list_containers_iter(10, '2/', None, None, '/')
self.assertEquals(len(listing), 1) self.assertEquals(len(listing), 1)
self.assertEquals([row[0] for row in listing], ['3/']) self.assertEquals([row[0] for row in listing], ['3/'])
listing = broker.list_containers_iter(10, '', '2', '/') listing = broker.list_containers_iter(10, '', None, '2', '/')
self.assertEquals(len(listing), 1) self.assertEquals(len(listing), 1)
self.assertEquals([row[0] for row in listing], ['2/']) self.assertEquals([row[0] for row in listing], ['2/'])
listing = broker.list_containers_iter(10, '2/0050', '2/', '/') listing = broker.list_containers_iter(10, '2/0050', None, '2/', '/')
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '2/0051') self.assertEquals(listing[0][0], '2/0051')
self.assertEquals(listing[1][0], '2/0051/') self.assertEquals(listing[1][0], '2/0051/')
self.assertEquals(listing[2][0], '2/0052') self.assertEquals(listing[2][0], '2/0052')
self.assertEquals(listing[-1][0], '2/0059') self.assertEquals(listing[-1][0], '2/0059')
listing = broker.list_containers_iter(10, '3/0045', '3/', '/') listing = broker.list_containers_iter(10, '3/0045', None, '3/', '/')
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['3/0045/', '3/0046', '3/0046/', '3/0047', ['3/0045/', '3/0046', '3/0046/', '3/0047',
@ -1704,21 +1715,21 @@ class TestAccountBroker(unittest.TestCase):
'3/0049/', '3/0050']) '3/0049/', '3/0050'])
broker.put_container('3/0049/', normalize_timestamp(time()), 0, 0, 0) broker.put_container('3/0049/', normalize_timestamp(time()), 0, 0, 0)
listing = broker.list_containers_iter(10, '3/0048', None, None) listing = broker.list_containers_iter(10, '3/0048', None, None, None)
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['3/0048/0049', '3/0049', '3/0049/', '3/0049/0049', ['3/0048/0049', '3/0049', '3/0049/', '3/0049/0049',
'3/0050', '3/0050/0049', '3/0051', '3/0051/0049', '3/0050', '3/0050/0049', '3/0051', '3/0051/0049',
'3/0052', '3/0052/0049']) '3/0052', '3/0052/0049'])
listing = broker.list_containers_iter(10, '3/0048', '3/', '/') listing = broker.list_containers_iter(10, '3/0048', None, '3/', '/')
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['3/0048/', '3/0049', '3/0049/', '3/0050', ['3/0048/', '3/0049', '3/0049/', '3/0050',
'3/0050/', '3/0051', '3/0051/', '3/0052', '3/0050/', '3/0051', '3/0051/', '3/0052',
'3/0052/', '3/0053']) '3/0052/', '3/0053'])
listing = broker.list_containers_iter(10, None, '3/0049/', '/') listing = broker.list_containers_iter(10, None, None, '3/0049/', '/')
self.assertEquals(len(listing), 2) self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['3/0049/', '3/0049/0049']) ['3/0049/', '3/0049/0049'])
@ -1738,20 +1749,20 @@ class TestAccountBroker(unittest.TestCase):
broker.put_container('b/a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('b/a', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('b/b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('b/b', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) broker.put_container('c', normalize_timestamp(time()), 0, 0, 0)
listing = broker.list_containers_iter(15, None, None, None) listing = broker.list_containers_iter(15, None, None, None, None)
self.assertEquals(len(listing), 10) self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['a', 'a/', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b', ['a', 'a/', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b',
'b/a', 'b/b', 'c']) 'b/a', 'b/b', 'c'])
listing = broker.list_containers_iter(15, None, '', '/') listing = broker.list_containers_iter(15, None, None, '', '/')
self.assertEquals(len(listing), 5) self.assertEquals(len(listing), 5)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['a', 'a/', 'b', 'b/', 'c']) ['a', 'a/', 'b', 'b/', 'c'])
listing = broker.list_containers_iter(15, None, 'a/', '/') listing = broker.list_containers_iter(15, None, None, 'a/', '/')
self.assertEquals(len(listing), 4) self.assertEquals(len(listing), 4)
self.assertEquals([row[0] for row in listing], self.assertEquals([row[0] for row in listing],
['a/', 'a/a', 'a/a/', 'a/b']) ['a/', 'a/a', 'a/a/', 'a/b'])
listing = broker.list_containers_iter(15, None, 'b/', '/') listing = broker.list_containers_iter(15, None, None, 'b/', '/')
self.assertEquals(len(listing), 2) self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['b/a', 'b/b']) self.assertEquals([row[0] for row in listing], ['b/a', 'b/b'])

10
test/unit/stats/test_log_processor.py
View File

@ -46,12 +46,16 @@ class DumbLogger(object):
pass pass
class DumbInternalProxy(object): class DumbInternalProxy(object):
def get_container_list(self, account, container, marker=None): def get_container_list(self, account, container, marker=None,
end_marker=None):
n = '2010/03/14/13/obj1' n = '2010/03/14/13/obj1'
if marker is None or n > marker: if marker is None or n > marker:
if end_marker and n <= end_marker:
return [{'name': n}]
elif end_marker:
return []
return [{'name': n}] return [{'name': n}]
else: return []
return []
def get_object(self, account, container, object_name): def get_object(self, account, container, object_name):
code = 200 code = 200