Change filters to use a generator pattern
By using a generator pattern, we avoid creating whole new lists each time, instead we iterate through the original list once (after it is sorted), constructing the final list only once. We also address the behavioral differences between the swift filtering results and our code so that ported unit tests work the same (non-slash objects, that is). Change-Id: If32c1987f24781ff81ab4c28c9ddfff17c2e7787 Signed-off-by: Peter Portante <peter.portante@redhat.com> Reviewed-on: http://review.gluster.org/5145 Tested-by: Luis Pabon <lpabon@redhat.com> Reviewed-by: Mohammed Junaid <junaid@redhat.com> Reviewed-by: Luis Pabon <lpabon@redhat.com>
This commit is contained in:
Peter Portante
Luis Pabon
parent
006c3599cb
commit
f80872e852
@ -55,72 +55,97 @@ def _read_metadata(dd):
|
||||
|
||||
def filter_prefix(objects, prefix):
|
||||
"""
|
||||
Accept sorted list.
|
||||
Accept a sorted list of strings, returning all strings starting with the
|
||||
given prefix.
|
||||
"""
|
||||
found = 0
|
||||
filtered_objs = []
|
||||
found = False
|
||||
for object_name in objects:
|
||||
if object_name.startswith(prefix):
|
||||
filtered_objs.append(object_name)
|
||||
found = 1
|
||||
yield object_name
|
||||
found = True
|
||||
else:
|
||||
# Since the list is assumed to be sorted, once we find an object
|
||||
# name that does not start with the prefix we know we won't find
|
||||
# any others, so we exit early.
|
||||
if found:
|
||||
break
|
||||
return filtered_objs
|
||||
|
||||
|
||||
def filter_delimiter(objects, delimiter, prefix):
|
||||
def filter_delimiter(objects, delimiter, prefix, marker, path=None):
|
||||
"""
|
||||
Accept sorted list.
|
||||
Objects should start with prefix.
|
||||
Accept a sorted list of strings, returning strings that:
|
||||
1. begin with "prefix" (empty string matches all)
|
||||
2. does not match the "path" argument
|
||||
3. does not contain the delimiter in the given prefix length
|
||||
4.
|
||||
be those that start with the prefix.
|
||||
"""
|
||||
filtered_objs = []
|
||||
assert delimiter
|
||||
assert prefix is not None
|
||||
skip_name = None
|
||||
for object_name in objects:
|
||||
tmp_obj = object_name.replace(prefix, '', 1)
|
||||
sufix = tmp_obj.split(delimiter, 1)
|
||||
new_obj = prefix + sufix[0]
|
||||
if new_obj and new_obj not in filtered_objs:
|
||||
filtered_objs.append(new_obj)
|
||||
|
||||
return filtered_objs
|
||||
if prefix and not object_name.startswith(prefix):
|
||||
break
|
||||
if path is not None:
|
||||
if object_name == path:
|
||||
continue
|
||||
if skip_name:
|
||||
if object_name < skip_name:
|
||||
continue
|
||||
else:
|
||||
skip_name = None
|
||||
end = object_name.find(delimiter, len(prefix))
|
||||
if end >= 0 and (len(object_name) > (end + 1)):
|
||||
skip_name = object_name[:end] + chr(ord(delimiter) + 1)
|
||||
continue
|
||||
else:
|
||||
if skip_name:
|
||||
if object_name < skip_name:
|
||||
continue
|
||||
else:
|
||||
skip_name = None
|
||||
end = object_name.find(delimiter, len(prefix))
|
||||
if end > 0:
|
||||
dir_name = object_name[:end + 1]
|
||||
if dir_name != marker:
|
||||
yield dir_name
|
||||
skip_name = object_name[:end] + chr(ord(delimiter) + 1)
|
||||
continue
|
||||
yield object_name
|
||||
|
||||
|
||||
def filter_marker(objects, marker):
|
||||
"""
|
||||
TODO: We can traverse in reverse order to optimize.
|
||||
Accept sorted list.
|
||||
Accept sorted list of strings, return all strings whose value is strictly
|
||||
greater than the given marker value.
|
||||
"""
|
||||
filtered_objs = []
|
||||
if objects[-1] < marker:
|
||||
return filtered_objs
|
||||
for object_name in objects:
|
||||
if object_name > marker:
|
||||
filtered_objs.append(object_name)
|
||||
yield object_name
|
||||
|
||||
return filtered_objs
|
||||
|
||||
def filter_prefix_as_marker(objects, prefix):
|
||||
"""
|
||||
Accept sorted list of strings, return all strings whose value is greater
|
||||
than or equal to the given prefix value.
|
||||
"""
|
||||
for object_name in objects:
|
||||
if object_name >= prefix:
|
||||
yield object_name
|
||||
|
||||
|
||||
def filter_end_marker(objects, end_marker):
|
||||
"""
|
||||
Accept sorted list.
|
||||
Accept a list of strings, sorted, and return all the strings that are
|
||||
strictly less than the given end_marker string. We perform this as a
|
||||
generator to avoid creating potentially large intermediate object lists.
|
||||
"""
|
||||
filtered_objs = []
|
||||
for object_name in objects:
|
||||
if object_name < end_marker:
|
||||
filtered_objs.append(object_name)
|
||||
yield object_name
|
||||
else:
|
||||
break
|
||||
|
||||
return filtered_objs
|
||||
|
||||
|
||||
def filter_limit(objects, limit):
|
||||
filtered_objs = []
|
||||
for i in range(0, limit):
|
||||
filtered_objs.append(objects[i])
|
||||
|
||||
return filtered_objs
|
||||
|
||||
|
||||
class DiskCommon(object):
|
||||
def is_deleted(self):
|
||||
@ -264,53 +289,82 @@ class DiskDir(DiskCommon):
|
||||
"""
|
||||
Returns tuple of name, created_at, size, content_type, etag.
|
||||
"""
|
||||
assert limit >= 0
|
||||
assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254)
|
||||
|
||||
if path is not None:
|
||||
prefix = path
|
||||
if path:
|
||||
prefix = path = path.rstrip('/') + '/'
|
||||
else:
|
||||
prefix = path
|
||||
delimiter = '/'
|
||||
elif delimiter and not prefix:
|
||||
prefix = ''
|
||||
|
||||
objects = self.update_object_count()
|
||||
container_list = []
|
||||
|
||||
objects = self.update_object_count()
|
||||
if objects:
|
||||
objects.sort()
|
||||
|
||||
if objects and prefix:
|
||||
objects = filter_prefix(objects, prefix)
|
||||
|
||||
if objects and delimiter:
|
||||
objects = filter_delimiter(objects, delimiter, prefix)
|
||||
|
||||
if objects and marker:
|
||||
objects = filter_marker(objects, marker)
|
||||
else:
|
||||
return container_list
|
||||
|
||||
if objects and end_marker:
|
||||
objects = filter_end_marker(objects, end_marker)
|
||||
|
||||
if objects and limit:
|
||||
if len(objects) > limit:
|
||||
objects = filter_limit(objects, limit)
|
||||
|
||||
container_list = []
|
||||
if objects:
|
||||
for obj in objects:
|
||||
obj_path = os.path.join(self.datadir, obj)
|
||||
metadata = read_metadata(obj_path)
|
||||
if not metadata or not validate_object(metadata):
|
||||
metadata = create_object_metadata(obj_path)
|
||||
if Glusterfs.OBJECT_ONLY and metadata \
|
||||
and metadata[X_CONTENT_TYPE] == DIR_TYPE:
|
||||
continue
|
||||
list_item = []
|
||||
list_item.append(obj)
|
||||
if metadata:
|
||||
list_item.append(metadata[X_TIMESTAMP])
|
||||
list_item.append(int(metadata[X_CONTENT_LENGTH]))
|
||||
list_item.append(metadata[X_CONTENT_TYPE])
|
||||
list_item.append(metadata[X_ETAG])
|
||||
container_list.append(list_item)
|
||||
if marker and marker >= prefix:
|
||||
objects = filter_marker(objects, marker)
|
||||
elif prefix:
|
||||
objects = filter_prefix_as_marker(objects, prefix)
|
||||
|
||||
if prefix is None:
|
||||
# No prefix, we don't need to apply the other arguments, we just
|
||||
# return what we have.
|
||||
pass
|
||||
else:
|
||||
# We have a non-None (for all intents and purposes it is a string)
|
||||
# prefix.
|
||||
if not delimiter:
|
||||
if not prefix:
|
||||
# We have nothing more to do
|
||||
pass
|
||||
else:
|
||||
objects = filter_prefix(objects, prefix)
|
||||
else:
|
||||
objects = filter_delimiter(objects, delimiter, prefix, marker,
|
||||
path)
|
||||
|
||||
count = 0
|
||||
for obj in objects:
|
||||
obj_path = os.path.join(self.datadir, obj)
|
||||
metadata = read_metadata(obj_path)
|
||||
if not metadata or not validate_object(metadata):
|
||||
if delimiter == '/' and obj_path[-1] == delimiter:
|
||||
clean_obj_path = obj_path[:-1]
|
||||
else:
|
||||
clean_obj_path = obj_path
|
||||
try:
|
||||
metadata = create_object_metadata(clean_obj_path)
|
||||
except OSError as e:
|
||||
# FIXME - total hack to get upstream swift ported unit
|
||||
# test cases working for now.
|
||||
if e.errno != errno.ENOENT:
|
||||
raise
|
||||
if Glusterfs.OBJECT_ONLY and metadata \
|
||||
and metadata[X_CONTENT_TYPE] == DIR_TYPE:
|
||||
continue
|
||||
list_item = []
|
||||
list_item.append(obj)
|
||||
if metadata:
|
||||
list_item.append(metadata[X_TIMESTAMP])
|
||||
list_item.append(int(metadata[X_CONTENT_LENGTH]))
|
||||
list_item.append(metadata[X_CONTENT_TYPE])
|
||||
list_item.append(metadata[X_ETAG])
|
||||
container_list.append(list_item)
|
||||
count += 1
|
||||
if count >= limit:
|
||||
break
|
||||
|
||||
return container_list
|
||||
|
||||
@ -525,49 +579,62 @@ class DiskAccount(DiskDir):
|
||||
if delimiter and not prefix:
|
||||
prefix = ''
|
||||
|
||||
account_list = []
|
||||
containers = self.update_container_count()
|
||||
|
||||
if containers:
|
||||
containers.sort()
|
||||
|
||||
if containers and prefix:
|
||||
containers = filter_prefix(containers, prefix)
|
||||
|
||||
if containers and delimiter:
|
||||
containers = filter_delimiter(containers, delimiter, prefix)
|
||||
|
||||
if containers and marker:
|
||||
containers = filter_marker(containers, marker)
|
||||
else:
|
||||
return account_list
|
||||
|
||||
if containers and end_marker:
|
||||
containers = filter_end_marker(containers, end_marker)
|
||||
|
||||
if containers and limit:
|
||||
if len(containers) > limit:
|
||||
containers = filter_limit(containers, limit)
|
||||
|
||||
account_list = []
|
||||
if containers:
|
||||
for cont in containers:
|
||||
list_item = []
|
||||
metadata = None
|
||||
list_item.append(cont)
|
||||
cont_path = os.path.join(self.datadir, cont)
|
||||
metadata = _read_metadata(cont_path)
|
||||
if not metadata or not validate_container(metadata):
|
||||
try:
|
||||
metadata = create_container_metadata(cont_path)
|
||||
except OSError as e:
|
||||
# FIXME - total hack to get port unit test cases
|
||||
# working for now.
|
||||
if e.errno != errno.ENOENT:
|
||||
raise
|
||||
if marker and marker >= prefix:
|
||||
containers = filter_marker(containers, marker)
|
||||
elif prefix:
|
||||
containers = filter_prefix_as_marker(containers, prefix)
|
||||
|
||||
if metadata:
|
||||
list_item.append(metadata[X_OBJECTS_COUNT][0])
|
||||
list_item.append(metadata[X_BYTES_USED][0])
|
||||
list_item.append(0)
|
||||
account_list.append(list_item)
|
||||
if prefix is None:
|
||||
# No prefix, we don't need to apply the other arguments, we just
|
||||
# return what we have.
|
||||
pass
|
||||
else:
|
||||
# We have a non-None (for all intents and purposes it is a string)
|
||||
# prefix.
|
||||
if not delimiter:
|
||||
if not prefix:
|
||||
# We have nothing more to do
|
||||
pass
|
||||
else:
|
||||
containers = filter_prefix(containers, prefix)
|
||||
else:
|
||||
containers = filter_delimiter(containers, delimiter, prefix,
|
||||
marker)
|
||||
|
||||
count = 0
|
||||
for cont in containers:
|
||||
list_item = []
|
||||
metadata = None
|
||||
list_item.append(cont)
|
||||
cont_path = os.path.join(self.datadir, cont)
|
||||
metadata = _read_metadata(cont_path)
|
||||
if not metadata or not validate_container(metadata):
|
||||
try:
|
||||
metadata = create_container_metadata(cont_path)
|
||||
except OSError as e:
|
||||
# FIXME - total hack to get port unit test cases
|
||||
# working for now.
|
||||
if e.errno != errno.ENOENT:
|
||||
raise
|
||||
if metadata:
|
||||
list_item.append(metadata[X_OBJECTS_COUNT][0])
|
||||
list_item.append(metadata[X_BYTES_USED][0])
|
||||
list_item.append(0)
|
||||
account_list.append(list_item)
|
||||
count += 1
|
||||
if count >= limit:
|
||||
break
|
||||
|
||||
return account_list
|
||||
|
||||
|
||||
@ -518,7 +518,7 @@ class TestContainerBroker(unittest.TestCase):
|
||||
# Test swift.common.db.ContainerBroker.get_info
|
||||
broker = self._get_broker(account='test1',
|
||||
container='test2')
|
||||
broker.initialize(normalize_timestamp('1'))
|
||||
broker.initialize(self.initial_ts)
|
||||
|
||||
info = broker.get_info()
|
||||
self.assertEquals(info['account'], 'test1')
|
||||
@ -565,7 +565,7 @@ class TestContainerBroker(unittest.TestCase):
|
||||
def test_set_x_syncs(self):
|
||||
broker = self._get_broker(account='test1',
|
||||
container='test2')
|
||||
broker.initialize(normalize_timestamp('1'))
|
||||
broker.initialize(self.initial_ts)
|
||||
|
||||
info = broker.get_info()
|
||||
self.assertEquals(info['x_container_sync_point1'], -1)
|
||||
@ -579,7 +579,7 @@ class TestContainerBroker(unittest.TestCase):
|
||||
def test_list_objects_iter(self):
|
||||
# Test swift.common.db.ContainerBroker.list_objects_iter
|
||||
broker = self._get_broker(account='a', container='c')
|
||||
broker.initialize(normalize_timestamp('1'))
|
||||
broker.initialize(self.initial_ts)
|
||||
|
||||
for obj1 in xrange(4):
|
||||
for obj2 in xrange(125):
|
||||
@ -634,17 +634,17 @@ class TestContainerBroker(unittest.TestCase):
|
||||
self.assertEquals(len(listing), 0)
|
||||
|
||||
listing = broker.list_objects_iter(10, '2.d/0050', None, '2.d/', '/')
|
||||
self.assertEquals(len(listing), 9)
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals(listing[0][0], '2.d/0051')
|
||||
self.assertEquals(listing[1][0], '2.d/0052')
|
||||
self.assertEquals(listing[-1][0], '2.d/0059')
|
||||
self.assertEquals(listing[-1][0], '2.d/0060')
|
||||
|
||||
listing = broker.list_objects_iter(10, '3.d/0045', None, '3.d/', '/')
|
||||
self.assertEquals(len(listing), 5)
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['3.d/0046', '3.d/0047',
|
||||
'3.d/0048', '3.d/0049',
|
||||
'3.d/0050'])
|
||||
['3.d/0046', '3.d/0047', '3.d/0048', '3.d/0049',
|
||||
'3.d/0050', '3.d/0051', '3.d/0052', '3.d/0053',
|
||||
'3.d/0054', '3.d/0055'])
|
||||
|
||||
# FIXME
|
||||
#broker.put_object('3/0049/', normalize_timestamp(time()), 0,
|
||||
@ -657,35 +657,141 @@ class TestContainerBroker(unittest.TestCase):
|
||||
# '3.d/0052', '3.d/0052.d/0049'])
|
||||
|
||||
listing = broker.list_objects_iter(10, '3.d/0048', None, '3.d/', '/')
|
||||
self.assertEquals(len(listing), 5)
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['3.d/0049', '3.d/0050',
|
||||
'3.d/0051', '3.d/0052', '3.d/0053'])
|
||||
['3.d/0049', '3.d/0050', '3.d/0051', '3.d/0052', '3.d/0053',
|
||||
'3.d/0054', '3.d/0055', '3.d/0056', '3.d/0057', '3.d/0058'])
|
||||
|
||||
listing = broker.list_objects_iter(10, None, None, '3.d/0049.d/', '/')
|
||||
self.assertEquals(len(listing), 1)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['3.d/0049.d/0049'])
|
||||
|
||||
# FIXME
|
||||
#listing = broker.list_objects_iter(10, None, None, None, None,
|
||||
# '3.d/0049')
|
||||
#self.assertEquals(len(listing), 1)
|
||||
#self.assertEquals([row[0] for row in listing], ['3.d/0049.d/0049'])
|
||||
listing = broker.list_objects_iter(10, None, None, None, None,
|
||||
'3.d/0049.d')
|
||||
self.assertEquals(len(listing), 1)
|
||||
self.assertEquals([row[0] for row in listing], ['3.d/0049.d/0049'])
|
||||
|
||||
listing = broker.list_objects_iter(2, None, None, '3.d/', '/')
|
||||
self.assertEquals(len(listing), 1)
|
||||
self.assertEquals([row[0] for row in listing], ['3.d/0000'])
|
||||
self.assertEquals(len(listing), 2)
|
||||
self.assertEquals([row[0] for row in listing], ['3.d/0000', '3.d/0001'])
|
||||
|
||||
# FIXME
|
||||
#listing = broker.list_objects_iter(2, None, None, None, None, '3')
|
||||
#self.assertEquals(len(listing), 2)
|
||||
#self.assertEquals([row[0] for row in listing], ['3.d/0000', '3.d/0001'])
|
||||
listing = broker.list_objects_iter(2, None, None, None, None, '3.d')
|
||||
self.assertEquals(len(listing), 2)
|
||||
self.assertEquals([row[0] for row in listing], ['3.d/0000', '3.d/0001'])
|
||||
|
||||
def test_list_objects_iter_non_slash(self):
|
||||
# Test swift.common.db.ContainerBroker.list_objects_iter using a
|
||||
# delimiter that is not a slash
|
||||
broker = self._get_broker(account='a', container='c')
|
||||
broker.initialize(self.initial_ts)
|
||||
|
||||
for obj1 in xrange(4):
|
||||
for obj2 in xrange(125):
|
||||
self._create_file('%d:%04d' % (obj1, obj2))
|
||||
for obj in xrange(125):
|
||||
self._create_file('2:0051:%04d' % obj)
|
||||
for obj in xrange(125):
|
||||
self._create_file('3:%04d:0049' % obj)
|
||||
|
||||
listing = broker.list_objects_iter(100, '', None, None, '')
|
||||
self.assertEquals(len(listing), 100)
|
||||
self.assertEquals(listing[0][0], '0:0000')
|
||||
self.assertEquals(listing[-1][0], '0:0099')
|
||||
|
||||
listing = broker.list_objects_iter(100, '', '0:0050', None, '')
|
||||
self.assertEquals(len(listing), 50)
|
||||
self.assertEquals(listing[0][0], '0:0000')
|
||||
self.assertEquals(listing[-1][0], '0:0049')
|
||||
|
||||
listing = broker.list_objects_iter(100, '0:0099', None, None, '')
|
||||
self.assertEquals(len(listing), 100)
|
||||
self.assertEquals(listing[0][0], '0:0100')
|
||||
self.assertEquals(listing[-1][0], '1:0074')
|
||||
|
||||
listing = broker.list_objects_iter(55, '1:0074', None, None, '')
|
||||
self.assertEquals(len(listing), 55)
|
||||
self.assertEquals(listing[0][0], '1:0075')
|
||||
self.assertEquals(listing[-1][0], '2:0004')
|
||||
|
||||
listing = broker.list_objects_iter(10, '', None, '0:01', '')
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals(listing[0][0], '0:0100')
|
||||
self.assertEquals(listing[-1][0], '0:0109')
|
||||
|
||||
listing = broker.list_objects_iter(10, '', None, '0:', ':')
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals(listing[0][0], '0:0000')
|
||||
self.assertEquals(listing[-1][0], '0:0009')
|
||||
|
||||
# Same as above, but using the path argument, so nothing should be
|
||||
# returned since path uses a '/' as a delimiter.
|
||||
listing = broker.list_objects_iter(10, '', None, None, '', '0')
|
||||
self.assertEquals(len(listing), 0)
|
||||
|
||||
listing = broker.list_objects_iter(10, '', None, '', ':')
|
||||
self.assertEquals(len(listing), 4)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['0:', '1:', '2:', '3:'])
|
||||
|
||||
listing = broker.list_objects_iter(10, '2', None, None, ':')
|
||||
self.assertEquals(len(listing), 2)
|
||||
self.assertEquals([row[0] for row in listing], ['2:', '3:'])
|
||||
|
||||
listing = broker.list_objects_iter(10, '2:', None, None, ':')
|
||||
self.assertEquals(len(listing), 1)
|
||||
self.assertEquals([row[0] for row in listing], ['3:'])
|
||||
|
||||
listing = broker.list_objects_iter(10, '2:0050', None, '2:', ':')
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals(listing[0][0], '2:0051')
|
||||
self.assertEquals(listing[1][0], '2:0051:')
|
||||
self.assertEquals(listing[2][0], '2:0052')
|
||||
self.assertEquals(listing[-1][0], '2:0059')
|
||||
|
||||
listing = broker.list_objects_iter(10, '3:0045', None, '3:', ':')
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['3:0045:', '3:0046', '3:0046:', '3:0047',
|
||||
'3:0047:', '3:0048', '3:0048:', '3:0049',
|
||||
'3:0049:', '3:0050'])
|
||||
|
||||
self._create_file('3:0049:')
|
||||
listing = broker.list_objects_iter(10, '3:0048', None, None, None)
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['3:0048:0049', '3:0049', '3:0049:',
|
||||
'3:0049:0049', '3:0050', '3:0050:0049', '3:0051', '3:0051:0049',
|
||||
'3:0052', '3:0052:0049'])
|
||||
|
||||
listing = broker.list_objects_iter(10, '3:0048', None, '3:', ':')
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['3:0048:', '3:0049', '3:0049:', '3:0050',
|
||||
'3:0050:', '3:0051', '3:0051:', '3:0052', '3:0052:', '3:0053'])
|
||||
|
||||
listing = broker.list_objects_iter(10, None, None, '3:0049:', ':')
|
||||
self.assertEquals(len(listing), 2)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['3:0049:', '3:0049:0049'])
|
||||
|
||||
# Same as above, but using the path argument, so nothing should be
|
||||
# returned since path uses a '/' as a delimiter.
|
||||
listing = broker.list_objects_iter(10, None, None, None, None,
|
||||
'3:0049')
|
||||
self.assertEquals(len(listing), 0)
|
||||
|
||||
listing = broker.list_objects_iter(2, None, None, '3:', ':')
|
||||
self.assertEquals(len(listing), 2)
|
||||
self.assertEquals([row[0] for row in listing], ['3:0000', '3:0000:'])
|
||||
|
||||
listing = broker.list_objects_iter(2, None, None, None, None, '3')
|
||||
self.assertEquals(len(listing), 0)
|
||||
|
||||
def test_list_objects_iter_prefix_delim(self):
|
||||
# Test swift.common.db.ContainerBroker.list_objects_iter
|
||||
broker = self._get_broker(account='a', container='c')
|
||||
broker.initialize(normalize_timestamp('1'))
|
||||
broker.initialize(self.initial_ts)
|
||||
|
||||
os.mkdir(os.path.join(self.container, 'pets'))
|
||||
os.mkdir(os.path.join(self.container, 'pets', 'dogs'))
|
||||
@ -723,7 +829,7 @@ class TestContainerBroker(unittest.TestCase):
|
||||
# Test swift.common.db.ContainerBroker.list_objects_iter for a
|
||||
# container that has an odd file with a trailing delimiter
|
||||
broker = self._get_broker(account='a', container='c')
|
||||
broker.initialize(normalize_timestamp('1'))
|
||||
broker.initialize(self.initial_ts)
|
||||
|
||||
self._create_file('a')
|
||||
self._create_file('a.d/a')
|
||||
@ -769,10 +875,65 @@ class TestContainerBroker(unittest.TestCase):
|
||||
self.assertEquals(len(listing), 2)
|
||||
self.assertEquals([row[0] for row in listing], ['b.d/a', 'b.d/b'])
|
||||
|
||||
def test_double_check_trailing_delimiter_non_slash(self):
|
||||
# Test swift.common.db.ContainerBroker.list_objects_iter for a
|
||||
# container that has an odd file with a trailing delimiter
|
||||
broker = self._get_broker(account='a', container='c')
|
||||
broker.initialize(self.initial_ts)
|
||||
|
||||
self._create_file('a')
|
||||
self._create_file('a:')
|
||||
self._create_file('a:a')
|
||||
self._create_file('a:a:a')
|
||||
self._create_file('a:a:b')
|
||||
self._create_file('a:b')
|
||||
self._create_file('b')
|
||||
self._create_file('b:a')
|
||||
self._create_file('b:b')
|
||||
self._create_file('c')
|
||||
self._create_file('a:0')
|
||||
self._create_file('0')
|
||||
self._create_file('0:')
|
||||
self._create_file('00')
|
||||
self._create_file('0:0')
|
||||
self._create_file('0:00')
|
||||
self._create_file('0:1')
|
||||
self._create_file('0:1:')
|
||||
self._create_file('0:1:0')
|
||||
self._create_file('1')
|
||||
self._create_file('1:')
|
||||
self._create_file('1:0')
|
||||
|
||||
listing = broker.list_objects_iter(25, None, None, None, None)
|
||||
self.assertEquals(len(listing), 22)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['0', '00', '0:', '0:0', '0:00', '0:1', '0:1:', '0:1:0', '1', '1:',
|
||||
'1:0', 'a', 'a:', 'a:0', 'a:a', 'a:a:a', 'a:a:b', 'a:b', 'b', 'b:a',
|
||||
'b:b', 'c'])
|
||||
listing = broker.list_objects_iter(25, None, None, '', ':')
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['0', '00', '0:', '1', '1:', 'a', 'a:', 'b', 'b:', 'c'])
|
||||
listing = broker.list_objects_iter(25, None, None, 'a:', ':')
|
||||
self.assertEquals(len(listing), 5)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['a:', 'a:0', 'a:a', 'a:a:', 'a:b'])
|
||||
listing = broker.list_objects_iter(25, None, None, '0:', ':')
|
||||
self.assertEquals(len(listing), 5)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['0:', '0:0', '0:00', '0:1', '0:1:'])
|
||||
listing = broker.list_objects_iter(25, None, None, '0:1:', ':')
|
||||
self.assertEquals(len(listing), 2)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['0:1:', '0:1:0'])
|
||||
listing = broker.list_objects_iter(25, None, None, 'b:', ':')
|
||||
self.assertEquals(len(listing), 2)
|
||||
self.assertEquals([row[0] for row in listing], ['b:a', 'b:b'])
|
||||
|
||||
def test_metadata(self):
|
||||
# Initializes a good broker for us
|
||||
broker = self._get_broker(account='a', container='c')
|
||||
broker.initialize(normalize_timestamp('1'))
|
||||
broker.initialize(self.initial_ts)
|
||||
|
||||
# Add our first item
|
||||
first_timestamp = normalize_timestamp(1)
|
||||
@ -916,7 +1077,7 @@ class TestAccountBroker(unittest.TestCase):
|
||||
def test_get_info(self):
|
||||
# Test swift.common.db.AccountBroker.get_info
|
||||
broker = self._get_broker(account='test1')
|
||||
broker.initialize(normalize_timestamp('1'))
|
||||
broker.initialize(self.initial_ts)
|
||||
|
||||
info = broker.get_info()
|
||||
self.assertEquals(info['account'], 'test1')
|
||||
@ -947,7 +1108,7 @@ class TestAccountBroker(unittest.TestCase):
|
||||
def test_list_containers_iter(self):
|
||||
# Test swift.common.db.AccountBroker.list_containers_iter
|
||||
broker = self._get_broker(account='a')
|
||||
broker.initialize(normalize_timestamp('1'))
|
||||
broker.initialize(self.initial_ts)
|
||||
for cont1 in xrange(4):
|
||||
for cont2 in xrange(125):
|
||||
self._create_container('%d-%04d' % (cont1, cont2))
|
||||
@ -994,27 +1155,29 @@ class TestAccountBroker(unittest.TestCase):
|
||||
listing = broker.list_containers_iter(10, '', None, '', '-')
|
||||
self.assertEquals(len(listing), 4)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['0', '1', '2', '3'])
|
||||
['0-', '1-', '2-', '3-'])
|
||||
|
||||
listing = broker.list_containers_iter(10, '2-', None, None, '-')
|
||||
self.assertEquals(len(listing), 1)
|
||||
self.assertEquals([row[0] for row in listing], ['3'])
|
||||
self.assertEquals([row[0] for row in listing], ['3-'])
|
||||
|
||||
listing = broker.list_containers_iter(10, '', None, '2', '-')
|
||||
self.assertEquals(len(listing), 1)
|
||||
self.assertEquals([row[0] for row in listing], ['2'])
|
||||
self.assertEquals([row[0] for row in listing], ['2-'])
|
||||
|
||||
listing = broker.list_containers_iter(10, '2-0050', None, '2-', '-')
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals(listing[0][0], '2-0051')
|
||||
self.assertEquals(listing[1][0], '2-0052')
|
||||
self.assertEquals(listing[-1][0], '2-0060')
|
||||
self.assertEquals(listing[1][0], '2-0051-')
|
||||
self.assertEquals(listing[2][0], '2-0052')
|
||||
self.assertEquals(listing[-1][0], '2-0059')
|
||||
|
||||
listing = broker.list_containers_iter(10, '3-0045', None, '3-', '-')
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['3-0046', '3-0047', '3-0048', '3-0049', '3-0050',
|
||||
'3-0051', '3-0052', '3-0053', '3-0054', '3-0055'])
|
||||
['3-0045-', '3-0046', '3-0046-', '3-0047',
|
||||
'3-0047-', '3-0048', '3-0048-', '3-0049',
|
||||
'3-0049-', '3-0050'])
|
||||
|
||||
self._create_container('3-0049-')
|
||||
listing = broker.list_containers_iter(10, '3-0048', None, None, None)
|
||||
@ -1027,8 +1190,9 @@ class TestAccountBroker(unittest.TestCase):
|
||||
listing = broker.list_containers_iter(10, '3-0048', None, '3-', '-')
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['3-0049', '3-0050', '3-0051', '3-0052', '3-0053',
|
||||
'3-0054', '3-0055', '3-0056', '3-0057', '3-0058'])
|
||||
['3-0048-', '3-0049', '3-0049-', '3-0050',
|
||||
'3-0050-', '3-0051', '3-0051-', '3-0052',
|
||||
'3-0052-', '3-0053'])
|
||||
|
||||
listing = broker.list_containers_iter(10, None, None, '3-0049-', '-')
|
||||
self.assertEquals(len(listing), 2)
|
||||
@ -1037,9 +1201,9 @@ class TestAccountBroker(unittest.TestCase):
|
||||
|
||||
def test_double_check_trailing_delimiter(self):
|
||||
# Test swift.common.db.AccountBroker.list_containers_iter for an
|
||||
# account that has an odd file with a trailing delimiter
|
||||
# account that has an odd container with a trailing delimiter
|
||||
broker = self._get_broker(account='a')
|
||||
broker.initialize(normalize_timestamp('1'))
|
||||
broker.initialize(self.initial_ts)
|
||||
self._create_container('a')
|
||||
self._create_container('a-')
|
||||
self._create_container('a-a')
|
||||
@ -1050,26 +1214,27 @@ class TestAccountBroker(unittest.TestCase):
|
||||
self._create_container('b-a')
|
||||
self._create_container('b-b')
|
||||
self._create_container('c')
|
||||
|
||||
listing = broker.list_containers_iter(15, None, None, None, None)
|
||||
self.assertEquals(len(listing), 10)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['a', 'a-', 'a-a', 'a-a-a', 'a-a-b', 'a-b', 'b',
|
||||
'b-a', 'b-b', 'c'])
|
||||
listing = broker.list_containers_iter(15, None, None, '', '-')
|
||||
self.assertEquals(len(listing), 3)
|
||||
self.assertEquals(len(listing), 5)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['a', 'b', 'c'])
|
||||
['a', 'a-', 'b', 'b-', 'c'])
|
||||
listing = broker.list_containers_iter(15, None, None, 'a-', '-')
|
||||
self.assertEquals(len(listing), 3)
|
||||
self.assertEquals(len(listing), 4)
|
||||
self.assertEquals([row[0] for row in listing],
|
||||
['a-', 'a-a', 'a-b'])
|
||||
['a-', 'a-a', 'a-a-', 'a-b'])
|
||||
listing = broker.list_containers_iter(15, None, None, 'b-', '-')
|
||||
self.assertEquals(len(listing), 2)
|
||||
self.assertEquals([row[0] for row in listing], ['b-a', 'b-b'])
|
||||
|
||||
def test_delete_db(self):
|
||||
broker = self._get_broker(account='a')
|
||||
broker.initialize(normalize_timestamp('1'))
|
||||
broker.initialize(self.initial_ts)
|
||||
self.assertEqual(broker.db_file, dd._db_file)
|
||||
self.assertEqual(os.path.basename(broker.db_file), 'db_file.db')
|
||||
broker.initialize(self.initial_ts)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user