diff --git a/doc/source/api/object_api_v1_overview.rst b/doc/source/api/object_api_v1_overview.rst index 0f6d7d9d22..a0a472869c 100644 --- a/doc/source/api/object_api_v1_overview.rst +++ b/doc/source/api/object_api_v1_overview.rst @@ -128,7 +128,24 @@ If you have a large number of containers or objects, you can use query parameters to page through large lists of containers or objects. Use the *``marker``*, *``limit``*, and *``end_marker``* query parameters to control how many items are returned in a list and where the list starts -or ends. +or ends. If you want to page through in reverse order, you can use the query +parameter *``reverse``*, noting that your marker and end_markers will be +applied to a reverse listing should be switched. I.e, for a list of objects +``[a, b, c, d, e]`` the non-reversed could be: + +.. code:: + + /v1/{account}/{container}/?marker=a&end_marker=d + b + c + +However, when reversed marker and end_marker are applied to a reversed list: + +.. code:: + + /v1/{account}/{container}/?marker=d&end_marker=a&reverse=on + c + b Object Storage HTTP requests have the following default constraints. Your service provider might use different default values. diff --git a/swift/account/backend.py b/swift/account/backend.py index c91f37fe9b..9a71eb8634 100644 --- a/swift/account/backend.py +++ b/swift/account/backend.py @@ -366,7 +366,7 @@ class AccountBroker(DatabaseBroker): ''').fetchone()) def list_containers_iter(self, limit, marker, end_marker, prefix, - delimiter): + delimiter, reverse=False): """ Get a list of containers sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not have @@ -377,15 +377,21 @@ class AccountBroker(DatabaseBroker): :param end_marker: end marker query :param prefix: prefix query :param delimiter: delimiter for query + :param reverse: reverse the result order. :returns: list of tuples of (name, object_count, bytes_used, 0) """ delim_force_gte = False (marker, end_marker, prefix, delimiter) = utf8encode( marker, end_marker, prefix, delimiter) + if reverse: + # Reverse the markers if we are reversing the listing. + marker, end_marker = end_marker, marker self._commit_puts_stale_ok() if delimiter and not prefix: prefix = '' + if prefix: + end_prefix = prefix[:-1] + chr(ord(prefix[-1]) + 1) orig_marker = marker with self.get() as conn: results = [] @@ -395,9 +401,13 @@ class AccountBroker(DatabaseBroker): FROM container WHERE """ query_args = [] - if end_marker: + if end_marker and (not prefix or end_marker < end_prefix): query += ' name < ? AND' query_args.append(end_marker) + elif prefix: + query += ' name < ? AND' + query_args.append(end_prefix) + if delim_force_gte: query += ' name >= ? AND' query_args.append(marker) @@ -413,38 +423,40 @@ class AccountBroker(DatabaseBroker): query += ' +deleted = 0' else: query += ' deleted = 0' - query += ' ORDER BY name LIMIT ?' + query += ' ORDER BY name %s LIMIT ?' % \ + ('DESC' if reverse else '') query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None - if prefix is None: - # A delimiter without a specified prefix is ignored + # Delimiters without a prefix is ignored, further if there + # is no delimiter then we can simply return the result as + # prefixes are now handled in the SQL statement. + if prefix is None or not delimiter: return [r for r in curs] - if not delimiter: - if not prefix: - # It is possible to have a delimiter but no prefix - # specified. As above, the prefix will be set to the - # empty string, so avoid performing the extra work to - # check against an empty prefix. - return [r for r in curs] - else: - return [r for r in curs if r[0].startswith(prefix)] # We have a delimiter and a prefix (possibly empty string) to # handle rowcount = 0 for row in curs: rowcount += 1 - marker = name = row[0] + name = row[0] + if reverse: + end_marker = name + else: + marker = name + if len(results) >= limit or not name.startswith(prefix): curs.close() return results end = name.find(delimiter, len(prefix)) if end > 0: - marker = name[:end] + chr(ord(delimiter) + 1) - # we want result to be inclusive of delim+1 - delim_force_gte = True + if reverse: + end_marker = name[:end + 1] + else: + marker = name[:end] + chr(ord(delimiter) + 1) + # we want result to be inclusive of delim+1 + delim_force_gte = True dir_name = name[:end + 1] if dir_name != orig_marker: results.append([dir_name, 0, 0, 1]) diff --git a/swift/account/server.py b/swift/account/server.py index c48b191795..8795844afc 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -191,6 +191,7 @@ class AccountController(BaseStorageServer): return HTTPPreconditionFailed(body='Bad delimiter') limit = constraints.ACCOUNT_LISTING_LIMIT given_limit = get_param(req, 'limit') + reverse = config_true_value(get_param(req, 'reverse')) if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > constraints.ACCOUNT_LISTING_LIMIT: @@ -211,7 +212,7 @@ class AccountController(BaseStorageServer): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response(account, req, out_content_type, broker, limit, marker, end_marker, prefix, - delimiter) + delimiter, reverse) @public @replication diff --git a/swift/account/utils.py b/swift/account/utils.py index d9b2b77396..4e83b4c3c7 100644 --- a/swift/account/utils.py +++ b/swift/account/utils.py @@ -70,14 +70,14 @@ def get_response_headers(broker): def account_listing_response(account, req, response_content_type, broker=None, limit='', marker='', end_marker='', prefix='', - delimiter=''): + delimiter='', reverse=False): if broker is None: broker = FakeAccountBroker() resp_headers = get_response_headers(broker) account_list = broker.list_containers_iter(limit, marker, end_marker, - prefix, delimiter) + prefix, delimiter, reverse) if response_content_type == 'application/json': data = [] for (name, object_count, bytes_used, is_subdir) in account_list: diff --git a/swift/container/backend.py b/swift/container/backend.py index 6df9b73498..70ce1f3396 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -557,7 +557,7 @@ class ContainerBroker(DatabaseBroker): conn.commit() def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, - path=None, storage_policy_index=0): + path=None, storage_policy_index=0, reverse=False): """ Get a list of objects sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not @@ -570,6 +570,7 @@ class ContainerBroker(DatabaseBroker): :param delimiter: delimiter for query :param path: if defined, will set the prefix and delimiter based on the path + :param reverse: reverse the result order. :returns: list of tuples of (name, created_at, size, content_type, etag) @@ -578,6 +579,9 @@ class ContainerBroker(DatabaseBroker): (marker, end_marker, prefix, delimiter, path) = utf8encode( marker, end_marker, prefix, delimiter, path) self._commit_puts_stale_ok() + if reverse: + # Reverse the markers if we are reversing the listing. + marker, end_marker = end_marker, marker if path is not None: prefix = path if path: @@ -585,6 +589,8 @@ class ContainerBroker(DatabaseBroker): delimiter = '/' elif delimiter and not prefix: prefix = '' + if prefix: + end_prefix = prefix[:-1] + chr(ord(prefix[-1]) + 1) orig_marker = marker with self.get() as conn: results = [] @@ -592,9 +598,13 @@ class ContainerBroker(DatabaseBroker): query = '''SELECT name, created_at, size, content_type, etag FROM object WHERE''' query_args = [] - if end_marker: + if end_marker and (not prefix or end_marker < end_prefix): query += ' name < ? AND' query_args.append(end_marker) + elif prefix: + query += ' name < ? AND' + query_args.append(end_prefix) + if delim_force_gte: query += ' name >= ? AND' query_args.append(marker) @@ -611,8 +621,8 @@ class ContainerBroker(DatabaseBroker): else: query += ' deleted = 0' orig_tail_query = ''' - ORDER BY name LIMIT ? - ''' + ORDER BY name %s LIMIT ? + ''' % ('DESC' if reverse else '') orig_tail_args = [limit - len(results)] # storage policy filter policy_tail_query = ''' @@ -633,25 +643,23 @@ class ContainerBroker(DatabaseBroker): tuple(query_args + tail_args)) curs.row_factory = None - if prefix is None: - # A delimiter without a specified prefix is ignored + # Delimiters without a prefix is ignored, further if there + # is no delimiter then we can simply return the result as + # prefixes are now handled in the SQL statement. + if prefix is None or not delimiter: return [r for r in curs] - if not delimiter: - if not prefix: - # It is possible to have a delimiter but no prefix - # specified. As above, the prefix will be set to the - # empty string, so avoid performing the extra work to - # check against an empty prefix. - return [r for r in curs] - else: - return [r for r in curs if r[0].startswith(prefix)] # We have a delimiter and a prefix (possibly empty string) to # handle rowcount = 0 for row in curs: rowcount += 1 - marker = name = row[0] + name = row[0] + if reverse: + end_marker = name + else: + marker = name + if len(results) >= limit or not name.startswith(prefix): curs.close() return results @@ -660,13 +668,19 @@ class ContainerBroker(DatabaseBroker): if name == path: continue if end >= 0 and len(name) > end + len(delimiter): - marker = name[:end] + chr(ord(delimiter) + 1) + if reverse: + end_marker = name[:end + 1] + else: + marker = name[:end] + chr(ord(delimiter) + 1) curs.close() break elif end > 0: - marker = name[:end] + chr(ord(delimiter) + 1) - # we want result to be inclusive of delim+1 - delim_force_gte = True + if reverse: + end_marker = name[:end + 1] + else: + marker = name[:end] + chr(ord(delimiter) + 1) + # we want result to be inclusive of delim+1 + delim_force_gte = True dir_name = name[:end + 1] if dir_name != orig_marker: results.append([dir_name, '0', 0, None, '']) diff --git a/swift/container/server.py b/swift/container/server.py index 293d27f489..74b0056752 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -452,6 +452,7 @@ class ContainerController(BaseStorageServer): end_marker = get_param(req, 'end_marker') limit = constraints.CONTAINER_LISTING_LIMIT given_limit = get_param(req, 'limit') + reverse = config_true_value(get_param(req, 'reverse')) if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > constraints.CONTAINER_LISTING_LIMIT: @@ -471,7 +472,7 @@ class ContainerController(BaseStorageServer): return HTTPNotFound(request=req, headers=resp_headers) container_list = broker.list_objects_iter( limit, marker, end_marker, prefix, delimiter, path, - storage_policy_index=info['storage_policy_index']) + storage_policy_index=info['storage_policy_index'], reverse=reverse) return self.create_listing(req, out_content_type, info, resp_headers, broker.metadata, container_list, container) diff --git a/test/functional/tests.py b/test/functional/tests.py index 8f7e2e8d17..911ef46d5d 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -327,6 +327,77 @@ class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers): set_up = False +class TestAccountSortingEnv(object): + @classmethod + def setUp(cls): + cls.conn = Connection(tf.config) + cls.conn.authenticate() + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) + cls.account.delete_containers() + + postfix = Utils.create_name() + cls.cont_items = ('a1', 'a2', 'A3', 'b1', 'B2', 'a10', 'b10', 'zz') + cls.cont_items = ['%s%s' % (x, postfix) for x in cls.cont_items] + + for container in cls.cont_items: + c = cls.account.container(container) + if not c.create(): + raise ResponseError(cls.conn.response) + + +class TestAccountSorting(Base): + env = TestAccountSortingEnv + set_up = False + + def testAccountContainerListSorting(self): + # name (byte order) sorting. + cont_list = sorted(self.env.cont_items) + cont_list.reverse() + cont_listing = self.env.account.containers(parms={'reverse': 'on'}) + self.assert_status(200) + self.assertEqual(cont_list, cont_listing) + + def testAccountContainerListSortingByPrefix(self): + cont_list = sorted(c for c in self.env.cont_items if c.startswith('a')) + cont_list.reverse() + cont_listing = self.env.account.containers(parms={ + 'reverse': 'on', 'prefix': 'a'}) + self.assert_status(200) + self.assertEqual(cont_list, cont_listing) + + def testAccountContainerListSortingByMarkersExclusive(self): + first_item = self.env.cont_items[3] # 'b1' + postfix + last_item = self.env.cont_items[4] # 'B2' + postfix + + cont_list = sorted(c for c in self.env.cont_items + if last_item < c < first_item) + cont_list.reverse() + cont_listing = self.env.account.containers(parms={ + 'reverse': 'on', 'marker': first_item, 'end_marker': last_item}) + self.assert_status(200) + self.assertEqual(cont_list, cont_listing) + + def testAccountContainerListSortingByMarkersInclusive(self): + first_item = self.env.cont_items[3] # 'b1' + postfix + last_item = self.env.cont_items[4] # 'B2' + postfix + + cont_list = sorted(c for c in self.env.cont_items + if last_item <= c <= first_item) + cont_list.reverse() + cont_listing = self.env.account.containers(parms={ + 'reverse': 'on', 'marker': first_item + '\x00', + 'end_marker': last_item[:-1] + chr(ord(last_item[-1]) - 1)}) + self.assert_status(200) + self.assertEqual(cont_list, cont_listing) + + def testAccountContainerListSortingByReversedMarkers(self): + cont_listing = self.env.account.containers(parms={ + 'reverse': 'on', 'marker': 'B', 'end_marker': 'b1'}) + self.assert_status(204) + self.assertEqual([], cont_listing) + + class TestContainerEnv(object): @classmethod def setUp(cls): @@ -647,6 +718,115 @@ class TestContainerUTF8(Base2, TestContainer): set_up = False +class TestContainerSortingEnv(object): + @classmethod + def setUp(cls): + cls.conn = Connection(tf.config) + cls.conn.authenticate() + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) + cls.account.delete_containers() + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.file_items = ('a1', 'a2', 'A3', 'b1', 'B2', 'a10', 'b10', 'zz') + cls.files = list() + cls.file_size = 128 + for name in cls.file_items: + file_item = cls.container.file(name) + file_item.write_random(cls.file_size) + cls.files.append(file_item.name) + + +class TestContainerSorting(Base): + env = TestContainerSortingEnv + set_up = False + + def testContainerFileListSortingReversed(self): + file_list = list(sorted(self.env.file_items)) + file_list.reverse() + cont_files = self.env.container.files(parms={'reverse': 'on'}) + self.assert_status(200) + self.assertEqual(file_list, cont_files) + + def testContainerFileSortingByPrefixReversed(self): + cont_list = sorted(c for c in self.env.file_items if c.startswith('a')) + cont_list.reverse() + cont_listing = self.env.container.files(parms={ + 'reverse': 'on', 'prefix': 'a'}) + self.assert_status(200) + self.assertEqual(cont_list, cont_listing) + + def testContainerFileSortingByMarkersExclusiveReversed(self): + first_item = self.env.file_items[3] # 'b1' + postfix + last_item = self.env.file_items[4] # 'B2' + postfix + + cont_list = sorted(c for c in self.env.file_items + if last_item < c < first_item) + cont_list.reverse() + cont_listing = self.env.container.files(parms={ + 'reverse': 'on', 'marker': first_item, 'end_marker': last_item}) + self.assert_status(200) + self.assertEqual(cont_list, cont_listing) + + def testContainerFileSortingByMarkersInclusiveReversed(self): + first_item = self.env.file_items[3] # 'b1' + postfix + last_item = self.env.file_items[4] # 'B2' + postfix + + cont_list = sorted(c for c in self.env.file_items + if last_item <= c <= first_item) + cont_list.reverse() + cont_listing = self.env.container.files(parms={ + 'reverse': 'on', 'marker': first_item + '\x00', + 'end_marker': last_item[:-1] + chr(ord(last_item[-1]) - 1)}) + self.assert_status(200) + self.assertEqual(cont_list, cont_listing) + + def testContainerFileSortingByReversedMarkersReversed(self): + cont_listing = self.env.container.files(parms={ + 'reverse': 'on', 'marker': 'B', 'end_marker': 'b1'}) + self.assert_status(204) + self.assertEqual([], cont_listing) + + def testContainerFileListSorting(self): + file_list = list(sorted(self.env.file_items)) + cont_files = self.env.container.files() + self.assert_status(200) + self.assertEqual(file_list, cont_files) + + # Lets try again but with reverse is specifically turned off + cont_files = self.env.container.files(parms={'reverse': 'off'}) + self.assert_status(200) + self.assertEqual(file_list, cont_files) + + cont_files = self.env.container.files(parms={'reverse': 'false'}) + self.assert_status(200) + self.assertEqual(file_list, cont_files) + + cont_files = self.env.container.files(parms={'reverse': 'no'}) + self.assert_status(200) + self.assertEqual(file_list, cont_files) + + cont_files = self.env.container.files(parms={'reverse': ''}) + self.assert_status(200) + self.assertEqual(file_list, cont_files) + + # Lets try again but with a incorrect reverse values + cont_files = self.env.container.files(parms={'reverse': 'foo'}) + self.assert_status(200) + self.assertEqual(file_list, cont_files) + + cont_files = self.env.container.files(parms={'reverse': 'hai'}) + self.assert_status(200) + self.assertEqual(file_list, cont_files) + + cont_files = self.env.container.files(parms={'reverse': 'o=[]::::>'}) + self.assert_status(200) + self.assertEqual(file_list, cont_files) + + class TestContainerPathsEnv(object): @classmethod def setUp(cls): diff --git a/test/unit/account/test_backend.py b/test/unit/account/test_backend.py index 8226195484..9efac1b4a3 100644 --- a/test/unit/account/test_backend.py +++ b/test/unit/account/test_backend.py @@ -416,20 +416,48 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') + listing = broker.list_containers_iter(10, '', None, '0-00', '-', + reverse=True) + self.assertEqual(len(listing), 10) + self.assertEqual(listing[0][0], '0-0099') + self.assertEqual(listing[-1][0], '0-0090') + listing = broker.list_containers_iter(10, '', None, '0-', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0009') + listing = broker.list_containers_iter(10, '', None, '0-', '-', + reverse=True) + self.assertEqual(len(listing), 10) + self.assertEqual(listing[0][0], '0-0124') + self.assertEqual(listing[-1][0], '0-0115') + listing = broker.list_containers_iter(10, '', None, '', '-') self.assertEqual(len(listing), 4) self.assertEqual([row[0] for row in listing], ['0-', '1-', '2-', '3-']) + listing = broker.list_containers_iter(10, '', None, '', '-', + reverse=True) + self.assertEqual(len(listing), 4) + self.assertEqual([row[0] for row in listing], + ['3-', '2-', '1-', '0-']) + listing = broker.list_containers_iter(10, '2-', None, None, '-') self.assertEqual(len(listing), 1) self.assertEqual([row[0] for row in listing], ['3-']) + listing = broker.list_containers_iter(10, '2-', None, None, '-', + reverse=True) + self.assertEqual(len(listing), 2) + self.assertEqual([row[0] for row in listing], ['1-', '0-']) + + listing = broker.list_containers_iter(10, '2.', None, None, '-', + reverse=True) + self.assertEqual(len(listing), 3) + self.assertEqual([row[0] for row in listing], ['2-', '1-', '0-']) + listing = broker.list_containers_iter(10, '', None, '2', '-') self.assertEqual(len(listing), 1) self.assertEqual([row[0] for row in listing], ['2-']) @@ -469,6 +497,147 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual([row[0] for row in listing], ['3-0049-', '3-0049-0049']) + def test_list_objects_iter_order_and_reverse(self): + # Test ContainerBroker.list_objects_iter + broker = AccountBroker(':memory:', account='a') + broker.initialize(Timestamp('1').internal, 0) + + broker.put_container( + 'c1', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx) + broker.put_container( + 'c10', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx) + broker.put_container( + 'C1', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx) + broker.put_container( + 'c2', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx) + broker.put_container( + 'c3', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx) + broker.put_container( + 'C4', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx) + + listing = broker.list_containers_iter(100, None, None, '', '', + reverse=False) + self.assertEqual([row[0] for row in listing], + ['C1', 'C4', 'c1', 'c10', 'c2', 'c3']) + listing = broker.list_containers_iter(100, None, None, '', '', + reverse=True) + self.assertEqual([row[0] for row in listing], + ['c3', 'c2', 'c10', 'c1', 'C4', 'C1']) + listing = broker.list_containers_iter(2, None, None, '', '', + reverse=True) + self.assertEqual([row[0] for row in listing], + ['c3', 'c2']) + listing = broker.list_containers_iter(100, 'c2', 'C4', '', '', + reverse=True) + self.assertEqual([row[0] for row in listing], + ['c10', 'c1']) + + def test_reverse_prefix_delim(self): + expectations = [ + { + 'containers': [ + 'topdir1-subdir1,0-c1', + 'topdir1-subdir1,1-c1', + 'topdir1-subdir1-c1', + ], + 'params': { + 'prefix': 'topdir1-', + 'delimiter': '-', + }, + 'expected': [ + 'topdir1-subdir1,0-', + 'topdir1-subdir1,1-', + 'topdir1-subdir1-', + ], + }, + { + 'containers': [ + 'topdir1-subdir1,0-c1', + 'topdir1-subdir1,1-c1', + 'topdir1-subdir1-c1', + 'topdir1-subdir1.', + 'topdir1-subdir1.-c1', + ], + 'params': { + 'prefix': 'topdir1-', + 'delimiter': '-', + }, + 'expected': [ + 'topdir1-subdir1,0-', + 'topdir1-subdir1,1-', + 'topdir1-subdir1-', + 'topdir1-subdir1.', + 'topdir1-subdir1.-', + ], + }, + { + 'containers': [ + 'topdir1-subdir1-c1', + 'topdir1-subdir1,0-c1', + 'topdir1-subdir1,1-c1', + ], + 'params': { + 'prefix': 'topdir1-', + 'delimiter': '-', + 'reverse': True, + }, + 'expected': [ + 'topdir1-subdir1-', + 'topdir1-subdir1,1-', + 'topdir1-subdir1,0-', + ], + }, + { + 'containers': [ + 'topdir1-subdir1.-c1', + 'topdir1-subdir1.', + 'topdir1-subdir1-c1', + 'topdir1-subdir1-', + 'topdir1-subdir1,', + 'topdir1-subdir1,0-c1', + 'topdir1-subdir1,1-c1', + ], + 'params': { + 'prefix': 'topdir1-', + 'delimiter': '-', + 'reverse': True, + }, + 'expected': [ + 'topdir1-subdir1.-', + 'topdir1-subdir1.', + 'topdir1-subdir1-', + 'topdir1-subdir1,1-', + 'topdir1-subdir1,0-', + 'topdir1-subdir1,', + ], + }, + ] + ts = make_timestamp_iter() + default_listing_params = { + 'limit': 10000, + 'marker': '', + 'end_marker': None, + 'prefix': None, + 'delimiter': None, + } + failures = [] + for expected in expectations: + broker = AccountBroker(':memory:', account='a') + broker.initialize(next(ts).internal, 0) + for name in expected['containers']: + broker.put_container(name, next(ts).internal, 0, 0, 0, + POLICIES.default.idx) + params = default_listing_params.copy() + params.update(expected['params']) + listing = list(c[0] for c in broker.list_containers_iter(**params)) + if listing != expected['expected']: + expected['listing'] = listing + failures.append( + "With containers %(containers)r, the params %(params)r " + "produced %(listing)r instead of %(expected)r" % expected) + self.assertFalse(failures, "Found the following failures:\n%s" % + '\n'.join(failures)) + def test_double_check_trailing_delimiter(self): # Test AccountBroker.list_containers_iter for an # account that has an odd container with a trailing delimiter diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py index b109d21c0d..7167b08917 100644 --- a/test/unit/container/test_backend.py +++ b/test/unit/container/test_backend.py @@ -34,7 +34,8 @@ from swift.common.storage_policy import POLICIES import mock -from test.unit import patch_policies, with_tempdir +from test.unit import (patch_policies, with_tempdir, make_timestamp_iter, + EMPTY_ETAG) from test.unit.common.test_db import TestExampleBroker @@ -773,6 +774,12 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(listing[0][0], '1/0075') self.assertEqual(listing[-1][0], '2/0004') + listing = broker.list_objects_iter(55, '2/0005', None, None, '', + reverse=True) + self.assertEqual(len(listing), 55) + self.assertEqual(listing[0][0], '2/0004') + self.assertEqual(listing[-1][0], '1/0075') + listing = broker.list_objects_iter(10, '', None, '0/01', '') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0/0100') @@ -783,17 +790,34 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(listing[0][0], '0/0000') self.assertEqual(listing[-1][0], '0/0009') + listing = broker.list_objects_iter(10, '', None, '0/', '/', + reverse=True) + self.assertEqual(len(listing), 10) + self.assertEqual(listing[0][0], '0/0124') + self.assertEqual(listing[-1][0], '0/0115') + # Same as above, but using the path argument. listing = broker.list_objects_iter(10, '', None, None, '', '0') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0/0000') self.assertEqual(listing[-1][0], '0/0009') + listing = broker.list_objects_iter(10, '', None, None, '', '0', + reverse=True) + self.assertEqual(len(listing), 10) + self.assertEqual(listing[0][0], '0/0124') + self.assertEqual(listing[-1][0], '0/0115') + listing = broker.list_objects_iter(10, '', None, '', '/') self.assertEqual(len(listing), 4) self.assertEqual([row[0] for row in listing], ['0/', '1/', '2/', '3/']) + listing = broker.list_objects_iter(10, '', None, '', '/', reverse=True) + self.assertEqual(len(listing), 4) + self.assertEqual([row[0] for row in listing], + ['3/', '2/', '1/', '0/']) + listing = broker.list_objects_iter(10, '2', None, None, '/') self.assertEqual(len(listing), 2) self.assertEqual([row[0] for row in listing], ['2/', '3/']) @@ -802,6 +826,16 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(len(listing), 1) self.assertEqual([row[0] for row in listing], ['3/']) + listing = broker.list_objects_iter(10, '2/', None, None, '/', + reverse=True) + self.assertEqual(len(listing), 2) + self.assertEqual([row[0] for row in listing], ['1/', '0/']) + + listing = broker.list_objects_iter(10, '20', None, None, '/', + reverse=True) + self.assertEqual(len(listing), 3) + self.assertEqual([row[0] for row in listing], ['2/', '1/', '0/']) + listing = broker.list_objects_iter(10, '2/0050', None, '2/', '/') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '2/0051') @@ -852,6 +886,113 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(len(listing), 2) self.assertEqual([row[0] for row in listing], ['3/0000', '3/0001']) + def test_reverse_prefix_delim(self): + expectations = [ + { + 'objects': [ + 'topdir1/subdir1.0/obj1', + 'topdir1/subdir1.1/obj1', + 'topdir1/subdir1/obj1', + ], + 'params': { + 'prefix': 'topdir1/', + 'delimiter': '/', + }, + 'expected': [ + 'topdir1/subdir1.0/', + 'topdir1/subdir1.1/', + 'topdir1/subdir1/', + ], + }, + { + 'objects': [ + 'topdir1/subdir1.0/obj1', + 'topdir1/subdir1.1/obj1', + 'topdir1/subdir1/obj1', + 'topdir1/subdir10', + 'topdir1/subdir10/obj1', + ], + 'params': { + 'prefix': 'topdir1/', + 'delimiter': '/', + }, + 'expected': [ + 'topdir1/subdir1.0/', + 'topdir1/subdir1.1/', + 'topdir1/subdir1/', + 'topdir1/subdir10', + 'topdir1/subdir10/', + ], + }, + { + 'objects': [ + 'topdir1/subdir1/obj1', + 'topdir1/subdir1.0/obj1', + 'topdir1/subdir1.1/obj1', + ], + 'params': { + 'prefix': 'topdir1/', + 'delimiter': '/', + 'reverse': True, + }, + 'expected': [ + 'topdir1/subdir1/', + 'topdir1/subdir1.1/', + 'topdir1/subdir1.0/', + ], + }, + { + 'objects': [ + 'topdir1/subdir10/obj1', + 'topdir1/subdir10', + 'topdir1/subdir1/obj1', + 'topdir1/subdir1.0/obj1', + 'topdir1/subdir1.1/obj1', + ], + 'params': { + 'prefix': 'topdir1/', + 'delimiter': '/', + 'reverse': True, + }, + 'expected': [ + 'topdir1/subdir10/', + 'topdir1/subdir10', + 'topdir1/subdir1/', + 'topdir1/subdir1.1/', + 'topdir1/subdir1.0/', + ], + }, + ] + ts = make_timestamp_iter() + default_listing_params = { + 'limit': 10000, + 'marker': '', + 'end_marker': None, + 'prefix': None, + 'delimiter': None, + } + obj_create_params = { + 'size': 0, + 'content_type': 'application/test', + 'etag': EMPTY_ETAG, + } + failures = [] + for expected in expectations: + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(next(ts).internal, 0) + for name in expected['objects']: + broker.put_object(name, next(ts).internal, **obj_create_params) + params = default_listing_params.copy() + params.update(expected['params']) + listing = list(o[0] for o in broker.list_objects_iter(**params)) + if listing != expected['expected']: + expected['listing'] = listing + failures.append( + "With objects %(objects)r, the params %(params)r " + "produced %(listing)r instead of %(expected)r" % expected) + self.assertFalse(failures, "Found the following failures:\n%s" % + '\n'.join(failures)) + def test_list_objects_iter_non_slash(self): # Test ContainerBroker.list_objects_iter using a # delimiter that is not a slash @@ -1006,6 +1147,47 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual([row[0] for row in listing], ['/pets/fish/a', '/pets/fish/b']) + def test_list_objects_iter_order_and_reverse(self): + # Test ContainerBroker.list_objects_iter + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(Timestamp('1').internal, 0) + + broker.put_object( + 'o1', Timestamp(0).internal, 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object( + 'o10', Timestamp(0).internal, 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object( + 'O1', Timestamp(0).internal, 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object( + 'o2', Timestamp(0).internal, 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object( + 'o3', Timestamp(0).internal, 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object( + 'O4', Timestamp(0).internal, 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + + listing = broker.list_objects_iter(100, None, None, '', '', + reverse=False) + self.assertEqual([row[0] for row in listing], + ['O1', 'O4', 'o1', 'o10', 'o2', 'o3']) + listing = broker.list_objects_iter(100, None, None, '', '', + reverse=True) + self.assertEqual([row[0] for row in listing], + ['o3', 'o2', 'o10', 'o1', 'O4', 'O1']) + listing = broker.list_objects_iter(2, None, None, '', '', + reverse=True) + self.assertEqual([row[0] for row in listing], + ['o3', 'o2']) + listing = broker.list_objects_iter(100, 'o2', 'O4', '', '', + reverse=True) + self.assertEqual([row[0] for row in listing], + ['o10', 'o1']) + def test_double_check_trailing_delimiter(self): # Test ContainerBroker.list_objects_iter for a # container that has an odd file with a trailing delimiter