diff --git a/swift/container/sharder.py b/swift/container/sharder.py index 7273410754..a3847c8194 100644 --- a/swift/container/sharder.py +++ b/swift/container/sharder.py @@ -1168,6 +1168,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator): params = params or {} params.setdefault('format', 'json') headers = {'X-Backend-Record-Type': 'shard', + 'X-Backend-Record-Shard-Format': 'full', 'X-Backend-Override-Deleted': 'true', 'X-Backend-Include-Deleted': str(include_deleted)} if newest: diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index f1673f268d..f19f160089 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -44,8 +44,8 @@ from swift.common.wsgi import make_pre_authed_env, make_pre_authed_request from swift.common.utils import Timestamp, WatchdogTimeout, config_true_value, \ public, split_path, list_from_csv, GreenthreadSafeIterator, \ GreenAsyncPile, quorum_size, parse_content_type, drain_and_close, \ - document_iters_to_http_response_body, ShardRange, cache_from_env, \ - CooperativeIterator, NamespaceBoundList + document_iters_to_http_response_body, cache_from_env, \ + CooperativeIterator, NamespaceBoundList, Namespace from swift.common.bufferedhttp import http_connect from swift.common import constraints from swift.common.exceptions import ChunkReadTimeout, ChunkWriteTimeout, \ @@ -2459,7 +2459,7 @@ class Controller(object): data = self._parse_listing_response(req, response) return data, response - def _parse_shard_ranges(self, req, listing, response): + def _parse_namespaces(self, req, listing, response): if listing is None: return None @@ -2471,10 +2471,15 @@ class Controller(object): return None try: - return [ShardRange.from_dict(shard_range) - for shard_range in listing] + # Note: a legacy container-server could return a list of + # ShardRanges, but that's ok: namespaces just need 'name', 'lower' + # and 'upper' keys. If we ever need to know we can look for a + # 'x-backend-record-shard-format' header from newer container + # servers. + return [Namespace(data['name'], data['lower'], data['upper']) + for data in listing] except (ValueError, TypeError, KeyError) as err: self.logger.error( - "Failed to get shard ranges from %s: invalid data: %r", + "Failed to get namespaces from %s: invalid data: %r", req.path_qs, err) return None diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py index f35bc3fcdc..e8e0ba5a50 100644 --- a/swift/proxy/controllers/container.py +++ b/swift/proxy/controllers/container.py @@ -19,8 +19,7 @@ import six from six.moves.urllib.parse import unquote from swift.common.utils import public, private, csv_append, Timestamp, \ - config_true_value, ShardRange, cache_from_env, filter_namespaces, \ - NamespaceBoundList + config_true_value, cache_from_env, filter_namespaces, NamespaceBoundList from swift.common.constraints import check_metadata, CONTAINER_LISTING_LIMIT from swift.common.http import HTTP_ACCEPTED, is_success from swift.common.request_helpers import get_sys_meta_prefix, get_param, \ @@ -191,7 +190,7 @@ class ContainerController(Controller): # available shard's lower. At worst, some misplaced objects, in the gap # above the shard's upper, may be included in the shard's response. data = self._parse_listing_response(req, resp) - backend_shard_ranges = self._parse_shard_ranges(req, data, resp) + backend_shard_ranges = self._parse_namespaces(req, data, resp) if backend_shard_ranges is None: return None @@ -248,6 +247,10 @@ class ContainerController(Controller): if (resp_record_type == 'shard' and sharding_state == 'sharded' and complete_listing): + # note: old container servers return a list of shard ranges, newer + # ones return a list of namespaces. If we ever need to know we can + # look for a 'x-backend-record-shard-format' header from newer + # container servers. ns_bound_list = self._store_shard_ranges_in_cache(req, resp) if ns_bound_list: resp.body = self._make_namespaces_response_body( @@ -353,6 +356,7 @@ class ContainerController(Controller): else: record_type = 'auto' req.headers['X-Backend-Record-Type'] = 'auto' + req.headers['X-Backend-Record-Shard-Format'] = 'namespace' params['states'] = 'listing' req.params = params @@ -394,10 +398,18 @@ class ContainerController(Controller): if all((req.method == "GET", record_type == 'auto', resp_record_type.lower() == 'shard')): - resp = self._get_from_shards(req, resp) + data = self._parse_listing_response(req, resp) + namespaces = self._parse_namespaces(req, data, resp) + if namespaces is not None: + # we got namespaces, so the container must be sharded; now + # build the listing from shards + # NB: the filtered namespaces list may be empty but we still + # need to build a response body with an empty list of shards + resp = self._get_from_shards(req, resp, namespaces) if orig_record_type not in ('object', 'shard'): resp.headers.pop('X-Backend-Record-Type', None) + resp.headers.pop('X-Backend-Record-Shard-Format', None) if not config_true_value( resp.headers.get('X-Backend-Cached-Results')): @@ -425,17 +437,28 @@ class ContainerController(Controller): 'False')) return resp - def _get_from_shards(self, req, resp): - # Construct listing using shards described by the response body. - # The history of containers that have returned shard ranges is + def _get_from_shards(self, req, resp, namespaces): + """ + Construct an object listing using shards described by the list of + namespaces. + + :param req: an instance of :class:`~swift.common.swob.Request`. + :param resp: an instance of :class:`~swift.common.swob.Response`. + :param namespaces: a list of :class:`~swift.common.utils.Namespace`. + :return: an instance of :class:`~swift.common.swob.Response`. If an + error is encountered while building the listing an instance of + ``HTTPServiceUnavailable`` may be returned. Otherwise, the given + ``resp`` is returned with a body that is an object listing. + """ + # The history of containers that have returned namespaces is # maintained in the request environ so that loops can be avoided by # forcing an object listing if the same container is visited again. # This can happen in at least two scenarios: - # 1. a container has filled a gap in its shard ranges with a - # shard range pointing to itself - # 2. a root container returns a (stale) shard range pointing to a + # 1. a container has filled a gap in its namespaces with a + # namespace pointing to itself + # 2. a root container returns a (stale) namespace pointing to a # shard that has shrunk into the root, in which case the shrunken - # shard may return the root's shard range. + # shard may return the root's namespace. shard_listing_history = req.environ.setdefault( 'swift.shard_listing_history', []) policy_key = 'X-Backend-Storage-Policy-Index' @@ -443,28 +466,15 @@ class ContainerController(Controller): # We're handling the original request to the root container: set # the root policy index in the request, unless it is already set, # so that shards will return listings for that policy index. - # Note: we only get here if the root responded with shard ranges, - # or if the shard ranges were cached and the cached root container + # Note: we only get here if the root responded with namespaces, + # or if the namespaces were cached and the cached root container # info has sharding_state==sharded; in both cases we can assume # that the response is "modern enough" to include # 'X-Backend-Storage-Policy-Index'. req.headers[policy_key] = resp.headers[policy_key] shard_listing_history.append((self.account_name, self.container_name)) - # Note: when the response body has been synthesised from cached data, - # each item in the list only has 'name', 'lower' and 'upper' keys. We - # therefore cannot use ShardRange.from_dict(), and the ShardRange - # instances constructed here will only have 'name', 'lower' and 'upper' - # attributes set. - # Ideally we would construct Namespace objects here, but later we use - # the ShardRange account and container properties to access parsed - # parts of the name. - shard_ranges = [ShardRange(**data) for data in json.loads(resp.body)] self.logger.debug('GET listing from %s shards for: %s', - len(shard_ranges), req.path_qs) - if not shard_ranges: - # can't find ranges or there was a problem getting the ranges. So - # return what we have. - return resp + len(namespaces), req.path_qs) objects = [] req_limit = constrain_req_limit(req, CONTAINER_LISTING_LIMIT) @@ -478,12 +488,12 @@ class ContainerController(Controller): limit = req_limit all_resp_status = [] - for i, shard_range in enumerate(shard_ranges): + for i, namespace in enumerate(namespaces): params['limit'] = limit # Always set marker to ensure that object names less than or equal # to those already in the listing are not fetched; if the listing # is empty then the original request marker, if any, is used. This - # allows misplaced objects below the expected shard range to be + # allows misplaced objects below the expected namespace to be # included in the listing. last_name = '' last_name_was_subdir = False @@ -502,18 +512,18 @@ class ContainerController(Controller): else: params['marker'] = '' # Always set end_marker to ensure that misplaced objects beyond the - # expected shard range are not fetched. This prevents a misplaced + # expected namespace are not fetched. This prevents a misplaced # object obscuring correctly placed objects in the next shard # range. - if end_marker and end_marker in shard_range: + if end_marker and end_marker in namespace: params['end_marker'] = str_to_wsgi(end_marker) elif reverse: - params['end_marker'] = str_to_wsgi(shard_range.lower_str) + params['end_marker'] = str_to_wsgi(namespace.lower_str) else: - params['end_marker'] = str_to_wsgi(shard_range.end_marker) + params['end_marker'] = str_to_wsgi(namespace.end_marker) headers = {} - if ((shard_range.account, shard_range.container) in + if ((namespace.account, namespace.container) in shard_listing_history): # directed back to same container - force GET of objects headers['X-Backend-Record-Type'] = 'object' @@ -521,26 +531,26 @@ class ContainerController(Controller): headers['X-Newest'] = 'true' if prefix: - if prefix > shard_range: + if prefix > namespace: continue try: just_past = prefix[:-1] + chr(ord(prefix[-1]) + 1) except ValueError: pass else: - if just_past < shard_range: + if just_past < namespace: continue if last_name_was_subdir and str( - shard_range.lower if reverse else shard_range.upper + namespace.lower if reverse else namespace.upper ).startswith(last_name): continue self.logger.debug( 'Getting listing part %d from shard %s %s with %s', - i, shard_range, shard_range.name, headers) + i, namespace, namespace.name, headers) objs, shard_resp = self._get_container_listing( - req, shard_range.account, shard_range.container, + req, namespace.account, namespace.container, headers=headers, params=params) all_resp_status.append(shard_resp.status_int) diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 33f78364bb..39ecf55e76 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -48,7 +48,7 @@ from swift.common.utils import ( normalize_delete_at_timestamp, public, get_expirer_container, document_iters_to_http_response_body, parse_content_range, quorum_size, reiterate, close_if_possible, safe_json_loads, md5, - find_namespace, NamespaceBoundList, CooperativeIterator, ShardRange) + NamespaceBoundList, CooperativeIterator) from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation from swift.common import constraints @@ -281,31 +281,32 @@ class BaseObjectController(Controller): """Handler for HTTP HEAD requests.""" return self.GETorHEAD(req) - def _get_updating_shard_ranges( + def _get_updating_namespaces( self, req, account, container, includes=None): """ - Fetch shard ranges in 'updating' states from given `account/container`. + Fetch namespaces in 'updating' states from given `account/container`. If `includes` is given then the shard range for that object name is - requested, otherwise all shard ranges are requested. + requested, otherwise all namespaces are requested. :param req: original Request instance. - :param account: account from which shard ranges should be fetched. - :param container: container from which shard ranges should be fetched. - :param includes: (optional) restricts the list of fetched shard ranges + :param account: account from which namespaces should be fetched. + :param container: container from which namespaces should be fetched. + :param includes: (optional) restricts the list of fetched namespaces to those which include the given name. - :return: a list of instances of :class:`swift.common.utils.ShardRange`, - or None if there was a problem fetching the shard ranges + :return: a list of instances of :class:`swift.common.utils.Namespace`, + or None if there was a problem fetching the namespaces. """ params = req.params.copy() params.pop('limit', None) params['format'] = 'json' params['states'] = 'updating' + headers = {'X-Backend-Record-Type': 'shard', + 'X-Backend-Record-Shard-Format': 'namespace'} if includes: params['includes'] = str_to_wsgi(includes) - headers = {'X-Backend-Record-Type': 'shard'} listing, response = self._get_container_listing( req, account, container, headers=headers, params=params) - return self._parse_shard_ranges(req, listing, response), response + return self._parse_namespaces(req, listing, response), response def _get_update_shard_caching_disabled(self, req, account, container, obj): """ @@ -316,17 +317,17 @@ class BaseObjectController(Controller): :param account: account from which shard ranges should be fetched. :param container: container from which shard ranges should be fetched. :param obj: object getting updated. - :return: an instance of :class:`swift.common.utils.ShardRange`, + :return: an instance of :class:`swift.common.utils.Namespace`, or None if the update should go back to the root """ # legacy behavior requests container server for includes=obj - shard_ranges, response = self._get_updating_shard_ranges( + namespaces, response = self._get_updating_namespaces( req, account, container, includes=obj) record_cache_op_metrics( self.logger, self.server_type.lower(), 'shard_updating', 'disabled', response) - # there will be only one shard range in the list if any - return shard_ranges[0] if shard_ranges else None + # there will be only one Namespace in the list if any + return namespaces[0] if namespaces else None def _get_update_shard(self, req, account, container, obj): """ @@ -340,7 +341,7 @@ class BaseObjectController(Controller): :param account: account from which shard ranges should be fetched. :param container: container from which shard ranges should be fetched. :param obj: object getting updated. - :return: an instance of :class:`swift.common.utils.ShardRange`, + :return: an instance of :class:`swift.common.utils.Namespace`, or None if the update should go back to the root """ if not self.app.recheck_updating_shard_ranges: @@ -354,20 +355,15 @@ class BaseObjectController(Controller): skip_chance = self.app.container_updating_shard_ranges_skip_cache ns_bound_list, get_cache_state = get_namespaces_from_cache( req, cache_key, skip_chance) - if ns_bound_list: - # found cached namespaces in either infocache or memcache - namespace = ns_bound_list.get_namespace(obj) - update_shard = ShardRange( - name=namespace.name, timestamp=0, lower=namespace.lower, - upper=namespace.upper) - else: - # pull full set of updating shard ranges from backend - shard_ranges, response = self._get_updating_shard_ranges( + if not ns_bound_list: + # namespaces not found in either infocache or memcache so pull full + # set of updating shard ranges from backend + namespaces, response = self._get_updating_namespaces( req, account, container) - if shard_ranges: + if namespaces: # only store the list of namespace lower bounds and names into # infocache and memcache. - ns_bound_list = NamespaceBoundList.parse(shard_ranges) + ns_bound_list = NamespaceBoundList.parse(namespaces) set_cache_state = set_namespaces_in_cache( req, cache_key, ns_bound_list, self.app.recheck_updating_shard_ranges) @@ -377,23 +373,22 @@ class BaseObjectController(Controller): if set_cache_state == 'set': self.logger.info( 'Caching updating shards for %s (%d shards)', - cache_key, len(shard_ranges)) - update_shard = find_namespace(obj, shard_ranges or []) + cache_key, len(namespaces)) record_cache_op_metrics( self.logger, self.server_type.lower(), 'shard_updating', get_cache_state, response) - return update_shard + return ns_bound_list.get_namespace(obj) if ns_bound_list else None def _get_update_target(self, req, container_info): # find the sharded container to which we'll send the update db_state = container_info.get('sharding_state', 'unsharded') if db_state in ('sharded', 'sharding'): - shard_range = self._get_update_shard( + update_shard_ns = self._get_update_shard( req, self.account_name, self.container_name, self.object_name) - if shard_range: + if update_shard_ns: partition, nodes = self.app.container_ring.get_nodes( - shard_range.account, shard_range.container) - return partition, nodes, shard_range.name + update_shard_ns.account, update_shard_ns.container) + return partition, nodes, update_shard_ns.name return container_info['partition'], container_info['nodes'], None diff --git a/test/probe/test_sharder.py b/test/probe/test_sharder.py index a129675cff..14a83746e6 100644 --- a/test/probe/test_sharder.py +++ b/test/probe/test_sharder.py @@ -194,6 +194,7 @@ class BaseTestContainerSharding(ReplProbeTest): self.assertEqual('object', resp_record_type) else: self.assertIsNone(resp_record_type) + self.assertNotIn('X-Backend-Record-Shard-Format', resp.headers) return json.loads(resp.body) def get_container_shard_ranges(self, account=None, container=None, diff --git a/test/unit/container/test_sharder.py b/test/unit/container/test_sharder.py index 8029f06e93..46ca3c916c 100644 --- a/test/unit/container/test_sharder.py +++ b/test/unit/container/test_sharder.py @@ -1283,7 +1283,8 @@ class TestSharder(BaseTestSharder): expected_headers = {'X-Backend-Record-Type': 'shard', 'X-Backend-Include-Deleted': 'False', - 'X-Backend-Override-Deleted': 'true'} + 'X-Backend-Override-Deleted': 'true', + 'X-Backend-Record-Shard-Format': 'full'} broker = self._make_broker() shard_ranges = self._make_shard_ranges((('', 'm'), ('m', ''))) @@ -6325,8 +6326,9 @@ class TestSharder(BaseTestSharder): as mocked, mock.patch.object( sharder, 'int_client') as mock_swift: mock_response = mock.MagicMock() - mock_response.headers = {'x-backend-record-type': - 'shard'} + mock_response.headers = { + 'x-backend-record-type': 'shard', + 'X-Backend-Record-Shard-Format': 'full'} shard_ranges.sort(key=ShardRange.sort_key) mock_response.body = json.dumps( [dict(sr) for sr in shard_ranges]) @@ -6348,7 +6350,8 @@ class TestSharder(BaseTestSharder): expected_headers = {'X-Backend-Record-Type': 'shard', 'X-Newest': 'true', 'X-Backend-Include-Deleted': 'True', - 'X-Backend-Override-Deleted': 'true'} + 'X-Backend-Override-Deleted': 'true', + 'X-Backend-Record-Shard-Format': 'full'} params = {'format': 'json', 'marker': marker, 'end_marker': end_marker, 'states': 'auditing'} mock_swift.make_request.assert_called_once_with( @@ -6434,7 +6437,8 @@ class TestSharder(BaseTestSharder): expected_headers = {'X-Backend-Record-Type': 'shard', 'X-Newest': 'true', 'X-Backend-Include-Deleted': 'True', - 'X-Backend-Override-Deleted': 'true'} + 'X-Backend-Override-Deleted': 'true', + 'X-Backend-Record-Shard-Format': 'full'} params = {'format': 'json', 'marker': 'j', 'end_marker': 'k', 'states': 'auditing'} mock_swift.make_request.assert_called_once_with( @@ -6471,7 +6475,8 @@ class TestSharder(BaseTestSharder): expected_headers = {'X-Backend-Record-Type': 'shard', 'X-Newest': 'true', 'X-Backend-Include-Deleted': 'True', - 'X-Backend-Override-Deleted': 'true'} + 'X-Backend-Override-Deleted': 'true', + 'X-Backend-Record-Shard-Format': 'full'} params = {'format': 'json', 'marker': 'k', 'end_marker': 't', 'states': 'auditing'} mock_swift.make_request.assert_called_once_with( @@ -6608,7 +6613,8 @@ class TestSharder(BaseTestSharder): expected_headers = {'X-Backend-Record-Type': 'shard', 'X-Newest': 'true', 'X-Backend-Include-Deleted': 'True', - 'X-Backend-Override-Deleted': 'true'} + 'X-Backend-Override-Deleted': 'true', + 'X-Backend-Record-Shard-Format': 'full'} params = {'format': 'json', 'marker': 'k', 'end_marker': 't', 'states': 'auditing'} mock_swift.make_request.assert_called_once_with( @@ -6690,7 +6696,8 @@ class TestSharder(BaseTestSharder): expected_headers = {'X-Backend-Record-Type': 'shard', 'X-Newest': 'true', 'X-Backend-Include-Deleted': 'True', - 'X-Backend-Override-Deleted': 'true'} + 'X-Backend-Override-Deleted': 'true', + 'X-Backend-Record-Shard-Format': 'full'} params = {'format': 'json', 'marker': 'a', 'end_marker': 'b', 'states': 'auditing'} mock_swift.make_request.assert_called_once_with( @@ -7391,7 +7398,8 @@ class TestSharder(BaseTestSharder): expected_headers = {'X-Backend-Record-Type': 'shard', 'X-Newest': 'true', 'X-Backend-Include-Deleted': 'True', - 'X-Backend-Override-Deleted': 'true'} + 'X-Backend-Override-Deleted': 'true', + 'X-Backend-Record-Shard-Format': 'full'} params = {'format': 'json', 'marker': 'a', 'end_marker': 'd', 'states': 'auditing'} mock_swift.make_request.assert_called_once_with( diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py index 60e5ba817a..1d27579cf2 100644 --- a/test/unit/proxy/controllers/test_container.py +++ b/test/unit/proxy/controllers/test_container.py @@ -21,6 +21,10 @@ import unittest from eventlet import Timeout import six from six.moves import urllib +if six.PY2: + from itertools import izip_longest as zip_longest +else: + from itertools import zip_longest from swift.common.constraints import CONTAINER_LISTING_LIMIT from swift.common.swob import Request, bytes_to_wsgi, str_to_wsgi, wsgi_quote @@ -42,8 +46,7 @@ from test.unit.proxy.test_server import node_error_count @patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) -class TestContainerController(TestRingBase): - +class BaseTestContainerController(TestRingBase): CONTAINER_REPLICAS = 3 def setUp(self): @@ -109,6 +112,8 @@ class TestContainerController(TestRingBase): 'Expected %s but got %s. Failed case: %s' % (expected, resp.status_int, str(responses))) + +class TestContainerController(BaseTestContainerController): def test_container_info_got_cached(self): memcache = FakeMemcache() controller = proxy_server.ContainerController(self.app, 'a', 'c') @@ -465,6 +470,72 @@ class TestContainerController(TestRingBase): ] self._assert_responses('POST', POST_TEST_CASES) + def test_GET_bad_requests(self): + # verify that the proxy controller enforces checks on request params + req = Request.blank( + '/v1/a/c?limit=%d' % (CONTAINER_LISTING_LIMIT + 1)) + self.assertEqual(412, req.get_response(self.app).status_int) + req = Request.blank('/v1/a/c?delimiter=%ff') + self.assertEqual(400, req.get_response(self.app).status_int) + req = Request.blank('/v1/a/c?marker=%ff') + self.assertEqual(400, req.get_response(self.app).status_int) + req = Request.blank('/v1/a/c?end_marker=%ff') + self.assertEqual(400, req.get_response(self.app).status_int) + req = Request.blank('/v1/a/c?prefix=%ff') + self.assertEqual(400, req.get_response(self.app).status_int) + req = Request.blank('/v1/a/c?format=%ff') + self.assertEqual(400, req.get_response(self.app).status_int) + req = Request.blank('/v1/a/c?path=%ff') + self.assertEqual(400, req.get_response(self.app).status_int) + req = Request.blank('/v1/a/c?includes=%ff') + self.assertEqual(400, req.get_response(self.app).status_int) + req = Request.blank('/v1/a/c?states=%ff') + self.assertEqual(400, req.get_response(self.app).status_int) + + +class TestGetShardedContainer(BaseTestContainerController): + RESP_SHARD_FORMAT_HEADERS = {'X-Backend-Record-Shard-Format': 'namespace'} + + def setUp(self): + super(TestGetShardedContainer, self).setUp() + + def _make_root_resp_hdrs(self, object_count, bytes_used, extra_hdrs=None, + extra_shard_hdrs=None): + # basic headers that backend will return... + root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', + 'X-Backend-Timestamp': '99', + 'X-Container-Object-Count': object_count, + 'X-Container-Bytes-Used': bytes_used, + 'X-Container-Meta-Flavour': 'peach', + 'X-Backend-Storage-Policy-Index': 0} + if extra_hdrs: + root_resp_hdrs.update(extra_hdrs) + + # headers returned when namespaces are returned... + root_shard_resp_hdrs = dict(root_resp_hdrs) + root_shard_resp_hdrs.update( + {'X-Backend-Record-Type': 'shard'}) + root_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) + if extra_shard_hdrs: + root_shard_resp_hdrs.update(extra_shard_hdrs) + return root_resp_hdrs, root_shard_resp_hdrs + + def _make_shard_resp_hdrs(self, sr_objs, extra_hdrs=None): + # headers returned from unsharded shard backends... + hdrs = [] + for i, _ in enumerate(sr_objs): + shard_hdrs = {'X-Backend-Sharding-State': 'unsharded', + 'X-Container-Object-Count': len(sr_objs[i]), + 'X-Container-Bytes-Used': + sum([obj['bytes'] for obj in sr_objs[i]]), + 'X-Container-Meta-Flavour': 'flavour%d' % i, + 'X-Backend-Storage-Policy-Index': 0, + 'X-Backend-Record-Type': 'object'} + if extra_hdrs: + shard_hdrs.update(extra_hdrs) + hdrs.append(shard_hdrs) + return hdrs + def _make_shard_objects(self, shard_range): if six.PY2: lower = ord(shard_range.lower.decode('utf8')[0] @@ -550,8 +621,9 @@ class TestContainerController(TestRingBase): req['headers']) return resp - def check_response(self, resp, root_resp_hdrs, expected_objects=None, - exp_sharding_state='sharded'): + def check_listing_response(self, resp, root_resp_hdrs, + expected_objects=None, + exp_sharding_state='sharded'): info_hdrs = dict(root_resp_hdrs) if expected_objects is None: # default is to expect whatever the root container sent @@ -569,6 +641,7 @@ class TestContainerController(TestRingBase): self.assertEqual(exp_sharding_state, resp.headers['X-Backend-Sharding-State']) self.assertNotIn('X-Backend-Record-Type', resp.headers) + self.assertNotIn('X-Backend-Record-Shard-Format', resp.headers) for k, v in root_resp_hdrs.items(): if k.lower().startswith('x-container-meta'): self.assertEqual(v, resp.headers[k]) @@ -576,23 +649,24 @@ class TestContainerController(TestRingBase): info = get_container_info(resp.request.environ, self.app) self.assertEqual(headers_to_container_info(info_hdrs), info) + def create_server_response_data(self, bounds, states=None): + if not isinstance(bounds[0], (list, tuple)): + bounds = [(l, u) for l, u in zip(bounds[:-1], bounds[1:])] + # some tests use bounds with '/' char, so replace this before using the + # upper bound to synthesize a valid container name + namespaces = [Namespace('.shards_a/c_%s' % upper.replace('/', '-'), + lower, upper) + for lower, upper in bounds] + ns_dicts = [dict(ns) for ns in namespaces] + ns_objs = [self._make_shard_objects(ns) for ns in namespaces] + return namespaces, ns_dicts, ns_objs + def test_GET_sharded_container_no_memcache(self): # Don't worry, ShardRange._encode takes care of unicode/bytes issues shard_bounds = ('', 'ham', 'pie', u'\N{SNOWMAN}', u'\U0001F334', '') - shard_ranges = [ - ShardRange('.shards_a/c_%s' % upper, Timestamp.now(), lower, upper) - for lower, upper in zip(shard_bounds[:-1], shard_bounds[1:])] - sr_dicts = [dict(sr, last_modified=sr.timestamp.isoformat) - for sr in shard_ranges] - sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] - shard_resp_hdrs = [ - {'X-Backend-Sharding-State': 'unsharded', - 'X-Container-Object-Count': len(sr_objs[i]), - 'X-Container-Bytes-Used': - sum([obj['bytes'] for obj in sr_objs[i]]), - 'X-Container-Meta-Flavour': 'flavour%d' % i, - 'X-Backend-Storage-Policy-Index': 0} - for i, _ in enumerate(shard_ranges)] + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + shard_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) all_objects = [] for objects in sr_objs: @@ -601,22 +675,16 @@ class TestContainerController(TestRingBase): num_all_objects = len(all_objects) limit = CONTAINER_LISTING_LIMIT expected_objects = all_objects - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Timestamp': '99', - # pretend root object stats are not yet updated - 'X-Container-Object-Count': num_all_objects - 1, - 'X-Container-Bytes-Used': size_all_objects - 1, - 'X-Container-Meta-Flavour': 'peach', - 'X-Backend-Storage-Policy-Index': 0} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + # pretend root object stats are not yet updated + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs( + num_all_objects - 1, size_all_objects - 1) # GET all objects # include some failed responses mock_responses = [ # status, body, headers (404, '', {}), - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (200, sr_objs[1], shard_resp_hdrs[1]), (200, sr_objs[2], shard_resp_hdrs[2]), @@ -629,29 +697,29 @@ class TestContainerController(TestRingBase): dict(states='listing')), # 404 ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[0].name)), + (wsgi_quote(str_to_wsgi(namespaces[0].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='', end_marker='ham\x00', limit=str(limit), states='listing')), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), + (wsgi_quote(str_to_wsgi(namespaces[1].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[2].name)), + (wsgi_quote(str_to_wsgi(namespaces[2].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1])))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='\xd1\xb0', end_marker='\xf0\x9f\x8c\xb4\x00', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1] + sr_objs[2])))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[4].name)), + (wsgi_quote(str_to_wsgi(namespaces[4].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='\xe2\xa8\x83', end_marker='', states='listing', @@ -662,26 +730,26 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests) # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, req_headers={'X-Backend-Record-Type': 'auto'}) - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, req_headers={'X-Backend-Record-Type': 'banana'}) - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) # GET all objects - sharding, final shard range points back to root root_range = ShardRange('a/c', Timestamp.now(), 'pie', '') mock_responses = [ # status, body, headers - (200, sr_dicts[:2] + [dict(root_range)], root_shard_resp_hdrs), + (200, ns_dicts[:2] + [dict(root_range)], root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (200, sr_objs[1], shard_resp_hdrs[1]), (200, sr_objs[2] + sr_objs[3] + sr_objs[4], root_resp_hdrs) @@ -690,12 +758,12 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 - (shard_ranges[0].name, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='', end_marker='ham\x00', limit=str(limit), states='listing')), # 200 - (shard_ranges[1].name, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='h', end_marker='pie\x00', states='listing', @@ -710,14 +778,14 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests) # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) # GET all objects in reverse and *blank* limit mock_responses = [ # status, body, headers # NB: the backend returns reversed shard range list - (200, list(reversed(sr_dicts)), root_shard_resp_hdrs), + (200, list(reversed(ns_dicts)), root_shard_resp_hdrs), (200, list(reversed(sr_objs[4])), shard_resp_hdrs[4]), (200, list(reversed(sr_objs[3])), shard_resp_hdrs[3]), (200, list(reversed(sr_objs[2])), shard_resp_hdrs[2]), @@ -728,31 +796,31 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing', reverse='true', limit='')), - (wsgi_quote(str_to_wsgi(shard_ranges[4].name)), + (wsgi_quote(str_to_wsgi(namespaces[4].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='', end_marker='\xf0\x9f\x8c\xb4', states='listing', reverse='true', limit=str(limit))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='\xf0\x9f\x8c\xb5', end_marker='\xe2\x98\x83', states='listing', reverse='true', limit=str(limit - len(sr_objs[4])))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[2].name)), + (wsgi_quote(str_to_wsgi(namespaces[2].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='\xe2\x98\x84', end_marker='pie', states='listing', reverse='true', limit=str(limit - len(sr_objs[4] + sr_objs[3])))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), + (wsgi_quote(str_to_wsgi(namespaces[1].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='q', end_marker='ham', states='listing', reverse='true', limit=str(limit - len(sr_objs[4] + sr_objs[3] + sr_objs[2])))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[0].name)), + (wsgi_quote(str_to_wsgi(namespaces[0].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='i', end_marker='', states='listing', reverse='true', @@ -765,15 +833,15 @@ class TestContainerController(TestRingBase): expected_requests, query_string='?reverse=true&limit=', reverse=True) # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) # GET with limit param limit = len(sr_objs[0]) + len(sr_objs[1]) + 1 expected_objects = all_objects[:limit] mock_responses = [ (404, '', {}), - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (200, sr_objs[1], shard_resp_hdrs[1]), (200, sr_objs[2][:1], shard_resp_hdrs[2]) @@ -783,17 +851,17 @@ class TestContainerController(TestRingBase): dict(limit=str(limit), states='listing')), # 404 ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(limit=str(limit), states='listing')), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[0].name)), + (wsgi_quote(str_to_wsgi(namespaces[0].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), - (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), + (wsgi_quote(str_to_wsgi(namespaces[1].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), - (wsgi_quote(str_to_wsgi(shard_ranges[2].name)), + (wsgi_quote(str_to_wsgi(namespaces[2].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing', @@ -802,7 +870,7 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, query_string='?limit=%s' % limit) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # GET with marker marker = bytes_to_wsgi(sr_objs[3][2]['name'].encode('utf8')) @@ -812,7 +880,7 @@ class TestContainerController(TestRingBase): expected_objects = all_objects[first_included:] mock_responses = [ (404, '', {}), - (200, sr_dicts[3:], root_shard_resp_hdrs), + (200, ns_dicts[3:], root_shard_resp_hdrs), (404, '', {}), (200, sr_objs[3][2:], shard_resp_hdrs[3]), (200, sr_objs[4], shard_resp_hdrs[4]), @@ -822,17 +890,17 @@ class TestContainerController(TestRingBase): dict(marker=marker, states='listing')), # 404 ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(marker=marker, states='listing')), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker=marker, end_marker='\xf0\x9f\x8c\xb4\x00', states='listing', limit=str(limit))), - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker=marker, end_marker='\xf0\x9f\x8c\xb4\x00', states='listing', limit=str(limit))), - (wsgi_quote(str_to_wsgi(shard_ranges[4].name)), + (wsgi_quote(str_to_wsgi(namespaces[4].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='\xe2\xa8\x83', end_marker='', states='listing', @@ -841,7 +909,7 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, query_string='?marker=%s' % marker) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # GET with end marker end_marker = bytes_to_wsgi(sr_objs[3][6]['name'].encode('utf8')) @@ -850,7 +918,7 @@ class TestContainerController(TestRingBase): expected_objects = all_objects[:first_excluded] mock_responses = [ (404, '', {}), - (200, sr_dicts[:4], root_shard_resp_hdrs), + (200, ns_dicts[:4], root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (404, '', {}), (200, sr_objs[1], shard_resp_hdrs[1]), @@ -863,33 +931,33 @@ class TestContainerController(TestRingBase): dict(end_marker=end_marker, states='listing')), # 404 ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(end_marker=end_marker, states='listing')), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[0].name)), + (wsgi_quote(str_to_wsgi(namespaces[0].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), - (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), + (wsgi_quote(str_to_wsgi(namespaces[1].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 404 dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), - (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), + (wsgi_quote(str_to_wsgi(namespaces[1].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), - (wsgi_quote(str_to_wsgi(shard_ranges[2].name)), + (wsgi_quote(str_to_wsgi(namespaces[2].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1])))), - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 404 dict(marker='\xd1\xb0', end_marker=end_marker, states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1] + sr_objs[2])))), - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='\xd1\xb0', end_marker=end_marker, states='listing', @@ -899,7 +967,7 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, query_string='?end_marker=%s' % end_marker) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # GET with prefix prefix = 'hat' @@ -908,7 +976,7 @@ class TestContainerController(TestRingBase): expected_objects = [] mock_responses = [ (404, '', {}), - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, [], shard_resp_hdrs[1]), ] expected_requests = [ @@ -916,7 +984,7 @@ class TestContainerController(TestRingBase): dict(prefix=prefix, states='listing')), # 404 ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(prefix=prefix, states='listing')), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), + (wsgi_quote(str_to_wsgi(namespaces[1].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 404 dict(prefix=prefix, marker='', end_marker='pie\x00', @@ -925,20 +993,20 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, query_string='?prefix=%s' % prefix) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # marker and end_marker and limit limit = 2 expected_objects = all_objects[first_included:first_excluded] mock_responses = [ - (200, sr_dicts[3:4], root_shard_resp_hdrs), + (200, ns_dicts[3:4], root_shard_resp_hdrs), (200, sr_objs[3][2:6], shard_resp_hdrs[1]) ] expected_requests = [ ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing', limit=str(limit), marker=marker, end_marker=end_marker)), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker=marker, end_marker=end_marker, states='listing', @@ -948,19 +1016,19 @@ class TestContainerController(TestRingBase): mock_responses, expected_objects, expected_requests, query_string='?marker=%s&end_marker=%s&limit=%s' % (marker, end_marker, limit)) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # reverse with marker, end_marker, and limit expected_objects.reverse() mock_responses = [ - (200, sr_dicts[3:4], root_shard_resp_hdrs), + (200, ns_dicts[3:4], root_shard_resp_hdrs), (200, list(reversed(sr_objs[3][2:6])), shard_resp_hdrs[1]) ] expected_requests = [ ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(marker=end_marker, reverse='true', end_marker=marker, limit=str(limit), states='listing',)), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker=end_marker, end_marker=marker, states='listing', @@ -970,26 +1038,15 @@ class TestContainerController(TestRingBase): mock_responses, expected_objects, expected_requests, query_string='?marker=%s&end_marker=%s&limit=%s&reverse=true' % (end_marker, marker, limit), reverse=True) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) def test_GET_sharded_container_with_memcache(self): # verify alternative code path in ContainerController when memcache is # available... shard_bounds = ('', 'ham', 'pie', u'\N{SNOWMAN}', u'\U0001F334', '') - shard_ranges = [ - ShardRange('.shards_a/c_%s' % upper, Timestamp.now(), lower, upper) - for lower, upper in zip(shard_bounds[:-1], shard_bounds[1:])] - sr_dicts = [dict(sr, last_modified=sr.timestamp.isoformat) - for sr in shard_ranges] - sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] - shard_resp_hdrs = [ - {'X-Backend-Sharding-State': 'unsharded', - 'X-Container-Object-Count': len(sr_objs[i]), - 'X-Container-Bytes-Used': - sum([obj['bytes'] for obj in sr_objs[i]]), - 'X-Container-Meta-Flavour': 'flavour%d' % i, - 'X-Backend-Storage-Policy-Index': 0} - for i, _ in enumerate(shard_ranges)] + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + shard_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) all_objects = [] for objects in sr_objs: @@ -998,23 +1055,17 @@ class TestContainerController(TestRingBase): num_all_objects = len(all_objects) limit = CONTAINER_LISTING_LIMIT expected_objects = all_objects - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Timestamp': '99', - # pretend root object stats are not yet updated - 'X-Container-Object-Count': num_all_objects - 1, - 'X-Container-Bytes-Used': size_all_objects - 1, - 'X-Container-Meta-Flavour': 'peach', - 'X-Backend-Storage-Policy-Index': 0, - 'X-Backend-Override-Shard-Name-Filter': 'true'} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + # pretend root object stats are not yet updated + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs( + num_all_objects - 1, size_all_objects - 1, + extra_shard_hdrs={'x-backend-override-shard-name-filter': 'true'}) # GET all objects # include some failed responses mock_responses = [ # status, body, headers (404, '', {}), - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (200, sr_objs[1], shard_resp_hdrs[1]), (200, sr_objs[2], shard_resp_hdrs[2]), @@ -1026,32 +1077,38 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto', 'X-Backend-Override-Shard-Name-Filter': 'sharded'}, dict(states='listing')), # 404 - ('a/c', {'X-Backend-Record-Type': 'auto'}, + ('a/c', {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded'}, dict(states='listing')), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[0].name)), + (wsgi_quote(str_to_wsgi(namespaces[0].name)), {'X-Backend-Record-Type': 'auto', - 'X-Backend-Storage-Policy-Index': '0'}, + 'X-Backend-Override-Shard-Name-Filter': 'sharded', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='', end_marker='ham\x00', limit=str(limit), states='listing')), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), + (wsgi_quote(str_to_wsgi(namespaces[1].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[2].name)), + (wsgi_quote(str_to_wsgi(namespaces[2].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1])))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='\xd1\xb0', end_marker='\xf0\x9f\x8c\xb4\x00', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1] + sr_objs[2])))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[4].name)), + (wsgi_quote(str_to_wsgi(namespaces[4].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='\xe2\xa8\x83', end_marker='', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1] + sr_objs[2] @@ -1061,26 +1118,26 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, memcache=True) # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, memcache=True, req_headers={'X-Backend-Record-Type': 'auto'}) - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, memcache=True, req_headers={'X-Backend-Record-Type': 'banana'}) - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) # GET all objects - sharding, final shard range points back to root root_range = ShardRange('a/c', Timestamp.now(), 'pie', '') mock_responses = [ # status, body, headers - (200, sr_dicts[:2] + [dict(root_range)], root_shard_resp_hdrs), + (200, ns_dicts[:2] + [dict(root_range)], root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (200, sr_objs[1], shard_resp_hdrs[1]), (200, sr_objs[2] + sr_objs[3] + sr_objs[4], root_resp_hdrs) @@ -1090,18 +1147,21 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto', 'X-Backend-Override-Shard-Name-Filter': 'sharded'}, dict(states='listing')), # 200 - (shard_ranges[0].name, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='', end_marker='ham\x00', limit=str(limit), states='listing')), # 200 - (shard_ranges[1].name, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # 200 (root_range.name, {'X-Backend-Record-Type': 'object', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='p', end_marker='', limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200 @@ -1110,13 +1170,13 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, memcache=True) # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) # GET all objects in reverse and *blank* limit mock_responses = [ # status, body, headers - (200, list(sr_dicts), root_shard_resp_hdrs), + (200, list(ns_dicts), root_shard_resp_hdrs), (200, list(reversed(sr_objs[4])), shard_resp_hdrs[4]), (200, list(reversed(sr_objs[3])), shard_resp_hdrs[3]), (200, list(reversed(sr_objs[2])), shard_resp_hdrs[2]), @@ -1128,32 +1188,37 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto', 'X-Backend-Override-Shard-Name-Filter': 'sharded'}, dict(states='listing', reverse='true', limit='')), - (wsgi_quote(str_to_wsgi(shard_ranges[4].name)), + (wsgi_quote(str_to_wsgi(namespaces[4].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='', end_marker='\xf0\x9f\x8c\xb4', states='listing', reverse='true', limit=str(limit))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='\xf0\x9f\x8c\xb5', end_marker='\xe2\x98\x83', states='listing', reverse='true', limit=str(limit - len(sr_objs[4])))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[2].name)), + (wsgi_quote(str_to_wsgi(namespaces[2].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='\xe2\x98\x84', end_marker='pie', states='listing', reverse='true', limit=str(limit - len(sr_objs[4] + sr_objs[3])))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), + (wsgi_quote(str_to_wsgi(namespaces[1].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='q', end_marker='ham', states='listing', reverse='true', limit=str(limit - len(sr_objs[4] + sr_objs[3] + sr_objs[2])))), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[0].name)), + (wsgi_quote(str_to_wsgi(namespaces[0].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='i', end_marker='', states='listing', reverse='true', limit=str(limit - len(sr_objs[4] + sr_objs[3] + sr_objs[2] @@ -1165,15 +1230,15 @@ class TestContainerController(TestRingBase): expected_requests, query_string='?reverse=true&limit=', reverse=True, memcache=True) # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) # GET with limit param limit = len(sr_objs[0]) + len(sr_objs[1]) + 1 expected_objects = all_objects[:limit] mock_responses = [ (404, '', {}), - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (200, sr_objs[1], shard_resp_hdrs[1]), (200, sr_objs[2][:1], shard_resp_hdrs[2]) @@ -1185,18 +1250,21 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto', 'X-Backend-Override-Shard-Name-Filter': 'sharded'}, dict(limit=str(limit), states='listing')), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[0].name)), + (wsgi_quote(str_to_wsgi(namespaces[0].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), - (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), + (wsgi_quote(str_to_wsgi(namespaces[1].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), - (wsgi_quote(str_to_wsgi(shard_ranges[2].name)), + (wsgi_quote(str_to_wsgi(namespaces[2].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1])))), @@ -1204,7 +1272,7 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, query_string='?limit=%s' % limit, memcache=True) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # GET with marker marker = bytes_to_wsgi(sr_objs[3][2]['name'].encode('utf8')) @@ -1214,7 +1282,9 @@ class TestContainerController(TestRingBase): expected_objects = all_objects[first_included:] mock_responses = [ (404, '', {}), - (200, sr_dicts[3:], root_shard_resp_hdrs), + # NB: proxy sent X-Backend-Override-Shard-Name-Filter so root + # returns complete shard listing despite marker + (200, ns_dicts, root_shard_resp_hdrs), (404, '', {}), (200, sr_objs[3][2:], shard_resp_hdrs[3]), (200, sr_objs[4], shard_resp_hdrs[4]), @@ -1226,18 +1296,21 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto', 'X-Backend-Override-Shard-Name-Filter': 'sharded'}, dict(marker=marker, states='listing')), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker=marker, end_marker='\xf0\x9f\x8c\xb4\x00', states='listing', limit=str(limit))), - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker=marker, end_marker='\xf0\x9f\x8c\xb4\x00', states='listing', limit=str(limit))), - (wsgi_quote(str_to_wsgi(shard_ranges[4].name)), + (wsgi_quote(str_to_wsgi(namespaces[4].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='\xe2\xa8\x83', end_marker='', states='listing', limit=str(limit - len(sr_objs[3][2:])))), @@ -1245,7 +1318,7 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, query_string='?marker=%s' % marker, memcache=True) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # GET with end marker end_marker = bytes_to_wsgi(sr_objs[3][6]['name'].encode('utf8')) @@ -1254,7 +1327,7 @@ class TestContainerController(TestRingBase): expected_objects = all_objects[:first_excluded] mock_responses = [ (404, '', {}), - (200, sr_dicts[:4], root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (404, '', {}), (200, sr_objs[1], shard_resp_hdrs[1]), @@ -1269,34 +1342,40 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto', 'X-Backend-Override-Shard-Name-Filter': 'sharded'}, dict(end_marker=end_marker, states='listing')), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[0].name)), + (wsgi_quote(str_to_wsgi(namespaces[0].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), - (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), + (wsgi_quote(str_to_wsgi(namespaces[1].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 404 dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), - (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), + (wsgi_quote(str_to_wsgi(namespaces[1].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), - (wsgi_quote(str_to_wsgi(shard_ranges[2].name)), + (wsgi_quote(str_to_wsgi(namespaces[2].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1])))), - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 404 dict(marker='\xd1\xb0', end_marker=end_marker, states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1] + sr_objs[2])))), - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='\xd1\xb0', end_marker=end_marker, states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1] @@ -1305,7 +1384,7 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, query_string='?end_marker=%s' % end_marker, memcache=True) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # GET with prefix prefix = 'hat' @@ -1314,7 +1393,7 @@ class TestContainerController(TestRingBase): expected_objects = [] mock_responses = [ (404, '', {}), - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, [], shard_resp_hdrs[1]), ] expected_requests = [ @@ -1324,8 +1403,9 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto', 'X-Backend-Override-Shard-Name-Filter': 'sharded'}, dict(prefix=prefix, states='listing')), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), + (wsgi_quote(str_to_wsgi(namespaces[1].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 404 dict(prefix=prefix, marker='', end_marker='pie\x00', states='listing', limit=str(limit))), @@ -1333,13 +1413,13 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, query_string='?prefix=%s' % prefix, memcache=True) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # marker and end_marker and limit limit = 2 expected_objects = all_objects[first_included:first_excluded] mock_responses = [ - (200, sr_dicts[3:4], root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[3][2:6], shard_resp_hdrs[1]) ] expected_requests = [ @@ -1347,8 +1427,9 @@ class TestContainerController(TestRingBase): 'X-Backend-Override-Shard-Name-Filter': 'sharded'}, dict(states='listing', limit=str(limit), marker=marker, end_marker=end_marker)), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker=marker, end_marker=end_marker, states='listing', limit=str(limit))), @@ -1357,12 +1438,12 @@ class TestContainerController(TestRingBase): mock_responses, expected_objects, expected_requests, query_string='?marker=%s&end_marker=%s&limit=%s' % (marker, end_marker, limit), memcache=True) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # reverse with marker, end_marker, and limit expected_objects.reverse() mock_responses = [ - (200, sr_dicts[3:4], root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, list(reversed(sr_objs[3][2:6])), shard_resp_hdrs[1]) ] expected_requests = [ @@ -1370,8 +1451,9 @@ class TestContainerController(TestRingBase): 'X-Backend-Override-Shard-Name-Filter': 'sharded'}, dict(marker=end_marker, reverse='true', end_marker=marker, limit=str(limit), states='listing',)), # 200 - (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), + (wsgi_quote(str_to_wsgi(namespaces[3].name)), {'X-Backend-Record-Type': 'auto', + 'X-Backend-Override-Shard-Name-Filter': 'sharded', 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker=end_marker, end_marker=marker, states='listing', limit=str(limit), reverse='true')), @@ -1380,42 +1462,25 @@ class TestContainerController(TestRingBase): mock_responses, expected_objects, expected_requests, query_string='?marker=%s&end_marker=%s&limit=%s&reverse=true' % (end_marker, marker, limit), reverse=True, memcache=True) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) def _do_test_GET_sharded_container_with_deleted_shards(self, shard_specs): # verify that if a shard fails to return its listing component then the # client response is 503 shard_bounds = (('a', 'b'), ('b', 'c'), ('c', '')) - shard_ranges = [ - ShardRange('.shards_a/c_%s' % upper, Timestamp.now(), lower, upper) - for lower, upper in shard_bounds] - sr_dicts = [dict(sr) for sr in shard_ranges] - sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] - shard_resp_hdrs = [ - {'X-Backend-Sharding-State': 'unsharded', - 'X-Container-Object-Count': len(sr_objs[i]), - 'X-Container-Bytes-Used': - sum([obj['bytes'] for obj in sr_objs[i]]), - 'X-Container-Meta-Flavour': 'flavour%d' % i, - 'X-Backend-Storage-Policy-Index': 0} - for i, _ in enumerate(shard_ranges)] + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + shard_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) all_objects = [] for objects in sr_objs: all_objects.extend(objects) - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Timestamp': '99', - # pretend root object stats are not yet updated - 'X-Container-Object-Count': 6, - 'X-Container-Bytes-Used': 12, - 'X-Backend-Storage-Policy-Index': 0} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' - + # pretend root object stats are not yet updated + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs(6, 12) mock_responses = [ # status, body, headers - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), ] for i, spec in enumerate(shard_specs): if spec == 200: @@ -1472,28 +1537,44 @@ class TestContainerController(TestRingBase): ['Aborting listing from shards due to bad response: %s' % ([200, 200, 503],)], errors[-1:]) + def test_GET_sharded_container_with_marker_beyond_end_marker(self): + # verify that if request params result in the filtered namespaces list + # being empty the response body still has an empty object list + shard_bounds = (('a', 'b'), ('b', 'c'), ('c', '')) + namespaces, ns_dicts, _ = self.create_server_response_data( + shard_bounds) + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs( + 6, 12, extra_shard_hdrs={ + 'x-backend-override-shard-name-filter': 'true'}) + + # NB: root returns complete shard listing + mock_responses = [ + # status, body, headers + (200, ns_dicts, root_shard_resp_hdrs), + ] + expected_requests = [ + ('a/c', {'X-Backend-Record-Type': 'auto'}, + dict(states='listing', marker='bb', end_marker='aa')), # 200 + ] + expected_objects = [] + resp = self._check_GET_shard_listing( + mock_responses, expected_objects, expected_requests, + query_string='?marker=bb&end_marker=aa', memcache=True) + self.check_listing_response(resp, root_resp_hdrs) + def test_GET_sharded_container_with_delimiter(self): shard_bounds = (('', 'ha/ppy'), ('ha/ppy', 'ha/ptic'), ('ha/ptic', 'ham'), ('ham', 'pie'), ('pie', '')) - shard_ranges = [ - ShardRange('.shards_a/c_%s' % upper.replace('/', ''), - Timestamp.now(), lower, upper) - for lower, upper in shard_bounds] - sr_dicts = [dict(sr) for sr in shard_ranges] + namespaces, ns_dicts, _ = self.create_server_response_data( + shard_bounds) shard_resp_hdrs = {'X-Backend-Sharding-State': 'unsharded', 'X-Container-Object-Count': 2, 'X-Container-Bytes-Used': 4, 'X-Backend-Storage-Policy-Index': 0} limit = CONTAINER_LISTING_LIMIT - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Timestamp': '99', - # pretend root object stats are not yet updated - 'X-Container-Object-Count': 6, - 'X-Container-Bytes-Used': 12, - 'X-Backend-Storage-Policy-Index': 0} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + # pretend root object stats are not yet updated + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs(6, 12) sr_0_obj = {'name': 'apple', 'bytes': 1, @@ -1510,7 +1591,7 @@ class TestContainerController(TestRingBase): subdir = {'subdir': 'ha/'} mock_responses = [ # status, body, headers - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, [sr_0_obj, subdir], shard_resp_hdrs), (200, [], shard_resp_hdrs), (200, [], shard_resp_hdrs), @@ -1519,16 +1600,16 @@ class TestContainerController(TestRingBase): expected_requests = [ ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing', delimiter='/')), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='ha/ppy\x00', limit=str(limit), states='listing', delimiter='/')), # 200 - (shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[2].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='ha/', end_marker='ham\x00', states='listing', limit=str(limit - 2), delimiter='/')), # 200 - (shard_ranges[3].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[3].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='ha/', end_marker='pie\x00', states='listing', limit=str(limit - 2), delimiter='/')), # 200 - (shard_ranges[4].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[4].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='ha/', end_marker='', states='listing', limit=str(limit - 2), delimiter='/')), # 200 ] @@ -1537,30 +1618,20 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, query_string='?delimiter=/') - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) def test_GET_sharded_container_with_delimiter_and_reverse(self): - shard_points = ('', 'ha.d', 'ha/ppy', 'ha/ptic', 'ham', 'pie', '') - shard_bounds = tuple(zip(shard_points, shard_points[1:])) - shard_ranges = [ - ShardRange('.shards_a/c_%s' % upper.replace('/', ''), - Timestamp.now(), lower, upper) - for lower, upper in shard_bounds] - sr_dicts = [dict(sr) for sr in shard_ranges] + shard_bounds = ('', 'ha.d', 'ha/ppy', 'ha/ptic', 'ham', 'pie', '') + namespaces, ns_dicts, _ = self.create_server_response_data( + shard_bounds) shard_resp_hdrs = {'X-Backend-Sharding-State': 'unsharded', 'X-Container-Object-Count': 2, 'X-Container-Bytes-Used': 4, 'X-Backend-Storage-Policy-Index': 0} limit = CONTAINER_LISTING_LIMIT - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Timestamp': '99', - # pretend root object stats are not yet updated - 'X-Container-Object-Count': 6, - 'X-Container-Bytes-Used': 12, - 'X-Backend-Storage-Policy-Index': 0} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + # pretend root object stats are not yet updated + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs(6, 12) sr_0_obj = {'name': 'apple', 'bytes': 1, @@ -1583,7 +1654,7 @@ class TestContainerController(TestRingBase): subdir = {'subdir': 'ha/'} mock_responses = [ # status, body, headers - (200, list(reversed(sr_dicts)), root_shard_resp_hdrs), + (200, list(reversed(ns_dicts)), root_shard_resp_hdrs), (200, [sr_5_obj], shard_resp_hdrs), (200, [], shard_resp_hdrs), (200, [subdir], shard_resp_hdrs), @@ -1593,19 +1664,19 @@ class TestContainerController(TestRingBase): expected_requests = [ ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing', delimiter='/', reverse='on')), # 200 - (shard_ranges[5].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[5].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='pie', states='listing', limit=str(limit), delimiter='/', reverse='on')), # 200 - (shard_ranges[4].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[4].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='pumpkin', end_marker='ham', states='listing', limit=str(limit - 1), delimiter='/', reverse='on')), # 200 - (shard_ranges[3].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[3].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='pumpkin', end_marker='ha/ptic', states='listing', limit=str(limit - 1), delimiter='/', reverse='on')), # 200 - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='ha/', end_marker='ha.d', limit=str(limit - 2), states='listing', delimiter='/', reverse='on')), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='ha.ggle', end_marker='', limit=str(limit - 3), states='listing', delimiter='/', reverse='on')), # 200 ] @@ -1614,7 +1685,7 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests, query_string='?delimiter=/&reverse=on', reverse=True) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) def test_GET_sharded_container_shard_redirects_to_root(self): # check that if the root redirects listing to a shard, but the shard @@ -1622,8 +1693,10 @@ class TestContainerController(TestRingBase): # the root) objects are requested from the root, rather than a loop. # single shard spanning entire namespace - shard_sr = ShardRange('.shards_a/c_xyz', Timestamp.now(), '', '') - all_objects = self._make_shard_objects(shard_sr) + shard_bounds = ('', '') + namespaces, _, sr_objs = self.create_server_response_data( + shard_bounds) + all_objects = sr_objs[0] size_all_objects = sum([obj['bytes'] for obj in all_objects]) num_all_objects = len(all_objects) limit = CONTAINER_LISTING_LIMIT @@ -1639,18 +1712,13 @@ class TestContainerController(TestRingBase): } # root still thinks it has a shard - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Timestamp': '99', - 'X-Container-Object-Count': num_all_objects, - 'X-Container-Bytes-Used': size_all_objects, - 'X-Backend-Storage-Policy-Index': 0} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs( + num_all_objects, size_all_objects) root_sr = ShardRange('a/c', Timestamp.now(), '', '') mock_responses = [ # status, body, headers - (200, [dict(shard_sr)], root_shard_resp_hdrs), # from root + (200, [dict(namespaces[0])], root_shard_resp_hdrs), # from root (200, [dict(root_sr)], shard_resp_hdrs), # from shard (200, all_objects, root_resp_hdrs), # from root ] @@ -1660,7 +1728,7 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # request to shard should specify auto record type - (wsgi_quote(str_to_wsgi(shard_sr.name)), + (wsgi_quote(str_to_wsgi(namespaces[0].name)), {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='', limit=str(limit), states='listing')), # 200 @@ -1672,10 +1740,10 @@ class TestContainerController(TestRingBase): expected_objects = all_objects resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests) - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) self.assertEqual( - [('a', 'c'), ('.shards_a', 'c_xyz')], + [('a', 'c'), ('.shards_a', 'c_')], resp.request.environ.get('swift.shard_listing_history')) lines = [line for line in self.app.logger.get_lines_for_level('debug') if line.startswith('Found 1024 objects in shard')] @@ -1689,30 +1757,23 @@ class TestContainerController(TestRingBase): # out of the loop (this isn't an expected scenario, but could perhaps # happen if multiple conflicting shard-shrinking decisions are made) shard_bounds = ('', 'a', 'b', '') - shard_ranges = [ - ShardRange('.shards_a/c_%s' % upper, Timestamp.now(), lower, upper) - for lower, upper in zip(shard_bounds[:-1], shard_bounds[1:])] + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) self.assertEqual([ '.shards_a/c_a', '.shards_a/c_b', '.shards_a/c_', - ], [sr.name for sr in shard_ranges]) - sr_dicts = [dict(sr) for sr in shard_ranges] - sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] + ], [sr.name for sr in namespaces]) all_objects = [] for objects in sr_objs: all_objects.extend(objects) size_all_objects = sum([obj['bytes'] for obj in all_objects]) num_all_objects = len(all_objects) - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Timestamp': '99', - 'X-Container-Object-Count': num_all_objects, - 'X-Container-Bytes-Used': size_all_objects, - 'X-Backend-Storage-Policy-Index': 0, - 'X-Backend-Record-Storage-Policy-Index': 0, - 'X-Backend-Record-Type': 'shard', - } + # pretend root object stats are not yet updated + _, root_shard_resp_hdrs = self._make_root_resp_hdrs( + num_all_objects, size_all_objects) + shard_resp_hdrs = {'X-Backend-Sharding-State': 'unsharded', 'X-Container-Object-Count': 2, 'X-Container-Bytes-Used': 4, @@ -1724,14 +1785,15 @@ class TestContainerController(TestRingBase): 'X-Backend-Record-Type': 'shard', 'X-Backend-Storage-Policy-Index': 0 } + shrinking_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) limit = CONTAINER_LISTING_LIMIT mock_responses = [ # status, body, headers - (200, sr_dicts, root_resp_hdrs), # from root + (200, ns_dicts, root_shard_resp_hdrs), # from root (200, sr_objs[0], shard_resp_hdrs), # objects from 1st shard - (200, [sr_dicts[2]], shrinking_resp_hdrs), # 2nd points to 3rd - (200, [sr_dicts[1]], shrinking_resp_hdrs), # 3rd points to 2nd + (200, [ns_dicts[2]], shrinking_resp_hdrs), # 2nd points to 3rd + (200, [ns_dicts[1]], shrinking_resp_hdrs), # 3rd points to 2nd (200, sr_objs[1], shard_resp_hdrs), # objects from 2nd (200, sr_objs[2], shard_resp_hdrs), # objects from 3rd ] @@ -1779,31 +1841,20 @@ class TestContainerController(TestRingBase): ] resp = self._check_GET_shard_listing( mock_responses, all_objects, expected_requests) - self.check_response(resp, root_resp_hdrs, - expected_objects=all_objects) + self.check_listing_response(resp, root_shard_resp_hdrs, + expected_objects=all_objects) self.assertEqual( [('a', 'c'), ('.shards_a', 'c_b'), ('.shards_a', 'c_')], resp.request.environ.get('swift.shard_listing_history')) def test_GET_sharded_container_overlapping_shards(self): # verify ordered listing even if unexpected overlapping shard ranges - shard_bounds = (('', 'ham', ShardRange.CLEAVED), - ('', 'pie', ShardRange.ACTIVE), - ('lemon', '', ShardRange.ACTIVE)) - shard_ranges = [ - ShardRange('.shards_a/c_' + upper, Timestamp.now(), lower, upper, - state=state) - for lower, upper, state in shard_bounds] - sr_dicts = [dict(sr) for sr in shard_ranges] - sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] - shard_resp_hdrs = [ - {'X-Backend-Sharding-State': 'unsharded', - 'X-Container-Object-Count': len(sr_objs[i]), - 'X-Container-Bytes-Used': - sum([obj['bytes'] for obj in sr_objs[i]]), - 'X-Container-Meta-Flavour': 'flavour%d' % i, - 'X-Backend-Storage-Policy-Index': 0} - for i in range(3)] + shard_bounds = (('', 'ham'), ('', 'pie'), ('lemon', '')) + shard_states = (ShardRange.CLEAVED, ShardRange.ACTIVE, + ShardRange.ACTIVE) + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds, states=shard_states) + shard_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) all_objects = [] for objects in sr_objs: @@ -1811,15 +1862,9 @@ class TestContainerController(TestRingBase): size_all_objects = sum([obj['bytes'] for obj in all_objects]) num_all_objects = len(all_objects) limit = CONTAINER_LISTING_LIMIT - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Timestamp': '99', - # pretend root object stats are not yet updated - 'X-Container-Object-Count': num_all_objects - 1, - 'X-Container-Bytes-Used': size_all_objects - 1, - 'X-Container-Meta-Flavour': 'peach', - 'X-Backend-Storage-Policy-Index': 0} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + # pretend root object stats are not yet updated + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs( + num_all_objects - 1, size_all_objects - 1) # forwards listing @@ -1829,7 +1874,7 @@ class TestContainerController(TestRingBase): objs_2 = [o for o in sr_objs[2] if o['name'] > sr_objs[1][-1]['name']] mock_responses = [ # status, body, headers - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (200, objs_1, shard_resp_hdrs[1]), (200, objs_2, shard_resp_hdrs[2]) @@ -1839,13 +1884,13 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), # 200 - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # 200 - (shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[2].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='p', end_marker='', states='listing', limit=str(limit - len(sr_objs[0] + objs_1)))) # 200 ] @@ -1854,8 +1899,8 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests) # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) # reverse listing @@ -1865,7 +1910,7 @@ class TestContainerController(TestRingBase): objs_1 = [o for o in sr_objs[1] if o['name'] < sr_objs[2][0]['name']] mock_responses = [ # status, body, headers - (200, list(reversed(sr_dicts)), root_shard_resp_hdrs), + (200, list(reversed(ns_dicts)), root_shard_resp_hdrs), (200, list(reversed(sr_objs[2])), shard_resp_hdrs[2]), (200, list(reversed(objs_1)), shard_resp_hdrs[1]), (200, list(reversed(objs_0)), shard_resp_hdrs[0]), @@ -1875,14 +1920,14 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing', reverse='true')), # 200 - (shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[2].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='lemon', states='listing', limit=str(limit), reverse='true')), # 200 - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='m', end_marker='', reverse='true', states='listing', limit=str(limit - len(sr_objs[2])))), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='A', end_marker='', reverse='true', states='listing', limit=str(limit - len(sr_objs[2] + objs_1)))) # 200 ] @@ -1892,25 +1937,15 @@ class TestContainerController(TestRingBase): mock_responses, expected_objects, expected_requests, query_string='?reverse=true', reverse=True) # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs, - expected_objects=expected_objects) + self.check_listing_response(resp, root_resp_hdrs, + expected_objects=expected_objects) def test_GET_sharded_container_gap_in_shards_no_memcache(self): # verify ordered listing even if unexpected gap between shard ranges shard_bounds = (('', 'ham'), ('onion', 'pie'), ('rhubarb', '')) - shard_ranges = [ - ShardRange('.shards_a/c_' + upper, Timestamp.now(), lower, upper) - for lower, upper in shard_bounds] - sr_dicts = [dict(sr) for sr in shard_ranges] - sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] - shard_resp_hdrs = [ - {'X-Backend-Sharding-State': 'unsharded', - 'X-Container-Object-Count': len(sr_objs[i]), - 'X-Container-Bytes-Used': - sum([obj['bytes'] for obj in sr_objs[i]]), - 'X-Container-Meta-Flavour': 'flavour%d' % i, - 'X-Backend-Storage-Policy-Index': 0} - for i in range(3)] + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + shard_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) all_objects = [] for objects in sr_objs: @@ -1918,19 +1953,13 @@ class TestContainerController(TestRingBase): size_all_objects = sum([obj['bytes'] for obj in all_objects]) num_all_objects = len(all_objects) limit = CONTAINER_LISTING_LIMIT - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Override-Shard-Name-Filter': 'true', - 'X-Backend-Timestamp': '99', - 'X-Container-Object-Count': num_all_objects, - 'X-Container-Bytes-Used': size_all_objects, - 'X-Container-Meta-Flavour': 'peach', - 'X-Backend-Storage-Policy-Index': 0} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + # pretend root object stats are not yet updated + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs( + num_all_objects, size_all_objects) mock_responses = [ # status, body, headers - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (200, sr_objs[1], shard_resp_hdrs[1]), (200, sr_objs[2], shard_resp_hdrs[2]) @@ -1940,13 +1969,13 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), # 200 - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # 200 - (shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[2].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='p', end_marker='', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200 ] @@ -1954,26 +1983,16 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, all_objects, expected_requests, memcache=False) # root object count will be overridden by actual length of listing - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) self.assertNotIn('swift.cache', resp.request.environ) def test_GET_sharding_container_gap_in_shards_memcache(self): # verify ordered listing even if unexpected gap between shard ranges; # root is sharding so shard ranges are not cached shard_bounds = (('', 'ham'), ('onion', 'pie'), ('rhubarb', '')) - shard_ranges = [ - ShardRange('.shards_a/c_' + upper, Timestamp.now(), lower, upper) - for lower, upper in shard_bounds] - sr_dicts = [dict(sr) for sr in shard_ranges] - sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] - shard_resp_hdrs = [ - {'X-Backend-Sharding-State': 'unsharded', - 'X-Container-Object-Count': len(sr_objs[i]), - 'X-Container-Bytes-Used': - sum([obj['bytes'] for obj in sr_objs[i]]), - 'X-Container-Meta-Flavour': 'flavour%d' % i, - 'X-Backend-Storage-Policy-Index': 0} - for i in range(3)] + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + shard_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) all_objects = [] for objects in sr_objs: @@ -1981,19 +2000,13 @@ class TestContainerController(TestRingBase): size_all_objects = sum([obj['bytes'] for obj in all_objects]) num_all_objects = len(all_objects) limit = CONTAINER_LISTING_LIMIT - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharding', - 'X-Backend-Override-Shard-Name-Filter': 'true', - 'X-Backend-Timestamp': '99', - 'X-Container-Object-Count': num_all_objects, - 'X-Container-Bytes-Used': size_all_objects, - 'X-Container-Meta-Flavour': 'peach', - 'X-Backend-Storage-Policy-Index': 0} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs( + num_all_objects, size_all_objects, + extra_hdrs={'X-Backend-Sharding-State': 'sharding'}) mock_responses = [ # status, body, headers - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (200, sr_objs[1], shard_resp_hdrs[1]), (200, sr_objs[2], shard_resp_hdrs[2]) @@ -2004,13 +2017,13 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), # 200 - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # 200 - (shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[2].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='p', end_marker='', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200 ] @@ -2018,8 +2031,8 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, all_objects, expected_requests, memcache=True) # root object count will be overridden by actual length of listing - self.check_response(resp, root_resp_hdrs, - exp_sharding_state='sharding') + self.check_listing_response(resp, root_resp_hdrs, + exp_sharding_state='sharding') self.assertIn('swift.cache', resp.request.environ) self.assertNotIn('shard-listing-v2/a/c', resp.request.environ['swift.cache'].store) @@ -2027,19 +2040,9 @@ class TestContainerController(TestRingBase): def test_GET_sharded_container_gap_in_shards_memcache(self): # verify ordered listing even if unexpected gap between shard ranges shard_bounds = (('', 'ham'), ('onion', 'pie'), ('rhubarb', '')) - shard_ranges = [ - ShardRange('.shards_a/c_' + upper, Timestamp.now(), lower, upper) - for lower, upper in shard_bounds] - sr_dicts = [dict(sr) for sr in shard_ranges] - sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] - shard_resp_hdrs = [ - {'X-Backend-Sharding-State': 'unsharded', - 'X-Container-Object-Count': len(sr_objs[i]), - 'X-Container-Bytes-Used': - sum([obj['bytes'] for obj in sr_objs[i]]), - 'X-Container-Meta-Flavour': 'flavour%d' % i, - 'X-Backend-Storage-Policy-Index': 0} - for i in range(3)] + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + shard_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) all_objects = [] for objects in sr_objs: @@ -2047,19 +2050,13 @@ class TestContainerController(TestRingBase): size_all_objects = sum([obj['bytes'] for obj in all_objects]) num_all_objects = len(all_objects) limit = CONTAINER_LISTING_LIMIT - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Override-Shard-Name-Filter': 'true', - 'X-Backend-Timestamp': '99', - 'X-Container-Object-Count': num_all_objects, - 'X-Container-Bytes-Used': size_all_objects, - 'X-Container-Meta-Flavour': 'peach', - 'X-Backend-Storage-Policy-Index': 0} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs( + num_all_objects, size_all_objects, + extra_shard_hdrs={'x-backend-override-shard-name-filter': 'true'}) mock_responses = [ # status, body, headers - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (200, sr_objs[1], shard_resp_hdrs[1]), (200, sr_objs[2], shard_resp_hdrs[2]) @@ -2071,13 +2068,13 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='onion\x00', states='listing', limit=str(limit))), # 200 - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='h', end_marker='rhubarb\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # 200 - (shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[2].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='p', end_marker='', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200 ] @@ -2085,7 +2082,7 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, all_objects, expected_requests, memcache=True) # root object count will be overridden by actual length of listing - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) self.assertIn('swift.cache', resp.request.environ) self.assertIn('shard-listing-v2/a/c', resp.request.environ['swift.cache'].store) @@ -2099,19 +2096,9 @@ class TestContainerController(TestRingBase): def test_GET_sharded_container_empty_shard(self): # verify ordered listing when a shard is empty shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', '')) - shard_ranges = [ - ShardRange('.shards_a/c_%s' % upper, Timestamp.now(), lower, upper) - for lower, upper in shard_bounds] - sr_dicts = [dict(sr) for sr in shard_ranges] - sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] - shard_resp_hdrs = [ - {'X-Backend-Sharding-State': 'unsharded', - 'X-Container-Object-Count': len(sr_objs[i]), - 'X-Container-Bytes-Used': - sum([obj['bytes'] for obj in sr_objs[i]]), - 'X-Container-Meta-Flavour': 'flavour%d' % i, - 'X-Backend-Storage-Policy-Index': 0} - for i in range(3)] + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + shard_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) empty_shard_resp_hdrs = { 'X-Backend-Sharding-State': 'unsharded', 'X-Container-Object-Count': 0, @@ -2122,18 +2109,13 @@ class TestContainerController(TestRingBase): # empty first shard range all_objects = sr_objs[1] + sr_objs[2] size_all_objects = sum([obj['bytes'] for obj in all_objects]) - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Timestamp': '99', - 'X-Container-Object-Count': len(all_objects), - 'X-Container-Bytes-Used': size_all_objects, - 'X-Container-Meta-Flavour': 'peach', - 'X-Backend-Storage-Policy-Index': 0} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + num_all_objects = len(all_objects) + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs( + num_all_objects, size_all_objects) mock_responses = [ # status, body, headers - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, [], empty_shard_resp_hdrs), (200, sr_objs[1], shard_resp_hdrs[1]), (200, sr_objs[2], shard_resp_hdrs[2]) @@ -2144,20 +2126,20 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), # 200 - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='pie\x00', states='listing', limit=str(limit))), # 200 - (shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[2].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='p', end_marker='', states='listing', limit=str(limit - len(sr_objs[1])))) # 200 ] resp = self._check_GET_shard_listing( mock_responses, sr_objs[1] + sr_objs[2], expected_requests) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # empty last shard range, reverse all_objects = sr_objs[0] + sr_objs[1] @@ -2170,10 +2152,11 @@ class TestContainerController(TestRingBase): 'X-Backend-Storage-Policy-Index': 0} root_shard_resp_hdrs = dict(root_resp_hdrs) root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + root_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) mock_responses = [ # status, body, headers - (200, list(reversed(sr_dicts)), root_shard_resp_hdrs), + (200, list(reversed(ns_dicts)), root_shard_resp_hdrs), (200, [], empty_shard_resp_hdrs), (200, list(reversed(sr_objs[1])), shard_resp_hdrs[1]), (200, list(reversed(sr_objs[0])), shard_resp_hdrs[0]), @@ -2183,13 +2166,13 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing', reverse='true')), # 200 - (shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[2].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='pie', states='listing', limit=str(limit), reverse='true')), # 200 - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='ham', states='listing', limit=str(limit), reverse='true')), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto'}, dict(marker=sr_objs[1][0]['name'], end_marker='', states='listing', reverse='true', limit=str(limit - len(sr_objs[1])))) # 200 @@ -2198,7 +2181,7 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, list(reversed(sr_objs[0] + sr_objs[1])), expected_requests, query_string='?reverse=true', reverse=True) - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # empty second shard range all_objects = sr_objs[0] + sr_objs[2] @@ -2211,10 +2194,11 @@ class TestContainerController(TestRingBase): 'X-Backend-Storage-Policy-Index': 0} root_shard_resp_hdrs = dict(root_resp_hdrs) root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + root_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) mock_responses = [ # status, body, headers - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), (200, [], empty_shard_resp_hdrs), (200, sr_objs[2], shard_resp_hdrs[2]) @@ -2225,13 +2209,13 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), # 200 - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # 200 - (shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[2].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='h', end_marker='', states='listing', limit=str(limit - len(sr_objs[0])))) # 200 ] @@ -2239,12 +2223,12 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, sr_objs[0] + sr_objs[2], expected_requests) # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # marker in empty second range mock_responses = [ # status, body, headers - (200, sr_dicts[1:], root_shard_resp_hdrs), + (200, ns_dicts[1:], root_shard_resp_hdrs), (200, [], empty_shard_resp_hdrs), (200, sr_objs[2], shard_resp_hdrs[2]) ] @@ -2253,10 +2237,10 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing', marker='koolaid')), # 200 - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='koolaid', end_marker='pie\x00', states='listing', limit=str(limit))), # 200 - (shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[2].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='koolaid', end_marker='', states='listing', limit=str(limit))) # 200 ] @@ -2265,12 +2249,12 @@ class TestContainerController(TestRingBase): mock_responses, sr_objs[2], expected_requests, query_string='?marker=koolaid') # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) # marker in empty second range, reverse mock_responses = [ # status, body, headers - (200, list(reversed(sr_dicts[:2])), root_shard_resp_hdrs), + (200, list(reversed(ns_dicts[:2])), root_shard_resp_hdrs), (200, [], empty_shard_resp_hdrs), (200, list(reversed(sr_objs[0])), shard_resp_hdrs[2]) ] @@ -2279,10 +2263,10 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing', marker='koolaid', reverse='true')), # 200 - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='koolaid', end_marker='ham', reverse='true', states='listing', limit=str(limit))), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='koolaid', end_marker='', reverse='true', states='listing', limit=str(limit))) # 200 ] @@ -2291,26 +2275,16 @@ class TestContainerController(TestRingBase): mock_responses, list(reversed(sr_objs[0])), expected_requests, query_string='?marker=koolaid&reverse=true', reverse=True) # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) def _check_GET_sharded_container_shard_error(self, error): # verify ordered listing when a shard is empty shard_bounds = (('', 'ham'), ('ham', 'pie'), ('lemon', '')) - shard_ranges = [ - ShardRange('.shards_a/c_%s' % upper, Timestamp.now(), lower, upper) - for lower, upper in shard_bounds] - sr_dicts = [dict(sr) for sr in shard_ranges] - sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) # empty second shard range sr_objs[1] = [] - shard_resp_hdrs = [ - {'X-Backend-Sharding-State': 'unsharded', - 'X-Container-Object-Count': len(sr_objs[i]), - 'X-Container-Bytes-Used': - sum([obj['bytes'] for obj in sr_objs[i]]), - 'X-Container-Meta-Flavour': 'flavour%d' % i, - 'X-Backend-Storage-Policy-Index': 0} - for i in range(3)] + shard_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) all_objects = [] for objects in sr_objs: @@ -2318,18 +2292,12 @@ class TestContainerController(TestRingBase): size_all_objects = sum([obj['bytes'] for obj in all_objects]) num_all_objects = len(all_objects) limit = CONTAINER_LISTING_LIMIT - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Timestamp': '99', - 'X-Container-Object-Count': num_all_objects, - 'X-Container-Bytes-Used': size_all_objects, - 'X-Container-Meta-Flavour': 'peach', - 'X-Backend-Storage-Policy-Index': 0} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs( + num_all_objects, size_all_objects) mock_responses = [ # status, body, headers - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0])] + \ [(error, [], {})] * 2 * self.CONTAINER_REPLICAS @@ -2338,10 +2306,10 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit)))] \ - + [(shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + + [(namespaces[1].name, {'X-Backend-Record-Type': 'auto'}, dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))) ] * 2 * self.CONTAINER_REPLICAS @@ -2357,21 +2325,12 @@ class TestContainerController(TestRingBase): def test_GET_sharded_container_sharding_shard(self): # one shard is in process of sharding shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', '')) - shard_ranges = [ - ShardRange('.shards_a/c_' + upper, Timestamp.now(), lower, upper) - for lower, upper in shard_bounds] - sr_dicts = [dict(sr) for sr in shard_ranges] - sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] - shard_resp_hdrs = [ - {'X-Backend-Sharding-State': 'unsharded', - 'X-Container-Object-Count': len(sr_objs[i]), - 'X-Container-Bytes-Used': - sum([obj['bytes'] for obj in sr_objs[i]]), - 'X-Container-Meta-Flavour': 'flavour%d' % i, - 'X-Backend-Storage-Policy-Index': 0} - for i in range(3)] + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + shard_resp_hdrs = self._make_shard_resp_hdrs(sr_objs) shard_1_shard_resp_hdrs = dict(shard_resp_hdrs[1]) shard_1_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) # second shard is sharding and has cleaved two out of three sub shards shard_resp_hdrs[1]['X-Backend-Sharding-State'] = 'sharding' @@ -2404,12 +2363,13 @@ class TestContainerController(TestRingBase): 'X-Backend-Storage-Policy-Index': 0} root_shard_resp_hdrs = dict(root_resp_hdrs) root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + root_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) mock_responses = [ # status, body, headers - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), - (200, sub_sr_dicts + [sr_dicts[1]], shard_1_shard_resp_hdrs), + (200, sub_sr_dicts + [ns_dicts[1]], shard_1_shard_resp_hdrs), (200, sub_sr_objs[0], sub_shard_resp_hdrs[0]), (200, sub_sr_objs[1], sub_shard_resp_hdrs[1]), (200, sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):], @@ -2422,13 +2382,13 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 # get first shard objects - (shard_ranges[0].name, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), # 200 # get second shard sub-shard ranges - (shard_ranges[1].name, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='h', end_marker='pie\x00', states='listing', @@ -2446,14 +2406,14 @@ class TestContainerController(TestRingBase): dict(marker='j', end_marker='lemon\x00', states='listing', limit=str(limit - len(sr_objs[0] + sub_sr_objs[0])))), # get remainder of first shard objects - (shard_ranges[1].name, + (namespaces[1].name, {'X-Backend-Record-Type': 'object', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='l', end_marker='pie\x00', limit=str(limit - len(sr_objs[0] + sub_sr_objs[0] + sub_sr_objs[1])))), # 200 # get third shard objects - (shard_ranges[2].name, + (namespaces[2].name, {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='p', end_marker='', states='listing', @@ -2465,7 +2425,7 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests) # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) @patch_policies([ StoragePolicy(0, 'zero', True, object_ring=FakeRing()), @@ -2475,22 +2435,15 @@ class TestContainerController(TestRingBase): # scenario: one shard is in process of sharding, shards have different # policy than root, expect listing to always request root policy index shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', '')) - shard_ranges = [ - ShardRange('.shards_a/c_' + upper, Timestamp.now(), lower, upper) - for lower, upper in shard_bounds] - sr_dicts = [dict(sr) for sr in shard_ranges] - sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] - shard_resp_hdrs = [ - {'X-Backend-Sharding-State': 'unsharded', - 'X-Container-Object-Count': len(sr_objs[i]), - 'X-Container-Bytes-Used': - sum([obj['bytes'] for obj in sr_objs[i]]), - 'X-Container-Meta-Flavour': 'flavour%d' % i, - 'X-Backend-Storage-Policy-Index': 1, - 'X-Backend-Record-Storage-Policy-Index': 0} - for i in range(3)] + namespaces, ns_dicts, sr_objs = self.create_server_response_data( + shard_bounds) + shard_resp_hdrs = self._make_shard_resp_hdrs( + sr_objs, extra_hdrs={ + 'X-Backend-Storage-Policy-Index': 1, + 'X-Backend-Record-Storage-Policy-Index': 0}) shard_1_shard_resp_hdrs = dict(shard_resp_hdrs[1]) shard_1_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) # second shard is sharding and has cleaved two out of three sub shards shard_resp_hdrs[1]['X-Backend-Sharding-State'] = 'sharding' @@ -2500,15 +2453,10 @@ class TestContainerController(TestRingBase): for lower, upper in sub_shard_bounds] sub_sr_dicts = [dict(sr) for sr in sub_shard_ranges] sub_sr_objs = [self._make_shard_objects(sr) for sr in sub_shard_ranges] - sub_shard_resp_hdrs = [ - {'X-Backend-Sharding-State': 'unsharded', - 'X-Container-Object-Count': len(sub_sr_objs[i]), - 'X-Container-Bytes-Used': - sum([obj['bytes'] for obj in sub_sr_objs[i]]), - 'X-Container-Meta-Flavour': 'flavour%d' % i, - 'X-Backend-Storage-Policy-Index': 1, - 'X-Backend-Record-Storage-Policy-Index': 0} - for i in range(2)] + sub_shard_resp_hdrs = self._make_shard_resp_hdrs( + sub_sr_objs, extra_hdrs={ + 'X-Backend-Storage-Policy-Index': 1, + 'X-Backend-Record-Storage-Policy-Index': 0}) all_objects = [] for objects in sr_objs: @@ -2524,12 +2472,13 @@ class TestContainerController(TestRingBase): 'X-Backend-Storage-Policy-Index': 0} root_shard_resp_hdrs = dict(root_resp_hdrs) root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + root_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS) mock_responses = [ # status, body, headers - (200, sr_dicts, root_shard_resp_hdrs), + (200, ns_dicts, root_shard_resp_hdrs), (200, sr_objs[0], shard_resp_hdrs[0]), - (200, sub_sr_dicts + [sr_dicts[1]], shard_1_shard_resp_hdrs), + (200, sub_sr_dicts + [ns_dicts[1]], shard_1_shard_resp_hdrs), (200, sub_sr_objs[0], sub_shard_resp_hdrs[0]), (200, sub_sr_objs[1], sub_shard_resp_hdrs[1]), (200, sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):], @@ -2542,13 +2491,13 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 # get first shard objects - (shard_ranges[0].name, + (namespaces[0].name, {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), # 200 # get second shard sub-shard ranges - (shard_ranges[1].name, + (namespaces[1].name, {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='h', end_marker='pie\x00', states='listing', @@ -2566,14 +2515,14 @@ class TestContainerController(TestRingBase): dict(marker='j', end_marker='lemon\x00', states='listing', limit=str(limit - len(sr_objs[0] + sub_sr_objs[0])))), # get remainder of second shard objects - (shard_ranges[1].name, + (namespaces[1].name, {'X-Backend-Record-Type': 'object', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='l', end_marker='pie\x00', limit=str(limit - len(sr_objs[0] + sub_sr_objs[0] + sub_sr_objs[1])))), # 200 # get third shard objects - (shard_ranges[2].name, + (namespaces[2].name, {'X-Backend-Record-Type': 'auto', 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='p', end_marker='', states='listing', @@ -2585,7 +2534,7 @@ class TestContainerController(TestRingBase): resp = self._check_GET_shard_listing( mock_responses, expected_objects, expected_requests) # root object count will overridden by actual length of listing - self.check_response(resp, root_resp_hdrs) + self.check_listing_response(resp, root_resp_hdrs) @patch_policies([ StoragePolicy(0, 'zero', True, object_ring=FakeRing()), @@ -2597,8 +2546,11 @@ class TestContainerController(TestRingBase): # policy index def do_test(shard_policy): # only need first shard for this test... - sr = ShardRange('.shards_a/c_pie', Timestamp.now(), '', 'pie') - sr_objs = self._make_shard_objects(sr) + shard_bounds = ('', 'pie') + namespaces, _, sr_objs = self.create_server_response_data( + shard_bounds) + sr = namespaces[0] + sr_objs = sr_objs[0] shard_resp_hdrs = { 'X-Backend-Sharding-State': 'unsharded', 'X-Container-Object-Count': len(sr_objs), @@ -2613,15 +2565,9 @@ class TestContainerController(TestRingBase): size_all_objects = sum([obj['bytes'] for obj in sr_objs]) num_all_objects = len(sr_objs) limit = CONTAINER_LISTING_LIMIT - root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', - 'X-Backend-Timestamp': '99', - 'X-Container-Object-Count': num_all_objects, - 'X-Container-Bytes-Used': size_all_objects, - 'X-Container-Meta-Flavour': 'peach', - # NB root policy 1 differes from shard policy - 'X-Backend-Storage-Policy-Index': 1} - root_shard_resp_hdrs = dict(root_resp_hdrs) - root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + root_resp_hdrs, root_shard_resp_hdrs = self._make_root_resp_hdrs( + num_all_objects, size_all_objects, + extra_hdrs={'X-Backend-Storage-Policy-Index': 1}) mock_responses = [ # status, body, headers @@ -2648,6 +2594,63 @@ class TestContainerController(TestRingBase): do_test(0) do_test(None) + +@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) +class TestGetShardedContainerLegacy(TestGetShardedContainer): + """ + Test all existing test cases to query Namespaces from container server but + get ShardRanges returned. This is to test backward compatibility that new + proxy-servers ask for new Namespace format from the older version container + servers who don't support Namespace format yet. + """ + # old container servers did not return this header + RESP_SHARD_FORMAT_HEADERS = {} + + def create_server_response_data(self, bounds, states=None): + if not isinstance(bounds[0], (list, tuple)): + bounds = [(l, u) for l, u in zip(bounds[:-1], bounds[1:])] + if not states: + states = [] + shard_ranges = [ + ShardRange('.shards_a/c_%s' % bound[1].replace('/', '-'), + Timestamp.now(), bound[0], bound[1], state=state) + for bound, state in zip_longest( + bounds, states, fillvalue=ShardRange.FOUND)] + sr_dicts = [dict(sr, last_modified=sr.timestamp.isoformat) + for sr in shard_ranges] + sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] + return shard_ranges, sr_dicts, sr_objs + + +class TestGetPathNamespaceCaching(BaseTestContainerController): + # These tests are verifying the content and caching of the backend + # namespace responses so we're not interested in gathering objects from the + # shards. We therefore mock _get_from_shards so that the response actually + # contains a fake listing and also capture the namespace listing passed to + # _get_from_shards. This avoids faking all the object listing responses + # from shards, and facilitates making assertions about the namespaces + # passed to _get_from_shards. + bogus_listing = [{'name': 'x'}, {'name': 'y'}] + bogus_listing_body = json.dumps(bogus_listing).encode('ascii') + + def setUp(self): + super(TestGetPathNamespaceCaching, self).setUp() + self._setup_shard_range_stubs() + self.get_from_shards_lists = [] + + def _fake_get_from_shards(self, req, resp, namespaces): + self.get_from_shards_lists.append(namespaces) + resp.body = self.bogus_listing_body + return resp + + def _call_app(self, req): + self.get_from_shards_lists = [] + with mock.patch( + 'swift.proxy.controllers.container.' + 'ContainerController._get_from_shards', + side_effect=self._fake_get_from_shards): + return req.get_response(self.app) + def _build_request(self, headers, params, infocache=None): # helper to make a GET request with caches set in environ query_string = '?' + '&'.join('%s=%s' % (k, v) @@ -2658,10 +2661,10 @@ class TestContainerController(TestRingBase): request.environ['swift.infocache'] = infocache if infocache else {} return request - def _check_response(self, resp, exp_shards, extra_hdrs): + def _check_response(self, resp, exp_listing, extra_hdrs): # helper to check a shard listing response actual_shards = json.loads(resp.body) - self.assertEqual(exp_shards, actual_shards) + self.assertEqual(exp_listing, actual_shards) exp_hdrs = dict(self.root_resp_hdrs) # x-put-timestamp is sent from backend but removed in proxy base # controller GETorHEAD_base so not expected in response from proxy @@ -2675,7 +2678,7 @@ class TestContainerController(TestRingBase): exp_hdrs.update( {'X-Storage-Policy': 'zero', # added in container controller 'Content-Length': - str(len(json.dumps(exp_shards).encode('ascii'))), + str(len(json.dumps(exp_listing).encode('ascii'))), } ) # we expect this header to be removed by proxy @@ -2695,7 +2698,7 @@ class TestContainerController(TestRingBase): with mocked_http_conn( *resp_status, body_iter=[resp_body] * num_resp, headers=[resp_hdrs] * num_resp) as fake_conn: - resp = req.get_response(self.app) + resp = self._call_app(req) self.assertEqual(resp_status[0], resp.status_int) self.assertEqual(num_resp, len(fake_conn.requests)) return fake_conn.requests[0], resp @@ -2740,8 +2743,7 @@ class TestContainerController(TestRingBase): for lower, upper in shard_bounds] self.namespaces = [Namespace(**ns) for ns in self.ns_dicts] self.ns_bound_list = NamespaceBoundList.parse(self.namespaces) - self.sr_dicts = [dict(ShardRange(timestamp=Timestamp.now(), **ns)) - for ns in self.ns_dicts] + self.sr_dicts = [dict(Namespace(**ns)) for ns in self.ns_dicts] self._stub_shards_dump = json.dumps(self.sr_dicts).encode('ascii') self.root_resp_hdrs = { 'Accept-Ranges': 'bytes', @@ -2758,8 +2760,14 @@ class TestContainerController(TestRingBase): 'X-Backend-Storage-Policy-Index': '0'} def _do_test_caching(self, record_type, exp_recheck_listing, - exp_record_type): + exp_record_type, exp_listing, + extra_backend_req_hdrs=None): # this test gets shard ranges into cache and then reads from cache + exp_backend_req_hdrs = { + 'X-Backend-Record-Type': record_type, + 'X-Backend-Override-Shard-Name-Filter': 'sharded'} + if extra_backend_req_hdrs: + exp_backend_req_hdrs.update(extra_backend_req_hdrs) sharding_state = 'sharded' exp_resp_headers = { 'X-Backend-Recheck-Container-Existence': '60', @@ -2784,10 +2792,8 @@ class TestContainerController(TestRingBase): 'X-Backend-Sharding-State': sharding_state, 'X-Backend-Override-Shard-Name-Filter': 'true'}) self._check_backend_req( - req, backend_req, - extra_hdrs={'X-Backend-Record-Type': record_type, - 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) - self._check_response(resp, self.ns_dicts, exp_resp_headers) + req, backend_req, extra_hdrs=exp_backend_req_hdrs) + self._check_response(resp, exp_listing, exp_resp_headers) cache_key = 'shard-listing-v2/a/c' self.assertEqual( @@ -2809,7 +2815,7 @@ class TestContainerController(TestRingBase): 'container.shard_listing.cache.bypass.200']) # container is sharded and proxy has that state cached, but - # no shard ranges cached; expect a cache miss and write-back + # no namespaces cached; expect a cache miss and write-back self.memcache.delete(cache_key) self.memcache.clear_calls() self.logger.clear() @@ -2821,10 +2827,8 @@ class TestContainerController(TestRingBase): 'X-Backend-Sharding-State': sharding_state, 'X-Backend-Override-Shard-Name-Filter': 'true'}) self._check_backend_req( - req, backend_req, - extra_hdrs={'X-Backend-Record-Type': record_type, - 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) - self._check_response(resp, self.ns_dicts, exp_resp_headers) + req, backend_req, extra_hdrs=exp_backend_req_hdrs) + self._check_response(resp, exp_listing, exp_resp_headers) self.assertEqual( [mock.call.get('container/a/c'), mock.call.get(cache_key, raise_on_error=True), @@ -2845,13 +2849,13 @@ class TestContainerController(TestRingBase): 'container.shard_listing.cache.miss.200']) # container is sharded and proxy does have that state cached and - # also has shard ranges cached; expect a read from cache + # also has namespaces cached; expect a read from cache self.memcache.clear_calls() self.logger.clear() req = self._build_request({'X-Backend-Record-Type': record_type}, {'states': 'listing'}, {}) resp = req.get_response(self.app) - self._check_response(resp, self.ns_dicts, exp_cache_resp_headers) + self._check_response(resp, exp_listing, exp_cache_resp_headers) self.assertEqual( [mock.call.get('container/a/c'), mock.call.get(cache_key, raise_on_error=True)], @@ -2879,10 +2883,8 @@ class TestContainerController(TestRingBase): 'X-Backend-Sharding-State': sharding_state, 'X-Backend-Override-Shard-Name-Filter': 'true'}) self._check_backend_req( - req, backend_req, - extra_hdrs={'X-Backend-Record-Type': record_type, - 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) - self._check_response(resp, self.ns_dicts, exp_resp_headers) + req, backend_req, extra_hdrs=exp_backend_req_hdrs) + self._check_response(resp, exp_listing, exp_resp_headers) self.assertEqual( [mock.call.get('container/a/c'), mock.call.set(cache_key, self.ns_bound_list.bounds, @@ -2908,7 +2910,7 @@ class TestContainerController(TestRingBase): {'states': 'listing'}, {}) with mock.patch('random.random', return_value=0.11): resp = req.get_response(self.app) - self._check_response(resp, self.ns_dicts, exp_cache_resp_headers) + self._check_response(resp, exp_listing, exp_cache_resp_headers) self.assertEqual( [mock.call.get('container/a/c'), mock.call.get(cache_key, raise_on_error=True)], @@ -2932,7 +2934,7 @@ class TestContainerController(TestRingBase): infocache=req.environ['swift.infocache']) with mock.patch('random.random', return_value=0.11): resp = req.get_response(self.app) - self._check_response(resp, self.ns_dicts, exp_cache_resp_headers) + self._check_response(resp, exp_listing, exp_cache_resp_headers) self.assertEqual([], self.memcache.calls) self.assertIn('swift.infocache', req.environ) self.assertIn(cache_key, req.environ['swift.infocache']) @@ -2961,7 +2963,6 @@ class TestContainerController(TestRingBase): self.memcache.calls) def test_get_from_shards_add_root_spi(self): - self._setup_shard_range_stubs() shard_resp = mock.MagicMock(status_int=204, headers={}) def mock_get_container_listing(self_, req, *args, **kargs): @@ -2971,7 +2972,7 @@ class TestContainerController(TestRingBase): # header in response -> header added to request captured_hdrs = {} req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) - resp = mock.MagicMock(body=self._stub_shards_dump, + resp = mock.MagicMock(status_int=200, headers=self.root_resp_hdrs, request=req) resp.headers['X-Backend-Storage-Policy-Index'] = '0' @@ -2980,7 +2981,7 @@ class TestContainerController(TestRingBase): mock_get_container_listing): controller_cls, d = self.app.get_controller(req) controller = controller_cls(self.app, **d) - controller._get_from_shards(req, resp) + controller._get_from_shards(req, resp, list(self.namespaces)) self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs) self.assertEqual( @@ -2988,7 +2989,7 @@ class TestContainerController(TestRingBase): captured_hdrs = {} req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) - resp = mock.MagicMock(body=self._stub_shards_dump, + resp = mock.MagicMock(status_int=200, headers=self.root_resp_hdrs, request=req) resp.headers['X-Backend-Storage-Policy-Index'] = '1' @@ -2997,7 +2998,7 @@ class TestContainerController(TestRingBase): mock_get_container_listing): controller_cls, d = self.app.get_controller(req) controller = controller_cls(self.app, **d) - controller._get_from_shards(req, resp) + controller._get_from_shards(req, resp, list(self.namespaces)) self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs) self.assertEqual( @@ -3010,7 +3011,7 @@ class TestContainerController(TestRingBase): 'REQUEST_METHOD': 'GET', 'swift.shard_listing_history': [('a', 'c')]} ) - resp = mock.MagicMock(body=self._stub_shards_dump, + resp = mock.MagicMock(status_int=200, headers=self.root_resp_hdrs, request=req) resp.headers['X-Backend-Storage-Policy-Index'] = '0' @@ -3019,7 +3020,7 @@ class TestContainerController(TestRingBase): mock_get_container_listing): controller_cls, d = self.app.get_controller(req) controller = controller_cls(self.app, **d) - controller._get_from_shards(req, resp) + controller._get_from_shards(req, resp, list(self.namespaces)) self.assertNotIn('X-Backend-Storage-Policy-Index', captured_hdrs) @@ -3027,7 +3028,7 @@ class TestContainerController(TestRingBase): captured_hdrs = {} req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) req.headers['X-Backend-Storage-Policy-Index'] = '0' - resp = mock.MagicMock(body=self._stub_shards_dump, + resp = mock.MagicMock(status_int=200, headers=self.root_resp_hdrs, request=req) resp.headers['X-Backend-Storage-Policy-Index'] = '1' @@ -3036,38 +3037,30 @@ class TestContainerController(TestRingBase): mock_get_container_listing): controller_cls, d = self.app.get_controller(req) controller = controller_cls(self.app, **d) - controller._get_from_shards(req, resp) + controller._get_from_shards(req, resp, list(self.namespaces)) self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs) self.assertEqual( captured_hdrs['X-Backend-Storage-Policy-Index'], '0') def test_GET_shard_ranges(self): - self._setup_shard_range_stubs() # expect shard ranges cache time to be default value of 600 - self._do_test_caching('shard', 600, 'shard') + self._do_test_caching('shard', 600, 'shard', self.ns_dicts) # expect shard ranges cache time to be configured value of 120 self.app.recheck_listing_shard_ranges = 120 - self._do_test_caching('shard', 120, 'shard') - - def mock_get_from_shards(self, req, resp): - # for the purposes of these tests we override _get_from_shards so - # that the response contains the shard listing even though the - # record_type is 'auto'; these tests are verifying the content and - # caching of the backend shard range response so we're not - # interested in gathering object from the shards - return resp + self._do_test_caching('shard', 120, 'shard', self.ns_dicts) with mock.patch('swift.proxy.controllers.container.' 'ContainerController._get_from_shards', - mock_get_from_shards): + self._fake_get_from_shards): self.app.recheck_listing_shard_ranges = 600 - self._do_test_caching('auto', 600, exp_record_type=None) + extra_req_headers = {'X-Backend-Record-Shard-Format': 'namespace'} + self._do_test_caching('auto', 600, None, self.bogus_listing, + extra_backend_req_hdrs=extra_req_headers) def test_GET_shard_ranges_404_response(self): # pre-warm cache with container info but not shard ranges so that the # backend request tries to get a cacheable listing, but backend 404's - self._setup_shard_range_stubs() self.memcache.delete_all() info = headers_to_container_info(self.root_resp_hdrs) info['status'] = 200 @@ -3098,7 +3091,6 @@ class TestContainerController(TestRingBase): self.logger.statsd_client.get_increment_counts()) def test_GET_shard_ranges_read_from_cache_error(self): - self._setup_shard_range_stubs() self.memcache = FakeMemcache() self.memcache.delete_all() self.logger.clear() @@ -3155,7 +3147,6 @@ class TestContainerController(TestRingBase): return resp def test_GET_shard_ranges_read_from_cache(self): - self._setup_shard_range_stubs() exp_hdrs = {'X-Backend-Cached-Results': 'true', 'X-Backend-Record-Type': 'shard', 'X-Backend-Override-Shard-Name-Filter': 'true', @@ -3184,36 +3175,40 @@ class TestContainerController(TestRingBase): {'states': 'listing', 'includes': 'egg'}, 'shard') self._check_response(resp, self.ns_dicts[:1], exp_hdrs) - # override _get_from_shards so that the response contains the shard - # listing that we want to verify even though the record_type is 'auto' - def mock_get_from_shards(self, req, resp): - return resp - # request with record-type=auto does not expect record-type in response del exp_hdrs['X-Backend-Record-Type'] with mock.patch('swift.proxy.controllers.container.' 'ContainerController._get_from_shards', - mock_get_from_shards): + side_effect=self._fake_get_from_shards): resp = self._do_test_GET_shard_ranges_read_from_cache( {'states': 'listing', 'reverse': 'true'}, 'auto') - exp_shards = list(self.ns_dicts) + exp_shards = list(self.namespaces) exp_shards.reverse() - self._check_response(resp, exp_shards, exp_hdrs) + self._check_response(resp, self.bogus_listing, exp_hdrs) + self.assertEqual([exp_shards], self.get_from_shards_lists) + self.get_from_shards_lists = [] resp = self._do_test_GET_shard_ranges_read_from_cache( {'states': 'listing', 'marker': 'jam'}, 'auto') - self._check_response(resp, self.ns_dicts[1:], exp_hdrs) + self._check_response(resp, self.bogus_listing, exp_hdrs) + self.assertEqual([self.namespaces[1:]], self.get_from_shards_lists) + self.get_from_shards_lists = [] resp = self._do_test_GET_shard_ranges_read_from_cache( {'states': 'listing', 'marker': 'jam', 'end_marker': 'kale'}, 'auto') - self._check_response(resp, self.ns_dicts[1:2], exp_hdrs) + self._check_response(resp, self.bogus_listing, exp_hdrs) + self.assertEqual([self.namespaces[1:2]], + self.get_from_shards_lists) + self.get_from_shards_lists = [] resp = self._do_test_GET_shard_ranges_read_from_cache( {'states': 'listing', 'includes': 'egg'}, 'auto') - self._check_response(resp, self.ns_dicts[:1], exp_hdrs) + self._check_response(resp, self.bogus_listing, exp_hdrs) + self.assertEqual([self.namespaces[:1]], self.get_from_shards_lists) - def _do_test_GET_shard_ranges_write_to_cache(self, params, record_type): + def _do_test_GET_shard_ranges_write_to_cache( + self, params, record_type, extra_backend_req_hdrs=None): # verify that shard range listing are written to cache when appropriate self.logger.clear() self.memcache.delete_all() @@ -3227,11 +3222,14 @@ class TestContainerController(TestRingBase): 'X-Backend-Sharding-State': 'sharded'} backend_req, resp = self._capture_backend_request( req, 200, self._stub_shards_dump, resp_hdrs) + extra_hdrs = {'X-Backend-Record-Type': record_type, + 'X-Backend-Override-Shard-Name-Filter': 'sharded'} + if extra_backend_req_hdrs: + extra_hdrs.update(extra_backend_req_hdrs) self._check_backend_req( req, backend_req, extra_params=params, - extra_hdrs={'X-Backend-Record-Type': record_type, - 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) + extra_hdrs=extra_hdrs) expected_hdrs = {'X-Backend-Recheck-Container-Existence': '60'} expected_hdrs.update(resp_hdrs) self.assertEqual( @@ -3252,7 +3250,6 @@ class TestContainerController(TestRingBase): return resp def test_GET_shard_ranges_write_to_cache(self): - self._setup_shard_range_stubs() exp_hdrs = {'X-Backend-Recheck-Container-Existence': '60', 'X-Backend-Record-Type': 'shard', 'X-Backend-Override-Shard-Name-Filter': 'true', @@ -3281,40 +3278,46 @@ class TestContainerController(TestRingBase): {'states': 'listing', 'includes': 'egg'}, 'shard') self._check_response(resp, self.ns_dicts[:1], exp_hdrs) - # override _get_from_shards so that the response contains the shard - # listing that we want to verify even though the record_type is 'auto' - def mock_get_from_shards(self, req, resp): - return resp - # request with record-type=auto does not expect record-type in response del exp_hdrs['X-Backend-Record-Type'] + extra_req_headers = {'X-Backend-Record-Shard-Format': 'namespace'} with mock.patch('swift.proxy.controllers.container.' 'ContainerController._get_from_shards', - mock_get_from_shards): + side_effect=self._fake_get_from_shards): resp = self._do_test_GET_shard_ranges_write_to_cache( - {'states': 'listing', 'reverse': 'true'}, 'auto') - exp_shards = list(self.ns_dicts) + {'states': 'listing', 'reverse': 'true'}, 'auto', + extra_backend_req_hdrs=extra_req_headers) + exp_shards = list(self.namespaces) exp_shards.reverse() - self._check_response(resp, exp_shards, exp_hdrs) + self.assertEqual([exp_shards], self.get_from_shards_lists) + self.get_from_shards_lists = [] resp = self._do_test_GET_shard_ranges_write_to_cache( - {'states': 'listing', 'marker': 'jam'}, 'auto') - self._check_response(resp, self.ns_dicts[1:], exp_hdrs) + {'states': 'listing', 'marker': 'jam'}, 'auto', + extra_backend_req_hdrs=extra_req_headers) + self._check_response(resp, self.bogus_listing, exp_hdrs) + self.assertEqual([self.namespaces[1:]], self.get_from_shards_lists) + + self.get_from_shards_lists = [] resp = self._do_test_GET_shard_ranges_write_to_cache( {'states': 'listing', 'marker': 'jam', 'end_marker': 'kale'}, - 'auto') - self._check_response(resp, self.ns_dicts[1:2], exp_hdrs) + 'auto', extra_backend_req_hdrs=extra_req_headers) + self._check_response(resp, self.bogus_listing, exp_hdrs) + self.assertEqual([self.namespaces[1:2]], + self.get_from_shards_lists) + self.get_from_shards_lists = [] resp = self._do_test_GET_shard_ranges_write_to_cache( - {'states': 'listing', 'includes': 'egg'}, 'auto') - self._check_response(resp, self.ns_dicts[:1], exp_hdrs) + {'states': 'listing', 'includes': 'egg'}, 'auto', + extra_backend_req_hdrs=extra_req_headers) + self._check_response(resp, self.bogus_listing, exp_hdrs) + self.assertEqual([self.namespaces[:1]], self.get_from_shards_lists) def test_GET_shard_ranges_write_to_cache_with_x_newest(self): # when x-newest is sent, verify that there is no cache lookup to check # sharding state but then backend requests are made requesting complete # shard list which can be cached - self._setup_shard_range_stubs() self.memcache.delete_all() self.memcache.clear_calls() req_hdrs = {'X-Backend-Record-Type': 'shard', @@ -3389,8 +3392,6 @@ class TestContainerController(TestRingBase): # lookup is only attempted when the cached sharding state and status # are suitable, and full set of headers can be constructed from cache; # Note: backend response has state unsharded so no shard ranges cached - self._setup_shard_range_stubs() - def do_test(info): self._setup_shard_range_stubs() self.memcache.set('container/a/c', info) @@ -3432,7 +3433,6 @@ class TestContainerController(TestRingBase): # 'X-Backend-Override-Shard-Name-Filter': 'true' to be returned unless # the sharding state is 'sharded' but include it in this test to check # that the state is checked by proxy controller - self._setup_shard_range_stubs() self._do_test_GET_shard_ranges_no_cache_write( {'X-Backend-Record-Type': 'shard', 'X-Backend-Override-Shard-Name-Filter': 'true', @@ -3454,7 +3454,6 @@ class TestContainerController(TestRingBase): # verify that shard ranges are not written to cache when container # response does not acknowledge x-backend-override-shard-name-filter # e.g. container server not upgraded - self._setup_shard_range_stubs() self._do_test_GET_shard_ranges_no_cache_write( {'X-Backend-Record-Type': 'shard', 'X-Backend-Sharding-State': 'sharded'}) @@ -3470,7 +3469,6 @@ class TestContainerController(TestRingBase): def test_GET_shard_ranges_no_cache_write_for_object_listing(self): # verify that shard ranges are not written to cache when container # response does not return shard ranges - self._setup_shard_range_stubs() self._do_test_GET_shard_ranges_no_cache_write( {'X-Backend-Record-Type': 'object', 'X-Backend-Override-Shard-Name-Filter': 'true', @@ -3533,7 +3531,7 @@ class TestContainerController(TestRingBase): [{'not': ' a shard range'}]) error_lines = self.logger.get_lines_for_level('error') self.assertEqual(1, len(error_lines), error_lines) - self.assertIn('Failed to get shard ranges', error_lines[0]) + self.assertIn('Failed to get namespaces', error_lines[0]) self.logger.clear() self._do_test_GET_shard_ranges_bad_response_body( @@ -3580,7 +3578,6 @@ class TestContainerController(TestRingBase): def test_GET_shard_ranges_no_cache_recheck_listing_shard_ranges(self): # verify that a GET for shards does not lookup or store in cache when # cache expiry time is set to zero - self._setup_shard_range_stubs() self.app.recheck_listing_shard_ranges = 0 self._do_test_GET_shards_no_cache_listing('unsharded') self._do_test_GET_shards_no_cache_listing('sharding') @@ -3601,7 +3598,6 @@ class TestContainerController(TestRingBase): def test_GET_shard_ranges_no_cache_when_requesting_updating_shards(self): # verify that a GET for shards in updating states does not lookup or # store in cache - self._setup_shard_range_stubs() self._do_test_GET_shards_no_cache_updating('unsharded') self._do_test_GET_shards_no_cache_updating('sharding') self._do_test_GET_shards_no_cache_updating('sharded') @@ -3611,7 +3607,6 @@ class TestContainerController(TestRingBase): def test_GET_shard_ranges_no_cache_when_include_deleted_shards(self): # verify that a GET for shards in listing states does not lookup or # store in cache if x-backend-include-deleted is true - self._setup_shard_range_stubs() self._do_test_GET_shards_no_cache( 'unsharded', {'states': 'listing'}, {'X-Backend-Include-Deleted': 'true'}) @@ -3631,7 +3626,6 @@ class TestContainerController(TestRingBase): def test_GET_objects_makes_no_cache_lookup(self): # verify that an object GET request does not lookup container metadata # in cache - self._setup_shard_range_stubs() self.memcache.delete_all() self.memcache.clear_calls() req_hdrs = {'X-Backend-Record-Type': 'object'} @@ -3658,7 +3652,6 @@ class TestContainerController(TestRingBase): self.memcache.calls[0][1][1]['sharding_state']) def test_GET_shard_ranges_no_memcache_available(self): - self._setup_shard_range_stubs() self.memcache.clear_calls() hdrs = {'X-Backend-Record-Type': 'shard'} params = {'states': 'listing'} @@ -3695,7 +3688,7 @@ class TestContainerController(TestRingBase): with mocked_http_conn( *resp_status, body_iter=[b''] * num_resp, headers=[{}] * num_resp): - resp = req.get_response(self.app) + resp = self._call_app(req) self.assertEqual(resp_status[0], resp.status_int) self.assertNotIn(cont_key, req.environ['swift.cache'].store) self.assertNotIn(shard_key, req.environ['swift.cache'].store) @@ -3703,28 +3696,6 @@ class TestContainerController(TestRingBase): do_test('POST', 204, self.CONTAINER_REPLICAS) do_test('PUT', 202, self.CONTAINER_REPLICAS) - def test_GET_bad_requests(self): - # verify that the proxy controller enforces checks on request params - req = Request.blank( - '/v1/a/c?limit=%d' % (CONTAINER_LISTING_LIMIT + 1)) - self.assertEqual(412, req.get_response(self.app).status_int) - req = Request.blank('/v1/a/c?delimiter=%ff') - self.assertEqual(400, req.get_response(self.app).status_int) - req = Request.blank('/v1/a/c?marker=%ff') - self.assertEqual(400, req.get_response(self.app).status_int) - req = Request.blank('/v1/a/c?end_marker=%ff') - self.assertEqual(400, req.get_response(self.app).status_int) - req = Request.blank('/v1/a/c?prefix=%ff') - self.assertEqual(400, req.get_response(self.app).status_int) - req = Request.blank('/v1/a/c?format=%ff') - self.assertEqual(400, req.get_response(self.app).status_int) - req = Request.blank('/v1/a/c?path=%ff') - self.assertEqual(400, req.get_response(self.app).status_int) - req = Request.blank('/v1/a/c?includes=%ff') - self.assertEqual(400, req.get_response(self.app).status_int) - req = Request.blank('/v1/a/c?states=%ff') - self.assertEqual(400, req.get_response(self.app).status_int) - @patch_policies( [StoragePolicy(0, 'zero', True, object_ring=FakeRing(replicas=4))]) diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 0a8c7fc888..9b17d0de77 100644 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -42,7 +42,7 @@ from swift.common import utils, swob, exceptions from swift.common.exceptions import ChunkWriteTimeout, ShortReadError, \ ChunkReadTimeout from swift.common.utils import Timestamp, list_from_csv, md5, FileLikeIter, \ - ShardRange + ShardRange, Namespace, NamespaceBoundList from swift.proxy import server as proxy_server from swift.proxy.controllers import obj from swift.proxy.controllers.base import \ @@ -56,7 +56,7 @@ from test.unit import ( FakeRing, fake_http_connect, patch_policies, SlowBody, FakeStatus, DEFAULT_TEST_EC_TYPE, encode_frag_archive_bodies, make_ec_object_stub, fake_ec_node_response, StubResponse, mocked_http_conn, - quiet_eventlet_exceptions, FakeSource, make_timestamp_iter) + quiet_eventlet_exceptions, FakeSource, make_timestamp_iter, FakeMemcache) from test.unit.proxy.test_server import node_error_count @@ -7241,95 +7241,223 @@ class TestECFragGetter(BaseObjectControllerMixin, unittest.TestCase): @patch_policies() -class TestGetUpdatingShardRanges(BaseObjectControllerMixin, unittest.TestCase): +class TestGetUpdateShard(BaseObjectControllerMixin, unittest.TestCase): + bound_prefix = 'x' + item = 'x1_test' + def setUp(self): - super(TestGetUpdatingShardRanges, self).setUp() + super(TestGetUpdateShard, self).setUp() + self.ctrl = obj.BaseObjectController(self.app, 'a', 'c', 'o') + self.memcache = FakeMemcache() + ts_iter = make_timestamp_iter() + # NB: these shard ranges have gaps + self.shard_ranges = [ShardRange( + '.sharded_a/sr%d' % i, next(ts_iter), + self.bound_prefix + u'%d_lower' % i, + self.bound_prefix + u'%d_upper' % i, + object_count=i, bytes_used=1024 * i, + meta_timestamp=next(ts_iter)) + for i in range(3)] + + def _create_response_data(self, shards, includes=None): + resp_headers = {'X-Backend-Record-Type': 'shard', + 'X-Backend-Record-Shard-Format': 'namespace'} + namespaces = [Namespace(sr.name, sr.lower, sr.upper) + for sr in shards] + if includes is not None: + namespaces = [ns for ns in namespaces if includes in ns] + body = json.dumps([dict(ns) for ns in namespaces]).encode('ascii') + return body, resp_headers + + def test_get_update_shard_cache_writing(self): + # verify case when complete set of shards is returned + req = Request.blank('/v1/a/c/o', method='PUT', + environ={'swift.cache': self.memcache}) + body, resp_headers = self._create_response_data(self.shard_ranges) + with mocked_http_conn( + 200, 200, body_iter=iter([b'', body]), + headers=resp_headers) as fake_conn: + actual = self.ctrl._get_update_shard(req, 'a', 'c', self.item) + + # account info + captured = fake_conn.requests + self.assertEqual('HEAD', captured[0]['method']) + self.assertEqual('a', captured[0]['path'][7:]) + # container GET + self.assertEqual('GET', captured[1]['method']) + self.assertEqual('a/c', captured[1]['path'][7:]) + params = sorted(captured[1]['qs'].split('&')) + self.assertEqual( + ['format=json', 'states=updating'], params) + captured_hdrs = captured[1]['headers'] + self.assertEqual('shard', captured_hdrs.get('X-Backend-Record-Type')) + self.assertEqual('namespace', + captured_hdrs.get('X-Backend-Record-Shard-Format')) + exp_bounds = NamespaceBoundList.parse(self.shard_ranges).bounds + self.assertEqual(json.loads(json.dumps(exp_bounds)), + self.memcache.get('shard-updating-v2/a/c')) + exp_ns = Namespace(self.shard_ranges[1].name, + self.shard_ranges[1].lower, + self.shard_ranges[2].lower) + self.assertEqual(exp_ns, actual) + self.assertFalse(self.app.logger.get_lines_for_level('error')) + + def test_get_update_shard_cache_reading(self): + # verify case when complete set of shards is in cache + cached_bounds = NamespaceBoundList.parse(self.shard_ranges).bounds + self.memcache.set('shard-updating-v2/a/c', cached_bounds) + req = Request.blank('/v1/a/c/o', method='PUT', + environ={'swift.cache': self.memcache}) + + actual = self.ctrl._get_update_shard(req, 'a', 'c', self.item) + + self.assertEqual(json.loads(json.dumps(cached_bounds)), + self.memcache.get('shard-updating-v2/a/c')) + exp_ns = Namespace(self.shard_ranges[1].name, + self.shard_ranges[1].lower, + self.shard_ranges[2].lower) + self.assertEqual(exp_ns, actual) + self.assertFalse(self.app.logger.get_lines_for_level('error')) + + def test_get_update_shard_cache_recheck_time_zero(self): + # verify case when shard caching is disabled + req = Request.blank('/v1/a/c/o', method='PUT', + environ={'swift.cache': self.memcache}) + body, resp_headers = self._create_response_data( + self.shard_ranges, self.item) + with mock.patch.object(self.app, 'recheck_updating_shard_ranges', 0): + with mocked_http_conn( + 200, 200, body_iter=iter([b'', body]), + headers=resp_headers) as fake_conn: + actual = self.ctrl._get_update_shard(req, 'a', 'c', self.item) + + # account info + captured = fake_conn.requests + self.assertEqual('HEAD', captured[0]['method']) + self.assertEqual('a', captured[0]['path'][7:]) + # container GET + self.assertEqual('GET', captured[1]['method']) + self.assertEqual('a/c', captured[1]['path'][7:]) + params = sorted(captured[1]['qs'].split('&')) + self.assertEqual( + ['format=json', 'includes=' + quote(self.item), 'states=updating'], + params) + captured_hdrs = captured[1]['headers'] + self.assertEqual('shard', captured_hdrs.get('X-Backend-Record-Type')) + self.assertEqual('namespace', + captured_hdrs.get('X-Backend-Record-Shard-Format')) + self.assertIsNone(self.memcache.get('shard-updating-v2/a/c')) + exp_ns = Namespace(self.shard_ranges[1].name, + self.shard_ranges[1].lower, + self.shard_ranges[1].upper) + self.assertEqual(exp_ns, actual) + self.assertFalse(self.app.logger.get_lines_for_level('error')) + + def test_get_update_shard_cache_not_available(self): + # verify case when memcache is not available + req = Request.blank('/v1/a/c/o', method='PUT') + body, resp_headers = self._create_response_data(self.shard_ranges) + with mocked_http_conn( + 200, 200, body_iter=iter([b'', body]), + headers=resp_headers) as fake_conn: + actual = self.ctrl._get_update_shard(req, 'a', 'c', self.item) + + # account info + captured = fake_conn.requests + self.assertEqual('HEAD', captured[0]['method']) + self.assertEqual('a', captured[0]['path'][7:]) + # container GET + self.assertEqual('GET', captured[1]['method']) + self.assertEqual('a/c', captured[1]['path'][7:]) + params = sorted(captured[1]['qs'].split('&')) + self.assertEqual( + ['format=json', 'states=updating'], params) + captured_hdrs = captured[1]['headers'] + self.assertEqual('shard', captured_hdrs.get('X-Backend-Record-Type')) + self.assertEqual('namespace', + captured_hdrs.get('X-Backend-Record-Shard-Format')) + self.assertIsNone(self.memcache.get('shard-updating-v2/a/c')) + exp_ns = Namespace(self.shard_ranges[1].name, + self.shard_ranges[1].lower, + self.shard_ranges[2].lower) + self.assertEqual(exp_ns, actual) + self.assertFalse(self.app.logger.get_lines_for_level('error')) + + def test_get_update_shard_empty_body(self): + # verify case when no shards are returned + req = Request.blank('/v1/a/c/o', method='PUT', + environ={'swift.cache': self.memcache}) + + body, resp_headers = self._create_response_data(self.shard_ranges) + with mocked_http_conn( + 200, 200, body_iter=[b'', b''], + headers=resp_headers) as fake_conn: + actual = self.ctrl._get_update_shard(req, 'a', 'c', self.item) + + # account info + captured = fake_conn.requests + self.assertEqual('HEAD', captured[0]['method']) + self.assertEqual('a', captured[0]['path'][7:]) + # container GET + self.assertEqual('GET', captured[1]['method']) + self.assertEqual('a/c', captured[1]['path'][7:]) + params = sorted(captured[1]['qs'].split('&')) + self.assertEqual( + ['format=json', 'states=updating'], params) + captured_hdrs = captured[1]['headers'] + self.assertEqual('shard', captured_hdrs.get('X-Backend-Record-Type')) + self.assertEqual('namespace', + captured_hdrs.get('X-Backend-Record-Shard-Format')) + self.assertIsNone(self.memcache.get('shard-updating-v2/a/c')) + self.assertIsNone(actual) + lines = self.app.logger.get_lines_for_level('error') + self.assertEqual(1, len(lines)) + self.assertIn('Problem with listing response from /v1/a/c/o', lines[0]) + + +class TestGetUpdateShardUTF8(TestGetUpdateShard): + bound_prefix = u'\u1234' + item = wsgi_to_str('\xe1\x88\xb41_test') + + +class TestGetUpdateShardLegacy(TestGetUpdateShard): + def _create_response_data(self, shards, includes=None): + # older container servers never return the shorter 'namespace' format + # nor the 'X-Backend-Record-Shard-Format' header + resp_headers = {'X-Backend-Record-Type': 'shard'} + if includes is not None: + shards = [sr for sr in shards if includes in sr] + body = json.dumps([dict(sr) for sr in shards]).encode('ascii') + return body, resp_headers + + +class TestGetUpdateShardLegacyUTF8(TestGetUpdateShard): + bound_prefix = u'\u1234' + item = wsgi_to_str('\xe1\x88\xb41_test') + + +@patch_policies() +class TestGetUpdatingNamespacesErrors(BaseObjectControllerMixin, + unittest.TestCase): + def setUp(self): + super(TestGetUpdatingNamespacesErrors, self).setUp() self.ctrl = obj.BaseObjectController(self.app, 'a', 'c', 'o') - def test_get_shard_ranges_for_object_put(self): - ts_iter = make_timestamp_iter() - shard_ranges = [dict(ShardRange( - '.sharded_a/sr%d' % i, next(ts_iter), '%d_lower' % i, - '%d_upper' % i, object_count=i, bytes_used=1024 * i, - meta_timestamp=next(ts_iter))) - for i in range(3)] - req = Request.blank('/v1/a/c/o', method='PUT') - resp_headers = {'X-Backend-Record-Type': 'shard'} - with mocked_http_conn( - 200, 200, - body_iter=iter([b'', - json.dumps(shard_ranges[1:2]).encode('ascii')]), - headers=resp_headers - ) as fake_conn: - actual, resp = self.ctrl._get_updating_shard_ranges( - req, 'a', 'c', '1_test') - self.assertEqual(200, resp.status_int) - - # account info - captured = fake_conn.requests - self.assertEqual('HEAD', captured[0]['method']) - self.assertEqual('a', captured[0]['path'][7:]) - # container GET - self.assertEqual('GET', captured[1]['method']) - self.assertEqual('a/c', captured[1]['path'][7:]) - params = sorted(captured[1]['qs'].split('&')) - self.assertEqual( - ['format=json', 'includes=1_test', 'states=updating'], params) - self.assertEqual( - 'shard', captured[1]['headers'].get('X-Backend-Record-Type')) - self.assertEqual(shard_ranges[1:2], [dict(pr) for pr in actual]) - self.assertFalse(self.app.logger.get_lines_for_level('error')) - - def test_get_shard_ranges_for_utf8_object_put(self): - ts_iter = make_timestamp_iter() - shard_ranges = [dict(ShardRange( - '.sharded_a/sr%d' % i, next(ts_iter), u'\u1234%d_lower' % i, - u'\u1234%d_upper' % i, object_count=i, bytes_used=1024 * i, - meta_timestamp=next(ts_iter))) - for i in range(3)] - req = Request.blank('/v1/a/c/o', method='PUT') - resp_headers = {'X-Backend-Record-Type': 'shard'} - with mocked_http_conn( - 200, 200, - body_iter=iter([b'', - json.dumps(shard_ranges[1:2]).encode('ascii')]), - headers=resp_headers - ) as fake_conn: - actual, resp = self.ctrl._get_updating_shard_ranges( - req, 'a', 'c', wsgi_to_str('\xe1\x88\xb41_test')) - self.assertEqual(200, resp.status_int) - - # account info - captured = fake_conn.requests - self.assertEqual('HEAD', captured[0]['method']) - self.assertEqual('a', captured[0]['path'][7:]) - # container GET - self.assertEqual('GET', captured[1]['method']) - self.assertEqual('a/c', captured[1]['path'][7:]) - params = sorted(captured[1]['qs'].split('&')) - self.assertEqual( - ['format=json', 'includes=%E1%88%B41_test', 'states=updating'], - params) - self.assertEqual( - 'shard', captured[1]['headers'].get('X-Backend-Record-Type')) - self.assertEqual(shard_ranges[1:2], [dict(pr) for pr in actual]) - self.assertFalse(self.app.logger.get_lines_for_level('error')) - - def _check_get_shard_ranges_bad_data(self, body): + def _check_get_namespaces_bad_data(self, body): req = Request.blank('/v1/a/c/o', method='PUT') # empty response resp_headers = {'X-Backend-Record-Type': 'shard'} with mocked_http_conn(200, 200, body_iter=iter([b'', body]), headers=resp_headers): - actual, resp = self.ctrl._get_updating_shard_ranges( + actual, resp = self.ctrl._get_updating_namespaces( req, 'a', 'c', '1_test') self.assertEqual(200, resp.status_int) self.assertIsNone(actual) lines = self.app.logger.get_lines_for_level('error') return lines - def test_get_shard_ranges_empty_body(self): - error_lines = self._check_get_shard_ranges_bad_data(b'') + def test_get_namespaces_empty_body(self): + error_lines = self._check_get_namespaces_bad_data(b'') self.assertIn('Problem with listing response', error_lines[0]) if six.PY2: self.assertIn('No JSON', error_lines[0]) @@ -7337,36 +7465,36 @@ class TestGetUpdatingShardRanges(BaseObjectControllerMixin, unittest.TestCase): self.assertIn('JSONDecodeError', error_lines[0]) self.assertFalse(error_lines[1:]) - def test_get_shard_ranges_not_a_list(self): + def test_get_namespaces_not_a_list(self): body = json.dumps({}).encode('ascii') - error_lines = self._check_get_shard_ranges_bad_data(body) + error_lines = self._check_get_namespaces_bad_data(body) self.assertIn('Problem with listing response', error_lines[0]) self.assertIn('not a list', error_lines[0]) self.assertFalse(error_lines[1:]) - def test_get_shard_ranges_key_missing(self): + def test_get_namespaces_key_missing(self): body = json.dumps([{}]).encode('ascii') - error_lines = self._check_get_shard_ranges_bad_data(body) - self.assertIn('Failed to get shard ranges', error_lines[0]) + error_lines = self._check_get_namespaces_bad_data(body) + self.assertIn('Failed to get namespaces', error_lines[0]) self.assertIn('KeyError', error_lines[0]) self.assertFalse(error_lines[1:]) - def test_get_shard_ranges_invalid_shard_range(self): - sr = ShardRange('a/c', Timestamp.now()) - bad_sr_data = dict(sr, name='bad_name') - body = json.dumps([bad_sr_data]).encode('ascii') - error_lines = self._check_get_shard_ranges_bad_data(body) - self.assertIn('Failed to get shard ranges', error_lines[0]) + def test_get_namespaces_invalid_shard_range(self): + # lower > upper ! + bad_ns_data = {'name': 'name', 'lower': 'z', 'upper': 'a'} + body = json.dumps([bad_ns_data]).encode('ascii') + error_lines = self._check_get_namespaces_bad_data(body) + self.assertIn('Failed to get namespaces', error_lines[0]) self.assertIn('ValueError', error_lines[0]) self.assertFalse(error_lines[1:]) - def test_get_shard_ranges_missing_record_type(self): + def test_get_namespaces_missing_record_type(self): req = Request.blank('/v1/a/c/o', method='PUT') - sr = ShardRange('a/c', Timestamp.now()) + sr = utils.Namespace('a/c', 'l', 'u') body = json.dumps([dict(sr)]).encode('ascii') with mocked_http_conn( 200, 200, body_iter=iter([b'', body])): - actual, resp = self.ctrl._get_updating_shard_ranges( + actual, resp = self.ctrl._get_updating_namespaces( req, 'a', 'c', '1_test') self.assertEqual(200, resp.status_int) self.assertIsNone(actual) @@ -7376,15 +7504,15 @@ class TestGetUpdatingShardRanges(BaseObjectControllerMixin, unittest.TestCase): self.assertIn('/a/c', error_lines[0]) self.assertFalse(error_lines[1:]) - def test_get_shard_ranges_wrong_record_type(self): + def test_get_namespaces_wrong_record_type(self): req = Request.blank('/v1/a/c/o', method='PUT') - sr = ShardRange('a/c', Timestamp.now()) + sr = utils.Namespace('a/c', 'l', 'u') body = json.dumps([dict(sr)]).encode('ascii') headers = {'X-Backend-Record-Type': 'object'} with mocked_http_conn( 200, 200, body_iter=iter([b'', body]), headers=headers): - actual, resp = self.ctrl._get_updating_shard_ranges( + actual, resp = self.ctrl._get_updating_namespaces( req, 'a', 'c', '1_test') self.assertEqual(200, resp.status_int) self.assertIsNone(actual) @@ -7394,10 +7522,10 @@ class TestGetUpdatingShardRanges(BaseObjectControllerMixin, unittest.TestCase): self.assertIn('/a/c', error_lines[0]) self.assertFalse(error_lines[1:]) - def test_get_shard_ranges_request_failed(self): + def test_get_namespaces_request_failed(self): req = Request.blank('/v1/a/c/o', method='PUT') with mocked_http_conn(200, 404, 404, 404): - actual, resp = self.ctrl._get_updating_shard_ranges( + actual, resp = self.ctrl._get_updating_namespaces( req, 'a', 'c', '1_test') self.assertEqual(404, resp.status_int) self.assertIsNone(actual)