diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py index 01f2189d70..f35bc3fcdc 100644 --- a/swift/proxy/controllers/container.py +++ b/swift/proxy/controllers/container.py @@ -342,10 +342,15 @@ class ContainerController(Controller): # the setter must be called for new params to update the query string. params = req.params params['format'] = 'json' - # x-backend-record-type may be sent via internal client e.g. from - # the sharder or in probe tests - record_type = req.headers.get('X-Backend-Record-Type', '').lower() - if not record_type: + # x-backend-record-type may be sent via internal client e.g. from the + # sharder, or by the proxy itself when making a recursive request, or + # in probe tests. If the header is present then the only values that + # the proxy respects are 'object' or 'shard'. However, the proxy may + # use the value 'auto' when making requests to container server. + orig_record_type = req.headers.get('X-Backend-Record-Type', '').lower() + if orig_record_type in ('object', 'shard'): + record_type = orig_record_type + else: record_type = 'auto' req.headers['X-Backend-Record-Type'] = 'auto' params['states'] = 'listing' @@ -391,6 +396,9 @@ class ContainerController(Controller): resp_record_type.lower() == 'shard')): resp = self._get_from_shards(req, resp) + if orig_record_type not in ('object', 'shard'): + resp.headers.pop('X-Backend-Record-Type', None) + if not config_true_value( resp.headers.get('X-Backend-Cached-Results')): # Cache container metadata. We just made a request to a storage diff --git a/test/probe/test_sharder.py b/test/probe/test_sharder.py index c1d4ce2e19..ad0022ef33 100644 --- a/test/probe/test_sharder.py +++ b/test/probe/test_sharder.py @@ -24,11 +24,12 @@ import six from six.moves.urllib.parse import quote from swift.common import direct_client, utils +from swift.common.header_key_dict import HeaderKeyDict from swift.common.internal_client import UnexpectedResponse from swift.common.manager import Manager from swift.common.memcached import MemcacheRing from swift.common.utils import ShardRange, parse_db_filename, get_db_files, \ - quorum_size, config_true_value, Timestamp, md5 + quorum_size, config_true_value, Timestamp, md5, Namespace from swift.container.backend import ContainerBroker, UNSHARDED, SHARDING, \ SHARDED from swift.container.sharder import CleavingContext, ContainerSharder @@ -173,16 +174,35 @@ class BaseTestContainerSharding(ReplProbeTest): else: conn.delete_object(self.container_name, obj) - def get_container_shard_ranges(self, account=None, container=None, - include_deleted=False): + def get_container_listing(self, account=None, container=None, + headers=None, params=None): account = account if account else self.account container = container if container else self.container_to_shard path = self.internal_client.make_path(account, container) - headers = {'X-Backend-Record-Type': 'shard'} - if include_deleted: - headers['X-Backend-Include-Deleted'] = 'true' - resp = self.internal_client.make_request( - 'GET', path + '?format=json', headers, [200]) + headers = headers or {} + return self.internal_client.make_request( + 'GET', path + '?format=json', headers, [200], params=params) + + def get_container_objects(self, account=None, container=None, + headers=None, params=None): + headers = HeaderKeyDict(headers) if headers else {} + resp = self.get_container_listing(account, container, headers, + params=params) + req_record_type = headers.get('X-Backend-Record-Type') + resp_record_type = resp.headers.get('X-Backend-Record-Type') + if req_record_type and req_record_type.lower() == 'object': + self.assertEqual('object', resp_record_type) + else: + self.assertIsNone(resp_record_type) + return json.loads(resp.body) + + def get_container_shard_ranges(self, account=None, container=None, + headers=None, params=None): + headers = dict(headers) if headers else {} + headers.update({'X-Backend-Record-Type': 'shard'}) + resp = self.get_container_listing(account, container, headers, + params=params) + self.assertEqual('shard', resp.headers.get('X-Backend-Record-Type')) return [ShardRange.from_dict(sr) for sr in json.loads(resp.body)] def direct_get_container_shard_ranges(self, account=None, container=None, @@ -386,6 +406,10 @@ class BaseTestContainerSharding(ReplProbeTest): expected_state, headers['X-Backend-Sharding-State']) return [ShardRange.from_dict(sr) for sr in shard_ranges] + def assert_container_states(self, expected_state, num_shard_ranges): + for node in self.brain.nodes: + self.assert_container_state(node, expected_state, num_shard_ranges) + def assert_subprocess_success(self, cmd_args): try: return subprocess.check_output(cmd_args, stderr=subprocess.STDOUT) @@ -434,6 +458,15 @@ class BaseTestContainerSharding(ReplProbeTest): return self.run_custom_daemon(ContainerSharder, 'container-sharder', conf_index, custom_conf, **kwargs) + def sharders_once_non_auto(self, **kwargs): + # inhibit auto_sharding regardless of the config setting + additional_args = kwargs.get('additional_args', []) + if not isinstance(additional_args, list): + additional_args = [additional_args] + additional_args.append('--no-auto-shard') + kwargs['additional_args'] = additional_args + self.sharders.once(**kwargs) + class BaseAutoContainerSharding(BaseTestContainerSharding): @@ -553,8 +586,7 @@ class TestContainerShardingNonUTF8(BaseAutoContainerSharding): number=n, additional_args='--partitions=%s' % self.brain.part) # sanity check shard range states - for node in self.brain.nodes: - self.assert_container_state(node, 'sharding', 4) + self.assert_container_states('sharding', 4) shard_ranges = self.get_container_shard_ranges() self.assertLengthEqual(shard_ranges, 4) self.assert_shard_range_state(ShardRange.CLEAVED, shard_ranges[:2]) @@ -584,8 +616,7 @@ class TestContainerShardingNonUTF8(BaseAutoContainerSharding): # run all the sharders again and the last two shard ranges get cleaved self.sharders.once(additional_args='--partitions=%s' % self.brain.part) - for node in self.brain.nodes: - self.assert_container_state(node, 'sharded', 4) + self.assert_container_states('sharded', 4) shard_ranges = self.get_container_shard_ranges() self.assert_shard_range_state(ShardRange.ACTIVE, shard_ranges) @@ -839,8 +870,7 @@ class TestContainerShardingObjectVersioning(BaseAutoContainerSharding): number=n, additional_args='--partitions=%s' % self.brain.part) # sanity check shard range states - for node in self.brain.nodes: - self.assert_container_state(node, 'sharding', 4) + self.assert_container_states('sharding', 4) shard_ranges = self.get_container_shard_ranges() self.assertLengthEqual(shard_ranges, 4) self.assert_shard_range_state(ShardRange.CLEAVED, shard_ranges[:2]) @@ -869,8 +899,7 @@ class TestContainerShardingObjectVersioning(BaseAutoContainerSharding): # run all the sharders again and the last two shard ranges get cleaved self.sharders.once(additional_args='--partitions=%s' % self.brain.part) - for node in self.brain.nodes: - self.assert_container_state(node, 'sharded', 4) + self.assert_container_states('sharded', 4) shard_ranges = self.get_container_shard_ranges() self.assert_shard_range_state(ShardRange.ACTIVE, shard_ranges) @@ -1918,8 +1947,7 @@ class TestContainerSharding(BaseAutoContainerSharding): self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity checks - for node in self.brain.nodes: - self.assert_container_state(node, 'sharded', 2) + self.assert_container_states('sharded', 2) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') @@ -2255,8 +2283,7 @@ class TestContainerSharding(BaseAutoContainerSharding): self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity checks - for node in self.brain.nodes: - self.assert_container_state(node, 'sharded', 2) + self.assert_container_states('sharded', 2) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') @@ -2373,8 +2400,7 @@ class TestContainerSharding(BaseAutoContainerSharding): self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity checks - for node in self.brain.nodes: - self.assert_container_state(node, 'sharded', 2) + self.assert_container_states('sharded', 2) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') @@ -2506,8 +2532,7 @@ class TestContainerSharding(BaseAutoContainerSharding): self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity checks - for node in self.brain.nodes: - self.assert_container_state(node, 'sharded', 2) + self.assert_container_states('sharded', 2) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') @@ -2620,8 +2645,7 @@ class TestContainerSharding(BaseAutoContainerSharding): self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity checks - for node in self.brain.nodes: - self.assert_container_state(node, 'sharded', 2) + self.assert_container_states('sharded', 2) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') @@ -2860,8 +2884,7 @@ class TestContainerSharding(BaseAutoContainerSharding): for node in self.brain.nodes[1:]: self.assert_container_state(node, 'sharding', 3) self.sharders.once() - for node in self.brain.nodes: - self.assert_container_state(node, 'sharded', 3) + self.assert_container_states('sharded', 3) self.assert_container_listing(obj_names) @@ -2900,6 +2923,137 @@ class TestContainerSharding(BaseAutoContainerSharding): self.assertEqual(0, int(metadata.get('x-account-bytes-used'))) +class TestShardedAPI(BaseTestContainerSharding): + def _assert_namespace_equivalence( + self, namespaces_list, other_namespaces_list): + # verify given lists are equivalent when cast to Namespaces + self.assertEqual(len(namespaces_list), len(other_namespaces_list)) + self.assertEqual( + [Namespace(sr.name, sr.lower, sr.upper) + for sr in namespaces_list], + [Namespace(sr.name, sr.lower, sr.upper) + for sr in other_namespaces_list]) + + def test_GET(self): + all_obj_names = self._make_object_names(10) + self.put_objects(all_obj_names) + + # unsharded container + objs = self.get_container_objects() + self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) + + objs = self.get_container_objects( + headers={'X-Backend-Record-Type': 'auto'}) + self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) + + objs = self.get_container_objects( + headers={'X-Backend-Record-Type': 'object'}) + self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) + + objs = self.get_container_objects( + headers={'X-Backend-Record-Type': 'banana'}) + self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) + + shard_ranges = self.get_container_shard_ranges() + self.assertFalse(shard_ranges) + + # Shard the container + client.post_container(self.url, self.admin_token, self.container_name, + headers={'X-Container-Sharding': 'on'}) + self.assert_subprocess_success([ + 'swift-manage-shard-ranges', + self.get_db_file(self.brain.part, self.brain.nodes[0]), + 'find_and_replace', '5', '--enable', '--minimum-shard-size', '5']) + self.replicators.once() + # "Run container-sharder on all nodes to shard the container." + # first pass cleaves 2 shards + self.sharders_once_non_auto( + additional_args='--partitions=%s' % self.brain.part) + # sanity check + self.assert_container_states('sharded', 2) + + orig_shard_ranges = self.get_container_shard_ranges() + self.assertEqual(2, len(orig_shard_ranges)) + + # the container is sharded so *all* shard ranges should satisfy + # updating and listing state aliases + shard_ranges = self.get_container_shard_ranges( + params={'states': 'updating'}) + self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) + + # XXX the states=listing param provokes the proxy to cache the backend + # values and then respond to the client with the cached *namespaces* !! + # shard_ranges = self.get_container_shard_ranges( + # params={'states': 'listing'}) + # self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) + + # XXX ditto... + # shard_ranges = self.get_container_shard_ranges( + # headers={'X-Newest': 'true'}, + # params={'states': 'listing'}) + # self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) + + # this is what the sharder requests... + shard_ranges = self.get_container_shard_ranges( + headers={'X-Newest': 'true'}, + params={'states': 'auditing'}) + own_ns = Namespace('%s/%s' % (self.account, self.container_name), + lower='', upper='') + self._assert_namespace_equivalence(orig_shard_ranges + [own_ns], + shard_ranges) + + shard_ranges = self.get_container_shard_ranges( + params={'includes': all_obj_names[1]}) + self._assert_namespace_equivalence(orig_shard_ranges[:1], shard_ranges) + + shard_ranges = self.get_container_shard_ranges( + # override 'includes' + headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'}, + params={'includes': all_obj_names[1]}) + self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) + + shard_ranges = self.get_container_shard_ranges( + params={'end_marker': all_obj_names[1]}) + self._assert_namespace_equivalence(orig_shard_ranges[:1], shard_ranges) + + shard_ranges = self.get_container_shard_ranges( + # override 'end_marker' + headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'}, + params={'end_marker': all_obj_names[1]}) + self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) + + shard_ranges = self.get_container_shard_ranges( + params={'reverse': 'true'}) + self._assert_namespace_equivalence(list(reversed(orig_shard_ranges)), + shard_ranges) + + shard_ranges = self.get_container_shard_ranges( + # override 'reverse' + headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'}, + params={'reverse': 'true'}) + self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) + + objs = self.get_container_objects() + self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) + + objs = self.get_container_objects( + headers={'X-Newest': 'true'}) + self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) + + objs = self.get_container_objects( + headers={'X-Backend-Record-Type': 'auto'}) + self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) + + objs = self.get_container_objects( + headers={'X-Backend-Record-Type': 'banana'}) + self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) + + # note: explicitly asking for the root object rows, but it has None + objs = self.get_container_objects( + headers={'X-Backend-Record-Type': 'object'}) + self.assertEqual([], objs) + + class TestContainerShardingMoreUTF8(TestContainerSharding): def _make_object_names(self, number): # override default with names that include non-ascii chars @@ -2925,16 +3079,7 @@ class TestContainerShardingMoreUTF8(TestContainerSharding): class TestManagedContainerSharding(BaseTestContainerSharding): - '''Test sharding using swift-manage-shard-ranges''' - - def sharders_once(self, **kwargs): - # inhibit auto_sharding regardless of the config setting - additional_args = kwargs.get('additional_args', []) - if not isinstance(additional_args, list): - additional_args = [additional_args] - additional_args.append('--no-auto-shard') - kwargs['additional_args'] = additional_args - self.sharders.once(**kwargs) + """Test sharding using swift-manage-shard-ranges""" def test_manage_shard_ranges(self): obj_names = self._make_object_names(10) @@ -2948,8 +3093,9 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # sanity check: we don't have nearly enough objects for this to shard # automatically - self.sharders_once(number=self.brain.node_numbers[0], - additional_args='--partitions=%s' % self.brain.part) + self.sharders_once_non_auto( + number=self.brain.node_numbers[0], + additional_args='--partitions=%s' % self.brain.part) self.assert_container_state(self.brain.nodes[0], 'unsharded', 0) self.assert_subprocess_success([ @@ -2962,7 +3108,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding): self.replicators.once() # "Run container-sharder on all nodes to shard the container." # first pass cleaves 2 shards - self.sharders_once(additional_args='--partitions=%s' % self.brain.part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % self.brain.part) self.assert_container_state(self.brain.nodes[0], 'sharding', 3) self.assert_container_state(self.brain.nodes[1], 'sharding', 3) shard_ranges = self.assert_container_state( @@ -2972,7 +3119,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # make the un-cleaved shard update the root container... self.assertEqual([3, 3, 4], [sr.object_count for sr in shard_ranges]) shard_part, nodes = self.get_part_and_node_numbers(shard_ranges[2]) - self.sharders_once(additional_args='--partitions=%s' % shard_part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % shard_part) shard_ranges = self.assert_container_state( self.brain.nodes[2], 'sharding', 3) # ...it does not report zero-stats despite being empty, because it has @@ -2980,7 +3128,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding): self.assertEqual([3, 3, 4], [sr.object_count for sr in shard_ranges]) # second pass cleaves final shard - self.sharders_once(additional_args='--partitions=%s' % self.brain.part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % self.brain.part) # Everybody's settled self.assert_container_state(self.brain.nodes[0], 'sharded', 3) @@ -3006,11 +3155,11 @@ class TestManagedContainerSharding(BaseTestContainerSharding): self.assert_container_state(self.brain.nodes[0], 'unsharded', 4) self.replicators.once() # run sharders twice to cleave all 4 shard ranges - self.sharders_once(additional_args='--partitions=%s' % self.brain.part) - self.sharders_once(additional_args='--partitions=%s' % self.brain.part) - self.assert_container_state(self.brain.nodes[0], 'sharded', 4) - self.assert_container_state(self.brain.nodes[1], 'sharded', 4) - self.assert_container_state(self.brain.nodes[2], 'sharded', 4) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % self.brain.part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % self.brain.part) + self.assert_container_states('sharded', 4) self.assert_container_listing(obj_names) # now compact some ranges; use --max-shrinking to allow 2 shrinking @@ -3025,7 +3174,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding): self.assertEqual([ShardRange.SHRINKING] * 2 + [ShardRange.ACTIVE] * 2, [sr.state for sr in shard_ranges]) self.replicators.once() - self.sharders_once() + self.sharders_once_non_auto() # check there's now just 2 remaining shard ranges shard_ranges = self.assert_container_state( self.brain.nodes[0], 'sharded', 2) @@ -3051,7 +3200,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding): self.assertEqual([ShardRange.SHRINKING] * 2, [sr.state for sr in shard_ranges]) self.replicators.once() - self.sharders_once() + self.sharders_once_non_auto() self.assert_container_state(self.brain.nodes[0], 'collapsed', 0) self.assert_container_listing(obj_names, req_hdrs={'X-Newest': 'True'}) @@ -3100,8 +3249,9 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # proceeds to cleave shard 0.0, but after 0.0 cleaving stalls because # next in iteration is shard range 1.0 in FOUND state from the other # replica that it cannot yet cleave. - self.sharders_once(number=self.brain.node_numbers[0], - additional_args='--partitions=%s' % self.brain.part) + self.sharders_once_non_auto( + number=self.brain.node_numbers[0], + additional_args='--partitions=%s' % self.brain.part) # On first pass the second replica passes audit (it has its own found # ranges and the first replica's created shard ranges but none in the @@ -3109,8 +3259,9 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # with the other replicas. All of the 7 shard ranges on this replica # are now in CREATED state so it proceeds to cleave the first two shard # ranges, 0.1 and 1.0. - self.sharders_once(number=self.brain.node_numbers[1], - additional_args='--partitions=%s' % self.brain.part) + self.sharders_once_non_auto( + number=self.brain.node_numbers[1], + additional_args='--partitions=%s' % self.brain.part) self.replicators.once() # Uh-oh @@ -3119,9 +3270,11 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # There's a race: the third replica may be sharding, may be unsharded # Try it again a few times - self.sharders_once(additional_args='--partitions=%s' % self.brain.part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % self.brain.part) self.replicators.once() - self.sharders_once(additional_args='--partitions=%s' % self.brain.part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % self.brain.part) # It's not really fixing itself... the sharder audit will detect # overlapping ranges which prevents cleaving proceeding; expect the @@ -3182,8 +3335,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # and the third replica may have cleaved no shards. We therefore need # two more passes of the sharder to get to a predictable state where # all replicas have cleaved all three 0.* shards. - self.sharders_once() - self.sharders_once() + self.sharders_once_non_auto() + self.sharders_once_non_auto() # now we expect all replicas to have just the three 1.* shards, with # the 0.* shards all deleted @@ -3230,7 +3383,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # because of the older epoch_0 in its db filename will now start to # shard again with a newer epoch_1 db, and will start to re-cleave the # 3 active shards, albeit with zero objects to cleave. - self.sharders_once() + self.sharders_once_non_auto() for node in (0, 1, 2): with annotate_failure('node %s' % node): broker = self.get_broker(self.brain.part, @@ -3274,7 +3427,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # to exist on the same node as the root because the roots cleaving # process doesn't think that it created the shard db and will therefore # replicate it as per a normal cleave. - self.sharders_once() + self.sharders_once_non_auto() for node in (0, 1, 2): with annotate_failure('node %s' % node): broker = self.get_broker(self.brain.part, @@ -3311,11 +3464,13 @@ class TestManagedContainerSharding(BaseTestContainerSharding): 'find_and_replace', '4', '--enable']) self.replicators.once() # cleave first two shards - self.sharders_once(additional_args='--partitions=%s' % self.brain.part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % self.brain.part) # cleave third shard - self.sharders_once(additional_args='--partitions=%s' % self.brain.part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % self.brain.part) # ensure all shards learn their ACTIVE state from root - self.sharders_once() + self.sharders_once_non_auto() for node in (0, 1, 2): with annotate_failure('node %d' % node): shard_ranges = self.assert_container_state( @@ -3368,9 +3523,12 @@ class TestManagedContainerSharding(BaseTestContainerSharding): self.replicators.once() # try hard to shard the shard... - self.sharders_once(additional_args='--partitions=%s' % shard_1_part) - self.sharders_once(additional_args='--partitions=%s' % shard_1_part) - self.sharders_once(additional_args='--partitions=%s' % shard_1_part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % shard_1_part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % shard_1_part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % shard_1_part) # sharding hasn't completed and there's overlaps in the shard and root: # the sub-shards will have been cleaved in the order listed above, but # sub-shards (10 -12) and one of (12 - 14) will be overlooked because @@ -3403,8 +3561,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding): ['swift-manage-shard-ranges', db_file, 'repair', '--yes', '--min-shard-age', '0']) self.replicators.once() - self.sharders_once() - self.sharders_once() + self.sharders_once_non_auto() + self.sharders_once_non_auto() # check root now has just 5 shard ranges root_shard_ranges = self.get_container_shard_ranges() @@ -3416,7 +3574,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # because the sub-shards report their state to the root; we cannot make # assertions about shrunk states in shard_1's shard range table) root_shard_ranges = self.get_container_shard_ranges( - include_deleted=True) + headers={'X-Backend-Include-Deleted': 'true'}) self.assertEqual(10, len(root_shard_ranges), root_shard_ranges) shrunk_shard_ranges = [sr for sr in root_shard_ranges if sr.state == ShardRange.SHRUNK] @@ -3452,13 +3610,12 @@ class TestManagedContainerSharding(BaseTestContainerSharding): 'find_and_replace', '2', '--enable']) self.container_replicators.once( additional_args='--partitions=%s' % self.brain.part) - for node in self.brain.nodes: - self.assert_container_state(node, 'unsharded', 2) - self.sharders_once(additional_args='--partitions=%s' % self.brain.part) + self.assert_container_states('unsharded', 2) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % self.brain.part) # get shards to update state from parent... - self.sharders_once() - for node in self.brain.nodes: - self.assert_container_state(node, 'sharded', 2) + self.sharders_once_non_auto() + self.assert_container_states('sharded', 2) # sanity check, all is well msg = self.assert_subprocess_success([ @@ -3483,9 +3640,10 @@ class TestManagedContainerSharding(BaseTestContainerSharding): self.assert_container_state( node, 'unsharded', 2, account=shard_ranges[0].account, container=shard_ranges[0].container, part=shard_part) - self.sharders_once(additional_args='--partitions=%s' % shard_part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % shard_part) # get shards to update state from parent... - self.sharders_once() + self.sharders_once_non_auto() for node in exclude_nodes(shard_nodes, self.brain.nodes[0]): self.assert_container_state( node, 'sharded', 2, account=shard_ranges[0].account, @@ -3528,7 +3686,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # TODO: how can we work around this? self.assertNotEqual(sub_shard_part, shard_part, 'You were unlucky, try again') - self.sharders_once(additional_args='--partitions=%s' % sub_shard_part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % sub_shard_part) # now root node 0 has the original shards plus one of the sub-shards # but all are active :( @@ -3558,8 +3717,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding): for sr in root_brokers[0].get_shard_ranges(include_deleted=True)]) # the transient overlap is 'fixed' in subsequent sharder cycles... - self.sharders_once() - self.sharders_once() + self.sharders_once_non_auto() + self.sharders_once_non_auto() self.container_replicators.once() for broker in root_brokers: @@ -3593,13 +3752,12 @@ class TestManagedContainerSharding(BaseTestContainerSharding): 'find_and_replace', '2', '--enable']) self.container_replicators.once( additional_args='--partitions=%s' % self.brain.part) - for node in self.brain.nodes: - self.assert_container_state(node, 'unsharded', 4) - self.sharders_once(additional_args='--partitions=%s' % self.brain.part) + self.assert_container_states('unsharded', 4) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % self.brain.part) # get shards to update state from parent... - self.sharders_once() - for node in self.brain.nodes: - self.assert_container_state(node, 'sharded', 4) + self.sharders_once_non_auto() + self.assert_container_states('sharded', 4) # sanity check, all is well msg = self.assert_subprocess_success([ @@ -3635,8 +3793,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding): '--yes']) self.assertIn(b'Repairs necessary to fill gaps.', msg) - self.sharders_once() - self.sharders_once() + self.sharders_once_non_auto() + self.sharders_once_non_auto() self.container_replicators.once() # yay! we fixed the gap (without creating an overlap) @@ -3662,7 +3820,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding): new_objs = [obj_names[4] + 'a'] self.put_objects(new_objs) # get root stats up to date - self.sharders_once() + self.sharders_once_non_auto() # new object is in listing but old objects in the gap have been lost - # don't delete shard ranges! self.assert_container_listing(obj_names[:4] + new_objs + obj_names[6:]) @@ -3702,7 +3860,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # Run container-sharder to shard the 2 primary replicas that did # receive the object PUTs for num in self.brain.primary_numbers: - self.sharders_once( + self.sharders_once_non_auto( number=num, additional_args='--partitions=%s' % self.brain.part) @@ -3711,7 +3869,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding): self.delete_objects(obj_names) # deal with DELETE's being misplaced in root db's... for num in self.brain.primary_numbers: - self.sharders_once( + self.sharders_once_non_auto( number=num, additional_args='--partitions=%s' % self.brain.part) @@ -3738,7 +3896,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # now shard the final DB for num in self.brain.handoff_numbers: - self.sharders_once( + self.sharders_once_non_auto( number=num, additional_args='--partitions=%s' % self.brain.part) @@ -3806,7 +3964,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # Run container-sharder to shard the 2 primary replicas that did # receive the object PUTs for num in self.brain.primary_numbers: - self.sharders_once( + self.sharders_once_non_auto( number=num, additional_args='--partitions=%s' % self.brain.part) @@ -3828,7 +3986,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # now shard the final DB for num in self.brain.handoff_numbers: - self.sharders_once( + self.sharders_once_non_auto( number=num, additional_args='--partitions=%s' % self.brain.part) shard_ranges = self.assert_container_state( @@ -3876,14 +4034,13 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # Run container-replicator to replicate them to other nodes. self.container_replicators.once( additional_args='--partitions=%s' % self.brain.part) - for node in self.brain.nodes: - self.assert_container_state(node, 'unsharded', 2) + self.assert_container_states('unsharded', 2) # Run container-sharder on all nodes to shard the container. - self.sharders_once(additional_args='--partitions=%s' % self.brain.part) + self.sharders_once_non_auto( + additional_args='--partitions=%s' % self.brain.part) # get shards to update state from parent... - self.sharders_once() - for node in self.brain.nodes: - self.assert_container_state(node, 'sharded', 2) + self.sharders_once_non_auto() + self.assert_container_states('sharded', 2) # shard first child shard into 2 grand-child-shards. c_shard_ranges = self.get_container_shard_ranges() @@ -3912,14 +4069,14 @@ class TestManagedContainerSharding(BaseTestContainerSharding): c_shard_dir = os.path.dirname(c_shard_brokers[2].db_file) c_shard_tmp_dir = c_shard_dir + ".tmp" os.rename(c_shard_dir, c_shard_tmp_dir) - self.sharders_once(additional_args='--partitions=%s' % - child_shard_part) + self.sharders_once_non_auto(additional_args='--partitions=%s' % + child_shard_part) for node in c_shard_nodes[:2]: self.assert_container_state( node, 'sharded', 2, account=c_shard_ranges[0].account, container=c_shard_ranges[0].container, part=child_shard_part) # get updates done... - self.sharders_once() + self.sharders_once_non_auto() # shard first grand-child shard into 2 grand-grand-child-shards. gc_shard_ranges = self.get_container_shard_ranges( @@ -3936,12 +4093,12 @@ class TestManagedContainerSharding(BaseTestContainerSharding): gc_shard_ranges[0].account, gc_shard_ranges[0].container) self.container_replicators.once( additional_args='--partitions=%s' % grandchild_shard_part) - self.sharders_once(additional_args='--partitions=%s' % - grandchild_shard_part) + self.sharders_once_non_auto(additional_args='--partitions=%s' % + grandchild_shard_part) # get shards to update state from parent... - self.sharders_once() - self.sharders_once() + self.sharders_once_non_auto() + self.sharders_once_non_auto() self.container_replicators.once( additional_args='--partitions=%s' % child_shard_part) @@ -3966,13 +4123,13 @@ class TestManagedContainerSharding(BaseTestContainerSharding): # now, finally, run the sharder on the child that is still waiting to # shard. It will get 2 great-grandchild ranges from root to replace # deleted grandchild. - self.sharders_once( + self.sharders_once_non_auto( additional_args=['--partitions=%s' % child_shard_part, '--devices=%s' % c_shard_nodes[2]['device']]) # batch size is 2 but this replicas has 3 shard ranges so we need two # runs of the sharder - self.sharders_once( + self.sharders_once_non_auto( additional_args=['--partitions=%s' % child_shard_part, '--devices=%s' % c_shard_nodes[2]['device']]) diff --git a/test/unit/container/test_sharder.py b/test/unit/container/test_sharder.py index 3e9da8762b..8029f06e93 100644 --- a/test/unit/container/test_sharder.py +++ b/test/unit/container/test_sharder.py @@ -14,7 +14,7 @@ # limitations under the License. import json import random -from argparse import Namespace +import argparse import eventlet import os @@ -47,7 +47,7 @@ from swift.container.sharder import ContainerSharder, sharding_enabled, \ update_own_shard_range_stats from swift.common.utils import ShardRange, Timestamp, hash_path, \ encode_timestamps, parse_db_filename, quorum_size, Everything, md5, \ - ShardName + ShardName, Namespace from test import annotate_failure from test.debug_logger import debug_logger @@ -1261,6 +1261,10 @@ class TestSharder(BaseTestSharder): do_test('') do_test(json.dumps({})) do_test(json.dumps([{'account': 'a', 'container': 'c'}])) + do_test(json.dumps([dict(Namespace('a/c', 'l', 'u'))])) + sr_dict = dict(ShardRange('a/c', next(self.ts_iter), 'l', 'u')) + sr_dict.pop('object_count') + do_test(json.dumps([sr_dict])) def test_fetch_shard_ranges_ok(self): def do_test(mock_resp_body, params): @@ -9696,11 +9700,11 @@ class TestContainerSharderConf(unittest.TestCase): # given namespace def assert_bad(conf): with self.assertRaises(ValueError): - ContainerSharderConf.validate_conf(Namespace(**conf)) + ContainerSharderConf.validate_conf(argparse.Namespace(**conf)) def assert_ok(conf): try: - ContainerSharderConf.validate_conf(Namespace(**conf)) + ContainerSharderConf.validate_conf(argparse.Namespace(**conf)) except ValueError as err: self.fail('Unexpected ValueError: %s' % err) diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py index 377570256b..60e5ba817a 100644 --- a/test/unit/proxy/controllers/test_container.py +++ b/test/unit/proxy/controllers/test_container.py @@ -486,7 +486,7 @@ class TestContainerController(TestRingBase): def _check_GET_shard_listing(self, mock_responses, expected_objects, expected_requests, query_string='', reverse=False, expected_status=200, - memcache=False): + memcache=False, req_headers=None): # mock_responses is a list of tuples (status, json body, headers) # expected objects is a list of dicts # expected_requests is a list of tuples (path, hdrs dict, params dict) @@ -506,6 +506,8 @@ class TestContainerController(TestRingBase): for resp in mock_responses]) exp_headers = [resp[2] for resp in mock_responses] request = Request.blank(container_path) + if req_headers: + request.headers.update(req_headers) if memcache: # memcache exists, which causes backend to ignore constraints and # reverse params for shard range GETs @@ -566,6 +568,7 @@ class TestContainerController(TestRingBase): int(resp.headers['X-Container-Object-Count'])) self.assertEqual(exp_sharding_state, resp.headers['X-Backend-Sharding-State']) + self.assertNotIn('X-Backend-Record-Type', resp.headers) for k, v in root_resp_hdrs.items(): if k.lower().startswith('x-container-meta'): self.assertEqual(v, resp.headers[k]) @@ -662,6 +665,18 @@ class TestContainerController(TestRingBase): self.check_response(resp, root_resp_hdrs, expected_objects=expected_objects) + resp = self._check_GET_shard_listing( + mock_responses, expected_objects, expected_requests, + req_headers={'X-Backend-Record-Type': 'auto'}) + self.check_response(resp, root_resp_hdrs, + expected_objects=expected_objects) + + resp = self._check_GET_shard_listing( + mock_responses, expected_objects, expected_requests, + req_headers={'X-Backend-Record-Type': 'banana'}) + self.check_response(resp, root_resp_hdrs, + expected_objects=expected_objects) + # GET all objects - sharding, final shard range points back to root root_range = ShardRange('a/c', Timestamp.now(), 'pie', '') mock_responses = [ @@ -1049,6 +1064,18 @@ class TestContainerController(TestRingBase): self.check_response(resp, root_resp_hdrs, expected_objects=expected_objects) + resp = self._check_GET_shard_listing( + mock_responses, expected_objects, expected_requests, memcache=True, + req_headers={'X-Backend-Record-Type': 'auto'}) + self.check_response(resp, root_resp_hdrs, + expected_objects=expected_objects) + + resp = self._check_GET_shard_listing( + mock_responses, expected_objects, expected_requests, memcache=True, + req_headers={'X-Backend-Record-Type': 'banana'}) + self.check_response(resp, root_resp_hdrs, + expected_objects=expected_objects) + # GET all objects - sharding, final shard range points back to root root_range = ShardRange('a/c', Timestamp.now(), 'pie', '') mock_responses = [ @@ -2730,9 +2757,20 @@ class TestContainerController(TestRingBase): 'X-Container-Bytes-Used': '12', 'X-Backend-Storage-Policy-Index': '0'} - def _do_test_caching(self, record_type, exp_recheck_listing): + def _do_test_caching(self, record_type, exp_recheck_listing, + exp_record_type): # this test gets shard ranges into cache and then reads from cache sharding_state = 'sharded' + exp_resp_headers = { + 'X-Backend-Recheck-Container-Existence': '60', + 'X-Backend-Sharding-State': sharding_state} + if exp_record_type: + exp_resp_headers['X-Backend-Record-Type'] = exp_record_type + exp_cache_resp_headers = { + 'X-Backend-Cached-Results': 'true', + 'X-Backend-Sharding-State': sharding_state} + if exp_record_type: + exp_cache_resp_headers['X-Backend-Record-Type'] = exp_record_type self.memcache.delete_all() # container is sharded but proxy does not have that state cached; # expect a backend request and expect shard ranges to be cached @@ -2749,10 +2787,7 @@ class TestContainerController(TestRingBase): req, backend_req, extra_hdrs={'X-Backend-Record-Type': record_type, 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) - self._check_response(resp, self.ns_dicts, { - 'X-Backend-Recheck-Container-Existence': '60', - 'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': sharding_state}) + self._check_response(resp, self.ns_dicts, exp_resp_headers) cache_key = 'shard-listing-v2/a/c' self.assertEqual( @@ -2789,10 +2824,7 @@ class TestContainerController(TestRingBase): req, backend_req, extra_hdrs={'X-Backend-Record-Type': record_type, 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) - self._check_response(resp, self.ns_dicts, { - 'X-Backend-Recheck-Container-Existence': '60', - 'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': sharding_state}) + self._check_response(resp, self.ns_dicts, exp_resp_headers) self.assertEqual( [mock.call.get('container/a/c'), mock.call.get(cache_key, raise_on_error=True), @@ -2819,10 +2851,7 @@ class TestContainerController(TestRingBase): req = self._build_request({'X-Backend-Record-Type': record_type}, {'states': 'listing'}, {}) resp = req.get_response(self.app) - self._check_response(resp, self.ns_dicts, { - 'X-Backend-Cached-Results': 'true', - 'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': sharding_state}) + self._check_response(resp, self.ns_dicts, exp_cache_resp_headers) self.assertEqual( [mock.call.get('container/a/c'), mock.call.get(cache_key, raise_on_error=True)], @@ -2853,10 +2882,7 @@ class TestContainerController(TestRingBase): req, backend_req, extra_hdrs={'X-Backend-Record-Type': record_type, 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) - self._check_response(resp, self.ns_dicts, { - 'X-Backend-Recheck-Container-Existence': '60', - 'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': sharding_state}) + self._check_response(resp, self.ns_dicts, exp_resp_headers) self.assertEqual( [mock.call.get('container/a/c'), mock.call.set(cache_key, self.ns_bound_list.bounds, @@ -2882,10 +2908,7 @@ class TestContainerController(TestRingBase): {'states': 'listing'}, {}) with mock.patch('random.random', return_value=0.11): resp = req.get_response(self.app) - self._check_response(resp, self.ns_dicts, { - 'X-Backend-Cached-Results': 'true', - 'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': sharding_state}) + self._check_response(resp, self.ns_dicts, exp_cache_resp_headers) self.assertEqual( [mock.call.get('container/a/c'), mock.call.get(cache_key, raise_on_error=True)], @@ -2909,10 +2932,7 @@ class TestContainerController(TestRingBase): infocache=req.environ['swift.infocache']) with mock.patch('random.random', return_value=0.11): resp = req.get_response(self.app) - self._check_response(resp, self.ns_dicts, { - 'X-Backend-Cached-Results': 'true', - 'X-Backend-Record-Type': 'shard', - 'X-Backend-Sharding-State': sharding_state}) + self._check_response(resp, self.ns_dicts, exp_cache_resp_headers) self.assertEqual([], self.memcache.calls) self.assertIn('swift.infocache', req.environ) self.assertIn(cache_key, req.environ['swift.infocache']) @@ -3025,10 +3045,10 @@ class TestContainerController(TestRingBase): def test_GET_shard_ranges(self): self._setup_shard_range_stubs() # expect shard ranges cache time to be default value of 600 - self._do_test_caching('shard', 600) + self._do_test_caching('shard', 600, 'shard') # expect shard ranges cache time to be configured value of 120 self.app.recheck_listing_shard_ranges = 120 - self._do_test_caching('shard', 120) + self._do_test_caching('shard', 120, 'shard') def mock_get_from_shards(self, req, resp): # for the purposes of these tests we override _get_from_shards so @@ -3042,7 +3062,7 @@ class TestContainerController(TestRingBase): 'ContainerController._get_from_shards', mock_get_from_shards): self.app.recheck_listing_shard_ranges = 600 - self._do_test_caching('auto', 600) + self._do_test_caching('auto', 600, exp_record_type=None) def test_GET_shard_ranges_404_response(self): # pre-warm cache with container info but not shard ranges so that the @@ -3169,6 +3189,8 @@ class TestContainerController(TestRingBase): def mock_get_from_shards(self, req, resp): return resp + # request with record-type=auto does not expect record-type in response + del exp_hdrs['X-Backend-Record-Type'] with mock.patch('swift.proxy.controllers.container.' 'ContainerController._get_from_shards', mock_get_from_shards): @@ -3264,6 +3286,8 @@ class TestContainerController(TestRingBase): def mock_get_from_shards(self, req, resp): return resp + # request with record-type=auto does not expect record-type in response + del exp_hdrs['X-Backend-Record-Type'] with mock.patch('swift.proxy.controllers.container.' 'ContainerController._get_from_shards', mock_get_from_shards):