# Copyright (c) 2017 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import pickle import shutil import subprocess import unittest import uuid from unittest import SkipTest import six from six.moves.urllib.parse import quote from swift.common import direct_client, utils from swift.common.header_key_dict import HeaderKeyDict from swift.common.internal_client import UnexpectedResponse from swift.common.manager import Manager from swift.common.memcached import MemcacheRing from swift.common.utils import ShardRange, parse_db_filename, quorum_size, \ config_true_value, Timestamp, md5, Namespace from swift.container.backend import ContainerBroker, UNSHARDED, SHARDING, \ SHARDED from swift.container.sharder import CleavingContext, ContainerSharder from swift.container.replicator import ContainerReplicator from swiftclient import client, get_auth, ClientException from swift.proxy.controllers.base import get_cache_key from swift.proxy.controllers.obj import num_container_updates from test import annotate_failure from test.debug_logger import debug_logger from test.probe import PROXY_BASE_URL from test.probe.brain import BrainSplitter from test.probe.common import ReplProbeTest, get_server_number, \ wait_for_server_to_hangup, ENABLED_POLICIES, exclude_nodes import mock try: from swiftclient.requests_compat import requests as client_requests except ImportError: # legacy location from swiftclient.client import requests as client_requests MIN_SHARD_CONTAINER_THRESHOLD = 4 MAX_SHARD_CONTAINER_THRESHOLD = 100 class ShardCollector(object): """ Returns map of node to tuples of (headers, shard ranges) returned from node """ def __init__(self): self.ranges = {} def __call__(self, cnode, cpart, account, container): self.ranges[cnode['id']] = direct_client.direct_get_container( cnode, cpart, account, container, headers={'X-Backend-Record-Type': 'shard'}) class BaseTestContainerSharding(ReplProbeTest): DELIM = '-' def _maybe_skip_test(self): try: self.cont_configs = [ utils.readconf(p, 'container-sharder') for p in self.configs['container-sharder'].values()] except ValueError: raise SkipTest('No [container-sharder] section found in ' 'container-server configs') self.max_shard_size = max( int(c.get('shard_container_threshold', '1000000')) for c in self.cont_configs) skip_reasons = [] if not (MIN_SHARD_CONTAINER_THRESHOLD <= self.max_shard_size <= MAX_SHARD_CONTAINER_THRESHOLD): skip_reasons.append( 'shard_container_threshold %d must be between %d and %d' % (self.max_shard_size, MIN_SHARD_CONTAINER_THRESHOLD, MAX_SHARD_CONTAINER_THRESHOLD)) def skip_check(reason_list, option, required): values = {int(c.get(option, required)) for c in self.cont_configs} if values != {required}: reason_list.append('%s must be %s' % (option, required)) skip_check(skip_reasons, 'shard_scanner_batch_size', 10) skip_check(skip_reasons, 'shard_batch_size', 2) if skip_reasons: raise SkipTest(', '.join(skip_reasons)) def _load_rings_and_configs(self): super(BaseTestContainerSharding, self)._load_rings_and_configs() # perform checks for skipping test before starting services self._maybe_skip_test() def _make_object_names(self, number, start=0): return ['obj%s%04d' % (self.DELIM, x) for x in range(start, start + number)] def _setup_container_name(self): # Container where we're PUTting objects self.container_name = 'container%s%s' % (self.DELIM, uuid.uuid4()) def setUp(self): client.logger.setLevel(client.logging.WARNING) client_requests.logging.getLogger().setLevel( client_requests.logging.WARNING) super(BaseTestContainerSharding, self).setUp() _, self.admin_token = get_auth( PROXY_BASE_URL + '/auth/v1.0', 'admin:admin', 'admin') self._setup_container_name() self.init_brain(self.container_name) self.sharders = Manager(['container-sharder']) self.internal_client = self.make_internal_client() self.logger = debug_logger('sharder-test') self.memcache = MemcacheRing(['127.0.0.1:11211'], logger=self.logger) self.container_replicators = Manager(['container-replicator']) def init_brain(self, container_name): self.container_to_shard = container_name self.brain = BrainSplitter( self.url, self.token, self.container_to_shard, None, 'container') self.brain.put_container(policy_index=int(self.policy)) def stop_container_servers(self, node_numbers=None): if node_numbers: ipports = [] server2ipport = {v: k for k, v in self.ipport2server.items()} for number in self.brain.node_numbers[node_numbers]: self.brain.servers.stop(number=number) server = 'container%d' % number ipports.append(server2ipport[server]) else: ipports = [k for k, v in self.ipport2server.items() if v.startswith('container')] self.brain.servers.stop() for ipport in ipports: wait_for_server_to_hangup(ipport) def put_objects(self, obj_names, contents=None): conn = client.Connection(preauthurl=self.url, preauthtoken=self.token) results = [] for obj in obj_names: rdict = {} conn.put_object(self.container_name, obj, contents=contents, response_dict=rdict) results.append((obj, rdict['headers'].get('x-object-version-id'))) return results def delete_objects(self, obj_names_and_versions): conn = client.Connection(preauthurl=self.url, preauthtoken=self.token) for obj in obj_names_and_versions: if isinstance(obj, tuple): obj, version = obj conn.delete_object(self.container_name, obj, query_string='version-id=%s' % version) else: conn.delete_object(self.container_name, obj) def get_container_listing(self, account=None, container=None, headers=None, params=None): account = account if account else self.account container = container if container else self.container_to_shard path = self.internal_client.make_path(account, container) headers = headers or {} return self.internal_client.make_request( 'GET', path + '?format=json', headers, [200], params=params) def get_container_objects(self, account=None, container=None, headers=None, params=None): headers = HeaderKeyDict(headers) if headers else {} resp = self.get_container_listing(account, container, headers, params=params) req_record_type = headers.get('X-Backend-Record-Type') resp_record_type = resp.headers.get('X-Backend-Record-Type') if req_record_type and req_record_type.lower() == 'object': self.assertEqual('object', resp_record_type) else: self.assertIsNone(resp_record_type) self.assertNotIn('X-Backend-Record-Shard-Format', resp.headers) return json.loads(resp.body) def get_container_shard_ranges(self, account=None, container=None, headers=None, params=None): headers = dict(headers) if headers else {} headers.update({'X-Backend-Record-Type': 'shard'}) resp = self.get_container_listing(account, container, headers, params=params) self.assertEqual('shard', resp.headers.get('X-Backend-Record-Type')) self.assertEqual('full', resp.headers.get('X-Backend-Record-Shard-Format')) return [ShardRange.from_dict(sr) for sr in json.loads(resp.body)] def get_container_namespaces(self, account=None, container=None, headers=None, params=None): headers = dict(headers) if headers else {} headers.update({'X-Backend-Record-Type': 'shard', 'X-Backend-Record-Shard-Format': 'namespace'}) resp = self.get_container_listing(account, container, headers, params=params) self.assertEqual('shard', resp.headers.get('X-Backend-Record-Type')) self.assertEqual('namespace', resp.headers.get('X-Backend-Record-Shard-Format')) return [Namespace(**ns) for ns in json.loads(resp.body)] def direct_get_container_shard_ranges(self, account=None, container=None, expect_failure=False): collector = ShardCollector() self.direct_container_op( collector, account, container, expect_failure) return collector.ranges def get_storage_dir(self, part, node, account=None, container=None): account = account or self.brain.account container = container or self.container_name server_type, config_number = get_server_number( (node['ip'], node['port']), self.ipport2server) assert server_type == 'container' repl_server = '%s-replicator' % server_type conf = utils.readconf(self.configs[repl_server][config_number], section_name=repl_server) datadir = os.path.join(conf['devices'], node['device'], 'containers') container_hash = utils.hash_path(account, container) return (utils.storage_directory(datadir, part, container_hash), container_hash) def get_db_file(self, part, node, account=None, container=None): container_dir, container_hash = self.get_storage_dir( part, node, account=account, container=container) for f in os.listdir(container_dir): path = os.path.join(container_dir, f) if path.endswith('.db'): return path def get_broker(self, part, node, account=None, container=None): return ContainerBroker( self.get_db_file(part, node, account, container)) def get_shard_broker(self, shard_range, node_index=0): shard_part, shard_nodes = self.brain.ring.get_nodes( shard_range.account, shard_range.container) return self.get_broker( shard_part, shard_nodes[node_index], shard_range.account, shard_range.container) def categorize_container_dir_content(self, account=None, container=None, more_nodes=False): account = account or self.brain.account container = container or self.container_name part, nodes = self.brain.ring.get_nodes(account, container) if more_nodes: nodes.extend(self.brain.ring.get_more_nodes(part)) storage_dirs = [ self.get_storage_dir(part, node, account=account, container=container)[0] for node in nodes] result = { 'shard_dbs': [], 'normal_dbs': [], 'pendings': [], 'locks': [], 'other': [], } for storage_dir in storage_dirs: for f in os.listdir(storage_dir): path = os.path.join(storage_dir, f) if path.endswith('.db'): hash_, epoch, ext = parse_db_filename(path) if epoch: result['shard_dbs'].append(path) else: result['normal_dbs'].append(path) elif path.endswith('.db.pending'): result['pendings'].append(path) elif path.endswith('/.lock'): result['locks'].append(path) else: result['other'].append(path) if result['other']: self.fail('Found unexpected files in storage directory:\n %s' % '\n '.join(result['other'])) return result def assert_dict_contains(self, expected_items, actual_dict): ignored = set(expected_items) ^ set(actual_dict) filtered_actual = {k: actual_dict[k] for k in actual_dict if k not in ignored} self.assertEqual(expected_items, filtered_actual) def assert_shard_ranges_contiguous(self, expected_number, shard_ranges, first_lower='', last_upper=''): if shard_ranges and isinstance(shard_ranges[0], ShardRange): actual_shard_ranges = sorted(shard_ranges) else: actual_shard_ranges = sorted(ShardRange.from_dict(d) for d in shard_ranges) self.assertLengthEqual(actual_shard_ranges, expected_number) if expected_number: with annotate_failure('Ranges %s.' % actual_shard_ranges): self.assertEqual(first_lower, actual_shard_ranges[0].lower_str) for x, y in zip(actual_shard_ranges, actual_shard_ranges[1:]): self.assertEqual(x.upper, y.lower) self.assertEqual(last_upper, actual_shard_ranges[-1].upper_str) def assert_shard_range_equal(self, expected, actual, excludes=None): excludes = excludes or [] expected_dict = dict(expected) actual_dict = dict(actual) for k in excludes: expected_dict.pop(k, None) actual_dict.pop(k, None) self.assertEqual(expected_dict, actual_dict) def assert_shard_range_lists_equal(self, expected, actual, excludes=None): self.assertEqual(len(expected), len(actual)) for expected, actual in zip(expected, actual): self.assert_shard_range_equal(expected, actual, excludes=excludes) def assert_shard_range_state(self, expected_state, shard_ranges): if shard_ranges and not isinstance(shard_ranges[0], ShardRange): shard_ranges = [ShardRange.from_dict(data) for data in shard_ranges] self.assertEqual([expected_state] * len(shard_ranges), [sr.state for sr in shard_ranges]) def assert_total_object_count(self, expected_object_count, shard_ranges): actual = sum(sr['object_count'] for sr in shard_ranges) self.assertEqual(expected_object_count, actual) def assert_container_listing(self, expected_listing, req_hdrs=None): req_hdrs = req_hdrs if req_hdrs else {} headers, actual_listing = client.get_container( self.url, self.token, self.container_name, headers=req_hdrs) self.assertIn('x-container-object-count', headers) expected_obj_count = len(expected_listing) self.assertEqual(expected_listing, [ x['name'].encode('utf-8') if six.PY2 else x['name'] for x in actual_listing]) self.assertEqual(str(expected_obj_count), headers['x-container-object-count']) return headers, actual_listing def assert_container_object_count(self, expected_obj_count): headers = client.head_container( self.url, self.token, self.container_name) self.assertIn('x-container-object-count', headers) self.assertEqual(str(expected_obj_count), headers['x-container-object-count']) def assert_container_post_ok(self, meta_value): key = 'X-Container-Meta-Assert-Post-Works' headers = {key: meta_value} client.post_container( self.url, self.token, self.container_name, headers=headers) resp_headers = client.head_container( self.url, self.token, self.container_name) self.assertEqual(meta_value, resp_headers.get(key.lower())) def assert_container_post_fails(self, meta_value): key = 'X-Container-Meta-Assert-Post-Works' headers = {key: meta_value} with self.assertRaises(ClientException) as cm: client.post_container( self.url, self.token, self.container_name, headers=headers) self.assertEqual(404, cm.exception.http_status) def assert_container_delete_fails(self): with self.assertRaises(ClientException) as cm: client.delete_container(self.url, self.token, self.container_name) self.assertEqual(409, cm.exception.http_status) def assert_container_not_found(self): with self.assertRaises(ClientException) as cm: client.get_container(self.url, self.token, self.container_name) self.assertEqual(404, cm.exception.http_status) # check for headers leaking out while deleted resp_headers = cm.exception.http_response_headers self.assertNotIn('X-Container-Object-Count', resp_headers) self.assertNotIn('X-Container-Bytes-Used', resp_headers) self.assertNotIn('X-Timestamp', resp_headers) self.assertNotIn('X-PUT-Timestamp', resp_headers) def assert_container_has_shard_sysmeta(self): node_headers = self.direct_head_container() for node_id, headers in node_headers.items(): with annotate_failure('%s in %s' % (node_id, node_headers.keys())): for k, v in headers.items(): if k.lower().startswith('x-container-sysmeta-shard'): break else: self.fail('No shard sysmeta found in %s' % headers) def assert_container_state(self, node, expected_state, num_shard_ranges, account=None, container=None, part=None, override_deleted=False): account = account or self.account container = container or self.container_to_shard part = part or self.brain.part headers = {'X-Backend-Record-Type': 'shard'} if override_deleted: headers['x-backend-override-deleted'] = True headers, shard_ranges = direct_client.direct_get_container( node, part, account, container, headers=headers) self.assertEqual(num_shard_ranges, len(shard_ranges)) self.assertIn('X-Backend-Sharding-State', headers) self.assertEqual( expected_state, headers['X-Backend-Sharding-State']) return [ShardRange.from_dict(sr) for sr in shard_ranges] def assert_container_states(self, expected_state, num_shard_ranges): for node in self.brain.nodes: self.assert_container_state(node, expected_state, num_shard_ranges) def assert_subprocess_success(self, cmd_args): try: return subprocess.check_output(cmd_args, stderr=subprocess.STDOUT) except Exception as exc: # why not 'except CalledProcessError'? because in my py3.6 tests # the CalledProcessError wasn't caught by that! despite type(exc) # being a CalledProcessError, isinstance(exc, CalledProcessError) # is False and the type has a different hash - could be # related to https://github.com/eventlet/eventlet/issues/413 try: # assume this is a CalledProcessError self.fail('%s with output:\n%s' % (exc, exc.output)) except AttributeError: raise exc def get_part_and_node_numbers(self, shard_range): """Return the partition and node numbers for a shard range.""" part, nodes = self.brain.ring.get_nodes( shard_range.account, shard_range.container) return part, [n['id'] + 1 for n in nodes] def run_sharders(self, shard_ranges, exclude_partitions=None): """Run the sharder on partitions for given shard ranges.""" if not isinstance(shard_ranges, (list, tuple, set)): shard_ranges = (shard_ranges,) exclude_partitions = exclude_partitions or [] shard_parts = [] for sr in shard_ranges: sr_part = self.get_part_and_node_numbers(sr)[0] if sr_part not in exclude_partitions: shard_parts.append(str(sr_part)) partitions = ','.join(shard_parts) self.sharders.once(additional_args='--partitions=%s' % partitions) def run_sharder_sequentially(self, shard_range=None): """Run sharder node by node on partition for given shard range.""" if shard_range: part, node_numbers = self.get_part_and_node_numbers(shard_range) else: part, node_numbers = self.brain.part, self.brain.node_numbers for node_number in node_numbers: self.sharders.once(number=node_number, additional_args='--partitions=%s' % part) def run_custom_sharder(self, conf_index, custom_conf, **kwargs): return self.run_custom_daemon(ContainerSharder, 'container-sharder', conf_index, custom_conf, **kwargs) def sharders_once_non_auto(self, **kwargs): # inhibit auto_sharding regardless of the config setting additional_args = kwargs.get('additional_args', []) if not isinstance(additional_args, list): additional_args = [additional_args] additional_args.append('--no-auto-shard') kwargs['additional_args'] = additional_args self.sharders.once(**kwargs) class BaseAutoContainerSharding(BaseTestContainerSharding): def _maybe_skip_test(self): super(BaseAutoContainerSharding, self)._maybe_skip_test() auto_shard = all(config_true_value(c.get('auto_shard', False)) for c in self.cont_configs) if not auto_shard: raise SkipTest('auto_shard must be true ' 'in all container_sharder configs') class TestContainerShardingNonUTF8(BaseAutoContainerSharding): def test_sharding_listing(self): # verify parameterised listing of a container during sharding all_obj_names = self._make_object_names(4 * self.max_shard_size) obj_names = all_obj_names[::2] obj_content = 'testing' self.put_objects(obj_names, contents=obj_content) # choose some names approx in middle of each expected shard range markers = [ obj_names[i] for i in range(self.max_shard_size // 4, 2 * self.max_shard_size, self.max_shard_size // 2)] def check_listing(objects, req_hdrs=None, **params): req_hdrs = req_hdrs if req_hdrs else {} qs = '&'.join('%s=%s' % (k, quote(str(v))) for k, v in params.items()) headers, listing = client.get_container( self.url, self.token, self.container_name, query_string=qs, headers=req_hdrs) listing = [x['name'].encode('utf-8') if six.PY2 else x['name'] for x in listing] if params.get('reverse'): marker = params.get('marker', ShardRange.MAX) end_marker = params.get('end_marker', ShardRange.MIN) expected = [o for o in objects if end_marker < o < marker] expected.reverse() else: marker = params.get('marker', ShardRange.MIN) end_marker = params.get('end_marker', ShardRange.MAX) expected = [o for o in objects if marker < o < end_marker] if 'limit' in params: expected = expected[:params['limit']] self.assertEqual(expected, listing) self.assertIn('x-timestamp', headers) self.assertIn('last-modified', headers) self.assertIn('x-trans-id', headers) self.assertEqual('bytes', headers.get('accept-ranges')) self.assertEqual('application/json; charset=utf-8', headers.get('content-type')) def check_listing_fails(exp_status, **params): qs = '&'.join(['%s=%s' % param for param in params.items()]) with self.assertRaises(ClientException) as cm: client.get_container( self.url, self.token, self.container_name, query_string=qs) self.assertEqual(exp_status, cm.exception.http_status) return cm.exception def do_listing_checks(objs, hdrs=None): hdrs = hdrs if hdrs else {} check_listing(objs, hdrs) check_listing(objs, hdrs, marker=markers[0], end_marker=markers[1]) check_listing(objs, hdrs, marker=markers[0], end_marker=markers[2]) check_listing(objs, hdrs, marker=markers[1], end_marker=markers[3]) check_listing(objs, hdrs, marker=markers[1], end_marker=markers[3], limit=self.max_shard_size // 4) check_listing(objs, hdrs, marker=markers[1], end_marker=markers[3], limit=self.max_shard_size // 4) check_listing(objs, hdrs, marker=markers[1], end_marker=markers[2], limit=self.max_shard_size // 2) check_listing(objs, hdrs, marker=markers[1], end_marker=markers[1]) check_listing(objs, hdrs, reverse=True) check_listing(objs, hdrs, reverse=True, end_marker=markers[1]) check_listing(objs, hdrs, reverse=True, marker=markers[3], end_marker=markers[1], limit=self.max_shard_size // 4) check_listing(objs, hdrs, reverse=True, marker=markers[3], end_marker=markers[1], limit=0) check_listing([], hdrs, marker=markers[0], end_marker=markers[0]) check_listing([], hdrs, marker=markers[0], end_marker=markers[1], reverse=True) check_listing(objs, hdrs, prefix='obj') check_listing([], hdrs, prefix='zzz') # delimiter headers, listing = client.get_container( self.url, self.token, self.container_name, query_string='delimiter=' + quote(self.DELIM), headers=hdrs) self.assertEqual([{'subdir': 'obj' + self.DELIM}], listing) headers, listing = client.get_container( self.url, self.token, self.container_name, query_string='delimiter=j' + quote(self.DELIM), headers=hdrs) self.assertEqual([{'subdir': 'obj' + self.DELIM}], listing) limit = self.cluster_info['swift']['container_listing_limit'] exc = check_listing_fails(412, limit=limit + 1) self.assertIn(b'Maximum limit', exc.http_response_content) exc = check_listing_fails(400, delimiter='%ff') self.assertIn(b'not valid UTF-8', exc.http_response_content) # sanity checks do_listing_checks(obj_names) # Shard the container client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # First run the 'leader' in charge of scanning, which finds all shard # ranges and cleaves first two self.sharders.once(number=self.brain.node_numbers[0], additional_args='--partitions=%s' % self.brain.part) # Then run sharder on other nodes which will also cleave first two # shard ranges for n in self.brain.node_numbers[1:]: self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity check shard range states self.assert_container_states('sharding', 4) shard_ranges = self.get_container_shard_ranges() self.assertLengthEqual(shard_ranges, 4) self.assert_shard_range_state(ShardRange.CLEAVED, shard_ranges[:2]) self.assert_shard_range_state(ShardRange.CREATED, shard_ranges[2:]) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() # confirm no sysmeta deleted self.assert_container_post_ok('sharding') do_listing_checks(obj_names) # put some new objects spread through entire namespace; object updates # should be directed to the shard container (both the cleaved and the # created shards) new_obj_names = all_obj_names[1::4] self.put_objects(new_obj_names, obj_content) # new objects that fell into the first two cleaved shard ranges are # reported in listing; new objects in the yet-to-be-cleaved shard # ranges are not yet included in listing because listings prefer the # root over the final two shards that are not yet-cleaved exp_obj_names = [o for o in obj_names + new_obj_names if o <= shard_ranges[1].upper] exp_obj_names += [o for o in obj_names if o > shard_ranges[1].upper] exp_obj_names.sort() do_listing_checks(exp_obj_names) # run all the sharders again and the last two shard ranges get cleaved self.sharders.once(additional_args='--partitions=%s' % self.brain.part) self.assert_container_states('sharded', 4) shard_ranges = self.get_container_shard_ranges() self.assert_shard_range_state(ShardRange.ACTIVE, shard_ranges) # listings are now gathered from all four shard ranges so should have # all the specified objects exp_obj_names = obj_names + new_obj_names exp_obj_names.sort() do_listing_checks(exp_obj_names) # shard ranges may now be cached by proxy so do listings checks again # forcing backend request do_listing_checks(exp_obj_names, hdrs={'X-Newest': 'true'}) # post more metadata to the container and check that it is read back # correctly from backend (using x-newest) and cache test_headers = {'x-container-meta-test': 'testing', 'x-container-read': 'read_acl', 'x-container-write': 'write_acl', 'x-container-sync-key': 'sync_key', # 'x-container-sync-to': 'sync_to', 'x-versions-location': 'versions', 'x-container-meta-access-control-allow-origin': 'aa', 'x-container-meta-access-control-expose-headers': 'bb', 'x-container-meta-access-control-max-age': '123'} client.post_container(self.url, self.admin_token, self.container_name, headers=test_headers) headers, listing = client.get_container( self.url, self.token, self.container_name, headers={'X-Newest': 'true'}) exp_headers = dict(test_headers) exp_headers.update({ 'x-container-object-count': str(len(exp_obj_names)), 'x-container-bytes-used': str(len(exp_obj_names) * len(obj_content)) }) for k, v in exp_headers.items(): self.assertIn(k, headers) self.assertEqual(v, headers[k], dict(headers)) cache_headers, listing = client.get_container( self.url, self.token, self.container_name) for k, v in exp_headers.items(): self.assertIn(k, cache_headers) self.assertEqual(v, cache_headers[k], dict(exp_headers)) # we don't expect any of these headers to be equal... for k in ('x-timestamp', 'last-modified', 'date', 'x-trans-id', 'x-openstack-request-id'): headers.pop(k, None) cache_headers.pop(k, None) self.assertEqual(headers, cache_headers) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') # delete original objects self.delete_objects(obj_names) do_listing_checks(new_obj_names) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') class TestContainerShardingFunkyNames(TestContainerShardingNonUTF8): DELIM = '\n' def _make_object_names(self, number, start=0): return ['obj\n%04d%%Ff' % x for x in range(start, start + number)] def _setup_container_name(self): self.container_name = 'container\n%%Ff\n%s' % uuid.uuid4() class TestContainerShardingUTF8(TestContainerShardingNonUTF8): def _make_object_names(self, number, start=0): # override default with names that include non-ascii chars name_length = self.cluster_info['swift']['max_object_name_length'] obj_names = [] for x in range(start, start + number): name = (u'obj-\u00e4\u00ea\u00ec\u00f2\u00fb\u1234-%04d' % x) name = name.encode('utf8').ljust(name_length, b'o') if not six.PY2: name = name.decode('utf8') obj_names.append(name) return obj_names def _setup_container_name(self): # override default with max length name that includes non-ascii chars super(TestContainerShardingUTF8, self)._setup_container_name() name_length = self.cluster_info['swift']['max_container_name_length'] cont_name = \ self.container_name + u'-\u00e4\u00ea\u00ec\u00f2\u00fb\u1234' self.container_name = cont_name.encode('utf8').ljust(name_length, b'x') if not six.PY2: self.container_name = self.container_name.decode('utf8') class TestContainerShardingObjectVersioning(BaseAutoContainerSharding): def _maybe_skip_test(self): super(TestContainerShardingObjectVersioning, self)._maybe_skip_test() try: vw_config = utils.readconf(self.configs['proxy-server'], 'filter:versioned_writes') except ValueError: raise SkipTest('No [filter:versioned_writes] section found in ' 'proxy-server configs') allow_object_versioning = config_true_value( vw_config.get('allow_object_versioning', False)) if not allow_object_versioning: raise SkipTest('allow_object_versioning must be true ' 'in all versioned_writes configs') def init_brain(self, container_name): client.put_container(self.url, self.token, container_name, headers={ 'X-Storage-Policy': self.policy.name, 'X-Versions-Enabled': 'true', }) self.container_to_shard = '\x00versions\x00' + container_name self.brain = BrainSplitter( self.url, self.token, self.container_to_shard, None, 'container') def test_sharding_listing(self): # verify parameterised listing of a container during sharding all_obj_names = self._make_object_names(3) * self.max_shard_size all_obj_names.extend(self._make_object_names(self.max_shard_size, start=3)) obj_names = all_obj_names[::2] obj_names_and_versions = self.put_objects(obj_names) def sort_key(obj_and_ver): obj, ver = obj_and_ver return obj, ~Timestamp(ver) obj_names_and_versions.sort(key=sort_key) # choose some names approx in middle of each expected shard range markers = [ obj_names_and_versions[i] for i in range(self.max_shard_size // 4, 2 * self.max_shard_size, self.max_shard_size // 2)] def check_listing(objects, **params): params['versions'] = '' qs = '&'.join('%s=%s' % param for param in params.items()) headers, listing = client.get_container( self.url, self.token, self.container_name, query_string=qs) listing = [(x['name'].encode('utf-8') if six.PY2 else x['name'], x['version_id']) for x in listing] if params.get('reverse'): marker = ( params.get('marker', ShardRange.MAX), ~Timestamp(params['version_marker']) if 'version_marker' in params else ~Timestamp('0'), ) end_marker = ( params.get('end_marker', ShardRange.MIN), Timestamp('0'), ) expected = [o for o in objects if end_marker < sort_key(o) < marker] expected.reverse() else: marker = ( params.get('marker', ShardRange.MIN), ~Timestamp(params['version_marker']) if 'version_marker' in params else Timestamp('0'), ) end_marker = ( params.get('end_marker', ShardRange.MAX), ~Timestamp('0'), ) expected = [o for o in objects if marker < sort_key(o) < end_marker] if 'limit' in params: expected = expected[:params['limit']] self.assertEqual(expected, listing) def check_listing_fails(exp_status, **params): params['versions'] = '' qs = '&'.join('%s=%s' % param for param in params.items()) with self.assertRaises(ClientException) as cm: client.get_container( self.url, self.token, self.container_name, query_string=qs) self.assertEqual(exp_status, cm.exception.http_status) return cm.exception def do_listing_checks(objects): check_listing(objects) check_listing(objects, marker=markers[0][0], version_marker=markers[0][1]) check_listing(objects, marker=markers[0][0], version_marker=markers[0][1], limit=self.max_shard_size // 10) check_listing(objects, marker=markers[0][0], version_marker=markers[0][1], limit=self.max_shard_size // 4) check_listing(objects, marker=markers[0][0], version_marker=markers[0][1], limit=self.max_shard_size // 2) check_listing(objects, marker=markers[1][0], version_marker=markers[1][1]) check_listing(objects, marker=markers[1][0], version_marker=markers[1][1], limit=self.max_shard_size // 10) check_listing(objects, marker=markers[2][0], version_marker=markers[2][1], limit=self.max_shard_size // 4) check_listing(objects, marker=markers[2][0], version_marker=markers[2][1], limit=self.max_shard_size // 2) check_listing(objects, reverse=True) check_listing(objects, reverse=True, marker=markers[1][0], version_marker=markers[1][1]) check_listing(objects, prefix='obj') check_listing([], prefix='zzz') # delimiter headers, listing = client.get_container( self.url, self.token, self.container_name, query_string='delimiter=-') self.assertEqual([{'subdir': 'obj-'}], listing) headers, listing = client.get_container( self.url, self.token, self.container_name, query_string='delimiter=j-') self.assertEqual([{'subdir': 'obj-'}], listing) limit = self.cluster_info['swift']['container_listing_limit'] exc = check_listing_fails(412, limit=limit + 1) self.assertIn(b'Maximum limit', exc.http_response_content) exc = check_listing_fails(400, delimiter='%ff') self.assertIn(b'not valid UTF-8', exc.http_response_content) # sanity checks do_listing_checks(obj_names_and_versions) # Shard the container. Use an internal_client so we get an implicit # X-Backend-Allow-Reserved-Names header self.internal_client.set_container_metadata( self.account, self.container_to_shard, { 'X-Container-Sysmeta-Sharding': 'True', }) # First run the 'leader' in charge of scanning, which finds all shard # ranges and cleaves first two self.sharders.once(number=self.brain.node_numbers[0], additional_args='--partitions=%s' % self.brain.part) # Then run sharder on other nodes which will also cleave first two # shard ranges for n in self.brain.node_numbers[1:]: self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity check shard range states self.assert_container_states('sharding', 4) shard_ranges = self.get_container_shard_ranges() self.assertLengthEqual(shard_ranges, 4) self.assert_shard_range_state(ShardRange.CLEAVED, shard_ranges[:2]) self.assert_shard_range_state(ShardRange.CREATED, shard_ranges[2:]) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() # confirm no sysmeta deleted self.assert_container_post_ok('sharding') do_listing_checks(obj_names_and_versions) # put some new objects spread through entire namespace new_obj_names = all_obj_names[1::4] new_obj_names_and_versions = self.put_objects(new_obj_names) # new objects that fell into the first two cleaved shard ranges are # reported in listing, new objects in the yet-to-be-cleaved shard # ranges are not yet included in listing exp_obj_names_and_versions = [ o for o in obj_names_and_versions + new_obj_names_and_versions if '\x00' + o[0] <= shard_ranges[1].upper] exp_obj_names_and_versions += [ o for o in obj_names_and_versions if '\x00' + o[0] > shard_ranges[1].upper] exp_obj_names_and_versions.sort(key=sort_key) do_listing_checks(exp_obj_names_and_versions) # run all the sharders again and the last two shard ranges get cleaved self.sharders.once(additional_args='--partitions=%s' % self.brain.part) self.assert_container_states('sharded', 4) shard_ranges = self.get_container_shard_ranges() self.assert_shard_range_state(ShardRange.ACTIVE, shard_ranges) exp_obj_names_and_versions = \ obj_names_and_versions + new_obj_names_and_versions exp_obj_names_and_versions.sort(key=sort_key) do_listing_checks(exp_obj_names_and_versions) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') # delete original objects self.delete_objects(obj_names_and_versions) new_obj_names_and_versions.sort(key=sort_key) do_listing_checks(new_obj_names_and_versions) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') class TestContainerSharding(BaseAutoContainerSharding): def _test_sharded_listing(self, run_replicators=False): obj_names = self._make_object_names(self.max_shard_size) self.put_objects(obj_names) # Verify that we start out with normal DBs, no shards found = self.categorize_container_dir_content() self.assertLengthEqual(found['normal_dbs'], 3) self.assertLengthEqual(found['shard_dbs'], 0) for db_file in found['normal_dbs']: broker = ContainerBroker(db_file) self.assertIs(True, broker.is_root_container()) self.assertEqual('unsharded', broker.get_db_state()) self.assertLengthEqual(broker.get_shard_ranges(), 0) headers, pre_sharding_listing = client.get_container( self.url, self.token, self.container_name) self.assertEqual(obj_names, [ x['name'].encode('utf-8') if six.PY2 else x['name'] for x in pre_sharding_listing]) # sanity # Shard it client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) pre_sharding_headers = client.head_container( self.url, self.admin_token, self.container_name) self.assertEqual('True', pre_sharding_headers.get('x-container-sharding')) # Only run the one in charge of scanning self.sharders.once(number=self.brain.node_numbers[0], additional_args='--partitions=%s' % self.brain.part) # Verify that we have one sharded db -- though the other normal DBs # received the shard ranges that got defined found = self.categorize_container_dir_content() self.assertLengthEqual(found['shard_dbs'], 1) broker = self.get_broker(self.brain.part, self.brain.nodes[0]) # sanity check - the shard db is on replica 0 self.assertEqual(found['shard_dbs'][0], broker.db_file) self.assertIs(True, broker.is_root_container()) self.assertEqual('sharded', broker.get_db_state()) orig_root_shard_ranges = [dict(sr) for sr in broker.get_shard_ranges()] self.assertLengthEqual(orig_root_shard_ranges, 2) self.assert_total_object_count(len(obj_names), orig_root_shard_ranges) self.assert_shard_ranges_contiguous(2, orig_root_shard_ranges) self.assertEqual([ShardRange.ACTIVE, ShardRange.ACTIVE], [sr['state'] for sr in orig_root_shard_ranges]) # Contexts should still be there, and should be complete contexts = set([ctx.done() for ctx, _ in CleavingContext.load_all(broker)]) self.assertEqual({True}, contexts) self.direct_delete_container(expect_failure=True) self.assertLengthEqual(found['normal_dbs'], 2) for db_file in found['normal_dbs']: broker = ContainerBroker(db_file) self.assertIs(True, broker.is_root_container()) self.assertEqual('unsharded', broker.get_db_state()) shard_ranges = [dict(sr) for sr in broker.get_shard_ranges()] self.assertEqual([ShardRange.CREATED, ShardRange.CREATED], [sr['state'] for sr in shard_ranges]) # the sharded db had shard range meta_timestamps and state updated # during cleaving, so we do not expect those to be equal on other # nodes self.assert_shard_range_lists_equal( orig_root_shard_ranges, shard_ranges, excludes=['meta_timestamp', 'state', 'state_timestamp']) contexts = list(CleavingContext.load_all(broker)) self.assertEqual([], contexts) # length check if run_replicators: Manager(['container-replicator']).once() # replication doesn't change the db file names found = self.categorize_container_dir_content() self.assertLengthEqual(found['shard_dbs'], 1) self.assertLengthEqual(found['normal_dbs'], 2) # Now that everyone has shard ranges, run *everyone* self.sharders.once(additional_args='--partitions=%s' % self.brain.part) # Verify that we only have shard dbs now found = self.categorize_container_dir_content() self.assertLengthEqual(found['shard_dbs'], 3) self.assertLengthEqual(found['normal_dbs'], 0) # Shards stayed the same for db_file in found['shard_dbs']: broker = ContainerBroker(db_file) self.assertIs(True, broker.is_root_container()) self.assertEqual('sharded', broker.get_db_state()) # Well, except for meta_timestamps, since the shards each reported self.assert_shard_range_lists_equal( orig_root_shard_ranges, broker.get_shard_ranges(), excludes=['meta_timestamp', 'state_timestamp']) for orig, updated in zip(orig_root_shard_ranges, broker.get_shard_ranges()): self.assertGreaterEqual(updated.state_timestamp, orig['state_timestamp']) self.assertGreaterEqual(updated.meta_timestamp, orig['meta_timestamp']) # Contexts should still be there, and should be complete contexts = set([ctx.done() for ctx, _ in CleavingContext.load_all(broker)]) self.assertEqual({True}, contexts) # Check that entire listing is available headers, actual_listing = self.assert_container_listing(obj_names) # ... and check some other container properties self.assertEqual(headers['last-modified'], pre_sharding_headers['last-modified']) # It even works in reverse! headers, listing = client.get_container(self.url, self.token, self.container_name, query_string='reverse=on') self.assertEqual(pre_sharding_listing[::-1], listing) # and repeat checks to use shard ranges now cached in proxy headers, actual_listing = self.assert_container_listing(obj_names) self.assertEqual(headers['last-modified'], pre_sharding_headers['last-modified']) headers, listing = client.get_container(self.url, self.token, self.container_name, query_string='reverse=on') self.assertEqual(pre_sharding_listing[::-1], listing) # Now put some new objects into first shard, taking its count to # 3 shard ranges' worth more_obj_names = [ 'beta%03d' % x for x in range(self.max_shard_size)] self.put_objects(more_obj_names) # The listing includes new objects (shard ranges haven't changed, just # their object content, so cached shard ranges are still correct)... headers, listing = self.assert_container_listing( more_obj_names + obj_names) self.assertEqual(pre_sharding_listing, listing[len(more_obj_names):]) # ...but root object count is out of date until the sharders run and # update the root self.assert_container_object_count(len(obj_names)) # run sharders on the shard to get root updated shard_1 = ShardRange.from_dict(orig_root_shard_ranges[0]) self.run_sharders(shard_1) self.assert_container_object_count(len(more_obj_names + obj_names)) # we've added objects enough that we need to shard the first shard # *again* into three new sub-shards, but nothing happens until the root # leader identifies shard candidate... root_shard_ranges = self.direct_get_container_shard_ranges() for node, (hdrs, root_shards) in root_shard_ranges.items(): self.assertLengthEqual(root_shards, 2) with annotate_failure('node %s. ' % node): self.assertEqual( [ShardRange.ACTIVE] * 2, [sr['state'] for sr in root_shards]) # orig shards 0, 1 should be contiguous self.assert_shard_ranges_contiguous(2, root_shards) # Now run the root leader to identify shard candidate...while one of # the shard container servers is down shard_1_part, shard_1_nodes = self.get_part_and_node_numbers(shard_1) self.brain.servers.stop(number=shard_1_nodes[2]) self.sharders.once(number=self.brain.node_numbers[0], additional_args='--partitions=%s' % self.brain.part) # ... so third replica of first shard state is not moved to sharding found_for_shard = self.categorize_container_dir_content( shard_1.account, shard_1.container) self.assertLengthEqual(found_for_shard['normal_dbs'], 3) self.assertEqual( [ShardRange.SHARDING, ShardRange.SHARDING, ShardRange.ACTIVE], [ContainerBroker(db_file).get_own_shard_range().state for db_file in found_for_shard['normal_dbs']]) # ...then run first cycle of first shard sharders in order, leader # first, to get to predictable state where all nodes have cleaved 2 out # of 3 ranges...starting with first two nodes for node_number in shard_1_nodes[:2]: self.sharders.once( number=node_number, additional_args='--partitions=%s' % shard_1_part) # ... first two replicas start sharding to sub-shards found_for_shard = self.categorize_container_dir_content( shard_1.account, shard_1.container) self.assertLengthEqual(found_for_shard['shard_dbs'], 2) for db_file in found_for_shard['shard_dbs'][:2]: broker = ContainerBroker(db_file) with annotate_failure('shard db file %s. ' % db_file): self.assertIs(False, broker.is_root_container()) self.assertEqual('sharding', broker.get_db_state()) self.assertEqual( ShardRange.SHARDING, broker.get_own_shard_range().state) shard_shards = broker.get_shard_ranges() self.assertEqual( [ShardRange.CLEAVED, ShardRange.CLEAVED, ShardRange.CREATED], [sr.state for sr in shard_shards]) self.assert_shard_ranges_contiguous( 3, shard_shards, first_lower=orig_root_shard_ranges[0]['lower'], last_upper=orig_root_shard_ranges[0]['upper']) contexts = list(CleavingContext.load_all(broker)) self.assertEqual(len(contexts), 1) context, _lm = contexts[0] self.assertIs(context.cleaving_done, False) self.assertIs(context.misplaced_done, True) self.assertEqual(context.ranges_done, 2) self.assertEqual(context.ranges_todo, 1) self.assertEqual(context.max_row, self.max_shard_size * 3 // 2) # but third replica still has no idea it should be sharding self.assertLengthEqual(found_for_shard['normal_dbs'], 3) broker = ContainerBroker(found_for_shard['normal_dbs'][2]) self.assertEqual(ShardRange.ACTIVE, broker.get_own_shard_range().state) # ...but once sharder runs on third replica it will learn its state and # fetch its sub-shard ranges durng audit; note that any root replica on # the stopped container server also won't know about the shards being # in sharding state, so leave that server stopped for now so that shard # fetches its state from an up-to-date root replica self.sharders.once( number=shard_1_nodes[2], additional_args='--partitions=%s' % shard_1_part) # third replica is sharding and has sub-shard ranges so can start # cleaving... found_for_shard = self.categorize_container_dir_content( shard_1.account, shard_1.container) self.assertLengthEqual(found_for_shard['shard_dbs'], 3) self.assertLengthEqual(found_for_shard['normal_dbs'], 3) sharding_broker = ContainerBroker(found_for_shard['normal_dbs'][2]) self.assertEqual('sharding', sharding_broker.get_db_state()) self.assertEqual( ShardRange.SHARDING, sharding_broker.get_own_shard_range().state) self.assertEqual(3, len(sharding_broker.get_shard_ranges())) # there may also be a sub-shard replica missing so run replicators on # all nodes to fix that if necessary self.brain.servers.start(number=shard_1_nodes[2]) self.replicators.once() # Now that the replicators have all run, third replica sees cleaving # contexts for the first two (plus its own cleaving context) contexts = list(CleavingContext.load_all(sharding_broker)) self.assertEqual(len(contexts), 3) broker_id = broker.get_info()['id'] self.assertIn(broker_id, [ctx[0].ref for ctx in contexts]) # check original first shard range state and sub-shards - all replicas # should now be in consistent state found_for_shard = self.categorize_container_dir_content( shard_1.account, shard_1.container) self.assertLengthEqual(found_for_shard['shard_dbs'], 3) self.assertLengthEqual(found_for_shard['normal_dbs'], 3) for db_file in found_for_shard['shard_dbs']: broker = ContainerBroker(db_file) with annotate_failure('shard db file %s. ' % db_file): self.assertIs(False, broker.is_root_container()) self.assertEqual('sharding', broker.get_db_state()) self.assertEqual( ShardRange.SHARDING, broker.get_own_shard_range().state) shard_shards = broker.get_shard_ranges() self.assertEqual( [ShardRange.CLEAVED, ShardRange.CLEAVED, ShardRange.CREATED], [sr.state for sr in shard_shards]) self.assert_shard_ranges_contiguous( 3, shard_shards, first_lower=orig_root_shard_ranges[0]['lower'], last_upper=orig_root_shard_ranges[0]['upper']) # check third sub-shard is in created state sub_shard = shard_shards[2] found_for_sub_shard = self.categorize_container_dir_content( sub_shard.account, sub_shard.container) self.assertFalse(found_for_sub_shard['shard_dbs']) self.assertLengthEqual(found_for_sub_shard['normal_dbs'], 3) for db_file in found_for_sub_shard['normal_dbs']: broker = ContainerBroker(db_file) with annotate_failure('sub shard db file %s. ' % db_file): self.assertIs(False, broker.is_root_container()) self.assertEqual('unsharded', broker.get_db_state()) self.assertEqual( ShardRange.CREATED, broker.get_own_shard_range().state) self.assertFalse(broker.get_shard_ranges()) # check root shard ranges root_shard_ranges = self.direct_get_container_shard_ranges() for node, (hdrs, root_shards) in root_shard_ranges.items(): self.assertLengthEqual(root_shards, 5) with annotate_failure('node %s. ' % node): # shard ranges are sorted by upper, state, lower, so expect: # sub-shards, orig shard 0, orig shard 1 self.assertEqual( [ShardRange.CLEAVED, ShardRange.CLEAVED, ShardRange.CREATED, ShardRange.SHARDING, ShardRange.ACTIVE], [sr['state'] for sr in root_shards]) # sub-shards 0, 1, 2, orig shard 1 should be contiguous self.assert_shard_ranges_contiguous( 4, root_shards[:3] + root_shards[4:]) # orig shards 0, 1 should be contiguous self.assert_shard_ranges_contiguous(2, root_shards[3:]) self.assert_container_listing(more_obj_names + obj_names) self.assert_container_object_count(len(more_obj_names + obj_names)) # Before writing, kill the cache self.memcache.delete(get_cache_key( self.account, self.container_name, shard='updating')) # add another object that lands in the first of the new sub-shards self.put_objects(['alpha']) # check that alpha object is in the first new shard shard_listings = self.direct_get_container(shard_shards[0].account, shard_shards[0].container) for node, (hdrs, listing) in shard_listings.items(): with annotate_failure(node): self.assertIn('alpha', [o['name'] for o in listing]) self.assert_container_listing(['alpha'] + more_obj_names + obj_names) # Run sharders again so things settle. self.run_sharders(shard_1) # Also run replicators to settle cleaving contexts self.replicators.once() # check original first shard range shards for db_file in found_for_shard['shard_dbs']: broker = ContainerBroker(db_file) with annotate_failure('shard db file %s. ' % db_file): self.assertIs(False, broker.is_root_container()) self.assertEqual('sharded', broker.get_db_state()) self.assertEqual( [ShardRange.ACTIVE] * 3, [sr.state for sr in broker.get_shard_ranges()]) # Contexts should still be there, and should be complete contexts = set([ctx.done() for ctx, _ in CleavingContext.load_all(broker)]) self.assertEqual({True}, contexts) # check root shard ranges root_shard_ranges = self.direct_get_container_shard_ranges() for node, (hdrs, root_shards) in root_shard_ranges.items(): # old first shard range should have been deleted self.assertLengthEqual(root_shards, 4) with annotate_failure('node %s. ' % node): self.assertEqual( [ShardRange.ACTIVE] * 4, [sr['state'] for sr in root_shards]) self.assert_shard_ranges_contiguous(4, root_shards) headers, final_listing = self.assert_container_listing( ['alpha'] + more_obj_names + obj_names) # check root found = self.categorize_container_dir_content() self.assertLengthEqual(found['shard_dbs'], 3) self.assertLengthEqual(found['normal_dbs'], 0) new_shard_ranges = None for db_file in found['shard_dbs']: broker = ContainerBroker(db_file) self.assertIs(True, broker.is_root_container()) self.assertEqual('sharded', broker.get_db_state()) if new_shard_ranges is None: new_shard_ranges = broker.get_shard_ranges( include_deleted=True) self.assertLengthEqual(new_shard_ranges, 5) # Second half is still there, and unchanged self.assertIn( dict(orig_root_shard_ranges[1], meta_timestamp=None, state_timestamp=None), [dict(sr, meta_timestamp=None, state_timestamp=None) for sr in new_shard_ranges]) # But the first half split in three, then deleted by_name = {sr.name: sr for sr in new_shard_ranges} self.assertIn(orig_root_shard_ranges[0]['name'], by_name) old_shard_range = by_name.pop( orig_root_shard_ranges[0]['name']) self.assertTrue(old_shard_range.deleted) self.assert_shard_ranges_contiguous(4, list(by_name.values())) else: # Everyone's on the same page. Well, except for # meta_timestamps, since the shards each reported other_shard_ranges = broker.get_shard_ranges( include_deleted=True) self.assert_shard_range_lists_equal( new_shard_ranges, other_shard_ranges, excludes=['meta_timestamp', 'state_timestamp']) for orig, updated in zip(orig_root_shard_ranges, other_shard_ranges): self.assertGreaterEqual(updated.meta_timestamp, orig['meta_timestamp']) self.assert_container_delete_fails() for obj in final_listing: client.delete_object( self.url, self.token, self.container_name, obj['name']) # the objects won't be listed anymore self.assert_container_listing([]) # but root container stats will not yet be aware of the deletions self.assert_container_delete_fails() # One server was down while the shard sharded its first two sub-shards, # so there may be undeleted handoff db(s) for sub-shard(s) that were # not fully replicated; run replicators now to clean up so they no # longer report bogus stats to root. self.replicators.once() # Run sharder so that shard containers update the root. Do not run # sharder on root container because that triggers shrinks which can # cause root object count to temporarily be non-zero and prevent the # final delete. self.run_sharders(self.get_container_shard_ranges()) # then root is empty and can be deleted self.assert_container_listing([]) self.assert_container_object_count(0) client.delete_container(self.url, self.token, self.container_name) def test_sharded_listing_no_replicators(self): self._test_sharded_listing() def test_sharded_listing_with_replicators(self): self._test_sharded_listing(run_replicators=True) def test_listing_under_populated_replica(self): # the leader node and one other primary have all the objects and will # cleave to 4 shard ranges, but the third primary only has 1 object in # the final shard range obj_names = self._make_object_names(2 * self.max_shard_size) self.brain.servers.stop(number=self.brain.node_numbers[2]) self.put_objects(obj_names) self.brain.servers.start(number=self.brain.node_numbers[2]) subset_obj_names = [obj_names[-1]] self.put_objects(subset_obj_names) self.brain.servers.stop(number=self.brain.node_numbers[2]) # sanity check: the first 2 primaries will list all objects self.assert_container_listing(obj_names, req_hdrs={'x-newest': 'true'}) # Run sharder on the fully populated nodes, starting with the leader client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) self.sharders.once(number=self.brain.node_numbers[0], additional_args='--partitions=%s' % self.brain.part) self.sharders.once(number=self.brain.node_numbers[1], additional_args='--partitions=%s' % self.brain.part) # Verify that the first 2 primary nodes have cleaved the first batch of # 2 shard ranges broker = self.get_broker(self.brain.part, self.brain.nodes[0]) self.assertEqual('sharding', broker.get_db_state()) shard_ranges = [dict(sr) for sr in broker.get_shard_ranges()] self.assertLengthEqual(shard_ranges, 4) self.assertEqual([ShardRange.CLEAVED, ShardRange.CLEAVED, ShardRange.CREATED, ShardRange.CREATED], [sr['state'] for sr in shard_ranges]) self.assertEqual( {False}, set([ctx.done() for ctx, _ in CleavingContext.load_all(broker)])) # listing is complete (from the fully populated primaries at least); # the root serves the listing parts for the last 2 shard ranges which # are not yet cleaved self.assert_container_listing(obj_names, req_hdrs={'x-newest': 'true'}) # Run the sharder on the under-populated node to get it fully # cleaved. self.brain.servers.start(number=self.brain.node_numbers[2]) Manager(['container-replicator']).once( number=self.brain.node_numbers[2]) self.sharders.once(number=self.brain.node_numbers[2], additional_args='--partitions=%s' % self.brain.part) broker = self.get_broker(self.brain.part, self.brain.nodes[2]) self.assertEqual('sharded', broker.get_db_state()) shard_ranges = [dict(sr) for sr in broker.get_shard_ranges()] self.assertLengthEqual(shard_ranges, 4) self.assertEqual([ShardRange.ACTIVE, ShardRange.ACTIVE, ShardRange.ACTIVE, ShardRange.ACTIVE], [sr['state'] for sr in shard_ranges]) self.assertEqual( {True, False}, set([ctx.done() for ctx, _ in CleavingContext.load_all(broker)])) # Get a consistent view of shard range states then check listing Manager(['container-replicator']).once( number=self.brain.node_numbers[2]) # oops, the listing is incomplete because the last 2 listing parts are # now served by the under-populated shard ranges. self.assert_container_listing( obj_names[:self.max_shard_size] + subset_obj_names, req_hdrs={'x-newest': 'true'}) # but once another replica has completed cleaving the listing is # complete again self.sharders.once(number=self.brain.node_numbers[1], additional_args='--partitions=%s' % self.brain.part) self.assert_container_listing(obj_names, req_hdrs={'x-newest': 'true'}) def assertInAsyncFile(self, async_path, expected): with open(async_path, 'rb') as fd: async_data = pickle.load(fd) errors = [] for k, v in expected.items(): if k not in async_data: errors.append("Key '%s' does not exist" % k) continue if async_data[k] != v: errors.append( "Exp value %s != %s" % (str(v), str(async_data[k]))) continue if errors: self.fail('\n'.join(errors)) def assertNotInAsyncFile(self, async_path, not_expect_keys): with open(async_path, 'rb') as fd: async_data = pickle.load(fd) errors = [] for k in not_expect_keys: if k in async_data: errors.append( "Key '%s' exists with value '%s'" % (k, async_data[k])) continue if errors: self.fail('\n'.join(errors)) def test_async_pendings(self): obj_names = self._make_object_names(self.max_shard_size * 2) # There are some updates *everyone* gets self.put_objects(obj_names[::5]) # But roll some outages so each container only get ~2/5 more object # records i.e. total of 3/5 updates per container; and async pendings # pile up for i, n in enumerate(self.brain.node_numbers, start=1): self.brain.servers.stop(number=n) self.put_objects(obj_names[i::5]) self.brain.servers.start(number=n) # Check the async pendings, they are unsharded so that's the db_state async_files = self.gather_async_pendings() self.assertTrue(async_files) for af in async_files: self.assertInAsyncFile(af, {'db_state': 'unsharded'}) self.assertNotInAsyncFile(af, ['container_path']) # But there are also 1/5 updates *no one* gets self.brain.servers.stop() self.put_objects(obj_names[4::5]) self.brain.servers.start() # Shard it client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) headers = client.head_container(self.url, self.admin_token, self.container_name) self.assertEqual('True', headers.get('x-container-sharding')) # sanity check found = self.categorize_container_dir_content() self.assertLengthEqual(found['shard_dbs'], 0) self.assertLengthEqual(found['normal_dbs'], 3) for db_file in found['normal_dbs']: broker = ContainerBroker(db_file) self.assertIs(True, broker.is_root_container()) self.assertEqual(len(obj_names) * 3 // 5, broker.get_info()['object_count']) # Only run the 'leader' in charge of scanning. # Each container has ~2 * max * 3/5 objects # which are distributed from obj000 to obj<2 * max - 1>, # so expect 3 shard ranges to be found: the first two will be complete # shards with max/2 objects and lower/upper bounds spaced by approx: # (2 * max - 1)/(2 * max * 3/5) * (max/2) =~ 5/6 * max # # Note that during this shard cycle the leader replicates to other # nodes so they will end up with ~2 * max * 4/5 objects. self.sharders.once(number=self.brain.node_numbers[0], additional_args='--partitions=%s' % self.brain.part) # Verify that we have one shard db -- though the other normal DBs # received the shard ranges that got defined found = self.categorize_container_dir_content() self.assertLengthEqual(found['shard_dbs'], 1) node_index_zero_db = found['shard_dbs'][0] broker = ContainerBroker(node_index_zero_db) self.assertIs(True, broker.is_root_container()) self.assertEqual(SHARDING, broker.get_db_state()) expected_shard_ranges = broker.get_shard_ranges() self.assertLengthEqual(expected_shard_ranges, 3) self.assertEqual( [ShardRange.CLEAVED, ShardRange.CLEAVED, ShardRange.CREATED], [sr.state for sr in expected_shard_ranges]) # Still have all three big DBs -- we've only cleaved 2 of the 3 shard # ranges that got defined self.assertLengthEqual(found['normal_dbs'], 3) db_states = [] for db_file in found['normal_dbs']: broker = ContainerBroker(db_file) self.assertIs(True, broker.is_root_container()) db_states.append(broker.get_db_state()) # the sharded db had shard range meta_timestamps updated during # cleaving, so we do not expect those to be equal on other nodes self.assert_shard_range_lists_equal( expected_shard_ranges, broker.get_shard_ranges(), excludes=['meta_timestamp', 'state_timestamp', 'state']) self.assertEqual(len(obj_names) * 3 // 5, broker.get_info()['object_count']) self.assertEqual([SHARDING, UNSHARDED, UNSHARDED], sorted(db_states)) # Run the other sharders so we're all in (roughly) the same state for n in self.brain.node_numbers[1:]: self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) found = self.categorize_container_dir_content() self.assertLengthEqual(found['shard_dbs'], 3) self.assertLengthEqual(found['normal_dbs'], 3) for db_file in found['normal_dbs']: broker = ContainerBroker(db_file) self.assertEqual(SHARDING, broker.get_db_state()) # no new rows self.assertEqual(len(obj_names) * 3 // 5, broker.get_info()['object_count']) # Run updaters to clear the async pendings Manager(['object-updater']).once() async_files = self.gather_async_pendings() self.assertFalse(async_files) # Our "big" dbs didn't take updates for db_file in found['normal_dbs']: broker = ContainerBroker(db_file) self.assertEqual(len(obj_names) * 3 // 5, broker.get_info()['object_count']) # confirm that the async pending updates got redirected to the shards for sr in expected_shard_ranges: shard_listings = self.direct_get_container(sr.account, sr.container) for node, (hdrs, listing) in shard_listings.items(): shard_listing_names = [ o['name'].encode('utf-8') if six.PY2 else o['name'] for o in listing] for obj in obj_names[4::5]: if obj in sr: self.assertIn(obj, shard_listing_names) else: self.assertNotIn(obj, shard_listing_names) # The entire listing is not yet available - we have two cleaved shard # ranges, complete with async updates, but for the remainder of the # namespace only what landed in the original container headers, listing = client.get_container(self.url, self.token, self.container_name) start_listing = [ o for o in obj_names if o <= expected_shard_ranges[1].upper] self.assertEqual( [x['name'].encode('utf-8') if six.PY2 else x['name'] for x in listing[:len(start_listing)]], start_listing) # we can't assert much about the remaining listing, other than that # there should be something self.assertTrue( [x['name'].encode('utf-8') if six.PY2 else x['name'] for x in listing[len(start_listing):]]) self.assertIn('x-container-object-count', headers) self.assertEqual(str(len(listing)), headers['x-container-object-count']) headers, listing = client.get_container(self.url, self.token, self.container_name, query_string='reverse=on') self.assertEqual([x['name'].encode('utf-8') if six.PY2 else x['name'] for x in listing[-len(start_listing):]], list(reversed(start_listing))) self.assertIn('x-container-object-count', headers) self.assertEqual(str(len(listing)), headers['x-container-object-count']) self.assertTrue( [x['name'].encode('utf-8') if six.PY2 else x['name'] for x in listing[:-len(start_listing)]]) # Run the sharders again to get everything to settle self.sharders.once() found = self.categorize_container_dir_content() self.assertLengthEqual(found['shard_dbs'], 3) self.assertLengthEqual(found['normal_dbs'], 0) # now all shards have been cleaved we should get the complete listing headers, listing = client.get_container(self.url, self.token, self.container_name) self.assertEqual([x['name'].encode('utf-8') if six.PY2 else x['name'] for x in listing], obj_names) # Create a few more objects in async pending. Check them, they should # now have the correct db_state as sharded more_obj_names = self._make_object_names(10, self.max_shard_size * 2) # No one should get these updates self.brain.servers.stop() self.put_objects(more_obj_names) self.brain.servers.start() async_files = self.gather_async_pendings() self.assertTrue(async_files) for af in async_files: # They should have a sharded db_state self.assertInAsyncFile(af, {'db_state': 'sharded'}) # But because the container-servers were down, they wont have # container-path (because it couldn't get a shard range back) self.assertNotInAsyncFile(af, ['container_path']) # they don't exist yet headers, listing = client.get_container(self.url, self.token, self.container_name) self.assertEqual([x['name'].encode('utf-8') if six.PY2 else x['name'] for x in listing], obj_names) # Now clear them out and they should now exist where we expect. Manager(['object-updater']).once() headers, listing = client.get_container(self.url, self.token, self.container_name) self.assertEqual([x['name'].encode('utf-8') if six.PY2 else x['name'] for x in listing], obj_names + more_obj_names) # And they're cleared up async_files = self.gather_async_pendings() self.assertFalse(async_files) # If we take 1/2 the nodes offline when we add some more objects, # we should get async pendings with container-path because there # was a container-server to respond. even_more_obj_names = self._make_object_names( 10, self.max_shard_size * 2 + 10) self.brain.stop_primary_half() self.put_objects(even_more_obj_names) self.brain.start_primary_half() async_files = self.gather_async_pendings() self.assertTrue(async_files) for af in async_files: # They should have a sharded db_state AND container_path self.assertInAsyncFile(af, {'db_state': 'sharded', 'container_path': mock.ANY}) Manager(['object-updater']).once() # And they're cleared up async_files = self.gather_async_pendings() self.assertFalse(async_files) def test_shrinking(self): int_client = self.make_internal_client() def check_node_data(node_data, exp_hdrs, exp_obj_count, exp_shards, exp_sharded_root_range=False): hdrs, range_data = node_data self.assert_dict_contains(exp_hdrs, hdrs) sharded_root_range = False other_range_data = [] for data in range_data: sr = ShardRange.from_dict(data) if (sr.account == self.account and sr.container == self.container_name and sr.state == ShardRange.SHARDED): # only expect one root range self.assertFalse(sharded_root_range, range_data) sharded_root_range = True self.assertEqual(ShardRange.MIN, sr.lower, sr) self.assertEqual(ShardRange.MAX, sr.upper, sr) else: # include active root range in further assertions other_range_data.append(data) self.assertEqual(exp_sharded_root_range, sharded_root_range) self.assert_shard_ranges_contiguous(exp_shards, other_range_data) self.assert_total_object_count(exp_obj_count, other_range_data) def check_shard_nodes_data(node_data, expected_state='unsharded', expected_shards=0, exp_obj_count=0, exp_sharded_root_range=False): # checks that shard range is consistent on all nodes root_path = '%s/%s' % (self.account, self.container_name) exp_shard_hdrs = { 'X-Container-Sysmeta-Shard-Quoted-Root': quote(root_path), 'X-Backend-Sharding-State': expected_state} object_counts = [] bytes_used = [] for node_id, node_data in node_data.items(): with annotate_failure('Node id %s.' % node_id): check_node_data( node_data, exp_shard_hdrs, exp_obj_count, expected_shards, exp_sharded_root_range) hdrs = node_data[0] object_counts.append(int(hdrs['X-Container-Object-Count'])) bytes_used.append(int(hdrs['X-Container-Bytes-Used'])) if len(set(object_counts)) != 1: self.fail('Inconsistent object counts: %s' % object_counts) if len(set(bytes_used)) != 1: self.fail('Inconsistent bytes used: %s' % bytes_used) return object_counts[0], bytes_used[0] repeat = [0] def do_shard_then_shrink(): repeat[0] += 1 obj_names = ['obj-%s-%03d' % (repeat[0], x) for x in range(self.max_shard_size)] self.put_objects(obj_names) # these two object names will fall at start of first shard range... alpha = 'alpha-%s' % repeat[0] beta = 'beta-%s' % repeat[0] # Enable sharding client.post_container( self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # sanity check self.assert_container_listing(obj_names) # Only run the one in charge of scanning self.sharders.once( number=self.brain.node_numbers[0], additional_args='--partitions=%s' % self.brain.part) # check root container root_nodes_data = self.direct_get_container_shard_ranges() self.assertEqual(3, len(root_nodes_data)) # nodes on which sharder has not run are still in unsharded state # but have had shard ranges replicated to them exp_obj_count = len(obj_names) exp_hdrs = {'X-Backend-Sharding-State': 'unsharded', 'X-Container-Object-Count': str(exp_obj_count)} node_id = self.brain.node_numbers[1] - 1 check_node_data( root_nodes_data[node_id], exp_hdrs, exp_obj_count, 2) node_id = self.brain.node_numbers[2] - 1 check_node_data( root_nodes_data[node_id], exp_hdrs, exp_obj_count, 2) # only one that ran sharder is in sharded state exp_hdrs['X-Backend-Sharding-State'] = 'sharded' node_id = self.brain.node_numbers[0] - 1 check_node_data( root_nodes_data[node_id], exp_hdrs, exp_obj_count, 2) orig_range_data = root_nodes_data[node_id][1] orig_shard_ranges = [ShardRange.from_dict(r) for r in orig_range_data] # check first shard shard_nodes_data = self.direct_get_container_shard_ranges( orig_shard_ranges[0].account, orig_shard_ranges[0].container) obj_count, bytes_used = check_shard_nodes_data(shard_nodes_data) total_shard_object_count = obj_count # check second shard shard_nodes_data = self.direct_get_container_shard_ranges( orig_shard_ranges[1].account, orig_shard_ranges[1].container) obj_count, bytes_used = check_shard_nodes_data(shard_nodes_data) total_shard_object_count += obj_count self.assertEqual(exp_obj_count, total_shard_object_count) # Now that everyone has shard ranges, run *everyone* self.sharders.once( additional_args='--partitions=%s' % self.brain.part) # all root container nodes should now be in sharded state root_nodes_data = self.direct_get_container_shard_ranges() self.assertEqual(3, len(root_nodes_data)) for node_id, node_data in root_nodes_data.items(): with annotate_failure('Node id %s.' % node_id): check_node_data(node_data, exp_hdrs, exp_obj_count, 2) # run updaters to update .sharded account; shard containers have # not updated account since having objects replicated to them self.updaters.once() shard_cont_count, shard_obj_count = int_client.get_account_info( orig_shard_ranges[0].account, [204]) self.assertEqual(2 * repeat[0], shard_cont_count) # the shards account should always have zero object count to avoid # double accounting self.assertEqual(0, shard_obj_count) # checking the listing also refreshes proxy container info cache so # that the proxy becomes aware that container is sharded and will # now look up the shard target for subsequent updates self.assert_container_listing(obj_names) # Before writing, kill the cache self.memcache.delete(get_cache_key( self.account, self.container_name, shard='updating')) # delete objects from first shard range first_shard_objects = [obj_name for obj_name in obj_names if obj_name <= orig_shard_ranges[0].upper] for obj in first_shard_objects: client.delete_object( self.url, self.token, self.container_name, obj) with self.assertRaises(ClientException): client.get_object( self.url, self.token, self.container_name, obj) second_shard_objects = [obj_name for obj_name in obj_names if obj_name > orig_shard_ranges[1].lower] self.assert_container_listing(second_shard_objects) # put a new object 'alpha' in first shard range self.put_objects([alpha]) second_shard_objects = [obj_name for obj_name in obj_names if obj_name > orig_shard_ranges[1].lower] self.assert_container_listing([alpha] + second_shard_objects) # while container servers are down, but proxy has container info in # cache from recent listing, put another object; this update will # lurk in async pending until the updaters run again; because all # the root container servers are down and therefore cannot respond # to a GET for a redirect target, the object update will default to # being targeted at the root container self.stop_container_servers() # Before writing, kill the cache self.memcache.delete(get_cache_key( self.account, self.container_name, shard='updating')) self.put_objects([beta]) self.brain.servers.start() async_pendings = self.gather_async_pendings() num_container_replicas = len(self.brain.nodes) num_obj_replicas = self.policy.object_ring.replica_count expected_num_updates = num_container_updates( num_container_replicas, quorum_size(num_container_replicas), num_obj_replicas, self.policy.quorum) expected_num_pendings = min(expected_num_updates, num_obj_replicas) # sanity check with annotate_failure('policy %s. ' % self.policy): self.assertLengthEqual(async_pendings, expected_num_pendings) # root object count is not updated... self.assert_container_object_count(len(obj_names)) self.assert_container_listing([alpha] + second_shard_objects) root_nodes_data = self.direct_get_container_shard_ranges() self.assertEqual(3, len(root_nodes_data)) for node_id, node_data in root_nodes_data.items(): with annotate_failure('Node id %s.' % node_id): check_node_data(node_data, exp_hdrs, exp_obj_count, 2) range_data = node_data[1] self.assert_shard_range_lists_equal( orig_range_data, range_data, excludes=['meta_timestamp', 'state_timestamp']) # ...until the sharders run and update root; reclaim tombstones so # that the shard is shrinkable shard_0_part = self.get_part_and_node_numbers( orig_shard_ranges[0])[0] for conf_index in self.configs['container-sharder'].keys(): self.run_custom_sharder(conf_index, {'reclaim_age': 0}, override_partitions=[shard_0_part]) exp_obj_count = len(second_shard_objects) + 1 self.assert_container_object_count(exp_obj_count) self.assert_container_listing([alpha] + second_shard_objects) # root sharder finds donor, acceptor pair and pushes changes self.sharders.once( additional_args='--partitions=%s' % self.brain.part) self.assert_container_listing([alpha] + second_shard_objects) # run sharder on donor to shrink and replicate to acceptor self.run_sharders(orig_shard_ranges[0]) self.assert_container_listing([alpha] + second_shard_objects) # run sharder on acceptor to update root with stats self.run_sharders(orig_shard_ranges[1]) self.assert_container_listing([alpha] + second_shard_objects) self.assert_container_object_count(len(second_shard_objects) + 1) # check root container root_nodes_data = self.direct_get_container_shard_ranges() self.assertEqual(3, len(root_nodes_data)) exp_hdrs['X-Container-Object-Count'] = str(exp_obj_count) for node_id, node_data in root_nodes_data.items(): with annotate_failure('Node id %s.' % node_id): # NB now only *one* shard range in root check_node_data(node_data, exp_hdrs, exp_obj_count, 1) # the acceptor shard is intact.. shard_nodes_data = self.direct_get_container_shard_ranges( orig_shard_ranges[1].account, orig_shard_ranges[1].container) obj_count, bytes_used = check_shard_nodes_data(shard_nodes_data) # all objects should now be in this shard self.assertEqual(exp_obj_count, obj_count) # the donor shard is also still intact donor = orig_shard_ranges[0] shard_nodes_data = self.direct_get_container_shard_ranges( donor.account, donor.container) # donor has the acceptor shard range but not the root shard range # because the root is still in ACTIVE state; # the donor's shard range will have the acceptor's projected stats obj_count, bytes_used = check_shard_nodes_data( shard_nodes_data, expected_state='sharded', expected_shards=1, exp_obj_count=len(second_shard_objects) + 1) # but the donor is empty and so reports zero stats self.assertEqual(0, obj_count) self.assertEqual(0, bytes_used) # check the donor own shard range state part, nodes = self.brain.ring.get_nodes( donor.account, donor.container) for node in nodes: with annotate_failure(node): broker = self.get_broker( part, node, donor.account, donor.container) own_sr = broker.get_own_shard_range() self.assertEqual(ShardRange.SHRUNK, own_sr.state) self.assertTrue(own_sr.deleted) # delete all the second shard's object apart from 'alpha' for obj in second_shard_objects: client.delete_object( self.url, self.token, self.container_name, obj) self.assert_container_listing([alpha]) # run sharders: second range should not shrink away yet because it # has tombstones self.sharders.once() # second shard updates root stats self.assert_container_listing([alpha]) self.sharders.once() # root finds shrinkable shard self.assert_container_listing([alpha]) self.sharders.once() # shards shrink themselves self.assert_container_listing([alpha]) # the acceptor shard is intact... shard_nodes_data = self.direct_get_container_shard_ranges( orig_shard_ranges[1].account, orig_shard_ranges[1].container) obj_count, bytes_used = check_shard_nodes_data(shard_nodes_data) self.assertEqual(1, obj_count) # run sharders to reclaim tombstones so that the second shard is # shrinkable shard_1_part = self.get_part_and_node_numbers( orig_shard_ranges[1])[0] for conf_index in self.configs['container-sharder'].keys(): self.run_custom_sharder(conf_index, {'reclaim_age': 0}, override_partitions=[shard_1_part]) self.assert_container_listing([alpha]) # run sharders so second range shrinks away, requires up to 2 # cycles self.sharders.once() # root finds shrinkable shard self.assert_container_listing([alpha]) self.sharders.once() # shards shrink themselves self.assert_container_listing([alpha]) # the second shard range has sharded and is empty shard_nodes_data = self.direct_get_container_shard_ranges( orig_shard_ranges[1].account, orig_shard_ranges[1].container) check_shard_nodes_data( shard_nodes_data, expected_state='sharded', expected_shards=1, exp_obj_count=0) # check root container root_nodes_data = self.direct_get_container_shard_ranges() self.assertEqual(3, len(root_nodes_data)) exp_hdrs = {'X-Backend-Sharding-State': 'collapsed', # just the alpha object 'X-Container-Object-Count': '1'} for node_id, node_data in root_nodes_data.items(): with annotate_failure('Node id %s.' % node_id): # NB now no shard ranges in root check_node_data(node_data, exp_hdrs, 0, 0) # delete the alpha object client.delete_object( self.url, self.token, self.container_name, alpha) # should now be able to delete the *apparently* empty container client.delete_container(self.url, self.token, self.container_name) self.assert_container_not_found() self.direct_head_container(expect_failure=True) # and the container stays deleted even after sharders run and shard # send updates self.sharders.once() self.assert_container_not_found() self.direct_head_container(expect_failure=True) # now run updaters to deal with the async pending for the beta # object self.updaters.once() # and the container is revived! self.assert_container_listing([beta]) # finally, clear out the container client.delete_object( self.url, self.token, self.container_name, beta) do_shard_then_shrink() # repeat from starting point of a collapsed and previously deleted # container do_shard_then_shrink() def test_delete_root_reclaim(self): all_obj_names = self._make_object_names(self.max_shard_size) self.put_objects(all_obj_names) # Shard the container client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) for n in self.brain.node_numbers: self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity checks self.assert_container_states('sharded', 2) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') self.assert_container_listing(all_obj_names) # delete all objects - updates redirected to shards self.delete_objects(all_obj_names) self.assert_container_listing([]) self.assert_container_post_ok('has objects') # root not yet updated with shard stats self.assert_container_object_count(len(all_obj_names)) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() # run sharder on shard containers to update root stats shard_ranges = self.get_container_shard_ranges() self.assertLengthEqual(shard_ranges, 2) self.run_sharders(shard_ranges) self.assert_container_listing([]) self.assert_container_post_ok('empty') self.assert_container_object_count(0) # and now we can delete it! client.delete_container(self.url, self.token, self.container_name) self.assert_container_post_fails('deleted') self.assert_container_not_found() # see if it will reclaim Manager(['container-updater']).once() for conf_file in self.configs['container-replicator'].values(): conf = utils.readconf(conf_file, 'container-replicator') conf['reclaim_age'] = 0 ContainerReplicator(conf).run_once() # we don't expect warnings from sharder root audits for conf_index in self.configs['container-sharder'].keys(): sharder = self.run_custom_sharder(conf_index, {}) self.assertEqual([], sharder.logger.get_lines_for_level('warning')) # until the root wants to start reclaiming but we haven't shrunk yet! found_warning = False for conf_index in self.configs['container-sharder'].keys(): sharder = self.run_custom_sharder(conf_index, {'reclaim_age': 0}) warnings = sharder.logger.get_lines_for_level('warning') if warnings: self.assertTrue(warnings[0].startswith( 'Reclaimable db stuck waiting for shrinking')) self.assertEqual(1, len(warnings)) found_warning = True self.assertTrue(found_warning) # TODO: shrink empty shards and assert everything reclaims def _setup_replication_scenario(self, num_shards, extra_objs=('alpha',)): # Get cluster to state where 2 replicas are sharding or sharded but 3rd # replica is unsharded and has an object that the first 2 are missing. # put objects while all servers are up obj_names = self._make_object_names( num_shards * self.max_shard_size // 2) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) node_numbers = self.brain.node_numbers # run replicators first time to get sync points set self.replicators.once() # stop the leader node and one other server self.stop_container_servers(slice(0, 2)) # ...then put one more object in first shard range namespace self.put_objects(extra_objs) # start leader and first other server, stop third server for number in node_numbers[:2]: self.brain.servers.start(number=number) self.brain.servers.stop(number=node_numbers[2]) self.assert_container_listing(obj_names) # sanity check # shard the container - first two shard ranges are cleaved for number in node_numbers[:2]: self.sharders.once( number=number, additional_args='--partitions=%s' % self.brain.part) self.assert_container_listing(obj_names) # sanity check return obj_names def test_replication_to_sharding_container(self): # verify that replication from an unsharded replica to a sharding # replica does not replicate rows but does replicate shard ranges obj_names = self._setup_replication_scenario(3) for node in self.brain.nodes[:2]: self.assert_container_state(node, 'sharding', 3) # bring third server back up, run replicator node_numbers = self.brain.node_numbers self.brain.servers.start(number=node_numbers[2]) # sanity check... self.assert_container_state(self.brain.nodes[2], 'unsharded', 0) self.replicators.once(number=node_numbers[2]) # check db files unchanged found = self.categorize_container_dir_content() self.assertLengthEqual(found['shard_dbs'], 2) self.assertLengthEqual(found['normal_dbs'], 3) # the 'alpha' object is NOT replicated to the two sharded nodes for node in self.brain.nodes[:2]: broker = self.get_broker(self.brain.part, node) with annotate_failure( 'Node id %s in %s' % (node['id'], self.brain.nodes[:2])): self.assertFalse(broker.get_objects()) self.assert_container_state(node, 'sharding', 3) self.brain.servers.stop(number=node_numbers[2]) self.assert_container_listing(obj_names) # all nodes now have shard ranges self.brain.servers.start(number=node_numbers[2]) node_data = self.direct_get_container_shard_ranges() for node, (hdrs, shard_ranges) in node_data.items(): with annotate_failure(node): self.assert_shard_ranges_contiguous(3, shard_ranges) # complete cleaving third shard range on first two nodes self.brain.servers.stop(number=node_numbers[2]) for number in node_numbers[:2]: self.sharders.once( number=number, additional_args='--partitions=%s' % self.brain.part) # ...and now they are in sharded state self.assert_container_state(self.brain.nodes[0], 'sharded', 3) self.assert_container_state(self.brain.nodes[1], 'sharded', 3) # ...still no 'alpha' object in listing self.assert_container_listing(obj_names) # run the sharder on the third server, alpha object is included in # shards that it cleaves self.brain.servers.start(number=node_numbers[2]) self.assert_container_state(self.brain.nodes[2], 'unsharded', 3) self.sharders.once(number=node_numbers[2], additional_args='--partitions=%s' % self.brain.part) self.assert_container_state(self.brain.nodes[2], 'sharding', 3) self.sharders.once(number=node_numbers[2], additional_args='--partitions=%s' % self.brain.part) self.assert_container_state(self.brain.nodes[2], 'sharded', 3) self.assert_container_listing(['alpha'] + obj_names) def test_replication_to_sharded_container(self): # verify that replication from an unsharded replica to a sharded # replica does not replicate rows but does replicate shard ranges obj_names = self._setup_replication_scenario(2) for node in self.brain.nodes[:2]: self.assert_container_state(node, 'sharded', 2) # sanity check found = self.categorize_container_dir_content() self.assertLengthEqual(found['shard_dbs'], 2) self.assertLengthEqual(found['normal_dbs'], 1) for node in self.brain.nodes[:2]: broker = self.get_broker(self.brain.part, node) info = broker.get_info() with annotate_failure( 'Node id %s in %s' % (node['id'], self.brain.nodes[:2])): self.assertEqual(len(obj_names), info['object_count']) self.assertFalse(broker.get_objects()) # bring third server back up, run replicator node_numbers = self.brain.node_numbers self.brain.servers.start(number=node_numbers[2]) # sanity check... self.assert_container_state(self.brain.nodes[2], 'unsharded', 0) self.replicators.once(number=node_numbers[2]) # check db files unchanged found = self.categorize_container_dir_content() self.assertLengthEqual(found['shard_dbs'], 2) self.assertLengthEqual(found['normal_dbs'], 1) # the 'alpha' object is NOT replicated to the two sharded nodes for node in self.brain.nodes[:2]: broker = self.get_broker(self.brain.part, node) with annotate_failure( 'Node id %s in %s' % (node['id'], self.brain.nodes[:2])): self.assertFalse(broker.get_objects()) self.assert_container_state(node, 'sharded', 2) self.brain.servers.stop(number=node_numbers[2]) self.assert_container_listing(obj_names) # all nodes now have shard ranges self.brain.servers.start(number=node_numbers[2]) node_data = self.direct_get_container_shard_ranges() for node, (hdrs, shard_ranges) in node_data.items(): with annotate_failure(node): self.assert_shard_ranges_contiguous(2, shard_ranges) # run the sharder on the third server, alpha object is included in # shards that it cleaves self.assert_container_state(self.brain.nodes[2], 'unsharded', 2) self.sharders.once(number=node_numbers[2], additional_args='--partitions=%s' % self.brain.part) self.assert_container_state(self.brain.nodes[2], 'sharded', 2) self.assert_container_listing(['alpha'] + obj_names) def test_sharding_requires_sufficient_replication(self): # verify that cleaving only progresses if each cleaved shard range is # sufficiently replicated # put enough objects for 4 shard ranges obj_names = self._make_object_names(2 * self.max_shard_size) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) node_numbers = self.brain.node_numbers leader_node = self.brain.nodes[0] leader_num = node_numbers[0] # run replicators first time to get sync points set self.replicators.once() # start sharding on the leader node self.sharders.once(number=leader_num, additional_args='--partitions=%s' % self.brain.part) shard_ranges = self.assert_container_state(leader_node, 'sharding', 4) self.assertEqual([ShardRange.CLEAVED] * 2 + [ShardRange.CREATED] * 2, [sr.state for sr in shard_ranges]) # Check the current progress. It shouldn't be complete. recon = direct_client.direct_get_recon(leader_node, "sharding") expected_in_progress = {'all': [{'account': 'AUTH_test', 'active': 0, 'cleaved': 2, 'created': 2, 'found': 0, 'db_state': 'sharding', 'state': 'sharding', 'error': None, 'file_size': mock.ANY, 'meta_timestamp': mock.ANY, 'node_index': 0, 'object_count': len(obj_names), 'container': mock.ANY, 'path': mock.ANY, 'root': mock.ANY}]} actual = recon['sharding_stats']['sharding']['sharding_in_progress'] self.assertEqual(expected_in_progress, actual) # stop *all* container servers for third shard range sr_part, sr_node_nums = self.get_part_and_node_numbers(shard_ranges[2]) for node_num in sr_node_nums: self.brain.servers.stop(number=node_num) # attempt to continue sharding on the leader node self.sharders.once(number=leader_num, additional_args='--partitions=%s' % self.brain.part) # no cleaving progress was made for node_num in sr_node_nums: self.brain.servers.start(number=node_num) shard_ranges = self.assert_container_state(leader_node, 'sharding', 4) self.assertEqual([ShardRange.CLEAVED] * 2 + [ShardRange.CREATED] * 2, [sr.state for sr in shard_ranges]) # stop two of the servers for third shard range, not including any # server that happens to be the leader node stopped = [] for node_num in sr_node_nums: if node_num != leader_num: self.brain.servers.stop(number=node_num) stopped.append(node_num) if len(stopped) >= 2: break self.assertLengthEqual(stopped, 2) # sanity check # attempt to continue sharding on the leader node self.sharders.once(number=leader_num, additional_args='--partitions=%s' % self.brain.part) # no cleaving progress was made for node_num in stopped: self.brain.servers.start(number=node_num) shard_ranges = self.assert_container_state(leader_node, 'sharding', 4) self.assertEqual([ShardRange.CLEAVED] * 2 + [ShardRange.CREATED] * 2, [sr.state for sr in shard_ranges]) # stop just one of the servers for third shard range stopped = [] for node_num in sr_node_nums: if node_num != leader_num: self.brain.servers.stop(number=node_num) stopped.append(node_num) break self.assertLengthEqual(stopped, 1) # sanity check # attempt to continue sharding the container self.sharders.once(number=leader_num, additional_args='--partitions=%s' % self.brain.part) # this time cleaving completed self.brain.servers.start(number=stopped[0]) shard_ranges = self.assert_container_state(leader_node, 'sharded', 4) self.assertEqual([ShardRange.ACTIVE] * 4, [sr.state for sr in shard_ranges]) # Check the leader's progress again, this time is should be complete recon = direct_client.direct_get_recon(leader_node, "sharding") expected_in_progress = {'all': [{'account': 'AUTH_test', 'active': 4, 'cleaved': 0, 'created': 0, 'found': 0, 'db_state': 'sharded', 'state': 'sharded', 'error': None, 'file_size': mock.ANY, 'meta_timestamp': mock.ANY, 'node_index': 0, 'object_count': len(obj_names), 'container': mock.ANY, 'path': mock.ANY, 'root': mock.ANY}]} actual = recon['sharding_stats']['sharding']['sharding_in_progress'] self.assertEqual(expected_in_progress, actual) def test_sharded_delete(self): all_obj_names = self._make_object_names(self.max_shard_size) self.put_objects(all_obj_names) # Shard the container client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) for n in self.brain.node_numbers: self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity checks self.assert_container_states('sharded', 2) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') self.assert_container_listing(all_obj_names) # delete all objects - updates redirected to shards self.delete_objects(all_obj_names) self.assert_container_listing([]) self.assert_container_post_ok('has objects') # root not yet updated with shard stats self.assert_container_object_count(len(all_obj_names)) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() # run sharder on shard containers to update root stats shard_ranges = self.get_container_shard_ranges() self.assertLengthEqual(shard_ranges, 2) self.run_sharders(shard_ranges) self.assert_container_listing([]) self.assert_container_post_ok('empty') self.assert_container_object_count(0) # put a new object - update redirected to shard self.put_objects(['alpha']) self.assert_container_listing(['alpha']) self.assert_container_object_count(0) # before root learns about new object in shard, delete the container client.delete_container(self.url, self.token, self.container_name) self.assert_container_post_fails('deleted') self.assert_container_not_found() # run the sharders to update root with shard stats self.run_sharders(shard_ranges) self.assert_container_listing(['alpha']) self.assert_container_object_count(1) self.assert_container_delete_fails() self.assert_container_post_ok('revived') def _do_test_sharded_can_get_objects_different_policy(self, policy_idx, new_policy_idx): # create sharded container client.delete_container(self.url, self.token, self.container_name) self.brain.put_container(policy_index=int(policy_idx)) all_obj_names = self._make_object_names(self.max_shard_size) self.put_objects(all_obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) for n in self.brain.node_numbers: self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # empty and delete self.delete_objects(all_obj_names) shard_ranges = self.get_container_shard_ranges() self.run_sharders(shard_ranges) client.delete_container(self.url, self.token, self.container_name) # re-create with new_policy_idx self.brain.put_container(policy_index=int(new_policy_idx)) # we re-use shard ranges new_shard_ranges = self.get_container_shard_ranges() self.assertEqual(shard_ranges, new_shard_ranges) self.put_objects(all_obj_names) # The shard is still on the old policy index, but the root spi # is passed to shard container server and is used to pull objects # of that index out. self.assert_container_listing(all_obj_names) # although a head request is getting object count for the shard spi self.assert_container_object_count(0) # we can force the listing to use the old policy index in which case we # expect no objects to be listed try: resp = self.internal_client.make_request( 'GET', path=self.internal_client.make_path( self.account, self.container_name), headers={'X-Backend-Storage-Policy-Index': str(policy_idx)}, acceptable_statuses=(2,), params={'format': 'json'} ) except UnexpectedResponse as exc: self.fail('Listing failed with %s' % exc.resp.status) self.assertEqual([], json.loads(b''.join(resp.app_iter))) @unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy") def test_sharded_can_get_objects_different_policy(self): policy_idx = self.policy.idx new_policy_idx = [pol.idx for pol in ENABLED_POLICIES if pol != self.policy.idx][0] self._do_test_sharded_can_get_objects_different_policy( policy_idx, new_policy_idx) @unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy") def test_sharded_can_get_objects_different_policy_reversed(self): policy_idx = [pol.idx for pol in ENABLED_POLICIES if pol != self.policy][0] new_policy_idx = self.policy.idx self._do_test_sharded_can_get_objects_different_policy( policy_idx, new_policy_idx) def test_object_update_redirection(self): all_obj_names = self._make_object_names(self.max_shard_size) self.put_objects(all_obj_names) # Shard the container client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) for n in self.brain.node_numbers: self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity checks self.assert_container_states('sharded', 2) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') self.assert_container_listing(all_obj_names) # delete all objects - updates redirected to shards self.delete_objects(all_obj_names) self.assert_container_listing([]) self.assert_container_post_ok('has objects') # run sharder on shard containers to update root stats; reclaim # the tombstones so that the shards appear to be shrinkable shard_ranges = self.get_container_shard_ranges() self.assertLengthEqual(shard_ranges, 2) shard_partitions = [self.get_part_and_node_numbers(sr)[0] for sr in shard_ranges] for conf_index in self.configs['container-sharder'].keys(): self.run_custom_sharder(conf_index, {'reclaim_age': 0}, override_partitions=shard_partitions) self.assert_container_object_count(0) # First, test a misplaced object moving from one shard to another. # with one shard server down, put a new 'alpha' object... shard_part, shard_nodes = self.get_part_and_node_numbers( shard_ranges[0]) self.brain.servers.stop(number=shard_nodes[2]) self.put_objects(['alpha']) self.assert_container_listing(['alpha']) self.assert_container_object_count(0) self.assertLengthEqual(self.gather_async_pendings(), 1) self.brain.servers.start(number=shard_nodes[2]) # run sharder on root to discover first shrink candidate self.sharders.once(additional_args='--partitions=%s' % self.brain.part) # then run sharder on the shard node without the alpha object self.sharders.once(additional_args='--partitions=%s' % shard_part, number=shard_nodes[2]) # root sees first shard has shrunk self.assertLengthEqual(self.get_container_shard_ranges(), 1) # cached shard ranges still show first shard range as active so listing # will include 'alpha' if the shard listing is fetched from node (0,1) # but not if fetched from node 2; to achieve predictability we use # x-newest to use shard ranges from the root so that only the second # shard range is used for listing, so alpha object not in listing self.assert_container_listing([], req_hdrs={'x-newest': 'true'}) self.assert_container_object_count(0) # run the updaters: the async pending update will be redirected from # shrunk shard to second shard self.updaters.once() self.assert_container_listing(['alpha']) self.assert_container_object_count(0) # root not yet updated # then run sharder on other shard nodes to complete shrinking for number in shard_nodes[:2]: self.sharders.once(additional_args='--partitions=%s' % shard_part, number=number) # and get root updated self.run_sharders(shard_ranges[1]) self.assert_container_listing(['alpha']) self.assert_container_object_count(1) self.assertLengthEqual(self.get_container_shard_ranges(), 1) # Now we have just one active shard, test a misplaced object moving # from that shard to the root. # with one shard server down, delete 'alpha' and put a 'beta' object... shard_part, shard_nodes = self.get_part_and_node_numbers( shard_ranges[1]) self.brain.servers.stop(number=shard_nodes[2]) # Before writing, kill the cache self.memcache.delete(get_cache_key( self.account, self.container_name, shard='updating')) self.delete_objects(['alpha']) self.put_objects(['beta']) self.assert_container_listing(['beta']) self.assert_container_object_count(1) self.assertLengthEqual(self.gather_async_pendings(), 2) self.brain.servers.start(number=shard_nodes[2]) # run sharder on root to discover second shrink candidate - root is not # yet aware of the beta object self.sharders.once(additional_args='--partitions=%s' % self.brain.part) # then run sharder on the shard node without the beta object, to shrink # it to root - note this moves stale copy of alpha to the root db self.sharders.once(additional_args='--partitions=%s' % shard_part, number=shard_nodes[2]) # now there are no active shards self.assertFalse(self.get_container_shard_ranges()) # with other two shard servers down, listing won't find beta object for number in shard_nodes[:2]: self.brain.servers.stop(number=number) self.assert_container_listing(['alpha']) self.assert_container_object_count(1) # run the updaters: the async pending update will be redirected from # shrunk shard to the root self.updaters.once() self.assert_container_listing(['beta']) self.assert_container_object_count(1) def test_misplaced_object_movement(self): def merge_object(shard_range, name, deleted=0): # it's hard to get a test to put a misplaced object into a shard, # so this hack is used force an object record directly into a shard # container db. Note: the actual object won't exist, we're just # using this to test object records in container dbs. shard_part, shard_nodes = self.brain.ring.get_nodes( shard_range.account, shard_range.container) shard_broker = self.get_broker( shard_part, shard_nodes[0], shard_range.account, shard_range.container) shard_broker.merge_items( [{'name': name, 'created_at': Timestamp.now().internal, 'size': 0, 'content_type': 'text/plain', 'etag': md5(usedforsecurity=False).hexdigest(), 'deleted': deleted, 'storage_policy_index': shard_broker.storage_policy_index}]) return shard_nodes[0] all_obj_names = self._make_object_names(self.max_shard_size) self.put_objects(all_obj_names) # Shard the container client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) for n in self.brain.node_numbers: self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity checks self.assert_container_states('sharded', 2) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') self.assert_container_listing(all_obj_names) # delete all objects in first shard range - updates redirected to shard shard_ranges = self.get_container_shard_ranges() self.assertLengthEqual(shard_ranges, 2) shard_0_objects = [name for name in all_obj_names if name in shard_ranges[0]] shard_1_objects = [name for name in all_obj_names if name in shard_ranges[1]] self.delete_objects(shard_0_objects) self.assert_container_listing(shard_1_objects) self.assert_container_post_ok('has objects') # run sharder on first shard container to update root stats; reclaim # the tombstones so that the shard appears to be shrinkable shard_0_part = self.get_part_and_node_numbers(shard_ranges[0])[0] for conf_index in self.configs['container-sharder'].keys(): self.run_custom_sharder(conf_index, {'reclaim_age': 0}, override_partitions=[shard_0_part]) self.assert_container_object_count(len(shard_1_objects)) # First, test a misplaced object moving from one shard to another. # run sharder on root to discover first shrink candidate self.sharders.once(additional_args='--partitions=%s' % self.brain.part) # then run sharder on first shard range to shrink it self.run_sharders(shard_ranges[0]) # force a misplaced object into the shrunken shard range to simulate # a client put that was in flight when it started to shrink misplaced_node = merge_object(shard_ranges[0], 'alpha', deleted=0) # root sees first shard has shrunk, only second shard range used for # listing so alpha object not in listing self.assertLengthEqual(self.get_container_shard_ranges(), 1) self.assert_container_listing(shard_1_objects) self.assert_container_object_count(len(shard_1_objects)) # until sharder runs on that node to move the misplaced object to the # second shard range shard_part, shard_nodes_numbers = self.get_part_and_node_numbers( shard_ranges[0]) self.sharders.once(additional_args='--partitions=%s' % shard_part, number=misplaced_node['id'] + 1) self.assert_container_listing(['alpha'] + shard_1_objects) # root not yet updated self.assert_container_object_count(len(shard_1_objects)) # run sharder to get root updated self.run_sharders(shard_ranges[1]) self.assert_container_listing(['alpha'] + shard_1_objects) self.assert_container_object_count(len(shard_1_objects) + 1) self.assertLengthEqual(self.get_container_shard_ranges(), 1) # Now we have just one active shard, test a misplaced object moving # from that shard to the root. # delete most objects from second shard range, reclaim the tombstones, # and run sharder on root to discover second shrink candidate self.delete_objects(shard_1_objects) shard_1_part = self.get_part_and_node_numbers(shard_ranges[1])[0] for conf_index in self.configs['container-sharder'].keys(): self.run_custom_sharder(conf_index, {'reclaim_age': 0}, override_partitions=[shard_1_part]) self.sharders.once(additional_args='--partitions=%s' % self.brain.part) # then run sharder on the shard node to shrink it to root - note this # moves alpha to the root db self.run_sharders(shard_ranges[1]) # now there are no active shards self.assertFalse(self.get_container_shard_ranges()) # force some misplaced object updates into second shrunk shard range merge_object(shard_ranges[1], 'alpha', deleted=1) misplaced_node = merge_object(shard_ranges[1], 'beta', deleted=0) # root is not yet aware of them self.assert_container_listing(['alpha']) self.assert_container_object_count(1) # until sharder runs on that node to move the misplaced object shard_part, shard_nodes_numbers = self.get_part_and_node_numbers( shard_ranges[1]) self.sharders.once(additional_args='--partitions=%s' % shard_part, number=misplaced_node['id'] + 1) self.assert_container_listing(['beta']) self.assert_container_object_count(1) self.assert_container_delete_fails() def test_misplaced_object_movement_from_deleted_shard(self): def merge_object(shard_range, name, deleted=0): # it's hard to get a test to put a misplaced object into a shard, # so this hack is used force an object record directly into a shard # container db. Note: the actual object won't exist, we're just # using this to test object records in container dbs. shard_part, shard_nodes = self.brain.ring.get_nodes( shard_range.account, shard_range.container) shard_broker = self.get_shard_broker(shard_range) # In this test we want to merge into a deleted container shard shard_broker.delete_db(Timestamp.now().internal) shard_broker.merge_items( [{'name': name, 'created_at': Timestamp.now().internal, 'size': 0, 'content_type': 'text/plain', 'etag': md5(usedforsecurity=False).hexdigest(), 'deleted': deleted, 'storage_policy_index': shard_broker.storage_policy_index}]) return shard_nodes[0] all_obj_names = self._make_object_names(self.max_shard_size) self.put_objects(all_obj_names) # Shard the container client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) for n in self.brain.node_numbers: self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity checks self.assert_container_states('sharded', 2) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') self.assert_container_listing(all_obj_names) # delete all objects in first shard range - updates redirected to shard shard_ranges = self.get_container_shard_ranges() self.assertLengthEqual(shard_ranges, 2) shard_0_objects = [name for name in all_obj_names if name in shard_ranges[0]] shard_1_objects = [name for name in all_obj_names if name in shard_ranges[1]] self.delete_objects(shard_0_objects) self.assert_container_listing(shard_1_objects) self.assert_container_post_ok('has objects') # run sharder on first shard container to update root stats shard_0_part = self.get_part_and_node_numbers(shard_ranges[0])[0] for conf_index in self.configs['container-sharder'].keys(): self.run_custom_sharder(conf_index, {'reclaim_age': 0}, override_partitions=[shard_0_part]) self.assert_container_object_count(len(shard_1_objects)) # First, test a misplaced object moving from one shard to another. # run sharder on root to discover first shrink candidate self.sharders.once(additional_args='--partitions=%s' % self.brain.part) # then run sharder on first shard range to shrink it self.run_sharders(shard_ranges[0]) # force a misplaced object into the shrunken shard range to simulate # a client put that was in flight when it started to shrink misplaced_node = merge_object(shard_ranges[0], 'alpha', deleted=0) # root sees first shard has shrunk, only second shard range used for # listing so alpha object not in listing self.assertLengthEqual(self.get_container_shard_ranges(), 1) self.assert_container_listing(shard_1_objects) self.assert_container_object_count(len(shard_1_objects)) # until sharder runs on that node to move the misplaced object to the # second shard range shard_part, shard_nodes_numbers = self.get_part_and_node_numbers( shard_ranges[0]) self.sharders.once(additional_args='--partitions=%s' % shard_part, number=misplaced_node['id'] + 1) self.assert_container_listing(['alpha'] + shard_1_objects) # root not yet updated self.assert_container_object_count(len(shard_1_objects)) # check the deleted shard did not push the wrong root path into the # other container for replica in 0, 1, 2: shard_x_broker = self.get_shard_broker(shard_ranges[1], replica) self.assertEqual("%s/%s" % (self.account, self.container_name), shard_x_broker.root_path) # run the sharder of the existing shard to update the root stats # to prove the misplaced object was moved to the other shard _and_ # the other shard still has the correct root because it updates root's # stats self.run_sharders(shard_ranges[1]) self.assert_container_object_count(len(shard_1_objects) + 1) def test_replication_to_sharded_container_from_unsharded_old_primary(self): primary_ids = [n['id'] for n in self.brain.nodes] handoff_node = next(n for n in self.brain.ring.devs if n['id'] not in primary_ids) # start with two sharded replicas and one unsharded with extra object obj_names = self._setup_replication_scenario(2) for node in self.brain.nodes[:2]: self.assert_container_state(node, 'sharded', 2) # Fake a ring change - copy unsharded db which has no shard ranges to a # handoff to create illusion of a new unpopulated primary node node_numbers = self.brain.node_numbers new_primary_node = self.brain.nodes[2] new_primary_node_number = node_numbers[2] new_primary_dir, container_hash = self.get_storage_dir( self.brain.part, new_primary_node) old_primary_dir, container_hash = self.get_storage_dir( self.brain.part, handoff_node) utils.mkdirs(os.path.dirname(old_primary_dir)) shutil.move(new_primary_dir, old_primary_dir) # make the cluster more or less "healthy" again self.brain.servers.start(number=new_primary_node_number) # get a db on every node... client.put_container(self.url, self.token, self.container_name) self.assertTrue(os.path.exists(os.path.join( new_primary_dir, container_hash + '.db'))) found = self.categorize_container_dir_content() self.assertLengthEqual(found['normal_dbs'], 1) # "new" primary self.assertLengthEqual(found['shard_dbs'], 2) # existing primaries # catastrophic failure! drive dies and is replaced on unchanged primary failed_node = self.brain.nodes[0] failed_dir, _container_hash = self.get_storage_dir( self.brain.part, failed_node) shutil.rmtree(failed_dir) # replicate the "old primary" to everybody except the "new primary" self.brain.servers.stop(number=new_primary_node_number) self.replicators.once(number=handoff_node['id'] + 1) # We're willing to rsync the retiring db to the failed primary. # This may or may not have shard ranges, depending on the order in # which we hit the primaries, but it definitely *doesn't* have an # epoch in its name yet. All objects are replicated. self.assertTrue(os.path.exists(os.path.join( failed_dir, container_hash + '.db'))) self.assertLengthEqual(os.listdir(failed_dir), 1) broker = self.get_broker(self.brain.part, failed_node) self.assertLengthEqual(broker.get_objects(), len(obj_names) + 1) # The other out-of-date primary is within usync range but objects are # not replicated to it because the handoff db learns about shard ranges broker = self.get_broker(self.brain.part, self.brain.nodes[1]) self.assertLengthEqual(broker.get_objects(), 0) # Handoff db still exists and now has shard ranges! self.assertTrue(os.path.exists(os.path.join( old_primary_dir, container_hash + '.db'))) broker = self.get_broker(self.brain.part, handoff_node) shard_ranges = broker.get_shard_ranges() self.assertLengthEqual(shard_ranges, 2) self.assert_container_state(handoff_node, 'unsharded', 2) # Replicate again, this time *including* "new primary" self.brain.servers.start(number=new_primary_node_number) self.replicators.once(number=handoff_node['id'] + 1) # Ordinarily, we would have rsync_then_merge'd to "new primary" # but instead we wait broker = self.get_broker(self.brain.part, new_primary_node) self.assertLengthEqual(broker.get_objects(), 0) shard_ranges = broker.get_shard_ranges() self.assertLengthEqual(shard_ranges, 2) # so the next time the sharder comes along, it can push rows out # and delete the big db self.sharders.once(number=handoff_node['id'] + 1, additional_args='--partitions=%s' % self.brain.part) self.assert_container_state(handoff_node, 'sharded', 2) self.assertFalse(os.path.exists(os.path.join( old_primary_dir, container_hash + '.db'))) # the sharded db hangs around until replication confirms durability # first attempt is not sufficiently successful self.brain.servers.stop(number=node_numbers[0]) self.replicators.once(number=handoff_node['id'] + 1) self.assertTrue(os.path.exists(old_primary_dir)) self.assert_container_state(handoff_node, 'sharded', 2) # second attempt is successful and handoff db is deleted self.brain.servers.start(number=node_numbers[0]) self.replicators.once(number=handoff_node['id'] + 1) self.assertFalse(os.path.exists(old_primary_dir)) # run all the sharders, get us into a consistent state self.sharders.once(additional_args='--partitions=%s' % self.brain.part) self.assert_container_listing(['alpha'] + obj_names) def test_replication_to_empty_new_primary_from_sharding_old_primary(self): primary_ids = [n['id'] for n in self.brain.nodes] handoff_node = next(n for n in self.brain.ring.devs if n['id'] not in primary_ids) num_shards = 3 obj_names = self._make_object_names( num_shards * self.max_shard_size // 2) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # run replicators first time to get sync points set self.replicators.once() # start sharding on only the leader node leader_node = self.brain.nodes[0] leader_node_number = self.brain.node_numbers[0] self.sharders.once(number=leader_node_number) self.assert_container_state(leader_node, 'sharding', 3) for node in self.brain.nodes[1:]: self.assert_container_state(node, 'unsharded', 3) # Fake a ring change - copy leader node db to a handoff to create # illusion of a new unpopulated primary leader node new_primary_dir, container_hash = self.get_storage_dir( self.brain.part, leader_node) old_primary_dir, container_hash = self.get_storage_dir( self.brain.part, handoff_node) utils.mkdirs(os.path.dirname(old_primary_dir)) shutil.move(new_primary_dir, old_primary_dir) self.assert_container_state(handoff_node, 'sharding', 3) # run replicator on handoff node to create a fresh db on new primary self.assertFalse(os.path.exists(new_primary_dir)) self.replicators.once(number=handoff_node['id'] + 1) self.assertTrue(os.path.exists(new_primary_dir)) self.assert_container_state(leader_node, 'sharded', 3) broker = self.get_broker(self.brain.part, leader_node) shard_ranges = broker.get_shard_ranges() self.assertLengthEqual(shard_ranges, 3) self.assertEqual( [ShardRange.CLEAVED, ShardRange.CLEAVED, ShardRange.CREATED], [sr.state for sr in shard_ranges]) # db still exists on handoff self.assertTrue(os.path.exists(old_primary_dir)) self.assert_container_state(handoff_node, 'sharding', 3) # continue sharding it... self.sharders.once(number=handoff_node['id'] + 1) self.assert_container_state(leader_node, 'sharded', 3) # now handoff is fully sharded the replicator will delete it self.replicators.once(number=handoff_node['id'] + 1) self.assertFalse(os.path.exists(old_primary_dir)) # all primaries now have active shard ranges but only one is in sharded # state self.assert_container_state(leader_node, 'sharded', 3) for node in self.brain.nodes[1:]: self.assert_container_state(node, 'unsharded', 3) node_data = self.direct_get_container_shard_ranges() for node_id, (hdrs, shard_ranges) in node_data.items(): with annotate_failure( 'node id %s from %s' % (node_id, node_data.keys)): self.assert_shard_range_state(ShardRange.ACTIVE, shard_ranges) # check handoff cleaved all objects before it was deleted - stop all # but leader node so that listing is fetched from shards for number in self.brain.node_numbers[1:3]: self.brain.servers.stop(number=number) self.assert_container_listing(obj_names) for number in self.brain.node_numbers[1:3]: self.brain.servers.start(number=number) self.sharders.once() self.assert_container_state(leader_node, 'sharded', 3) for node in self.brain.nodes[1:]: self.assert_container_state(node, 'sharding', 3) self.sharders.once() self.assert_container_states('sharded', 3) self.assert_container_listing(obj_names) def test_sharded_account_updates(self): # verify that .shards account updates have zero object count and bytes # to avoid double accounting all_obj_names = self._make_object_names(self.max_shard_size) self.put_objects(all_obj_names, contents='xyz') # Shard the container into 2 shards client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) for n in self.brain.node_numbers: self.sharders.once( number=n, additional_args='--partitions=%s' % self.brain.part) # sanity checks for node in self.brain.nodes: shard_ranges = self.assert_container_state(node, 'sharded', 2) self.assert_container_delete_fails() self.assert_container_has_shard_sysmeta() self.assert_container_post_ok('sharded') self.assert_container_listing(all_obj_names) # run the updaters to get account stats updated self.updaters.once() # check user account stats metadata = self.internal_client.get_account_metadata(self.account) self.assertEqual(1, int(metadata.get('x-account-container-count'))) self.assertEqual(self.max_shard_size, int(metadata.get('x-account-object-count'))) self.assertEqual(3 * self.max_shard_size, int(metadata.get('x-account-bytes-used'))) # check hidden .shards account stats metadata = self.internal_client.get_account_metadata( shard_ranges[0].account) self.assertEqual(2, int(metadata.get('x-account-container-count'))) self.assertEqual(0, int(metadata.get('x-account-object-count'))) self.assertEqual(0, int(metadata.get('x-account-bytes-used'))) class TestShardedAPI(BaseTestContainerSharding): def _assert_namespace_equivalence( self, namespaces_list, other_namespaces_list): # verify given lists are equivalent when cast to Namespaces self.assertEqual(len(namespaces_list), len(other_namespaces_list)) self.assertEqual( [Namespace(sr.name, sr.lower, sr.upper) for sr in namespaces_list], [Namespace(sr.name, sr.lower, sr.upper) for sr in other_namespaces_list]) def test_GET(self): all_obj_names = self._make_object_names(10) self.put_objects(all_obj_names) # unsharded container objs = self.get_container_objects() self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) objs = self.get_container_objects( headers={'X-Backend-Record-Type': 'auto'}) self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) objs = self.get_container_objects( headers={'X-Backend-Record-Type': 'object'}) self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) objs = self.get_container_objects( headers={'X-Backend-Record-Type': 'banana'}) self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) shard_ranges = self.get_container_shard_ranges() self.assertFalse(shard_ranges) # Shard the container client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, self.brain.nodes[0]), 'find_and_replace', '5', '--enable', '--minimum-shard-size', '5']) self.replicators.once() # "Run container-sharder on all nodes to shard the container." # first pass cleaves 2 shards self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) # sanity check self.assert_container_states('sharded', 2) orig_shard_ranges = self.get_container_shard_ranges() self.assertEqual(2, len(orig_shard_ranges)) namespaces = self.get_container_namespaces() self._assert_namespace_equivalence(orig_shard_ranges, namespaces) # the container is sharded so *all* shard ranges should satisfy # updating and listing state aliases shard_ranges = self.get_container_shard_ranges( params={'states': 'updating'}) self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) shard_ranges = self.get_container_shard_ranges( params={'states': 'listing'}) self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) shard_ranges = self.get_container_shard_ranges( headers={'X-Newest': 'true'}, params={'states': 'listing'}) self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) # this is what the sharder requests... shard_ranges = self.get_container_shard_ranges( headers={'X-Newest': 'true'}, params={'states': 'auditing'}) own_ns = Namespace('%s/%s' % (self.account, self.container_name), lower='', upper='') self._assert_namespace_equivalence(orig_shard_ranges + [own_ns], shard_ranges) shard_ranges = self.get_container_shard_ranges( params={'includes': all_obj_names[1]}) self._assert_namespace_equivalence(orig_shard_ranges[:1], shard_ranges) namespaces = self.get_container_namespaces( params={'includes': all_obj_names[1]}) self._assert_namespace_equivalence(shard_ranges, namespaces) shard_ranges = self.get_container_shard_ranges( # override 'includes' headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'}, params={'includes': all_obj_names[1]}) self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) namespaces = self.get_container_namespaces( # override 'includes' headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'}, params={'includes': all_obj_names[1]}) self._assert_namespace_equivalence(shard_ranges, namespaces) shard_ranges = self.get_container_shard_ranges( params={'end_marker': all_obj_names[1]}) self._assert_namespace_equivalence(orig_shard_ranges[:1], shard_ranges) namespaces = self.get_container_namespaces( params={'end_marker': all_obj_names[1]}) self._assert_namespace_equivalence(shard_ranges, namespaces) shard_ranges = self.get_container_shard_ranges( # override 'end_marker' headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'}, params={'end_marker': all_obj_names[1]}) self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) namespaces = self.get_container_namespaces( # override 'end_marker' headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'}, params={'end_marker': all_obj_names[1]}) self._assert_namespace_equivalence(shard_ranges, namespaces) shard_ranges = self.get_container_shard_ranges( params={'reverse': 'true'}) self._assert_namespace_equivalence(list(reversed(orig_shard_ranges)), shard_ranges) namespaces = self.get_container_namespaces( params={'reverse': 'true'}) self._assert_namespace_equivalence(shard_ranges, namespaces) shard_ranges = self.get_container_shard_ranges( # override 'reverse' headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'}, params={'reverse': 'true'}) self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges) namespaces = self.get_container_namespaces( # override 'reverse' headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'}, params={'reverse': 'true'}) self._assert_namespace_equivalence(shard_ranges, namespaces) objs = self.get_container_objects() self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) objs = self.get_container_objects( headers={'X-Newest': 'true'}) self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) objs = self.get_container_objects( headers={'X-Backend-Record-Type': 'auto'}) self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) objs = self.get_container_objects( headers={'X-Backend-Record-Type': 'banana'}) self.assertEqual(all_obj_names, [obj['name'] for obj in objs]) # note: explicitly asking for the root object rows, but it has None objs = self.get_container_objects( headers={'X-Backend-Record-Type': 'object'}) self.assertEqual([], objs) class TestContainerShardingMoreUTF8(TestContainerSharding): def _make_object_names(self, number, start=0): # override default with names that include non-ascii chars name_length = self.cluster_info['swift']['max_object_name_length'] obj_names = [] for x in range(start, start + number): name = (u'obj-\u00e4\u00ea\u00ec\u00f2\u00fb-%04d' % x) name = name.encode('utf8').ljust(name_length, b'o') if not six.PY2: name = name.decode('utf8') obj_names.append(name) return obj_names def _setup_container_name(self): # override default with max length name that includes non-ascii chars super(TestContainerShardingMoreUTF8, self)._setup_container_name() name_length = self.cluster_info['swift']['max_container_name_length'] cont_name = \ self.container_name + u'-\u00e4\u00ea\u00ec\u00f2\u00fb\u1234' self.container_name = cont_name.encode('utf8').ljust(name_length, b'x') if not six.PY2: self.container_name = self.container_name.decode('utf8') class TestManagedContainerSharding(BaseTestContainerSharding): """Test sharding using swift-manage-shard-ranges""" def test_manage_shard_ranges(self): obj_names = self._make_object_names(10) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # run replicators first time to get sync points set self.replicators.once() # sanity check: we don't have nearly enough objects for this to shard # automatically self.sharders_once_non_auto( number=self.brain.node_numbers[0], additional_args='--partitions=%s' % self.brain.part) self.assert_container_state(self.brain.nodes[0], 'unsharded', 0) self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, self.brain.nodes[0]), 'find_and_replace', '3', '--enable', '--minimum-shard-size', '2']) self.assert_container_state(self.brain.nodes[0], 'unsharded', 3) # "Run container-replicator to replicate them to other nodes." self.replicators.once() # "Run container-sharder on all nodes to shard the container." # first pass cleaves 2 shards self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) self.assert_container_state(self.brain.nodes[0], 'sharding', 3) self.assert_container_state(self.brain.nodes[1], 'sharding', 3) shard_ranges = self.assert_container_state( self.brain.nodes[2], 'sharding', 3) self.assert_container_listing(obj_names) # make the un-cleaved shard update the root container... self.assertEqual([3, 3, 4], [sr.object_count for sr in shard_ranges]) shard_part, nodes = self.get_part_and_node_numbers(shard_ranges[2]) self.sharders_once_non_auto( additional_args='--partitions=%s' % shard_part) shard_ranges = self.assert_container_state( self.brain.nodes[2], 'sharding', 3) # ...it does not report zero-stats despite being empty, because it has # not yet reached CLEAVED state self.assertEqual([3, 3, 4], [sr.object_count for sr in shard_ranges]) # second pass cleaves final shard self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) # Everybody's settled self.assert_container_state(self.brain.nodes[0], 'sharded', 3) self.assert_container_state(self.brain.nodes[1], 'sharded', 3) shard_ranges = self.assert_container_state( self.brain.nodes[2], 'sharded', 3) self.assertEqual([3, 3, 4], [sr.object_count for sr in shard_ranges]) self.assert_container_listing(obj_names) def test_manage_shard_ranges_compact(self): # verify shard range compaction using swift-manage-shard-ranges obj_names = self._make_object_names(8) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # run replicators first time to get sync points set, and get container # sharded into 4 shards self.replicators.once() self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, self.brain.nodes[0]), 'find_and_replace', '2', '--enable']) self.assert_container_state(self.brain.nodes[0], 'unsharded', 4) self.replicators.once() # run sharders twice to cleave all 4 shard ranges self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) self.assert_container_states('sharded', 4) self.assert_container_listing(obj_names) # now compact some ranges; use --max-shrinking to allow 2 shrinking # shards self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, self.brain.nodes[0]), 'compact', '--max-expanding', '1', '--max-shrinking', '2', '--yes']) shard_ranges = self.assert_container_state( self.brain.nodes[0], 'sharded', 4) self.assertEqual([ShardRange.SHRINKING] * 2 + [ShardRange.ACTIVE] * 2, [sr.state for sr in shard_ranges]) self.replicators.once() self.sharders_once_non_auto() # check there's now just 2 remaining shard ranges shard_ranges = self.assert_container_state( self.brain.nodes[0], 'sharded', 2) self.assertEqual([ShardRange.ACTIVE] * 2, [sr.state for sr in shard_ranges]) self.assert_container_listing(obj_names, req_hdrs={'X-Newest': 'True'}) # root container own shard range should still be SHARDED for i, node in enumerate(self.brain.nodes): with annotate_failure('node[%d]' % i): broker = self.get_broker(self.brain.part, self.brain.nodes[0]) self.assertEqual(ShardRange.SHARDED, broker.get_own_shard_range().state) # now compact the final two shard ranges to the root; use # --max-shrinking to allow 2 shrinking shards self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, self.brain.nodes[0]), 'compact', '--yes', '--max-shrinking', '2']) shard_ranges = self.assert_container_state( self.brain.nodes[0], 'sharded', 2) self.assertEqual([ShardRange.SHRINKING] * 2, [sr.state for sr in shard_ranges]) self.replicators.once() self.sharders_once_non_auto() self.assert_container_state(self.brain.nodes[0], 'collapsed', 0) self.assert_container_listing(obj_names, req_hdrs={'X-Newest': 'True'}) # root container own shard range should now be ACTIVE for i, node in enumerate(self.brain.nodes): with annotate_failure('node[%d]' % i): broker = self.get_broker(self.brain.part, self.brain.nodes[0]) self.assertEqual(ShardRange.ACTIVE, broker.get_own_shard_range().state) def test_manage_shard_ranges_repair_root(self): # provoke overlaps in root container and repair obj_names = self._make_object_names(16) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # run replicators first time to get sync points set self.replicators.once() # find 4 shard ranges on nodes[0] - let's denote these ranges 0.0, 0.1, # 0.2 and 0.3 that are installed with epoch_0 self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, self.brain.nodes[0]), 'find_and_replace', '4', '--enable']) shard_ranges_0 = self.assert_container_state(self.brain.nodes[0], 'unsharded', 4) # *Also* go find 3 shard ranges on *another node*, like a dumb-dumb - # let's denote these ranges 1.0, 1.1 and 1.2 that are installed with # epoch_1 self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, self.brain.nodes[1]), 'find_and_replace', '7', '--enable']) shard_ranges_1 = self.assert_container_state(self.brain.nodes[1], 'unsharded', 3) # Run sharder in specific order so that the replica with the older # epoch_0 starts sharding first - this will prove problematic later! # On first pass the first replica passes audit, creates shards and then # syncs shard ranges with the other replicas, so it has a mix of 0.* # shard ranges in CLEAVED state and 1.* ranges in FOUND state. It # proceeds to cleave shard 0.0, but after 0.0 cleaving stalls because # next in iteration is shard range 1.0 in FOUND state from the other # replica that it cannot yet cleave. self.sharders_once_non_auto( number=self.brain.node_numbers[0], additional_args='--partitions=%s' % self.brain.part) # On first pass the second replica passes audit (it has its own found # ranges and the first replica's created shard ranges but none in the # same state overlap), creates its shards and then syncs shard ranges # with the other replicas. All of the 7 shard ranges on this replica # are now in CREATED state so it proceeds to cleave the first two shard # ranges, 0.1 and 1.0. self.sharders_once_non_auto( number=self.brain.node_numbers[1], additional_args='--partitions=%s' % self.brain.part) self.replicators.once() # Uh-oh self.assert_container_state(self.brain.nodes[0], 'sharding', 7) self.assert_container_state(self.brain.nodes[1], 'sharding', 7) # There's a race: the third replica may be sharding, may be unsharded # Try it again a few times self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) self.replicators.once() self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) # It's not really fixing itself... the sharder audit will detect # overlapping ranges which prevents cleaving proceeding; expect the # shard ranges to be mostly still in created state, with one or two # possibly cleaved during first pass before the sharding got stalled shard_ranges = self.assert_container_state(self.brain.nodes[0], 'sharding', 7) self.assertEqual([ShardRange.CLEAVED] * 2 + [ShardRange.CREATED] * 5, [sr.state for sr in shard_ranges]) shard_ranges = self.assert_container_state(self.brain.nodes[1], 'sharding', 7) self.assertEqual([ShardRange.CLEAVED] * 2 + [ShardRange.CREATED] * 5, [sr.state for sr in shard_ranges]) # But hey, at least listings still work! They're just going to get # horribly out of date as more objects are added self.assert_container_listing(obj_names) # 'swift-manage-shard-ranges repair' will choose the second set of 3 # shard ranges (1.*) over the first set of 4 (0.*) because that's the # path with most cleaving progress, and so shrink shard ranges 0.*. db_file = self.get_db_file(self.brain.part, self.brain.nodes[0]) self.assert_subprocess_success( ['swift-manage-shard-ranges', db_file, 'repair', '--yes', '--min-shard-age', '0']) # make sure all root replicas now sync their shard ranges self.replicators.once() # Run sharder on the shrinking shards. This should not change the state # of any of the acceptors, particularly the ones that have yet to have # object cleaved from the roots, because we don't want the as yet # uncleaved acceptors becoming prematurely active and creating 'holes' # in listings. The shrinking shard ranges should however get deleted in # root container table. self.run_sharders(shard_ranges_0) shard_ranges = self.assert_container_state(self.brain.nodes[1], 'sharding', 3) self.assertEqual([ShardRange.CLEAVED] * 1 + [ShardRange.CREATED] * 2, [sr.state for sr in shard_ranges]) self.assert_container_listing(obj_names) # check the unwanted shards did shrink away... for shard_range in shard_ranges_0: with annotate_failure(shard_range): found_for_shard = self.categorize_container_dir_content( shard_range.account, shard_range.container) self.assertLengthEqual(found_for_shard['shard_dbs'], 3) actual = [] for shard_db in found_for_shard['shard_dbs']: broker = ContainerBroker(shard_db) own_sr = broker.get_own_shard_range() actual.append( (broker.get_db_state(), own_sr.state, own_sr.deleted)) self.assertEqual([(SHARDED, ShardRange.SHRUNK, True)] * 3, actual) # At this point one of the first two replicas may have done some useful # cleaving of 1.* shards, the other may have only cleaved 0.* shards, # and the third replica may have cleaved no shards. We therefore need # two more passes of the sharder to get to a predictable state where # all replicas have cleaved all three 0.* shards. self.sharders_once_non_auto() self.sharders_once_non_auto() # now we expect all replicas to have just the three 1.* shards, with # the 0.* shards all deleted brokers = {} exp_shard_ranges = sorted( [sr.copy(state=ShardRange.SHRUNK, deleted=True) for sr in shard_ranges_0] + [sr.copy(state=ShardRange.ACTIVE) for sr in shard_ranges_1], key=ShardRange.sort_key) for node in (0, 1, 2): with annotate_failure('node %s' % node): broker = self.get_broker(self.brain.part, self.brain.nodes[node]) brokers[node] = broker shard_ranges = broker.get_shard_ranges() self.assertEqual(shard_ranges_1, shard_ranges) shard_ranges = broker.get_shard_ranges(include_deleted=True) self.assertLengthEqual(shard_ranges, len(exp_shard_ranges)) self.maxDiff = None self.assertEqual(exp_shard_ranges, shard_ranges) self.assertEqual(ShardRange.SHARDED, broker.get_own_shard_range().state) # Sadly, the first replica to start sharding is still reporting its db # state to be 'unsharded' because, although it has sharded, its shard # db epoch (epoch_0) does not match its own shard range epoch # (epoch_1), and that is because the second replica (with epoch_1) # updated the own shard range and replicated it to all other replicas. # If we had run the sharder on the second replica before the first # replica, then by the time the first replica started sharding it would # have learnt the newer epoch_1 and we wouldn't see this inconsistency. self.assertEqual(UNSHARDED, brokers[0].get_db_state()) self.assertEqual(SHARDED, brokers[1].get_db_state()) self.assertEqual(SHARDED, brokers[2].get_db_state()) epoch_1 = brokers[1].db_epoch self.assertEqual(epoch_1, brokers[2].db_epoch) self.assertLess(brokers[0].db_epoch, epoch_1) # the root replica that thinks it is unsharded is problematic - it will # not return shard ranges for listings, but has no objects, so it's # luck of the draw whether we get a listing or not at this point :( # Run the sharders again: the first replica that is still 'unsharded' # because of the older epoch_0 in its db filename will now start to # shard again with a newer epoch_1 db, and will start to re-cleave the # 3 active shards, albeit with zero objects to cleave. self.sharders_once_non_auto() for node in (0, 1, 2): with annotate_failure('node %s' % node): broker = self.get_broker(self.brain.part, self.brain.nodes[node]) brokers[node] = broker shard_ranges = broker.get_shard_ranges() self.assertEqual(shard_ranges_1, shard_ranges) shard_ranges = broker.get_shard_ranges(include_deleted=True) self.assertLengthEqual(shard_ranges, len(exp_shard_ranges)) self.assertEqual(exp_shard_ranges, shard_ranges) self.assertEqual(ShardRange.SHARDED, broker.get_own_shard_range().state) self.assertEqual(epoch_1, broker.db_epoch) self.assertIn(brokers[0].get_db_state(), (SHARDING, SHARDED)) self.assertEqual(SHARDED, brokers[1].get_db_state()) self.assertEqual(SHARDED, brokers[2].get_db_state()) # This cycle of the sharders also guarantees that all shards have had # their state updated to ACTIVE from the root; this was not necessarily # true at end of the previous sharder pass because a shard audit (when # the shard is updated from a root) may have happened before all roots # have had their shard ranges transitioned to ACTIVE. for shard_range in shard_ranges_1: with annotate_failure(shard_range): found_for_shard = self.categorize_container_dir_content( shard_range.account, shard_range.container) self.assertLengthEqual(found_for_shard['normal_dbs'], 3) actual = [] for shard_db in found_for_shard['normal_dbs']: broker = ContainerBroker(shard_db) own_sr = broker.get_own_shard_range() actual.append( (broker.get_db_state(), own_sr.state, own_sr.deleted)) self.assertEqual([(UNSHARDED, ShardRange.ACTIVE, False)] * 3, actual) # We may need one more pass of the sharder before all three shard # ranges are cleaved (2 per pass) and all the root replicas are # predictably in sharded state. Note: the accelerated cleaving of >2 # zero-object shard ranges per cycle is defeated if a shard happens # to exist on the same node as the root because the roots cleaving # process doesn't think that it created the shard db and will therefore # replicate it as per a normal cleave. self.sharders_once_non_auto() for node in (0, 1, 2): with annotate_failure('node %s' % node): broker = self.get_broker(self.brain.part, self.brain.nodes[node]) brokers[node] = broker shard_ranges = broker.get_shard_ranges() self.assertEqual(shard_ranges_1, shard_ranges) shard_ranges = broker.get_shard_ranges(include_deleted=True) self.assertLengthEqual(shard_ranges, len(exp_shard_ranges)) self.assertEqual(exp_shard_ranges, shard_ranges) self.assertEqual(ShardRange.SHARDED, broker.get_own_shard_range().state) self.assertEqual(epoch_1, broker.db_epoch) self.assertEqual(SHARDED, broker.get_db_state()) # Finally, with all root replicas in a consistent state, the listing # will be be predictably correct self.assert_container_listing(obj_names) def test_manage_shard_ranges_repair_shard(self): # provoke overlaps in a shard container and repair them obj_names = self._make_object_names(24) initial_obj_names = obj_names[::2] # put 12 objects in container self.put_objects(initial_obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # run replicators first time to get sync points set self.replicators.once() # find 3 shard ranges on root nodes[0] and get the root sharded self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, self.brain.nodes[0]), 'find_and_replace', '4', '--enable']) self.replicators.once() # cleave first two shards self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) # cleave third shard self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) # ensure all shards learn their ACTIVE state from root self.sharders_once_non_auto() for node in (0, 1, 2): with annotate_failure('node %d' % node): shard_ranges = self.assert_container_state( self.brain.nodes[node], 'sharded', 3) for sr in shard_ranges: self.assertEqual(ShardRange.ACTIVE, sr.state) self.assert_container_listing(initial_obj_names) # add objects to second shard range so it has 8 objects ; this range # has bounds (obj-0006,obj-0014] root_shard_ranges = self.get_container_shard_ranges() self.assertEqual(3, len(root_shard_ranges)) shard_1 = root_shard_ranges[1] self.assertEqual(obj_names[6], shard_1.lower) self.assertEqual(obj_names[14], shard_1.upper) more_obj_names = obj_names[7:15:2] self.put_objects(more_obj_names) expected_obj_names = sorted(initial_obj_names + more_obj_names) self.assert_container_listing(expected_obj_names) shard_1_part, shard_1_nodes = self.brain.ring.get_nodes( shard_1.account, shard_1.container) # find 3 sub-shards on one shard node; use --force-commits to ensure # the recently PUT objects are included when finding the shard range # pivot points self.assert_subprocess_success([ 'swift-manage-shard-ranges', '--force-commits', self.get_db_file(shard_1_part, shard_1_nodes[1], shard_1.account, shard_1.container), 'find_and_replace', '3', '--enable']) # ... and mistakenly find 4 shard ranges on a different shard node :( self.assert_subprocess_success([ 'swift-manage-shard-ranges', '--force-commits', self.get_db_file(shard_1_part, shard_1_nodes[2], shard_1.account, shard_1.container), 'find_and_replace', '2', '--enable']) # replicate the muddle of shard ranges between shard replicas, merged # result is: # '' - 6 shard ACTIVE # 6 - 8 sub-shard FOUND # 6 - 9 sub-shard FOUND # 8 - 10 sub-shard FOUND # 9 - 12 sub-shard FOUND # 10 - 12 sub-shard FOUND # 12 - 14 sub-shard FOUND # 12 - 14 sub-shard FOUND # 6 - 14 shard SHARDING # 14 - '' shard ACTIVE self.replicators.once() # try hard to shard the shard... self.sharders_once_non_auto( additional_args='--partitions=%s' % shard_1_part) self.sharders_once_non_auto( additional_args='--partitions=%s' % shard_1_part) self.sharders_once_non_auto( additional_args='--partitions=%s' % shard_1_part) # sharding hasn't completed and there's overlaps in the shard and root: # the sub-shards will have been cleaved in the order listed above, but # sub-shards (10 -12) and one of (12 - 14) will be overlooked because # the cleave cursor will have moved past their namespace before they # were yielded by the shard range iterator, so we now have: # '' - 6 shard ACTIVE # 6 - 8 sub-shard ACTIVE # 6 - 9 sub-shard ACTIVE # 8 - 10 sub-shard ACTIVE # 10 - 12 sub-shard CREATED # 9 - 12 sub-shard ACTIVE # 12 - 14 sub-shard CREATED # 12 - 14 sub-shard ACTIVE # 14 - '' shard ACTIVE sub_shard_ranges = self.get_container_shard_ranges( shard_1.account, shard_1.container) self.assertEqual(7, len(sub_shard_ranges), sub_shard_ranges) root_shard_ranges = self.get_container_shard_ranges() self.assertEqual(9, len(root_shard_ranges), root_shard_ranges) self.assertEqual([ShardRange.ACTIVE] * 4 + [ShardRange.CREATED, ShardRange.ACTIVE] * 2 + [ShardRange.ACTIVE], [sr.state for sr in root_shard_ranges]) # fix the overlaps - a set of 3 ACTIVE sub-shards will be chosen and 4 # other sub-shards will be shrunk away; apply the fix at the root # container db_file = self.get_db_file(self.brain.part, self.brain.nodes[0]) self.assert_subprocess_success( ['swift-manage-shard-ranges', db_file, 'repair', '--yes', '--min-shard-age', '0']) self.replicators.once() self.sharders_once_non_auto() self.sharders_once_non_auto() # check root now has just 5 shard ranges root_shard_ranges = self.get_container_shard_ranges() self.assertEqual(5, len(root_shard_ranges), root_shard_ranges) self.assertEqual([ShardRange.ACTIVE] * 5, [sr.state for sr in root_shard_ranges]) # check there are 1 sharded shard and 4 shrunk sub-shard ranges in the # root (note, shard_1's shard ranges aren't updated once it has sharded # because the sub-shards report their state to the root; we cannot make # assertions about shrunk states in shard_1's shard range table) root_shard_ranges = self.get_container_shard_ranges( headers={'X-Backend-Include-Deleted': 'true'}) self.assertEqual(10, len(root_shard_ranges), root_shard_ranges) shrunk_shard_ranges = [sr for sr in root_shard_ranges if sr.state == ShardRange.SHRUNK] self.assertEqual(4, len(shrunk_shard_ranges), root_shard_ranges) self.assertEqual([True] * 4, [sr.deleted for sr in shrunk_shard_ranges]) sharded_shard_ranges = [sr for sr in root_shard_ranges if sr.state == ShardRange.SHARDED] self.assertEqual(1, len(sharded_shard_ranges), root_shard_ranges) self.assert_container_listing(expected_obj_names) def test_manage_shard_ranges_repair_parent_child_ranges(self): # Test repairing a transient parent-child shard range overlap in the # root container, expect no repairs to be done. # note: be careful not to add a container listing to this test which # would get shard ranges into memcache obj_names = self._make_object_names(4) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # run replicators first time to get sync points set self.container_replicators.once( additional_args='--partitions=%s' % self.brain.part) # shard root root_0_db_file = self.get_db_file(self.brain.part, self.brain.nodes[0]) self.assert_subprocess_success([ 'swift-manage-shard-ranges', root_0_db_file, 'find_and_replace', '2', '--enable']) self.container_replicators.once( additional_args='--partitions=%s' % self.brain.part) self.assert_container_states('unsharded', 2) self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) # get shards to update state from parent... self.sharders_once_non_auto() self.assert_container_states('sharded', 2) # sanity check, all is well msg = self.assert_subprocess_success([ 'swift-manage-shard-ranges', root_0_db_file, 'repair', '--gaps', '--dry-run']) self.assertIn(b'No repairs necessary.', msg) # shard first shard into 2 sub-shards while root node 0 is disabled self.stop_container_servers(node_numbers=slice(0, 1)) shard_ranges = self.get_container_shard_ranges() shard_brokers = [self.get_shard_broker(shard_ranges[0], node_index=i) for i in range(3)] self.assert_subprocess_success([ 'swift-manage-shard-ranges', shard_brokers[0].db_file, 'find_and_replace', '1', '--enable']) shard_part, shard_nodes = self.brain.ring.get_nodes( shard_ranges[0].account, shard_ranges[0].container) self.container_replicators.once( additional_args='--partitions=%s' % shard_part) for node in exclude_nodes(shard_nodes, self.brain.nodes[0]): self.assert_container_state( node, 'unsharded', 2, account=shard_ranges[0].account, container=shard_ranges[0].container, part=shard_part) self.sharders_once_non_auto( additional_args='--partitions=%s' % shard_part) # get shards to update state from parent... self.sharders_once_non_auto() for node in exclude_nodes(shard_nodes, self.brain.nodes[0]): self.assert_container_state( node, 'sharded', 2, account=shard_ranges[0].account, container=shard_ranges[0].container, part=shard_part) # put an object into the second of the 2 sub-shards so that the shard # will update the root next time the sharder is run; do this before # restarting root node 0 so that the object update is definitely # redirected to a sub-shard by root node 1 or 2. new_obj_name = obj_names[0] + 'a' self.put_objects([new_obj_name]) # restart root node 0 self.brain.servers.start(number=self.brain.node_numbers[0]) # node 0 DB doesn't know about the sub-shards root_brokers = [self.get_broker(self.brain.part, node) for node in self.brain.nodes] broker = root_brokers[0] self.assertEqual( [(ShardRange.ACTIVE, False, ShardRange.MIN, obj_names[1]), (ShardRange.ACTIVE, False, obj_names[1], ShardRange.MAX)], [(sr.state, sr.deleted, sr.lower, sr.upper) for sr in broker.get_shard_ranges(include_deleted=True)]) for broker in root_brokers[1:]: self.assertEqual( [(ShardRange.ACTIVE, False, ShardRange.MIN, obj_names[0]), (ShardRange.ACTIVE, False, obj_names[0], obj_names[1]), (ShardRange.SHARDED, True, ShardRange.MIN, obj_names[1]), (ShardRange.ACTIVE, False, obj_names[1], ShardRange.MAX)], [(sr.state, sr.deleted, sr.lower, sr.upper) for sr in broker.get_shard_ranges(include_deleted=True)]) sub_shard = root_brokers[1].get_shard_ranges()[1] self.assertEqual(obj_names[0], sub_shard.lower) self.assertEqual(obj_names[1], sub_shard.upper) sub_shard_part, nodes = self.get_part_and_node_numbers(sub_shard) # we want the sub-shard to update root node 0 but not the sharded # shard, but there is a small chance the two will be in same partition # TODO: how can we work around this? self.assertNotEqual(sub_shard_part, shard_part, 'You were unlucky, try again') self.sharders_once_non_auto( additional_args='--partitions=%s' % sub_shard_part) # now root node 0 has the original shards plus one of the sub-shards # but all are active :( self.assertEqual( [(ShardRange.ACTIVE, False, ShardRange.MIN, obj_names[1]), # note: overlap! (ShardRange.ACTIVE, False, obj_names[0], obj_names[1]), (ShardRange.ACTIVE, False, obj_names[1], ShardRange.MAX)], [(sr.state, sr.deleted, sr.lower, sr.upper) for sr in root_brokers[0].get_shard_ranges(include_deleted=True)]) # try to fix the overlap and expect no repair has been done. msg = self.assert_subprocess_success( ['swift-manage-shard-ranges', root_0_db_file, 'repair', '--yes', '--min-shard-age', '0']) self.assertIn( b'1 donor shards ignored due to parent-child relationship checks', msg) # verify parent-child checks has prevented repair to be done. self.assertEqual( [(ShardRange.ACTIVE, False, ShardRange.MIN, obj_names[1]), # note: overlap! (ShardRange.ACTIVE, False, obj_names[0], obj_names[1]), (ShardRange.ACTIVE, False, obj_names[1], ShardRange.MAX)], [(sr.state, sr.deleted, sr.lower, sr.upper) for sr in root_brokers[0].get_shard_ranges(include_deleted=True)]) # the transient overlap is 'fixed' in subsequent sharder cycles... self.sharders_once_non_auto() self.sharders_once_non_auto() self.container_replicators.once() for broker in root_brokers: self.assertEqual( [(ShardRange.ACTIVE, False, ShardRange.MIN, obj_names[0]), (ShardRange.ACTIVE, False, obj_names[0], obj_names[1]), (ShardRange.SHARDED, True, ShardRange.MIN, obj_names[1]), (ShardRange.ACTIVE, False, obj_names[1], ShardRange.MAX)], [(sr.state, sr.deleted, sr.lower, sr.upper) for sr in broker.get_shard_ranges(include_deleted=True)]) def test_manage_shard_ranges_repair_root_gap(self): # create a gap in root container; repair the gap. # note: be careful not to add a container listing to this test which # would get shard ranges into memcache obj_names = self._make_object_names(8) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # run replicators first time to get sync points set self.container_replicators.once( additional_args='--partitions=%s' % self.brain.part) # shard root root_0_db_file = self.get_db_file(self.brain.part, self.brain.nodes[0]) self.assert_subprocess_success([ 'swift-manage-shard-ranges', root_0_db_file, 'find_and_replace', '2', '--enable']) self.container_replicators.once( additional_args='--partitions=%s' % self.brain.part) self.assert_container_states('unsharded', 4) self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) # get shards to update state from parent... self.sharders_once_non_auto() self.assert_container_states('sharded', 4) # sanity check, all is well msg = self.assert_subprocess_success([ 'swift-manage-shard-ranges', root_0_db_file, 'repair', '--gaps', '--dry-run']) self.assertIn(b'No repairs necessary.', msg) # deliberately create a gap in root shard ranges (don't ever do this # for real) # TODO: replace direct broker modification with s-m-s-r merge root_brokers = [self.get_broker(self.brain.part, node) for node in self.brain.nodes] shard_ranges = root_brokers[0].get_shard_ranges() self.assertEqual(4, len(shard_ranges)) shard_ranges[2].set_deleted() root_brokers[0].merge_shard_ranges(shard_ranges) shard_ranges = root_brokers[0].get_shard_ranges() self.assertEqual(3, len(shard_ranges)) self.container_replicators.once() # confirm that we made a gap. for broker in root_brokers: self.assertEqual( [(ShardRange.ACTIVE, False, ShardRange.MIN, obj_names[1]), (ShardRange.ACTIVE, False, obj_names[1], obj_names[3]), (ShardRange.ACTIVE, True, obj_names[3], obj_names[5]), (ShardRange.ACTIVE, False, obj_names[5], ShardRange.MAX)], [(sr.state, sr.deleted, sr.lower, sr.upper) for sr in broker.get_shard_ranges(include_deleted=True)]) msg = self.assert_subprocess_success([ 'swift-manage-shard-ranges', root_0_db_file, 'repair', '--gaps', '--yes']) self.assertIn(b'Repairs necessary to fill gaps.', msg) self.sharders_once_non_auto() self.sharders_once_non_auto() self.container_replicators.once() # yay! we fixed the gap (without creating an overlap) for broker in root_brokers: self.assertEqual( [(ShardRange.ACTIVE, False, ShardRange.MIN, obj_names[1]), (ShardRange.ACTIVE, False, obj_names[1], obj_names[3]), (ShardRange.ACTIVE, True, obj_names[3], obj_names[5]), (ShardRange.ACTIVE, False, obj_names[3], ShardRange.MAX)], [(sr.state, sr.deleted, sr.lower, sr.upper) for sr in broker.get_shard_ranges(include_deleted=True)]) msg = self.assert_subprocess_success([ 'swift-manage-shard-ranges', root_0_db_file, 'repair', '--dry-run', '--min-shard-age', '0']) self.assertIn(b'No repairs necessary.', msg) msg = self.assert_subprocess_success([ 'swift-manage-shard-ranges', root_0_db_file, 'repair', '--gaps', '--dry-run']) self.assertIn(b'No repairs necessary.', msg) # put an object into the gap namespace new_objs = [obj_names[4] + 'a'] self.put_objects(new_objs) # get root stats up to date self.sharders_once_non_auto() # new object is in listing but old objects in the gap have been lost - # don't delete shard ranges! self.assert_container_listing(obj_names[:4] + new_objs + obj_names[6:]) def test_manage_shard_ranges_unsharded_deleted_root(self): # verify that a deleted DB will still be sharded # choose a node that will not be sharded initially sharded_nodes = [] unsharded_node = None for node in self.brain.nodes: if self.brain.node_numbers[node['index']] \ in self.brain.handoff_numbers: unsharded_node = node else: sharded_nodes.append(node) # put some objects - not enough to trigger auto-sharding obj_names = self._make_object_names(MIN_SHARD_CONTAINER_THRESHOLD - 1) self.put_objects(obj_names) # run replicators first time to get sync points set and commit updates self.replicators.once() # setup sharding... self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, sharded_nodes[0]), 'find_and_replace', '2', '--enable', '--minimum-shard-size', '1']) # Run container-replicator to replicate shard ranges self.container_replicators.once() self.assert_container_state(sharded_nodes[0], 'unsharded', 2) self.assert_container_state(sharded_nodes[1], 'unsharded', 2) self.assert_container_state(unsharded_node, 'unsharded', 2) # Run container-sharder to shard the 2 primary replicas that did # receive the object PUTs for num in self.brain.primary_numbers: self.sharders_once_non_auto( number=num, additional_args='--partitions=%s' % self.brain.part) # delete the objects - the proxy's will have cached container info with # out-of-date db_state=unsharded, so updates go to the root DBs self.delete_objects(obj_names) # deal with DELETE's being misplaced in root db's... for num in self.brain.primary_numbers: self.sharders_once_non_auto( number=num, additional_args='--partitions=%s' % self.brain.part) self.assert_container_state(sharded_nodes[0], 'sharded', 2) self.assert_container_state(sharded_nodes[1], 'sharded', 2) shard_ranges = self.assert_container_state( unsharded_node, 'unsharded', 2) # get root stats updated - but avoid sharding the remaining root DB self.run_sharders(shard_ranges, exclude_partitions=[self.brain.part]) self.assert_container_listing([]) # delete the empty container client.delete_container(self.url, self.admin_token, self.container_name) # sanity check - unsharded DB is deleted broker = self.get_broker(self.brain.part, unsharded_node, self.account, self.container_name) self.assertEqual(UNSHARDED, broker.get_db_state()) self.assertTrue(broker.is_deleted()) self.assertEqual(0, broker.get_info()['object_count']) self.assertEqual(0, broker.get_shard_usage()['object_count']) # now shard the final DB for num in self.brain.handoff_numbers: self.sharders_once_non_auto( number=num, additional_args='--partitions=%s' % self.brain.part) # all DBs should now be sharded and still deleted for node in self.brain.nodes: with annotate_failure( 'node %s in %s' % (node['index'], [n['index'] for n in self.brain.nodes])): self.assert_container_state(node, 'sharded', 2, override_deleted=True) broker = self.get_broker(self.brain.part, node, self.account, self.container_name) self.assertEqual(SHARDED, broker.get_db_state()) self.assertEqual(0, broker.get_info()['object_count']) self.assertEqual(0, broker.get_shard_usage()['object_count']) self.assertTrue(broker.is_deleted()) def test_manage_shard_ranges_unsharded_deleted_root_gets_undeleted(self): # verify that an apparently deleted DB (no object rows in root db) will # still be sharded and also become undeleted when objects are # discovered in the shards # choose a node that will not be sharded initially sharded_nodes = [] unsharded_node = None for node in self.brain.nodes: if self.brain.node_numbers[node['index']] \ in self.brain.handoff_numbers: unsharded_node = node else: sharded_nodes.append(node) # put some objects, but only to 2 replicas - not enough to trigger # auto-sharding self.brain.stop_handoff_half() obj_names = self._make_object_names(MIN_SHARD_CONTAINER_THRESHOLD - 1) self.put_objects(obj_names) # run replicators first time to get sync points set and commit puts self.replicators.once() # setup sharding... self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, sharded_nodes[0]), 'find_and_replace', '2', '--enable', '--minimum-shard-size', '1']) # Run container-replicator to replicate shard ranges - object rows will # not be sync'd now there are shard ranges for num in self.brain.primary_numbers: self.container_replicators.once(number=num) self.assert_container_state(sharded_nodes[0], 'unsharded', 2) self.assert_container_state(sharded_nodes[1], 'unsharded', 2) # revive the stopped node self.brain.start_handoff_half() self.assert_container_state(unsharded_node, 'unsharded', 0) # delete the empty replica direct_client.direct_delete_container( unsharded_node, self.brain.part, self.account, self.container_name) # Run container-sharder to shard the 2 primary replicas that did # receive the object PUTs for num in self.brain.primary_numbers: self.sharders_once_non_auto( number=num, additional_args='--partitions=%s' % self.brain.part) self.assert_container_state(sharded_nodes[0], 'sharded', 2) self.assert_container_state(sharded_nodes[1], 'sharded', 2) # the sharder syncs shard ranges ... self.assert_container_state(unsharded_node, 'unsharded', 2, override_deleted=True) # sanity check - unsharded DB is empty and deleted broker = self.get_broker(self.brain.part, unsharded_node, self.account, self.container_name) self.assertEqual(UNSHARDED, broker.get_db_state()) self.assertEqual(0, broker.get_info()['object_count']) # the shard ranges do have object count but are in CREATED state so # not reported in shard usage... self.assertEqual(0, broker.get_shard_usage()['object_count']) self.assertTrue(broker.is_deleted()) # now shard the final DB for num in self.brain.handoff_numbers: self.sharders_once_non_auto( number=num, additional_args='--partitions=%s' % self.brain.part) shard_ranges = self.assert_container_state( unsharded_node, 'sharded', 2, override_deleted=True) # and get roots updated and sync'd self.container_replicators.once() self.run_sharders(shard_ranges, exclude_partitions=[self.brain.part]) # all DBs should now be sharded and NOT deleted for node in self.brain.nodes: with annotate_failure( 'node %s in %s' % (node['index'], [n['index'] for n in self.brain.nodes])): broker = self.get_broker(self.brain.part, node, self.account, self.container_name) self.assertEqual(SHARDED, broker.get_db_state()) self.assertEqual(3, broker.get_info()['object_count']) self.assertEqual(3, broker.get_shard_usage()['object_count']) self.assertFalse(broker.is_deleted()) def test_handoff_replication_does_not_cause_reset_epoch(self): obj_names = self._make_object_names(100) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # run replicators first time to get sync points set self.replicators.once() # sanity check: we don't have nearly enough objects for this to shard # automatically self.sharders_once_non_auto( number=self.brain.node_numbers[0], additional_args='--partitions=%s' % self.brain.part) self.assert_container_state(self.brain.nodes[0], 'unsharded', 0) self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, self.brain.nodes[0]), 'find_and_replace', '50', '--enable', '--minimum-shard-size', '40']) self.assert_container_state(self.brain.nodes[0], 'unsharded', 2) # "Run container-replicator to replicate them to other nodes." self.replicators.once() # "Run container-sharder on all nodes to shard the container." self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) # Everybody's settled self.assert_container_state(self.brain.nodes[0], 'sharded', 2) self.assert_container_state(self.brain.nodes[1], 'sharded', 2) self.assert_container_state(self.brain.nodes[2], 'sharded', 2) self.assert_container_listing(obj_names) # now lets put the container again and make sure it lands on a handoff self.brain.stop_primary_half() self.brain.put_container(policy_index=int(self.policy)) self.brain.start_primary_half() dir_content = self.categorize_container_dir_content(more_nodes=True) # the handoff node is considered normal because it doesn't have an # epoch self.assertEqual(len(dir_content['normal_dbs']), 1) self.assertEqual(len(dir_content['shard_dbs']), 3) # let's replicate self.replicators.once() self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) # let's now check the handoff broker it should have all the shards handoff_broker = ContainerBroker(dir_content['normal_dbs'][0]) self.assertEqual(len(handoff_broker.get_shard_ranges()), 2) handoff_osr = handoff_broker.get_own_shard_range(no_default=True) self.assertIsNotNone(handoff_osr.epoch) def test_force_replication_of_a_reset_own_shard_range(self): obj_names = self._make_object_names(100) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # run replicators first time to get sync points set self.replicators.once() # sanity check: we don't have nearly enough objects for this to shard # automatically self.sharders_once_non_auto( number=self.brain.node_numbers[0], additional_args='--partitions=%s' % self.brain.part) self.assert_container_state(self.brain.nodes[0], 'unsharded', 0) self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, self.brain.nodes[0]), 'find_and_replace', '50', '--enable', '--minimum-shard-size', '40']) self.assert_container_state(self.brain.nodes[0], 'unsharded', 2) # "Run container-replicator to replicate them to other nodes." self.replicators.once() # "Run container-sharder on all nodes to shard the container." self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) # Everybody's settled self.assert_container_state(self.brain.nodes[0], 'sharded', 2) self.assert_container_state(self.brain.nodes[1], 'sharded', 2) self.assert_container_state(self.brain.nodes[2], 'sharded', 2) self.assert_container_listing(obj_names) # Lets delete a primary to simulate a new primary and force an # own_shard_range reset. new_primary = self.brain.nodes[2] db_file = self.get_db_file(self.brain.part, new_primary) os.remove(db_file) # issue a new PUT to create the "new" primary container self.brain.put_container(policy_index=int(self.policy)) # put a bunch of objects that should land in the primary so it'll be # shardable (in case this makes any kind of difference). self.put_objects(obj_names) # The new primary isn't considered a shard_db because it hasn't # sunk with the other primaries yet. dir_content = self.categorize_container_dir_content() self.assertEqual(len(dir_content['normal_dbs']), 1) self.assertEqual(len(dir_content['shard_dbs']), 2) # run the sharders incase this will trigger a reset osr self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) new_primary_broker = self.get_broker(self.brain.part, new_primary) # Nope, still no default/reset osr self.assertIsNone( new_primary_broker.get_own_shard_range(no_default=True)) # Let's reset the osr by hand. reset_osr = new_primary_broker.get_own_shard_range() self.assertIsNone(reset_osr.epoch) self.assertEqual(reset_osr.state, ShardRange.ACTIVE) new_primary_broker.merge_shard_ranges(reset_osr) # now let's replicate with the old primaries self.replicators.once() # Pull an old primary own_shard_range dir_content = self.categorize_container_dir_content() old_broker = ContainerBroker(dir_content['shard_dbs'][0]) old_osr = old_broker.get_own_shard_range() new_primary_broker = ContainerBroker(dir_content['normal_dbs'][0]) new_osr = new_primary_broker.get_own_shard_range() # This version stops replicating a remote non-epoch osr over a local # epoched osr. But it doesn't do the other way. So it means the # primary with non-epoched OSR get's stuck with it, if it is newer then # the other epoched versions. self.assertIsNotNone(old_osr.epoch) self.assertEqual(old_osr.state, ShardRange.SHARDED) self.assertIsNone(new_osr.epoch) self.assertGreater(new_osr.timestamp, old_osr.timestamp) def test_manage_shard_ranges_missing_epoch_no_false_positives(self): # when one replica of a shard is sharding before the others, it's epoch # is not None but it is normal for the other replica to replicate to it # sending their own shard ranges with epoch=None until they also shard obj_names = self._make_object_names(4) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # run replicators first time to get sync points set, and get container # sharded into 4 shards self.replicators.once() self.assert_subprocess_success([ 'swift-manage-shard-ranges', self.get_db_file(self.brain.part, self.brain.nodes[0]), 'find_and_replace', '2', '--enable']) ranges = self.assert_container_state( self.brain.nodes[0], 'unsharded', 2) # "Run container-replicator to replicate them to other nodes." self.replicators.once() # "Run container-sharder on all nodes to shard the container." self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) # Run them again, just so the shards themselves can pull down the # latest sharded versions of their OSRs. self.sharders_once_non_auto() # Everybody's settled self.assert_container_state(self.brain.nodes[0], 'sharded', 2) self.assert_container_state(self.brain.nodes[1], 'sharded', 2) ranges = self.assert_container_state(self.brain.nodes[2], 'sharded', 2) self.assert_container_listing(obj_names) # Now we need to shard a shard. A shard's OSR always exist and should # have an epoch of None, so we should get some false positives. # we'll shard ranges[1] which have a range of objs-0002 - MAX shard_obj_names = ['objs-0001%d' % i for i in range(2)] self.put_objects(shard_obj_names) part, shard_node_numbers = self.get_part_and_node_numbers(ranges[1]) shard_nodes = self.brain.ring.get_part_nodes(part) shard_broker = self.get_shard_broker(ranges[1], 0) # set the account, container instance variables shard_broker.get_info() self.replicators.once() self.assert_subprocess_success([ 'swift-manage-shard-ranges', shard_broker.db_file, 'find_and_replace', '2', '--enable']) self.assert_container_state( shard_nodes[0], 'unsharded', 2, shard_broker.account, shard_broker.container, part) # index 0 has an epoch now but 1 and 2 don't for idx in 1, 2: sb = self.get_shard_broker(ranges[1], idx) osr = sb.get_own_shard_range(no_default=True) self.assertIsNone(osr.epoch) expected_false_positive_line_snippet = 'Ignoring remote osr w/o epoch:' # run the replicator on the node with an epoch and it'll complain the # others dont have an epoch and not set it. replicator = self.run_custom_daemon( ContainerReplicator, 'container-replicator', shard_node_numbers[0], {}) warnings = replicator.logger.get_lines_for_level('warning') self.assertFalse([w for w in warnings if expected_false_positive_line_snippet in w]) # But it does send the new OSR with an epoch so the others should all # have it now. for idx in 1, 2: sb = self.get_shard_broker(ranges[1], idx) osr = sb.get_own_shard_range(no_default=True) self.assertIsNotNone(osr.epoch) def test_manage_shard_ranges_deleted_child_and_parent_gap(self): # Test to produce a scenario where a parent container is stuck at # sharding because of a gap in shard ranges. And the gap is caused by # deleted child shard range which finishes sharding before its parent # does. # note: be careful not to add a container listing to this test which # would get shard ranges into memcache. obj_names = self._make_object_names(20) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) # run replicators first time to get sync points set. self.container_replicators.once( additional_args='--partitions=%s' % self.brain.part) # shard root into two child-shards. root_0_db_file = self.get_db_file(self.brain.part, self.brain.nodes[0]) self.assert_subprocess_success([ 'swift-manage-shard-ranges', root_0_db_file, 'find_and_replace', '10', '--enable']) # Run container-replicator to replicate them to other nodes. self.container_replicators.once( additional_args='--partitions=%s' % self.brain.part) self.assert_container_states('unsharded', 2) # Run container-sharder on all nodes to shard the container. self.sharders_once_non_auto( additional_args='--partitions=%s' % self.brain.part) # get shards to update state from parent... self.sharders_once_non_auto() self.assert_container_states('sharded', 2) # shard first child shard into 2 grand-child-shards. c_shard_ranges = self.get_container_shard_ranges() c_shard_brokers = [self.get_shard_broker( c_shard_ranges[0], node_index=i) for i in range(3)] self.assert_subprocess_success([ 'swift-manage-shard-ranges', c_shard_brokers[0].db_file, 'find_and_replace', '5', '--enable']) child_shard_part, c_shard_nodes = self.brain.ring.get_nodes( c_shard_ranges[0].account, c_shard_ranges[0].container) self.container_replicators.once( additional_args='--partitions=%s' % child_shard_part) for node in c_shard_nodes: self.assert_container_state( node, 'unsharded', 2, account=c_shard_ranges[0].account, container=c_shard_ranges[0].container, part=child_shard_part) # run sharder on only 2 of the child replicas by renaming the third # replica's DB file directory. # NOTE: if we only rename the retiring DB file, other replicas will # create a "fresh" DB with timestamp during replication, and then # after we restore the retiring DB back, there will be two DB files # in the same folder, and container state will appear to be "sharding" # instead of "unsharded". c_shard_dir = os.path.dirname(c_shard_brokers[2].db_file) c_shard_tmp_dir = c_shard_dir + ".tmp" os.rename(c_shard_dir, c_shard_tmp_dir) self.sharders_once_non_auto(additional_args='--partitions=%s' % child_shard_part) for node in c_shard_nodes[:2]: self.assert_container_state( node, 'sharded', 2, account=c_shard_ranges[0].account, container=c_shard_ranges[0].container, part=child_shard_part) # get updates done... self.sharders_once_non_auto() # shard first grand-child shard into 2 grand-grand-child-shards. gc_shard_ranges = self.get_container_shard_ranges( account=c_shard_ranges[0].account, container=c_shard_ranges[0].container) shard_brokers = [self.get_shard_broker( gc_shard_ranges[0], node_index=i) for i in range(3)] self.assert_subprocess_success([ 'swift-manage-shard-ranges', shard_brokers[0].db_file, 'find_and_replace', '3', '--enable']) grandchild_shard_part, gc_shard_nodes = self.brain.ring.get_nodes( gc_shard_ranges[0].account, gc_shard_ranges[0].container) self.container_replicators.once( additional_args='--partitions=%s' % grandchild_shard_part) self.sharders_once_non_auto(additional_args='--partitions=%s' % grandchild_shard_part) # get shards to update state from parent... self.sharders_once_non_auto() self.sharders_once_non_auto() self.container_replicators.once( additional_args='--partitions=%s' % child_shard_part) # restore back the DB file directory of the disable child replica. shutil.rmtree(c_shard_dir, ignore_errors=True) os.rename(c_shard_tmp_dir, c_shard_dir) # the 2 child shards that sharded earlier still have their original # grand-child shards because they stopped updating form root once # sharded. for node in c_shard_nodes[:2]: self.assert_container_state( node, 'sharded', 2, account=c_shard_ranges[0].account, container=c_shard_ranges[0].container, part=child_shard_part) # the child shard that did not shard earlier has not been touched by # the sharder since, so still has two grand-child shards. self.assert_container_state( c_shard_nodes[2], 'unsharded', 2, account=c_shard_ranges[0].account, container=c_shard_ranges[0].container, part=child_shard_part) # now, finally, run the sharder on the child that is still waiting to # shard. It will get 2 great-grandchild ranges from root to replace # deleted grandchild. self.sharders_once_non_auto( additional_args=['--partitions=%s' % child_shard_part, '--devices=%s' % c_shard_nodes[2]['device']]) # batch size is 2 but this replicas has 3 shard ranges so we need two # runs of the sharder self.sharders_once_non_auto( additional_args=['--partitions=%s' % child_shard_part, '--devices=%s' % c_shard_nodes[2]['device']]) self.assert_container_state( c_shard_nodes[2], 'sharded', 3, account=c_shard_ranges[0].account, container=c_shard_ranges[0].container, part=child_shard_part)