diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index ffdb6e37dc..32eaf4fcf5 100644 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -194,21 +194,6 @@ class BaseObjectControllerMixin(object): policy = policy or POLICIES.default return policy.quorum - # Add a few helper methods for EC tests. - def _make_ec_archive_bodies(self, test_body, policy=None): - policy = policy or self.policy - return encode_frag_archive_bodies(policy, test_body) - - def _make_ec_object_stub(self, pattern='test', policy=None, - timestamp=None): - policy = policy or self.policy - test_body = pattern * policy.ec_segment_size - test_body = test_body[:-random.randint(1, 1000)] - return make_ec_object_stub(test_body, policy, timestamp) - - def _fake_ec_node_response(self, node_frags): - return fake_ec_node_response(node_frags, self.policy) - def test_iter_nodes_local_first_noops_when_no_affinity(self): # this test needs a stable node order - most don't self.app.sort_nodes = lambda l, *args, **kwargs: l @@ -1625,8 +1610,109 @@ def capture_http_requests(get_response): yield fake_conn +class ECObjectControllerMixin(BaseObjectControllerMixin): + # Add a few helper methods for EC tests. + def _make_ec_archive_bodies(self, test_body, policy=None): + policy = policy or self.policy + return encode_frag_archive_bodies(policy, test_body) + + def _make_ec_object_stub(self, pattern='test', policy=None, + timestamp=None): + policy = policy or self.policy + test_body = pattern * policy.ec_segment_size + test_body = test_body[:-random.randint(1, 1000)] + return make_ec_object_stub(test_body, policy, timestamp) + + def _fake_ec_node_response(self, node_frags): + return fake_ec_node_response(node_frags, self.policy) + + def test_GET_with_duplicate_but_sufficient_frag_indexes(self): + obj1 = self._make_ec_object_stub() + # proxy should ignore duplicated frag indexes and continue search for + # a set of unique indexes, finding last one on a handoff + node_frags = [ + {'obj': obj1, 'frag': 0}, + {'obj': obj1, 'frag': 0}, # duplicate frag + {'obj': obj1, 'frag': 1}, + {'obj': obj1, 'frag': 1}, # duplicate frag + {'obj': obj1, 'frag': 2}, + {'obj': obj1, 'frag': 2}, # duplicate frag + {'obj': obj1, 'frag': 3}, + {'obj': obj1, 'frag': 3}, # duplicate frag + {'obj': obj1, 'frag': 4}, + {'obj': obj1, 'frag': 4}, # duplicate frag + {'obj': obj1, 'frag': 10}, + {'obj': obj1, 'frag': 11}, + {'obj': obj1, 'frag': 12}, + {'obj': obj1, 'frag': 13}, + ] * self.policy.ec_duplication_factor + node_frags.append({'obj': obj1, 'frag': 5}) # first handoff + + fake_response = self._fake_ec_node_response(node_frags) + + req = swob.Request.blank('/v1/a/c/o') + with capture_http_requests(fake_response) as log: + resp = req.get_response(self.app) + + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers['etag'], obj1['etag']) + self.assertEqual(md5(resp.body).hexdigest(), obj1['etag']) + + # expect a request to all primaries plus one handoff + self.assertEqual(self.replicas() + 1, len(log)) + collected_indexes = defaultdict(list) + for conn in log: + fi = conn.resp.headers.get('X-Object-Sysmeta-Ec-Frag-Index') + if fi is not None: + collected_indexes[fi].append(conn) + self.assertEqual(len(collected_indexes), self.policy.ec_ndata) + + def test_GET_with_duplicate_but_insufficient_frag_indexes(self): + obj1 = self._make_ec_object_stub() + # proxy should ignore duplicated frag indexes and continue search for + # a set of unique indexes, but fails to find one + node_frags = [ + {'obj': obj1, 'frag': 0}, + {'obj': obj1, 'frag': 0}, # duplicate frag + {'obj': obj1, 'frag': 1}, + {'obj': obj1, 'frag': 1}, # duplicate frag + {'obj': obj1, 'frag': 2}, + {'obj': obj1, 'frag': 2}, # duplicate frag + {'obj': obj1, 'frag': 3}, + {'obj': obj1, 'frag': 3}, # duplicate frag + {'obj': obj1, 'frag': 4}, + {'obj': obj1, 'frag': 4}, # duplicate frag + {'obj': obj1, 'frag': 10}, + {'obj': obj1, 'frag': 11}, + {'obj': obj1, 'frag': 12}, + {'obj': obj1, 'frag': 13}, + ] + + # ... and the rests are 404s which is limited by request_count + # (2 * replicas in default) rather than max_extra_requests limitation + # because the retries will be in ResumingGetter if the responses + # are 404s + node_frags += [[]] * (self.replicas() * 2 - len(node_frags)) + fake_response = self._fake_ec_node_response(node_frags) + + req = swob.Request.blank('/v1/a/c/o') + with capture_http_requests(fake_response) as log: + resp = req.get_response(self.app) + + self.assertEqual(resp.status_int, 404) + + # expect a request to all nodes + self.assertEqual(2 * self.replicas(), len(log)) + collected_indexes = defaultdict(list) + for conn in log: + fi = conn.resp.headers.get('X-Object-Sysmeta-Ec-Frag-Index') + if fi is not None: + collected_indexes[fi].append(conn) + self.assertEqual(len(collected_indexes), self.policy.ec_ndata - 1) + + @patch_policies(with_ec_default=True) -class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): +class TestECObjController(ECObjectControllerMixin, unittest.TestCase): container_info = { 'status': 200, 'read_acl': None, @@ -2719,47 +2805,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): 'collected %s frags for etag %s' % ( len(frags), etag)) - def test_GET_with_duplicate_but_sufficient_frag_indexes(self): - obj1 = self._make_ec_object_stub() - # proxy should ignore duplicated frag indexes and continue search for - # a set of unique indexes, finding last one on a handoff - node_frags = [ - {'obj': obj1, 'frag': 0}, - {'obj': obj1, 'frag': 0}, # duplicate frag - {'obj': obj1, 'frag': 1}, - {'obj': obj1, 'frag': 1}, # duplicate frag - {'obj': obj1, 'frag': 2}, - {'obj': obj1, 'frag': 2}, # duplicate frag - {'obj': obj1, 'frag': 3}, - {'obj': obj1, 'frag': 3}, # duplicate frag - {'obj': obj1, 'frag': 4}, - {'obj': obj1, 'frag': 4}, # duplicate frag - {'obj': obj1, 'frag': 10}, - {'obj': obj1, 'frag': 11}, - {'obj': obj1, 'frag': 12}, - {'obj': obj1, 'frag': 13}, - {'obj': obj1, 'frag': 5}, # handoff - ] - - fake_response = self._fake_ec_node_response(node_frags) - - req = swob.Request.blank('/v1/a/c/o') - with capture_http_requests(fake_response) as log: - resp = req.get_response(self.app) - - self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.headers['etag'], obj1['etag']) - self.assertEqual(md5(resp.body).hexdigest(), obj1['etag']) - - # expect a request to all primaries plus one handoff - self.assertEqual(self.replicas() + 1, len(log)) - collected_indexes = defaultdict(list) - for conn in log: - fi = conn.resp.headers.get('X-Object-Sysmeta-Ec-Frag-Index') - if fi is not None: - collected_indexes[fi].append(conn) - self.assertEqual(len(collected_indexes), self.policy.ec_ndata) - def test_GET_with_duplicate_and_hidden_frag_indexes(self): obj1 = self._make_ec_object_stub() # proxy should ignore duplicated frag indexes and continue search for @@ -2803,48 +2848,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): collected_indexes[fi].append(conn) self.assertEqual(len(collected_indexes), self.policy.ec_ndata) - def test_GET_with_duplicate_but_insufficient_frag(self): - obj1 = self._make_ec_object_stub() - # proxy should ignore duplicated frag indexes and continue search for - # a set of unique indexes, but fails to find one - node_frags = [ - {'obj': obj1, 'frag': 0}, - {'obj': obj1, 'frag': 0}, # duplicate frag - {'obj': obj1, 'frag': 1}, - {'obj': obj1, 'frag': 1}, # duplicate frag - {'obj': obj1, 'frag': 2}, - {'obj': obj1, 'frag': 2}, # duplicate frag - {'obj': obj1, 'frag': 3}, - {'obj': obj1, 'frag': 3}, # duplicate frag - {'obj': obj1, 'frag': 4}, - {'obj': obj1, 'frag': 4}, # duplicate frag - {'obj': obj1, 'frag': 10}, - {'obj': obj1, 'frag': 11}, - {'obj': obj1, 'frag': 12}, - {'obj': obj1, 'frag': 13}, - ] + [[]] * 14 # 404 from handoffs - - fake_response = self._fake_ec_node_response(node_frags) - - req = swob.Request.blank('/v1/a/c/o') - with capture_http_requests(fake_response) as log: - resp = req.get_response(self.app) - - self.assertEqual(resp.status_int, 404) - - # expect a request to all nodes - self.assertEqual(2 * self.replicas(), len(log)) - collected_indexes = defaultdict(list) - collected_etags = set() - for conn in log: - etag = conn.resp.headers['X-Object-Sysmeta-Ec-Etag'] - collected_etags.add(etag) # will be None from handoffs - fi = conn.resp.headers.get('X-Object-Sysmeta-Ec-Frag-Index') - if fi is not None: - collected_indexes[fi].append(conn) - self.assertEqual(len(collected_indexes), self.policy.ec_ndata - 1) - self.assertEqual({obj1['etag'], None}, collected_etags) - def test_GET_with_missing_and_mixed_frags_may_503(self): obj1 = self._make_ec_object_stub(pattern='obj1') obj2 = self._make_ec_object_stub(pattern='obj2') @@ -3930,7 +3933,7 @@ class TestECFunctions(unittest.TestCase): StoragePolicy(1, name='unu')], fake_ring_args=[{'replicas': 28}, {}]) class TestECDuplicationObjController( - BaseObjectControllerMixin, unittest.TestCase): + ECObjectControllerMixin, unittest.TestCase): container_info = { 'status': 200, 'read_acl': None, @@ -4089,49 +4092,6 @@ class TestECDuplicationObjController( 'collected %s frags for etag %s' % ( len(frags), etag)) - def test_GET_with_duplicate_but_insufficient_frag(self): - obj1 = self._make_ec_object_stub() - # proxy should ignore duplicated frag indexes and continue search for - # a set of unique indexes, but fails to find one - node_frags = [ - {'obj': obj1, 'frag': 0}, - {'obj': obj1, 'frag': 0}, # duplicate frag - {'obj': obj1, 'frag': 1}, - {'obj': obj1, 'frag': 1}, # duplicate frag - {'obj': obj1, 'frag': 2}, - {'obj': obj1, 'frag': 2}, # duplicate frag - {'obj': obj1, 'frag': 3}, - {'obj': obj1, 'frag': 3}, # duplicate frag - {'obj': obj1, 'frag': 4}, - {'obj': obj1, 'frag': 4}, # duplicate frag - {'obj': obj1, 'frag': 10}, - {'obj': obj1, 'frag': 11}, - {'obj': obj1, 'frag': 12}, - {'obj': obj1, 'frag': 13}, - ] - - # ... and the rests are 404s which is limited by request_count - # (2 * replicas in default) rather than max_extra_requests limitation - # because the retries will be in ResumingGetter if the responses - # are 404s - node_frags += [[]] * (self.replicas() * 2 - len(node_frags)) - fake_response = self._fake_ec_node_response(node_frags) - - req = swob.Request.blank('/v1/a/c/o') - with capture_http_requests(fake_response) as log: - resp = req.get_response(self.app) - - self.assertEqual(resp.status_int, 404) - - # expect a request to all nodes - self.assertEqual(2 * self.replicas(), len(log)) - collected_indexes = defaultdict(list) - for conn in log: - fi = conn.resp.headers.get('X-Object-Sysmeta-Ec-Frag-Index') - if fi is not None: - collected_indexes[fi].append(conn) - self.assertEqual(len(collected_indexes), self.policy.ec_ndata - 1) - def test_GET_with_many_missed_overwrite_will_need_handoff(self): obj1 = self._make_ec_object_stub(pattern='obj1') obj2 = self._make_ec_object_stub(pattern='obj2') @@ -4189,54 +4149,6 @@ class TestECDuplicationObjController( 'collected %s frags for etag %s' % ( len(frags), etag)) - def test_GET_with_duplicate_but_sufficient_frag_indexes(self): - obj1 = self._make_ec_object_stub() - # proxy should ignore duplicated frag indexes and continue search for - # a set of unique indexes, finding last one on a handoff - node_frags = [ - {'obj': obj1, 'frag': 0}, - {'obj': obj1, 'frag': 0}, # duplicate frag - {'obj': obj1, 'frag': 1}, - {'obj': obj1, 'frag': 1}, # duplicate frag - {'obj': obj1, 'frag': 2}, - {'obj': obj1, 'frag': 2}, # duplicate frag - {'obj': obj1, 'frag': 3}, - {'obj': obj1, 'frag': 3}, # duplicate frag - {'obj': obj1, 'frag': 4}, - {'obj': obj1, 'frag': 4}, # duplicate frag - {'obj': obj1, 'frag': 10}, - {'obj': obj1, 'frag': 11}, - {'obj': obj1, 'frag': 12}, - {'obj': obj1, 'frag': 13}, - ] - - # proxy will access randomly to a node in the second set - # so to ensure the GET fragment meets what it needs. - node_frags += [{'obj': obj1, 'frag': 5}] - # rests are 404s - node_frags += [[]] * (self.replicas() - len(node_frags)) - - fake_response = self._fake_ec_node_response(node_frags) - - req = swob.Request.blank('/v1/a/c/o') - with capture_http_requests(fake_response) as log: - resp = req.get_response(self.app) - - self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.headers['etag'], obj1['etag']) - self.assertEqual(md5(resp.body).hexdigest(), obj1['etag']) - - # expect a request to all primaries plus one handoff - self.assertGreaterEqual( - len(log), self.policy.ec_n_unique_fragments + 1) - self.assertLessEqual(len(log), self.replicas()) - collected_indexes = defaultdict(list) - for conn in log: - fi = conn.resp.headers.get('X-Object-Sysmeta-Ec-Frag-Index') - if fi is not None: - collected_indexes[fi].append(conn) - self.assertEqual(len(collected_indexes), self.policy.ec_ndata) - def test_GET_with_missing_and_mixed_frags_will_dig_deep_but_succeed(self): obj1 = self._make_ec_object_stub(pattern='obj1', timestamp=self.ts())