From b75d2a4e37d4c86763a2cc56c6dd53ebe2e0de19 Mon Sep 17 00:00:00 2001 From: Bill Huber Date: Mon, 17 Aug 2015 13:54:44 -0500 Subject: [PATCH] Quorum on durable response is too low Increase the .durable quorum from 2 to "parity + 1" to guarantee that we will never fail to rebuild an object. Otherwise, with low durable responses back (< parity + 1), the putter objects return with failed attribute set to true, thereby failing the rebuild of fragments for an object. Change-Id: I80d666f61273e589d0990baa78fd657b3470785d Closes-Bug: 1484565 --- swift/proxy/controllers/obj.py | 7 +++---- test/unit/proxy/controllers/test_obj.py | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 2aac83f2e5..043a7f3b0a 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -2370,10 +2370,9 @@ class ECObjectController(BaseObjectController): need_quorum = False # The .durable file will propagate in a replicated fashion; if # one exists, the reconstructor will spread it around. Thus, we - # don't require as many .durable files to be successfully - # written as we do fragment archives in order to call the PUT a - # success. - min_conns = 2 + # require "parity + 1" .durable files to be successfully written + # as we do fragment archives in order to call the PUT a success. + min_conns = policy.ec_nparity + 1 putters = [p for p in putters if not p.failed] # ignore response etags, and quorum boolean statuses, reasons, bodies, _etags, _quorum = \ diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 3cc9ce65dc..ea4b165c70 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -1479,7 +1479,7 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): codes = [FakeStatus(201, response_sleep=response_sleep) for i in range(self.replicas())] # swap out some with regular fast responses - number_of_fast_responses_needed_to_be_quick_enough = 2 + number_of_fast_responses_needed_to_be_quick_enough = 5 fast_indexes = random.sample( range(self.replicas()), number_of_fast_responses_needed_to_be_quick_enough) @@ -1496,6 +1496,21 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): self.assertEqual(resp.status_int, 201) self.assertTrue(response_time < response_sleep) + def test_PUT_with_less_durable_responses(self): + req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT', + body='') + + codes = [201] * self.policy.ec_nparity + codes += [503] * (self.policy.ec_ndata - 1) + random.shuffle(codes) + expect_headers = { + 'X-Obj-Metadata-Footer': 'yes', + 'X-Obj-Multiphase-Commit': 'yes' + } + with set_http_connect(*codes, expect_headers=expect_headers): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 503) + def test_COPY_with_ranges(self): req = swift.common.swob.Request.blank( '/v1/a/c/o', method='COPY',