diff --git a/swift/obj/updater.py b/swift/obj/updater.py index 743cf850dc..a8504dd576 100644 --- a/swift/obj/updater.py +++ b/swift/obj/updater.py @@ -31,8 +31,7 @@ from swift.common.utils import get_logger, renamer, write_pickle, \ from swift.common.daemon import Daemon from swift.common.storage_policy import split_policy_string, PolicyError from swift.obj.diskfile import get_tmp_dir, ASYNCDIR_BASE -from swift.common.http import is_success, HTTP_NOT_FOUND, \ - HTTP_INTERNAL_SERVER_ERROR +from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR class ObjectUpdater(Daemon): @@ -269,8 +268,13 @@ class ObjectUpdater(Daemon): with Timeout(self.node_timeout): resp = conn.getresponse() resp.read() - success = (is_success(resp.status) or - resp.status == HTTP_NOT_FOUND) + success = is_success(resp.status) + if not success: + self.logger.error( + _('Error code %(status)d is returned from remote ' + 'server %(ip)s: %(port)s / %(device)s'), + {'status': resp.status, 'ip': node['ip'], + 'port': node['port'], 'device': node['device']}) return (success, node['id']) except (Exception, Timeout): self.logger.exception(_('ERROR with remote server ' diff --git a/test/probe/test_object_async_update.py b/test/probe/test_object_async_update.py index bab7286424..42a11d90b3 100644 --- a/test/probe/test_object_async_update.py +++ b/test/probe/test_object_async_update.py @@ -18,7 +18,10 @@ from io import StringIO from unittest import main from uuid import uuid4 +from nose import SkipTest + from swiftclient import client +from swiftclient.exceptions import ClientException from swift.common import direct_client from swift.common.manager import Manager @@ -58,6 +61,64 @@ class TestObjectAsyncUpdate(ReplProbeTest): cnode, cpart, self.account, container)[1]] self.assertTrue(obj in objs) + def test_missing_container(self): + # In this test, we need to put container at handoff devices, so we + # need container devices more than replica count + if len(self.container_ring.devs) <= self.container_ring.replica_count: + raise SkipTest('Need devices more that replica count') + + container = 'container-%s' % uuid4() + cpart, cnodes = self.container_ring.get_nodes(self.account, container) + + # Kill all primary container servers + for cnode in cnodes: + kill_server((cnode['ip'], cnode['port']), self.ipport2server) + + # Create container, and all of its replicas are placed at handoff + # device + try: + client.put_container(self.url, self.token, container) + except ClientException as err: + # if the cluster doesn't have enough devices, swift may return + # error (ex. When we only have 4 devices in 3-replica cluster). + self.assertEqual(err.http_status, 503) + + # Assert handoff device has a container replica + another_cnode = self.container_ring.get_more_nodes(cpart).next() + direct_client.direct_get_container( + another_cnode, cpart, self.account, container) + + # Restart all primary container servers + for cnode in cnodes: + start_server((cnode['ip'], cnode['port']), self.ipport2server) + + # Create container/obj + obj = 'object-%s' % uuid4() + client.put_object(self.url, self.token, container, obj, '') + + # Run the object-updater + Manager(['object-updater']).once() + + # Run the container-replicator, and now, container replicas + # at handoff device get moved to primary servers + Manager(['container-replicator']).once() + + # Assert container replicas in primary servers, just moved by + # replicator don't know about the object + for cnode in cnodes: + self.assertFalse(direct_client.direct_get_container( + cnode, cpart, self.account, container)[1]) + + # Re-run the object-updaters and now container replicas in primary + # container servers should get updated + Manager(['object-updater']).once() + + # Assert all primary container servers know about container/obj + for cnode in cnodes: + objs = [o['name'] for o in direct_client.direct_get_container( + cnode, cpart, self.account, container)[1]] + self.assertIn(obj, objs) + class TestUpdateOverrides(ReplProbeTest): """ diff --git a/test/unit/obj/test_updater.py b/test/unit/obj/test_updater.py index 1990e4a6b3..12be96c99b 100644 --- a/test/unit/obj/test_updater.py +++ b/test/unit/obj/test_updater.py @@ -362,7 +362,7 @@ class TestObjectUpdater(unittest.TestCase): self.assertEqual([0], pickle.load(open(op_path)).get('successes')) - event = spawn(accept, [404, 500]) + event = spawn(accept, [404, 201]) cu.logger._clear() cu.run_once() err = event.wait() @@ -371,7 +371,7 @@ class TestObjectUpdater(unittest.TestCase): self.assertTrue(os.path.exists(op_path)) self.assertEqual(cu.logger.get_increment_counts(), {'failures': 1}) - self.assertEqual([0, 1], + self.assertEqual([0, 2], pickle.load(open(op_path)).get('successes')) event = spawn(accept, [201])