Ensure update of the container by object-updater

Object-updater regards 404 from container-server as success.
This causes wrong giving up and generates object which never appears
at container, for example when all container replicas are placed in
handoff devices at the moment.

This patch makes object-updater try updating until it gets 201s from
all container servers, and ensures the update.

Change-Id: Ic5d55ff368cec62ab2eb9fd883801ba4ab043dd4
Closes-Bug: #1328735

swift/obj/updater.py

@@ -31,8 +31,7 @@ from swift.common.utils import get_logger, renamer, write_pickle, \
 from swift.common.daemon import Daemon
 from swift.common.storage_policy import split_policy_string, PolicyError
 from swift.obj.diskfile import get_tmp_dir, ASYNCDIR_BASE
-from swift.common.http import is_success, HTTP_NOT_FOUND, \
-    HTTP_INTERNAL_SERVER_ERROR
+from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR
 
 
 class ObjectUpdater(Daemon):
@@ -269,8 +268,13 @@ class ObjectUpdater(Daemon):
             with Timeout(self.node_timeout):
                 resp = conn.getresponse()
                 resp.read()
-                success = (is_success(resp.status) or
-                           resp.status == HTTP_NOT_FOUND)
+                success = is_success(resp.status)
+                if not success:
+                    self.logger.error(
+                        _('Error code %(status)d is returned from remote '
+                          'server %(ip)s: %(port)s / %(device)s'),
+                        {'status': resp.status, 'ip': node['ip'],
+                         'port': node['port'], 'device': node['device']})
                 return (success, node['id'])
         except (Exception, Timeout):
             self.logger.exception(_('ERROR with remote server '

test/probe/test_object_async_update.py

@@ -18,7 +18,10 @@ from io import StringIO
 from unittest import main
 from uuid import uuid4
 
+from nose import SkipTest
+
 from swiftclient import client
+from swiftclient.exceptions import ClientException
 
 from swift.common import direct_client
 from swift.common.manager import Manager
@@ -58,6 +61,64 @@ class TestObjectAsyncUpdate(ReplProbeTest):
             cnode, cpart, self.account, container)[1]]
         self.assertTrue(obj in objs)
 
+    def test_missing_container(self):
+        # In this test, we need to put container at handoff devices, so we
+        # need container devices more than replica count
+        if len(self.container_ring.devs) <= self.container_ring.replica_count:
+            raise SkipTest('Need devices more that replica count')
+
+        container = 'container-%s' % uuid4()
+        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
+
+        # Kill all primary container servers
+        for cnode in cnodes:
+            kill_server((cnode['ip'], cnode['port']), self.ipport2server)
+
+        # Create container, and all of its replicas are placed at handoff
+        # device
+        try:
+            client.put_container(self.url, self.token, container)
+        except ClientException as err:
+            # if the cluster doesn't have enough devices, swift may return
+            # error (ex. When we only have 4 devices in 3-replica cluster).
+            self.assertEqual(err.http_status, 503)
+
+        # Assert handoff device has a container replica
+        another_cnode = self.container_ring.get_more_nodes(cpart).next()
+        direct_client.direct_get_container(
+            another_cnode, cpart, self.account, container)
+
+        # Restart all primary container servers
+        for cnode in cnodes:
+            start_server((cnode['ip'], cnode['port']), self.ipport2server)
+
+        # Create container/obj
+        obj = 'object-%s' % uuid4()
+        client.put_object(self.url, self.token, container, obj, '')
+
+        # Run the object-updater
+        Manager(['object-updater']).once()
+
+        # Run the container-replicator, and now, container replicas
+        # at handoff device get moved to primary servers
+        Manager(['container-replicator']).once()
+
+        # Assert container replicas in primary servers, just moved by
+        # replicator don't know about the object
+        for cnode in cnodes:
+            self.assertFalse(direct_client.direct_get_container(
+                cnode, cpart, self.account, container)[1])
+
+        # Re-run the object-updaters and now container replicas in primary
+        # container servers should get updated
+        Manager(['object-updater']).once()
+
+        # Assert all primary container servers know about container/obj
+        for cnode in cnodes:
+            objs = [o['name'] for o in direct_client.direct_get_container(
+                    cnode, cpart, self.account, container)[1]]
+            self.assertIn(obj, objs)
+
 
 class TestUpdateOverrides(ReplProbeTest):
     """

test/unit/obj/test_updater.py

@@ -362,7 +362,7 @@ class TestObjectUpdater(unittest.TestCase):
         self.assertEqual([0],
                          pickle.load(open(op_path)).get('successes'))
 
-        event = spawn(accept, [404, 500])
+        event = spawn(accept, [404, 201])
         cu.logger._clear()
         cu.run_once()
         err = event.wait()
@@ -371,7 +371,7 @@ class TestObjectUpdater(unittest.TestCase):
         self.assertTrue(os.path.exists(op_path))
         self.assertEqual(cu.logger.get_increment_counts(),
                          {'failures': 1})
-        self.assertEqual([0, 1],
+        self.assertEqual([0, 2],
                          pickle.load(open(op_path)).get('successes'))
 
         event = spawn(accept, [201])