From 69e9a1acbf6d6bb2e0c86a1b42393fcc11033fd2 Mon Sep 17 00:00:00 2001
From: Tim Burke <tim.burke@gmail.com>
Date: Tue, 15 Jun 2021 14:05:12 -0700
Subject: [PATCH] reconciler: Tolerate 503s on HEAD

Change-Id: Ie53ebb90db4127b798df17fc7303aefc8dcc7f2a
---
 swift/container/reconciler.py          | 12 ++++++--
 test/unit/container/test_reconciler.py | 38 ++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/swift/container/reconciler.py b/swift/container/reconciler.py
index 346032ad91..269e1ba6f0 100644
--- a/swift/container/reconciler.py
+++ b/swift/container/reconciler.py
@@ -489,9 +489,15 @@ class ContainerReconciler(Daemon):
                           container_policy_index)
         headers = {
             'X-Backend-Storage-Policy-Index': container_policy_index}
-        dest_obj = self.swift.get_object_metadata(account, container, obj,
-                                                  headers=headers,
-                                                  acceptable_statuses=(2, 4))
+        try:
+            dest_obj = self.swift.get_object_metadata(
+                account, container, obj, headers=headers,
+                acceptable_statuses=(2, 4))
+        except UnexpectedResponse:
+            self.stats_log('unavailable_destination', '%r (%f) unable to '
+                           'determine the destination timestamp, if any',
+                           path, q_ts)
+            return False
         dest_ts = Timestamp(dest_obj.get('x-backend-timestamp', 0))
         if dest_ts >= q_ts:
             self.stats_log('found_object', '%r (%f) in policy_index %s '
diff --git a/test/unit/container/test_reconciler.py b/test/unit/container/test_reconciler.py
index c707a705f3..91c48df587 100644
--- a/test/unit/container/test_reconciler.py
+++ b/test/unit/container/test_reconciler.py
@@ -1281,6 +1281,44 @@ class TestReconciler(unittest.TestCase):
         self.assertEqual(deleted_container_entries, [])
         self.assertEqual(self.reconciler.stats['retry'], 1)
 
+    def test_object_move_fails_preflight(self):
+        # setup the cluster
+        self._mock_listing({
+            (None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3600.123456,
+            (1, '/AUTH_bob/c/o1'): 3600.123457,  # slightly newer
+        })
+        self._mock_oldest_spi({'c': 0})  # destination
+
+        # make the HEAD blow up
+        self.fake_swift.storage_policy[0].register(
+            'HEAD', '/v1/AUTH_bob/c/o1', swob.HTTPServiceUnavailable, {})
+        # turn the crank
+        deleted_container_entries = self._run_once()
+
+        # we did some listings...
+        self.assertEqual(
+            self.fake_swift.calls,
+            [('GET', self.current_container_path),
+             ('GET', '/v1/.misplaced_objects' + listing_qs('')),
+             ('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
+             ('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
+             ('GET', '/v1/.misplaced_objects/3600' +
+              listing_qs('1:/AUTH_bob/c/o1'))])
+        # ...but we can't even tell whether anything's misplaced or not
+        self.assertEqual(self.reconciler.stats['misplaced_object'], 0)
+        self.assertEqual(self.reconciler.stats['unavailable_destination'], 1)
+        # so we don't try to do any sort of move or cleanup
+        self.assertEqual(self.reconciler.stats['copy_attempt'], 0)
+        self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
+        self.assertEqual(self.reconciler.stats['pop_queue'], 0)
+        self.assertEqual(deleted_container_entries, [])
+        # and we'll have to try again later
+        self.assertEqual(self.reconciler.stats['retry'], 1)
+        self.assertEqual(self.fake_swift.storage_policy[1].calls, [])
+        self.assertEqual(
+            self.fake_swift.storage_policy[0].calls,
+            [('HEAD', '/v1/AUTH_bob/c/o1')])
+
     def test_object_move_fails_cleanup(self):
         # setup the cluster
         self._mock_listing({