From f31b6f7353f6f8e91a452bf3586032b94b7898c8 Mon Sep 17 00:00:00 2001
From: Tim Burke <tim.burke@gmail.com>
Date: Wed, 20 Mar 2024 12:18:04 -0700
Subject: [PATCH] recon-cron: Tolerate missing directories

Any of these directories may get unlinked between when we saw them in
their parent's directory listing and when we go to descend.

Change-Id: I1dfc0ee1d9e70cb0600557cde980bd5880bd40b3
---
 swift/cli/recon_cron.py          | 10 +++++-----
 test/unit/cli/test_recon_cron.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/swift/cli/recon_cron.py b/swift/cli/recon_cron.py
index d2ba80361a..bd1dd22052 100644
--- a/swift/cli/recon_cron.py
+++ b/swift/cli/recon_cron.py
@@ -18,18 +18,18 @@ import time
 from eventlet import Timeout
 
 from swift.common.utils import get_logger, dump_recon_cache, readconf, \
-    lock_path
+    lock_path, listdir
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 from swift.obj.diskfile import ASYNCDIR_BASE
 
 
 def get_async_count(device_dir):
     async_count = 0
-    for i in os.listdir(device_dir):
+    for i in listdir(device_dir):
         device = os.path.join(device_dir, i)
         if not os.path.isdir(device):
             continue
-        for asyncdir in os.listdir(device):
+        for asyncdir in listdir(device):
             # skip stuff like "accounts", "containers", etc.
             if not (asyncdir == ASYNCDIR_BASE or
                     asyncdir.startswith(ASYNCDIR_BASE + '-')):
@@ -37,10 +37,10 @@ def get_async_count(device_dir):
             async_pending = os.path.join(device, asyncdir)
 
             if os.path.isdir(async_pending):
-                for entry in os.listdir(async_pending):
+                for entry in listdir(async_pending):
                     if os.path.isdir(os.path.join(async_pending, entry)):
                         async_hdir = os.path.join(async_pending, entry)
-                        async_count += len(os.listdir(async_hdir))
+                        async_count += len(listdir(async_hdir))
     return async_count
 
 
diff --git a/test/unit/cli/test_recon_cron.py b/test/unit/cli/test_recon_cron.py
index 8e53d1e0b0..b377038a6b 100644
--- a/test/unit/cli/test_recon_cron.py
+++ b/test/unit/cli/test_recon_cron.py
@@ -16,6 +16,7 @@
 import tempfile
 import shutil
 import os
+import mock
 
 from unittest import TestCase
 from swift.cli.recon_cron import get_async_count
@@ -48,3 +49,32 @@ class TestReconCron(TestCase):
 
         count = get_async_count(device_dir)
         self.assertEqual(count, 3)
+
+    def test_get_async_count_deleted(self):
+        device_dir = os.path.join(self.temp_dir, 'device')
+        device_index = os.path.join(device_dir, '1')
+        async_dir = os.path.join(device_index, ASYNCDIR_BASE)
+        entry1 = os.path.join(async_dir, 'entry1')
+        entry2 = os.path.join(async_dir, 'entry2')
+        os.makedirs(entry1)
+        os.makedirs(entry2)
+
+        pending_file1 = os.path.join(entry1, 'pending_file1')
+        pending_file2 = os.path.join(entry1, 'pending_file2')
+        pending_file3 = os.path.join(entry2, 'pending_file3')
+        open(pending_file1, 'w').close()
+        open(pending_file2, 'w').close()
+        open(pending_file3, 'w').close()
+
+        orig_isdir = os.path.isdir
+
+        def racy_isdir(d):
+            result = orig_isdir(d)
+            if d == entry1:
+                # clean it up before caller has a chance to descend
+                shutil.rmtree(entry1)
+            return result
+
+        with mock.patch('os.path.isdir', racy_isdir):
+            count = get_async_count(device_dir)
+        self.assertEqual(count, 1)