openstack/swift
Code Issues Proposed changes

recon-cron: Tolerate missing directories

Browse Source 
Any of these directories may get unlinked between when we saw them in
their parent's directory listing and when we go to descend.

Change-Id: I1dfc0ee1d9e70cb0600557cde980bd5880bd40b3
This commit is contained in:
Tim Burke 2024-03-20 12:18:04 -07:00
parent b6dc24dbc0
commit f31b6f7353
2 changed files with 35 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
swift/cli/recon_cron.py
View File

@ -18,18 +18,18 @@ import time
from eventlet import Timeout
from swift.common.utils import get_logger, dump_recon_cache, readconf, \
lock_path
lock_path, listdir
from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
from swift.obj.diskfile import ASYNCDIR_BASE
def get_async_count(device_dir):
async_count = 0
for i in os.listdir(device_dir):
for i in listdir(device_dir):
device = os.path.join(device_dir, i)
if not os.path.isdir(device):
continue
for asyncdir in os.listdir(device):
for asyncdir in listdir(device):
# skip stuff like "accounts", "containers", etc.
if not (asyncdir == ASYNCDIR_BASE or
asyncdir.startswith(ASYNCDIR_BASE + '-')):
@ -37,10 +37,10 @@ def get_async_count(device_dir):
async_pending = os.path.join(device, asyncdir)
if os.path.isdir(async_pending):
for entry in os.listdir(async_pending):
for entry in listdir(async_pending):
if os.path.isdir(os.path.join(async_pending, entry)):
async_hdir = os.path.join(async_pending, entry)
async_count += len(os.listdir(async_hdir))
async_count += len(listdir(async_hdir))
return async_count

30
test/unit/cli/test_recon_cron.py
View File

@ -16,6 +16,7 @@
import tempfile
import shutil
import os
import mock
from unittest import TestCase
from swift.cli.recon_cron import get_async_count
@ -48,3 +49,32 @@ class TestReconCron(TestCase):
count = get_async_count(device_dir)
self.assertEqual(count, 3)
def test_get_async_count_deleted(self):
device_dir = os.path.join(self.temp_dir, 'device')
device_index = os.path.join(device_dir, '1')
async_dir = os.path.join(device_index, ASYNCDIR_BASE)
entry1 = os.path.join(async_dir, 'entry1')
entry2 = os.path.join(async_dir, 'entry2')
os.makedirs(entry1)
os.makedirs(entry2)
pending_file1 = os.path.join(entry1, 'pending_file1')
pending_file2 = os.path.join(entry1, 'pending_file2')
pending_file3 = os.path.join(entry2, 'pending_file3')
open(pending_file1, 'w').close()
open(pending_file2, 'w').close()
open(pending_file3, 'w').close()
orig_isdir = os.path.isdir
def racy_isdir(d):
result = orig_isdir(d)
if d == entry1:
# clean it up before caller has a chance to descend
shutil.rmtree(entry1)
return result
with mock.patch('os.path.isdir', racy_isdir):
count = get_async_count(device_dir)
self.assertEqual(count, 1)