openstack/swift
Code Issues Proposed changes

Merge "Rename hash_cleanup_listdir tests"

Browse Source
This commit is contained in:
Jenkins 2016-05-11 21:20:28 +00:00 committed by Gerrit Code Review
commit ffef6105cd
3 changed files with 112 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
swift/obj/diskfile.py
View File

@ -2641,7 +2641,7 @@ class ECDiskFile(BaseDiskFile):
reverting it to its primary node. reverting it to its primary node.
The hash will be invalidated, and if empty or invalid the The hash will be invalidated, and if empty or invalid the
hsh_path will be removed on next hash_cleanup_listdir. hsh_path will be removed on next cleanup_ondisk_files.
:param timestamp: the object timestamp, an instance of :param timestamp: the object timestamp, an instance of
:class:`~swift.common.utils.Timestamp` :class:`~swift.common.utils.Timestamp`

219
test/unit/obj/test_diskfile.py
View File

@ -623,7 +623,7 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.fail('%s with files %s' % (str(e), files)) self.fail('%s with files %s' % (str(e), files))
shuffle(files) shuffle(files)
def _test_hash_cleanup_listdir_files(self, scenarios, policy, def _test_cleanup_ondisk_files_files(self, scenarios, policy,
reclaim_age=None): reclaim_age=None):
# check that expected files are left in hashdir after cleanup # check that expected files are left in hashdir after cleanup
for test in scenarios: for test in scenarios:
@ -638,12 +638,12 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
if (f[2] if len(f) > 2 else f[1])]) if (f[2] if len(f) > 2 else f[1])])
if reclaim_age: if reclaim_age:
class_under_test.cleanup_ondisk_files( class_under_test.cleanup_ondisk_files(
hashdir, reclaim_age=reclaim_age)['files'] hashdir, reclaim_age=reclaim_age)
else: else:
with mock.patch('swift.obj.diskfile.time') as mock_time: with mock.patch('swift.obj.diskfile.time') as mock_time:
# don't reclaim anything # don't reclaim anything
mock_time.time.return_value = 0.0 mock_time.time.return_value = 0.0
class_under_test.cleanup_ondisk_files(hashdir)['files'] class_under_test.cleanup_ondisk_files(hashdir)
after_cleanup = set(os.listdir(hashdir)) after_cleanup = set(os.listdir(hashdir))
errmsg = "expected %r, got %r for test %r" % ( errmsg = "expected %r, got %r for test %r" % (
sorted(expected_after_cleanup), sorted(after_cleanup), test sorted(expected_after_cleanup), sorted(after_cleanup), test
@ -652,7 +652,7 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
def _test_yield_hashes_cleanup(self, scenarios, policy): def _test_yield_hashes_cleanup(self, scenarios, policy):
# opportunistic test to check that yield_hashes cleans up dir using # opportunistic test to check that yield_hashes cleans up dir using
# same scenarios as passed to _test_hash_cleanup_listdir_files # same scenarios as passed to _test_cleanup_ondisk_files_files
for test in scenarios: for test in scenarios:
class_under_test = self.df_router[policy] class_under_test = self.df_router[policy]
# list(zip(...)) for py3 compatibility (zip is lazy there) # list(zip(...)) for py3 compatibility (zip is lazy there)
@ -724,7 +724,7 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
'Unexpected file %s' 'Unexpected file %s'
% os.path.join(datadir, unexpected))) % os.path.join(datadir, unexpected)))
def test_hash_cleanup_listdir_reclaim_non_data_files(self): def test_cleanup_ondisk_files_reclaim_non_data_files(self):
# Each scenario specifies a list of (filename, extension, [survives]) # Each scenario specifies a list of (filename, extension, [survives])
# tuples. If extension is set or 'survives' is True, the filename # tuples. If extension is set or 'survives' is True, the filename
# should still be in the dir after cleanup. # should still be in the dir after cleanup.
@ -755,7 +755,7 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
[('%s.meta' % older, False, False), [('%s.meta' % older, False, False),
('%s.ts' % older, False, False)]] ('%s.ts' % older, False, False)]]
self._test_hash_cleanup_listdir_files(scenarios, POLICIES.default, self._test_cleanup_ondisk_files_files(scenarios, POLICIES.default,
reclaim_age=1000) reclaim_age=1000)
def test_construct_dev_path(self): def test_construct_dev_path(self):
@ -848,9 +848,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value=None) self.df_mgr.get_dev_path = mock.MagicMock(return_value=None)
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'cleanup_ondisk_files')) as hclistdir, \ 'cleanup_ondisk_files')) as cleanup, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = {'files': ['1381679759.90941.data']} cleanup.return_value = {'files': ['1381679759.90941.data']}
readmeta.return_value = {'name': '/a/c/o'} readmeta.return_value = {'name': '/a/c/o'}
self.assertRaises( self.assertRaises(
DiskFileDeviceUnavailable, DiskFileDeviceUnavailable,
@ -861,13 +861,13 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'cleanup_ondisk_files')) as hclistdir, \ 'cleanup_ondisk_files')) as cleanup, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta, \ mock.patch('swift.obj.diskfile.read_metadata') as readmeta, \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'quarantine_renamer')) as quarantine_renamer: 'quarantine_renamer')) as quarantine_renamer:
osexc = OSError() osexc = OSError()
osexc.errno = errno.ENOTDIR osexc.errno = errno.ENOTDIR
hclistdir.side_effect = osexc cleanup.side_effect = osexc
readmeta.return_value = {'name': '/a/c/o'} readmeta.return_value = {'name': '/a/c/o'}
self.assertRaises( self.assertRaises(
DiskFileNotExist, DiskFileNotExist,
@ -881,11 +881,11 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'cleanup_ondisk_files')) as hclistdir, \ 'cleanup_ondisk_files')) as cleanup, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
osexc = OSError() osexc = OSError()
osexc.errno = errno.ENOENT osexc.errno = errno.ENOENT
hclistdir.side_effect = osexc cleanup.side_effect = osexc
readmeta.return_value = {'name': '/a/c/o'} readmeta.return_value = {'name': '/a/c/o'}
self.assertRaises( self.assertRaises(
DiskFileNotExist, DiskFileNotExist,
@ -896,10 +896,10 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'cleanup_ondisk_files')) as hclistdir, \ 'cleanup_ondisk_files')) as cleanup, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
osexc = OSError() osexc = OSError()
hclistdir.side_effect = osexc cleanup.side_effect = osexc
readmeta.return_value = {'name': '/a/c/o'} readmeta.return_value = {'name': '/a/c/o'}
self.assertRaises( self.assertRaises(
OSError, OSError,
@ -910,9 +910,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'cleanup_ondisk_files')) as hclistdir, \ 'cleanup_ondisk_files')) as cleanup, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = {'files': []} cleanup.return_value = {'files': []}
readmeta.return_value = {'name': '/a/c/o'} readmeta.return_value = {'name': '/a/c/o'}
self.assertRaises( self.assertRaises(
DiskFileNotExist, DiskFileNotExist,
@ -923,9 +923,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'cleanup_ondisk_files')) as hclistdir, \ 'cleanup_ondisk_files')) as cleanup, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = {'files': ['1381679759.90941.data']} cleanup.return_value = {'files': ['1381679759.90941.data']}
readmeta.side_effect = EOFError() readmeta.side_effect = EOFError()
self.assertRaises( self.assertRaises(
DiskFileNotExist, DiskFileNotExist,
@ -936,9 +936,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'cleanup_ondisk_files')) as hclistdir, \ 'cleanup_ondisk_files')) as cleanup, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = {'files': ['1381679759.90941.data']} cleanup.return_value = {'files': ['1381679759.90941.data']}
readmeta.return_value = {} readmeta.return_value = {}
try: try:
self.df_mgr.get_diskfile_from_hash( self.df_mgr.get_diskfile_from_hash(
@ -952,9 +952,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'cleanup_ondisk_files')) as hclistdir, \ 'cleanup_ondisk_files')) as cleanup, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = {'files': ['1381679759.90941.data']} cleanup.return_value = {'files': ['1381679759.90941.data']}
readmeta.return_value = {'name': 'bad'} readmeta.return_value = {'name': 'bad'}
try: try:
self.df_mgr.get_diskfile_from_hash( self.df_mgr.get_diskfile_from_hash(
@ -968,16 +968,16 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')) as dfclass, \ with mock.patch(self._manager_mock('diskfile_cls')) as dfclass, \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'cleanup_ondisk_files')) as hclistdir, \ 'cleanup_ondisk_files')) as cleanup, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = {'files': ['1381679759.90941.data']} cleanup.return_value = {'files': ['1381679759.90941.data']}
readmeta.return_value = {'name': '/a/c/o'} readmeta.return_value = {'name': '/a/c/o'}
self.df_mgr.get_diskfile_from_hash( self.df_mgr.get_diskfile_from_hash(
'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0]) 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0])
dfclass.assert_called_once_with( dfclass.assert_called_once_with(
self.df_mgr, '/srv/dev/', '9', self.df_mgr, '/srv/dev/', '9',
'a', 'c', 'o', policy=POLICIES[0]) 'a', 'c', 'o', policy=POLICIES[0])
hclistdir.assert_called_once_with( cleanup.assert_called_once_with(
'/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900', '/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900',
604800) 604800)
readmeta.assert_called_once_with( readmeta.assert_called_once_with(
@ -1167,7 +1167,7 @@ class TestDiskFileManager(DiskFileManagerMixin, unittest.TestCase):
] ]
self._test_get_ondisk_files(scenarios, POLICIES[0], None) self._test_get_ondisk_files(scenarios, POLICIES[0], None)
self._test_hash_cleanup_listdir_files(scenarios, POLICIES[0]) self._test_cleanup_ondisk_files_files(scenarios, POLICIES[0])
self._test_yield_hashes_cleanup(scenarios, POLICIES[0]) self._test_yield_hashes_cleanup(scenarios, POLICIES[0])
def test_get_ondisk_files_with_stray_meta(self): def test_get_ondisk_files_with_stray_meta(self):
@ -1225,7 +1225,7 @@ class TestDiskFileManager(DiskFileManagerMixin, unittest.TestCase):
self.assertEqual("Invalid Timestamp value in filename 'junk'", self.assertEqual("Invalid Timestamp value in filename 'junk'",
str(cm.exception)) str(cm.exception))
def test_hash_cleanup_listdir_reclaim_with_data_files(self): def test_cleanup_ondisk_files_reclaim_with_data_files(self):
# Each scenario specifies a list of (filename, extension, [survives]) # Each scenario specifies a list of (filename, extension, [survives])
# tuples. If extension is set or 'survives' is True, the filename # tuples. If extension is set or 'survives' is True, the filename
# should still be in the dir after cleanup. # should still be in the dir after cleanup.
@ -1250,7 +1250,7 @@ class TestDiskFileManager(DiskFileManagerMixin, unittest.TestCase):
[('%s.meta' % older, '.meta'), [('%s.meta' % older, '.meta'),
('%s.data' % much_older, '.data')]] ('%s.data' % much_older, '.data')]]
self._test_hash_cleanup_listdir_files(scenarios, POLICIES.default, self._test_cleanup_ondisk_files_files(scenarios, POLICIES.default,
reclaim_age=1000) reclaim_age=1000)
def test_yield_hashes(self): def test_yield_hashes(self):
@ -1498,7 +1498,7 @@ class TestECDiskFileManager(DiskFileManagerMixin, unittest.TestCase):
] ]
self._test_get_ondisk_files(scenarios, POLICIES.default, None) self._test_get_ondisk_files(scenarios, POLICIES.default, None)
self._test_hash_cleanup_listdir_files(scenarios, POLICIES.default) self._test_cleanup_ondisk_files_files(scenarios, POLICIES.default)
self._test_yield_hashes_cleanup(scenarios, POLICIES.default) self._test_yield_hashes_cleanup(scenarios, POLICIES.default)
def test_get_ondisk_files_with_ec_policy_and_frag_index(self): def test_get_ondisk_files_with_ec_policy_and_frag_index(self):
@ -1565,10 +1565,10 @@ class TestECDiskFileManager(DiskFileManagerMixin, unittest.TestCase):
] ]
self._test_get_ondisk_files(scenarios, POLICIES.default, frag_index=1) self._test_get_ondisk_files(scenarios, POLICIES.default, frag_index=1)
# note: not calling self._test_hash_cleanup_listdir_files(scenarios, 0) # note: not calling self._test_cleanup_ondisk_files_files(scenarios, 0)
# here due to the anomalous scenario as commented above # here due to the anomalous scenario as commented above
def test_hash_cleanup_listdir_reclaim_with_data_files(self): def test_cleanup_ondisk_files_reclaim_with_data_files(self):
# Each scenario specifies a list of (filename, extension, [survives]) # Each scenario specifies a list of (filename, extension, [survives])
# tuples. If extension is set or 'survives' is True, the filename # tuples. If extension is set or 'survives' is True, the filename
# should still be in the dir after cleanup. # should still be in the dir after cleanup.
@ -1624,7 +1624,7 @@ class TestECDiskFileManager(DiskFileManagerMixin, unittest.TestCase):
[('%s.meta' % older, False, False), [('%s.meta' % older, False, False),
('%s.durable' % much_older, False, False)]] ('%s.durable' % much_older, False, False)]]
self._test_hash_cleanup_listdir_files(scenarios, POLICIES.default, self._test_cleanup_ondisk_files_files(scenarios, POLICIES.default,
reclaim_age=1000) reclaim_age=1000)
def test_get_ondisk_files_with_stray_meta(self): def test_get_ondisk_files_with_stray_meta(self):
@ -3176,12 +3176,12 @@ class DiskFileMixin(BaseDiskFileTestMixin):
if policy.policy_type == EC_POLICY: if policy.policy_type == EC_POLICY:
self.assertTrue(isinstance(mock_fsync.call_args[0][0], int)) self.assertTrue(isinstance(mock_fsync.call_args[0][0], int))
def test_commit_ignores_hash_cleanup_listdir_error(self): def test_commit_ignores_cleanup_ondisk_files_error(self):
for policy in POLICIES: for policy in POLICIES:
# Check OSError from hash_cleanup_listdir is caught and ignored # Check OSError from cleanup_ondisk_files is caught and ignored
mock_hcl = mock.MagicMock(side_effect=OSError) mock_cleanup = mock.MagicMock(side_effect=OSError)
df = self._simple_get_diskfile(account='a', container='c', df = self._simple_get_diskfile(account='a', container='c',
obj='o_hcl_error', policy=policy) obj='o_error', policy=policy)
timestamp = Timestamp(time()) timestamp = Timestamp(time())
with df.create() as writer: with df.create() as writer:
@ -3192,13 +3192,13 @@ class DiskFileMixin(BaseDiskFileTestMixin):
} }
writer.put(metadata) writer.put(metadata)
with mock.patch(self._manager_mock( with mock.patch(self._manager_mock(
'cleanup_ondisk_files', df), mock_hcl): 'cleanup_ondisk_files', df), mock_cleanup):
writer.commit(timestamp) writer.commit(timestamp)
expected = { expected = {
EC_POLICY: 1, EC_POLICY: 1,
REPL_POLICY: 0, REPL_POLICY: 0,
}[policy.policy_type] }[policy.policy_type]
self.assertEqual(expected, mock_hcl.call_count) self.assertEqual(expected, mock_cleanup.call_count)
expected = ['%s.data' % timestamp.internal] expected = ['%s.data' % timestamp.internal]
if policy.policy_type == EC_POLICY: if policy.policy_type == EC_POLICY:
expected = ['%s#2.data' % timestamp.internal, expected = ['%s#2.data' % timestamp.internal,
@ -3208,8 +3208,8 @@ class DiskFileMixin(BaseDiskFileTestMixin):
'Unexpected dir listing %s' % dl) 'Unexpected dir listing %s' % dl)
self.assertEqual(sorted(expected), sorted(dl)) self.assertEqual(sorted(expected), sorted(dl))
def test_number_calls_to_hash_cleanup_listdir_during_create(self): def test_number_calls_to_cleanup_ondisk_files_during_create(self):
# Check how many calls are made to hash_cleanup_listdir, and when, # Check how many calls are made to cleanup_ondisk_files, and when,
# during put(), commit() sequence # during put(), commit() sequence
for policy in POLICIES: for policy in POLICIES:
expected = { expected = {
@ -3217,7 +3217,7 @@ class DiskFileMixin(BaseDiskFileTestMixin):
REPL_POLICY: (1, 0), REPL_POLICY: (1, 0),
}[policy.policy_type] }[policy.policy_type]
df = self._simple_get_diskfile(account='a', container='c', df = self._simple_get_diskfile(account='a', container='c',
obj='o_hcl_error', policy=policy) obj='o_error', policy=policy)
timestamp = Timestamp(time()) timestamp = Timestamp(time())
with df.create() as writer: with df.create() as writer:
metadata = { metadata = {
@ -3226,16 +3226,16 @@ class DiskFileMixin(BaseDiskFileTestMixin):
'Content-Length': '0', 'Content-Length': '0',
} }
with mock.patch(self._manager_mock( with mock.patch(self._manager_mock(
'cleanup_ondisk_files', df)) as mock_hcl: 'cleanup_ondisk_files', df)) as mock_cleanup:
writer.put(metadata) writer.put(metadata)
self.assertEqual(expected[0], mock_hcl.call_count) self.assertEqual(expected[0], mock_cleanup.call_count)
with mock.patch(self._manager_mock( with mock.patch(self._manager_mock(
'cleanup_ondisk_files', df)) as mock_hcl: 'cleanup_ondisk_files', df)) as mock_cleanup:
writer.commit(timestamp) writer.commit(timestamp)
self.assertEqual(expected[1], mock_hcl.call_count) self.assertEqual(expected[1], mock_cleanup.call_count)
def test_number_calls_to_hash_cleanup_listdir_during_delete(self): def test_number_calls_to_cleanup_ondisk_files_during_delete(self):
# Check how many calls are made to hash_cleanup_listdir, and when, # Check how many calls are made to cleanup_ondisk_files, and when,
# for delete() and necessary prerequisite steps # for delete() and necessary prerequisite steps
for policy in POLICIES: for policy in POLICIES:
expected = { expected = {
@ -3243,7 +3243,7 @@ class DiskFileMixin(BaseDiskFileTestMixin):
REPL_POLICY: (1, 0, 1), REPL_POLICY: (1, 0, 1),
}[policy.policy_type] }[policy.policy_type]
df = self._simple_get_diskfile(account='a', container='c', df = self._simple_get_diskfile(account='a', container='c',
obj='o_hcl_error', policy=policy) obj='o_error', policy=policy)
timestamp = Timestamp(time()) timestamp = Timestamp(time())
with df.create() as writer: with df.create() as writer:
metadata = { metadata = {
@ -3252,18 +3252,18 @@ class DiskFileMixin(BaseDiskFileTestMixin):
'Content-Length': '0', 'Content-Length': '0',
} }
with mock.patch(self._manager_mock( with mock.patch(self._manager_mock(
'cleanup_ondisk_files', df)) as mock_hcl: 'cleanup_ondisk_files', df)) as mock_cleanup:
writer.put(metadata) writer.put(metadata)
self.assertEqual(expected[0], mock_hcl.call_count) self.assertEqual(expected[0], mock_cleanup.call_count)
with mock.patch(self._manager_mock( with mock.patch(self._manager_mock(
'cleanup_ondisk_files', df)) as mock_hcl: 'cleanup_ondisk_files', df)) as mock_cleanup:
writer.commit(timestamp) writer.commit(timestamp)
self.assertEqual(expected[1], mock_hcl.call_count) self.assertEqual(expected[1], mock_cleanup.call_count)
with mock.patch(self._manager_mock( with mock.patch(self._manager_mock(
'cleanup_ondisk_files', df)) as mock_hcl: 'cleanup_ondisk_files', df)) as mock_cleanup:
timestamp = Timestamp(time()) timestamp = Timestamp(time())
df.delete(timestamp) df.delete(timestamp)
self.assertEqual(expected[2], mock_hcl.call_count) self.assertEqual(expected[2], mock_cleanup.call_count)
def test_delete(self): def test_delete(self):
for policy in POLICIES: for policy in POLICIES:
@ -3686,15 +3686,16 @@ class DiskFileMixin(BaseDiskFileTestMixin):
# open() was attempted, but no data file so expect None # open() was attempted, but no data file so expect None
self.assertIsNone(df.durable_timestamp) self.assertIsNone(df.durable_timestamp)
def test_error_in_hash_cleanup_listdir(self): def test_error_in_cleanup_ondisk_files(self):
def mock_hcl(*args, **kwargs): def mock_cleanup(*args, **kwargs):
raise OSError() raise OSError()
df = self._get_open_disk_file() df = self._get_open_disk_file()
file_count = len(os.listdir(df._datadir)) file_count = len(os.listdir(df._datadir))
ts = time() ts = time()
with mock.patch(self._manager_mock('cleanup_ondisk_files'), mock_hcl): with mock.patch(
self._manager_mock('cleanup_ondisk_files'), mock_cleanup):
try: try:
df.delete(ts) df.delete(ts)
except OSError: except OSError:
@ -4490,27 +4491,27 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase):
class TestSuffixHashes(unittest.TestCase): class TestSuffixHashes(unittest.TestCase):
""" """
This tests all things related to hashing suffixes and therefore This tests all things related to hashing suffixes and therefore
there's also few test methods for hash_cleanup_listdir as well there's also few test methods for cleanup_ondisk_files as well
(because it's used by hash_suffix). (because it's used by hash_suffix).
The public interface to suffix hashing is on the Manager:: The public interface to suffix hashing is on the Manager::
* hash_cleanup_listdir(hsh_path) * cleanup_ondisk_files(hsh_path)
* get_hashes(device, partition, suffixes, policy) * get_hashes(device, partition, suffixes, policy)
* invalidate_hash(suffix_dir) * invalidate_hash(suffix_dir)
The Manager.get_hashes method (used by the REPLICATE verb) The Manager.get_hashes method (used by the REPLICATE verb)
calls Manager._get_hashes (which may be an alias to the module calls Manager._get_hashes (which may be an alias to the module
method get_hashes), which calls hash_suffix, which calls method get_hashes), which calls hash_suffix, which calls
hash_cleanup_listdir. cleanup_ondisk_files.
Outside of that, hash_cleanup_listdir and invalidate_hash are Outside of that, cleanup_ondisk_files and invalidate_hash are
used mostly after writing new files via PUT or DELETE. used mostly after writing new files via PUT or DELETE.
Test methods are organized by:: Test methods are organized by::
* hash_cleanup_listdir tests - behaviors * cleanup_ondisk_files tests - behaviors
* hash_cleanup_listdir tests - error handling * cleanup_ondisk_files tests - error handling
* invalidate_hash tests - behavior * invalidate_hash tests - behavior
* invalidate_hash tests - error handling * invalidate_hash tests - error handling
* get_hashes tests - hash_suffix behaviors * get_hashes tests - hash_suffix behaviors
@ -4583,7 +4584,7 @@ class TestSuffixHashes(unittest.TestCase):
filename += '.meta' filename += '.meta'
return filename return filename
def check_hash_cleanup_listdir(self, policy, input_files, output_files): def check_cleanup_ondisk_files(self, policy, input_files, output_files):
orig_unlink = os.unlink orig_unlink = os.unlink
file_list = list(input_files) file_list = list(input_files)
@ -4611,17 +4612,17 @@ class TestSuffixHashes(unittest.TestCase):
files = df_mgr.cleanup_ondisk_files('/whatever')['files'] files = df_mgr.cleanup_ondisk_files('/whatever')['files']
self.assertEqual(files, output_files) self.assertEqual(files, output_files)
# hash_cleanup_listdir tests - behaviors # cleanup_ondisk_files tests - behaviors
def test_hash_cleanup_listdir_purge_data_newer_ts(self): def test_cleanup_ondisk_files_purge_data_newer_ts(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
# purge .data if there's a newer .ts # purge .data if there's a newer .ts
file1 = self._datafilename(self.ts(), policy) file1 = self._datafilename(self.ts(), policy)
file2 = self.ts().internal + '.ts' file2 = self.ts().internal + '.ts'
file_list = [file1, file2] file_list = [file1, file2]
self.check_hash_cleanup_listdir(policy, file_list, [file2]) self.check_cleanup_ondisk_files(policy, file_list, [file2])
def test_hash_cleanup_listdir_purge_expired_ts(self): def test_cleanup_ondisk_files_purge_expired_ts(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
# purge older .ts files if there's a newer .data # purge older .ts files if there's a newer .data
file1 = self.ts().internal + '.ts' file1 = self.ts().internal + '.ts'
@ -4635,9 +4636,9 @@ class TestSuffixHashes(unittest.TestCase):
EC_POLICY: [file3, file2], EC_POLICY: [file3, file2],
REPL_POLICY: [file3], REPL_POLICY: [file3],
}[policy.policy_type] }[policy.policy_type]
self.check_hash_cleanup_listdir(policy, file_list, expected) self.check_cleanup_ondisk_files(policy, file_list, expected)
def test_hash_cleanup_listdir_purge_ts_newer_data(self): def test_cleanup_ondisk_files_purge_ts_newer_data(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
# purge .ts if there's a newer .data # purge .ts if there's a newer .data
file1 = self.ts().internal + '.ts' file1 = self.ts().internal + '.ts'
@ -4651,9 +4652,9 @@ class TestSuffixHashes(unittest.TestCase):
EC_POLICY: [durable_file, file2], EC_POLICY: [durable_file, file2],
REPL_POLICY: [file2], REPL_POLICY: [file2],
}[policy.policy_type] }[policy.policy_type]
self.check_hash_cleanup_listdir(policy, file_list, expected) self.check_cleanup_ondisk_files(policy, file_list, expected)
def test_hash_cleanup_listdir_purge_older_ts(self): def test_cleanup_ondisk_files_purge_older_ts(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
file1 = self.ts().internal + '.ts' file1 = self.ts().internal + '.ts'
file2 = self.ts().internal + '.ts' file2 = self.ts().internal + '.ts'
@ -4667,9 +4668,9 @@ class TestSuffixHashes(unittest.TestCase):
REPL_POLICY: [file4, file3], REPL_POLICY: [file4, file3],
}[policy.policy_type] }[policy.policy_type]
file_list = [file1, file2, file3, file4] file_list = [file1, file2, file3, file4]
self.check_hash_cleanup_listdir(policy, file_list, expected) self.check_cleanup_ondisk_files(policy, file_list, expected)
def test_hash_cleanup_listdir_keep_meta_data_purge_ts(self): def test_cleanup_ondisk_files_keep_meta_data_purge_ts(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
file1 = self.ts().internal + '.ts' file1 = self.ts().internal + '.ts'
file2 = self.ts().internal + '.ts' file2 = self.ts().internal + '.ts'
@ -4686,17 +4687,17 @@ class TestSuffixHashes(unittest.TestCase):
EC_POLICY: [file4, durable_filename, file3], EC_POLICY: [file4, durable_filename, file3],
REPL_POLICY: [file4, file3], REPL_POLICY: [file4, file3],
}[policy.policy_type] }[policy.policy_type]
self.check_hash_cleanup_listdir(policy, file_list, expected) self.check_cleanup_ondisk_files(policy, file_list, expected)
def test_hash_cleanup_listdir_keep_one_ts(self): def test_cleanup_ondisk_files_keep_one_ts(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
file1, file2, file3 = [self.ts().internal + '.ts' file1, file2, file3 = [self.ts().internal + '.ts'
for i in range(3)] for i in range(3)]
file_list = [file1, file2, file3] file_list = [file1, file2, file3]
# keep only latest of multiple .ts files # keep only latest of multiple .ts files
self.check_hash_cleanup_listdir(policy, file_list, [file3]) self.check_cleanup_ondisk_files(policy, file_list, [file3])
def test_hash_cleanup_listdir_multi_data_file(self): def test_cleanup_ondisk_files_multi_data_file(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
file1 = self._datafilename(self.ts(), policy, 1) file1 = self._datafilename(self.ts(), policy, 1)
file2 = self._datafilename(self.ts(), policy, 2) file2 = self._datafilename(self.ts(), policy, 2)
@ -4708,9 +4709,9 @@ class TestSuffixHashes(unittest.TestCase):
REPL_POLICY: [file3] REPL_POLICY: [file3]
}[policy.policy_type] }[policy.policy_type]
file_list = [file1, file2, file3] file_list = [file1, file2, file3]
self.check_hash_cleanup_listdir(policy, file_list, expected) self.check_cleanup_ondisk_files(policy, file_list, expected)
def test_hash_cleanup_listdir_keeps_one_datafile(self): def test_cleanup_ondisk_files_keeps_one_datafile(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
timestamps = [self.ts() for i in range(3)] timestamps = [self.ts() for i in range(3)]
file1 = self._datafilename(timestamps[0], policy, 1) file1 = self._datafilename(timestamps[0], policy, 1)
@ -4727,9 +4728,9 @@ class TestSuffixHashes(unittest.TestCase):
# keep only latest of multiple .data files # keep only latest of multiple .data files
REPL_POLICY: [file3] REPL_POLICY: [file3]
}[policy.policy_type] }[policy.policy_type]
self.check_hash_cleanup_listdir(policy, file_list, expected) self.check_cleanup_ondisk_files(policy, file_list, expected)
def test_hash_cleanup_listdir_keep_one_meta(self): def test_cleanup_ondisk_files_keep_one_meta(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
# keep only latest of multiple .meta files # keep only latest of multiple .meta files
t_data = self.ts() t_data = self.ts()
@ -4743,50 +4744,50 @@ class TestSuffixHashes(unittest.TestCase):
EC_POLICY: [file3, durable_file, file1], EC_POLICY: [file3, durable_file, file1],
REPL_POLICY: [file3, file1] REPL_POLICY: [file3, file1]
}[policy.policy_type] }[policy.policy_type]
self.check_hash_cleanup_listdir(policy, file_list, expected) self.check_cleanup_ondisk_files(policy, file_list, expected)
def test_hash_cleanup_listdir_only_meta(self): def test_cleanup_ondisk_files_only_meta(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
file1, file2 = [self.ts().internal + '.meta' for i in range(2)] file1, file2 = [self.ts().internal + '.meta' for i in range(2)]
file_list = [file1, file2] file_list = [file1, file2]
self.check_hash_cleanup_listdir(policy, file_list, [file2]) self.check_cleanup_ondisk_files(policy, file_list, [file2])
def test_hash_cleanup_listdir_ignore_orphaned_ts(self): def test_cleanup_ondisk_files_ignore_orphaned_ts(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
# A more recent orphaned .meta file will prevent old .ts files # A more recent orphaned .meta file will prevent old .ts files
# from being cleaned up otherwise # from being cleaned up otherwise
file1, file2 = [self.ts().internal + '.ts' for i in range(2)] file1, file2 = [self.ts().internal + '.ts' for i in range(2)]
file3 = self.ts().internal + '.meta' file3 = self.ts().internal + '.meta'
file_list = [file1, file2, file3] file_list = [file1, file2, file3]
self.check_hash_cleanup_listdir(policy, file_list, [file3, file2]) self.check_cleanup_ondisk_files(policy, file_list, [file3, file2])
def test_hash_cleanup_listdir_purge_old_data_only(self): def test_cleanup_ondisk_files_purge_old_data_only(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
# Oldest .data will be purge, .meta and .ts won't be touched # Oldest .data will be purge, .meta and .ts won't be touched
file1 = self._datafilename(self.ts(), policy) file1 = self._datafilename(self.ts(), policy)
file2 = self.ts().internal + '.ts' file2 = self.ts().internal + '.ts'
file3 = self.ts().internal + '.meta' file3 = self.ts().internal + '.meta'
file_list = [file1, file2, file3] file_list = [file1, file2, file3]
self.check_hash_cleanup_listdir(policy, file_list, [file3, file2]) self.check_cleanup_ondisk_files(policy, file_list, [file3, file2])
def test_hash_cleanup_listdir_purge_old_ts(self): def test_cleanup_ondisk_files_purge_old_ts(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
# A single old .ts file will be removed # A single old .ts file will be removed
old_float = time() - (diskfile.ONE_WEEK + 1) old_float = time() - (diskfile.ONE_WEEK + 1)
file1 = Timestamp(old_float).internal + '.ts' file1 = Timestamp(old_float).internal + '.ts'
file_list = [file1] file_list = [file1]
self.check_hash_cleanup_listdir(policy, file_list, []) self.check_cleanup_ondisk_files(policy, file_list, [])
def test_hash_cleanup_listdir_keep_isolated_meta_purge_old_ts(self): def test_cleanup_ondisk_files_keep_isolated_meta_purge_old_ts(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
# A single old .ts file will be removed despite presence of a .meta # A single old .ts file will be removed despite presence of a .meta
old_float = time() - (diskfile.ONE_WEEK + 1) old_float = time() - (diskfile.ONE_WEEK + 1)
file1 = Timestamp(old_float).internal + '.ts' file1 = Timestamp(old_float).internal + '.ts'
file2 = Timestamp(time() + 2).internal + '.meta' file2 = Timestamp(time() + 2).internal + '.meta'
file_list = [file1, file2] file_list = [file1, file2]
self.check_hash_cleanup_listdir(policy, file_list, [file2]) self.check_cleanup_ondisk_files(policy, file_list, [file2])
def test_hash_cleanup_listdir_keep_single_old_data(self): def test_cleanup_ondisk_files_keep_single_old_data(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
old_float = time() - (diskfile.ONE_WEEK + 1) old_float = time() - (diskfile.ONE_WEEK + 1)
file1 = self._datafilename(Timestamp(old_float), policy) file1 = self._datafilename(Timestamp(old_float), policy)
@ -4798,33 +4799,33 @@ class TestSuffixHashes(unittest.TestCase):
else: else:
# A single old .data file will not be removed # A single old .data file will not be removed
expected = file_list expected = file_list
self.check_hash_cleanup_listdir(policy, file_list, expected) self.check_cleanup_ondisk_files(policy, file_list, expected)
def test_hash_cleanup_listdir_drops_isolated_durable(self): def test_cleanup_ondisk_files_drops_isolated_durable(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
if policy.policy_type == EC_POLICY: if policy.policy_type == EC_POLICY:
file1 = Timestamp(time()).internal + '.durable' file1 = Timestamp(time()).internal + '.durable'
file_list = [file1] file_list = [file1]
self.check_hash_cleanup_listdir(policy, file_list, []) self.check_cleanup_ondisk_files(policy, file_list, [])
def test_hash_cleanup_listdir_purges_single_old_meta(self): def test_cleanup_ondisk_files_purges_single_old_meta(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
# A single old .meta file will be removed # A single old .meta file will be removed
old_float = time() - (diskfile.ONE_WEEK + 1) old_float = time() - (diskfile.ONE_WEEK + 1)
file1 = Timestamp(old_float).internal + '.meta' file1 = Timestamp(old_float).internal + '.meta'
file_list = [file1] file_list = [file1]
self.check_hash_cleanup_listdir(policy, file_list, []) self.check_cleanup_ondisk_files(policy, file_list, [])
# hash_cleanup_listdir tests - error handling # cleanup_ondisk_files tests - error handling
def test_hash_cleanup_listdir_hsh_path_enoent(self): def test_cleanup_ondisk_files_hsh_path_enoent(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
df_mgr = self.df_router[policy] df_mgr = self.df_router[policy]
# common.utils.listdir *completely* mutes ENOENT # common.utils.listdir *completely* mutes ENOENT
path = os.path.join(self.testdir, 'does-not-exist') path = os.path.join(self.testdir, 'does-not-exist')
self.assertEqual(df_mgr.cleanup_ondisk_files(path)['files'], []) self.assertEqual(df_mgr.cleanup_ondisk_files(path)['files'], [])
def test_hash_cleanup_listdir_hsh_path_other_oserror(self): def test_cleanup_ondisk_files_hsh_path_other_oserror(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
df_mgr = self.df_router[policy] df_mgr = self.df_router[policy]
with mock.patch('os.listdir') as mock_listdir: with mock.patch('os.listdir') as mock_listdir:
@ -4834,22 +4835,22 @@ class TestSuffixHashes(unittest.TestCase):
self.assertRaises(OSError, df_mgr.cleanup_ondisk_files, self.assertRaises(OSError, df_mgr.cleanup_ondisk_files,
path) path)
def test_hash_cleanup_listdir_reclaim_tombstone_remove_file_error(self): def test_cleanup_ondisk_files_reclaim_tombstone_remove_file_error(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
# Timestamp 1 makes the check routine pretend the file # Timestamp 1 makes the check routine pretend the file
# disappeared after listdir before unlink. # disappeared after listdir before unlink.
file1 = '0000000001.00000.ts' file1 = '0000000001.00000.ts'
file_list = [file1] file_list = [file1]
self.check_hash_cleanup_listdir(policy, file_list, []) self.check_cleanup_ondisk_files(policy, file_list, [])
def test_hash_cleanup_listdir_older_remove_file_error(self): def test_cleanup_ondisk_files_older_remove_file_error(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
# Timestamp 1 makes the check routine pretend the file # Timestamp 1 makes the check routine pretend the file
# disappeared after listdir before unlink. # disappeared after listdir before unlink.
file1 = self._datafilename(Timestamp(1), policy) file1 = self._datafilename(Timestamp(1), policy)
file2 = '0000000002.00000.ts' file2 = '0000000002.00000.ts'
file_list = [file1, file2] file_list = [file1, file2]
self.check_hash_cleanup_listdir(policy, file_list, []) self.check_cleanup_ondisk_files(policy, file_list, [])
# invalidate_hash tests - behavior # invalidate_hash tests - behavior
@ -5472,7 +5473,7 @@ class TestSuffixHashes(unittest.TestCase):
(os.path.join(part_path, '123'), False), (os.path.join(part_path, '123'), False),
]) ])
def test_hash_suffix_hash_cleanup_listdir_enotdir_quarantined(self): def test_hash_suffix_cleanup_ondisk_files_enotdir_quarantined(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
df = self.df_router[policy].get_diskfile( df = self.df_router[policy].get_diskfile(
self.existing_device, '0', 'a', 'c', 'o', policy=policy) self.existing_device, '0', 'a', 'c', 'o', policy=policy)
@ -5501,7 +5502,7 @@ class TestSuffixHashes(unittest.TestCase):
) )
self.assertTrue(os.path.exists(quarantine_path)) self.assertTrue(os.path.exists(quarantine_path))
def test_hash_suffix_hash_cleanup_listdir_other_oserror(self): def test_hash_suffix_cleanup_ondisk_files_other_oserror(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
timestamp = self.ts() timestamp = self.ts()
df_mgr = self.df_router[policy] df_mgr = self.df_router[policy]
@ -5531,7 +5532,7 @@ class TestSuffixHashes(unittest.TestCase):
listdir_calls.append(path) listdir_calls.append(path)
if path == datadir_path: if path == datadir_path:
# we want the part and suffix listdir calls to pass and # we want the part and suffix listdir calls to pass and
# make the hash_cleanup_listdir raise an exception # make the cleanup_ondisk_files raise an exception
raise OSError(errno.EACCES, os.strerror(errno.EACCES)) raise OSError(errno.EACCES, os.strerror(errno.EACCES))
return orig_os_listdir(path) return orig_os_listdir(path)

2
test/unit/obj/test_server.py
View File

@ -1126,7 +1126,7 @@ class TestObjectController(unittest.TestCase):
# data file doesn't exist there (This is sanity because # data file doesn't exist there (This is sanity because
# if .data written unexpectedly, it will be removed # if .data written unexpectedly, it will be removed
# by hash_cleanup_list_dir) # by cleanup_ondisk_files)
datafile = os.path.join( datafile = os.path.join(
self.testdir, 'sda1', self.testdir, 'sda1',
storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p', storage_directory(diskfile.get_data_dir(POLICIES[0]), 'p',