From 62a9dbca7613a760bc9ea179ab030f433b76f4a6 Mon Sep 17 00:00:00 2001 From: kim woo seok Date: Fri, 22 Sep 2023 02:56:48 +0900 Subject: [PATCH] Add unittest of swift-recon-cron Moving bin script of swift-recon-cron to cli module for unittest Delete unused `logger` parameter in get_async_count function Partial-Bug: #1743656 Change-Id: I4ca91e3b519a99f3096b95b286779a183e936eb7 --- bin/swift-recon-cron | 62 +-------------------------- swift/cli/recon_cron.py | 73 ++++++++++++++++++++++++++++++++ test/unit/cli/test_recon_cron.py | 53 +++++++++++++++++++++++ 3 files changed, 128 insertions(+), 60 deletions(-) create mode 100644 swift/cli/recon_cron.py create mode 100644 test/unit/cli/test_recon_cron.py diff --git a/bin/swift-recon-cron b/bin/swift-recon-cron index f17223feef..cdb0fd95ed 100755 --- a/bin/swift-recon-cron +++ b/bin/swift-recon-cron @@ -16,67 +16,9 @@ swift-recon-cron.py """ -import os import sys -import time - -from eventlet import Timeout - -from swift.common.utils import get_logger, dump_recon_cache, readconf, \ - lock_path -from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH -from swift.obj.diskfile import ASYNCDIR_BASE - - -def get_async_count(device_dir, logger): - async_count = 0 - for i in os.listdir(device_dir): - device = os.path.join(device_dir, i) - if not os.path.isdir(device): - continue - for asyncdir in os.listdir(device): - # skip stuff like "accounts", "containers", etc. - if not (asyncdir == ASYNCDIR_BASE or - asyncdir.startswith(ASYNCDIR_BASE + '-')): - continue - async_pending = os.path.join(device, asyncdir) - - if os.path.isdir(async_pending): - for entry in os.listdir(async_pending): - if os.path.isdir(os.path.join(async_pending, entry)): - async_hdir = os.path.join(async_pending, entry) - async_count += len(os.listdir(async_hdir)) - return async_count - - -def main(): - try: - conf_path = sys.argv[1] - except Exception: - print("Usage: %s CONF_FILE" % sys.argv[0].split('/')[-1]) - print("ex: swift-recon-cron /etc/swift/object-server.conf") - sys.exit(1) - conf = readconf(conf_path, 'filter:recon') - device_dir = conf.get('devices', '/srv/node') - recon_cache_path = conf.get('recon_cache_path', DEFAULT_RECON_CACHE_PATH) - recon_lock_path = conf.get('recon_lock_path', '/var/lock') - cache_file = os.path.join(recon_cache_path, RECON_OBJECT_FILE) - lock_dir = os.path.join(recon_lock_path, "swift-recon-object-cron") - conf['log_name'] = conf.get('log_name', 'recon-cron') - logger = get_logger(conf, log_route='recon-cron') - try: - with lock_path(lock_dir): - asyncs = get_async_count(device_dir, logger) - dump_recon_cache({ - 'async_pending': asyncs, - 'async_pending_last': time.time(), - }, cache_file, logger) - except (Exception, Timeout) as err: - msg = 'Exception during recon-cron while accessing devices' - logger.exception(msg) - print('%s: %s' % (msg, err)) - sys.exit(1) +from swift.cli.recon_cron import main if __name__ == '__main__': - main() + sys.exit(main()) diff --git a/swift/cli/recon_cron.py b/swift/cli/recon_cron.py new file mode 100644 index 0000000000..b7cf3566d5 --- /dev/null +++ b/swift/cli/recon_cron.py @@ -0,0 +1,73 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import time + +from eventlet import Timeout + +from swift.common.utils import get_logger, dump_recon_cache, readconf, \ + lock_path +from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH +from swift.obj.diskfile import ASYNCDIR_BASE + + +def get_async_count(device_dir): + async_count = 0 + for i in os.listdir(device_dir): + device = os.path.join(device_dir, i) + if not os.path.isdir(device): + continue + for asyncdir in os.listdir(device): + # skip stuff like "accounts", "containers", etc. + if not (asyncdir == ASYNCDIR_BASE or + asyncdir.startswith(ASYNCDIR_BASE + '-')): + continue + async_pending = os.path.join(device, asyncdir) + + if os.path.isdir(async_pending): + for entry in os.listdir(async_pending): + if os.path.isdir(os.path.join(async_pending, entry)): + async_hdir = os.path.join(async_pending, entry) + async_count += len(os.listdir(async_hdir)) + return async_count + + +def main(): + try: + conf_path = sys.argv[1] + except Exception: + print("Usage: %s CONF_FILE" % sys.argv[0].split('/')[-1]) + print("ex: swift-recon-cron /etc/swift/object-server.conf") + sys.exit(1) + conf = readconf(conf_path, 'filter:recon') + device_dir = conf.get('devices', '/srv/node') + recon_cache_path = conf.get('recon_cache_path', DEFAULT_RECON_CACHE_PATH) + recon_lock_path = conf.get('recon_lock_path', '/var/lock') + cache_file = os.path.join(recon_cache_path, RECON_OBJECT_FILE) + lock_dir = os.path.join(recon_lock_path, "swift-recon-object-cron") + conf['log_name'] = conf.get('log_name', 'recon-cron') + logger = get_logger(conf, log_route='recon-cron') + try: + with lock_path(lock_dir): + asyncs = get_async_count(device_dir) + dump_recon_cache({ + 'async_pending': asyncs, + 'async_pending_last': time.time(), + }, cache_file, logger) + except (Exception, Timeout) as err: + msg = 'Exception during recon-cron while accessing devices' + logger.exception(msg) + print('%s: %s' % (msg, err)) + sys.exit(1) diff --git a/test/unit/cli/test_recon_cron.py b/test/unit/cli/test_recon_cron.py new file mode 100644 index 0000000000..4a24fd5fc2 --- /dev/null +++ b/test/unit/cli/test_recon_cron.py @@ -0,0 +1,53 @@ +# Copyright (c) 2010-2022 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile +import shutil +import os + +from unittest import TestCase +from swift.cli.recon_cron import get_async_count +from swift.obj.diskfile import ASYNCDIR_BASE + + +class TestReconCron(TestCase): + + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_get_async_count(self): + device_dir = os.path.join(self.temp_dir, 'device') + os.makedirs(device_dir) + device_index = os.path.join(device_dir, '1') + os.makedirs(device_index) + async_dir = os.path.join(device_index, ASYNCDIR_BASE) + os.makedirs(async_dir) + entry1 = os.path.join(async_dir, 'entry1') + entry2 = os.path.join(async_dir, 'entry2') + os.makedirs(entry1) + os.makedirs(entry2) + + pending_file1 = os.path.join(entry1, 'pending_file1') + pending_file2 = os.path.join(entry1, 'pending_file2') + pending_file3 = os.path.join(entry2, 'pending_file3') + open(pending_file1, 'w').close() + open(pending_file2, 'w').close() + open(pending_file3, 'w').close() + + count = get_async_count(device_dir) + self.assertEqual(count, 3)