From 8f9d7db7f12527f0d848d0c89c60755ee14bc58b Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 17 Jun 2024 13:14:53 -0700 Subject: [PATCH] Use entry_points for a bunch more executables See https://review.opendev.org/c/openstack/swift/+/918365 for motivation. A handful of [files]scripts entries still remain, but they're more involved to change over. These ones were all fairly mechanical to fix. Related-Change: Ifcc8138e7b55d5b82bea0d411ec6bfcca2c77c83 Change-Id: Ia43d7cd3921bc6c9ff0cee3100ef5f486fd3edcb --- bin/swift-account-auditor | 7 ++---- bin/swift-account-reaper | 7 ++---- bin/swift-account-replicator | 18 ++------------ bin/swift-container-auditor | 7 ++---- bin/swift-container-reconciler | 7 ++---- bin/swift-container-replicator | 18 ++------------ bin/swift-container-sharder | 21 ++-------------- bin/swift-container-sync | 7 ++---- bin/swift-container-updater | 7 ++---- bin/swift-object-auditor | 13 ++-------- bin/swift-object-expirer | 17 ++----------- bin/swift-object-reconstructor | 17 ++----------- bin/swift-object-relinker | 3 +-- bin/swift-object-replicator | 21 ++-------------- bin/swift-object-updater | 7 ++---- bin/swift-ring-builder | 21 ++-------------- setup.cfg | 44 +++++++++++++++++----------------- swift/account/auditor.py | 11 +++++++++ swift/account/reaper.py | 12 +++++++++- swift/account/replicator.py | 22 +++++++++++++++++ swift/cli/relinker.py | 2 +- swift/cli/ringbuilder.py | 24 +++++++++++++++++++ swift/container/auditor.py | 11 +++++++++ swift/container/reconciler.py | 13 ++++++++-- swift/container/replicator.py | 23 +++++++++++++++++- swift/container/sharder.py | 26 +++++++++++++++++++- swift/container/sync.py | 12 +++++++++- swift/container/updater.py | 13 ++++++++-- swift/obj/auditor.py | 20 ++++++++++++++-- swift/obj/expirer.py | 23 ++++++++++++++++-- swift/obj/reconstructor.py | 23 ++++++++++++++++-- swift/obj/replicator.py | 27 +++++++++++++++++++-- swift/obj/updater.py | 13 ++++++++-- 33 files changed, 309 insertions(+), 208 deletions(-) diff --git a/bin/swift-account-auditor b/bin/swift-account-auditor index f7895c8ccf..4581c5052b 100755 --- a/bin/swift-account-auditor +++ b/bin/swift-account-auditor @@ -14,10 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.account.auditor import AccountAuditor -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon +from swift.account import auditor if __name__ == '__main__': - conf_file, options = parse_options(once=True) - run_daemon(AccountAuditor, conf_file, **options) + auditor.main() diff --git a/bin/swift-account-reaper b/bin/swift-account-reaper index 885c4fd75f..3ea731bfbd 100755 --- a/bin/swift-account-reaper +++ b/bin/swift-account-reaper @@ -14,10 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.account.reaper import AccountReaper -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon +from swift.account import reaper if __name__ == '__main__': - conf_file, options = parse_options(once=True) - run_daemon(AccountReaper, conf_file, **options) + reaper.main() diff --git a/bin/swift-account-replicator b/bin/swift-account-replicator index 072b6e0315..9f500f0bb8 100755 --- a/bin/swift-account-replicator +++ b/bin/swift-account-replicator @@ -14,21 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import optparse - -from swift.account.replicator import AccountReplicator -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon +from swift.account import replicator if __name__ == '__main__': - parser = optparse.OptionParser("%prog CONFIG [options]") - parser.add_option('-d', '--devices', - help=('Replicate only given devices. ' - 'Comma-separated list. ' - 'Only has effect if --once is used.')) - parser.add_option('-p', '--partitions', - help=('Replicate only given partitions. ' - 'Comma-separated list. ' - 'Only has effect if --once is used.')) - conf_file, options = parse_options(parser=parser, once=True) - run_daemon(AccountReplicator, conf_file, **options) + replicator.main() diff --git a/bin/swift-container-auditor b/bin/swift-container-auditor index 5021513f07..a56911f24e 100755 --- a/bin/swift-container-auditor +++ b/bin/swift-container-auditor @@ -14,10 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.container.auditor import ContainerAuditor -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon +from swift.container import auditor if __name__ == '__main__': - conf_file, options = parse_options(once=True) - run_daemon(ContainerAuditor, conf_file, **options) + auditor.main() diff --git a/bin/swift-container-reconciler b/bin/swift-container-reconciler index 29f35d8a82..194936fdbc 100755 --- a/bin/swift-container-reconciler +++ b/bin/swift-container-reconciler @@ -12,10 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.container.reconciler import ContainerReconciler -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon +from swift.container import reconciler if __name__ == '__main__': - conf_file, options = parse_options(once=True) - run_daemon(ContainerReconciler, conf_file, **options) + reconciler.main() diff --git a/bin/swift-container-replicator b/bin/swift-container-replicator index d1990216c5..bd08307bb1 100755 --- a/bin/swift-container-replicator +++ b/bin/swift-container-replicator @@ -14,21 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import optparse - -from swift.container.replicator import ContainerReplicator -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon +from swift.container import replicator if __name__ == '__main__': - parser = optparse.OptionParser("%prog CONFIG [options]") - parser.add_option('-d', '--devices', - help=('Replicate only given devices. ' - 'Comma-separated list. ' - 'Only has effect if --once is used.')) - parser.add_option('-p', '--partitions', - help=('Replicate only given partitions. ' - 'Comma-separated list. ' - 'Only has effect if --once is used.')) - conf_file, options = parse_options(parser=parser, once=True) - run_daemon(ContainerReplicator, conf_file, **options) + replicator.main() diff --git a/bin/swift-container-sharder b/bin/swift-container-sharder index 0374b8d91b..ce59e338d0 100755 --- a/bin/swift-container-sharder +++ b/bin/swift-container-sharder @@ -14,24 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.container.sharder import ContainerSharder -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon -from optparse import OptionParser +from swift.container import sharder if __name__ == '__main__': - parser = OptionParser("%prog CONFIG [options]") - parser.add_option('-d', '--devices', - help='Shard containers only on given devices. ' - 'Comma-separated list. ' - 'Only has effect if --once is used.') - parser.add_option('-p', '--partitions', - help='Shard containers only in given partitions. ' - 'Comma-separated list. ' - 'Only has effect if --once is used.') - parser.add_option('--no-auto-shard', action='store_false', - dest='auto_shard', default=None, - help='Disable auto-sharding. Overrides the auto_shard ' - 'value in the config file.') - conf_file, options = parse_options(parser=parser, once=True) - run_daemon(ContainerSharder, conf_file, **options) + sharder.main() diff --git a/bin/swift-container-sync b/bin/swift-container-sync index b885015703..38acac881e 100755 --- a/bin/swift-container-sync +++ b/bin/swift-container-sync @@ -14,10 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.container.sync import ContainerSync -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon +from swift.container import sync if __name__ == '__main__': - conf_file, options = parse_options(once=True) - run_daemon(ContainerSync, conf_file, **options) + sync.main() diff --git a/bin/swift-container-updater b/bin/swift-container-updater index 96c38f04be..11fb11fde7 100755 --- a/bin/swift-container-updater +++ b/bin/swift-container-updater @@ -14,10 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.container.updater import ContainerUpdater -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon +from swift.container import updater if __name__ == '__main__': - conf_file, options = parse_options(once=True) - run_daemon(ContainerUpdater, conf_file, **options) + updater.main() diff --git a/bin/swift-object-auditor b/bin/swift-object-auditor index 55d0054b77..dc56d53ae6 100755 --- a/bin/swift-object-auditor +++ b/bin/swift-object-auditor @@ -14,16 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.obj.auditor import ObjectAuditor -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon -from optparse import OptionParser +from swift.obj import auditor if __name__ == '__main__': - parser = OptionParser("%prog CONFIG [options]") - parser.add_option('-z', '--zero_byte_fps', - help='Audit only zero byte files at specified files/sec') - parser.add_option('-d', '--devices', - help='Audit only given devices. Comma-separated list') - conf_file, options = parse_options(parser=parser, once=True) - run_daemon(ObjectAuditor, conf_file, **options) + auditor.main() diff --git a/bin/swift-object-expirer b/bin/swift-object-expirer index 7deaf57508..735af596ac 100755 --- a/bin/swift-object-expirer +++ b/bin/swift-object-expirer @@ -14,20 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.common.daemon import run_daemon -from swift.common.utils import parse_options -from swift.obj.expirer import ObjectExpirer -from optparse import OptionParser +from swift.obj import expirer if __name__ == '__main__': - parser = OptionParser("%prog CONFIG [options]") - parser.add_option('--processes', dest='processes', - help="Number of processes to use to do the work, don't " - "use this option to do all the work in one process") - parser.add_option('--process', dest='process', - help="Process number for this process, don't use " - "this option to do all the work in one process, this " - "is used to determine which part of the work this " - "process should do") - conf_file, options = parse_options(parser=parser, once=True) - run_daemon(ObjectExpirer, conf_file, **options) + expirer.main() diff --git a/bin/swift-object-reconstructor b/bin/swift-object-reconstructor index 2c28437cd8..9271e62e5b 100755 --- a/bin/swift-object-reconstructor +++ b/bin/swift-object-reconstructor @@ -14,20 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.obj.reconstructor import ObjectReconstructor -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon -from optparse import OptionParser +from swift.obj import reconstructor if __name__ == '__main__': - parser = OptionParser("%prog CONFIG [options]") - parser.add_option('-d', '--devices', - help='Reconstruct only given devices. ' - 'Comma-separated list. ' - 'Only has effect if --once is used.') - parser.add_option('-p', '--partitions', - help='Reconstruct only given partitions. ' - 'Comma-separated list. ' - 'Only has effect if --once is used.') - conf_file, options = parse_options(parser=parser, once=True) - run_daemon(ObjectReconstructor, conf_file, **options) + reconstructor.main() diff --git a/bin/swift-object-relinker b/bin/swift-object-relinker index ff7e227a64..5457c93f65 100755 --- a/bin/swift-object-relinker +++ b/bin/swift-object-relinker @@ -18,6 +18,5 @@ import sys from swift.cli.relinker import main - if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + sys.exit(main()) diff --git a/bin/swift-object-replicator b/bin/swift-object-replicator index 7a53470a8b..99a8afd1b0 100755 --- a/bin/swift-object-replicator +++ b/bin/swift-object-replicator @@ -14,24 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.obj.replicator import ObjectReplicator -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon -from optparse import OptionParser +from swift.obj import replicator if __name__ == '__main__': - parser = OptionParser("%prog CONFIG [options]") - parser.add_option('-d', '--devices', - help='Replicate only given devices. ' - 'Comma-separated list. ' - 'Only has effect if --once is used.') - parser.add_option('-p', '--partitions', - help='Replicate only given partitions. ' - 'Comma-separated list. ' - 'Only has effect if --once is used.') - parser.add_option('-i', '--policies', - help='Replicate only given policy indices. ' - 'Comma-separated list. ' - 'Only has effect if --once is used.') - conf_file, options = parse_options(parser=parser, once=True) - run_daemon(ObjectReplicator, conf_file, **options) + replicator.main() diff --git a/bin/swift-object-updater b/bin/swift-object-updater index 14ed55c2b2..3294d9d0f1 100755 --- a/bin/swift-object-updater +++ b/bin/swift-object-updater @@ -14,10 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.obj.updater import ObjectUpdater -from swift.common.utils import parse_options -from swift.common.daemon import run_daemon +from swift.obj import updater if __name__ == '__main__': - conf_file, options = parse_options(once=True) - run_daemon(ObjectUpdater, conf_file, **options) + updater.main() diff --git a/bin/swift-ring-builder b/bin/swift-ring-builder index f89b474414..9b38415787 100755 --- a/bin/swift-ring-builder +++ b/bin/swift-ring-builder @@ -14,24 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - -import sys -import traceback - -# We exit code 1 on WARNING statuses, 2 on ERROR. This means we need -# to handle any uncaught exceptions by printing the usual backtrace, -# but then exiting 2 (not 1 as is usual for a python -# exception). - - -def exit_with_status_two(tp, val, tb): - traceback.print_exception(tp, val, tb) - sys.exit(2) - - -sys.excepthook = exit_with_status_two - -from swift.cli.ringbuilder import main +from swift.cli.ringbuilder import error_handling_main if __name__ == "__main__": - sys.exit(main()) + error_handling_main() diff --git a/setup.cfg b/setup.cfg index 8c577c679d..817588aeb8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,39 +43,17 @@ packages = [files] scripts = bin/swift-account-audit - bin/swift-account-auditor bin/swift-account-info - bin/swift-account-reaper - bin/swift-account-replicator bin/swift-config - bin/swift-container-auditor bin/swift-container-info - bin/swift-container-replicator - bin/swift-container-sharder - bin/swift-container-sync - bin/swift-container-updater - bin/swift-container-reconciler bin/swift-reconciler-enqueue bin/swift-dispersion-populate - bin/swift-dispersion-report bin/swift-drive-audit - bin/swift-form-signature bin/swift-get-nodes bin/swift-init - bin/swift-object-auditor - bin/swift-object-expirer bin/swift-object-info - bin/swift-object-replicator - bin/swift-object-reconstructor - bin/swift-object-relinker - bin/swift-object-updater bin/swift-oldies bin/swift-orphans - bin/swift-recon - bin/swift-recon-cron - bin/swift-ring-builder - bin/swift-ring-builder-analyzer - bin/swift-ring-composer [extras] kms_keymaster = @@ -90,13 +68,35 @@ keystone = [entry_points] console_scripts = + swift-account-auditor = swift.account.auditor:main + swift-account-reaper = swift.account.reaper:main + swift-account-replicator = swift.account.replicator:main swift-account-server = swift.account.server:main + swift-container-auditor = swift.container.auditor:main swift-container-deleter = swift.cli.container_deleter:main + swift-container-replicator = swift.container.replicator:main + swift-container-reconciler = swift.container.reconciler:main swift-container-server = swift.container.server:main + swift-container-sharder = swift.container.sharder:main + swift-container-sync = swift.container.sync:main + swift-container-updater = swift.container.updater:main + swift-dispersion-report = swift.cli.dispersion_report:main + swift-form-signature = swift.cli.form_signature:main swift-manage-shard-ranges = swift.cli.manage_shard_ranges:main + swift-object-auditor = swift.obj.auditor:main + swift-object-expirer = swift.obj.expirer:main + swift-object-reconstructor = swift.obj.reconstructor:main + swift-object-relinker = swift.cli.relinker:main + swift-object-replicator = swift.obj.replicator:main swift-object-server = swift.obj.server:main + swift-object-updater = swift.obj.updater:main swift-proxy-server = swift.proxy.server:main + swift-recon = swift.cli.recon:main + swift-recon-cron = swift.cli.recon_cron:main swift-reload = swift.cli.reload:main + swift-ring-builder = swift.cli.ringbuilder:error_handling_main + swift-ring-builder-analyzer = swift.cli.ring_builder_analyzer:main + swift-ring-composer = swift.cli.ringcomposer:main paste.app_factory = proxy = swift.proxy.server:app_factory diff --git a/swift/account/auditor.py b/swift/account/auditor.py index 7aa26c0757..0b0799c5bc 100644 --- a/swift/account/auditor.py +++ b/swift/account/auditor.py @@ -16,7 +16,9 @@ from swift.account.backend import AccountBroker from swift.common.exceptions import InvalidAccountInfo +from swift.common.daemon import run_daemon from swift.common.db_auditor import DatabaseAuditor +from swift.common.utils import parse_options class AccountAuditor(DatabaseAuditor): @@ -45,3 +47,12 @@ class AccountAuditor(DatabaseAuditor): 'does not match the sum of %(key)s across policies (%(sum)s)' % {'key': key, 'account': info.get('account'), 'total': info[key], 'sum': policy_totals[key]}) + + +def main(): + conf_file, options = parse_options(once=True) + run_daemon(AccountAuditor, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 2afe4b56ce..41f7a440b7 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -27,6 +27,7 @@ import six import swift.common.db from swift.account.backend import AccountBroker, DATADIR from swift.common.constraints import check_drive +from swift.common.daemon import run_daemon from swift.common.direct_client import direct_delete_container, \ direct_delete_object, direct_get_container from swift.common.exceptions import ClientException @@ -34,7 +35,7 @@ from swift.common.request_helpers import USE_REPLICATION_NETWORK_HEADER from swift.common.ring import Ring from swift.common.ring.utils import is_local_device from swift.common.utils import get_logger, whataremyips, config_true_value, \ - Timestamp, md5, node_to_string + Timestamp, md5, node_to_string, parse_options from swift.common.daemon import Daemon from swift.common.storage_policy import POLICIES, PolicyError @@ -525,3 +526,12 @@ class AccountReaper(Daemon): else: self.stats_objects_possibly_remaining += 1 self.logger.increment('objects_possibly_remaining') + + +def main(): + conf_file, options = parse_options(once=True) + run_daemon(AccountReaper, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/account/replicator.py b/swift/account/replicator.py index 773a3fb206..71be882e02 100644 --- a/swift/account/replicator.py +++ b/swift/account/replicator.py @@ -13,8 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import optparse + from swift.account.backend import AccountBroker, DATADIR from swift.common import db_replicator +from swift.common.daemon import run_daemon +from swift.common.utils import parse_options class AccountReplicator(db_replicator.Replicator): @@ -22,3 +26,21 @@ class AccountReplicator(db_replicator.Replicator): brokerclass = AccountBroker datadir = DATADIR default_port = 6202 + + +def main(): + parser = optparse.OptionParser("%prog CONFIG [options]") + parser.add_option('-d', '--devices', + help=('Replicate only given devices. ' + 'Comma-separated list. ' + 'Only has effect if --once is used.')) + parser.add_option('-p', '--partitions', + help=('Replicate only given partitions. ' + 'Comma-separated list. ' + 'Only has effect if --once is used.')) + conf_file, options = parse_options(parser=parser, once=True) + run_daemon(AccountReplicator, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/cli/relinker.py b/swift/cli/relinker.py index b090e333d2..ec02e47901 100644 --- a/swift/cli/relinker.py +++ b/swift/cli/relinker.py @@ -743,7 +743,7 @@ def auto_or_int(value): return config_auto_int_value(value, default='auto') -def main(args): +def main(args=None): parser = argparse.ArgumentParser( description='Relink and cleanup objects to increase partition power') parser.add_argument('action', choices=['relink', 'cleanup']) diff --git a/swift/cli/ringbuilder.py b/swift/cli/ringbuilder.py index 62b9560233..374f462356 100644 --- a/swift/cli/ringbuilder.py +++ b/swift/cli/ringbuilder.py @@ -22,9 +22,11 @@ from itertools import islice from operator import itemgetter from os import mkdir from os.path import basename, abspath, dirname, exists, join as pathjoin +import sys from sys import argv as sys_argv, exit, stderr, stdout from textwrap import wrap from time import time +import traceback from datetime import timedelta import optparse import math @@ -1698,3 +1700,25 @@ def main(arguments=None): exit(2) else: getattr(Commands, command, Commands.unknown)() + + +def error_handling_main(): + # We exit code 1 on WARNING statuses, 2 on ERROR. This means we need + # to handle any uncaught exceptions by printing the usual backtrace, + # but then exiting 2 (not 1 as is usual for a python + # exception). + + # We *don't* want to do this in main(), however, because we don't want to + # pollute the test environment or cause a bunch of test churn to mock out + # sys.excepthook + + def exit_with_status_two(tp, val, tb): + traceback.print_exception(tp, val, tb) + exit(2) + + sys.excepthook = exit_with_status_two + main() + + +if __name__ == '__main__': + error_handling_main() diff --git a/swift/container/auditor.py b/swift/container/auditor.py index 9638c4e0fc..14b0537f4f 100644 --- a/swift/container/auditor.py +++ b/swift/container/auditor.py @@ -15,7 +15,9 @@ from swift.container.backend import ContainerBroker +from swift.common.daemon import run_daemon from swift.common.db_auditor import DatabaseAuditor +from swift.common.utils import parse_options class ContainerAuditor(DatabaseAuditor): @@ -26,3 +28,12 @@ class ContainerAuditor(DatabaseAuditor): def _audit(self, job, broker): return None + + +def main(): + conf_file, options = parse_options(once=True) + run_daemon(ContainerAuditor, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/container/reconciler.py b/swift/container/reconciler.py index 30bc3501c1..e3caaac5e7 100644 --- a/swift/container/reconciler.py +++ b/swift/container/reconciler.py @@ -22,7 +22,7 @@ from eventlet import GreenPile, GreenPool, Timeout import six from swift.common import constraints -from swift.common.daemon import Daemon +from swift.common.daemon import Daemon, run_daemon from swift.common.direct_client import ( direct_head_container, direct_delete_container_object, direct_put_container_object, ClientException) @@ -31,7 +31,7 @@ from swift.common.request_helpers import MISPLACED_OBJECTS_ACCOUNT, \ USE_REPLICATION_NETWORK_HEADER from swift.common.utils import get_logger, split_path, majority_size, \ FileLikeIter, Timestamp, last_modified_date_to_timestamp, \ - LRUCache, decode_timestamps, hash_path + LRUCache, decode_timestamps, hash_path, parse_options from swift.common.storage_policy import POLICIES MISPLACED_OBJECTS_CONTAINER_DIVISOR = 3600 # 1 hour @@ -860,3 +860,12 @@ class ContainerReconciler(Daemon): self.stats = defaultdict(int) self.logger.info('sleeping between intervals (%ss)', self.interval) time.sleep(self.interval) + + +def main(): + conf_file, options = parse_options(once=True) + run_daemon(ContainerReconciler, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/container/replicator.py b/swift/container/replicator.py index 411d297736..ee464bfd00 100644 --- a/swift/container/replicator.py +++ b/swift/container/replicator.py @@ -17,6 +17,7 @@ import os import json from collections import defaultdict from eventlet import Timeout +import optparse from random import choice from swift.container.sync_store import ContainerSyncStore @@ -26,10 +27,12 @@ from swift.container.reconciler import ( MISPLACED_OBJECTS_ACCOUNT, incorrect_policy_index, get_reconciler_container_name, get_row_to_q_entry_translator) from swift.common import db_replicator +from swift.common.daemon import run_daemon from swift.common.storage_policy import POLICIES from swift.common.swob import HTTPOk, HTTPAccepted from swift.common.http import is_success -from swift.common.utils import Timestamp, majority_size, get_db_files +from swift.common.utils import Timestamp, majority_size, get_db_files, \ + parse_options def check_merge_own_shard_range(shards, broker, logger, source): @@ -440,3 +443,21 @@ class ContainerReplicatorRpc(db_replicator.ReplicatorRpc): def get_shard_ranges(self, broker, args): return HTTPOk(headers={'Content-Type': 'application/json'}, body=json.dumps(broker.get_all_shard_range_data())) + + +def main(): + parser = optparse.OptionParser("%prog CONFIG [options]") + parser.add_option('-d', '--devices', + help=('Replicate only given devices. ' + 'Comma-separated list. ' + 'Only has effect if --once is used.')) + parser.add_option('-p', '--partitions', + help=('Replicate only given partitions. ' + 'Comma-separated list. ' + 'Only has effect if --once is used.')) + conf_file, options = parse_options(parser=parser, once=True) + run_daemon(ContainerReplicator, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/container/sharder.py b/swift/container/sharder.py index a3847c8194..fd0139ffdb 100644 --- a/swift/container/sharder.py +++ b/swift/container/sharder.py @@ -17,6 +17,7 @@ import errno import json import logging import operator +from optparse import OptionParser import time from collections import defaultdict from operator import itemgetter @@ -32,6 +33,7 @@ from swift.common import internal_client from swift.common.constraints import check_drive, AUTO_CREATE_ACCOUNT_PREFIX from swift.common.direct_client import (direct_put_container, DirectClientException) +from swift.common.daemon import run_daemon from swift.common.request_helpers import USE_REPLICATION_NETWORK_HEADER from swift.common.ring.utils import is_local_device from swift.common.swob import str_to_wsgi @@ -39,7 +41,7 @@ from swift.common.utils import get_logger, config_true_value, \ dump_recon_cache, whataremyips, Timestamp, ShardRange, GreenAsyncPile, \ config_positive_int_value, quorum_size, parse_override_options, \ Everything, config_auto_int_value, ShardRangeList, config_percent_value, \ - node_to_string + node_to_string, parse_options from swift.container.backend import ContainerBroker, \ RECORD_TYPE_SHARD, UNSHARDED, SHARDING, SHARDED, COLLAPSED, \ SHARD_UPDATE_STATES, sift_shard_ranges, SHARD_UPDATE_STAT_STATES @@ -2574,3 +2576,25 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator): elapsed = time.time() - begin self.logger.info( 'Container sharder "once" mode completed: %.02fs', elapsed) + + +def main(): + parser = OptionParser("%prog CONFIG [options]") + parser.add_option('-d', '--devices', + help='Shard containers only on given devices. ' + 'Comma-separated list. ' + 'Only has effect if --once is used.') + parser.add_option('-p', '--partitions', + help='Shard containers only in given partitions. ' + 'Comma-separated list. ' + 'Only has effect if --once is used.') + parser.add_option('--no-auto-shard', action='store_false', + dest='auto_shard', default=None, + help='Disable auto-sharding. Overrides the auto_shard ' + 'value in the config file.') + conf_file, options = parse_options(parser=parser, once=True) + run_daemon(ContainerSharder, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/container/sync.py b/swift/container/sync.py index 55a7a77ecc..124d7ba4e3 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -29,6 +29,7 @@ from swift.common.db import DatabaseConnectionError from swift.container.backend import ContainerBroker from swift.container.sync_store import ContainerSyncStore from swift.common.container_sync_realms import ContainerSyncRealms +from swift.common.daemon import run_daemon from swift.common.internal_client import ( delete_object, put_object, head_object, InternalClient, UnexpectedResponse) @@ -39,7 +40,7 @@ from swift.common.swob import normalize_etag from swift.common.utils import ( clean_content_type, config_true_value, FileLikeIter, get_logger, hash_path, quote, validate_sync_to, - whataremyips, Timestamp, decode_timestamps) + whataremyips, Timestamp, decode_timestamps, parse_options) from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT from swift.common.wsgi import ConfigString @@ -650,3 +651,12 @@ class ContainerSync(Daemon): def select_http_proxy(self): return choice(self.http_proxies) if self.http_proxies else None + + +def main(): + conf_file, options = parse_options(once=True) + run_daemon(ContainerSync, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/container/updater.py b/swift/container/updater.py index 6b2be12864..46a8a06a87 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -32,8 +32,8 @@ from swift.common.exceptions import ConnectionTimeout, LockTimeout from swift.common.ring import Ring from swift.common.utils import get_logger, config_true_value, \ dump_recon_cache, majority_size, Timestamp, EventletRateLimiter, \ - eventlet_monkey_patch, node_to_string -from swift.common.daemon import Daemon + eventlet_monkey_patch, node_to_string, parse_options +from swift.common.daemon import Daemon, run_daemon from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR from swift.common.recon import RECON_CONTAINER_FILE, DEFAULT_RECON_CACHE_PATH @@ -357,3 +357,12 @@ class ContainerUpdater(Daemon): return HTTP_INTERNAL_SERVER_ERROR finally: conn.close() + + +def main(): + conf_file, options = parse_options(once=True) + run_daemon(ContainerUpdater, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 1f94f97c9a..a1ddefbe99 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -18,6 +18,7 @@ import os import sys import time import signal +from optparse import OptionParser from os.path import basename, dirname, join from random import shuffle from contextlib import closing @@ -26,12 +27,13 @@ from eventlet import Timeout from swift.obj import diskfile, replicator from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \ DiskFileDeleted, DiskFileExpired, QuarantineRequest -from swift.common.daemon import Daemon +from swift.common.daemon import Daemon, run_daemon from swift.common.storage_policy import POLICIES from swift.common.utils import ( config_auto_int_value, dump_recon_cache, get_logger, list_from_csv, listdir, load_pkg_resource, parse_prefixed_conf, EventletRateLimiter, - readconf, round_robin_iter, unlink_paths_older_than, PrefixLoggerAdapter) + readconf, round_robin_iter, unlink_paths_older_than, PrefixLoggerAdapter, + parse_options) from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH @@ -537,3 +539,17 @@ class WatcherWrapper(object): except (Exception, Timeout): self.logger.exception('Error ending watcher') self.watcher_in_error = True + + +def main(): + parser = OptionParser("%prog CONFIG [options]") + parser.add_option('-z', '--zero_byte_fps', + help='Audit only zero byte files at specified files/sec') + parser.add_option('-d', '--devices', + help='Audit only given devices. Comma-separated list') + conf_file, options = parse_options(parser=parser, once=True) + run_daemon(ObjectAuditor, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/obj/expirer.py b/swift/obj/expirer.py index 180aa2302b..0a0ad101b9 100644 --- a/swift/obj/expirer.py +++ b/swift/obj/expirer.py @@ -18,6 +18,7 @@ from six.moves import urllib from random import random from time import time +from optparse import OptionParser from os.path import join from collections import defaultdict, deque @@ -25,12 +26,12 @@ from eventlet import sleep, Timeout from eventlet.greenpool import GreenPool from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX -from swift.common.daemon import Daemon +from swift.common.daemon import Daemon, run_daemon from swift.common.internal_client import InternalClient, UnexpectedResponse from swift.common.utils import get_logger, dump_recon_cache, split_path, \ Timestamp, config_true_value, normalize_delete_at_timestamp, \ RateLimitedIterator, md5, non_negative_float, non_negative_int, \ - parse_content_type + parse_content_type, parse_options from swift.common.http import HTTP_NOT_FOUND, HTTP_CONFLICT, \ HTTP_PRECONDITION_FAILED from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH @@ -581,3 +582,21 @@ class ObjectExpirer(Daemon): self.swift.delete_object(*split_path('/' + actual_obj, 3, 3, True), headers=headers, acceptable_statuses=acceptable_statuses) + + +def main(): + parser = OptionParser("%prog CONFIG [options]") + parser.add_option('--processes', dest='processes', + help="Number of processes to use to do the work, don't " + "use this option to do all the work in one process") + parser.add_option('--process', dest='process', + help="Process number for this process, don't use " + "this option to do all the work in one process, this " + "is used to determine which part of the work this " + "process should do") + conf_file, options = parse_options(parser=parser, once=True) + run_daemon(ObjectExpirer, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/obj/reconstructor.py b/swift/obj/reconstructor.py index 774be4d840..6e8dc0679b 100644 --- a/swift/obj/reconstructor.py +++ b/swift/obj/reconstructor.py @@ -15,6 +15,7 @@ import itertools import json import errno +from optparse import OptionParser import os from os.path import join import random @@ -33,10 +34,10 @@ from swift.common.utils import ( GreenAsyncPile, Timestamp, remove_file, node_to_string, load_recon_cache, parse_override_options, distribute_evenly, PrefixLoggerAdapter, remove_directory, config_request_node_count_value, - non_negative_int) + non_negative_int, parse_options) from swift.common.header_key_dict import HeaderKeyDict from swift.common.bufferedhttp import http_connect -from swift.common.daemon import Daemon +from swift.common.daemon import Daemon, run_daemon from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH from swift.common.ring.utils import is_local_device from swift.obj.ssync_sender import Sender as ssync_sender @@ -1561,3 +1562,21 @@ class ObjectReconstructor(Daemon): self.logger.debug('reconstruction sleeping for %s seconds.', self.interval) sleep(self.interval) + + +def main(): + parser = OptionParser("%prog CONFIG [options]") + parser.add_option('-d', '--devices', + help='Reconstruct only given devices. ' + 'Comma-separated list. ' + 'Only has effect if --once is used.') + parser.add_option('-p', '--partitions', + help='Reconstruct only given partitions. ' + 'Comma-separated list. ' + 'Only has effect if --once is used.') + conf_file, options = parse_options(parser=parser, once=True) + run_daemon(ObjectReconstructor, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index 4a82521aad..03a3f56b34 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -14,6 +14,7 @@ # limitations under the License. from collections import defaultdict +from optparse import OptionParser import os import errno from os.path import isdir, isfile, join, dirname @@ -35,9 +36,9 @@ from swift.common.utils import whataremyips, unlink_older_than, \ rsync_module_interpolation, mkdirs, config_true_value, \ config_auto_int_value, storage_directory, \ load_recon_cache, PrefixLoggerAdapter, parse_override_options, \ - distribute_evenly, listdir, node_to_string + distribute_evenly, listdir, node_to_string, parse_options from swift.common.bufferedhttp import http_connect -from swift.common.daemon import Daemon +from swift.common.daemon import Daemon, run_daemon from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH from swift.obj import ssync_sender @@ -1155,3 +1156,25 @@ class ObjectReplicator(Daemon): # This method is called after run_once using multiple workers. update = self.aggregate_recon_update() dump_recon_cache(update, self.rcache, self.logger) + + +def main(): + parser = OptionParser("%prog CONFIG [options]") + parser.add_option('-d', '--devices', + help='Replicate only given devices. ' + 'Comma-separated list. ' + 'Only has effect if --once is used.') + parser.add_option('-p', '--partitions', + help='Replicate only given partitions. ' + 'Comma-separated list. ' + 'Only has effect if --once is used.') + parser.add_option('-i', '--policies', + help='Replicate only given policy indices. ' + 'Comma-separated list. ' + 'Only has effect if --once is used.') + conf_file, options = parse_options(parser=parser, once=True) + run_daemon(ObjectReplicator, conf_file, **options) + + +if __name__ == '__main__': + main() diff --git a/swift/obj/updater.py b/swift/obj/updater.py index 562c2847a1..00db49c2d7 100644 --- a/swift/obj/updater.py +++ b/swift/obj/updater.py @@ -34,8 +34,8 @@ from swift.common.utils import get_logger, renamer, write_pickle, \ dump_recon_cache, config_true_value, RateLimitedIterator, split_path, \ eventlet_monkey_patch, get_redirect_data, ContextPool, hash_path, \ non_negative_float, config_positive_int_value, non_negative_int, \ - EventletRateLimiter, node_to_string -from swift.common.daemon import Daemon + EventletRateLimiter, node_to_string, parse_options +from swift.common.daemon import Daemon, run_daemon from swift.common.header_key_dict import HeaderKeyDict from swift.common.storage_policy import split_policy_string, PolicyError from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH @@ -758,3 +758,12 @@ class ObjectUpdater(Daemon): self.logger.timing('updater.timing.status.%s' % status, elapsed * 1000) return HTTP_INTERNAL_SERVER_ERROR, node['id'], redirect + + +def main(): + conf_file, options = parse_options(once=True) + run_daemon(ObjectUpdater, conf_file, **options) + + +if __name__ == '__main__': + main()