From 9bda92d54a4d80b7c274fa75f02a1c4ae293d6e7 Mon Sep 17 00:00:00 2001 From: Darrell Bishop Date: Sat, 25 Aug 2012 16:02:45 -0700 Subject: [PATCH] Misc. swift-bench improvements. swift-bench now honors the environment variables, ST_AUTH, ST_USER, and ST_KEY like python-swiftclient does. Added --lower-object-size (or -l) command-line option which, if specified, will turn a specified --object-size into --upper-object-size. It will raise a ValueError if --object-size is not specified or is <= --lower-object-size. BenchController how handles SIGINT (KeyboardInterrupt) similarly to the swift command-line client: the first Ctrl-C will make it fast-track to completion (no new PUT or GET operations are started, but everything PUT is DELETE'ed). A second Ctrl-C will immediately exit. The behavior for SIGTERM is unchanged (a single SIGTERM will immediately terminate the process). Added a sample configuration file for swift-bench, with documenting comments. Change-Id: I6f394ad995300fc8af3d565d95c3b45559ee510a --- bin/swift-bench | 43 +++++++++++++++----------- etc/swift-bench.conf-sample | 60 +++++++++++++++++++++++++++++++++++++ swift/common/bench.py | 27 ++++++++++++++++- 3 files changed, 112 insertions(+), 18 deletions(-) create mode 100644 etc/swift-bench.conf-sample diff --git a/bin/swift-bench b/bin/swift-bench index df0e404f68..b69671ab81 100755 --- a/bin/swift-bench +++ b/bin/swift-bench @@ -26,43 +26,43 @@ from swift.common.utils import readconf, LogAdapter # The defaults should be sufficient to run swift-bench on a SAIO CONF_DEFAULTS = { - 'auth': '', - 'user': '', - 'key': '', - 'object_sources': '', + 'auth': os.environ.get('ST_AUTH', ''), + 'user': os.environ.get('ST_USER', ''), + 'key': os.environ.get('ST_KEY', ''), + 'auth_version': '1.0', + 'use_proxy': 'yes', 'put_concurrency': '10', 'get_concurrency': '10', 'del_concurrency': '10', - 'concurrency': '', - 'object_size': '1', - 'lower_object_size': '10', + 'concurrency': '', # set all 3 in one shot + 'object_sources': '', # set of file contents to read and use for PUTs + 'lower_object_size': '10', # bounded random size used if these differ 'upper_object_size': '10', + 'object_size': '1', # only if not object_sources and lower == upper 'num_objects': '1000', 'num_gets': '10000', 'delete': 'yes', - 'container_name': uuid.uuid4().hex, + 'container_name': uuid.uuid4().hex, # really "container name base" 'num_containers': '20', - 'use_proxy': 'yes', - 'url': '', - 'account': '', - 'devices': 'sdb1', + 'url': '', # used when use_proxy = no or overrides auth X-Storage-Url + 'account': '', # used when use_proxy = no + 'devices': 'sdb1', # space-sep list 'log_level': 'INFO', 'timeout': '10', - 'auth_version': '1.0', - } +} SAIO_DEFAULTS = { 'auth': 'http://localhost:8080/auth/v1.0', 'user': 'test:tester', 'key': 'testing', - } +} if __name__ == '__main__': usage = "usage: %prog [OPTIONS] [CONF_FILE]" usage += """\n\nConf file with SAIO defaults: [bench] - auth = http://localhost:8080/v1.0 + auth = http://localhost:8080/auth/v1.0 user = test:tester key = testing concurrency = 10 @@ -87,6 +87,9 @@ if __name__ == '__main__': help='Number of concurrent connections to use') parser.add_option('-s', '--object-size', dest='object_size', help='Size of objects to PUT (in bytes)') + parser.add_option('-l', '--lower-object-size', dest='lower_object_size', + help=('Lower size of objects (in bytes); ' + '--object-size will be upper-object-size')) parser.add_option('-n', '--num-objects', dest='num_objects', help='Number of objects to PUT') parser.add_option('-g', '--num-gets', dest='num_gets', @@ -102,12 +105,18 @@ if __name__ == '__main__': options, args = parser.parse_args() if options.saio: CONF_DEFAULTS.update(SAIO_DEFAULTS) + if getattr(options, 'lower_object_size', None): + if options.object_size <= options.lower_object_size: + raise ValueError('--lower-object-size (%s) must be ' + '< --object-size (%s)' % + (options.lower_object_size, options.object_size)) + CONF_DEFAULTS['upper_object_size'] = options.object_size if args: conf = args[0] if not os.path.exists(conf): sys.exit("No such conf file: %s" % conf) conf = readconf(conf, 'bench', log_name='swift-bench', - defaults=CONF_DEFAULTS) + defaults=CONF_DEFAULTS) else: conf = CONF_DEFAULTS parser.set_defaults(**conf) diff --git a/etc/swift-bench.conf-sample b/etc/swift-bench.conf-sample new file mode 100644 index 0000000000..423608ec48 --- /dev/null +++ b/etc/swift-bench.conf-sample @@ -0,0 +1,60 @@ +[bench] +# auth = http://localhost:8080/auth/v1.0 +# user = test:tester +# key = testing +# auth_version = 1.0 +# log-level = INFO +# timeout = 10 + +# You can configure PUT, GET, and DELETE concurrency independently or set all +# three with "concurrency" +# put_concurrency = 10 +# get_concurrency = 10 +# del_concurrency = 10 +# concurrency = + +# A space-sep list of files whose contents will be read and randomly chosen +# as the body (object contents) for each PUT. +# object_sources = + +# If object_sources is not set and lower_object_size != upper_object_size, +# each PUT will randomly select an object size between the two values. Units +# are bytes. +# lower_object_size = 10 +# upper_object_size = 10 + +# If object_sources is not set and lower_object_size == upper_object_size, +# every object PUT will contain this many bytes. +# object_size = 1 + +# num_objects = 1000 +# num_gets = 10000 +# num_containers = 20 + +# The base name for created containers. +# container_name = (randomly-chosen uuid4) + +# Should swift-bench benchmark DELETEing the created objects and then delete +# all created containers? +# delete = yes + +# Without use_proxy, swift-bench will talk directly to the backend Swift +# servers. Doing that will require "url", "account", and at least one +# "devices" entry. +# use_proxy = yes + +# If use_proxy = yes, this will override any returned X-Storage-Url returned +# by authenticaion (the account name will still be extracted from +# X-Storage-Url though and may NOT be set with the "account" conf var). If +# use_proxy = no, this setting is required and used as the X-Storage-Url when +# deleting containers and as a source for IP and port for back-end Swift server +# connections. The IP and port specified in this setting must have local +# storage access to every device specified in "devices". +# url = + +# Only used (and required) when use_proxy = no. +# account = + +# A space-sep list of devices names; only relevant (and required) when +# use_proxy = no. +# devices = sdb1 diff --git a/swift/common/bench.py b/swift/common/bench.py index 6a0af88237..3edfaecccc 100644 --- a/swift/common/bench.py +++ b/swift/common/bench.py @@ -13,9 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys import uuid import time import random +import signal from contextlib import contextmanager import eventlet.pools @@ -41,6 +43,7 @@ class Bench(object): def __init__(self, logger, conf, names): self.logger = logger + self.aborted = False self.user = conf.user self.key = conf.key self.auth_url = conf.auth @@ -116,6 +119,8 @@ class Bench(object): self.failures = 0 self.complete = 0 for i in xrange(self.total): + if self.aborted: + break pool.spawn_n(self._run, i) pool.waitall() self._log_status(self.msg + ' **FINAL**') @@ -132,15 +137,35 @@ class BenchController(object): self.names = [] self.delete = conf.delete.lower() in TRUE_VALUES self.gets = int(conf.num_gets) + self.aborted = False + + def sigint1(self, signum, frame): + if self.delete: + print >>sys.stderr, ( + 'SIGINT received; finishing up and running DELETE.\n' + 'Send one more SIGINT to exit *immediately*.') + self.aborted = True + if self.running and not isinstance(self.running, BenchDELETE): + self.running.aborted = True + signal.signal(signal.SIGINT, self.sigint2) + else: + self.sigint2(signum, frame) + + def sigint2(self, signum, frame): + sys.exit('Final SIGINT received.') def run(self): + signal.signal(signal.SIGINT, self.sigint1) puts = BenchPUT(self.logger, self.conf, self.names) + self.running = puts puts.run() - if self.gets: + if self.gets and not self.aborted: gets = BenchGET(self.logger, self.conf, self.names) + self.running = gets gets.run() if self.delete: dels = BenchDELETE(self.logger, self.conf, self.names) + self.running = dels dels.run()