diff --git a/bin/swift-bench b/bin/swift-bench index 93bb2b5707..447d82724d 100755 --- a/bin/swift-bench +++ b/bin/swift-bench @@ -22,7 +22,7 @@ import uuid from optparse import OptionParser from swift.common.bench import BenchController -from swift.common.utils import readconf, NamedLogger +from swift.common.utils import readconf, LogAdapter, NamedFormatter # The defaults should be sufficient to run swift-bench on a SAIO CONF_DEFAULTS = { @@ -124,10 +124,11 @@ if __name__ == '__main__': 'critical': logging.CRITICAL}.get( options.log_level.lower(), logging.INFO)) loghandler = logging.StreamHandler() - logformat = logging.Formatter('%(asctime)s %(levelname)s %(message)s') - loghandler.setFormatter(logformat) logger.addHandler(loghandler) - logger = NamedLogger(logger, 'swift-bench') + logger = LogAdapter(logger) + logformat = NamedFormatter('swift-bench', logger, + fmt='%(server)s %(asctime)s %(levelname)s %(message)s') + loghandler.setFormatter(logformat) controller = BenchController(logger, options) controller.run() diff --git a/bin/swift-ring-builder b/bin/swift-ring-builder index 347ba4825e..c448bea5ca 100755 --- a/bin/swift-ring-builder +++ b/bin/swift-ring-builder @@ -20,20 +20,40 @@ from gzip import GzipFile from os import mkdir from os.path import basename, dirname, exists, join as pathjoin from sys import argv, exit +from textwrap import wrap from time import time from swift.common.ring import RingBuilder MAJOR_VERSION = 1 -MINOR_VERSION = 1 +MINOR_VERSION = 2 EXIT_RING_CHANGED = 0 EXIT_RING_UNCHANGED = 1 -EXIT_ERROR = 2 +EXIT_ERROR = 2 def search_devs(builder, search_value): - # dz-:/_ + """ +The can be of the form: + dz-:/_ + Any part is optional, but you must include at least one part. + Examples: + d74 Matches the device id 74 + z1 Matches devices in zone 1 + z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4 + 1.2.3.4 Matches devices in any zone with the ip 1.2.3.4 + z1:5678 Matches devices in zone 1 using port 5678 + :5678 Matches devices that use port 5678 + /sdb1 Matches devices with the device name sdb1 + _shiny Matches devices with shiny in the meta data + _"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data + Most specific example: + d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8" + Nerd explanation: + All items require their single character prefix except the ip, in which + case the - is optional unless the device id or zone is also included. + """ orig_search_value = search_value match = [] if search_value.startswith('d'): @@ -72,7 +92,8 @@ def search_devs(builder, search_value): match.append(('meta', search_value[1:])) search_value = '' if search_value: - raise ValueError('Invalid : %s' % repr(orig_search_value)) + raise ValueError('Invalid : %s' % + repr(orig_search_value)) devs = [] for dev in builder.devs: if not dev: @@ -89,142 +110,22 @@ def search_devs(builder, search_value): return devs -SEARCH_VALUE_HELP = ''' - The can be of the form: - dz-:/_ - Any part is optional, but you must include at least one part. - Examples: - d74 Matches the device id 74 - z1 Matches devices in zone 1 - z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4 - 1.2.3.4 Matches devices in any zone with the ip 1.2.3.4 - z1:5678 Matches devices in zone 1 using port 5678 - :5678 Matches devices that use port 5678 - /sdb1 Matches devices with the device name sdb1 - _shiny Matches devices with shiny in the meta data - _"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data - Most specific example: - d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8" - Nerd explanation: - All items require their single character prefix except the ip, in which - case the - is optional unless the device id or zone is also included. -'''.strip() +class Commands: -CREATE_HELP = ''' -swift-ring-builder create + def unknown(): + print 'Unknown command: %s' % argv[2] + exit(EXIT_ERROR) + + def create(): + """ +swift-ring-builder create + Creates with 2^ partitions and . is number of hours to restrict moving a partition more than once. -'''.strip() - -SEARCH_HELP = ''' -swift-ring-builder search - Shows information about matching devices. - - %(SEARCH_VALUE_HELP)s -'''.strip() % globals() - -ADD_HELP = ''' -swift-ring-builder add z-:/_ - Adds a device to the ring with the given information. No partitions will be - assigned to the new device until after running 'rebalance'. This is so you - can make multiple device changes and rebalance them all just once. -'''.strip() - -SET_WEIGHT_HELP = ''' -swift-ring-builder set_weight - Resets the device's weight. No partitions will be reassigned to or from the - device until after running 'rebalance'. This is so you can make multiple - device changes and rebalance them all just once. - - %(SEARCH_VALUE_HELP)s -'''.strip() % globals() - -SET_INFO_HELP = ''' -swift-ring-builder set_info - :/_ - Resets the device's information. This information isn't used to assign - partitions, so you can use 'write_ring' afterward to rewrite the current - ring with the newer device information. Any of the parts are optional - in the final :/_ parameter; just give what you - want to change. For instance set_info d74 _"snet: 5.6.7.8" would just - update the meta data for device id 74. - - %(SEARCH_VALUE_HELP)s -'''.strip() % globals() - -REMOVE_HELP = ''' -swift-ring-builder remove - Removes the device(s) from the ring. This should normally just be used for - a device that has failed. For a device you wish to decommission, it's best - to set its weight to 0, wait for it to drain all its data, then use this - remove command. This will not take effect until after running 'rebalance'. - This is so you can make multiple device changes and rebalance them all just - once. - - %(SEARCH_VALUE_HELP)s -'''.strip() % globals() - -SET_MIN_PART_HOURS_HELP = ''' -swift-ring-builder set_min_part_hours - Changes the to the given . This should be set to - however long a full replication/update cycle takes. We're working on a way - to determine this more easily than scanning logs. -'''.strip() - - -if __name__ == '__main__': - if len(argv) < 2: - print ''' -swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s - -%(CREATE_HELP)s - -swift-ring-builder - Shows information about the ring and the devices within. - -%(SEARCH_HELP)s - -%(ADD_HELP)s - -%(SET_WEIGHT_HELP)s - -%(SET_INFO_HELP)s - -%(REMOVE_HELP)s - -swift-ring-builder rebalance - Attempts to rebalance the ring by reassigning partitions that haven't been - recently reassigned. - -swift-ring-builder validate - Just runs the validation routines on the ring. - -swift-ring-builder write_ring - Just rewrites the distributable ring file. This is done automatically after - a successful rebalance, so really this is only useful after one or more - 'set_info' calls when no rebalance is needed but you want to send out the - new device information. - -%(SET_MIN_PART_HOURS_HELP)s - -Quick list: create search add set_weight set_info remove rebalance write_ring - set_min_part_hours -Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error -'''.strip() % globals() - exit(EXIT_RING_UNCHANGED) - - if exists(argv[1]): - builder = pickle.load(open(argv[1], 'rb')) - for dev in builder.devs: - if dev and 'meta' not in dev: - dev['meta'] = '' - elif len(argv) < 3 or argv[2] != 'create': - print 'Ring Builder file does not exist: %s' % argv[1] - exit(EXIT_ERROR) - elif argv[2] == 'create': + """ if len(argv) < 6: - print CREATE_HELP + print Commands.create.__doc__.strip() exit(EXIT_RING_UNCHANGED) builder = RingBuilder(int(argv[3]), int(argv[4]), int(argv[5])) backup_dir = pathjoin(dirname(argv[1]), 'backups') @@ -238,19 +139,11 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error pickle.dump(builder, open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_CHANGED) - backup_dir = pathjoin(dirname(argv[1]), 'backups') - try: - mkdir(backup_dir) - except OSError, err: - if err.errno != EEXIST: - raise - - ring_file = argv[1] - if ring_file.endswith('.builder'): - ring_file = ring_file[:-len('.builder')] - ring_file += '.ring.gz' - - if len(argv) == 2: + def default(): + """ +swift-ring-builder + Shows information about the ring and the devices within. + """ print '%s, build version %d' % (argv[1], builder.version) zones = 0 balance = 0 @@ -284,9 +177,15 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error dev['meta']) exit(EXIT_RING_UNCHANGED) - if argv[2] == 'search': + def search(): + """ +swift-ring-builder search + Shows information about matching devices. + """ if len(argv) < 4: - print SEARCH_HELP + print Commands.search.__doc__.strip() + print + print search_devs.__doc__.strip() exit(EXIT_RING_UNCHANGED) devs = search_devs(builder, argv[3]) if not devs: @@ -311,10 +210,16 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error dev['meta']) exit(EXIT_RING_UNCHANGED) - elif argv[2] == 'add': - # add z-:/_ + def add(): + """ +swift-ring-builder add z-:/_ + + Adds a device to the ring with the given information. No partitions will be + assigned to the new device until after running 'rebalance'. This is so you + can make multiple device changes and rebalance them all just once. + """ if len(argv) < 5: - print ADD_HELP + print Commands.add.__doc__.strip() exit(EXIT_RING_UNCHANGED) if not argv[3].startswith('z'): @@ -379,9 +284,17 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error pickle.dump(builder, open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) - elif argv[2] == 'set_weight': + def set_weight(): + """ +swift-ring-builder set_weight + Resets the device's weight. No partitions will be reassigned to or from the + device until after running 'rebalance'. This is so you can make multiple + device changes and rebalance them all just once. + """ if len(argv) != 5: - print SET_WEIGHT_HELP + print Commands.set_weight.__doc__.strip() + print + print search_devs.__doc__.strip() exit(EXIT_RING_UNCHANGED) devs = search_devs(builder, argv[3]) weight = float(argv[4]) @@ -404,9 +317,21 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error pickle.dump(builder, open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) - elif argv[2] == 'set_info': + def set_info(): + """ +swift-ring-builder set_info + :/_ + Resets the device's information. This information isn't used to assign + partitions, so you can use 'write_ring' afterward to rewrite the current + ring with the newer device information. Any of the parts are optional + in the final :/_ parameter; just give what you + want to change. For instance set_info d74 _"snet: 5.6.7.8" would just + update the meta data for device id 74. + """ if len(argv) != 5: - print SET_INFO_HELP + print Commands.set_info.__doc__.strip() + print + print search_devs.__doc__.strip() exit(EXIT_RING_UNCHANGED) devs = search_devs(builder, argv[3]) change_value = argv[4] @@ -471,9 +396,20 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error pickle.dump(builder, open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) - elif argv[2] == 'remove': + def remove(): + """ +swift-ring-builder remove + Removes the device(s) from the ring. This should normally just be used for + a device that has failed. For a device you wish to decommission, it's best + to set its weight to 0, wait for it to drain all its data, then use this + remove command. This will not take effect until after running 'rebalance'. + This is so you can make multiple device changes and rebalance them all just + once. + """ if len(argv) < 4: - print REMOVE_HELP + print Commands.remove.__doc__.strip() + print + print search_devs.__doc__.strip() exit(EXIT_RING_UNCHANGED) devs = search_devs(builder, argv[3]) if not devs: @@ -491,11 +427,17 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error for dev in devs: builder.remove_dev(dev['id']) print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \ - 'marked for removal and will be removed next rebalance.' % dev + 'marked for removal and will be removed next rebalance.' \ + % dev pickle.dump(builder, open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) - elif argv[2] == 'rebalance': + def rebalance(): + """ +swift-ring-builder rebalance + Attempts to rebalance the ring by reassigning partitions that haven't been + recently reassigned. + """ devs_changed = builder.devs_changed last_balance = builder.get_balance() parts, balance = builder.rebalance() @@ -528,31 +470,50 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error pickle.dump(builder, open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_CHANGED) - elif argv[2] == 'validate': + def validate(): + """ +swift-ring-builder validate + Just runs the validation routines on the ring. + """ builder.validate() exit(EXIT_RING_UNCHANGED) - elif argv[2] == 'write_ring': + def write_ring(): + """ +swift-ring-builder write_ring + Just rewrites the distributable ring file. This is done automatically after + a successful rebalance, so really this is only useful after one or more + 'set_info' calls when no rebalance is needed but you want to send out the + new device information. + """ ring_data = builder.get_ring() if not ring_data._replica2part2dev_id: - if ring_data.devs: - print 'Warning: Writing a ring with no partition assignments but with devices; did you forget to run "rebalance"?' - else: - print 'Warning: Writing an empty ring' + if ring_data.devs: + print 'Warning: Writing a ring with no partition ' \ + 'assignments but with devices; did you forget to run ' \ + '"rebalance"?' + else: + print 'Warning: Writing an empty ring' pickle.dump(ring_data, GzipFile(pathjoin(backup_dir, '%d.' % time() + basename(ring_file)), 'wb'), protocol=2) pickle.dump(ring_data, GzipFile(ring_file, 'wb'), protocol=2) exit(EXIT_RING_CHANGED) - elif argv[2] == 'pretend_min_part_hours_passed': + def pretend_min_part_hours_passed(): builder.pretend_min_part_hours_passed() pickle.dump(builder, open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) - elif argv[2] == 'set_min_part_hours': + def set_min_part_hours(): + """ +swift-ring-builder set_min_part_hours + Changes the to the given . This should be set to + however long a full replication/update cycle takes. We're working on a way + to determine this more easily than scanning logs. + """ if len(argv) < 4: - print SET_MIN_PART_HOURS_HELP + print Commands.set_min_part_hours.__doc__.strip() exit(EXIT_RING_UNCHANGED) builder.change_min_part_hours(int(argv[3])) print 'The minimum number of hours before a partition can be ' \ @@ -560,5 +521,51 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error pickle.dump(builder, open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) - print 'Unknown command: %s' % argv[2] - exit(EXIT_ERROR) + +if __name__ == '__main__': + if len(argv) < 2: + print "swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s\n" % \ + globals() + print Commands.default.__doc__.strip() + print + cmds = [c for c, f in Commands.__dict__.iteritems() + if f.__doc__ and c[0] != '_' and c != 'default'] + cmds.sort() + for cmd in cmds: + print Commands.__dict__[cmd].__doc__.strip() + print + print search_devs.__doc__.strip() + print + for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ', + subsequent_indent=' '): + print line + print 'Exit codes: 0 = ring changed, 1 = ring did not change, ' \ + '2 = error' + exit(EXIT_RING_UNCHANGED) + + if exists(argv[1]): + builder = pickle.load(open(argv[1], 'rb')) + for dev in builder.devs: + if dev and 'meta' not in dev: + dev['meta'] = '' + elif len(argv) < 3 or argv[2] != 'create': + print 'Ring Builder file does not exist: %s' % argv[1] + exit(EXIT_ERROR) + + backup_dir = pathjoin(dirname(argv[1]), 'backups') + try: + mkdir(backup_dir) + except OSError, err: + if err.errno != EEXIST: + raise + + ring_file = argv[1] + if ring_file.endswith('.builder'): + ring_file = ring_file[:-len('.builder')] + ring_file += '.ring.gz' + + if len(argv) == 2: + command = "default" + else: + command = argv[2] + Commands.__dict__.get(command, Commands.unknown)() diff --git a/swift/common/middleware/catch_errors.py b/swift/common/middleware/catch_errors.py index e2287fdbed..10d8614194 100644 --- a/swift/common/middleware/catch_errors.py +++ b/swift/common/middleware/catch_errors.py @@ -26,7 +26,11 @@ class CatchErrorMiddleware(object): def __init__(self, app, conf): self.app = app - self.logger = get_logger(conf) + # if the application already has a logger we should use that one + self.logger = getattr(app, 'logger', None) + if not self.logger: + # and only call get_logger if we have to + self.logger = get_logger(conf) def __call__(self, env, start_response): try: diff --git a/swift/common/utils.py b/swift/common/utils.py index 6b138ee9ee..b3f640c3d3 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -284,11 +284,15 @@ class LoggerFileObject(object): class LogAdapter(object): - """Cheesy version of the LoggerAdapter available in Python 3""" + """ + A Logger like object which performs some reformatting on calls to + :meth:`exception`. Can be used to store a threadlocal transaction id. + """ + + _txn_id = threading.local() def __init__(self, logger): self.logger = logger - self._txn_id = threading.local() for proxied_method in ('debug', 'log', 'warn', 'warning', 'error', 'critical', 'info'): setattr(self, proxied_method, getattr(logger, proxied_method)) @@ -334,18 +338,45 @@ class LogAdapter(object): class NamedFormatter(logging.Formatter): - def __init__(self, server, logger): - logging.Formatter.__init__(self) + """ + NamedFormatter is used to add additional information to log messages. + Normally it will simply add the server name as an attribute on the + LogRecord and the default format string will include it at the + begining of the log message. Additionally, if the transaction id is + available and not already included in the message, NamedFormatter will + add it. + + NamedFormatter may be initialized with a format string which makes use + of the standard LogRecord attributes. In addition the format string + may include the following mapping key: + + +----------------+---------------------------------------------+ + | Format | Description | + +================+=============================================+ + | %(server)s | Name of the swift server doing logging | + +----------------+---------------------------------------------+ + + :param server: the swift server name, a string. + :param logger: a Logger or :class:`LogAdapter` instance, additional + context may be pulled from attributes on this logger if + available. + :param fmt: the format string used to construct the message, if none is + supplied it defaults to ``"%(server)s %(message)s"`` + """ + + def __init__(self, server, logger, + fmt="%(server)s %(message)s"): + logging.Formatter.__init__(self, fmt) self.server = server self.logger = logger def format(self, record): + record.server = self.server msg = logging.Formatter.format(self, record) if self.logger.txn_id and (record.levelno != logging.INFO or self.logger.txn_id not in msg): - return '%s %s (txn: %s)' % (self.server, msg, self.logger.txn_id) - else: - return '%s %s' % (self.server, msg) + msg = "%s (txn: %s)" % (msg, self.logger.txn_id) + return msg def get_logger(conf, name=None, log_to_console=False): @@ -386,7 +417,10 @@ def get_logger(conf, name=None, log_to_console=False): root_logger.setLevel( getattr(logging, conf.get('log_level', 'INFO').upper(), logging.INFO)) adapted_logger = LogAdapter(root_logger) - get_logger.handler.setFormatter(NamedFormatter(name, adapted_logger)) + formatter = NamedFormatter(name, adapted_logger) + get_logger.handler.setFormatter(formatter) + if hasattr(get_logger, 'console'): + get_logger.console.setFormatter(formatter) return adapted_logger