diff --git a/bin/swift-ring-builder b/bin/swift-ring-builder index 8bb3d19883..b13fa4b406 100755 --- a/bin/swift-ring-builder +++ b/bin/swift-ring-builder @@ -18,6 +18,7 @@ import cPickle as pickle from array import array from errno import EEXIST from gzip import GzipFile +from itertools import islice, izip from os import mkdir from os.path import basename, dirname, exists, join as pathjoin from sys import argv, exit, modules @@ -267,208 +268,246 @@ swift-ring-builder list_parts [] .. def add(): """ -swift-ring-builder add z-:/_ - - Adds a device to the ring with the given information. No partitions will be +swift-ring-builder add + z-:/_ + [z-:/_ ] ... + + Adds devices to the ring with the given information. No partitions will be assigned to the new device until after running 'rebalance'. This is so you can make multiple device changes and rebalance them all just once. """ - if len(argv) < 5: + if len(argv) < 5 or len(argv) % 2 != 1: print Commands.add.__doc__.strip() exit(EXIT_ERROR) - if not argv[3].startswith('z'): - print 'Invalid add value: %s' % argv[3] - exit(EXIT_ERROR) - i = 1 - while i < len(argv[3]) and argv[3][i].isdigit(): - i += 1 - zone = int(argv[3][1:i]) - rest = argv[3][i:] + devs_and_weights = izip(islice(argv, 3, len(argv), 2), + islice(argv, 4, len(argv), 2)) + for devstr, weightstr in devs_and_weights: + if not devstr.startswith('z'): + print 'Invalid add value: %s' % devstr + exit(EXIT_ERROR) + i = 1 + while i < len(devstr) and devstr[i].isdigit(): + i += 1 + zone = int(devstr[1:i]) + rest = devstr[i:] - if not rest.startswith('-'): - print 'Invalid add value: %s' % argv[3] - exit(EXIT_ERROR) - i = 1 - if rest[i] == '[': - i += 1 - while i < len(rest) and rest[i] != ']': + if not rest.startswith('-'): + print 'Invalid add value: %s' % devstr + print "The on-disk ring builder is unchanged.\n" + exit(EXIT_ERROR) + i = 1 + if rest[i] == '[': i += 1 - i += 1 - ip = rest[1:i].lstrip('[').rstrip(']') - rest = rest[i:] - else: - while i < len(rest) and rest[i] in '0123456789.': + while i < len(rest) and rest[i] != ']': + i += 1 i += 1 - ip = rest[1:i] + ip = rest[1:i].lstrip('[').rstrip(']') + rest = rest[i:] + else: + while i < len(rest) and rest[i] in '0123456789.': + i += 1 + ip = rest[1:i] + rest = rest[i:] + + if not rest.startswith(':'): + print 'Invalid add value: %s' % devstr + print "The on-disk ring builder is unchanged.\n" + exit(EXIT_ERROR) + i = 1 + while i < len(rest) and rest[i].isdigit(): + i += 1 + port = int(rest[1:i]) rest = rest[i:] - if not rest.startswith(':'): - print 'Invalid add value: %s' % argv[3] - exit(EXIT_ERROR) - i = 1 - while i < len(rest) and rest[i].isdigit(): - i += 1 - port = int(rest[1:i]) - rest = rest[i:] + if not rest.startswith('/'): + print 'Invalid add value: %s' % devstr + print "The on-disk ring builder is unchanged.\n" + exit(EXIT_ERROR) + i = 1 + while i < len(rest) and rest[i] != '_': + i += 1 + device_name = rest[1:i] + rest = rest[i:] - if not rest.startswith('/'): - print 'Invalid add value: %s' % argv[3] - exit(EXIT_ERROR) - i = 1 - while i < len(rest) and rest[i] != '_': - i += 1 - device_name = rest[1:i] - rest = rest[i:] + meta = '' + if rest.startswith('_'): + meta = rest[1:] - meta = '' - if rest.startswith('_'): - meta = rest[1:] - - weight = float(argv[4]) - - for dev in builder.devs: - if dev is None: - continue - if dev['ip'] == ip and dev['port'] == port and \ - dev['device'] == device_name: - print 'Device %d already uses %s:%d/%s.' % \ - (dev['id'], dev['ip'], dev['port'], dev['device']) + try: + weight = float(weightstr) + except ValueError: + print 'Invalid weight value: %s' % weightstr + print "The on-disk ring builder is unchanged.\n" exit(EXIT_ERROR) - next_dev_id = 0 - if builder.devs: - next_dev_id = max(d['id'] for d in builder.devs if d) + 1 - builder.add_dev({'id': next_dev_id, 'zone': zone, 'ip': ip, - 'port': port, 'device': device_name, 'weight': weight, - 'meta': meta}) - if ':' in ip: - print 'Device z%s-[%s]:%s/%s_"%s" with %s weight got id %s' % \ - (zone, ip, port, device_name, meta, weight, next_dev_id) - else: - print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \ - (zone, ip, port, device_name, meta, weight, next_dev_id) + if weight < 0: + print 'Invalid weight value (must be positive): %s' % weightstr + print "The on-disk ring builder is unchanged.\n" + exit(EXIT_ERROR) + + for dev in builder.devs: + if dev is None: + continue + if dev['ip'] == ip and dev['port'] == port and \ + dev['device'] == device_name: + print 'Device %d already uses %s:%d/%s.' % \ + (dev['id'], dev['ip'], dev['port'], dev['device']) + print "The on-disk ring builder is unchanged.\n" + exit(EXIT_ERROR) + + next_dev_id = 0 + if builder.devs: + next_dev_id = max(d['id'] for d in builder.devs if d) + 1 + builder.add_dev({'id': next_dev_id, 'zone': zone, 'ip': ip, + 'port': port, 'device': device_name, + 'weight': weight, 'meta': meta}) + if ':' in ip: + print 'Device z%s-[%s]:%s/%s_"%s" with %s weight got id %s' % \ + (zone, ip, port, device_name, meta, weight, next_dev_id) + else: + print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \ + (zone, ip, port, device_name, meta, weight, next_dev_id) pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_SUCCESS) def set_weight(): """ swift-ring-builder set_weight - Resets the device's weight. No partitions will be reassigned to or from the - device until after running 'rebalance'. This is so you can make multiple - device changes and rebalance them all just once. + [ 1: - print 'Matched more than one device:' - for dev in devs: - print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ - '"%(meta)s"' % dev - if raw_input('Are you sure you want to update the weight for ' - 'these %s devices? (y/N) ' % len(devs)) != 'y': - print 'Aborting device modifications' + + devs_and_weights = izip(islice(argv, 3, len(argv), 2), + islice(argv, 4, len(argv), 2)) + for devstr, weightstr in devs_and_weights: + devs = search_devs(builder, devstr) + weight = float(weightstr) + if not devs: + print("Search value \"%s\" matched 0 devices.\n" + "The on-disk ring builder is unchanged.\n" + % devstr) exit(EXIT_ERROR) - for dev in devs: - builder.set_dev_weight(dev['id'], weight) - print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \ - 'weight set to %(weight)s' % dev + if len(devs) > 1: + print 'Matched more than one device:' + for dev in devs: + print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ + '"%(meta)s"' % dev + if raw_input('Are you sure you want to update the weight for ' + 'these %s devices? (y/N) ' % len(devs)) != 'y': + print 'Aborting device modifications' + exit(EXIT_ERROR) + for dev in devs: + builder.set_dev_weight(dev['id'], weight) + print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ + '"%(meta)s" weight set to %(weight)s' % dev pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_SUCCESS) def set_info(): """ -swift-ring-builder set_info - :/_ - Resets the device's information. This information isn't used to assign - partitions, so you can use 'write_ring' afterward to rewrite the current - ring with the newer device information. Any of the parts are optional - in the final :/_ parameter; just give what you - want to change. For instance set_info d74 _"snet: 5.6.7.8" would just - update the meta data for device id 74. +swift-ring-builder set_info + :/_ + [ :/_] ... + + For each search-value, resets the matched device's information. + This information isn't used to assign partitions, so you can use + 'write_ring' afterward to rewrite the current ring with the newer + device information. Any of the parts are optional in the final + :/_ parameter; just give what you + want to change. For instance set_info d74 _"snet: 5.6.7.8" would + just update the meta data for device id 74. """ - if len(argv) != 5: + if len(argv) < 5 or len(argv) % 2 != 1: print Commands.set_info.__doc__.strip() print print search_devs.__doc__.strip() exit(EXIT_ERROR) - devs = search_devs(builder, argv[3]) - change_value = argv[4] - change = [] - if len(change_value) and change_value[0].isdigit(): - i = 1 - while i < len(change_value) and change_value[i] in '0123456789.': + + searches_and_changes = izip(islice(argv, 3, len(argv), 2), + islice(argv, 4, len(argv), 2)) + + for search_value, change_value in searches_and_changes: + devs = search_devs(builder, search_value) + change = [] + if len(change_value) and change_value[0].isdigit(): + i = 1 + while (i < len(change_value) and + change_value[i] in '0123456789.'): + i += 1 + change.append(('ip', change_value[:i])) + change_value = change_value[i:] + elif len(change_value) and change_value[0] == '[': + i = 1 + while i < len(change_value) and change_value[i] != ']': + i += 1 i += 1 - change.append(('ip', change_value[:i])) - change_value = change_value[i:] - elif len(change_value) and change_value[0] == '[': - i = 1 - while i < len(change_value) and change_value[i] != ']': - i += 1 - i += 1 - change.append(('ip', change_value[:i].lstrip('[').rstrip(']'))) - change_value = change_value[i:] - if change_value.startswith(':'): - i = 1 - while i < len(change_value) and change_value[i].isdigit(): - i += 1 - change.append(('port', int(change_value[1:i]))) - change_value = change_value[i:] - if change_value.startswith('/'): - i = 1 - while i < len(change_value) and change_value[i] != '_': - i += 1 - change.append(('device', change_value[1:i])) - change_value = change_value[i:] - if change_value.startswith('_'): - change.append(('meta', change_value[1:])) - change_value = '' - if change_value or not change: - raise ValueError('Invalid set info change value: %s' % - repr(argv[4])) - if not devs: - print 'No matching devices found' - exit(EXIT_ERROR) - if len(devs) > 1: - print 'Matched more than one device:' - for dev in devs: - print ' %s' % format_device(dev) - if raw_input('Are you sure you want to update the info for ' - 'these %s devices? (y/N) ' % len(devs)) != 'y': - print 'Aborting device modifications' + change.append(('ip', change_value[:i].lstrip('[').rstrip(']'))) + change_value = change_value[i:] + if change_value.startswith(':'): + i = 1 + while i < len(change_value) and change_value[i].isdigit(): + i += 1 + change.append(('port', int(change_value[1:i]))) + change_value = change_value[i:] + if change_value.startswith('/'): + i = 1 + while i < len(change_value) and change_value[i] != '_': + i += 1 + change.append(('device', change_value[1:i])) + change_value = change_value[i:] + if change_value.startswith('_'): + change.append(('meta', change_value[1:])) + change_value = '' + if change_value or not change: + raise ValueError('Invalid set info change value: %s' % + repr(argv[4])) + if not devs: + print("Search value \"%s\" matched 0 devices.\n" + "The on-disk ring builder is unchanged.\n" + % search_value) exit(EXIT_ERROR) - for dev in devs: - orig_dev_string = format_device(dev) - test_dev = dict(dev) - for key, value in change: - test_dev[key] = value - for check_dev in builder.devs: - if not check_dev or check_dev['id'] == test_dev['id']: - continue - if check_dev['ip'] == test_dev['ip'] and \ - check_dev['port'] == test_dev['port'] and \ - check_dev['device'] == test_dev['device']: - print 'Device %d already uses %s:%d/%s.' % \ - (check_dev['id'], check_dev['ip'], check_dev['port'], - check_dev['device']) + if len(devs) > 1: + print 'Matched more than one device:' + for dev in devs: + print ' %s' % format_device(dev) + if raw_input('Are you sure you want to update the info for ' + 'these %s devices? (y/N) ' % len(devs)) != 'y': + print 'Aborting device modifications' exit(EXIT_ERROR) - for key, value in change: - dev[key] = value - print 'Device %s is now %s' % (orig_dev_string, format_device(dev)) + for dev in devs: + orig_dev_string = format_device(dev) + test_dev = dict(dev) + for key, value in change: + test_dev[key] = value + for check_dev in builder.devs: + if not check_dev or check_dev['id'] == test_dev['id']: + continue + if check_dev['ip'] == test_dev['ip'] and \ + check_dev['port'] == test_dev['port'] and \ + check_dev['device'] == test_dev['device']: + print 'Device %d already uses %s:%d/%s.' % \ + (check_dev['id'], check_dev['ip'], + check_dev['port'], check_dev['device']) + exit(EXIT_ERROR) + for key, value in change: + dev[key] = value + print 'Device %s is now %s' % (orig_dev_string, + format_device(dev)) pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_SUCCESS) def remove(): """ -swift-ring-builder remove +swift-ring-builder remove [search-value ...] Removes the device(s) from the ring. This should normally just be used for a device that has failed. For a device you wish to decommission, it's best to set its weight to 0, wait for it to drain all its data, then use this @@ -481,38 +520,42 @@ swift-ring-builder remove print print search_devs.__doc__.strip() exit(EXIT_ERROR) - devs = search_devs(builder, argv[3]) - if not devs: - print 'No matching devices found' - exit(EXIT_ERROR) - if len(devs) > 1: - print 'Matched more than one device:' - for dev in devs: - print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ - '"%(meta)s"' % dev - if raw_input('Are you sure you want to remove these %s devices? ' - '(y/N) ' % len(devs)) != 'y': - print 'Aborting device removals' - exit(EXIT_ERROR) - for dev in devs: - try: - builder.remove_dev(dev['id']) - except exceptions.RingBuilderError, e: - print '-' * 79 - print ("An error occurred while removing device with id %d\n" - "This usually means that you attempted to remove the\n" - "last device in a ring. If this is the case, consider\n" - "creating a new ring instead.\n" - "The on-disk ring builder is unchanged\n" - "Original exception message: %s" % - (dev['id'], e.message) - ) - print '-' * 79 - exit(EXIT_ERROR) - print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \ - 'marked for removal and will be removed next rebalance.' \ - % dev + for search_value in argv[3:]: + devs = search_devs(builder, search_value) + if not devs: + print("Search value \"%s\" matched 0 devices.\n" + "The on-disk ring builder is unchanged.\n" % devstr) + exit(EXIT_ERROR) + if len(devs) > 1: + print 'Matched more than one device:' + for dev in devs: + print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ + '"%(meta)s"' % dev + if raw_input('Are you sure you want to remove these %s ' + 'devices? (y/N) ' % len(devs)) != 'y': + print 'Aborting device removals' + exit(EXIT_ERROR) + for dev in devs: + try: + builder.remove_dev(dev['id']) + except exceptions.RingBuilderError, e: + print '-' * 79 + print( + "An error occurred while removing device with id %d\n" + "This usually means that you attempted to remove\n" + "the last device in a ring. If this is the case,\n" + "consider creating a new ring instead.\n" + "The on-disk ring builder is unchanged.\n" + "Original exception message: %s" % + (dev['id'], e.message) + ) + print '-' * 79 + exit(EXIT_ERROR) + + print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ + '"%(meta)s" marked for removal and will be removed' \ + ' next rebalance.' % dev pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_SUCCESS)