openstack/swift
Code Issues Proposed changes

Merge "Let some swift-ring-builder commands take >1 arg."

Browse Source
This commit is contained in:
Jenkins 2012-05-11 15:15:03 +00:00 committed by Gerrit Code Review
commit 86f37c47d7
1 changed files with 231 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
bin/swift-ring-builder
View File

@ -18,6 +18,7 @@ import cPickle as pickle
from array import array from array import array
from errno import EEXIST from errno import EEXIST
from gzip import GzipFile from gzip import GzipFile
from itertools import islice, izip
from os import mkdir from os import mkdir
from os.path import basename, dirname, exists, join as pathjoin from os.path import basename, dirname, exists, join as pathjoin
from sys import argv, exit, modules from sys import argv, exit, modules
@ -267,27 +268,33 @@ swift-ring-builder <builder_file> list_parts <search-value> [<search-value>] ..
def add(): def add():
""" """
swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta> swift-ring-builder <builder_file> add
<wght> z<zone>-<ip>:<port>/<device_name>_<meta> <weight>
Adds a device to the ring with the given information. No partitions will be [z<zone>-<ip>:<port>/<device_name>_<meta> <weight>] ...
Adds devices to the ring with the given information. No partitions will be
assigned to the new device until after running 'rebalance'. This is so you assigned to the new device until after running 'rebalance'. This is so you
can make multiple device changes and rebalance them all just once. can make multiple device changes and rebalance them all just once.
""" """
if len(argv) < 5: if len(argv) < 5 or len(argv) % 2 != 1:
print Commands.add.__doc__.strip() print Commands.add.__doc__.strip()
exit(EXIT_ERROR) exit(EXIT_ERROR)
if not argv[3].startswith('z'): devs_and_weights = izip(islice(argv, 3, len(argv), 2),
print 'Invalid add value: %s' % argv[3] islice(argv, 4, len(argv), 2))
for devstr, weightstr in devs_and_weights:
if not devstr.startswith('z'):
print 'Invalid add value: %s' % devstr
exit(EXIT_ERROR) exit(EXIT_ERROR)
i = 1 i = 1
while i < len(argv[3]) and argv[3][i].isdigit(): while i < len(devstr) and devstr[i].isdigit():
i += 1 i += 1
zone = int(argv[3][1:i]) zone = int(devstr[1:i])
rest = argv[3][i:] rest = devstr[i:]
if not rest.startswith('-'): if not rest.startswith('-'):
print 'Invalid add value: %s' % argv[3] print 'Invalid add value: %s' % devstr
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR) exit(EXIT_ERROR)
i = 1 i = 1
if rest[i] == '[': if rest[i] == '[':
@ -304,7 +311,8 @@ swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta>
rest = rest[i:] rest = rest[i:]
if not rest.startswith(':'): if not rest.startswith(':'):
print 'Invalid add value: %s' % argv[3] print 'Invalid add value: %s' % devstr
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR) exit(EXIT_ERROR)
i = 1 i = 1
while i < len(rest) and rest[i].isdigit(): while i < len(rest) and rest[i].isdigit():
@ -313,7 +321,8 @@ swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta>
rest = rest[i:] rest = rest[i:]
if not rest.startswith('/'): if not rest.startswith('/'):
print 'Invalid add value: %s' % argv[3] print 'Invalid add value: %s' % devstr
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR) exit(EXIT_ERROR)
i = 1 i = 1
while i < len(rest) and rest[i] != '_': while i < len(rest) and rest[i] != '_':
@ -325,7 +334,17 @@ swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta>
if rest.startswith('_'): if rest.startswith('_'):
meta = rest[1:] meta = rest[1:]
weight = float(argv[4]) try:
weight = float(weightstr)
except ValueError:
print 'Invalid weight value: %s' % weightstr
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR)
if weight < 0:
print 'Invalid weight value (must be positive): %s' % weightstr
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR)
for dev in builder.devs: for dev in builder.devs:
if dev is None: if dev is None:
@ -334,14 +353,15 @@ swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta>
dev['device'] == device_name: dev['device'] == device_name:
print 'Device %d already uses %s:%d/%s.' % \ print 'Device %d already uses %s:%d/%s.' % \
(dev['id'], dev['ip'], dev['port'], dev['device']) (dev['id'], dev['ip'], dev['port'], dev['device'])
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR) exit(EXIT_ERROR)
next_dev_id = 0 next_dev_id = 0
if builder.devs: if builder.devs:
next_dev_id = max(d['id'] for d in builder.devs if d) + 1 next_dev_id = max(d['id'] for d in builder.devs if d) + 1
builder.add_dev({'id': next_dev_id, 'zone': zone, 'ip': ip, builder.add_dev({'id': next_dev_id, 'zone': zone, 'ip': ip,
'port': port, 'device': device_name, 'weight': weight, 'port': port, 'device': device_name,
'meta': meta}) 'weight': weight, 'meta': meta})
if ':' in ip: if ':' in ip:
print 'Device z%s-[%s]:%s/%s_"%s" with %s weight got id %s' % \ print 'Device z%s-[%s]:%s/%s_"%s" with %s weight got id %s' % \
(zone, ip, port, device_name, meta, weight, next_dev_id) (zone, ip, port, device_name, meta, weight, next_dev_id)
@ -354,19 +374,27 @@ swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta>
def set_weight(): def set_weight():
""" """
swift-ring-builder <builder_file> set_weight <search-value> <weight> swift-ring-builder <builder_file> set_weight <search-value> <weight>
Resets the device's weight. No partitions will be reassigned to or from the [<search-value> <weight] ...
device until after running 'rebalance'. This is so you can make multiple
device changes and rebalance them all just once. Resets the devices' weights. No partitions will be reassigned to or from
the device until after running 'rebalance'. This is so you can make
multiple device changes and rebalance them all just once.
""" """
if len(argv) != 5: if len(argv) < 5 or len(argv) % 2 != 1:
print Commands.set_weight.__doc__.strip() print Commands.set_weight.__doc__.strip()
print print
print search_devs.__doc__.strip() print search_devs.__doc__.strip()
exit(EXIT_ERROR) exit(EXIT_ERROR)
devs = search_devs(builder, argv[3])
weight = float(argv[4]) devs_and_weights = izip(islice(argv, 3, len(argv), 2),
islice(argv, 4, len(argv), 2))
for devstr, weightstr in devs_and_weights:
devs = search_devs(builder, devstr)
weight = float(weightstr)
if not devs: if not devs:
print 'No matching devices found' print("Search value \"%s\" matched 0 devices.\n"
"The on-disk ring builder is unchanged.\n"
% devstr)
exit(EXIT_ERROR) exit(EXIT_ERROR)
if len(devs) > 1: if len(devs) > 1:
print 'Matched more than one device:' print 'Matched more than one device:'
@ -379,33 +407,41 @@ swift-ring-builder <builder_file> set_weight <search-value> <weight>
exit(EXIT_ERROR) exit(EXIT_ERROR)
for dev in devs: for dev in devs:
builder.set_dev_weight(dev['id'], weight) builder.set_dev_weight(dev['id'], weight)
print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \ print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \
'weight set to %(weight)s' % dev '"%(meta)s" weight set to %(weight)s' % dev
pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
exit(EXIT_SUCCESS) exit(EXIT_SUCCESS)
def set_info(): def set_info():
""" """
swift-ring-builder <builder_file> set_info <search-value> swift-ring-builder <builder_file> set_info
<ip>:<port>/<device_name>_<meta> <search-value> <ip>:<port>/<device_name>_<meta>
Resets the device's information. This information isn't used to assign [<search-value> <ip>:<port>/<device_name>_<meta>] ...
partitions, so you can use 'write_ring' afterward to rewrite the current
ring with the newer device information. Any of the parts are optional For each search-value, resets the matched device's information.
in the final <ip>:<port>/<device_name>_<meta> parameter; just give what you This information isn't used to assign partitions, so you can use
want to change. For instance set_info d74 _"snet: 5.6.7.8" would just 'write_ring' afterward to rewrite the current ring with the newer
update the meta data for device id 74. device information. Any of the parts are optional in the final
<ip>:<port>/<device_name>_<meta> parameter; just give what you
want to change. For instance set_info d74 _"snet: 5.6.7.8" would
just update the meta data for device id 74.
""" """
if len(argv) != 5: if len(argv) < 5 or len(argv) % 2 != 1:
print Commands.set_info.__doc__.strip() print Commands.set_info.__doc__.strip()
print print
print search_devs.__doc__.strip() print search_devs.__doc__.strip()
exit(EXIT_ERROR) exit(EXIT_ERROR)
devs = search_devs(builder, argv[3])
change_value = argv[4] searches_and_changes = izip(islice(argv, 3, len(argv), 2),
islice(argv, 4, len(argv), 2))
for search_value, change_value in searches_and_changes:
devs = search_devs(builder, search_value)
change = [] change = []
if len(change_value) and change_value[0].isdigit(): if len(change_value) and change_value[0].isdigit():
i = 1 i = 1
while i < len(change_value) and change_value[i] in '0123456789.': while (i < len(change_value) and
change_value[i] in '0123456789.'):
i += 1 i += 1
change.append(('ip', change_value[:i])) change.append(('ip', change_value[:i]))
change_value = change_value[i:] change_value = change_value[i:]
@ -435,7 +471,9 @@ swift-ring-builder <builder_file> set_info <search-value>
raise ValueError('Invalid set info change value: %s' % raise ValueError('Invalid set info change value: %s' %
repr(argv[4])) repr(argv[4]))
if not devs: if not devs:
print 'No matching devices found' print("Search value \"%s\" matched 0 devices.\n"
"The on-disk ring builder is unchanged.\n"
% search_value)
exit(EXIT_ERROR) exit(EXIT_ERROR)
if len(devs) > 1: if len(devs) > 1:
print 'Matched more than one device:' print 'Matched more than one device:'
@ -457,18 +495,19 @@ swift-ring-builder <builder_file> set_info <search-value>
check_dev['port'] == test_dev['port'] and \ check_dev['port'] == test_dev['port'] and \
check_dev['device'] == test_dev['device']: check_dev['device'] == test_dev['device']:
print 'Device %d already uses %s:%d/%s.' % \ print 'Device %d already uses %s:%d/%s.' % \
(check_dev['id'], check_dev['ip'], check_dev['port'], (check_dev['id'], check_dev['ip'],
check_dev['device']) check_dev['port'], check_dev['device'])
exit(EXIT_ERROR) exit(EXIT_ERROR)
for key, value in change: for key, value in change:
dev[key] = value dev[key] = value
print 'Device %s is now %s' % (orig_dev_string, format_device(dev)) print 'Device %s is now %s' % (orig_dev_string,
format_device(dev))
pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
exit(EXIT_SUCCESS) exit(EXIT_SUCCESS)
def remove(): def remove():
""" """
swift-ring-builder <builder_file> remove <search-value> swift-ring-builder <builder_file> remove <search-value> [search-value ...]
Removes the device(s) from the ring. This should normally just be used for Removes the device(s) from the ring. This should normally just be used for
a device that has failed. For a device you wish to decommission, it's best a device that has failed. For a device you wish to decommission, it's best
to set its weight to 0, wait for it to drain all its data, then use this to set its weight to 0, wait for it to drain all its data, then use this
@ -481,17 +520,20 @@ swift-ring-builder <builder_file> remove <search-value>
print print
print search_devs.__doc__.strip() print search_devs.__doc__.strip()
exit(EXIT_ERROR) exit(EXIT_ERROR)
devs = search_devs(builder, argv[3])
for search_value in argv[3:]:
devs = search_devs(builder, search_value)
if not devs: if not devs:
print 'No matching devices found' print("Search value \"%s\" matched 0 devices.\n"
"The on-disk ring builder is unchanged.\n" % devstr)
exit(EXIT_ERROR) exit(EXIT_ERROR)
if len(devs) > 1: if len(devs) > 1:
print 'Matched more than one device:' print 'Matched more than one device:'
for dev in devs: for dev in devs:
print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \
'"%(meta)s"' % dev '"%(meta)s"' % dev
if raw_input('Are you sure you want to remove these %s devices? ' if raw_input('Are you sure you want to remove these %s '
'(y/N) ' % len(devs)) != 'y': 'devices? (y/N) ' % len(devs)) != 'y':
print 'Aborting device removals' print 'Aborting device removals'
exit(EXIT_ERROR) exit(EXIT_ERROR)
for dev in devs: for dev in devs:
@ -499,20 +541,21 @@ swift-ring-builder <builder_file> remove <search-value>
builder.remove_dev(dev['id']) builder.remove_dev(dev['id'])
except exceptions.RingBuilderError, e: except exceptions.RingBuilderError, e:
print '-' * 79 print '-' * 79
print ("An error occurred while removing device with id %d\n" print(
"This usually means that you attempted to remove the\n" "An error occurred while removing device with id %d\n"
"last device in a ring. If this is the case, consider\n" "This usually means that you attempted to remove\n"
"creating a new ring instead.\n" "the last device in a ring. If this is the case,\n"
"The on-disk ring builder is unchanged\n" "consider creating a new ring instead.\n"
"The on-disk ring builder is unchanged.\n"
"Original exception message: %s" % "Original exception message: %s" %
(dev['id'], e.message) (dev['id'], e.message)
) )
print '-' * 79 print '-' * 79
exit(EXIT_ERROR) exit(EXIT_ERROR)
print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \ print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \
'marked for removal and will be removed next rebalance.' \ '"%(meta)s" marked for removal and will be removed' \
% dev ' next rebalance.' % dev
pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
exit(EXIT_SUCCESS) exit(EXIT_SUCCESS)