Breakout search_devs & add get_builder() for reuse
This moves search_devs into RingBuilder to make it accessible to other utils that need to search the builder. Along the same lines this also adds a load() call to get a RingBuilder instance when working with the builder files. - This adds python-mock >= 0.7 as a dependency for unittests. On Ubuntu 10.04 you'll have to pip install it, on 12.04 you can apt-get install it. Fedora 17+ should be able to yum install it. - new pep8 compliance - Fixed a small issue (undefined var) in swift-ring-builder when remove was called but failed to find a match. Change-Id: I2e02684235aa2f4e901a00858ae037091594c545
This commit is contained in:
parent
7f89e50eaf
commit
c0537ac6e0
@ -17,11 +17,10 @@
|
|||||||
import cPickle as pickle
|
import cPickle as pickle
|
||||||
from array import array
|
from array import array
|
||||||
from errno import EEXIST
|
from errno import EEXIST
|
||||||
from gzip import GzipFile
|
|
||||||
from itertools import islice, izip
|
from itertools import islice, izip
|
||||||
from os import mkdir
|
from os import mkdir
|
||||||
from os.path import basename, dirname, exists, join as pathjoin
|
from os.path import basename, dirname, exists, join as pathjoin
|
||||||
from sys import argv, exit, modules
|
from sys import argv, exit
|
||||||
from textwrap import wrap
|
from textwrap import wrap
|
||||||
from time import time
|
from time import time
|
||||||
|
|
||||||
@ -36,92 +35,6 @@ EXIT_WARNING = 1
|
|||||||
EXIT_ERROR = 2
|
EXIT_ERROR = 2
|
||||||
|
|
||||||
|
|
||||||
def search_devs(builder, search_value):
|
|
||||||
"""
|
|
||||||
The <search-value> can be of the form:
|
|
||||||
d<device_id>z<zone>-<ip>:<port>/<device_name>_<meta>
|
|
||||||
Any part is optional, but you must include at least one part.
|
|
||||||
Examples:
|
|
||||||
d74 Matches the device id 74
|
|
||||||
z1 Matches devices in zone 1
|
|
||||||
z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4
|
|
||||||
1.2.3.4 Matches devices in any zone with the ip 1.2.3.4
|
|
||||||
z1:5678 Matches devices in zone 1 using port 5678
|
|
||||||
:5678 Matches devices that use port 5678
|
|
||||||
/sdb1 Matches devices with the device name sdb1
|
|
||||||
_shiny Matches devices with shiny in the meta data
|
|
||||||
_"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data
|
|
||||||
[::1] Matches devices in any zone with the ip ::1
|
|
||||||
z1-[::1]:5678 Matches devices in zone 1 with ip ::1 and port 5678
|
|
||||||
Most specific example:
|
|
||||||
d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8"
|
|
||||||
Nerd explanation:
|
|
||||||
All items require their single character prefix except the ip, in which
|
|
||||||
case the - is optional unless the device id or zone is also included.
|
|
||||||
"""
|
|
||||||
orig_search_value = search_value
|
|
||||||
match = []
|
|
||||||
if search_value.startswith('d'):
|
|
||||||
i = 1
|
|
||||||
while i < len(search_value) and search_value[i].isdigit():
|
|
||||||
i += 1
|
|
||||||
match.append(('id', int(search_value[1:i])))
|
|
||||||
search_value = search_value[i:]
|
|
||||||
if search_value.startswith('z'):
|
|
||||||
i = 1
|
|
||||||
while i < len(search_value) and search_value[i].isdigit():
|
|
||||||
i += 1
|
|
||||||
match.append(('zone', int(search_value[1:i])))
|
|
||||||
search_value = search_value[i:]
|
|
||||||
if search_value.startswith('-'):
|
|
||||||
search_value = search_value[1:]
|
|
||||||
if len(search_value) and search_value[0].isdigit():
|
|
||||||
i = 1
|
|
||||||
while i < len(search_value) and search_value[i] in '0123456789.':
|
|
||||||
i += 1
|
|
||||||
match.append(('ip', search_value[:i]))
|
|
||||||
search_value = search_value[i:]
|
|
||||||
elif len(search_value) and search_value[0] == '[':
|
|
||||||
i = 1
|
|
||||||
while i < len(search_value) and search_value[i] != ']':
|
|
||||||
i += 1
|
|
||||||
i += 1
|
|
||||||
match.append(('ip', search_value[:i].lstrip('[').rstrip(']')))
|
|
||||||
search_value = search_value[i:]
|
|
||||||
if search_value.startswith(':'):
|
|
||||||
i = 1
|
|
||||||
while i < len(search_value) and search_value[i].isdigit():
|
|
||||||
i += 1
|
|
||||||
match.append(('port', int(search_value[1:i])))
|
|
||||||
search_value = search_value[i:]
|
|
||||||
if search_value.startswith('/'):
|
|
||||||
i = 1
|
|
||||||
while i < len(search_value) and search_value[i] != '_':
|
|
||||||
i += 1
|
|
||||||
match.append(('device', search_value[1:i]))
|
|
||||||
search_value = search_value[i:]
|
|
||||||
if search_value.startswith('_'):
|
|
||||||
match.append(('meta', search_value[1:]))
|
|
||||||
search_value = ''
|
|
||||||
if search_value:
|
|
||||||
raise ValueError('Invalid <search-value>: %s' %
|
|
||||||
repr(orig_search_value))
|
|
||||||
devs = []
|
|
||||||
for dev in builder.devs:
|
|
||||||
if not dev:
|
|
||||||
continue
|
|
||||||
matched = True
|
|
||||||
for key, value in match:
|
|
||||||
if key == 'meta':
|
|
||||||
if value not in dev.get(key):
|
|
||||||
matched = False
|
|
||||||
elif dev.get(key) != value:
|
|
||||||
matched = False
|
|
||||||
if matched:
|
|
||||||
devs.append(dev)
|
|
||||||
return devs
|
|
||||||
|
|
||||||
|
|
||||||
def format_device(dev):
|
def format_device(dev):
|
||||||
"""
|
"""
|
||||||
Format a device for display.
|
Format a device for display.
|
||||||
@ -157,7 +70,7 @@ swift-ring-builder <builder_file> create <part_power> <replicas>
|
|||||||
if err.errno != EEXIST:
|
if err.errno != EEXIST:
|
||||||
raise
|
raise
|
||||||
pickle.dump(builder.to_dict(), open(pathjoin(backup_dir,
|
pickle.dump(builder.to_dict(), open(pathjoin(backup_dir,
|
||||||
'%d.' % time() + basename(argv[1])), 'wb'), protocol=2)
|
'%d.' % time() + basename(argv[1])), 'wb'), protocol=2)
|
||||||
pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
|
pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
|
||||||
exit(EXIT_SUCCESS)
|
exit(EXIT_SUCCESS)
|
||||||
|
|
||||||
@ -192,7 +105,7 @@ swift-ring-builder <builder_file>
|
|||||||
balance = 0
|
balance = 0
|
||||||
else:
|
else:
|
||||||
balance = 100.0 * dev['parts'] / \
|
balance = 100.0 * dev['parts'] / \
|
||||||
(dev['weight'] * weighted_parts) - 100.0
|
(dev['weight'] * weighted_parts) - 100.0
|
||||||
print ' %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % \
|
print ' %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % \
|
||||||
(dev['id'], dev['zone'], dev['ip'], dev['port'],
|
(dev['id'], dev['zone'], dev['ip'], dev['port'],
|
||||||
dev['device'], dev['weight'], dev['parts'], balance,
|
dev['device'], dev['weight'], dev['parts'], balance,
|
||||||
@ -207,9 +120,9 @@ swift-ring-builder <builder_file> search <search-value>
|
|||||||
if len(argv) < 4:
|
if len(argv) < 4:
|
||||||
print Commands.search.__doc__.strip()
|
print Commands.search.__doc__.strip()
|
||||||
print
|
print
|
||||||
print search_devs.__doc__.strip()
|
print builder.search_devs.__doc__.strip()
|
||||||
exit(EXIT_ERROR)
|
exit(EXIT_ERROR)
|
||||||
devs = search_devs(builder, argv[3])
|
devs = builder.search_devs(argv[3])
|
||||||
if not devs:
|
if not devs:
|
||||||
print 'No matching devices found'
|
print 'No matching devices found'
|
||||||
exit(EXIT_ERROR)
|
exit(EXIT_ERROR)
|
||||||
@ -225,7 +138,7 @@ swift-ring-builder <builder_file> search <search-value>
|
|||||||
balance = 0
|
balance = 0
|
||||||
else:
|
else:
|
||||||
balance = 100.0 * dev['parts'] / \
|
balance = 100.0 * dev['parts'] / \
|
||||||
(dev['weight'] * weighted_parts) - 100.0
|
(dev['weight'] * weighted_parts) - 100.0
|
||||||
print ' %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % \
|
print ' %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % \
|
||||||
(dev['id'], dev['zone'], dev['ip'], dev['port'],
|
(dev['id'], dev['zone'], dev['ip'], dev['port'],
|
||||||
dev['device'], dev['weight'], dev['parts'], balance,
|
dev['device'], dev['weight'], dev['parts'], balance,
|
||||||
@ -245,11 +158,11 @@ swift-ring-builder <builder_file> list_parts <search-value> [<search-value>] ..
|
|||||||
if len(argv) < 4:
|
if len(argv) < 4:
|
||||||
print Commands.list_parts.__doc__.strip()
|
print Commands.list_parts.__doc__.strip()
|
||||||
print
|
print
|
||||||
print search_devs.__doc__.strip()
|
print builder.search_devs.__doc__.strip()
|
||||||
exit(EXIT_ERROR)
|
exit(EXIT_ERROR)
|
||||||
devs = []
|
devs = []
|
||||||
for arg in argv[3:]:
|
for arg in argv[3:]:
|
||||||
devs.extend(search_devs(builder, arg) or [])
|
devs.extend(builder.search_devs(arg) or [])
|
||||||
if not devs:
|
if not devs:
|
||||||
print 'No matching devices found'
|
print 'No matching devices found'
|
||||||
exit(EXIT_ERROR)
|
exit(EXIT_ERROR)
|
||||||
@ -383,13 +296,13 @@ swift-ring-builder <builder_file> set_weight <search-value> <weight>
|
|||||||
if len(argv) < 5 or len(argv) % 2 != 1:
|
if len(argv) < 5 or len(argv) % 2 != 1:
|
||||||
print Commands.set_weight.__doc__.strip()
|
print Commands.set_weight.__doc__.strip()
|
||||||
print
|
print
|
||||||
print search_devs.__doc__.strip()
|
print builder.search_devs.__doc__.strip()
|
||||||
exit(EXIT_ERROR)
|
exit(EXIT_ERROR)
|
||||||
|
|
||||||
devs_and_weights = izip(islice(argv, 3, len(argv), 2),
|
devs_and_weights = izip(islice(argv, 3, len(argv), 2),
|
||||||
islice(argv, 4, len(argv), 2))
|
islice(argv, 4, len(argv), 2))
|
||||||
for devstr, weightstr in devs_and_weights:
|
for devstr, weightstr in devs_and_weights:
|
||||||
devs = search_devs(builder, devstr)
|
devs = builder.search_devs(devstr)
|
||||||
weight = float(weightstr)
|
weight = float(weightstr)
|
||||||
if not devs:
|
if not devs:
|
||||||
print("Search value \"%s\" matched 0 devices.\n"
|
print("Search value \"%s\" matched 0 devices.\n"
|
||||||
@ -429,14 +342,14 @@ swift-ring-builder <builder_file> set_info
|
|||||||
if len(argv) < 5 or len(argv) % 2 != 1:
|
if len(argv) < 5 or len(argv) % 2 != 1:
|
||||||
print Commands.set_info.__doc__.strip()
|
print Commands.set_info.__doc__.strip()
|
||||||
print
|
print
|
||||||
print search_devs.__doc__.strip()
|
print builder.search_devs.__doc__.strip()
|
||||||
exit(EXIT_ERROR)
|
exit(EXIT_ERROR)
|
||||||
|
|
||||||
searches_and_changes = izip(islice(argv, 3, len(argv), 2),
|
searches_and_changes = izip(islice(argv, 3, len(argv), 2),
|
||||||
islice(argv, 4, len(argv), 2))
|
islice(argv, 4, len(argv), 2))
|
||||||
|
|
||||||
for search_value, change_value in searches_and_changes:
|
for search_value, change_value in searches_and_changes:
|
||||||
devs = search_devs(builder, search_value)
|
devs = builder.search_devs(search_value)
|
||||||
change = []
|
change = []
|
||||||
if len(change_value) and change_value[0].isdigit():
|
if len(change_value) and change_value[0].isdigit():
|
||||||
i = 1
|
i = 1
|
||||||
@ -518,14 +431,14 @@ swift-ring-builder <builder_file> remove <search-value> [search-value ...]
|
|||||||
if len(argv) < 4:
|
if len(argv) < 4:
|
||||||
print Commands.remove.__doc__.strip()
|
print Commands.remove.__doc__.strip()
|
||||||
print
|
print
|
||||||
print search_devs.__doc__.strip()
|
print builder.search_devs.__doc__.strip()
|
||||||
exit(EXIT_ERROR)
|
exit(EXIT_ERROR)
|
||||||
|
|
||||||
for search_value in argv[3:]:
|
for search_value in argv[3:]:
|
||||||
devs = search_devs(builder, search_value)
|
devs = builder.search_devs(search_value)
|
||||||
if not devs:
|
if not devs:
|
||||||
print("Search value \"%s\" matched 0 devices.\n"
|
print("Search value \"%s\" matched 0 devices.\n"
|
||||||
"The on-disk ring builder is unchanged.\n" % devstr)
|
"The on-disk ring builder is unchanged." % search_value)
|
||||||
exit(EXIT_ERROR)
|
exit(EXIT_ERROR)
|
||||||
if len(devs) > 1:
|
if len(devs) > 1:
|
||||||
print 'Matched more than one device:'
|
print 'Matched more than one device:'
|
||||||
@ -549,7 +462,7 @@ swift-ring-builder <builder_file> remove <search-value> [search-value ...]
|
|||||||
"The on-disk ring builder is unchanged.\n"
|
"The on-disk ring builder is unchanged.\n"
|
||||||
"Original exception message: %s" %
|
"Original exception message: %s" %
|
||||||
(dev['id'], e.message)
|
(dev['id'], e.message)
|
||||||
)
|
)
|
||||||
print '-' * 79
|
print '-' * 79
|
||||||
exit(EXIT_ERROR)
|
exit(EXIT_ERROR)
|
||||||
|
|
||||||
@ -613,7 +526,7 @@ swift-ring-builder <builder_file> rebalance
|
|||||||
builder.get_ring().save(
|
builder.get_ring().save(
|
||||||
pathjoin(backup_dir, '%d.' % ts + basename(ring_file)))
|
pathjoin(backup_dir, '%d.' % ts + basename(ring_file)))
|
||||||
pickle.dump(builder.to_dict(), open(pathjoin(backup_dir,
|
pickle.dump(builder.to_dict(), open(pathjoin(backup_dir,
|
||||||
'%d.' % ts + basename(argv[1])), 'wb'), protocol=2)
|
'%d.' % ts + basename(argv[1])), 'wb'), protocol=2)
|
||||||
builder.get_ring().save(ring_file)
|
builder.get_ring().save(ring_file)
|
||||||
pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
|
pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
|
||||||
exit(status)
|
exit(status)
|
||||||
@ -676,37 +589,23 @@ if __name__ == '__main__':
|
|||||||
print Commands.default.__doc__.strip()
|
print Commands.default.__doc__.strip()
|
||||||
print
|
print
|
||||||
cmds = [c for c, f in Commands.__dict__.iteritems()
|
cmds = [c for c, f in Commands.__dict__.iteritems()
|
||||||
if f.__doc__ and c[0] != '_' and c != 'default']
|
if f.__doc__ and c[0] != '_' and c != 'default']
|
||||||
cmds.sort()
|
cmds.sort()
|
||||||
for cmd in cmds:
|
for cmd in cmds:
|
||||||
print Commands.__dict__[cmd].__doc__.strip()
|
print Commands.__dict__[cmd].__doc__.strip()
|
||||||
print
|
print
|
||||||
print search_devs.__doc__.strip()
|
print RingBuilder.search_devs.__doc__.strip()
|
||||||
print
|
print
|
||||||
for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ',
|
for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ',
|
||||||
subsequent_indent=' '):
|
subsequent_indent=' '):
|
||||||
print line
|
print line
|
||||||
print ('Exit codes: 0 = operation successful\n'
|
print ('Exit codes: 0 = operation successful\n'
|
||||||
' 1 = operation completed with warnings\n' \
|
' 1 = operation completed with warnings\n'
|
||||||
' 2 = error'
|
' 2 = error')
|
||||||
)
|
|
||||||
exit(EXIT_SUCCESS)
|
exit(EXIT_SUCCESS)
|
||||||
|
|
||||||
if exists(argv[1]):
|
if exists(argv[1]):
|
||||||
try:
|
builder = RingBuilder.load(argv[1])
|
||||||
builder = pickle.load(open(argv[1], 'rb'))
|
|
||||||
if not hasattr(builder, 'devs'):
|
|
||||||
builder_dict = builder
|
|
||||||
builder = RingBuilder(1, 1, 1)
|
|
||||||
builder.copy_from(builder_dict)
|
|
||||||
except ImportError: # Happens with really old builder pickles
|
|
||||||
modules['swift.ring_builder'] = \
|
|
||||||
modules['swift.common.ring.builder']
|
|
||||||
builder = RingBuilder(1, 1, 1)
|
|
||||||
builder.copy_from(pickle.load(open(argv[1], 'rb')))
|
|
||||||
for dev in builder.devs:
|
|
||||||
if dev and 'meta' not in dev:
|
|
||||||
dev['meta'] = ''
|
|
||||||
elif len(argv) < 3 or argv[2] != 'create':
|
elif len(argv) < 3 or argv[2] != 'create':
|
||||||
print 'Ring Builder file does not exist: %s' % argv[1]
|
print 'Ring Builder file does not exist: %s' % argv[1]
|
||||||
exit(EXIT_ERROR)
|
exit(EXIT_ERROR)
|
||||||
|
@ -31,7 +31,8 @@ Installing dependencies and the core code
|
|||||||
#. `apt-get install curl gcc git-core memcached python-configobj
|
#. `apt-get install curl gcc git-core memcached python-configobj
|
||||||
python-coverage python-dev python-nose python-setuptools python-simplejson
|
python-coverage python-dev python-nose python-setuptools python-simplejson
|
||||||
python-xattr sqlite3 xfsprogs python-webob python-eventlet
|
python-xattr sqlite3 xfsprogs python-webob python-eventlet
|
||||||
python-greenlet python-pastedeploy python-netifaces`
|
python-greenlet python-pastedeploy python-netifaces python-pip`
|
||||||
|
#. `pip install mock`
|
||||||
#. Install anything else you want, like screen, ssh, vim, etc.
|
#. Install anything else you want, like screen, ssh, vim, etc.
|
||||||
|
|
||||||
* On Fedora, log in as root and do:
|
* On Fedora, log in as root and do:
|
||||||
@ -40,7 +41,7 @@ Installing dependencies and the core code
|
|||||||
openstack-swift-account openstack-swift-container openstack-swift-object`
|
openstack-swift-account openstack-swift-container openstack-swift-object`
|
||||||
#. `yum install xinetd rsync`
|
#. `yum install xinetd rsync`
|
||||||
#. `yum install memcached`
|
#. `yum install memcached`
|
||||||
#. `yum install python-netifaces python-nose`
|
#. `yum install python-netifaces python-nose python-mock`
|
||||||
|
|
||||||
This installs all necessary dependencies, and also creates user `swift`
|
This installs all necessary dependencies, and also creates user `swift`
|
||||||
and group `swift`. So, `swift:swift` ought to be used in every place where
|
and group `swift`. So, `swift:swift` ought to be used in every place where
|
||||||
|
@ -16,8 +16,11 @@
|
|||||||
import bisect
|
import bisect
|
||||||
import itertools
|
import itertools
|
||||||
import math
|
import math
|
||||||
|
import cPickle as pickle
|
||||||
|
|
||||||
|
|
||||||
from array import array
|
from array import array
|
||||||
|
from sys import modules
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from random import randint, shuffle
|
from random import randint, shuffle
|
||||||
from time import time
|
from time import time
|
||||||
@ -83,7 +86,7 @@ class RingBuilder(object):
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
return self.parts * self.replicas / \
|
return self.parts * self.replicas / \
|
||||||
sum(d['weight'] for d in self._iter_devs())
|
sum(d['weight'] for d in self._iter_devs())
|
||||||
except ZeroDivisionError:
|
except ZeroDivisionError:
|
||||||
raise exceptions.EmptyRingError('There are no devices in this '
|
raise exceptions.EmptyRingError('There are no devices in this '
|
||||||
'ring, or all devices have been '
|
'ring, or all devices have been '
|
||||||
@ -190,8 +193,9 @@ class RingBuilder(object):
|
|||||||
self._ring = RingData([], devs, 32 - self.part_power)
|
self._ring = RingData([], devs, 32 - self.part_power)
|
||||||
else:
|
else:
|
||||||
self._ring = \
|
self._ring = \
|
||||||
RingData([array('H', p2d) for p2d in self._replica2part2dev],
|
RingData([array('H', p2d) for p2d in
|
||||||
devs, 32 - self.part_power)
|
self._replica2part2dev],
|
||||||
|
devs, 32 - self.part_power)
|
||||||
return self._ring
|
return self._ring
|
||||||
|
|
||||||
def add_dev(self, dev):
|
def add_dev(self, dev):
|
||||||
@ -222,7 +226,7 @@ class RingBuilder(object):
|
|||||||
"""
|
"""
|
||||||
if dev['id'] < len(self.devs) and self.devs[dev['id']] is not None:
|
if dev['id'] < len(self.devs) and self.devs[dev['id']] is not None:
|
||||||
raise exceptions.DuplicateDeviceError(
|
raise exceptions.DuplicateDeviceError(
|
||||||
'Duplicate device id: %d' % dev['id'])
|
'Duplicate device id: %d' % dev['id'])
|
||||||
# Add holes to self.devs to ensure self.devs[dev['id']] will be the dev
|
# Add holes to self.devs to ensure self.devs[dev['id']] will be the dev
|
||||||
while dev['id'] >= len(self.devs):
|
while dev['id'] >= len(self.devs):
|
||||||
self.devs.append(None)
|
self.devs.append(None)
|
||||||
@ -454,7 +458,7 @@ class RingBuilder(object):
|
|||||||
"""
|
"""
|
||||||
self._replica2part2dev = \
|
self._replica2part2dev = \
|
||||||
[array('H', (0 for _junk in xrange(self.parts)))
|
[array('H', (0 for _junk in xrange(self.parts)))
|
||||||
for _junk in xrange(self.replicas)]
|
for _junk in xrange(self.replicas)]
|
||||||
|
|
||||||
replicas = range(self.replicas)
|
replicas = range(self.replicas)
|
||||||
self._last_part_moves = array('B', (0 for _junk in xrange(self.parts)))
|
self._last_part_moves = array('B', (0 for _junk in xrange(self.parts)))
|
||||||
@ -518,7 +522,8 @@ class RingBuilder(object):
|
|||||||
removed_replica = False
|
removed_replica = False
|
||||||
for tier in tiers_for_dev(dev):
|
for tier in tiers_for_dev(dev):
|
||||||
if (replicas_at_tier[tier] > max_allowed_replicas[tier] and
|
if (replicas_at_tier[tier] > max_allowed_replicas[tier] and
|
||||||
self._last_part_moves[part] >= self.min_part_hours):
|
self._last_part_moves[part] >=
|
||||||
|
self.min_part_hours):
|
||||||
self._last_part_moves[part] = 0
|
self._last_part_moves[part] = 0
|
||||||
spread_out_parts[part].append(replica)
|
spread_out_parts[part].append(replica)
|
||||||
dev['parts_wanted'] += 1
|
dev['parts_wanted'] += 1
|
||||||
@ -737,3 +742,107 @@ class RingBuilder(object):
|
|||||||
mr.update(walk_tree(subtier, submax))
|
mr.update(walk_tree(subtier, submax))
|
||||||
return mr
|
return mr
|
||||||
return walk_tree((), self.replicas)
|
return walk_tree((), self.replicas)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load(cls, builder_file, open=open):
|
||||||
|
"""
|
||||||
|
Obtain RingBuilder instance of the provided builder file
|
||||||
|
|
||||||
|
:param builder_file: path to builder file to load
|
||||||
|
:return: RingBuilder instance
|
||||||
|
"""
|
||||||
|
builder = pickle.load(open(builder_file, 'rb'))
|
||||||
|
if not hasattr(builder, 'devs'):
|
||||||
|
builder_dict = builder
|
||||||
|
builder = RingBuilder(1, 1, 1)
|
||||||
|
builder.copy_from(builder_dict)
|
||||||
|
for dev in builder.devs:
|
||||||
|
#really old rings didn't have meta keys
|
||||||
|
if dev and 'meta' not in dev:
|
||||||
|
dev['meta'] = ''
|
||||||
|
return builder
|
||||||
|
|
||||||
|
def search_devs(self, search_value):
|
||||||
|
"""
|
||||||
|
The <search-value> can be of the form:
|
||||||
|
d<device_id>z<zone>-<ip>:<port>/<device_name>_<meta>
|
||||||
|
Any part is optional, but you must include at least one part.
|
||||||
|
Examples:
|
||||||
|
d74 Matches the device id 74
|
||||||
|
z1 Matches devices in zone 1
|
||||||
|
z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4
|
||||||
|
1.2.3.4 Matches devices in any zone with the ip 1.2.3.4
|
||||||
|
z1:5678 Matches devices in zone 1 using port 5678
|
||||||
|
:5678 Matches devices that use port 5678
|
||||||
|
/sdb1 Matches devices with the device name sdb1
|
||||||
|
_shiny Matches devices with shiny in the meta data
|
||||||
|
_"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data
|
||||||
|
[::1] Matches devices in any zone with the ip ::1
|
||||||
|
z1-[::1]:5678 Matches devices in zone 1 with ip ::1 and port 5678
|
||||||
|
Most specific example:
|
||||||
|
d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8"
|
||||||
|
Nerd explanation:
|
||||||
|
All items require their single character prefix except the ip, in which
|
||||||
|
case the - is optional unless the device id or zone is also included.
|
||||||
|
"""
|
||||||
|
orig_search_value = search_value
|
||||||
|
match = []
|
||||||
|
if search_value.startswith('d'):
|
||||||
|
i = 1
|
||||||
|
while i < len(search_value) and search_value[i].isdigit():
|
||||||
|
i += 1
|
||||||
|
match.append(('id', int(search_value[1:i])))
|
||||||
|
search_value = search_value[i:]
|
||||||
|
if search_value.startswith('z'):
|
||||||
|
i = 1
|
||||||
|
while i < len(search_value) and search_value[i].isdigit():
|
||||||
|
i += 1
|
||||||
|
match.append(('zone', int(search_value[1:i])))
|
||||||
|
search_value = search_value[i:]
|
||||||
|
if search_value.startswith('-'):
|
||||||
|
search_value = search_value[1:]
|
||||||
|
if len(search_value) and search_value[0].isdigit():
|
||||||
|
i = 1
|
||||||
|
while i < len(search_value) and search_value[i] in '0123456789.':
|
||||||
|
i += 1
|
||||||
|
match.append(('ip', search_value[:i]))
|
||||||
|
search_value = search_value[i:]
|
||||||
|
elif len(search_value) and search_value[0] == '[':
|
||||||
|
i = 1
|
||||||
|
while i < len(search_value) and search_value[i] != ']':
|
||||||
|
i += 1
|
||||||
|
i += 1
|
||||||
|
match.append(('ip', search_value[:i].lstrip('[').rstrip(']')))
|
||||||
|
search_value = search_value[i:]
|
||||||
|
if search_value.startswith(':'):
|
||||||
|
i = 1
|
||||||
|
while i < len(search_value) and search_value[i].isdigit():
|
||||||
|
i += 1
|
||||||
|
match.append(('port', int(search_value[1:i])))
|
||||||
|
search_value = search_value[i:]
|
||||||
|
if search_value.startswith('/'):
|
||||||
|
i = 1
|
||||||
|
while i < len(search_value) and search_value[i] != '_':
|
||||||
|
i += 1
|
||||||
|
match.append(('device', search_value[1:i]))
|
||||||
|
search_value = search_value[i:]
|
||||||
|
if search_value.startswith('_'):
|
||||||
|
match.append(('meta', search_value[1:]))
|
||||||
|
search_value = ''
|
||||||
|
if search_value:
|
||||||
|
raise ValueError('Invalid <search-value>: %s' %
|
||||||
|
repr(orig_search_value))
|
||||||
|
matched_devs = []
|
||||||
|
for dev in self.devs:
|
||||||
|
if not dev:
|
||||||
|
continue
|
||||||
|
matched = True
|
||||||
|
for key, value in match:
|
||||||
|
if key == 'meta':
|
||||||
|
if value not in dev.get(key):
|
||||||
|
matched = False
|
||||||
|
elif dev.get(key) != value:
|
||||||
|
matched = False
|
||||||
|
if matched:
|
||||||
|
matched_devs.append(dev)
|
||||||
|
return matched_devs
|
||||||
|
@ -17,7 +17,7 @@ import array
|
|||||||
import cPickle as pickle
|
import cPickle as pickle
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from gzip import GzipFile
|
from gzip import GzipFile
|
||||||
from os.path import getmtime, join as pathjoin
|
from os.path import getmtime
|
||||||
import struct
|
import struct
|
||||||
from time import time
|
from time import time
|
||||||
import os
|
import os
|
||||||
|
@ -15,13 +15,16 @@
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import unittest
|
import unittest
|
||||||
|
import cPickle as pickle
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from shutil import rmtree
|
from shutil import rmtree
|
||||||
|
from mock import Mock, call as mock_call
|
||||||
|
|
||||||
from swift.common import exceptions
|
from swift.common import exceptions
|
||||||
from swift.common import ring
|
from swift.common import ring
|
||||||
from swift.common.ring import RingBuilder, RingData
|
from swift.common.ring import RingBuilder, RingData
|
||||||
|
|
||||||
|
|
||||||
class TestRingBuilder(unittest.TestCase):
|
class TestRingBuilder(unittest.TestCase):
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
@ -38,7 +41,7 @@ class TestRingBuilder(unittest.TestCase):
|
|||||||
self.assertEquals(rb.part_power, 8)
|
self.assertEquals(rb.part_power, 8)
|
||||||
self.assertEquals(rb.replicas, 3)
|
self.assertEquals(rb.replicas, 3)
|
||||||
self.assertEquals(rb.min_part_hours, 1)
|
self.assertEquals(rb.min_part_hours, 1)
|
||||||
self.assertEquals(rb.parts, 2**8)
|
self.assertEquals(rb.parts, 2 ** 8)
|
||||||
self.assertEquals(rb.devs, [])
|
self.assertEquals(rb.devs, [])
|
||||||
self.assertEquals(rb.devs_changed, False)
|
self.assertEquals(rb.devs_changed, False)
|
||||||
self.assertEquals(rb.version, 0)
|
self.assertEquals(rb.version, 0)
|
||||||
@ -159,10 +162,10 @@ class TestRingBuilder(unittest.TestCase):
|
|||||||
|
|
||||||
def test_shuffled_gather(self):
|
def test_shuffled_gather(self):
|
||||||
if self._shuffled_gather_helper() and \
|
if self._shuffled_gather_helper() and \
|
||||||
self._shuffled_gather_helper():
|
self._shuffled_gather_helper():
|
||||||
raise AssertionError('It is highly likely the ring is no '
|
raise AssertionError('It is highly likely the ring is no '
|
||||||
'longer shuffling the set of partitions to reassign on a '
|
'longer shuffling the set of partitions '
|
||||||
'rebalance.')
|
'to reassign on a rebalance.')
|
||||||
|
|
||||||
def _shuffled_gather_helper(self):
|
def _shuffled_gather_helper(self):
|
||||||
rb = ring.RingBuilder(8, 3, 1)
|
rb = ring.RingBuilder(8, 3, 1)
|
||||||
@ -493,6 +496,84 @@ class TestRingBuilder(unittest.TestCase):
|
|||||||
|
|
||||||
rb.rebalance()
|
rb.rebalance()
|
||||||
|
|
||||||
|
def test_load(self):
|
||||||
|
rb = ring.RingBuilder(8, 3, 1)
|
||||||
|
devs = [{'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0',
|
||||||
|
'port': 10000, 'device': 'sda1', 'meta': 'meta0'},
|
||||||
|
{'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1',
|
||||||
|
'port': 10001, 'device': 'sdb1', 'meta': 'meta1'},
|
||||||
|
{'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2',
|
||||||
|
'port': 10002, 'device': 'sdc1', 'meta': 'meta2'},
|
||||||
|
{'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3',
|
||||||
|
'port': 10003, 'device': 'sdd1'}]
|
||||||
|
for d in devs:
|
||||||
|
rb.add_dev(d)
|
||||||
|
rb.rebalance()
|
||||||
|
|
||||||
|
real_pickle = pickle.load
|
||||||
|
try:
|
||||||
|
#test a legit builder
|
||||||
|
fake_pickle = Mock(return_value=rb)
|
||||||
|
fake_open = Mock(return_value=None)
|
||||||
|
pickle.load = fake_pickle
|
||||||
|
builder = RingBuilder.load('fake.builder', open=fake_open)
|
||||||
|
self.assertEquals(fake_pickle.call_count, 1)
|
||||||
|
fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
|
||||||
|
self.assertEquals(builder, rb)
|
||||||
|
fake_pickle.reset_mock()
|
||||||
|
fake_open.reset_mock()
|
||||||
|
|
||||||
|
#test old style builder
|
||||||
|
fake_pickle.return_value = rb.to_dict()
|
||||||
|
pickle.load = fake_pickle
|
||||||
|
builder = RingBuilder.load('fake.builder', open=fake_open)
|
||||||
|
fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
|
||||||
|
self.assertEquals(builder.devs, rb.devs)
|
||||||
|
fake_pickle.reset_mock()
|
||||||
|
fake_open.reset_mock()
|
||||||
|
|
||||||
|
#test old devs but no meta
|
||||||
|
no_meta_builder = rb
|
||||||
|
for dev in no_meta_builder.devs:
|
||||||
|
del(dev['meta'])
|
||||||
|
print no_meta_builder.devs
|
||||||
|
fake_pickle.return_value = no_meta_builder
|
||||||
|
pickle.load = fake_pickle
|
||||||
|
builder = RingBuilder.load('fake.builder', open=fake_open)
|
||||||
|
fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
|
||||||
|
self.assertEquals(builder.devs, rb.devs)
|
||||||
|
fake_pickle.reset_mock()
|
||||||
|
finally:
|
||||||
|
pickle.load = real_pickle
|
||||||
|
|
||||||
|
def test_search_devs(self):
|
||||||
|
rb = ring.RingBuilder(8, 3, 1)
|
||||||
|
devs = [{'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0',
|
||||||
|
'port': 10000, 'device': 'sda1', 'meta': 'meta0'},
|
||||||
|
{'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1',
|
||||||
|
'port': 10001, 'device': 'sdb1', 'meta': 'meta1'},
|
||||||
|
{'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2',
|
||||||
|
'port': 10002, 'device': 'sdc1', 'meta': 'meta2'},
|
||||||
|
{'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3',
|
||||||
|
'port': 10003, 'device': 'sdd1', 'meta': 'meta3'}]
|
||||||
|
for d in devs:
|
||||||
|
rb.add_dev(d)
|
||||||
|
rb.rebalance()
|
||||||
|
res = rb.search_devs('d1')
|
||||||
|
self.assertEquals(res, [devs[1]])
|
||||||
|
res = rb.search_devs('z1')
|
||||||
|
self.assertEquals(res, [devs[1]])
|
||||||
|
res = rb.search_devs('-127.0.0.1')
|
||||||
|
self.assertEquals(res, [devs[1]])
|
||||||
|
res = rb.search_devs('-[127.0.0.1]:10001')
|
||||||
|
self.assertEquals(res, [devs[1]])
|
||||||
|
res = rb.search_devs(':10001')
|
||||||
|
self.assertEquals(res, [devs[1]])
|
||||||
|
res = rb.search_devs('/sdb1')
|
||||||
|
self.assertEquals(res, [devs[1]])
|
||||||
|
res = rb.search_devs('_meta1')
|
||||||
|
self.assertRaises(ValueError, rb.search_devs, 'OMGPONIES')
|
||||||
|
|
||||||
def test_validate(self):
|
def test_validate(self):
|
||||||
rb = ring.RingBuilder(8, 3, 1)
|
rb = ring.RingBuilder(8, 3, 1)
|
||||||
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1',
|
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1',
|
||||||
|
@ -5,3 +5,4 @@ openstack.nose_plugin
|
|||||||
nosehtmloutput
|
nosehtmloutput
|
||||||
pep8==0.6.1
|
pep8==0.6.1
|
||||||
sphinx>=1.1.2
|
sphinx>=1.1.2
|
||||||
|
mock>=0.7.0
|
||||||
|
Loading…
x
Reference in New Issue
Block a user