openstack/swift
Code Issues Proposed changes

Merge "Breakout search_devs & add get_builder() for reuse"

Browse Source
This commit is contained in:
Jenkins 2012-09-07 22:09:58 +00:00 committed by Gerrit Code Review
commit 67f21f5e4a
6 changed files with 227 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
bin/swift-ring-builder
View File

@ -17,11 +17,10 @@
import cPickle as pickle
from array import array
from errno import EEXIST
from gzip import GzipFile
from itertools import islice, izip
from os import mkdir
from os.path import basename, dirname, exists, join as pathjoin
from sys import argv, exit, modules
from sys import argv, exit
from textwrap import wrap
from time import time
@ -36,92 +35,6 @@ EXIT_WARNING = 1
EXIT_ERROR = 2
def search_devs(builder, search_value):
"""
The <search-value> can be of the form:
d<device_id>z<zone>-<ip>:<port>/<device_name>_<meta>
Any part is optional, but you must include at least one part.
Examples:
d74 Matches the device id 74
z1 Matches devices in zone 1
z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4
1.2.3.4 Matches devices in any zone with the ip 1.2.3.4
z1:5678 Matches devices in zone 1 using port 5678
:5678 Matches devices that use port 5678
/sdb1 Matches devices with the device name sdb1
_shiny Matches devices with shiny in the meta data
_"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data
[::1] Matches devices in any zone with the ip ::1
z1-[::1]:5678 Matches devices in zone 1 with ip ::1 and port 5678
Most specific example:
d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8"
Nerd explanation:
All items require their single character prefix except the ip, in which
case the - is optional unless the device id or zone is also included.
"""
orig_search_value = search_value
match = []
if search_value.startswith('d'):
i = 1
while i < len(search_value) and search_value[i].isdigit():
i += 1
match.append(('id', int(search_value[1:i])))
search_value = search_value[i:]
if search_value.startswith('z'):
i = 1
while i < len(search_value) and search_value[i].isdigit():
i += 1
match.append(('zone', int(search_value[1:i])))
search_value = search_value[i:]
if search_value.startswith('-'):
search_value = search_value[1:]
if len(search_value) and search_value[0].isdigit():
i = 1
while i < len(search_value) and search_value[i] in '0123456789.':
i += 1
match.append(('ip', search_value[:i]))
search_value = search_value[i:]
elif len(search_value) and search_value[0] == '[':
i = 1
while i < len(search_value) and search_value[i] != ']':
i += 1
i += 1
match.append(('ip', search_value[:i].lstrip('[').rstrip(']')))
search_value = search_value[i:]
if search_value.startswith(':'):
i = 1
while i < len(search_value) and search_value[i].isdigit():
i += 1
match.append(('port', int(search_value[1:i])))
search_value = search_value[i:]
if search_value.startswith('/'):
i = 1
while i < len(search_value) and search_value[i] != '_':
i += 1
match.append(('device', search_value[1:i]))
search_value = search_value[i:]
if search_value.startswith('_'):
match.append(('meta', search_value[1:]))
search_value = ''
if search_value:
raise ValueError('Invalid <search-value>: %s' %
repr(orig_search_value))
devs = []
for dev in builder.devs:
if not dev:
continue
matched = True
for key, value in match:
if key == 'meta':
if value not in dev.get(key):
matched = False
elif dev.get(key) != value:
matched = False
if matched:
devs.append(dev)
return devs
def format_device(dev):
"""
Format a device for display.
@ -207,9 +120,9 @@ swift-ring-builder <builder_file> search <search-value>
if len(argv) < 4:
print Commands.search.__doc__.strip()
print
print search_devs.__doc__.strip()
print builder.search_devs.__doc__.strip()
exit(EXIT_ERROR)
devs = search_devs(builder, argv[3])
devs = builder.search_devs(argv[3])
if not devs:
print 'No matching devices found'
exit(EXIT_ERROR)
@ -245,11 +158,11 @@ swift-ring-builder <builder_file> list_parts <search-value> [<search-value>] ..
if len(argv) < 4:
print Commands.list_parts.__doc__.strip()
print
print search_devs.__doc__.strip()
print builder.search_devs.__doc__.strip()
exit(EXIT_ERROR)
devs = []
for arg in argv[3:]:
devs.extend(search_devs(builder, arg) or [])
devs.extend(builder.search_devs(arg) or [])
if not devs:
print 'No matching devices found'
exit(EXIT_ERROR)
@ -383,13 +296,13 @@ swift-ring-builder <builder_file> set_weight <search-value> <weight>
if len(argv) < 5 or len(argv) % 2 != 1:
print Commands.set_weight.__doc__.strip()
print
print search_devs.__doc__.strip()
print builder.search_devs.__doc__.strip()
exit(EXIT_ERROR)
devs_and_weights = izip(islice(argv, 3, len(argv), 2),
islice(argv, 4, len(argv), 2))
for devstr, weightstr in devs_and_weights:
devs = search_devs(builder, devstr)
devs = builder.search_devs(devstr)
weight = float(weightstr)
if not devs:
print("Search value \"%s\" matched 0 devices.\n"
@ -429,14 +342,14 @@ swift-ring-builder <builder_file> set_info
if len(argv) < 5 or len(argv) % 2 != 1:
print Commands.set_info.__doc__.strip()
print
print search_devs.__doc__.strip()
print builder.search_devs.__doc__.strip()
exit(EXIT_ERROR)
searches_and_changes = izip(islice(argv, 3, len(argv), 2),
islice(argv, 4, len(argv), 2))
for search_value, change_value in searches_and_changes:
devs = search_devs(builder, search_value)
devs = builder.search_devs(search_value)
change = []
if len(change_value) and change_value[0].isdigit():
i = 1
@ -518,14 +431,14 @@ swift-ring-builder <builder_file> remove <search-value> [search-value ...]
if len(argv) < 4:
print Commands.remove.__doc__.strip()
print
print search_devs.__doc__.strip()
print builder.search_devs.__doc__.strip()
exit(EXIT_ERROR)
for search_value in argv[3:]:
devs = search_devs(builder, search_value)
devs = builder.search_devs(search_value)
if not devs:
print("Search value \"%s\" matched 0 devices.\n"
"The on-disk ring builder is unchanged.\n" % devstr)
"The on-disk ring builder is unchanged." % search_value)
exit(EXIT_ERROR)
if len(devs) > 1:
print 'Matched more than one device:'
@ -681,32 +594,18 @@ if __name__ == '__main__':
for cmd in cmds:
print Commands.__dict__[cmd].__doc__.strip()
print
print search_devs.__doc__.strip()
print RingBuilder.search_devs.__doc__.strip()
print
for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ',
subsequent_indent=' '):
print line
print ('Exit codes: 0 = operation successful\n'
' 1 = operation completed with warnings\n' \
' 2 = error'
)
' 1 = operation completed with warnings\n'
' 2 = error')
exit(EXIT_SUCCESS)
if exists(argv[1]):
try:
builder = pickle.load(open(argv[1], 'rb'))
if not hasattr(builder, 'devs'):
builder_dict = builder
builder = RingBuilder(1, 1, 1)
builder.copy_from(builder_dict)
except ImportError: # Happens with really old builder pickles
modules['swift.ring_builder'] = \
modules['swift.common.ring.builder']
builder = RingBuilder(1, 1, 1)
builder.copy_from(pickle.load(open(argv[1], 'rb')))
for dev in builder.devs:
if dev and 'meta' not in dev:
dev['meta'] = ''
builder = RingBuilder.load(argv[1])
elif len(argv) < 3 or argv[2] != 'create':
print 'Ring Builder file does not exist: %s' % argv[1]
exit(EXIT_ERROR)

5
doc/source/development_saio.rst
View File

@ -31,7 +31,8 @@ Installing dependencies and the core code
#. `apt-get install curl gcc git-core memcached python-configobj
python-coverage python-dev python-nose python-setuptools python-simplejson
python-xattr sqlite3 xfsprogs python-webob python-eventlet
python-greenlet python-pastedeploy python-netifaces`
python-greenlet python-pastedeploy python-netifaces python-pip`
#. `pip install mock`
#. Install anything else you want, like screen, ssh, vim, etc.
* On Fedora, log in as root and do:
@ -40,7 +41,7 @@ Installing dependencies and the core code
openstack-swift-account openstack-swift-container openstack-swift-object`
#. `yum install xinetd rsync`
#. `yum install memcached`
#. `yum install python-netifaces python-nose`
#. `yum install python-netifaces python-nose python-mock`
This installs all necessary dependencies, and also creates user `swift`
and group `swift`. So, `swift:swift` ought to be used in every place where

113
swift/common/ring/builder.py
View File

@ -16,8 +16,11 @@
import bisect
import itertools
import math
import cPickle as pickle
from array import array
from sys import modules
from collections import defaultdict
from random import randint, shuffle
from time import time
@ -190,7 +193,8 @@ class RingBuilder(object):
self._ring = RingData([], devs, 32 - self.part_power)
else:
self._ring = \
RingData([array('H', p2d) for p2d in self._replica2part2dev],
RingData([array('H', p2d) for p2d in
self._replica2part2dev],
devs, 32 - self.part_power)
return self._ring
@ -518,7 +522,8 @@ class RingBuilder(object):
removed_replica = False
for tier in tiers_for_dev(dev):
if (replicas_at_tier[tier] > max_allowed_replicas[tier] and
self._last_part_moves[part] >= self.min_part_hours):
self._last_part_moves[part] >=
self.min_part_hours):
self._last_part_moves[part] = 0
spread_out_parts[part].append(replica)
dev['parts_wanted'] += 1
@ -737,3 +742,107 @@ class RingBuilder(object):
mr.update(walk_tree(subtier, submax))
return mr
return walk_tree((), self.replicas)
@classmethod
def load(cls, builder_file, open=open):
"""
Obtain RingBuilder instance of the provided builder file
:param builder_file: path to builder file to load
:return: RingBuilder instance
"""
builder = pickle.load(open(builder_file, 'rb'))
if not hasattr(builder, 'devs'):
builder_dict = builder
builder = RingBuilder(1, 1, 1)
builder.copy_from(builder_dict)
for dev in builder.devs:
#really old rings didn't have meta keys
if dev and 'meta' not in dev:
dev['meta'] = ''
return builder
def search_devs(self, search_value):
"""
The <search-value> can be of the form:
d<device_id>z<zone>-<ip>:<port>/<device_name>_<meta>
Any part is optional, but you must include at least one part.
Examples:
d74 Matches the device id 74
z1 Matches devices in zone 1
z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4
1.2.3.4 Matches devices in any zone with the ip 1.2.3.4
z1:5678 Matches devices in zone 1 using port 5678
:5678 Matches devices that use port 5678
/sdb1 Matches devices with the device name sdb1
_shiny Matches devices with shiny in the meta data
_"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data
[::1] Matches devices in any zone with the ip ::1
z1-[::1]:5678 Matches devices in zone 1 with ip ::1 and port 5678
Most specific example:
d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8"
Nerd explanation:
All items require their single character prefix except the ip, in which
case the - is optional unless the device id or zone is also included.
"""
orig_search_value = search_value
match = []
if search_value.startswith('d'):
i = 1
while i < len(search_value) and search_value[i].isdigit():
i += 1
match.append(('id', int(search_value[1:i])))
search_value = search_value[i:]
if search_value.startswith('z'):
i = 1
while i < len(search_value) and search_value[i].isdigit():
i += 1
match.append(('zone', int(search_value[1:i])))
search_value = search_value[i:]
if search_value.startswith('-'):
search_value = search_value[1:]
if len(search_value) and search_value[0].isdigit():
i = 1
while i < len(search_value) and search_value[i] in '0123456789.':
i += 1
match.append(('ip', search_value[:i]))
search_value = search_value[i:]
elif len(search_value) and search_value[0] == '[':
i = 1
while i < len(search_value) and search_value[i] != ']':
i += 1
i += 1
match.append(('ip', search_value[:i].lstrip('[').rstrip(']')))
search_value = search_value[i:]
if search_value.startswith(':'):
i = 1
while i < len(search_value) and search_value[i].isdigit():
i += 1
match.append(('port', int(search_value[1:i])))
search_value = search_value[i:]
if search_value.startswith('/'):
i = 1
while i < len(search_value) and search_value[i] != '_':
i += 1
match.append(('device', search_value[1:i]))
search_value = search_value[i:]
if search_value.startswith('_'):
match.append(('meta', search_value[1:]))
search_value = ''
if search_value:
raise ValueError('Invalid <search-value>: %s' %
repr(orig_search_value))
matched_devs = []
for dev in self.devs:
if not dev:
continue
matched = True
for key, value in match:
if key == 'meta':
if value not in dev.get(key):
matched = False
elif dev.get(key) != value:
matched = False
if matched:
matched_devs.append(dev)
return matched_devs

2
swift/common/ring/ring.py
View File

@ -17,7 +17,7 @@ import array
import cPickle as pickle
from collections import defaultdict
from gzip import GzipFile
from os.path import getmtime, join as pathjoin
from os.path import getmtime
import struct
from time import time
import os

87
test/unit/common/ring/test_builder.py
View File

@ -15,13 +15,16 @@
import os
import unittest
import cPickle as pickle
from collections import defaultdict
from shutil import rmtree
from mock import Mock, call as mock_call
from swift.common import exceptions
from swift.common import ring
from swift.common.ring import RingBuilder, RingData
class TestRingBuilder(unittest.TestCase):
def setUp(self):
@ -38,7 +41,7 @@ class TestRingBuilder(unittest.TestCase):
self.assertEquals(rb.part_power, 8)
self.assertEquals(rb.replicas, 3)
self.assertEquals(rb.min_part_hours, 1)
self.assertEquals(rb.parts, 2**8)
self.assertEquals(rb.parts, 2 ** 8)
self.assertEquals(rb.devs, [])
self.assertEquals(rb.devs_changed, False)
self.assertEquals(rb.version, 0)
@ -161,8 +164,8 @@ class TestRingBuilder(unittest.TestCase):
if self._shuffled_gather_helper() and \
self._shuffled_gather_helper():
raise AssertionError('It is highly likely the ring is no '
'longer shuffling the set of partitions to reassign on a '
'rebalance.')
'longer shuffling the set of partitions '
'to reassign on a rebalance.')
def _shuffled_gather_helper(self):
rb = ring.RingBuilder(8, 3, 1)
@ -493,6 +496,84 @@ class TestRingBuilder(unittest.TestCase):
rb.rebalance()
def test_load(self):
rb = ring.RingBuilder(8, 3, 1)
devs = [{'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0',
'port': 10000, 'device': 'sda1', 'meta': 'meta0'},
{'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1',
'port': 10001, 'device': 'sdb1', 'meta': 'meta1'},
{'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2',
'port': 10002, 'device': 'sdc1', 'meta': 'meta2'},
{'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3',
'port': 10003, 'device': 'sdd1'}]
for d in devs:
rb.add_dev(d)
rb.rebalance()
real_pickle = pickle.load
try:
#test a legit builder
fake_pickle = Mock(return_value=rb)
fake_open = Mock(return_value=None)
pickle.load = fake_pickle
builder = RingBuilder.load('fake.builder', open=fake_open)
self.assertEquals(fake_pickle.call_count, 1)
fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
self.assertEquals(builder, rb)
fake_pickle.reset_mock()
fake_open.reset_mock()
#test old style builder
fake_pickle.return_value = rb.to_dict()
pickle.load = fake_pickle
builder = RingBuilder.load('fake.builder', open=fake_open)
fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
self.assertEquals(builder.devs, rb.devs)
fake_pickle.reset_mock()
fake_open.reset_mock()
#test old devs but no meta
no_meta_builder = rb
for dev in no_meta_builder.devs:
del(dev['meta'])
print no_meta_builder.devs
fake_pickle.return_value = no_meta_builder
pickle.load = fake_pickle
builder = RingBuilder.load('fake.builder', open=fake_open)
fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
self.assertEquals(builder.devs, rb.devs)
fake_pickle.reset_mock()
finally:
pickle.load = real_pickle
def test_search_devs(self):
rb = ring.RingBuilder(8, 3, 1)
devs = [{'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0',
'port': 10000, 'device': 'sda1', 'meta': 'meta0'},
{'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1',
'port': 10001, 'device': 'sdb1', 'meta': 'meta1'},
{'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2',
'port': 10002, 'device': 'sdc1', 'meta': 'meta2'},
{'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3',
'port': 10003, 'device': 'sdd1', 'meta': 'meta3'}]
for d in devs:
rb.add_dev(d)
rb.rebalance()
res = rb.search_devs('d1')
self.assertEquals(res, [devs[1]])
res = rb.search_devs('z1')
self.assertEquals(res, [devs[1]])
res = rb.search_devs('-127.0.0.1')
self.assertEquals(res, [devs[1]])
res = rb.search_devs('-[127.0.0.1]:10001')
self.assertEquals(res, [devs[1]])
res = rb.search_devs(':10001')
self.assertEquals(res, [devs[1]])
res = rb.search_devs('/sdb1')
self.assertEquals(res, [devs[1]])
res = rb.search_devs('_meta1')
self.assertRaises(ValueError, rb.search_devs, 'OMGPONIES')
def test_validate(self):
rb = ring.RingBuilder(8, 3, 1)
rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1',

1
tools/test-requires
View File

@ -5,3 +5,4 @@ openstack.nose_plugin
nosehtmloutput
pep8==0.6.1
sphinx>=1.1.2
mock>=0.7.0