diff --git a/bin/swift-ring-builder b/bin/swift-ring-builder index fb1649f430..dbf3d0fec4 100755 --- a/bin/swift-ring-builder +++ b/bin/swift-ring-builder @@ -17,11 +17,10 @@ import cPickle as pickle from array import array from errno import EEXIST -from gzip import GzipFile from itertools import islice, izip from os import mkdir from os.path import basename, dirname, exists, join as pathjoin -from sys import argv, exit, modules +from sys import argv, exit from textwrap import wrap from time import time @@ -36,92 +35,6 @@ EXIT_WARNING = 1 EXIT_ERROR = 2 -def search_devs(builder, search_value): - """ -The can be of the form: - dz-:/_ - Any part is optional, but you must include at least one part. - Examples: - d74 Matches the device id 74 - z1 Matches devices in zone 1 - z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4 - 1.2.3.4 Matches devices in any zone with the ip 1.2.3.4 - z1:5678 Matches devices in zone 1 using port 5678 - :5678 Matches devices that use port 5678 - /sdb1 Matches devices with the device name sdb1 - _shiny Matches devices with shiny in the meta data - _"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data - [::1] Matches devices in any zone with the ip ::1 - z1-[::1]:5678 Matches devices in zone 1 with ip ::1 and port 5678 - Most specific example: - d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8" - Nerd explanation: - All items require their single character prefix except the ip, in which - case the - is optional unless the device id or zone is also included. - """ - orig_search_value = search_value - match = [] - if search_value.startswith('d'): - i = 1 - while i < len(search_value) and search_value[i].isdigit(): - i += 1 - match.append(('id', int(search_value[1:i]))) - search_value = search_value[i:] - if search_value.startswith('z'): - i = 1 - while i < len(search_value) and search_value[i].isdigit(): - i += 1 - match.append(('zone', int(search_value[1:i]))) - search_value = search_value[i:] - if search_value.startswith('-'): - search_value = search_value[1:] - if len(search_value) and search_value[0].isdigit(): - i = 1 - while i < len(search_value) and search_value[i] in '0123456789.': - i += 1 - match.append(('ip', search_value[:i])) - search_value = search_value[i:] - elif len(search_value) and search_value[0] == '[': - i = 1 - while i < len(search_value) and search_value[i] != ']': - i += 1 - i += 1 - match.append(('ip', search_value[:i].lstrip('[').rstrip(']'))) - search_value = search_value[i:] - if search_value.startswith(':'): - i = 1 - while i < len(search_value) and search_value[i].isdigit(): - i += 1 - match.append(('port', int(search_value[1:i]))) - search_value = search_value[i:] - if search_value.startswith('/'): - i = 1 - while i < len(search_value) and search_value[i] != '_': - i += 1 - match.append(('device', search_value[1:i])) - search_value = search_value[i:] - if search_value.startswith('_'): - match.append(('meta', search_value[1:])) - search_value = '' - if search_value: - raise ValueError('Invalid : %s' % - repr(orig_search_value)) - devs = [] - for dev in builder.devs: - if not dev: - continue - matched = True - for key, value in match: - if key == 'meta': - if value not in dev.get(key): - matched = False - elif dev.get(key) != value: - matched = False - if matched: - devs.append(dev) - return devs - - def format_device(dev): """ Format a device for display. @@ -157,7 +70,7 @@ swift-ring-builder create if err.errno != EEXIST: raise pickle.dump(builder.to_dict(), open(pathjoin(backup_dir, - '%d.' % time() + basename(argv[1])), 'wb'), protocol=2) + '%d.' % time() + basename(argv[1])), 'wb'), protocol=2) pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_SUCCESS) @@ -192,7 +105,7 @@ swift-ring-builder balance = 0 else: balance = 100.0 * dev['parts'] / \ - (dev['weight'] * weighted_parts) - 100.0 + (dev['weight'] * weighted_parts) - 100.0 print ' %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % \ (dev['id'], dev['zone'], dev['ip'], dev['port'], dev['device'], dev['weight'], dev['parts'], balance, @@ -207,9 +120,9 @@ swift-ring-builder search if len(argv) < 4: print Commands.search.__doc__.strip() print - print search_devs.__doc__.strip() + print builder.search_devs.__doc__.strip() exit(EXIT_ERROR) - devs = search_devs(builder, argv[3]) + devs = builder.search_devs(argv[3]) if not devs: print 'No matching devices found' exit(EXIT_ERROR) @@ -225,7 +138,7 @@ swift-ring-builder search balance = 0 else: balance = 100.0 * dev['parts'] / \ - (dev['weight'] * weighted_parts) - 100.0 + (dev['weight'] * weighted_parts) - 100.0 print ' %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % \ (dev['id'], dev['zone'], dev['ip'], dev['port'], dev['device'], dev['weight'], dev['parts'], balance, @@ -245,11 +158,11 @@ swift-ring-builder list_parts [] .. if len(argv) < 4: print Commands.list_parts.__doc__.strip() print - print search_devs.__doc__.strip() + print builder.search_devs.__doc__.strip() exit(EXIT_ERROR) devs = [] for arg in argv[3:]: - devs.extend(search_devs(builder, arg) or []) + devs.extend(builder.search_devs(arg) or []) if not devs: print 'No matching devices found' exit(EXIT_ERROR) @@ -383,13 +296,13 @@ swift-ring-builder set_weight if len(argv) < 5 or len(argv) % 2 != 1: print Commands.set_weight.__doc__.strip() print - print search_devs.__doc__.strip() + print builder.search_devs.__doc__.strip() exit(EXIT_ERROR) devs_and_weights = izip(islice(argv, 3, len(argv), 2), islice(argv, 4, len(argv), 2)) for devstr, weightstr in devs_and_weights: - devs = search_devs(builder, devstr) + devs = builder.search_devs(devstr) weight = float(weightstr) if not devs: print("Search value \"%s\" matched 0 devices.\n" @@ -429,14 +342,14 @@ swift-ring-builder set_info if len(argv) < 5 or len(argv) % 2 != 1: print Commands.set_info.__doc__.strip() print - print search_devs.__doc__.strip() + print builder.search_devs.__doc__.strip() exit(EXIT_ERROR) searches_and_changes = izip(islice(argv, 3, len(argv), 2), islice(argv, 4, len(argv), 2)) for search_value, change_value in searches_and_changes: - devs = search_devs(builder, search_value) + devs = builder.search_devs(search_value) change = [] if len(change_value) and change_value[0].isdigit(): i = 1 @@ -518,14 +431,14 @@ swift-ring-builder remove [search-value ...] if len(argv) < 4: print Commands.remove.__doc__.strip() print - print search_devs.__doc__.strip() + print builder.search_devs.__doc__.strip() exit(EXIT_ERROR) for search_value in argv[3:]: - devs = search_devs(builder, search_value) + devs = builder.search_devs(search_value) if not devs: print("Search value \"%s\" matched 0 devices.\n" - "The on-disk ring builder is unchanged.\n" % devstr) + "The on-disk ring builder is unchanged." % search_value) exit(EXIT_ERROR) if len(devs) > 1: print 'Matched more than one device:' @@ -549,7 +462,7 @@ swift-ring-builder remove [search-value ...] "The on-disk ring builder is unchanged.\n" "Original exception message: %s" % (dev['id'], e.message) - ) + ) print '-' * 79 exit(EXIT_ERROR) @@ -613,7 +526,7 @@ swift-ring-builder rebalance builder.get_ring().save( pathjoin(backup_dir, '%d.' % ts + basename(ring_file))) pickle.dump(builder.to_dict(), open(pathjoin(backup_dir, - '%d.' % ts + basename(argv[1])), 'wb'), protocol=2) + '%d.' % ts + basename(argv[1])), 'wb'), protocol=2) builder.get_ring().save(ring_file) pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(status) @@ -676,37 +589,23 @@ if __name__ == '__main__': print Commands.default.__doc__.strip() print cmds = [c for c, f in Commands.__dict__.iteritems() - if f.__doc__ and c[0] != '_' and c != 'default'] + if f.__doc__ and c[0] != '_' and c != 'default'] cmds.sort() for cmd in cmds: print Commands.__dict__[cmd].__doc__.strip() print - print search_devs.__doc__.strip() + print RingBuilder.search_devs.__doc__.strip() print for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ', subsequent_indent=' '): print line print ('Exit codes: 0 = operation successful\n' - ' 1 = operation completed with warnings\n' \ - ' 2 = error' - ) + ' 1 = operation completed with warnings\n' + ' 2 = error') exit(EXIT_SUCCESS) if exists(argv[1]): - try: - builder = pickle.load(open(argv[1], 'rb')) - if not hasattr(builder, 'devs'): - builder_dict = builder - builder = RingBuilder(1, 1, 1) - builder.copy_from(builder_dict) - except ImportError: # Happens with really old builder pickles - modules['swift.ring_builder'] = \ - modules['swift.common.ring.builder'] - builder = RingBuilder(1, 1, 1) - builder.copy_from(pickle.load(open(argv[1], 'rb'))) - for dev in builder.devs: - if dev and 'meta' not in dev: - dev['meta'] = '' + builder = RingBuilder.load(argv[1]) elif len(argv) < 3 or argv[2] != 'create': print 'Ring Builder file does not exist: %s' % argv[1] exit(EXIT_ERROR) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 54fe16ceab..496fa3b3a8 100755 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -31,7 +31,8 @@ Installing dependencies and the core code #. `apt-get install curl gcc git-core memcached python-configobj python-coverage python-dev python-nose python-setuptools python-simplejson python-xattr sqlite3 xfsprogs python-webob python-eventlet - python-greenlet python-pastedeploy python-netifaces` + python-greenlet python-pastedeploy python-netifaces python-pip` + #. `pip install mock` #. Install anything else you want, like screen, ssh, vim, etc. * On Fedora, log in as root and do: @@ -40,7 +41,7 @@ Installing dependencies and the core code openstack-swift-account openstack-swift-container openstack-swift-object` #. `yum install xinetd rsync` #. `yum install memcached` - #. `yum install python-netifaces python-nose` + #. `yum install python-netifaces python-nose python-mock` This installs all necessary dependencies, and also creates user `swift` and group `swift`. So, `swift:swift` ought to be used in every place where diff --git a/swift/common/ring/builder.py b/swift/common/ring/builder.py index 89ae9d2ec1..4e33169aaa 100644 --- a/swift/common/ring/builder.py +++ b/swift/common/ring/builder.py @@ -16,8 +16,11 @@ import bisect import itertools import math +import cPickle as pickle + from array import array +from sys import modules from collections import defaultdict from random import randint, shuffle from time import time @@ -83,7 +86,7 @@ class RingBuilder(object): """ try: return self.parts * self.replicas / \ - sum(d['weight'] for d in self._iter_devs()) + sum(d['weight'] for d in self._iter_devs()) except ZeroDivisionError: raise exceptions.EmptyRingError('There are no devices in this ' 'ring, or all devices have been ' @@ -190,8 +193,9 @@ class RingBuilder(object): self._ring = RingData([], devs, 32 - self.part_power) else: self._ring = \ - RingData([array('H', p2d) for p2d in self._replica2part2dev], - devs, 32 - self.part_power) + RingData([array('H', p2d) for p2d in + self._replica2part2dev], + devs, 32 - self.part_power) return self._ring def add_dev(self, dev): @@ -222,7 +226,7 @@ class RingBuilder(object): """ if dev['id'] < len(self.devs) and self.devs[dev['id']] is not None: raise exceptions.DuplicateDeviceError( - 'Duplicate device id: %d' % dev['id']) + 'Duplicate device id: %d' % dev['id']) # Add holes to self.devs to ensure self.devs[dev['id']] will be the dev while dev['id'] >= len(self.devs): self.devs.append(None) @@ -454,7 +458,7 @@ class RingBuilder(object): """ self._replica2part2dev = \ [array('H', (0 for _junk in xrange(self.parts))) - for _junk in xrange(self.replicas)] + for _junk in xrange(self.replicas)] replicas = range(self.replicas) self._last_part_moves = array('B', (0 for _junk in xrange(self.parts))) @@ -518,7 +522,8 @@ class RingBuilder(object): removed_replica = False for tier in tiers_for_dev(dev): if (replicas_at_tier[tier] > max_allowed_replicas[tier] and - self._last_part_moves[part] >= self.min_part_hours): + self._last_part_moves[part] >= + self.min_part_hours): self._last_part_moves[part] = 0 spread_out_parts[part].append(replica) dev['parts_wanted'] += 1 @@ -737,3 +742,107 @@ class RingBuilder(object): mr.update(walk_tree(subtier, submax)) return mr return walk_tree((), self.replicas) + + @classmethod + def load(cls, builder_file, open=open): + """ + Obtain RingBuilder instance of the provided builder file + + :param builder_file: path to builder file to load + :return: RingBuilder instance + """ + builder = pickle.load(open(builder_file, 'rb')) + if not hasattr(builder, 'devs'): + builder_dict = builder + builder = RingBuilder(1, 1, 1) + builder.copy_from(builder_dict) + for dev in builder.devs: + #really old rings didn't have meta keys + if dev and 'meta' not in dev: + dev['meta'] = '' + return builder + + def search_devs(self, search_value): + """ +The can be of the form: + dz-:/_ + Any part is optional, but you must include at least one part. + Examples: + d74 Matches the device id 74 + z1 Matches devices in zone 1 + z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4 + 1.2.3.4 Matches devices in any zone with the ip 1.2.3.4 + z1:5678 Matches devices in zone 1 using port 5678 + :5678 Matches devices that use port 5678 + /sdb1 Matches devices with the device name sdb1 + _shiny Matches devices with shiny in the meta data + _"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data + [::1] Matches devices in any zone with the ip ::1 + z1-[::1]:5678 Matches devices in zone 1 with ip ::1 and port 5678 + Most specific example: + d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8" + Nerd explanation: + All items require their single character prefix except the ip, in which + case the - is optional unless the device id or zone is also included. + """ + orig_search_value = search_value + match = [] + if search_value.startswith('d'): + i = 1 + while i < len(search_value) and search_value[i].isdigit(): + i += 1 + match.append(('id', int(search_value[1:i]))) + search_value = search_value[i:] + if search_value.startswith('z'): + i = 1 + while i < len(search_value) and search_value[i].isdigit(): + i += 1 + match.append(('zone', int(search_value[1:i]))) + search_value = search_value[i:] + if search_value.startswith('-'): + search_value = search_value[1:] + if len(search_value) and search_value[0].isdigit(): + i = 1 + while i < len(search_value) and search_value[i] in '0123456789.': + i += 1 + match.append(('ip', search_value[:i])) + search_value = search_value[i:] + elif len(search_value) and search_value[0] == '[': + i = 1 + while i < len(search_value) and search_value[i] != ']': + i += 1 + i += 1 + match.append(('ip', search_value[:i].lstrip('[').rstrip(']'))) + search_value = search_value[i:] + if search_value.startswith(':'): + i = 1 + while i < len(search_value) and search_value[i].isdigit(): + i += 1 + match.append(('port', int(search_value[1:i]))) + search_value = search_value[i:] + if search_value.startswith('/'): + i = 1 + while i < len(search_value) and search_value[i] != '_': + i += 1 + match.append(('device', search_value[1:i])) + search_value = search_value[i:] + if search_value.startswith('_'): + match.append(('meta', search_value[1:])) + search_value = '' + if search_value: + raise ValueError('Invalid : %s' % + repr(orig_search_value)) + matched_devs = [] + for dev in self.devs: + if not dev: + continue + matched = True + for key, value in match: + if key == 'meta': + if value not in dev.get(key): + matched = False + elif dev.get(key) != value: + matched = False + if matched: + matched_devs.append(dev) + return matched_devs diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 9fdad3070a..1f97ee2ccb 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -17,7 +17,7 @@ import array import cPickle as pickle from collections import defaultdict from gzip import GzipFile -from os.path import getmtime, join as pathjoin +from os.path import getmtime import struct from time import time import os diff --git a/test/unit/common/ring/test_builder.py b/test/unit/common/ring/test_builder.py index 52670ade8b..76f0c768d6 100644 --- a/test/unit/common/ring/test_builder.py +++ b/test/unit/common/ring/test_builder.py @@ -15,13 +15,16 @@ import os import unittest +import cPickle as pickle from collections import defaultdict from shutil import rmtree +from mock import Mock, call as mock_call from swift.common import exceptions from swift.common import ring from swift.common.ring import RingBuilder, RingData + class TestRingBuilder(unittest.TestCase): def setUp(self): @@ -38,7 +41,7 @@ class TestRingBuilder(unittest.TestCase): self.assertEquals(rb.part_power, 8) self.assertEquals(rb.replicas, 3) self.assertEquals(rb.min_part_hours, 1) - self.assertEquals(rb.parts, 2**8) + self.assertEquals(rb.parts, 2 ** 8) self.assertEquals(rb.devs, []) self.assertEquals(rb.devs_changed, False) self.assertEquals(rb.version, 0) @@ -159,10 +162,10 @@ class TestRingBuilder(unittest.TestCase): def test_shuffled_gather(self): if self._shuffled_gather_helper() and \ - self._shuffled_gather_helper(): + self._shuffled_gather_helper(): raise AssertionError('It is highly likely the ring is no ' - 'longer shuffling the set of partitions to reassign on a ' - 'rebalance.') + 'longer shuffling the set of partitions ' + 'to reassign on a rebalance.') def _shuffled_gather_helper(self): rb = ring.RingBuilder(8, 3, 1) @@ -493,6 +496,84 @@ class TestRingBuilder(unittest.TestCase): rb.rebalance() + def test_load(self): + rb = ring.RingBuilder(8, 3, 1) + devs = [{'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0', + 'port': 10000, 'device': 'sda1', 'meta': 'meta0'}, + {'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10001, 'device': 'sdb1', 'meta': 'meta1'}, + {'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2', + 'port': 10002, 'device': 'sdc1', 'meta': 'meta2'}, + {'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3', + 'port': 10003, 'device': 'sdd1'}] + for d in devs: + rb.add_dev(d) + rb.rebalance() + + real_pickle = pickle.load + try: + #test a legit builder + fake_pickle = Mock(return_value=rb) + fake_open = Mock(return_value=None) + pickle.load = fake_pickle + builder = RingBuilder.load('fake.builder', open=fake_open) + self.assertEquals(fake_pickle.call_count, 1) + fake_open.assert_has_calls([mock_call('fake.builder', 'rb')]) + self.assertEquals(builder, rb) + fake_pickle.reset_mock() + fake_open.reset_mock() + + #test old style builder + fake_pickle.return_value = rb.to_dict() + pickle.load = fake_pickle + builder = RingBuilder.load('fake.builder', open=fake_open) + fake_open.assert_has_calls([mock_call('fake.builder', 'rb')]) + self.assertEquals(builder.devs, rb.devs) + fake_pickle.reset_mock() + fake_open.reset_mock() + + #test old devs but no meta + no_meta_builder = rb + for dev in no_meta_builder.devs: + del(dev['meta']) + print no_meta_builder.devs + fake_pickle.return_value = no_meta_builder + pickle.load = fake_pickle + builder = RingBuilder.load('fake.builder', open=fake_open) + fake_open.assert_has_calls([mock_call('fake.builder', 'rb')]) + self.assertEquals(builder.devs, rb.devs) + fake_pickle.reset_mock() + finally: + pickle.load = real_pickle + + def test_search_devs(self): + rb = ring.RingBuilder(8, 3, 1) + devs = [{'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0', + 'port': 10000, 'device': 'sda1', 'meta': 'meta0'}, + {'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10001, 'device': 'sdb1', 'meta': 'meta1'}, + {'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2', + 'port': 10002, 'device': 'sdc1', 'meta': 'meta2'}, + {'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3', + 'port': 10003, 'device': 'sdd1', 'meta': 'meta3'}] + for d in devs: + rb.add_dev(d) + rb.rebalance() + res = rb.search_devs('d1') + self.assertEquals(res, [devs[1]]) + res = rb.search_devs('z1') + self.assertEquals(res, [devs[1]]) + res = rb.search_devs('-127.0.0.1') + self.assertEquals(res, [devs[1]]) + res = rb.search_devs('-[127.0.0.1]:10001') + self.assertEquals(res, [devs[1]]) + res = rb.search_devs(':10001') + self.assertEquals(res, [devs[1]]) + res = rb.search_devs('/sdb1') + self.assertEquals(res, [devs[1]]) + res = rb.search_devs('_meta1') + self.assertRaises(ValueError, rb.search_devs, 'OMGPONIES') + def test_validate(self): rb = ring.RingBuilder(8, 3, 1) rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', diff --git a/tools/test-requires b/tools/test-requires index ceee2d41cd..647e28eb39 100644 --- a/tools/test-requires +++ b/tools/test-requires @@ -5,3 +5,4 @@ openstack.nose_plugin nosehtmloutput pep8==0.6.1 sphinx>=1.1.2 +mock>=0.7.0