openstack/swift
Code Issues Proposed changes

Experimental swift-ring-composer CLI to build composite rings

Browse Source 
Provides a simple, experimental, CLI tool to generate a
composite ring from a list of component builder files.

For example:

  swift-ring-composer <composite-file> compose \
      <builder-file> <builder-file> --output <ring-file>

Commands available:

- compose: compose a list of builder file to a composite ring
- show: show the metadata for a composite ring

Co-Authored-By: Kota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>
Co-Authored-By: Matthew Oliver <matt@oliver.net.au>
Change-Id: I25a79e71c13af352e19e4358f60545265b51584f
This commit is contained in:
Alistair Coles 2017-06-19 14:46:14 +01:00
parent e5ce83b967
commit 6b626f2f98
6 changed files with 462 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
bin/swift-ring-composer Executable file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env python
# Copyright (c) 2017 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift.cli.ringcomposer import main
if __name__ == "__main__":
sys.exit(main())

40
doc/manpages/swift-ring-composer.1 Normal file
View File

@ -0,0 +1,40 @@
.TH swift-ring-composer "1" "June 2018" "Linux" "OpenStack Swift"
.SH NAME
.B swift-ring-composer
\- manual page for swift-ring-composer
.SH SYNOPSIS
.LP
.B swift-ring-composer
[\-h] <composite_builder_file> {show,compose} ...
.SH DESCRIPTION
This is a tool for building a composite ring file from other existing ring
builder files. The component ring builders must all have the same partition
power. Each device must only be used in a single component builder. Each
region must only be used in a single component builder.
.PP
.B NOTE:
This tool is for experimental use and may be removed in future versions of Swift.
.PP
.SS "positional arguments:"
.TP
<composite_builder_file>
Name of composite builder file
.SS "optional arguments:"
.TP
\fB\-h\fR, \fB\-\-help\fR
show this help message and exit
.SH "COMMANDS"
.TP
.SS "\fBshow\fR [-h]"
show composite ring builder metadata
.TP
.SS "\fBcompose\fR [-h] [<builder_file> <builder_file> [<builder_file> ...] --output <ring_file> [--force]"
compose composite ring
.PP
.SH DOCUMENTATION
.LP
More in depth documentation about the swift ring and also OpenStack Swift as a
whole can be found at
.BI https://swift.openstack.org

5
doc/source/overview_ring.rst
View File

@ -361,6 +361,11 @@ Composite Rings
---------------
.. automodule:: swift.common.ring.composite_builder
**********************************
swift-ring-composer (Experimental)
**********************************
.. automodule:: swift.cli.ringcomposer
---------------------
Ring Builder Analyzer
---------------------

1
setup.cfg
View File

@ -61,6 +61,7 @@ scripts =
bin/swift-recon-cron
bin/swift-ring-builder
bin/swift-ring-builder-analyzer
bin/swift-ring-composer
[extras]
kms_keymaster =

183
swift/cli/ringcomposer.py Normal file
View File

@ -0,0 +1,183 @@
# Copyright (c) 2017 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
``swift-ring-composer`` is an experimental tool for building a composite ring
file from other existing component ring builder files. Its CLI, name or
implementation may change or be removed altogether in future versions of Swift.
Currently its interface is similar to that of the ``swift-ring-builder``. The
command structure takes the form of::
swift-ring-composer <composite builder file> <sub-command> <options>
where ``<composite builder file>`` is a special builder which stores a json
blob of composite ring metadata. This metadata describes the component
``RingBuilder``'s used in the composite ring, their order and version.
There are currently 2 sub-commands: ``show`` and ``compose``. The ``show``
sub-command takes no additional arguments and displays the current contents of
of the composite builder file::
swift-ring-composer <composite builder file> show
The ``compose`` sub-command is the one that actually stitches the component
ring builders together to create both the composite ring file and composite
builder file. The command takes the form::
swift-ring-composer <composite builder file> compose <builder1> \\
<builder2> [<builder3> .. <builderN>] --output <composite ring file> \\
[--force]
There may look like there is a lot going on there but it's actually quite
simple. The ``compose`` command takes in the list of builders to stitch
together and the filename for the composite ring file via the ``--output``
option. The ``--force`` option overrides checks on the ring composition.
To change ring devices, first add or remove devices from the component ring
builders and then use the ``compose`` sub-command to create a new composite
ring file.
.. note::
``swift-ring-builder`` cannot be used to inspect the generated composite
ring file because there is no conventional builder file corresponding to
the composite ring file name. You can either programmatically look inside
the composite ring file using the swift ring classes or create a temporary
builder file from the composite ring file using::
swift-ring-builder <composite ring file> write_builder
Do not use this builder file to manage ring devices.
For further details use::
swift-ring-composer -h
"""
from __future__ import print_function
import argparse
import json
import os
import sys
from swift.common.ring.composite_builder import CompositeRingBuilder
EXIT_SUCCESS = 0
EXIT_ERROR = 2
WARNING = """
NOTE: This tool is for experimental use and may be
removed in future versions of Swift.
"""
DESCRIPTION = """
This is a tool for building a composite ring file from other existing ring
builder files. The component ring builders must all have the same partition
power. Each device must only be used in a single component builder. Each region
must only be used in a single component builder.
"""
def _print_to_stderr(msg):
print(msg, file=sys.stderr)
def _print_err(msg, err):
_print_to_stderr('%s\nOriginal exception message:\n%s' % (msg, err))
def show(composite_builder, args):
print(json.dumps(composite_builder.to_dict(), indent=4, sort_keys=True))
return EXIT_SUCCESS
def compose(composite_builder, args):
composite_builder = composite_builder or CompositeRingBuilder()
try:
ring_data = composite_builder.compose(
args.builder_files, force=args.force, require_modified=True)
except Exception as err:
_print_err(
'An error occurred while composing the ring.', err)
return EXIT_ERROR
try:
ring_data.save(args.output)
except Exception as err:
_print_err(
'An error occurred while writing the composite ring file.', err)
return EXIT_ERROR
try:
composite_builder.save(args.composite_builder_file)
except Exception as err:
_print_err(
'An error occurred while writing the composite builder file.', err)
return EXIT_ERROR
return EXIT_SUCCESS
def main(arguments=None):
if arguments is not None:
argv = arguments
else:
argv = sys.argv
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument(
'composite_builder_file',
metavar='composite_builder_file', type=str,
help='Name of composite builder file')
subparsers = parser.add_subparsers(
help='subcommand help', title='subcommands')
# show
show_parser = subparsers.add_parser(
'show', help='show composite ring builder metadata')
show_parser.set_defaults(func=show)
# compose
compose_parser = subparsers.add_parser(
'compose', help='compose composite ring',
usage='%(prog)s [-h] '
'[builder_file builder_file [builder_file ...] '
'--output ring_file [--force]')
bf_help = ('Paths to component ring builder files to include in composite '
'ring')
compose_parser.add_argument('builder_files', metavar='builder_file',
nargs='*', type=str, help=bf_help)
compose_parser.add_argument('--output', metavar='output_file', type=str,
required=True, help='Name of output ring file')
compose_parser.add_argument(
'--force', action='store_true',
help='Force new composite ring file to be written')
compose_parser.set_defaults(func=compose)
_print_to_stderr(WARNING)
args = parser.parse_args(argv[1:])
composite_builder = None
if args.func != compose or os.path.exists(args.composite_builder_file):
try:
composite_builder = CompositeRingBuilder.load(
args.composite_builder_file)
except Exception as err:
_print_err(
'An error occurred while loading the composite builder file.',
err)
exit(EXIT_ERROR)
exit(args.func(composite_builder, args))
if __name__ == '__main__':
main()

211
test/unit/cli/test_ringcomposer.py Normal file
View File

@ -0,0 +1,211 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import shutil
import tempfile
import unittest
import six
from mock import mock
from swift.cli import ringcomposer
from swift.common.ring import RingBuilder
class TestCommands(unittest.TestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
self.composite_builder_file = os.path.join(self.tmpdir,
'composite.builder')
self.composite_ring_file = os.path.join(self.tmpdir,
'composite.ring')
def tearDown(self):
shutil.rmtree(self.tmpdir)
def _write_stub_builder(self, region):
replicas = 3
builder = RingBuilder(8, replicas, 1)
for i in range(replicas):
dev = {'weight': 100,
'region': '%d' % region,
'zone': '1',
'ip': '10.0.0.%d' % region,
'port': '3600',
'device': 'sdb%d' % i}
builder.add_dev(dev)
builder.rebalance()
builder_file = os.path.join(self.tmpdir, '%d.builder' % region)
builder.save(builder_file)
return builder, builder_file
def _run_composer(self, args):
mock_stdout = six.StringIO()
mock_stderr = six.StringIO()
with mock.patch("sys.stdout", mock_stdout):
with mock.patch("sys.stderr", mock_stderr):
with self.assertRaises(SystemExit) as cm:
ringcomposer.main(args)
return (cm.exception.code,
mock_stdout.getvalue(),
mock_stderr.getvalue())
def test_unknown_command(self):
args = ('', self.composite_builder_file, 'unknown')
exit_code, stdout, stderr = self._run_composer(args)
self.assertEqual(2, exit_code)
self.assertIn('invalid choice', stderr)
args = ('', 'non-existent-file', 'unknown')
exit_code, stdout, stderr = self._run_composer(args)
self.assertEqual(2, exit_code)
self.assertIn('invalid choice', stderr)
def test_bad_composite_builder_file(self):
cmds = (('', self.composite_builder_file, 'show'),
('', self.composite_builder_file, 'compose',
'b1_file', 'b2_file', '--output', self.composite_ring_file))
for cmd in cmds:
try:
with open(self.composite_builder_file, 'wb') as fd:
fd.write('not json')
exit_code, stdout, stderr = self._run_composer(cmd)
self.assertEqual(2, exit_code)
self.assertIn('An error occurred while loading the composite '
'builder file', stderr)
self.assertIn(
'File does not contain valid composite ring data', stderr)
except AssertionError as err:
self.fail('Failed testing command %r due to: %s' % (cmd, err))
def test_compose(self):
b1, b1_file = self._write_stub_builder(1)
b2, b2_file = self._write_stub_builder(2)
args = ('', self.composite_builder_file, 'compose', b1_file, b2_file,
'--output', self.composite_ring_file)
exit_code, stdout, stderr = self._run_composer(args)
self.assertEqual(0, exit_code)
self.assertTrue(os.path.exists(self.composite_builder_file))
self.assertTrue(os.path.exists(self.composite_ring_file))
def test_compose_existing(self):
b1, b1_file = self._write_stub_builder(1)
b2, b2_file = self._write_stub_builder(2)
args = ('', self.composite_builder_file, 'compose', b1_file, b2_file,
'--output', self.composite_ring_file)
exit_code, stdout, stderr = self._run_composer(args)
self.assertEqual(0, exit_code)
os.unlink(self.composite_ring_file)
# no changes - expect failure
args = ('', self.composite_builder_file, 'compose',
'--output', self.composite_ring_file)
exit_code, stdout, stderr = self._run_composer(args)
self.assertEqual(2, exit_code)
self.assertFalse(os.path.exists(self.composite_ring_file))
# --force should force output
args = ('', self.composite_builder_file, 'compose',
'--output', self.composite_ring_file, '--force')
exit_code, stdout, stderr = self._run_composer(args)
self.assertEqual(0, exit_code)
self.assertTrue(os.path.exists(self.composite_ring_file))
def test_compose_insufficient_component_builder_files(self):
b1, b1_file = self._write_stub_builder(1)
args = ('', self.composite_builder_file, 'compose', b1_file,
'--output', self.composite_ring_file)
exit_code, stdout, stderr = self._run_composer(args)
self.assertEqual(2, exit_code)
self.assertIn('An error occurred while composing the ring', stderr)
self.assertIn('Two or more component builders are required', stderr)
self.assertFalse(os.path.exists(self.composite_builder_file))
self.assertFalse(os.path.exists(self.composite_ring_file))
def test_compose_nonexistent_component_builder_file(self):
b1, b1_file = self._write_stub_builder(1)
bad_file = os.path.join(self.tmpdir, 'non-existent-file')
args = ('', self.composite_builder_file, 'compose', b1_file, bad_file,
'--output', self.composite_ring_file)
exit_code, stdout, stderr = self._run_composer(args)
self.assertIn('An error occurred while composing the ring', stderr)
self.assertIn('Ring Builder file does not exist', stderr)
self.assertEqual(2, exit_code)
self.assertFalse(os.path.exists(self.composite_builder_file))
self.assertFalse(os.path.exists(self.composite_ring_file))
def test_compose_fails_to_write_composite_ring_file(self):
b1, b1_file = self._write_stub_builder(1)
b2, b2_file = self._write_stub_builder(2)
args = ('', self.composite_builder_file, 'compose', b1_file, b2_file,
'--output', self.composite_ring_file)
with mock.patch('swift.common.ring.RingData.save',
side_effect=IOError('io error')):
exit_code, stdout, stderr = self._run_composer(args)
self.assertEqual(2, exit_code)
self.assertIn(
'An error occurred while writing the composite ring file', stderr)
self.assertIn('io error', stderr)
self.assertFalse(os.path.exists(self.composite_builder_file))
self.assertFalse(os.path.exists(self.composite_ring_file))
def test_compose_fails_to_write_composite_builder_file(self):
b1, b1_file = self._write_stub_builder(1)
b2, b2_file = self._write_stub_builder(2)
args = ('', self.composite_builder_file, 'compose', b1_file, b2_file,
'--output', self.composite_ring_file)
func = 'swift.common.ring.composite_builder.CompositeRingBuilder.save'
with mock.patch(func, side_effect=IOError('io error')):
exit_code, stdout, stderr = self._run_composer(args)
self.assertEqual(2, exit_code)
self.assertIn(
'An error occurred while writing the composite builder file',
stderr)
self.assertIn('io error', stderr)
self.assertFalse(os.path.exists(self.composite_builder_file))
self.assertTrue(os.path.exists(self.composite_ring_file))
def test_show(self):
b1, b1_file = self._write_stub_builder(1)
b2, b2_file = self._write_stub_builder(2)
args = ('', self.composite_builder_file, 'compose', b1_file, b2_file,
'--output', self.composite_ring_file)
exit_code, stdout, stderr = self._run_composer(args)
self.assertEqual(0, exit_code)
args = ('', self.composite_builder_file, 'show')
exit_code, stdout, stderr = self._run_composer(args)
self.assertEqual(0, exit_code)
expected = {'component_builder_files': {b1.id: b1_file,
b2.id: b2_file},
'components': [
{'id': b1.id,
'replicas': b1.replicas,
# added replicas devices plus rebalance
'version': b1.replicas + 1},
{'id': b2.id,
'replicas': b2.replicas,
# added replicas devices plus rebalance
'version': b2.replicas + 1}],
'version': 1
}
self.assertEqual(expected, json.loads(stdout))
def test_show_nonexistent_composite_builder_file(self):
args = ('', 'non-existent-file', 'show')
exit_code, stdout, stderr = self._run_composer(args)
self.assertEqual(2, exit_code)
self.assertIn(
'An error occurred while loading the composite builder file',
stderr)
self.assertIn("No such file or directory: 'non-existent-file'", stderr)