Experimental swift-ring-composer CLI to build composite rings

Provides a simple, experimental, CLI tool to generate a
composite ring from a list of component builder files.

For example:

  swift-ring-composer <composite-file> compose \
      <builder-file> <builder-file> --output <ring-file>

Commands available:

- compose: compose a list of builder file to a composite ring
- show: show the metadata for a composite ring

Co-Authored-By: Kota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>
Co-Authored-By: Matthew Oliver <matt@oliver.net.au>
Change-Id: I25a79e71c13af352e19e4358f60545265b51584f

bin/swift-ring-composer

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# Copyright (c) 2017 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+
+from swift.cli.ringcomposer import main
+
+
+if __name__ == "__main__":
+    sys.exit(main())

doc/manpages/swift-ring-composer.1

@@ -0,0 +1,40 @@
+.TH swift-ring-composer "1" "June 2018" "Linux" "OpenStack Swift"
+.SH NAME
+.B swift-ring-composer
+\- manual page for swift-ring-composer
+
+.SH SYNOPSIS
+.LP
+.B swift-ring-composer
+[\-h] <composite_builder_file> {show,compose} ...
+
+.SH DESCRIPTION
+This is a tool for building a composite ring file from other existing ring
+builder files. The component ring builders must all have the same partition
+power. Each device must only be used in a single component builder. Each
+region must only be used in a single component builder.
+.PP
+.B NOTE:
+This tool is for experimental use and may be removed in future versions of Swift.
+.PP
+.SS "positional arguments:"
+.TP
+<composite_builder_file>
+Name of composite builder file
+.SS "optional arguments:"
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.SH "COMMANDS"
+.TP
+.SS "\fBshow\fR [-h]"
+show composite ring builder metadata
+.TP
+.SS "\fBcompose\fR [-h] [<builder_file> <builder_file> [<builder_file> ...] --output <ring_file> [--force]"
+compose composite ring
+.PP
+.SH DOCUMENTATION
+.LP
+More in depth documentation about the swift ring and also OpenStack Swift as a
+whole can be found at
+.BI https://swift.openstack.org

doc/source/overview_ring.rst

@@ -361,6 +361,11 @@ Composite Rings
 ---------------
 .. automodule:: swift.common.ring.composite_builder
 
+**********************************
+swift-ring-composer (Experimental)
+**********************************
+.. automodule:: swift.cli.ringcomposer
+
 ---------------------
 Ring Builder Analyzer
 ---------------------

setup.cfg

@@ -61,6 +61,7 @@ scripts =
     bin/swift-recon-cron
     bin/swift-ring-builder
     bin/swift-ring-builder-analyzer
+    bin/swift-ring-composer
 
 [extras]
 kms_keymaster =

swift/cli/ringcomposer.py

@@ -0,0 +1,183 @@
+# Copyright (c) 2017 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+``swift-ring-composer`` is an experimental tool for building a composite ring
+file from other existing component ring builder files. Its CLI, name or
+implementation may change or be removed altogether in future versions of Swift.
+
+Currently its interface is similar to that of the ``swift-ring-builder``. The
+command structure takes the form of::
+
+    swift-ring-composer <composite builder file> <sub-command> <options>
+
+where ``<composite builder file>`` is a special builder which stores a json
+blob of composite ring metadata. This metadata describes the component
+``RingBuilder``'s used in the composite ring, their order and version.
+
+There are currently 2 sub-commands: ``show`` and ``compose``. The ``show``
+sub-command takes no additional arguments and displays the current contents of
+of the composite builder file::
+
+    swift-ring-composer <composite builder file> show
+
+The ``compose`` sub-command is the one that actually stitches the component
+ring builders together to create both the composite ring file and composite
+builder file. The command takes the form::
+
+    swift-ring-composer <composite builder file> compose <builder1> \\
+    <builder2> [<builder3> .. <builderN>] --output <composite ring file> \\
+    [--force]
+
+There may look like there is a lot going on there but it's actually quite
+simple. The ``compose`` command takes in the list of builders to stitch
+together and the filename for the composite ring file via the ``--output``
+option. The ``--force`` option overrides checks on the ring composition.
+
+To change ring devices, first add or remove devices from the component ring
+builders and then use the ``compose`` sub-command to create a new composite
+ring file.
+
+.. note::
+
+    ``swift-ring-builder`` cannot be used to inspect the generated composite
+    ring file because there is no conventional builder file corresponding to
+    the composite ring file name. You can either programmatically look inside
+    the composite ring file using the swift ring classes or create a temporary
+    builder file from the composite ring file using::
+
+        swift-ring-builder <composite ring file> write_builder
+
+    Do not use this builder file to manage ring devices.
+
+For further details use::
+
+  swift-ring-composer -h
+"""
+from __future__ import print_function
+import argparse
+import json
+import os
+import sys
+
+from swift.common.ring.composite_builder import CompositeRingBuilder
+
+EXIT_SUCCESS = 0
+EXIT_ERROR = 2
+
+WARNING = """
+NOTE: This tool is for experimental use and may be
+      removed in future versions of Swift.
+"""
+
+DESCRIPTION = """
+This is a tool for building a composite ring file from other existing ring
+builder files. The component ring builders must all have the same partition
+power. Each device must only be used in a single component builder. Each region
+must only be used in a single component builder.
+"""
+
+
+def _print_to_stderr(msg):
+    print(msg, file=sys.stderr)
+
+
+def _print_err(msg, err):
+    _print_to_stderr('%s\nOriginal exception message:\n%s' % (msg, err))
+
+
+def show(composite_builder, args):
+    print(json.dumps(composite_builder.to_dict(), indent=4, sort_keys=True))
+    return EXIT_SUCCESS
+
+
+def compose(composite_builder, args):
+    composite_builder = composite_builder or CompositeRingBuilder()
+    try:
+        ring_data = composite_builder.compose(
+            args.builder_files, force=args.force, require_modified=True)
+    except Exception as err:
+        _print_err(
+            'An error occurred while composing the ring.', err)
+        return EXIT_ERROR
+    try:
+        ring_data.save(args.output)
+    except Exception as err:
+        _print_err(
+            'An error occurred while writing the composite ring file.', err)
+        return EXIT_ERROR
+    try:
+        composite_builder.save(args.composite_builder_file)
+    except Exception as err:
+        _print_err(
+            'An error occurred while writing the composite builder file.', err)
+        return EXIT_ERROR
+    return EXIT_SUCCESS
+
+
+def main(arguments=None):
+    if arguments is not None:
+        argv = arguments
+    else:
+        argv = sys.argv
+
+    parser = argparse.ArgumentParser(description=DESCRIPTION)
+    parser.add_argument(
+        'composite_builder_file',
+        metavar='composite_builder_file', type=str,
+        help='Name of composite builder file')
+
+    subparsers = parser.add_subparsers(
+        help='subcommand help', title='subcommands')
+
+    # show
+    show_parser = subparsers.add_parser(
+        'show', help='show composite ring builder metadata')
+    show_parser.set_defaults(func=show)
+
+    # compose
+    compose_parser = subparsers.add_parser(
+        'compose', help='compose composite ring',
+        usage='%(prog)s [-h] '
+              '[builder_file builder_file [builder_file ...] '
+              '--output ring_file [--force]')
+    bf_help = ('Paths to component ring builder files to include in composite '
+               'ring')
+    compose_parser.add_argument('builder_files', metavar='builder_file',
+                                nargs='*', type=str, help=bf_help)
+    compose_parser.add_argument('--output', metavar='output_file', type=str,
+                                required=True, help='Name of output ring file')
+    compose_parser.add_argument(
+        '--force', action='store_true',
+        help='Force new composite ring file to be written')
+    compose_parser.set_defaults(func=compose)
+
+    _print_to_stderr(WARNING)
+    args = parser.parse_args(argv[1:])
+    composite_builder = None
+    if args.func != compose or os.path.exists(args.composite_builder_file):
+        try:
+            composite_builder = CompositeRingBuilder.load(
+                args.composite_builder_file)
+        except Exception as err:
+            _print_err(
+                'An error occurred while loading the composite builder file.',
+                err)
+            exit(EXIT_ERROR)
+
+    exit(args.func(composite_builder, args))
+
+
+if __name__ == '__main__':
+    main()

test/unit/cli/test_ringcomposer.py

@@ -0,0 +1,211 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import six
+from mock import mock
+
+from swift.cli import ringcomposer
+from swift.common.ring import RingBuilder
+
+
+class TestCommands(unittest.TestCase):
+
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+        self.composite_builder_file = os.path.join(self.tmpdir,
+                                                   'composite.builder')
+        self.composite_ring_file = os.path.join(self.tmpdir,
+                                                'composite.ring')
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+
+    def _write_stub_builder(self, region):
+        replicas = 3
+        builder = RingBuilder(8, replicas, 1)
+        for i in range(replicas):
+            dev = {'weight': 100,
+                   'region': '%d' % region,
+                   'zone': '1',
+                   'ip': '10.0.0.%d' % region,
+                   'port': '3600',
+                   'device': 'sdb%d' % i}
+            builder.add_dev(dev)
+        builder.rebalance()
+        builder_file = os.path.join(self.tmpdir, '%d.builder' % region)
+        builder.save(builder_file)
+        return builder, builder_file
+
+    def _run_composer(self, args):
+        mock_stdout = six.StringIO()
+        mock_stderr = six.StringIO()
+        with mock.patch("sys.stdout", mock_stdout):
+            with mock.patch("sys.stderr", mock_stderr):
+                with self.assertRaises(SystemExit) as cm:
+                    ringcomposer.main(args)
+        return (cm.exception.code,
+                mock_stdout.getvalue(),
+                mock_stderr.getvalue())
+
+    def test_unknown_command(self):
+        args = ('', self.composite_builder_file, 'unknown')
+        exit_code, stdout, stderr = self._run_composer(args)
+        self.assertEqual(2, exit_code)
+        self.assertIn('invalid choice', stderr)
+
+        args = ('', 'non-existent-file', 'unknown')
+        exit_code, stdout, stderr = self._run_composer(args)
+        self.assertEqual(2, exit_code)
+        self.assertIn('invalid choice', stderr)
+
+    def test_bad_composite_builder_file(self):
+        cmds = (('', self.composite_builder_file, 'show'),
+                ('', self.composite_builder_file, 'compose',
+                 'b1_file', 'b2_file', '--output', self.composite_ring_file))
+        for cmd in cmds:
+            try:
+                with open(self.composite_builder_file, 'wb') as fd:
+                    fd.write('not json')
+                exit_code, stdout, stderr = self._run_composer(cmd)
+                self.assertEqual(2, exit_code)
+                self.assertIn('An error occurred while loading the composite '
+                              'builder file', stderr)
+                self.assertIn(
+                    'File does not contain valid composite ring data', stderr)
+            except AssertionError as err:
+                self.fail('Failed testing command %r due to: %s' % (cmd, err))
+
+    def test_compose(self):
+        b1, b1_file = self._write_stub_builder(1)
+        b2, b2_file = self._write_stub_builder(2)
+        args = ('', self.composite_builder_file, 'compose', b1_file, b2_file,
+                '--output', self.composite_ring_file)
+        exit_code, stdout, stderr = self._run_composer(args)
+        self.assertEqual(0, exit_code)
+        self.assertTrue(os.path.exists(self.composite_builder_file))
+        self.assertTrue(os.path.exists(self.composite_ring_file))
+
+    def test_compose_existing(self):
+        b1, b1_file = self._write_stub_builder(1)
+        b2, b2_file = self._write_stub_builder(2)
+        args = ('', self.composite_builder_file, 'compose', b1_file, b2_file,
+                '--output', self.composite_ring_file)
+        exit_code, stdout, stderr = self._run_composer(args)
+        self.assertEqual(0, exit_code)
+        os.unlink(self.composite_ring_file)
+        # no changes - expect failure
+        args = ('', self.composite_builder_file, 'compose',
+                '--output', self.composite_ring_file)
+        exit_code, stdout, stderr = self._run_composer(args)
+        self.assertEqual(2, exit_code)
+        self.assertFalse(os.path.exists(self.composite_ring_file))
+        # --force should force output
+        args = ('', self.composite_builder_file, 'compose',
+                '--output', self.composite_ring_file, '--force')
+        exit_code, stdout, stderr = self._run_composer(args)
+        self.assertEqual(0, exit_code)
+        self.assertTrue(os.path.exists(self.composite_ring_file))
+
+    def test_compose_insufficient_component_builder_files(self):
+        b1, b1_file = self._write_stub_builder(1)
+        args = ('', self.composite_builder_file, 'compose', b1_file,
+                '--output', self.composite_ring_file)
+        exit_code, stdout, stderr = self._run_composer(args)
+        self.assertEqual(2, exit_code)
+        self.assertIn('An error occurred while composing the ring', stderr)
+        self.assertIn('Two or more component builders are required', stderr)
+        self.assertFalse(os.path.exists(self.composite_builder_file))
+        self.assertFalse(os.path.exists(self.composite_ring_file))
+
+    def test_compose_nonexistent_component_builder_file(self):
+        b1, b1_file = self._write_stub_builder(1)
+        bad_file = os.path.join(self.tmpdir, 'non-existent-file')
+        args = ('', self.composite_builder_file, 'compose', b1_file, bad_file,
+                '--output', self.composite_ring_file)
+        exit_code, stdout, stderr = self._run_composer(args)
+        self.assertIn('An error occurred while composing the ring', stderr)
+        self.assertIn('Ring Builder file does not exist', stderr)
+        self.assertEqual(2, exit_code)
+        self.assertFalse(os.path.exists(self.composite_builder_file))
+        self.assertFalse(os.path.exists(self.composite_ring_file))
+
+    def test_compose_fails_to_write_composite_ring_file(self):
+        b1, b1_file = self._write_stub_builder(1)
+        b2, b2_file = self._write_stub_builder(2)
+        args = ('', self.composite_builder_file, 'compose', b1_file, b2_file,
+                '--output', self.composite_ring_file)
+        with mock.patch('swift.common.ring.RingData.save',
+                        side_effect=IOError('io error')):
+            exit_code, stdout, stderr = self._run_composer(args)
+        self.assertEqual(2, exit_code)
+        self.assertIn(
+            'An error occurred while writing the composite ring file', stderr)
+        self.assertIn('io error', stderr)
+        self.assertFalse(os.path.exists(self.composite_builder_file))
+        self.assertFalse(os.path.exists(self.composite_ring_file))
+
+    def test_compose_fails_to_write_composite_builder_file(self):
+        b1, b1_file = self._write_stub_builder(1)
+        b2, b2_file = self._write_stub_builder(2)
+        args = ('', self.composite_builder_file, 'compose', b1_file, b2_file,
+                '--output', self.composite_ring_file)
+        func = 'swift.common.ring.composite_builder.CompositeRingBuilder.save'
+        with mock.patch(func, side_effect=IOError('io error')):
+            exit_code, stdout, stderr = self._run_composer(args)
+        self.assertEqual(2, exit_code)
+        self.assertIn(
+            'An error occurred while writing the composite builder file',
+            stderr)
+        self.assertIn('io error', stderr)
+        self.assertFalse(os.path.exists(self.composite_builder_file))
+        self.assertTrue(os.path.exists(self.composite_ring_file))
+
+    def test_show(self):
+        b1, b1_file = self._write_stub_builder(1)
+        b2, b2_file = self._write_stub_builder(2)
+        args = ('', self.composite_builder_file, 'compose', b1_file, b2_file,
+                '--output', self.composite_ring_file)
+        exit_code, stdout, stderr = self._run_composer(args)
+        self.assertEqual(0, exit_code)
+        args = ('', self.composite_builder_file, 'show')
+        exit_code, stdout, stderr = self._run_composer(args)
+        self.assertEqual(0, exit_code)
+        expected = {'component_builder_files': {b1.id: b1_file,
+                                                b2.id: b2_file},
+                    'components': [
+                        {'id': b1.id,
+                         'replicas': b1.replicas,
+                         # added replicas devices plus rebalance
+                         'version': b1.replicas + 1},
+                        {'id': b2.id,
+                         'replicas': b2.replicas,
+                         # added replicas devices plus rebalance
+                         'version': b2.replicas + 1}],
+                    'version': 1
+                    }
+        self.assertEqual(expected, json.loads(stdout))
+
+    def test_show_nonexistent_composite_builder_file(self):
+        args = ('', 'non-existent-file', 'show')
+        exit_code, stdout, stderr = self._run_composer(args)
+        self.assertEqual(2, exit_code)
+        self.assertIn(
+            'An error occurred while loading the composite builder file',
+            stderr)
+        self.assertIn("No such file or directory: 'non-existent-file'", stderr)