Merge "Add support for creating vfat and iso images"

etc/ironic/ironic.conf.sample

@@ -251,6 +251,13 @@
 # Force backing images to raw format. (boolean value)
 #force_raw_images=true
 
+# Path to isolinux binary file. (string value)
+#isolinux_bin=/usr/lib/syslinux/isolinux.bin
+
+# Template file for isolinux configuration file. (string
+# value)
+#isolinux_config_template=$pybasedir/common/isolinux_config.template
+
 
 #
 # Options defined in ironic.common.paths

etc/ironic/rootwrap.d/ironic-utils.filters

@@ -4,13 +4,15 @@
 [Filters]
 # ironic/drivers/modules/deploy_utils.py
 iscsiadm: CommandFilter, iscsiadm, root
-dd: CommandFilter, dd, root
 blkid: CommandFilter, blkid, root
 blockdev: CommandFilter, blockdev, root
 
 # ironic/common/utils.py
 mkswap: CommandFilter, mkswap, root
 mkfs: CommandFilter, mkfs, root
+mount: CommandFilter, mount, root
+umount: CommandFilter, umount, root
+dd: CommandFilter, dd, root
 
 # ironic/common/disk_partitioner.py
 fuser: CommandFilter, fuser, root

ironic/common/exception.py

@@ -437,3 +437,7 @@ class InsufficientDiskSpace(IronicException):
     message = _("Disk volume where '%(path)s' is located doesn't have "
                 "enough disk space. Required %(required)d MiB, "
                 "only %(actual)d MiB available space present.")
+
+
+class ImageCreationFailed(IronicException):
+    message = _('Creating %(image_type)s image failed: %(error)s')

ironic/common/images.py

@@ -20,28 +20,202 @@ Handling of VM disk images.
 """
 
 import os
+import shutil
 
+import jinja2
 from oslo.config import cfg
 
 from ironic.common import exception
+from ironic.common import i18n
 from ironic.common import image_service as service
+from ironic.common import paths
 from ironic.common import utils
 from ironic.openstack.common import fileutils
 from ironic.openstack.common import imageutils
 from ironic.openstack.common import log as logging
+from ironic.openstack.common import processutils
 
 LOG = logging.getLogger(__name__)
 
+_LE = i18n._LE
+
 image_opts = [
     cfg.BoolOpt('force_raw_images',
                 default=True,
                 help='Force backing images to raw format.'),
+    cfg.StrOpt('isolinux_bin',
+                default='/usr/lib/syslinux/isolinux.bin',
+                help='Path to isolinux binary file.'),
+    cfg.StrOpt('isolinux_config_template',
+                default=paths.basedir_def('common/isolinux_config.template'),
+                help='Template file for isolinux configuration file.'),
 ]
 
 CONF = cfg.CONF
 CONF.register_opts(image_opts)
 
 
+def _create_root_fs(root_directory, files_info):
+    """Creates a filesystem root in given directory.
+
+    Given a mapping of absolute path of files to their relative paths
+    within the filesystem, this method copies the files to their
+    destination.
+
+    :param root_directory: the filesystem root directory.
+    :param files_info: A dict containing absolute path of file to be copied
+        -> relative path within the vfat image. For example,
+        {
+         '/absolute/path/to/file' -> 'relative/path/within/root'
+         ...
+        }
+    :raises: OSError, if creation of any directory failed.
+    :raises: IOError, if copying any of the files failed.
+    """
+    for src_file, path in files_info.items():
+        target_file = os.path.join(root_directory, path)
+        dirname = os.path.dirname(target_file)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+
+        shutil.copyfile(src_file, target_file)
+
+
+def create_vfat_image(output_file, files_info=None, parameters=None,
+                      parameters_file='parameters.txt', fs_size_kib=100):
+    """Creates the fat fs image on the desired file.
+
+    This method copies the given files to a root directory (optional),
+    writes the parameters specified to the parameters file within the
+    root directory (optional), and then creates a vfat image of the root
+    directory.
+
+    :param output_file: The path to the file where the fat fs image needs
+        to be created.
+    :param files_info: A dict containing absolute path of file to be copied
+        -> relative path within the vfat image. For example,
+        {
+         '/absolute/path/to/file' -> 'relative/path/within/root'
+         ...
+        }
+    :param parameters: A dict containing key-value pairs of parameters.
+    :param parameters_file: The filename for the parameters file.
+    :param fs_size_kib: size of the vfat filesystem in KiB.
+    :raises: ImageCreationFailed, if image creation failed while doing any
+        of filesystem manipulation activities like creating dirs, mounting,
+        creating filesystem, copying files, etc.
+    """
+    try:
+        utils.dd('/dev/zero', output_file, 'count=1', "bs=%dKiB" % fs_size_kib)
+    except processutils.ProcessExecutionError as e:
+        raise exception.ImageCreationFailed(image_type='vfat', error=e)
+
+    with utils.tempdir() as tmpdir:
+
+        try:
+            utils.mkfs('vfat', output_file)
+            utils.mount(output_file, tmpdir, '-o', 'umask=0')
+        except processutils.ProcessExecutionError as e:
+            raise exception.ImageCreationFailed(image_type='vfat', error=e)
+
+        try:
+            if files_info:
+                _create_root_fs(tmpdir, files_info)
+
+            if parameters:
+                parameters_file = os.path.join(tmpdir, parameters_file)
+                params_list = ['%(key)s=%(val)s' % {'key': k, 'val': v}
+                           for k, v in parameters.items()]
+                file_contents = '\n'.join(params_list)
+                utils.write_to_file(parameters_file, file_contents)
+
+        except Exception as e:
+            LOG.exception(_LE("vfat image creation failed. Error: %s"), e)
+            raise exception.ImageCreationFailed(image_type='vfat', error=e)
+
+        finally:
+            try:
+                utils.umount(tmpdir)
+            except processutils.ProcessExecutionError as e:
+                raise exception.ImageCreationFailed(image_type='vfat', error=e)
+
+
+def _generate_isolinux_cfg(kernel_params):
+    """Generates a isolinux configuration file.
+
+    Given a given a list of strings containing kernel parameters, this method
+    returns the kernel cmdline string.
+    :param kernel_params: a list of strings(each element being a string like
+        'K=V' or 'K' or combination of them like 'K1=V1 K2 K3=V3') to be added
+        as the kernel cmdline.
+    :returns: a string containing the contents of the isolinux configuration
+        file.
+    """
+    if not kernel_params:
+        kernel_params = []
+    kernel_params_str = ' '.join(kernel_params)
+
+    template = CONF.isolinux_config_template
+    tmpl_path, tmpl_file = os.path.split(template)
+    env = jinja2.Environment(loader=jinja2.FileSystemLoader(tmpl_path))
+    template = env.get_template(tmpl_file)
+
+    options = {'kernel': '/vmlinuz', 'ramdisk': '/initrd',
+               'kernel_params': kernel_params_str}
+
+    cfg = template.render(options)
+    return cfg
+
+
+def create_isolinux_image(output_file, kernel, ramdisk, kernel_params=None):
+    """Creates an isolinux image on the specified file.
+
+    Copies the provided kernel, ramdisk to a directory, generates the isolinux
+    configuration file using the kernel parameters provided, and then generates
+    a bootable ISO image.
+
+    :param output_file: the path to the file where the iso image needs to be
+        created.
+    :param kernel: the kernel to use.
+    :param ramdisk: the ramdisk to use.
+    :param kernel_params: a list of strings(each element being a string like
+        'K=V' or 'K' or combination of them like 'K1=V1,K2,...') to be added
+        as the kernel cmdline.
+    :raises: ImageCreationFailed, if image creation failed while copying files
+        or while running command to generate iso.
+    """
+    ISOLINUX_BIN = 'isolinux/isolinux.bin'
+    ISOLINUX_CFG = 'isolinux/isolinux.cfg'
+
+    with utils.tempdir() as tmpdir:
+
+        files_info = {
+                      kernel: 'vmlinuz',
+                      ramdisk: 'initrd',
+                      CONF.isolinux_bin: ISOLINUX_BIN,
+                     }
+
+        try:
+            _create_root_fs(tmpdir, files_info)
+        except (OSError, IOError) as e:
+            LOG.exception(_LE("Creating the filesystem root failed."))
+            raise exception.ImageCreationFailed(image_type='iso', error=e)
+
+        cfg = _generate_isolinux_cfg(kernel_params)
+
+        isolinux_cfg = os.path.join(tmpdir, ISOLINUX_CFG)
+        utils.write_to_file(isolinux_cfg, cfg)
+
+        try:
+            utils.execute('mkisofs', '-r', '-V', "BOOT IMAGE",
+                          '-cache-inodes', '-J', '-l', '-no-emul-boot',
+                          '-boot-load-size', '4', '-boot-info-table',
+                          '-b', ISOLINUX_BIN, '-o', output_file, tmpdir)
+        except processutils.ProcessExecutionError as e:
+            LOG.exception(_LE("Creating ISO image failed."))
+            raise exception.ImageCreationFailed(image_type='iso', error=e)
+
+
 def qemu_img_info(path):
     """Return an object containing the parsed output from qemu-img info."""
     if not os.path.exists(path):

ironic/common/isolinux_config.template

@@ -0,0 +1,5 @@
+default boot
+
+label boot
+kernel {{ kernel }}
+append initrd={{ ramdisk }} text {{ kernel_params }} --

ironic/common/utils.py

@@ -504,3 +504,44 @@ def is_uuid_like(val):
         return str(uuid.UUID(val)) == val
     except (TypeError, ValueError, AttributeError):
         return False
+
+
+def mount(src, dest, *args):
+    """Mounts a device/image file on specified location.
+
+    :param src: the path to the source file for mounting
+    :param dest: the path where it needs to be mounted.
+    :param args: a tuple containing the arguments to be
+        passed to mount command.
+    :raises: processutils.ProcessExecutionError if it failed
+        to run the process.
+    """
+    args = ('mount', ) + args + (src, dest)
+    execute(*args, run_as_root=True, check_exit_code=[0])
+
+
+def umount(loc, *args):
+    """Umounts a mounted location.
+
+    :param loc: the path to be unmounted.
+    :param args: a tuple containing the argumnets to be
+        passed to the umount command.
+    :raises: processutils.ProcessExecutionError if it failed
+        to run the process.
+    """
+    args = ('umount', ) + args + (loc, )
+    execute(*args, run_as_root=True, check_exit_code=[0])
+
+
+def dd(src, dst, *args):
+    """Execute dd from src to dst.
+
+    :param src: the input file for dd command.
+    :param dst: the output file for dd command.
+    :param args: a tuple containing the arguments to be
+        passed to dd command.
+    :raises: processutils.ProcessExecutionError if it failed
+        to run the process.
+    """
+    execute('dd', 'if=%s' % src, 'of=%s' % dst, *args,
+            run_as_root=True, check_exit_code=[0])

ironic/drivers/modules/deploy_utils.py

@@ -139,13 +139,7 @@ def is_block_device(dev):
 
 def dd(src, dst):
     """Execute dd from src to dst."""
-    utils.execute('dd',
-                  'if=%s' % src,
-                  'of=%s' % dst,
-                  'bs=1M',
-                  'oflag=direct',
-                  run_as_root=True,
-                  check_exit_code=[0])
+    utils.dd(src, dst, 'bs=1M', 'oflag=direct')
 
 
 def mkswap(dev, label='swap1'):

ironic/drivers/modules/iscsi_deploy.py

@@ -354,7 +354,8 @@ def validate_glance_image_properties(ctx, deploy_info, properties):
     :param ctx: security context
     :param deploy_info: the deploy_info to be validated
     :param properties: the list of image meta-properties to be validated.
-    :raises: InvalidParameterValue if the glance image doesn't exist.
+    :raises: InvalidParameterValue if connection to glance failed or
+        authorization for accessing image failed or if image doesn't exist.
     :raises: MissingParameterValue if the glance image doesn't contain
         the mentioned properties.
     """

ironic/tests/test_images.py

@@ -18,13 +18,21 @@
 
 import contextlib
 import fixtures
+import mock
+import os
+import shutil
 
+from oslo.config import cfg
 from oslo.utils import excutils
 
 from ironic.common import exception
 from ironic.common import images
+from ironic.common import utils
+from ironic.openstack.common import processutils
 from ironic.tests import base
 
+CONF = cfg.CONF
+
 
 class IronicImagesTestCase(base.TestCase):
     def test_fetch_raw_image(self):
@@ -111,3 +119,212 @@ class IronicImagesTestCase(base.TestCase):
         self.assertEqual(expected_commands, self.executes)
 
         del self.executes
+
+
+class FsImageTestCase(base.TestCase):
+
+    @mock.patch.object(shutil, 'copyfile')
+    @mock.patch.object(os, 'makedirs')
+    @mock.patch.object(os.path, 'dirname')
+    @mock.patch.object(os.path, 'exists')
+    def test__create_root_fs(self, path_exists_mock,
+                            dirname_mock, mkdir_mock, cp_mock):
+
+        path_exists_mock_func = lambda path: path == 'root_dir'
+
+        files_info = {
+                'a1': 'b1',
+                'a2': 'b2',
+                'a3': 'sub_dir/b3'}
+
+        path_exists_mock.side_effect = path_exists_mock_func
+        dirname_mock.side_effect = ['root_dir', 'root_dir',
+                                    'root_dir/sub_dir', 'root_dir/sub_dir']
+        images._create_root_fs('root_dir', files_info)
+        cp_mock.assert_any_call('a1', 'root_dir/b1')
+        cp_mock.assert_any_call('a2', 'root_dir/b2')
+        cp_mock.assert_any_call('a3', 'root_dir/sub_dir/b3')
+
+        path_exists_mock.assert_any_call('root_dir/sub_dir')
+        dirname_mock.assert_any_call('root_dir/b1')
+        dirname_mock.assert_any_call('root_dir/b2')
+        dirname_mock.assert_any_call('root_dir/sub_dir/b3')
+        mkdir_mock.assert_called_once_with('root_dir/sub_dir')
+
+    @mock.patch.object(images, '_create_root_fs')
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'write_to_file')
+    @mock.patch.object(utils, 'dd')
+    @mock.patch.object(utils, 'umount')
+    @mock.patch.object(utils, 'mount')
+    @mock.patch.object(utils, 'mkfs')
+    def test_create_vfat_image(self, mkfs_mock, mount_mock, umount_mock,
+            dd_mock, write_mock, tempdir_mock, create_root_fs_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tempdir'
+        tempdir_mock.return_value = mock_file_handle
+
+        parameters = {'p1': 'v1'}
+        files_info = {'a': 'b'}
+        images.create_vfat_image('tgt_file', parameters=parameters,
+                files_info=files_info, parameters_file='qwe',
+                fs_size_kib=1000)
+
+        dd_mock.assert_called_once_with('/dev/zero',
+                                         'tgt_file',
+                                         'count=1',
+                                         'bs=1000KiB')
+
+        mkfs_mock.assert_called_once_with('vfat', 'tgt_file')
+        mount_mock.assert_called_once_with('tgt_file', 'tempdir',
+                                           '-o', 'umask=0')
+
+        parameters_file_path = os.path.join('tempdir', 'qwe')
+        write_mock.assert_called_once_with(parameters_file_path, 'p1=v1')
+        create_root_fs_mock.assert_called_once_with('tempdir', files_info)
+        umount_mock.assert_called_once_with('tempdir')
+
+    @mock.patch.object(images, '_create_root_fs')
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'dd')
+    @mock.patch.object(utils, 'umount')
+    @mock.patch.object(utils, 'mount')
+    @mock.patch.object(utils, 'mkfs')
+    def test_create_vfat_image_always_umount(self, mkfs_mock, mount_mock,
+            umount_mock, dd_mock, tempdir_mock, create_root_fs_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tempdir'
+        tempdir_mock.return_value = mock_file_handle
+        files_info = {'a': 'b'}
+        create_root_fs_mock.side_effect = OSError()
+        self.assertRaises(exception.ImageCreationFailed,
+                          images.create_vfat_image, 'tgt_file',
+                          files_info=files_info)
+
+        umount_mock.assert_called_once_with('tempdir')
+
+    @mock.patch.object(utils, 'dd')
+    def test_create_vfat_image_dd_fails(self, dd_mock):
+
+        dd_mock.side_effect = processutils.ProcessExecutionError
+        self.assertRaises(exception.ImageCreationFailed,
+                          images.create_vfat_image, 'tgt_file')
+
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'dd')
+    @mock.patch.object(utils, 'mkfs')
+    def test_create_vfat_image_mkfs_fails(self, mkfs_mock, dd_mock,
+                                          tempdir_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tempdir'
+        tempdir_mock.return_value = mock_file_handle
+
+        mkfs_mock.side_effect = processutils.ProcessExecutionError
+        self.assertRaises(exception.ImageCreationFailed,
+                          images.create_vfat_image, 'tgt_file')
+
+    @mock.patch.object(images, '_create_root_fs')
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'dd')
+    @mock.patch.object(utils, 'umount')
+    @mock.patch.object(utils, 'mount')
+    @mock.patch.object(utils, 'mkfs')
+    def test_create_vfat_image_umount_fails(self, mkfs_mock, mount_mock,
+            umount_mock, dd_mock, tempdir_mock, create_root_fs_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tempdir'
+        tempdir_mock.return_value = mock_file_handle
+        umount_mock.side_effect = processutils.ProcessExecutionError
+
+        self.assertRaises(exception.ImageCreationFailed,
+                          images.create_vfat_image, 'tgt_file')
+
+    def test__generate_isolinux_cfg(self):
+
+        kernel_params = ['key1=value1', 'key2']
+        expected_cfg = ("default boot\n"
+                        "\n"
+                        "label boot\n"
+                        "kernel /vmlinuz\n"
+                        "append initrd=/initrd text key1=value1 key2 --")
+        cfg = images._generate_isolinux_cfg(kernel_params)
+        self.assertEqual(expected_cfg, cfg)
+
+    @mock.patch.object(images, '_create_root_fs')
+    @mock.patch.object(utils, 'write_to_file')
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'execute')
+    @mock.patch.object(images, '_generate_isolinux_cfg')
+    def test_create_isolinux_image(self, gen_cfg_mock, utils_mock,
+                                   tempdir_mock, write_to_file_mock,
+                                   create_root_fs_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tmpdir'
+        tempdir_mock.return_value = mock_file_handle
+
+        cfg = "cfg"
+        cfg_file = 'tmpdir/isolinux/isolinux.cfg'
+        gen_cfg_mock.return_value = cfg
+
+        params = ['a=b', 'c']
+
+        images.create_isolinux_image('tgt_file', 'path/to/kernel',
+                'path/to/ramdisk', kernel_params=params)
+
+        files_info = {
+                'path/to/kernel': 'vmlinuz',
+                'path/to/ramdisk': 'initrd',
+                CONF.isolinux_bin: 'isolinux/isolinux.bin'
+                }
+        create_root_fs_mock.assert_called_once_with('tmpdir', files_info)
+        gen_cfg_mock.assert_called_once_with(params)
+        write_to_file_mock.assert_called_once_with(cfg_file, cfg)
+
+        utils_mock.assert_called_once_with('mkisofs', '-r', '-V',
+                 "BOOT IMAGE", '-cache-inodes', '-J', '-l',
+                 '-no-emul-boot', '-boot-load-size',
+                 '4', '-boot-info-table', '-b', 'isolinux/isolinux.bin',
+                 '-o', 'tgt_file', 'tmpdir')
+
+    @mock.patch.object(images, '_create_root_fs')
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'execute')
+    def test_create_isolinux_image_rootfs_fails(self, utils_mock,
+                                                tempdir_mock,
+                                                create_root_fs_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tmpdir'
+        tempdir_mock.return_value = mock_file_handle
+        create_root_fs_mock.side_effect = IOError
+
+        self.assertRaises(exception.ImageCreationFailed,
+                          images.create_isolinux_image,
+                          'tgt_file', 'path/to/kernel',
+                          'path/to/ramdisk')
+
+    @mock.patch.object(images, '_create_root_fs')
+    @mock.patch.object(utils, 'write_to_file')
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'execute')
+    @mock.patch.object(images, '_generate_isolinux_cfg')
+    def test_create_isolinux_image_mkisofs_fails(self, gen_cfg_mock,
+                                                 utils_mock,
+                                                 tempdir_mock,
+                                                 write_to_file_mock,
+                                                 create_root_fs_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tmpdir'
+        tempdir_mock.return_value = mock_file_handle
+        utils_mock.side_effect = processutils.ProcessExecutionError
+
+        self.assertRaises(exception.ImageCreationFailed,
+                          images.create_isolinux_image,
+                          'tgt_file', 'path/to/kernel',
+                          'path/to/ramdisk')