Add get/set_boot_image to libvirt driver

This change introduces fully-functional Virtual Media emulation.
Libvirt domains can be booted off the ISO image inserted into
the virtual media device.

Story: 2005149
Task: 29858
Change-Id: I8880dd9ae545d963e7bacf9aa78e4c5ea6b769cb

doc/source/user/dynamic-emulator.rst

@@ -150,6 +150,23 @@ Now you can run `sushy-emulator` with the updated configuration file:
 
    The images you will serve to your VMs need to be UEFI-bootable.
 
+Settable boot image
+~~~~~~~~~~~~~~~~~~~
+
+The `libvirt` system emulation backend supports setting custom boot images,
+so that libvirt domains (representing bare metal nodes) can boot from user
+images.
+
+This feature enables system boot from virtual media device.
+
+The limitations:
+
+* Only ISO images are supported
+* Remote libvirt hypervisor is not supported
+
+See *VirtualMedia* resource section for more information on how to perform
+virtual media boot.
+
 Systems resource driver: OpenStack
 ++++++++++++++++++++++++++++++++++
 
@@ -431,3 +448,116 @@ Redfish client can eject image from virtual media device:
         -H "Content-Type: application/json" \
         -X POST \
         http://localhost:8000/redfish/v1/Managers/58893887-8974-2487-2389-841168418919/VirtualMedia/Cd/Actions/VirtualMedia.EjectMedia
+
+Virtual media boot
+++++++++++++++++++
+
+To boot a system from a virtual media device the client first needs to figure
+out which manager is responsible for the system of interest:
+
+.. code-block:: bash
+
+    $ curl http://localhost:8000/redfish/v1/Systems/281c2fc3-dd34-439a-9f0f-63df45e2c998
+    {
+    ...
+    "Links": {
+        "Chassis": [
+        ],
+        "ManagedBy": [
+            {
+                "@odata.id": "/redfish/v1/Managers/58893887-8974-2487-2389-841168418919"
+            }
+        ]
+    },
+    ...
+
+Exploring the Redfish API links, the client can learn the virtual media devices
+being offered:
+
+.. code-block:: bash
+
+    $ curl http://localhost:8000/redfish/v1/Managers/58893887-894-2487-2389-841168418919/VirtualMedia
+    ...
+    "Members": [
+    {
+        "@odata.id": "/redfish/v1/Managers/58893887-8974-2487-2389-841168418919/VirtualMedia/Cd"
+    },
+    ...
+
+Knowing virtual media device name, the client can check out its present state:
+
+.. code-block:: bash
+
+    $ curl http://localhost:8000/redfish/v1/Managers/58893887-8974-2487-2389-841168418919/VirtualMedia/Cd
+    {
+        ...
+        "Name": "Virtual CD",
+        "MediaTypes": [
+            "CD",
+            "DVD"
+        ],
+        "Image": "",
+        "ImageName": "",
+        "ConnectedVia": "URI",
+        "Inserted": false,
+        "WriteProtected": false,
+        ...
+
+Assuming `http://localhost/var/tmp/mini.iso` URL points to a bootable UEFI or
+hybrid ISO, the following Redfish REST API call will insert the image into the
+virtual CD drive:
+
+.. code-block:: bash
+
+    $ curl -d \
+        '{"Image":"http:://localhost/var/tmp/mini.iso", "Inserted": true}' \
+         -H "Content-Type: application/json" \
+         -X POST \
+         http://localhost:8000/redfish/v1/Managers/58893887-8974-2487-2389-841168418919/VirtualMedia/Cd/Actions/VirtualMedia.InsertMedia
+
+Querying again, the emulator should have it in the drive:
+
+.. code-block:: bash
+
+    $ curl http://localhost:8000/redfish/v1/Managers/58893887-8974-2487-2389-841168418919/VirtualMedia/Cd
+    {
+        ...
+        "Name": "Virtual CD",
+        "MediaTypes": [
+            "CD",
+            "DVD"
+        ],
+        "Image": "http://localhost/var/tmp/mini.iso",
+        "ImageName": "mini.iso",
+        "ConnectedVia": "URI",
+        "Inserted": true,
+        "WriteProtected": true,
+        ...
+
+Next, the node needs to be configured to boot from its local CD drive
+over UEFI:
+
+.. code-block:: bash
+
+   $ curl -X PATCH -H 'Content-Type: application/json' \
+       -d '{
+         "Boot": {
+             "BootSourceOverrideTarget": "Cd",
+             "BootSourceOverrideMode": "Uefi",
+             "BootSourceOverrideEnabled": "Continuous"
+         }
+       }' \
+       http://localhost:8000/redfish/v1/Systems/281c2fc3-dd34-439a-9f0f-63df45e2c998
+
+By this point the system will boot off the virtual CD drive when powering it on:
+
+.. code-block:: bash
+
+   curl -d '{"ResetType":"On"}' \
+       -H "Content-Type: application/json" -X POST \
+        http://localhost:8000/redfish/v1/Systems/281c2fc3-dd34-439a-9f0f-63df45e2c998/Actions/ComputerSystem.Reset
+
+.. note::
+
+   ISO files to boot from must be UEFI-bootable, libvirtd should be running on the same
+   machine with sushy-emulator.

releasenotes/notes/add-virtual-media-4a137a5fb5017031.yaml

@@ -10,3 +10,10 @@ features:
     Each Manager automatically gets its own instance of the above
     mentioned virtual media device collection. HTTP/S-hosted images
     can be inserted into and ejected from virtual media devices.
+
+    If libvirt virtualization backend is being used, once ISO image
+    is inserted into the respective Manager, any System under that
+    Manager can boot from that image by booting from its local ``cdrom``
+    over UEFI.
+
+    The ISO images must be UEFI-bootable or hybrid.

sushy_tools/emulator/main.py

@@ -378,7 +378,7 @@ def virtual_media_insert(identity, device):
                     system, device, boot_image=image_path,
                     write_protected=write_protected)
 
-            except error.FishyError as ex:
+            except error.NotSupportedError as ex:
                 app.logger.warning(
                     'System %s failed to set boot image %s on device %s: '
                     '%s', system, image_path, device, ex)
@@ -402,7 +402,7 @@ def virtual_media_eject(identity, device):
             try:
                 resources.systems.set_boot_image(system, device)
 
-            except error.FishyError as ex:
+            except error.NotSupportedError as ex:
                 app.logger.warning(
                     'System %s failed to remove boot image from device %s: '
                     '%s', system, device, ex)

sushy_tools/emulator/resources/systems/libvirtdriver.py

@@ -15,6 +15,7 @@
 
 from collections import namedtuple
 import logging
+import os
 import uuid
 import xml.etree.ElementTree as ET
 
@@ -100,11 +101,39 @@ class LibvirtDriver(AbstractSystemsDriver):
 
     }
 
+    DEVICE_TYPE_MAP = {
+        constants.DEVICE_TYPE_CD: 'cdrom',
+        constants.DEVICE_TYPE_FLOPPY: 'floppy',
+    }
+
+    DEVICE_TYPE_MAP_REV = {v: k for k, v in DEVICE_TYPE_MAP.items()}
+
+    # target device, controller ID, controller unit number
+    DEVICE_TARGET_MAP = {
+        constants.DEVICE_TYPE_FLOPPY: ('fda', 'fdc', '0'),
+        constants.DEVICE_TYPE_CD: ('hdc', 'ide', '1')
+    }
+
     DEFAULT_BIOS_ATTRIBUTES = {"BootMode": "Uefi",
                                "EmbeddedSata": "Raid",
                                "NicBoot1": "NetworkBoot",
                                "ProcTurboMode": "Enabled"}
 
+    STORAGE_POOL = 'default'
+
+    STORAGE_POOL_XML = """
+<volume type='file'>
+  <name>%(name)s</name>
+  <key>%(name)s</key>
+  <capacity unit='bytes'>%(size)i</capacity>
+  <physical unit='bytes'>%(size)i</physical>
+  <target>
+    <path>%(path)s</path>
+    <format type='raw'/>
+  </target>
+</volume>
+"""
+
     @classmethod
     def initialize(cls, config, uri=None):
         cls._config = config
@@ -611,7 +640,174 @@ class LibvirtDriver(AbstractSystemsDriver):
         :returns: a `tuple` of (boot_image, write_protected, inserted)
         :raises: `error.FishyError` if boot device can't be accessed
         """
-        raise error.FishyError('Not implemented')
+        domain = self._get_domain(identity, readonly=True)
+
+        tree = ET.fromstring(domain.XMLDesc())
+
+        device_element = tree.find('devices')
+        if device_element is None:
+            msg = ('Missing "devices" tag in the libvirt domain '
+                   '"%(identity)s" configuration' % {'identity': identity})
+            raise error.FishyError(msg)
+
+        for disk_element in device_element.findall('disk'):
+            dev_type = disk_element.attrib.get('device')
+            if (dev_type not in self.DEVICE_TYPE_MAP_REV or
+                    dev_type != self.DEVICE_TYPE_MAP.get(device)):
+                continue
+
+            source_element = disk_element.find('source')
+            if source_element is None:
+                continue
+
+            boot_image = source_element.attrib.get('file')
+            if boot_image is None:
+                continue
+
+            read_only = disk_element.find('readonly') or False
+
+            inserted = (self.get_boot_device(identity) ==
+                        constants.DEVICE_TYPE_CD)
+            if inserted:
+                inserted = self.get_boot_mode(identity) == 'Uefi'
+
+            return boot_image, read_only, inserted
+
+        return '', False, False
+
+    def _upload_image(self, domain, conn, boot_image):
+        pool = conn.storagePoolLookupByName(self.STORAGE_POOL)
+
+        pool_tree = ET.fromstring(pool.XMLDesc())
+
+        # Find out path to images
+        pool_path_element = pool_tree.find('target/path')
+        if pool_path_element is None:
+            msg = ('Missing "target/path" tag in the libvirt '
+                   'storage pool "%(pool)s"'
+                   '' % {'pool': self.STORAGE_POOL})
+            raise error.FishyError(msg)
+
+        image_name = '%s-%s.img' % (
+            os.path.basename(boot_image).replace('.', '-'),
+            domain.UUIDString())
+
+        image_path = os.path.join(
+            pool_path_element.text, image_name)
+
+        image_size = os.stat(boot_image).st_size
+
+        # Remove already existing volume
+
+        volumes_names = [v.name() for v in pool.listAllVolumes()]
+        if image_name in volumes_names:
+            volume = pool.storageVolLookupByName(image_name)
+            volume.delete()
+
+        # Create new volume
+
+        volume = pool.createXML(
+            self.STORAGE_POOL_XML % {
+                'name': image_name, 'path': image_path,
+                'size': image_size})
+
+        # Upload image to hypervisor
+
+        stream = conn.newStream()
+        volume.upload(stream, 0, image_size)
+
+        def read_file(stream, nbytes, fl):
+            return fl.read(nbytes)
+
+        stream.sendAll(read_file, open(boot_image, 'rb'))
+
+        stream.finish()
+
+        return image_path
+
+    def _add_boot_image(self, domain, domain_tree, device,
+                        boot_image, write_protected):
+
+        identity = domain.UUIDString()
+
+        device_element = domain_tree.find('devices')
+        if device_element is None:
+            msg = ('Missing "devices" tag in the libvirt domain '
+                   '"%(identity)s" configuration' % {'identity': identity})
+            raise error.FishyError(msg)
+
+        with libvirt_open(self._uri) as conn:
+
+            image_path = self._upload_image(domain, conn, boot_image)
+
+            try:
+                lv_device = self.BOOT_DEVICE_MAP[device]
+
+            except KeyError:
+                raise error.FishyError(
+                    'Unknown device %s at %s' % (device, identity))
+
+            # Add disk element pointing to the boot image
+
+            disk_element = ET.SubElement(device_element, 'disk')
+            disk_element.set('type', 'file')
+            disk_element.set('device', lv_device)
+
+            tgt_dev, tgt_bus, tgt_unit = self.DEVICE_TARGET_MAP[device]
+
+            target_element = ET.SubElement(disk_element, 'target')
+            target_element.set('dev', tgt_dev)
+            target_element.set('bus', tgt_bus)
+
+            driver_element = ET.SubElement(disk_element, 'driver')
+            driver_element.set('name', 'qemu')
+            driver_element.set('type', 'raw')
+
+            address_element = ET.SubElement(disk_element, 'address')
+            address_element.set('type', 'drive')
+            address_element.set('controller', '0')
+            address_element.set('bus', '0')
+            address_element.set('target', '0')
+            address_element.set('unit', tgt_unit)
+
+            source_element = ET.SubElement(disk_element, 'source')
+            source_element.set('file', image_path)
+
+            if write_protected:
+                ET.SubElement(disk_element, 'readonly')
+
+            conn.defineXML(ET.tostring(domain_tree).decode('utf-8'))
+
+    def _remove_boot_images(self, domain, domain_tree, device):
+
+        identity = domain.UUIDString()
+
+        device_element = domain_tree.find('devices')
+        if device_element is None:
+            msg = ('Missing "devices" tag in the libvirt domain '
+                   '"%(identity)s" configuration' % {'identity': identity})
+            raise error.FishyError(msg)
+
+        try:
+            lv_device = self.BOOT_DEVICE_MAP[device]
+
+        except KeyError:
+            raise error.FishyError(
+                'Unknown device %s at %s' % (device, identity))
+
+        device_element = domain_tree.find('devices')
+        if device_element is None:
+            msg = ('Missing "devices" tag in the libvirt domain '
+                   '"%(identity)s" configuration' % {'identity': identity})
+            raise error.FishyError(msg)
+
+        # First, remove all existing devices
+        disk_elements = device_element.findall('disk')
+
+        for disk_element in disk_elements:
+            dev_type = disk_element.attrib.get('device')
+            if dev_type == lv_device:
+                device_element.remove(disk_element)
 
     def set_boot_image(self, identity, device, boot_image=None,
                        write_protected=True):
@@ -626,4 +822,20 @@ class LibvirtDriver(AbstractSystemsDriver):
 
         :raises: `error.FishyError` if boot device can't be set
         """
-        raise error.FishyError('Not implemented')
+        domain = self._get_domain(identity)
+
+        domain_tree = ET.fromstring(domain.XMLDesc())
+
+        self._remove_boot_images(domain, domain_tree, device)
+
+        if boot_image:
+
+            try:
+                self._add_boot_image(domain, domain_tree, device,
+                                     boot_image, write_protected)
+
+            except libvirt.libvirtError as e:
+                msg = ('Error changing boot image at libvirt URI "%(uri)s": '
+                       '%(error)s' % {'uri': self._uri, 'error': e})
+
+                raise error.FishyError(msg)

sushy_tools/emulator/resources/systems/novadriver.py

@@ -355,7 +355,7 @@ class OpenStackDriver(AbstractSystemsDriver):
         :returns: a `tuple` of (boot_image, write_protected, inserted)
         :raises: `error.FishyError` if boot device can't be accessed
         """
-        raise error.FishyError('Not implemented')
+        raise error.NotSupportedError('Not implemented')
 
     def set_boot_image(self, identity, device, boot_image=None,
                        write_protected=True):
@@ -370,4 +370,4 @@ class OpenStackDriver(AbstractSystemsDriver):
 
         :raises: `error.FishyError` if boot device can't be set
         """
-        raise error.FishyError('Not implemented')
+        raise error.NotSupportedError('Not implemented')

sushy_tools/error.py

@@ -20,3 +20,7 @@ class FishyError(Exception):
 
 class AliasAccessError(FishyError):
     """Node access attempted via an alias, not UUID"""
+
+
+class NotSupportedError(FishyError):
+    """Feature not supported by resource driver"""

sushy_tools/tests/unit/emulator/pool.xml

@@ -0,0 +1,17 @@
+<pool type='dir'>
+  <name>default</name>
+  <uuid>267e1242-d53f-46dd-adb3-9f3992c55f6f</uuid>
+  <capacity unit='bytes'>166318571520</capacity>
+  <allocation unit='bytes'>13143412736</allocation>
+  <available unit='bytes'>153175158784</available>
+  <source>
+  </source>
+  <target>
+    <path>/var/lib/libvirt/images</path>
+    <permissions>
+      <mode>0711</mode>
+      <owner>0</owner>
+      <group>0</group>
+    </permissions>
+  </target>
+</pool>

sushy_tools/tests/unit/emulator/resources/systems/test_libvirt.py

@@ -341,6 +341,55 @@ class LibvirtDriverTestCase(base.BaseTestCase):
         # NOTE(etingof): should enforce default loader
         self.assertIsNone(os_element.find('loader'))
 
+    @mock.patch('libvirt.openReadOnly', autospec=True)
+    def test_get_boot_image(self, libvirt_mock):
+        with open('sushy_tools/tests/unit/emulator/domain.xml', 'r') as f:
+            data = f.read()
+
+        conn_mock = libvirt_mock.return_value
+        domain_mock = conn_mock.lookupByUUID.return_value
+        domain_mock.XMLDesc.return_value = data
+
+        image_info = self.test_driver.get_boot_image(self.uuid, 'Cd')
+
+        expected = '/home/user/boot.iso', False, False
+
+        self.assertEqual(expected, image_info)
+
+    @mock.patch('sushy_tools.emulator.resources.systems.libvirtdriver'
+                '.os.stat', autospec=True)
+    @mock.patch('sushy_tools.emulator.resources.systems.libvirtdriver'
+                '.open')
+    @mock.patch('libvirt.open', autospec=True)
+    @mock.patch('libvirt.openReadOnly', autospec=True)
+    def test_set_boot_image(self, libvirt_mock, libvirt_rw_mock,
+                            open_mock, stat_mock):
+        with open('sushy_tools/tests/unit/emulator/domain.xml', 'r') as f:
+            data = f.read()
+
+        conn_mock = libvirt_rw_mock.return_value
+        domain_mock = conn_mock.lookupByUUID.return_value
+        domain_mock.XMLDesc.return_value = data
+
+        pool_mock = conn_mock.storagePoolLookupByName.return_value
+
+        with open('sushy_tools/tests/unit/emulator/pool.xml', 'r') as f:
+            data = f.read()
+
+        pool_mock.XMLDesc.return_value = data
+
+        self.test_driver.set_boot_image(self.uuid, 'Cd', '/tmp/image.iso')
+
+        conn_mock = libvirt_rw_mock.return_value
+        pool_mock.listAllVolumes.assert_called_once_with()
+        stat_mock.assert_called_once_with('/tmp/image.iso')
+        pool_mock.createXML.assert_called_once_with(mock.ANY)
+
+        volume_mock = pool_mock.createXML.return_value
+        volume_mock.upload.assert_called_once_with(mock.ANY, 0, mock.ANY)
+
+        conn_mock.defineXML.assert_called_once_with(mock.ANY)
+
     @mock.patch('libvirt.openReadOnly', autospec=True)
     def test_get_total_memory(self, libvirt_mock):
         conn_mock = libvirt_mock.return_value