Merge "Add inspection hooks"

ironic/conf/inspector.py

@@ -128,7 +128,14 @@ opts = [
                 help=_('Mapping of IP subnet CIDR to physical network. When '
                        'the phyical-network inspection hook is enabled, the '
                        '"physical_network" property of corresponding '
-                       'baremetal ports is populated based on this mapping.'))
+                       'baremetal ports is populated based on this mapping.')),
+    cfg.BoolOpt('disk_partitioning_spacing',
+                default=True,
+                help=_('Whether to leave 1 GiB of disk size untouched for '
+                       'partitioning. Only has effect when used with the IPA '
+                       'as a ramdisk, for older ramdisk local_gb is '
+                       'calculated on the ramdisk side. This configuration '
+                       'option is used by the "root-device" inspection hook.'))
 ]
 
 

ironic/drivers/modules/inspector/hooks/raid_device.py

@@ -0,0 +1,84 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from oslo_log import log as logging
+
+from ironic.common import exception
+from ironic.drivers.modules.inspector.hooks import base
+from ironic.objects import node_inventory
+
+LOG = logging.getLogger(__name__)
+
+
+class RaidDeviceHook(base.InspectionHook):
+    """Hook for learning the root device after RAID creation.
+
+    This hook can figure out the root device in 2 runs. In the first run, the
+    node's inventory is saved as usual, and the hook does not do anything.
+    The second run will check the difference between the recently discovered
+    block devices (as reported by the inspection results) and the previously
+    saved ones (from the previously saved inventory). If there is exactly one
+    new block device, its serial number is saved in node.properties under the
+    'root_device' key.
+
+    This way, it helps to figure out the root device hint in cases when Ironic
+    doesn't have enough information to do so otherwise. One such usecase is
+    DRAC RAID configuration, where the BMC doesn't provide any useful
+    information about the created RAID disks. Using this hook immediately
+    before and after creating the root RAID device will solve the issue of
+    root device hints.
+    """
+
+    def _get_serials(self, inventory):
+        if inventory.get('disks'):
+            return [x['serial'] for x in inventory.get('disks')
+                    if x.get('serial')]
+
+    def __call__(self, task, inventory, plugin_data):
+        node = task.node
+
+        if 'root_device' in node.properties:
+            LOG.info('Root device is already known for node %s', node.uuid)
+            return
+
+        current_devices = self._get_serials(inventory)
+        if not current_devices:
+            LOG.warning('No block device information was received from the '
+                        'ramdisk for node %s', node.uuid)
+            return
+
+        try:
+            previous_inventory = node_inventory.NodeInventory.get_by_node_id(
+                task.context, node.id)
+        except exception.NodeInventoryNotFound:
+            LOG.debug('Inventory for node %s not found in the database. Raid '
+                      'device hook exiting.', task.node.uuid)
+            return
+        previous_devices = self._get_serials(previous_inventory.get(
+            'inventory_data'))
+
+        # Compare previously discovered devices with the current ones
+        new_devices = [device for device in current_devices
+                       if device not in previous_devices]
+        if len(new_devices) > 1:
+            LOG.warning('Root device cannot be identified because multiple '
+                        'new devices were found for node %s', node.uuid)
+            return
+        elif len(new_devices) == 0:
+            LOG.warning('No new devices were found for node %s', node.uuid)
+            return
+
+        node.set_property('root_device', {'serial': new_devices[0]})
+        node.save()
+        LOG.info('"root_device" property set for node %s', node.uuid)

ironic/drivers/modules/inspector/hooks/root_device.py

@@ -0,0 +1,109 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ironic_lib import utils as il_utils
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_utils import units
+
+from ironic.common import exception
+from ironic.common.i18n import _
+from ironic.drivers.modules.inspector.hooks import base
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+
+class RootDeviceHook(base.InspectionHook):
+    """Smarter root disk selection using Ironic root device hints."""
+
+    def _get_skip_list_for_node(self, node, block_devices):
+        skip_list_hints = node.properties.get("skip_block_devices", [])
+        if not skip_list_hints:
+            return
+        skip_list = set()
+
+        for hint in skip_list_hints:
+            skipped_devs = il_utils.find_devices_by_hints(block_devices, hint)
+            excluded_devs = {dev['name'] for dev in skipped_devs}
+            skipped_devices = excluded_devs.difference(skip_list)
+            skip_list = skip_list.union(excluded_devs)
+
+            if skipped_devices:
+                LOG.warning("Using hint %(hint)s skipping devices: %(devs)s",
+                            {'hint': hint, 'devs': ','.join(skipped_devices)})
+        return skip_list
+
+    def _process_root_device_hints(self, node, inventory, plugin_data):
+        """Detect root disk from root device hints and IPA inventory."""
+
+        hints = node.properties.get('root_device')
+        if not hints:
+            LOG.debug('Root device hints are not provided for node %s',
+                      node.uuid)
+            return
+
+        skip_list = self._get_skip_list_for_node(node, inventory['disks'])
+        if skip_list:
+            inventory_disks = [d for d in inventory['disks']
+                               if d['name'] not in skip_list]
+        else:
+            inventory_disks = inventory['disks']
+
+        try:
+            root_device = il_utils.match_root_device_hints(inventory_disks,
+                                                           hints)
+        except (TypeError, ValueError) as e:
+            raise exception.HardwareInspectionFailure(
+                _('No disks could be found using root device hints %(hints)s '
+                  'for node %(node)s because they failed to validate. '
+                  'Error: %(error)s') % {'hints': hints, 'node': node.uuid,
+                                         'error': e})
+        if not root_device:
+            raise exception.HardwareInspectionFailure(_(
+                'No disks satisfied root device hints for node %s') %
+                node.uuid)
+
+        LOG.debug('Disk %(disk)s of size %(size)s satisfies root device '
+                  'hints. Node: %s', {'disk': root_device.get('name'),
+                                      'size': root_device['size'],
+                                      'node': node.uuid})
+        plugin_data['root_disk'] = root_device
+
+    def __call__(self, task, inventory, plugin_data):
+        """Process root disk information."""
+        self._process_root_device_hints(task.node, inventory, plugin_data)
+
+        root_disk = plugin_data.get('root_disk')
+        if root_disk:
+            local_gb = root_disk['size'] // units.Gi
+            if not local_gb:
+                LOG.warning('The requested root disk is too small (smaller '
+                            'than 1 GiB) or its size cannot be detected. '
+                            'Root disk: %s, Node: %s', root_disk,
+                            task.node.uuid)
+            else:
+                if CONF.inspector.disk_partitioning_spacing:
+                    local_gb -= 1
+                LOG.info('Root disk %(disk)s, local_gb %(local_gb)s GiB, '
+                         'Node: %(node)s', {'disk': root_disk,
+                                            'local_gb': local_gb,
+                                            'node': task.node.uuid})
+        else:
+            local_gb = 0
+            LOG.info('No root device found for node %s. Assuming node is '
+                     'diskless.', task.node.uuid)
+
+        plugin_data['local_gb'] = local_gb
+        task.node.set_property('local_gb', str(local_gb))
+        task.node.save()

ironic/tests/unit/drivers/modules/inspector/hooks/test_raid_device.py

@@ -0,0 +1,107 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from unittest import mock
+
+from ironic.common import exception
+from ironic.conductor import task_manager
+from ironic.conf import CONF
+from ironic.drivers.modules.inspector.hooks import raid_device \
+    as raid_device_hook
+from ironic.objects.node_inventory import NodeInventory
+from ironic.tests.unit.db import base as db_base
+from ironic.tests.unit.objects import utils as obj_utils
+
+
+class RaidDeviceTestCase(db_base.DbTestCase):
+    def setUp(self):
+        super().setUp()
+        CONF.set_override('enabled_inspect_interfaces',
+                          ['agent', 'no-inspect'])
+        self.node = obj_utils.create_test_node(self.context,
+                                               inspect_interface='agent')
+        self.inventory_1 = {'disks': [{'name': '/dev/sda', 'serial': '1111'},
+                                      {'name': '/dev/sdb', 'serial': '2222'}]}
+        self.inventory_2 = {'disks': [{'name': '/dev/sdb', 'serial': '2222'},
+                                      {'name': '/dev/sdc', 'serial': '3333'}]}
+        self.plugin_data = {'plugin_data': 'fake-plugin-data'}
+
+    def test_root_device_already_set(self):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            self.node.properties = {'root_device': 'any'}
+            raid_device_hook.RaidDeviceHook().__call__(task,
+                                                       self.inventory_1,
+                                                       self.plugin_data)
+            self.assertEqual(self.node.properties.get('root_device'), 'any')
+
+    def test_no_serials(self):
+        self.inventory_1['disks'][0]['serial'] = None
+        with task_manager.acquire(self.context, self.node.id) as task:
+            raid_device_hook.RaidDeviceHook().__call__(task,
+                                                       self.inventory_1,
+                                                       self.plugin_data)
+            self.node.refresh()
+            self.assertIsNone(self.node.properties.get('root_device'))
+
+    @mock.patch.object(NodeInventory, 'get_by_node_id', autospec=True)
+    def test_no_previous_inventory(self, mock_get_by_node_id):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            mock_get_by_node_id.side_effect = exception.NodeInventoryNotFound()
+            raid_device_hook.RaidDeviceHook().__call__(task,
+                                                       self.inventory_1,
+                                                       self.plugin_data)
+            self.node.refresh()
+            self.assertIsNone(self.node.properties.get('root_device'))
+
+    @mock.patch.object(NodeInventory, 'get_by_node_id', autospec=True)
+    def test_no_new_root_devices(self, mock_get_by_node_id):
+        with task_manager.acquire(self.context, self.node.id) as task:
+
+            mock_get_by_node_id.return_value = NodeInventory(
+                task.context, id=1, node_id=self.node.id,
+                inventory_data=self.inventory_1, plugin_data=self.plugin_data)
+
+            raid_device_hook.RaidDeviceHook().__call__(task,
+                                                       self.inventory_1,
+                                                       self.plugin_data)
+            self.node.refresh()
+            result = self.node.properties.get('root_device')
+            self.assertIsNone(result)
+
+    @mock.patch.object(NodeInventory, 'get_by_node_id', autospec=True)
+    def test_root_device_found(self, mock_get_by_node_id):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            mock_get_by_node_id.return_value = NodeInventory(
+                task.context, id=1, node_id=self.node.id,
+                inventory_data=self.inventory_1, plugin_data=self.plugin_data)
+            raid_device_hook.RaidDeviceHook().__call__(task,
+                                                       self.inventory_2,
+                                                       self.plugin_data)
+            self.node.refresh()
+            result = self.node.properties.get('root_device')
+            self.assertEqual(result, {'serial': '3333'})
+
+    @mock.patch.object(NodeInventory, 'get_by_node_id', autospec=True)
+    def test_multiple_new_root_devices(self, mock_get_by_node_id):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            mock_get_by_node_id.return_value = NodeInventory(
+                task.context, id=1, node_id=self.node.id,
+                inventory_data=self.inventory_1, plugin_data=self.plugin_data)
+            self.inventory_2['disks'].append({'name': '/dev/sdd',
+                                              'serial': '4444'})
+            raid_device_hook.RaidDeviceHook().__call__(task,
+                                                       self.inventory_2,
+                                                       self.plugin_data)
+            self.node.refresh()
+            result = self.node.properties.get('root_device')
+            self.assertIsNone(result)

ironic/tests/unit/drivers/modules/inspector/hooks/test_root_device.py

@@ -0,0 +1,178 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_utils import units
+
+from ironic.common import exception
+from ironic.conductor import task_manager
+from ironic.conf import CONF
+from ironic.drivers.modules.inspector.hooks import root_device as \
+    root_device_hook
+from ironic.tests.unit.db import base as db_base
+from ironic.tests.unit.objects import utils as obj_utils
+
+
+class RootDeviceTestCase(db_base.DbTestCase):
+    def setUp(self):
+        super().setUp()
+        CONF.set_override('enabled_inspect_interfaces',
+                          ['agent', 'no-inspect'])
+        self.node = obj_utils.create_test_node(self.context,
+                                               inspect_interface='agent')
+        self.inventory = {'disks': [
+            {'name': '/dev/sda', 'model': 'Model 1', 'size': 1000 * units.Gi,
+             'serial': '1111'},
+            {'name': '/dev/sdb', 'model': 'Model 2', 'size': 10 * units.Gi,
+             'serial': '2222'},
+            {'name': '/dev/sdc', 'model': 'Model 1', 'size': 20 * units.Gi,
+             'serial': '3333'},
+            {'name': '/dev/sdd', 'model': 'Model 3', 'size': 0,
+             'serial': '4444'}]}
+        self.plugin_data = {}
+
+    def test_no_hints(self):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            root_device_hook.RootDeviceHook().__call__(task,
+                                                       self.inventory,
+                                                       self.plugin_data)
+            self.assertNotIn('root_disk', self.plugin_data)
+            self.assertEqual(self.plugin_data['local_gb'], 0)
+            self.assertEqual(task.node.properties.get('local_gb'), '0')
+
+    def test_root_device_skip_list(self):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            task.node.properties['root_device'] = {'serial': '1111'}
+            task.node.properties['skip_block_devices'] = [{'size': 1000}]
+
+            self.assertRaisesRegex(exception.HardwareInspectionFailure,
+                                   'No disks satisfied root device hints for '
+                                   'node %s' % self.node.id,
+                                   root_device_hook.RootDeviceHook().__call__,
+                                   task, self.inventory, self.plugin_data)
+            self.assertNotIn('root_disk', self.plugin_data)
+            self.assertNotIn('local_gb', self.plugin_data)
+            # The default value of the `local_gb` property is left unchanged
+            self.assertEqual(task.node.properties.get('local_gb'), '10')
+
+    def test_first_match_on_skip_list_use_second(self):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            task.node.properties['root_device'] = {'model': 'Model 1'}
+            task.node.properties['skip_block_devices'] = [{'size': 1000}]
+            root_device_hook.RootDeviceHook().__call__(task,
+                                                       self.inventory,
+                                                       self.plugin_data)
+            task.node.refresh()
+
+            expected_root_device = self.inventory['disks'][2].copy()
+            self.assertEqual(self.plugin_data['root_disk'],
+                             expected_root_device)
+
+            expected_local_gb = (expected_root_device['size'] // units.Gi) - 1
+            self.assertEqual(self.plugin_data['local_gb'],
+                             expected_local_gb)
+            self.assertEqual(task.node.properties.get('local_gb'),
+                             str(expected_local_gb))
+
+    def test_one_matches(self):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            task.node.properties['root_device'] = {'serial': '1111'}
+
+            root_device_hook.RootDeviceHook().__call__(task,
+                                                       self.inventory,
+                                                       self.plugin_data)
+            task.node.refresh()
+
+            self.assertEqual(self.plugin_data['root_disk'],
+                             self.inventory['disks'][0])
+            self.assertEqual(self.plugin_data['local_gb'], 999)
+            self.assertEqual(task.node.properties.get('local_gb'), '999')
+
+    def test_local_gb_without_spacing(self):
+        CONF.set_override('disk_partitioning_spacing', False, 'inspector')
+        with task_manager.acquire(self.context, self.node.id) as task:
+            task.node.properties['root_device'] = {'serial': '1111'}
+            root_device_hook.RootDeviceHook().__call__(task,
+                                                       self.inventory,
+                                                       self.plugin_data)
+            task.node.refresh()
+
+            self.assertEqual(self.plugin_data['root_disk'],
+                             self.inventory['disks'][0])
+            self.assertEqual(self.plugin_data['local_gb'], 1000)
+            self.assertEqual(task.node.properties.get('local_gb'), '1000')
+
+    def test_zero_size(self):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            task.node.properties['root_device'] = {'name': '/dev/sdd'}
+            root_device_hook.RootDeviceHook().__call__(task,
+                                                       self.inventory,
+                                                       self.plugin_data)
+            task.node.refresh()
+            self.assertEqual(self.plugin_data['root_disk'],
+                             self.inventory['disks'][3])
+            self.assertEqual(self.plugin_data['local_gb'], 0)
+            self.assertEqual(task.node.properties.get('local_gb'), '0')
+
+    def test_all_match(self):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            task.node.properties['root_device'] = {'size': 20,
+                                                   'model': 'Model 1'}
+            root_device_hook.RootDeviceHook().__call__(task,
+                                                       self.inventory,
+                                                       self.plugin_data)
+            task.node.refresh()
+            self.assertEqual(self.plugin_data['root_disk'],
+                             self.inventory['disks'][2])
+            self.assertEqual(self.plugin_data['local_gb'], 19)
+            self.assertEqual(task.node.properties.get('local_gb'), '19')
+
+    def test_incorrect_hint(self):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            task.node.properties['root_device'] = {'size': 20,
+                                                   'model': 'Model 42'}
+            self.assertRaisesRegex(exception.HardwareInspectionFailure,
+                                   'No disks satisfied root device hints for '
+                                   'node %s' % task.node.uuid,
+                                   root_device_hook.RootDeviceHook().__call__,
+                                   task, self.inventory, self.plugin_data)
+            self.assertNotIn('root_disk', self.plugin_data)
+            self.assertNotIn('local_gb', self.plugin_data)
+            # The default value of the `local_gb` property is unchanged
+            self.assertEqual(task.node.properties.get('local_gb'), '10')
+
+    def test_size_string(self):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            task.node.properties['root_device'] = {'size': '20'}
+            root_device_hook.RootDeviceHook().__call__(task,
+                                                       self.inventory,
+                                                       self.plugin_data)
+            task.node.refresh()
+            self.assertEqual(self.plugin_data['root_disk'],
+                             self.inventory['disks'][2])
+            self.assertEqual(self.plugin_data['local_gb'], 19)
+            self.assertEqual(task.node.properties.get('local_gb'), '19')
+
+    def test_size_invalid(self):
+        with task_manager.acquire(self.context, self.node.id) as task:
+            for bad_size in ('foo', None, {}):
+                task.node.properties['root_device'] = {'size': bad_size}
+                self.assertRaisesRegex(
+                    exception.HardwareInspectionFailure,
+                    'No disks could be found using root device hints',
+                    root_device_hook.RootDeviceHook().__call__,
+                    task, self.inventory, self.plugin_data)
+
+                self.assertNotIn('root_disk', self.plugin_data)
+                self.assertNotIn('local_gb', self.plugin_data)
+                # The default value of the `local_gb` property is left
+                # unchanged
+                self.assertEqual(task.node.properties.get('local_gb'), '10')

setup.cfg

@@ -209,6 +209,8 @@ ironic.inspection.hooks =
     memory = ironic.drivers.modules.inspector.hooks.memory:MemoryHook
     pci-devices = ironic.drivers.modules.inspector.hooks.pci_devices:PciDevicesHook
     physical-network = ironic.drivers.modules.inspector.hooks.physical_network:PhysicalNetworkHook
+    raid-device = ironic.drivers.modules.inspector.hooks.raid_device:RaidDeviceHook
+    root-device = ironic.drivers.modules.inspector.hooks.root_device:RootDeviceHook
 
 [egg_info]
 tag_build =