Wait until iDRAC is ready before out-of-band cleaning

When out-of-band cleaning is initiated, the node is PXE booted and the
ramdisk is loaded.  After in-band cleaning completes, the node is
rebooted.  At that point, the iDRAC automatically creates and runs an
"Export Configuration" job.  Out-of-band cleaning then starts: either
RAID configuration creation or deletion.  If the export job has not
finished by the time the RAID deletion or creation job is attempted to
be created, then the RAID job creation fails.

This patch causes RAID configuration creation and deletion to wait
until the iDRAC declares itself to be ready before proceeding with
out-of-band cleaning.  This ensures that the export job has completed
before creating another job.

Change-Id: I79faba2206b86288ae636c46468a8b2dc321f979
Closes-Bug: 1691808
Depends-On: I929deada3dda7b09a6f29033fff89d9b0382aef8

driver-requirements.txt

@@ -10,7 +10,7 @@ python-ironic-inspector-client>=1.5.0
 python-oneviewclient<3.0.0,>=2.5.2
 python-scciclient>=0.4.0
 UcsSdk==0.8.2.2
-python-dracclient>=0.1.0
+python-dracclient>=1.2.0
 
 # The CIMC drivers use the Cisco IMC SDK version 0.7.2 or greater
 ImcSdk>=0.7.2

ironic/drivers/modules/drac/raid.py

@@ -705,6 +705,11 @@ class DracRAID(base.RAIDInterface):
         """
         node = task.node
 
+        # The node is rebooting at this point, so wait for the iDRAC to
+        # enter the ready state before proceeding with cleaning
+        client = drac_common.get_drac_client(node)
+        client.wait_until_idrac_is_ready()
+
         logical_disks = node.target_raid_config['logical_disks']
         for disk in logical_disks:
             if (disk['size_gb'] == 'MAX' and 'physical_disks' not in disk):
@@ -756,6 +761,11 @@ class DracRAID(base.RAIDInterface):
         """
         node = task.node
 
+        # The node is rebooting at this point, so wait for the iDRAC to
+        # enter the ready state before proceeding with cleaning
+        client = drac_common.get_drac_client(node)
+        client.wait_until_idrac_is_ready()
+
         controllers = set()
         for disk in list_virtual_disks(node):
             controllers.add(disk.controller)

ironic/tests/unit/drivers/modules/drac/test_raid.py

@@ -615,6 +615,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
             task.driver.raid.create_configuration(
                 task, create_root_volume=True, create_nonroot_volumes=False)
 
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
             mock_client.create_virtual_disk.assert_called_once_with(
                 'RAID.Integrated.1-1',
                 ['Disk.Bay.0:Enclosure.Internal.0-1:RAID.Integrated.1-1',
@@ -647,6 +648,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
             return_value = task.driver.raid.create_configuration(
                 task, create_root_volume=False, create_nonroot_volumes=False)
 
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
             self.assertEqual(0, mock_client.create_virtual_disk.call_count)
             self.assertEqual(0, mock_commit_config.call_count)
 
@@ -688,6 +690,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
             task.driver.raid.create_configuration(
                 task, create_root_volume=True, create_nonroot_volumes=True)
 
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
             mock_client.create_virtual_disk.assert_called_once_with(
                 'RAID.Integrated.1-1',
                 ['Disk.Bay.0:Enclosure.Internal.0-1:RAID.Integrated.1-1',
@@ -729,6 +732,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
             task.driver.raid.create_configuration(
                 task, create_root_volume=True, create_nonroot_volumes=True)
 
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
             mock_client.create_virtual_disk.assert_has_calls(
                 [mock.call(
                     'controller-2',
@@ -800,6 +804,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
             task.driver.raid.create_configuration(
                 task, create_root_volume=True, create_nonroot_volumes=True)
 
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
             mock_client.create_virtual_disk.assert_has_calls(
                 [mock.call(
                     'RAID.Integrated.1-1',
@@ -859,6 +864,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
             task.driver.raid.create_configuration(
                 task, create_root_volume=True, create_nonroot_volumes=True)
 
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
             mock_client.create_virtual_disk.assert_has_calls(
                 [mock.call(
                     'RAID.Integrated.1-1',
@@ -919,6 +925,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
             task.driver.raid.create_configuration(
                 task, create_root_volume=True, create_nonroot_volumes=True)
 
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
             mock_client.create_virtual_disk.assert_has_calls(
                 [mock.call(
                     'RAID.Integrated.1-1',
@@ -975,6 +982,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
                 exception.InvalidParameterValue,
                 task.driver.raid.create_configuration,
                 task)
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
 
     @mock.patch.object(drac_common, 'get_drac_client', spec_set=True,
                        autospec=True)
@@ -1007,6 +1015,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
             task.driver.raid.create_configuration(
                 task, create_root_volume=True, create_nonroot_volumes=True)
 
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
             mock_client.create_virtual_disk.assert_has_calls(
                 [mock.call(
                     'RAID.Integrated.1-1',
@@ -1061,6 +1070,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
                 exception.DracOperationError,
                 task.driver.raid.create_configuration,
                 task, create_root_volume=True, create_nonroot_volumes=True)
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
 
     @mock.patch.object(drac_common, 'get_drac_client', spec_set=True,
                        autospec=True)
@@ -1098,6 +1108,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
             task.driver.raid.create_configuration(
                 task, create_root_volume=True, create_nonroot_volumes=True)
 
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
             mock_client.create_virtual_disk.assert_has_calls(
                 [mock.call(
                     'RAID.Integrated.1-1',
@@ -1163,6 +1174,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
                 exception.DracOperationError,
                 task.driver.raid.create_configuration,
                 task, create_root_volume=True, create_nonroot_volumes=True)
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
 
     @mock.patch.object(drac_common, 'get_drac_client', spec_set=True,
                        autospec=True)
@@ -1198,6 +1210,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
                 exception.DracOperationError,
                 task.driver.raid.create_configuration,
                 task, create_root_volume=True, create_nonroot_volumes=True)
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
 
     @mock.patch.object(drac_common, 'get_drac_client', spec_set=True,
                        autospec=True)
@@ -1235,6 +1248,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
                 exception.DracOperationError,
                 task.driver.raid.create_configuration,
                 task, create_root_volume=True, create_nonroot_volumes=True)
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
 
     @mock.patch.object(drac_common, 'get_drac_client', spec_set=True,
                        autospec=True)
@@ -1269,6 +1283,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
                                   shared=False) as task:
             return_value = task.driver.raid.delete_configuration(task)
 
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
             mock_client.delete_virtual_disk.assert_called_once_with(
                 'Disk.Virtual.0:RAID.Integrated.1-1')
             mock_commit_config.assert_called_once_with(
@@ -1298,6 +1313,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase):
                                   shared=False) as task:
             return_value = task.driver.raid.delete_configuration(task)
 
+            mock_client.wait_until_idrac_is_ready.assert_called_once()
             self.assertEqual(0, mock_client.delete_virtual_disk.call_count)
             self.assertEqual(0, mock_commit_config.call_count)
 

releasenotes/notes/drac-fix-oob-cleaning-b4b717895e243c9b.yaml

@@ -0,0 +1,7 @@
+---
+fixes:
+  - Fixes issue where RAID creation/deletion done as part of out-of-band
+    cleaning would frequently fail when using the iDRAC driver due to
+    Export Configuration job running.  For more information, see
+    https://bugs.launchpad.net/ironic/+bug/1691808.  Note that this fix
+    requires python-dracclient version 1.2.0 or higher.