Merge "DRAC: add retry capability to wsman client operations"

ironic/drivers/modules/drac/client.py

@@ -15,15 +15,20 @@
 Wrapper for pywsman.Client
 """
 
+import time
 from xml.etree import ElementTree
 
 from oslo_utils import importutils
 
 from ironic.common import exception
+from ironic.common.i18n import _LW
 from ironic.drivers.modules.drac import common as drac_common
+from ironic.openstack.common import log as logging
 
 pywsman = importutils.try_import('pywsman')
 
+LOG = logging.getLogger(__name__)
+
 _SOAP_ENVELOPE_URI = 'http://www.w3.org/2003/05/soap-envelope'
 
 # Filter Dialects, see (Section 2.3.1):
@@ -36,6 +41,9 @@ RET_SUCCESS = '0'
 RET_ERROR = '2'
 RET_CREATED = '4096'
 
+RETRY_COUNT = 5
+RETRY_DELAY = 5
+
 
 def get_wsman_client(node):
     """Return a DRAC client object.
@@ -53,6 +61,29 @@ def get_wsman_client(node):
     return client
 
 
+def retry_on_empty_response(client, action, *args, **kwargs):
+    """Wrapper to retry an action on failure."""
+
+    func = getattr(client, action)
+    for i in range(RETRY_COUNT):
+        response = func(*args, **kwargs)
+        if response:
+            return response
+        else:
+            LOG.warning(_LW('Empty response on calling %(action)s on client. '
+                            'Last error (cURL error code): %(last_error)s, '
+                            'fault string: "%(fault_string)s" '
+                            'response_code: %(response_code)s. '
+                            'Retry attempt %(count)d') %
+                        {'action': action,
+                         'last_error': client.last_error(),
+                         'fault_string': client.fault_string(),
+                         'response_code': client.response_code(),
+                         'count': i + 1})
+
+            time.sleep(RETRY_DELAY)
+
+
 class Client(object):
 
     def __init__(self, drac_host, drac_port, drac_path, drac_protocol,
@@ -96,15 +127,16 @@ class Client(object):
         options.set_flags(pywsman.FLAG_ENUMERATION_OPTIMIZATION)
         options.set_max_elements(100)
 
-        doc = self.client.enumerate(options, filter_, resource_uri)
+        doc = retry_on_empty_response(self.client, 'enumerate',
+                                      options, filter_, resource_uri)
         root = self._get_root(doc)
 
         final_xml = root
         find_query = './/{%s}Body' % _SOAP_ENVELOPE_URI
         insertion_point = final_xml.find(find_query)
         while doc.context() is not None:
-            doc = self.client.pull(options, None, resource_uri,
-                                   str(doc.context()))
+            doc = retry_on_empty_response(self.client, 'pull', options, None,
+                                          resource_uri, str(doc.context()))
             root = self._get_root(doc)
             for result in root.findall(find_query):
                 for child in list(result):
@@ -160,7 +192,9 @@ class Client(object):
             for name, value in properties.items():
                 options.add_property(name, value)
 
-        doc = self.client.invoke(options, resource_uri, method, xml_doc)
+        doc = retry_on_empty_response(self.client, 'invoke', options,
+                                      resource_uri, method, xml_doc)
+
         root = self._get_root(doc)
 
         return_value = drac_common.find_xml(root, 'ReturnValue',

ironic/tests/drivers/drac/test_client.py

@@ -51,6 +51,24 @@ class DracClientTestCase(base.TestCase):
             None, self.resource_uri)
         mock_xml.context.assert_called_once_with()
 
+    def test_wsman_enumerate_retry(self, mock_client_pywsman):
+        mock_xml = test_utils.mock_wsman_root('<test></test>')
+        mock_pywsman_client = mock_client_pywsman.Client.return_value
+        mock_pywsman_client.enumerate.side_effect = [None, mock_xml]
+
+        client = drac_client.Client(**INFO_DICT)
+        client.wsman_enumerate(self.resource_uri)
+
+        mock_options = mock_client_pywsman.ClientOptions.return_value
+        mock_options.set_flags.assert_called_once_with(
+            mock_client_pywsman.FLAG_ENUMERATION_OPTIMIZATION)
+        mock_options.set_max_elements.assert_called_once_with(100)
+        mock_pywsman_client.enumerate.assert_has_calls([
+            mock.call(mock_options, None, self.resource_uri),
+            mock.call(mock_options, None, self.resource_uri)
+        ])
+        mock_xml.context.assert_called_once_with()
+
     def test_wsman_enumerate_with_additional_pull(self, mock_client_pywsman):
         mock_root = mock.Mock()
         mock_root.string.side_effect = [test_utils.build_soap_xml(
@@ -118,6 +136,23 @@ class DracClientTestCase(base.TestCase):
         mock_pywsman_client.invoke.assert_called_once_with(mock_options,
             self.resource_uri, method_name, None)
 
+    def test_wsman_invoke_retry(self, mock_client_pywsman):
+        result_xml = test_utils.build_soap_xml(
+            [{'ReturnValue': drac_client.RET_SUCCESS}], self.resource_uri)
+        mock_xml = test_utils.mock_wsman_root(result_xml)
+        mock_pywsman_client = mock_client_pywsman.Client.return_value
+        mock_pywsman_client.invoke.side_effect = [None, mock_xml]
+
+        method_name = 'method'
+        client = drac_client.Client(**INFO_DICT)
+        client.wsman_invoke(self.resource_uri, method_name)
+
+        mock_options = mock_client_pywsman.ClientOptions.return_value
+        mock_pywsman_client.invoke.assert_has_calls([
+            mock.call(mock_options, self.resource_uri, method_name, None),
+            mock.call(mock_options, self.resource_uri, method_name, None)
+        ])
+
     def test_wsman_invoke_with_selectors(self, mock_client_pywsman):
         result_xml = test_utils.build_soap_xml(
             [{'ReturnValue': drac_client.RET_SUCCESS}], self.resource_uri)