diff --git a/ironic/drivers/modules/drac/client.py b/ironic/drivers/modules/drac/client.py index 3b4b889293..3b40bd8d66 100644 --- a/ironic/drivers/modules/drac/client.py +++ b/ironic/drivers/modules/drac/client.py @@ -15,15 +15,20 @@ Wrapper for pywsman.Client """ +import time from xml.etree import ElementTree from oslo_utils import importutils from ironic.common import exception +from ironic.common.i18n import _LW from ironic.drivers.modules.drac import common as drac_common +from ironic.openstack.common import log as logging pywsman = importutils.try_import('pywsman') +LOG = logging.getLogger(__name__) + _SOAP_ENVELOPE_URI = 'http://www.w3.org/2003/05/soap-envelope' # Filter Dialects, see (Section 2.3.1): @@ -36,6 +41,9 @@ RET_SUCCESS = '0' RET_ERROR = '2' RET_CREATED = '4096' +RETRY_COUNT = 5 +RETRY_DELAY = 5 + def get_wsman_client(node): """Return a DRAC client object. @@ -53,6 +61,29 @@ def get_wsman_client(node): return client +def retry_on_empty_response(client, action, *args, **kwargs): + """Wrapper to retry an action on failure.""" + + func = getattr(client, action) + for i in range(RETRY_COUNT): + response = func(*args, **kwargs) + if response: + return response + else: + LOG.warning(_LW('Empty response on calling %(action)s on client. ' + 'Last error (cURL error code): %(last_error)s, ' + 'fault string: "%(fault_string)s" ' + 'response_code: %(response_code)s. ' + 'Retry attempt %(count)d') % + {'action': action, + 'last_error': client.last_error(), + 'fault_string': client.fault_string(), + 'response_code': client.response_code(), + 'count': i + 1}) + + time.sleep(RETRY_DELAY) + + class Client(object): def __init__(self, drac_host, drac_port, drac_path, drac_protocol, @@ -96,15 +127,16 @@ class Client(object): options.set_flags(pywsman.FLAG_ENUMERATION_OPTIMIZATION) options.set_max_elements(100) - doc = self.client.enumerate(options, filter_, resource_uri) + doc = retry_on_empty_response(self.client, 'enumerate', + options, filter_, resource_uri) root = self._get_root(doc) final_xml = root find_query = './/{%s}Body' % _SOAP_ENVELOPE_URI insertion_point = final_xml.find(find_query) while doc.context() is not None: - doc = self.client.pull(options, None, resource_uri, - str(doc.context())) + doc = retry_on_empty_response(self.client, 'pull', options, None, + resource_uri, str(doc.context())) root = self._get_root(doc) for result in root.findall(find_query): for child in list(result): @@ -160,7 +192,9 @@ class Client(object): for name, value in properties.items(): options.add_property(name, value) - doc = self.client.invoke(options, resource_uri, method, xml_doc) + doc = retry_on_empty_response(self.client, 'invoke', options, + resource_uri, method, xml_doc) + root = self._get_root(doc) return_value = drac_common.find_xml(root, 'ReturnValue', diff --git a/ironic/tests/drivers/drac/test_client.py b/ironic/tests/drivers/drac/test_client.py index a5df57bd62..6d3edd58e1 100644 --- a/ironic/tests/drivers/drac/test_client.py +++ b/ironic/tests/drivers/drac/test_client.py @@ -51,6 +51,24 @@ class DracClientTestCase(base.TestCase): None, self.resource_uri) mock_xml.context.assert_called_once_with() + def test_wsman_enumerate_retry(self, mock_client_pywsman): + mock_xml = test_utils.mock_wsman_root('') + mock_pywsman_client = mock_client_pywsman.Client.return_value + mock_pywsman_client.enumerate.side_effect = [None, mock_xml] + + client = drac_client.Client(**INFO_DICT) + client.wsman_enumerate(self.resource_uri) + + mock_options = mock_client_pywsman.ClientOptions.return_value + mock_options.set_flags.assert_called_once_with( + mock_client_pywsman.FLAG_ENUMERATION_OPTIMIZATION) + mock_options.set_max_elements.assert_called_once_with(100) + mock_pywsman_client.enumerate.assert_has_calls([ + mock.call(mock_options, None, self.resource_uri), + mock.call(mock_options, None, self.resource_uri) + ]) + mock_xml.context.assert_called_once_with() + def test_wsman_enumerate_with_additional_pull(self, mock_client_pywsman): mock_root = mock.Mock() mock_root.string.side_effect = [test_utils.build_soap_xml( @@ -118,6 +136,23 @@ class DracClientTestCase(base.TestCase): mock_pywsman_client.invoke.assert_called_once_with(mock_options, self.resource_uri, method_name, None) + def test_wsman_invoke_retry(self, mock_client_pywsman): + result_xml = test_utils.build_soap_xml( + [{'ReturnValue': drac_client.RET_SUCCESS}], self.resource_uri) + mock_xml = test_utils.mock_wsman_root(result_xml) + mock_pywsman_client = mock_client_pywsman.Client.return_value + mock_pywsman_client.invoke.side_effect = [None, mock_xml] + + method_name = 'method' + client = drac_client.Client(**INFO_DICT) + client.wsman_invoke(self.resource_uri, method_name) + + mock_options = mock_client_pywsman.ClientOptions.return_value + mock_pywsman_client.invoke.assert_has_calls([ + mock.call(mock_options, self.resource_uri, method_name, None), + mock.call(mock_options, self.resource_uri, method_name, None) + ]) + def test_wsman_invoke_with_selectors(self, mock_client_pywsman): result_xml = test_utils.build_soap_xml( [{'ReturnValue': drac_client.RET_SUCCESS}], self.resource_uri)