Add helper to wait until baremetal locks clear

Add a halper to allow the methods to wait until locks
have cleared before proceeding.

Change-Id: I36175cc570fec47484297fd7fc7b6aa7da466d9f

shade/operatorcloud.py

@@ -782,6 +782,53 @@ class OperatorCloud(openstackcloud.OpenStackCloud):
                                             json=patch,
                                             error_message=msg)
 
+    def wait_for_baremetal_node_lock(self, node, timeout=30):
+        """Wait for a baremetal node to have no lock.
+
+        Baremetal nodes in ironic have a reservation lock that
+        is used to represent that a conductor has locked the node
+        while performing some sort of action, such as changing
+        configuration as a result of a machine state change.
+
+        This lock can occur during power syncronization, and prevents
+        updates to objects attached to the node, such as ports.
+
+        In the vast majority of cases, locks should clear in a few
+        seconds, and as such this method will only wait for 30 seconds.
+        The default wait is two seconds between checking if the lock
+        has cleared.
+
+        This method is intended for use by methods that need to
+        gracefully block without genreating errors, however this
+        method does prevent another client or a timer from
+        triggering a lock immediately after we see the lock as
+        having cleared.
+
+        :param node: The json representation of the node,
+                     specificially looking for the node
+                     'uuid' and 'reservation' fields.
+        :param timeout: Integer in seconds to wait for the
+                        lock to clear. Default: 30
+
+        :raises: OpenStackCloudException upon client failure.
+        :returns: None
+        """
+        # TODO(TheJulia): This _can_ still fail with a race
+        # condition in that between us checking the status,
+        # a conductor where the conductor could still obtain
+        # a lock before we are able to obtain a lock.
+        # This means we should handle this with such conections
+
+        if node['reservation'] is None:
+            return
+        else:
+            msg = 'Waiting for lock to be released for node {node}'.format(
+                node=node['uuid'])
+            for count in _utils._iterate_timeout(timeout, msg, 2):
+                current_node = self.get_machine(node['uuid'])
+                if current_node['reservation'] is None:
+                    return
+
     @_utils.valid_kwargs('type', 'service_type', 'description')
     def create_service(self, name, enabled=True, **kwargs):
         """Create a service.

shade/tests/unit/test_baremetal_node.py

@@ -779,6 +779,55 @@ class TestBaremetalNode(base.IronicTestCase):
         self.assertEqual(available_node, return_value)
         self.assert_calls()
 
+    def test_wait_for_baremetal_node_lock_locked(self):
+        self.fake_baremetal_node['reservation'] = 'conductor0'
+        unlocked_node = self.fake_baremetal_node.copy()
+        unlocked_node['reservation'] = None
+        self.register_uris([
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     resource='nodes',
+                     append=[self.fake_baremetal_node['uuid']]),
+                 json=self.fake_baremetal_node),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     resource='nodes',
+                     append=[self.fake_baremetal_node['uuid']]),
+                 json=unlocked_node),
+        ])
+        self.assertIsNone(
+            self.op_cloud.wait_for_baremetal_node_lock(
+                self.fake_baremetal_node,
+                timeout=1))
+
+        self.assert_calls()
+
+    def test_wait_for_baremetal_node_lock_not_locked(self):
+        self.fake_baremetal_node['reservation'] = None
+        self.assertIsNone(
+            self.op_cloud.wait_for_baremetal_node_lock(
+                self.fake_baremetal_node,
+                timeout=1))
+
+        self.assertEqual(0, len(self.adapter.request_history))
+
+    def test_wait_for_baremetal_node_lock_timeout(self):
+        self.fake_baremetal_node['reservation'] = 'conductor0'
+        self.register_uris([
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     resource='nodes',
+                     append=[self.fake_baremetal_node['uuid']]),
+                 json=self.fake_baremetal_node),
+        ])
+        self.assertRaises(
+            exc.OpenStackCloudException,
+            self.op_cloud.wait_for_baremetal_node_lock,
+            self.fake_baremetal_node,
+            timeout=0.001)
+
+        self.assert_calls()
+
     def test_activate_node(self):
         self.fake_baremetal_node['provision_state'] = 'active'
         self.register_uris([