Check node trait before creating server

Placement may fail to update trait because of Conflict
the trait may be updated by the Nova compute
update_available_resource periodic task.
We need node status is enabled, so we check the node
trait and delay if it is not the correct status.
the max delay time is 10 minutes.

Depends-on: Ie98e2c000568ecac63e8981a8b7087029c0d3705

Change-Id: I3e45d4a66a6e1bf55499def8550da38ddf01b638

requirements.txt

@@ -7,3 +7,4 @@ six>=1.10.0 # MIT
 oslo.log>=3.36.0 # Apache-2.0
 oslo.utils>=3.33.0 # Apache-2.0
 tempest>=17.1.0 # Apache-2.0
+os-traits>=0.15.0 # Apache-2.0

watcher_tempest_plugin/infra_optim_clients.py

@@ -16,10 +16,13 @@
 
 import abc
 
+from oslo_serialization import jsonutils as json
 import six
 from tempest import clients
 from tempest.common import credentials_factory as creds_factory
 from tempest import config
+from tempest.lib.common import rest_client
+from tempest.lib.services.placement import placement_client
 
 from watcher_tempest_plugin.services.infra_optim.v1.json import client as ioc
 from watcher_tempest_plugin.services.metric.v1.json import client as gc
@@ -36,6 +39,8 @@ class BaseManager(clients.Manager):
             self.auth_provider, 'infra-optim', CONF.identity.region)
         self.gn_client = gc.GnocchiClientJSON(
             self.auth_provider, 'metric', CONF.identity.region)
+        self.placement_client = ExtendPlacementClient(
+            self.auth_provider, 'placement', CONF.identity.region)
 
 
 class AdminManager(BaseManager):
@@ -43,3 +48,20 @@ class AdminManager(BaseManager):
         super(AdminManager, self).__init__(
             creds_factory.get_configured_admin_credentials(),
         )
+
+
+class ExtendPlacementClient(placement_client.PlacementClient):
+
+    def list_provider_traits(self, rp_uuid):
+        """List resource provider traits.
+
+        For full list of available parameters, please refer to the official
+        API reference:
+        https://docs.openstack.org/api-ref/placement/#
+        list-resource-provider-traits-detail
+        """
+        url = '/resource_providers/%s/traits' % rp_uuid
+        resp, body = self.get(url)
+        self.expected_success(200, resp.status)
+        body = json.loads(body)
+        return rest_client.ResponseBody(resp, body)

watcher_tempest_plugin/tests/scenario/base.py

@@ -24,6 +24,7 @@ import time
 
 from datetime import datetime
 from datetime import timedelta
+import os_traits
 from oslo_log import log
 from tempest.common import waiters
 from tempest import config
@@ -60,11 +61,14 @@ class BaseInfraOptimScenarioTest(manager.ScenarioTest):
         super(BaseInfraOptimScenarioTest, cls).setup_clients()
         cls.client = cls.mgr.io_client
         cls.gnocchi = cls.mgr.gn_client
+        cls.placement_client = cls.mgr.placement_client
 
     def setUp(self):
         super(BaseInfraOptimScenarioTest, self).setUp()
         self.useFixture(api_microversion_fixture.APIMicroversionFixture(
             compute_microversion=CONF.compute.min_microversion))
+        self.useFixture(api_microversion_fixture.APIMicroversionFixture(
+            placement_microversion=CONF.placement.min_microversion))
 
     @classmethod
     def resource_setup(cls):
@@ -202,8 +206,28 @@ class BaseInfraOptimScenarioTest(manager.ScenarioTest):
         if instances:
             return instances
 
+        hypervisors = self.get_hypervisors_setup()
         created_instances = []
-        for _ in compute_nodes[:CONF.compute.min_compute_nodes]:
+        for node in compute_nodes[:CONF.compute.min_compute_nodes]:
+            hyp_id = [
+                hyp['id'] for hyp in hypervisors
+                if hyp['hypervisor_hostname'] == node['host']]
+            # Placement may fail to update trait because of Conflict
+            # the trait may be updated by the Nova compute
+            # update_available_resource periodic task.
+            # We need node status is enabled, so we check the node
+            # trait and delay if it is not the correct status.
+            # the max delay time is 10 minutes.
+            node_trait = os_traits.COMPUTE_STATUS_DISABLED
+            retry = 20
+            trait_status = True
+            while trait_status and retry:
+                trait_status = self.check_node_trait(hyp_id[0], node_trait)
+                if not trait_status:
+                    break
+                time.sleep(30)
+                retry -= 1
+            self.assertNotEqual(0, retry)
             # by getting to active state here, this means this has
             # landed on the host in question.
             instance = self.create_server(image_id=CONF.compute.image_ref,
@@ -520,3 +544,16 @@ class BaseInfraOptimScenarioTest(manager.ScenarioTest):
 
         for action in action_list['actions']:
             self.assertEqual('SUCCEEDED', action.get('state'))
+
+    def check_node_trait(self, node_id, trait):
+        """Check if trait is in node traits
+
+        :param node_id: The unique identifier of the node.
+        :param trait: node trait
+        :return: True if node has the trait else False
+        """
+        traits = self.placement_client.list_provider_traits(node_id)
+        if trait in traits.get('traits'):
+            return True
+        else:
+            return False