Wait for resources to become available

This will wait for any resources that are created to become available
before exiting the script. This allows you to avoid a race condition
where a server could be created before the resource tracker had been
updated with the new resources; server creation would fail.

Change-Id: I57f8c93cb1ebbc284b96ef1ced2c4edd59b27795
Story: 2004274
Task: 27823
Depends-On: https://review.openstack.org/617642

ansible/deploy.yml

@@ -28,5 +28,9 @@
   import_playbook: flavor_registration.yml
   tags: openstack
 
+- name: Wait for resources to become available
+  import_playbook: resource_wait.yml
+  tags: openstack, resource
+
 - name: Clean up Tenks state
   import_playbook: cleanup_state.yml

ansible/resource_wait.yml

@@ -0,0 +1,46 @@
+---
+- hosts: localhost
+  tasks:
+    - name: Check that OpenStack credentials exist in the environment
+      fail:
+        msg: >
+          $OS_USERNAME was not found in the environment. Ensure the OpenStack
+          credentials exist in your environment, perhaps by sourcing your RC file.
+      when: not lookup('env', 'OS_USERNAME')
+
+    - name: Gather list of OpenStack services
+      command: >-
+        {{ virtualenv_path }}/bin/openstack service list -f json
+      register: service_list_output
+      changed_when: false
+
+    - block:
+      - name: Set default value for expected resources
+        set_fact:
+          tenks_expected_resources: []
+
+      - name: Build list of expected resources
+        # takes the form: [{ resource_class: CUSTOM_TEST_RC, amount: 2, traits: [] }, ]
+        vars:
+          resource:
+            amount: "{{ spec.count | int }}"  # this gets converted back to a string
+            resource_class: "{{ 'CUSTOM_' ~ spec.ironic_config.resource_class | upper | replace('-', '_') }}"
+            traits: "{{ spec.ironic_config.traits | default([])}}"
+        set_fact:
+          tenks_expected_resources: >-
+            {{ tenks_expected_resources + [resource] }}
+        loop: "{{ specs }}"
+        when: "'ironic_config' in spec"
+        loop_control:
+          loop_var: spec
+
+      - name: Include the wait-for-resources role
+        include_role:
+          name: wait-for-resources
+        vars:
+          wait_for_resources_required_resources: "{{ tenks_expected_resources }}"
+          wait_for_resources_venv: "{{ virtualenv_path }}"
+          wait_for_resources_python_upper_constraints_url: >-
+            {{ python_upper_constraints_url }}
+      # Only attempt to wait for resources when the placement service is running
+      when: service_list_output.stdout | from_json | selectattr('Type', 'equalto', 'placement') | list | length >= 1

ansible/roles/wait-for-resources/defaults/main.yml

@@ -0,0 +1,28 @@
+---
+
+# A list of dictionaries, each with the following keys:
+# * resource_class - the name of the resource class that you want to be
+#   available
+# * traits - a list of traits a provider requires before that provider is
+#   considered to provide that resource
+# * amount - natural number representing the amount of the resource that
+#   should be available. This will be tested against the total resource count
+#   minus the amount that is reserved.
+wait_for_resources_required_resources:
+
+# Virtualenv containing OpenStack client
+wait_for_resources_venv:
+
+# The resources are polled this many times. The delay between each iteration
+# is also configurable, but defaults to 10 seconds
+wait_for_resources_retry_limit: 15
+
+# The delay between each iteration.
+wait_for_resources_delay:
+
+# The URL of the upper constraints file to pass to pip when installing Python
+# packages.
+wait_for_resources_python_upper_constraints_url:
+
+# Path on the remote machine to copy the python requirements file to
+wait_for_resources_python_requirements_path: /tmp/wait-for-python-requirements.txt

ansible/roles/wait-for-resources/files/requirements.txt

@@ -0,0 +1,4 @@
+# This file contains the Python packages that are needed in the
+# what-for-resource virtual env.
+
+osc-placement>=1.3.0 # Apache License 2.0

ansible/roles/wait-for-resources/library/wait_for_resources.py

@@ -0,0 +1,383 @@
+#!/usr/bin/python
+
+# Copyright (c) 2018 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+DOCUMENTATION = """
+module: wait_for_resources
+short_description: Waits for a set of resources to become available in
+placement
+author: Will Szumski(will@stackhpc.com)
+options:
+  - option-name: resources
+    description: List of dictionaries describing the resources that should
+    be available. Each dictionary should contain the keys: resource_class,
+    amount, and optionally, a list of traits that the provider needs to provide
+    before it is considered to provide the resource.
+    required: True
+    type: list
+  - option-name: venv
+    description: Path to a virtualenv containing the OpenStack CLI. It is a
+    requirement for the placement api plugin (osc-placement) to be installed.
+    required: False
+    type: string
+  - option-name: maximum_retries
+    description: The maximum number of iterations to poll for the resources
+    required: False
+    type: int
+  - option-name: delay
+    description: Delay between each iteration of the polling loop in seconds
+    required: False
+    type: int
+requirements:
+  - python-openstackclient
+  - osc-placement
+"""
+
+EXAMPLES = """
+- name: Wait for resources to become available
+  wait_for_resources:
+    resources: {'resource_class: CUSTOM_TEST_RC, 'traits': [], 'amount': 2}
+    delay: 10
+    maximum_retries: 15
+    venv: /path/to/venv
+"""
+
+RETURN = """
+iterations:
+  description: The Number of iterations required before the resource became
+  available
+  returned: success
+  type: int
+  sample: 9
+"""
+
+# Need to disable PEP8, as it wants all imports at top of file
+from ansible.module_utils.basic import AnsibleModule  # noqa
+from collections import namedtuple  # noqa
+import six  # noqa
+import os  # noqa
+import time  # noqa
+
+_RETRY_LIMIT_FAILURE_TEMPLATE = "exceeded retry limit of {max_retries} " \
+                                "whilst waiting for resources to become " \
+                                "available"
+
+Specifier = namedtuple("Specifier", "name traits")
+Provider = namedtuple("Provider", "uuid inventory_list traits")
+
+# Store a list of import errors to report to the user.
+IMPORT_ERRORS = []
+try:
+    import json
+except Exception as e:
+    IMPORT_ERRORS.append(e)
+
+
+def meets_criteria(actual, requested):
+    """For each resource, determines whether the total satisfies the amount
+    requested, ignoring any unrequested resource_classes"""
+    for specifier in requested.keys():
+        if specifier not in actual:
+            return False
+        if actual[specifier] < requested[specifier]:
+            return False
+    return True
+
+
+def get_openstack_binary_path(module):
+    """Returns the path to the openstack binary taking into account the
+    virtualenv that was specified (if available)"""
+    venv = module.params["venv"]
+    if venv:
+        return os.path.join(venv, "bin", "openstack")
+    # use openstack in PATH
+    return "openstack"
+
+
+def get_inventory(module, provider_uuid):
+    """
+    Gets inventory of resources for a give provider UUID
+    :param module: ansible module
+    :param provider_uuid: provider to query
+    :return: list of dictionaries of the form:
+    [
+      {
+        "allocation_ratio": 16.0,
+        "total": 24,
+        "reserved": 0,
+        "resource_class": "VCPU",
+        "step_size": 1,
+        "min_unit": 1,
+        "max_unit": 24
+      },
+    ]
+    """
+    cmd = "{openstack} resource provider inventory list {uuid} -f json" \
+        .format(uuid=provider_uuid,
+                openstack=get_openstack_binary_path(module))
+
+    rc, out, err = module.run_command(cmd)
+    if rc != 0:
+        msg = "{} failed with return code: {}, stderr: {}".format(cmd, rc, err)
+        module.fail_json(msg=msg)
+    return json.loads(out)
+
+
+def get_providers(module):
+    """
+    Gets a list of resource providers
+    :param module: ansible module
+    :return: list of dictionaries of the form:
+    [
+      {
+        "generation": 2,
+        "uuid": "657c4ab0-de82-4def-b7b0-d13ce672bfd0",
+        "name": "kayobe-will-master"
+      },
+    ]
+    """
+    cmd = "{openstack} resource provider list -f json".format(
+        openstack=get_openstack_binary_path(module)
+    )
+    rc, out, err = module.run_command(cmd)
+    if rc != 0:
+        msg = "{} failed with return code: {}, stderr: {}".format(cmd, rc, err)
+        module.fail_json(msg=msg)
+    return json.loads(out)
+
+
+def get_traits(module, provider_uuid):
+    """
+    Gets a list of traits for a resource provider
+    :param provider_uuid: the uuid of the provider
+    :param module: ansible module
+    :return: set of traits of the form:
+    {
+        "HW_CPU_X86_SSE2",
+    }
+    """
+    cmd = "{openstack} resource provider trait list {uuid} " \
+          "--os-placement-api-version 1.6 -f json " \
+        .format(uuid=provider_uuid,
+                openstack=get_openstack_binary_path(module))
+
+    rc, out, err = module.run_command(cmd)
+    if rc != 0:
+        msg = "{} failed with return code: {}, stderr: {}".format(cmd, rc, err)
+        module.fail_json(msg=msg)
+    raw = json.loads(out)
+    return set([x["name"] for x in raw])
+
+
+def merge(x, y, f):
+    """ Merges two dictionaries. If a key appears in both dictionaries, the
+    common values are merged using the function ``f``"""
+    # Start with symmetric difference; keys either in A or B, but not both
+    merged = {k: x.get(k, y.get(k)) for k in six.viewkeys(x) ^ six.viewkeys(y)}
+    # Update with `f()` applied to the intersection
+    merged.update(
+        {k: f(x[k], y[k]) for k in six.viewkeys(x) & six.viewkeys(y)})
+    return merged
+
+
+def collect(specifiers, provider):
+    """Given a specifier and a provider, gets the amount of resource that is
+     available for that given provider"""
+    inventory = {}
+
+    # we want to be able to look up items by resource_class
+    for item in provider.inventory_list:
+        inventory[item["resource_class"]] = item
+    result = {}
+    for specifier in specifiers:
+        if specifier.traits != provider.traits:
+            continue
+        if specifier.name in inventory:
+            reserved = inventory[specifier.name]["reserved"]
+            total_available = inventory[specifier.name]["total"]
+            result[specifier] = total_available - reserved
+            break
+    return result
+
+
+def get_totals(specifiers, providers):
+    """Loops over the providers adding up all of the resources that
+    are available"""
+    totals = {}
+
+    for specifier in specifiers:
+        # initialise totals so the combine function does not blow up
+        totals[specifier] = 0
+
+    for provider in providers:
+        current = collect(specifiers, provider)
+        totals = merge(totals, current, lambda x, y: x + y)
+
+    return totals
+
+
+def are_resources_available(module, specifiers, expected):
+    """
+    Determines whether or not a set of resources are available
+    :param module: Ansible module
+    :param specifiers: tuples of the form: (resource_class, traits)
+    :param expected: dictionary of the form:
+    {(resource_class, traits): amount }
+    :return: True if resource available, otherwise False
+    """
+    providers_raw = get_providers(module)
+    providers = []
+    for provider in providers_raw:
+        uuid = provider["uuid"]
+        traits = get_traits(module, uuid)
+        inventory = get_inventory(module, uuid)
+        provider = Provider(
+            uuid=uuid,
+            inventory_list=inventory,
+            traits=traits
+        )
+        providers.append(provider)
+    actual = get_totals(specifiers, providers)
+    return meets_criteria(actual, expected)
+
+
+def wait_for_resources(module):
+    """
+    Waits for a set of resources to become available
+    :param module: Ansible module
+    :return: the number of attempts needed
+    """
+    max_retries = module.params["maximum_retries"]
+    delay = module.params["delay"]
+    resources = module.params["resources"]
+    expected = {}
+    specifiers = []
+    for resource in resources:
+        # default value for traits
+        traits = resource["traits"] if resource["traits"] else []
+        specifier = Specifier(name=resource["resource_class"],
+                              traits=frozenset(traits))
+        specifiers.append(specifier)
+        expected[specifier] = resource["amount"]
+
+    for i in range(max_retries):
+        if are_resources_available(module, specifiers, expected):
+            return i + 1
+        time.sleep(delay)
+
+    fail_msg = _RETRY_LIMIT_FAILURE_TEMPLATE.format(
+        max_retries=max_retries)
+    module.fail_json(msg=fail_msg)
+
+
+def validate_spec(module, resource, field, type_):
+    check_resource_msg = (
+        "Please check the dictionaries in your resources list meet the "
+        "required specification.")
+    unset_field_msg = (
+        "One of your resources does not have the field, {field}, set."
+    )
+    field_type_msg = (
+        "The field, {field}, should be type {type}."
+    )
+    if field not in resource:
+        msg = "{} {}".format(unset_field_msg, check_resource_msg)
+        module.fail_json(msg=msg.format(field=field))
+    elif not isinstance(resource[field], type_):
+        msg = "{} {}".format(field_type_msg, check_resource_msg)
+        module.fail_json(msg=msg.format(field=field, type=type_.__name__))
+
+
+def validate_resources(module, specs):
+    resources = module.params["resources"]
+    for resource in resources:
+        for spec in specs:
+            validate_spec(module, resource, spec["field"], type_=spec["type"])
+
+
+def get_module():
+    """Creates and returns an Ansible module"""
+    module = AnsibleModule(
+        argument_spec=dict(
+            resources=dict(required=True, type='list'),
+            maximum_retries=dict(required=False, type='int', default=15),
+            delay=dict(required=False, type='int', default=10),
+            venv=dict(required=False, type='str', default="")
+        ),
+        supports_check_mode=True,
+    )
+
+    # note(wszumski): amount seems to get converted back to a string
+    # https://github.com/ansible/ansible/issues/18095
+    resources = module.params["resources"]
+    for resource in resources:
+        if 'amount' not in resource:
+            continue
+        amount = resource["amount"]
+        if isinstance(amount, str):
+            try:
+                amount_int = int(amount)
+                resource["amount"] = amount_int
+            except ValueError:
+                # this will get picked up in validate_resources
+                pass
+
+    # Validate resources list
+    resources_specs = [
+        {
+            'field': 'resource_class',
+            'type': str
+        },
+        {
+            'field': 'traits',
+            'type': list
+        },
+        {
+            'field': 'amount',
+            'type': int
+        },
+    ]
+
+    validate_resources(module, resources_specs)
+
+    # Fail if there were any exceptions when importing modules.
+    if IMPORT_ERRORS:
+        module.fail_json(msg="Import errors: %s" %
+                             ", ".join([repr(e) for e in IMPORT_ERRORS]))
+
+    return module
+
+
+def main():
+    module = get_module()
+
+    attempts_needed = 0
+
+    # In check mode, only perform validation of parameters
+    if not module.check_mode:
+        attempts_needed = wait_for_resources(module)
+
+    # This module doesn't really change anything, it just waits for
+    # something to change externally
+    result = {
+        "changed": False,
+        "iterations": attempts_needed
+    }
+
+    module.exit_json(**result)
+
+
+if __name__ == '__main__':
+    main()

ansible/roles/wait-for-resources/tasks/main.yml

@@ -0,0 +1,38 @@
+---
+
+# Waits for for the resource tracker to be updated with a given resource
+# class
+
+- name: Validate that the virtualenv variable is set
+  fail:
+    msg: >-
+      You must set the variable, wait_for_resources_venv, to use this role.
+  when: wait_for_resources_venv is none
+
+- name: Validate resources
+  include_tasks: validate.yml
+  vars:
+    wait_for_resources_amount: "{{ item.amount }}"
+    wait_for_resources_resource_class: "{{ item.resource_class }}"
+  loop: "{{ wait_for_resources_required_resources }}"
+
+- name: Copy over requirements file
+  copy:
+    src: requirements.txt
+    dest: "{{ wait_for_resources_python_requirements_path }}"
+
+- name: Install dependencies in supplied venv
+  pip:
+    requirements: "{{ wait_for_resources_python_requirements_path }}"
+    extra_args: >-
+      {%- if wait_for_resources_python_upper_constraints_url -%}
+      -c {{ wait_for_resources_python_upper_constraints_url }}
+      {%- endif -%}
+    virtualenv: "{{ wait_for_resources_venv }}"
+
+- name: Call wait_for_resources module
+  wait_for_resources:
+    venv: "{{ wait_for_resources_venv }}"
+    resources: "{{ wait_for_resources_required_resources }}"
+    delay: "{{ wait_for_resources_delay | default(omit, true) }}"
+    maximum_retries: "{{ wait_for_resources_retry_limit }}"

ansible/roles/wait-for-resources/tasks/validate.yml

@@ -0,0 +1,17 @@
+---
+
+- name: Validate that the target resource class is set
+  fail:
+    msg: >-
+      Each item in the dictionary wait_for_resources_required_resources must contain
+      the key, resource_class.
+  when: wait_for_resources_class is none
+
+- name: Validate expected resource count
+  vars:
+    count: "{{ wait_for_resources_amount | int(default=-1) }}"
+  fail:
+    msg: >-
+      Each item in the dictionary wait_for_resources_required_resources must contain
+      the key, amount.
+  when: wait_for_resources_amount is none or count | int < 0

tests/test_tenks_wait_for_resources.py

@@ -0,0 +1,674 @@
+# Copyright (c) 2018 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+import copy
+import imp
+import json
+import os
+import random
+from itertools import repeat, chain, cycle
+
+from ansible.module_utils import basic
+
+from tests.utils import ModuleTestCase, set_module_args, AnsibleExitJson, \
+    AnsibleFailJson
+
+# Python 2/3 compatibility.
+try:
+    from unittest.mock import MagicMock, patch
+except ImportError:
+    from mock import MagicMock, patch  # noqa
+
+# Import method lifted from kolla_ansible's test_merge_config.py
+PROJECT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
+PLUGIN_FILE = os.path.join(PROJECT_DIR,
+                           'ansible/roles/wait-for-resources/library'
+                           '/wait_for_resources.py')
+
+wait_for = imp.load_source('wait_for_resources', PLUGIN_FILE)
+
+meets_criteria = wait_for.meets_criteria
+get_providers = wait_for.get_providers
+get_inventory = wait_for.get_inventory
+get_traits = wait_for.get_traits
+merge = wait_for.merge
+get_openstack_binary_path = wait_for.get_openstack_binary_path
+Specifier = wait_for.Specifier
+
+inventory_list_out = """[
+  {
+    "allocation_ratio": 1.0, 
+    "total": 1, 
+    "reserved": 0, 
+    "resource_class": "CUSTOM_TEST_RC", 
+    "step_size": 1, 
+    "min_unit": 1, 
+    "max_unit": 1
+  }
+]"""  # noqa
+
+inventory_custom_b_out = """[
+  {
+    "allocation_ratio": 1.0, 
+    "total": 1, 
+    "reserved": 0, 
+    "resource_class": "CUSTOM_B", 
+    "step_size": 1, 
+    "min_unit": 1, 
+    "max_unit": 1
+  }
+]"""  # noqa
+
+inventory_reserved_out = """[
+  {
+    "allocation_ratio": 1.0, 
+    "total": 1, 
+    "reserved": 1, 
+    "resource_class": "CUSTOM_TEST_RC", 
+    "step_size": 1, 
+    "min_unit": 1, 
+    "max_unit": 1
+  }
+]""" # noqa
+
+inventory_list = [{'allocation_ratio': 1.0,
+                   'max_unit': 1,
+                   'min_unit': 1,
+                   'reserved': 0,
+                   'resource_class': 'CUSTOM_TEST_RC',
+                   'step_size': 1,
+                   'total': 1}]
+
+resource_provider_list_out = """[
+  {
+    "generation": 2, 
+    "uuid": "657c4ab0-de82-4def-b7b0-d13ce672bfd0", 
+    "name": "kayobe-will-master"
+  }, 
+  {
+    "generation": 1, 
+    "uuid": "e2e78f98-d3ec-466a-862b-42d7ef5dca7d", 
+    "name": "e2e78f98-d3ec-466a-862b-42d7ef5dca7d"
+  }, 
+  {
+    "generation": 1, 
+    "uuid": "07072aea-cc2b-4135-a7e2-3a4dd3a9f629", 
+    "name": "07072aea-cc2b-4135-a7e2-3a4dd3a9f629"
+  }
+]""" # noqa
+
+resource_provider_list = [{'generation': 2, 'name': 'kayobe-will-master',
+                           'uuid': '657c4ab0-de82-4def-b7b0-d13ce672bfd0'},
+                          {'generation': 1,
+                           'name': 'e2e78f98-d3ec-466a-862b-42d7ef5dca7d',
+                           'uuid': 'e2e78f98-d3ec-466a-862b-42d7ef5dca7d'},
+                          {'generation': 1,
+                           'name': '07072aea-cc2b-4135-a7e2-3a4dd3a9f629',
+                           'uuid': '07072aea-cc2b-4135-a7e2-3a4dd3a9f629'}]
+
+resource_provider_traits_out = """[
+  {
+    "name": "HW_CPU_X86_SSE2"
+  }, 
+  {
+    "name": "HW_CPU_X86_CLMUL"
+  }, 
+  {
+    "name": "HW_CPU_X86_SSE"
+  }, 
+  {
+    "name": "HW_CPU_X86_ABM"
+  }, 
+  {
+    "name": "HW_CPU_X86_MMX"
+  }, 
+  {
+    "name": "HW_CPU_X86_AVX2"
+  }, 
+  {
+    "name": "HW_CPU_X86_SSE41"
+  }, 
+  {
+    "name": "HW_CPU_X86_SSE42"
+  }, 
+  {
+    "name": "HW_CPU_X86_AESNI"
+  }, 
+  {
+    "name": "HW_CPU_X86_AVX"
+  }, 
+  {
+    "name": "HW_CPU_X86_VMX"
+  }, 
+  {
+    "name": "HW_CPU_X86_BMI2"
+  }, 
+  {
+    "name": "HW_CPU_X86_FMA3"
+  }, 
+  {
+    "name": "HW_CPU_X86_SSSE3"
+  }, 
+  {
+    "name": "HW_CPU_X86_F16C"
+  }, 
+  {
+    "name": "HW_CPU_X86_BMI"
+  }
+]"""  # noqa
+
+
+resource_provider_no_traits_out = """[]
+"""
+
+resource_provider_traits = {'HW_CPU_X86_SSE2',
+                            'HW_CPU_X86_CLMUL',
+                            'HW_CPU_X86_SSE',
+                            'HW_CPU_X86_ABM',
+                            'HW_CPU_X86_MMX',
+                            'HW_CPU_X86_AVX2',
+                            'HW_CPU_X86_SSE41',
+                            'HW_CPU_X86_SSE42',
+                            'HW_CPU_X86_AESNI',
+                            'HW_CPU_X86_AVX',
+                            'HW_CPU_X86_VMX',
+                            'HW_CPU_X86_BMI2',
+                            'HW_CPU_X86_FMA3',
+                            'HW_CPU_X86_SSSE3',
+                            'HW_CPU_X86_F16C',
+                            'HW_CPU_X86_BMI'}
+
+
+def traits_to_json(traits):
+    return json.dumps(
+        [{'name': x} for x in traits]
+    )
+
+
+def traits(*args):
+    # converts list to frozenset
+    return frozenset(*args)
+
+
+resources_expected = {
+    Specifier('CUSTOM_B', traits()): 2,
+    Specifier('CUSTOM_A', traits('TRAIT_A')): 3
+}
+
+resources_not_enough = {
+    Specifier('CUSTOM_B', traits()): 1,
+    Specifier('CUSTOM_A', traits('TRAIT_A')): 2
+}
+
+resources_one_ok = {
+    Specifier('CUSTOM_B', traits()): 2,
+    Specifier('CUSTOM_A', traits('TRAIT_A')): 2
+}
+
+resources_both_ok = {
+    Specifier('CUSTOM_B', traits()): 2,
+    Specifier('CUSTOM_A', traits('TRAIT_A')): 3
+}
+
+resources_too_many = {
+    Specifier('CUSTOM_B', traits()): 5,
+    Specifier('CUSTOM_A', traits('TRAIT_A')): 6
+}
+
+resources_extra_keys = {
+    Specifier('CUSTOM_B', traits()): 2,
+    Specifier('CUSTOM_A', traits('TRAIT_A')): 3,
+    Specifier('CUSTOM_C', traits('TRAIT_A')): 3,
+}
+
+resources_key_missing = {
+    'CUSTOM_B': 2
+}
+
+dummy_resources = [
+    {
+        "resource_class": "CUSTOM_TEST_RC",
+        "traits": [],
+        "amount": 3
+    },
+]
+
+# Scenario definitions - we define these outside the test as we want to pass
+# them into the 'patch' decorator
+
+# scenario - resource class is incorrect on first iteration
+
+inventory_resource_class_incorrect = chain(
+    repeat(inventory_custom_b_out, 3),
+    repeat(inventory_list_out, 3)
+)
+
+# scenario - resource class is reserved on first iteration
+
+inventory_resource_class_reserved = chain(
+    repeat(inventory_reserved_out, 3),
+    repeat(inventory_list_out, 3)
+)
+
+# scenario - same resource class, different traits - success
+
+# Arbitrary subset
+resource_provider_traits_subset = random.sample(resource_provider_traits, 3)
+
+resource_provider_traits_subset_out = traits_to_json(
+    resource_provider_traits_subset
+)
+
+inventory_same_resource_different_traits = chain(
+    # We are using a provider output with 3 providers, so we need three
+    # inventory strings - one for each
+    repeat(inventory_reserved_out, 1), repeat(inventory_list_out, 2)
+)
+
+traits_same_resource_different_traits = chain(
+    # We are using a provider output with 3 providers, so we need three
+    # trait strings - one for each
+    repeat(resource_provider_no_traits_out, 1),
+    repeat(resource_provider_traits_out, 1),
+    repeat(resource_provider_traits_subset_out, 1)
+)
+
+
+def get_traits_mocked(start=0, stop=None):
+    def mocked_function(_module, _provider):
+        all_traits = list(resource_provider_traits)
+        stop_ = stop if stop else len(all_traits)
+        return frozenset(all_traits[start:stop_])
+    return mocked_function
+
+
+def get_dummy_resources(resource_class="CUSTOM_TEST_RC", amount=3,
+                        traits=[]):
+    result = [
+        {
+            "resource_class": resource_class,
+            "traits": traits,
+            "amount": amount
+        },
+    ]
+    return result
+
+
+def get_dummy_module_args(resources,
+                          maximum_retries=10,
+                          delay=10,
+                          venv="/my/dummy/venv"):
+    result = {
+        'resources': resources,
+        'maximum_retries': maximum_retries,
+        'delay': delay,
+        'venv': venv
+    }
+    return result
+
+
+def pop_output(a):
+    next_ = next(a, None)
+    if next_:
+        return 0, next_, ""
+    else:
+        return 1, "", "Ran out of input"
+
+
+def noop(*_args, **_kwargs):
+    pass
+
+
+def create_run_cmd(providers, inventories, traits):
+    def dummy_run_command(*args, **_kwargs):
+        print(args)
+        if "resource provider list" in args[1]:
+            return pop_output(providers)
+        elif "resource provider trait list" in args[1]:
+            return pop_output(traits)
+        elif "resource provider inventory list" in args[1]:
+            return pop_output(inventories)
+        else:
+            raise ValueError("{} not expected", args)
+
+    return dummy_run_command
+
+
+class TestTenksWaitForResource(ModuleTestCase):
+
+    def setUp(self):
+        super(TestTenksWaitForResource, self).setUp()
+
+    def test_meets_criteria_not_enough(self):
+        self.assertFalse(meets_criteria(actual=resources_not_enough,
+                                        requested=resources_expected))
+
+    def test_meets_criteria_one_ok(self):
+        self.assertFalse(meets_criteria(actual=resources_one_ok,
+                                        requested=resources_expected))
+
+    def test_meets_criteria_both_ok(self):
+        self.assertTrue(meets_criteria(actual=resources_both_ok,
+                                       requested=resources_expected))
+
+    def test_meets_criteria_too_many(self):
+        self.assertTrue(meets_criteria(actual=resources_too_many,
+                                       requested=resources_expected))
+
+    def test_meets_criteria_keys_we_dont_care_about(self):
+        self.assertTrue(meets_criteria(actual=resources_extra_keys,
+                                       requested=resources_expected))
+
+    def test_meets_criteria_missing_key(self):
+        self.assertFalse(meets_criteria(actual=resources_key_missing,
+                                        requested=resources_expected))
+
+    @patch.object(basic.AnsibleModule, 'run_command')
+    def test_resource_provider_list(self, run_command):
+        dummy_module_arguments = get_dummy_module_args(
+            resources=dummy_resources
+        )
+        run_command.return_value = 0, resource_provider_list_out, ''
+        set_module_args(dummy_module_arguments)
+        module = wait_for.get_module()
+        providers = get_providers(module)
+        calls = [call for call in run_command.call_args_list if
+                 "openstack resource provider list" in call[0][0]]
+        self.assertGreater(len(calls), 0)
+        self.assertListEqual(resource_provider_list, providers)
+
+    @patch.object(basic.AnsibleModule, 'run_command')
+    def test_inventory_list(self, run_command):
+        dummy_module_arguments = get_dummy_module_args(
+            resources=dummy_resources
+        )
+        run_command.return_value = 0, inventory_list_out, ''
+        set_module_args(dummy_module_arguments)
+        module = wait_for.get_module()
+        expected = get_inventory(module, "provider-uuid")
+        calls = [call for call in run_command.call_args_list if
+                 "openstack resource provider inventory list" in call[0][0]]
+        self.assertGreater(len(calls), 0)
+        self.assertListEqual(expected, inventory_list)
+
+    @patch.object(basic.AnsibleModule, 'run_command')
+    def test_traits(self, run_command):
+        dummy_module_arguments = get_dummy_module_args(
+            resources=dummy_resources
+        )
+        run_command.return_value = 0, resource_provider_traits_out, ''
+        set_module_args(dummy_module_arguments)
+        module = wait_for.get_module()
+        expected = get_traits(module, "provider-uuid")
+        calls = [call for call in run_command.call_args_list if
+                 "openstack resource provider trait list" in call[0][0]]
+        self.assertGreater(len(calls), 0)
+        self.assertSetEqual(expected, resource_provider_traits)
+
+    def test_venv_path(self):
+        dummy_module_arguments = get_dummy_module_args(
+            resources=dummy_resources
+        )
+        set_module_args(dummy_module_arguments)
+        module = wait_for.get_module()
+        openstack_binary_path = get_openstack_binary_path(module)
+        venv = dummy_module_arguments["venv"]
+        expected = os.path.join(venv, "bin", "openstack")
+        self.assertEqual(expected, openstack_binary_path)
+
+    def test_venv_path_unset(self):
+        dummy_module_arguments = get_dummy_module_args(
+            resources=dummy_resources
+        )
+        args = copy.copy(dummy_module_arguments)
+        del args["venv"]
+        set_module_args(args)
+        module = wait_for.get_module()
+        openstack_binary_path = get_openstack_binary_path(module)
+        expected = "openstack"
+        self.assertEqual(expected, openstack_binary_path)
+
+    @patch.object(basic.AnsibleModule, 'run_command',
+                  create_run_cmd(
+                      providers=cycle(repeat(resource_provider_list_out, 1)),
+                      inventories=cycle(repeat(inventory_reserved_out, 1)),
+                      traits=cycle(repeat(resource_provider_no_traits_out, 3))
+                  ))
+    @patch('time.sleep', noop)
+    def test_main_failure_exhaust_retries(self):
+        dummy_module_arguments = get_dummy_module_args(
+            resources=dummy_resources
+        )
+        set_module_args(dummy_module_arguments)
+        expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format(
+            max_retries=dummy_module_arguments["maximum_retries"])
+        self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
+
+    @patch.object(basic.AnsibleModule, 'run_command',
+                  create_run_cmd(
+                      providers=cycle(repeat(resource_provider_list_out, 1)),
+                      inventories=cycle(repeat(inventory_list_out, 1)),
+                      traits=cycle(repeat(resource_provider_traits_out, 3))
+                  ))
+    @patch('time.sleep', noop)
+    @patch.object(wait_for, 'get_traits', get_traits_mocked(0, 2))
+    def test_main_failure_provider_does_not_provide_all_traits(
+            self):
+        expected_traits = list(resource_provider_traits)
+        resources = get_dummy_resources(traits=expected_traits)
+        dummy_module_arguments = get_dummy_module_args(
+            resources=resources,
+        )
+        set_module_args(dummy_module_arguments)
+        expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format(
+            max_retries=dummy_module_arguments["maximum_retries"])
+        self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
+
+    @patch.object(basic.AnsibleModule, 'run_command',
+                  create_run_cmd(
+                      providers=cycle(repeat(resource_provider_list_out, 1)),
+                      inventories=cycle(repeat(inventory_list_out, 1)),
+                      traits=cycle(repeat(resource_provider_traits_out, 3))
+                  ))
+    @patch('time.sleep', noop)
+    def test_main_failure_request_subset_of_traits(
+            self):
+        # pick an arbitrary sub range of traits
+        expected_traits = list(resource_provider_traits)[3:6]
+        resources = get_dummy_resources(traits=expected_traits)
+        dummy_module_arguments = get_dummy_module_args(
+            resources=resources,
+        )
+        set_module_args(dummy_module_arguments)
+        expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format(
+            max_retries=dummy_module_arguments["maximum_retries"])
+        self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
+
+    @patch.object(basic.AnsibleModule, 'run_command',
+                  create_run_cmd(
+                      providers=cycle(repeat(resource_provider_list_out, 1)),
+                      inventories=cycle(repeat(inventory_reserved_out, 1)),
+                      traits=cycle(repeat(resource_provider_no_traits_out, 3))
+                  ))
+    @patch('time.sleep', noop)
+    def test_main_failure_exhaust_retries_traits_not_matched(self):
+        resources = get_dummy_resources(traits=["WE_NEED_THIS"])
+        dummy_module_arguments = get_dummy_module_args(
+            resources=resources
+        )
+        set_module_args(dummy_module_arguments)
+        expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format(
+            max_retries=dummy_module_arguments["maximum_retries"])
+        self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
+
+    @patch.object(basic.AnsibleModule, 'run_command',
+                  create_run_cmd(
+                      providers=repeat(resource_provider_list_out, 1),
+                      # one per provider
+                      inventories=repeat(inventory_list_out, 3),
+                      traits=repeat(resource_provider_no_traits_out, 3)
+                  ))
+    def test_main_success_one_iteration(self):
+        dummy_module_arguments = get_dummy_module_args(
+            resources=dummy_resources
+        )
+        set_module_args(dummy_module_arguments)
+        with self.assertRaises(AnsibleExitJson) as cm:
+            wait_for.main()
+        exception = cm.exception
+        self.assertFalse(exception.args[0]['changed'])
+        self.assertEqual(1, exception.args[0]["iterations"])
+
+    @patch.object(basic.AnsibleModule, 'run_command',
+                  create_run_cmd(
+                      providers=repeat(resource_provider_list_out, 1),
+                      inventories=inventory_same_resource_different_traits,
+                      traits=traits_same_resource_different_traits
+                  ))
+    def test_main_success_same_resource_class_different_traits(self):
+        resource_a = get_dummy_resources(
+            traits=list(resource_provider_traits_subset),
+            amount=1
+        )
+        resource_b = get_dummy_resources(
+            traits=list(resource_provider_traits),
+            amount=1
+        )
+        dummy_module_arguments = get_dummy_module_args(
+            resources=resource_a + resource_b,
+        )
+        set_module_args(dummy_module_arguments)
+        try:
+            wait_for.main()
+        except AnsibleExitJson as result:
+            self.assertFalse(result.args[0]['changed'])
+            self.assertEqual(1, result.args[0]["iterations"])
+        else:
+            self.fail("Should have thrown AnsibleExitJson")
+
+    @patch.object(basic.AnsibleModule, 'run_command',
+                  create_run_cmd(
+                      providers=repeat(resource_provider_list_out, 1),
+                      # one per provider
+                      inventories=repeat(inventory_list_out, 3),
+                      traits=repeat(resource_provider_no_traits_out, 3)
+                  ))
+    def test_main_success_no_traits(self):
+        # if the resource provider doesn't have any traits and no traits
+        # were requested, this should still pass
+        dummy_module_arguments = get_dummy_module_args(
+            resources=dummy_resources
+        )
+        set_module_args(dummy_module_arguments)
+        try:
+            wait_for.main()
+        except AnsibleExitJson as result:
+            self.assertFalse(result.args[0]['changed'])
+            self.assertEqual(1, result.args[0]["iterations"])
+        else:
+            self.fail("Should have thrown AnsibleExitJson")
+
+    @patch.object(basic.AnsibleModule, 'run_command',
+                  create_run_cmd(
+                      providers=repeat(resource_provider_list_out, 2),
+                      # one per provider * 2 iterations
+                      inventories=inventory_resource_class_incorrect,
+                      traits=repeat(resource_provider_no_traits_out, 6)))
+    @patch('time.sleep', noop)
+    def test_success_first_iteration_wrong_resource_class(self):
+        # different resource class to the one we are looking for on first
+        # iteration
+        dummy_module_arguments = get_dummy_module_args(
+            resources=dummy_resources
+        )
+        set_module_args(dummy_module_arguments)
+        try:
+            wait_for.main()
+        except AnsibleExitJson as result:
+            self.assertFalse(result.args[0]['changed'])
+            self.assertEqual(2, result.args[0]["iterations"])
+        else:
+            self.fail("Should have thrown AnsibleExitJson")
+
+    @patch.object(basic.AnsibleModule, 'run_command',
+                  create_run_cmd(
+                      providers=repeat(resource_provider_list_out, 2),
+                      inventories=inventory_resource_class_reserved,
+                      # one per provider * 2 iterations
+                      traits=repeat(resource_provider_no_traits_out, 6)))
+    @patch('time.sleep', noop)
+    def test_success_resource_class_reserved(self):
+        # resource class is reserved on the first iteration
+        dummy_module_arguments = get_dummy_module_args(
+            resources=dummy_resources
+        )
+        set_module_args(dummy_module_arguments)
+        try:
+            wait_for.main()
+        except AnsibleExitJson as result:
+            self.assertFalse(result.args[0]['changed'])
+            self.assertEqual(2, result.args[0]["iterations"])
+        else:
+            self.fail("Should have thrown AnsibleExitJson")
+
+    @patch.object(basic.AnsibleModule, 'run_command',
+                  create_run_cmd(
+                      providers=repeat(resource_provider_list_out, 1),
+                      # one per provider
+                      inventories=repeat(inventory_list_out, 3),
+                      traits=repeat(resource_provider_no_traits_out, 3)
+                  ))
+    def test_validation_amount_is_str_repr_of_int(self):
+        dummy_module_arguments = get_dummy_module_args(
+            # amount should be an int
+            resources=get_dummy_resources(amount="3")
+        )
+        set_module_args(dummy_module_arguments)
+        try:
+            wait_for.main()
+        except AnsibleExitJson as result:
+            self.assertFalse(result.args[0]['changed'])
+            self.assertEqual(1, result.args[0]["iterations"])
+        else:
+            self.fail("Should have thrown AnsibleExitJson")
+
+    def test_validation_amount_not_int(self):
+        dummy_module_arguments = get_dummy_module_args(
+            # amount should be an int
+            resources=get_dummy_resources(amount="not_a_number")
+        )
+        set_module_args(dummy_module_arguments)
+        expected_msg = "amount, should be type int"
+        self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
+
+    def test_validation_amount_missing(self):
+        resources = get_dummy_resources()
+        for resource in resources:
+            del resource["amount"]
+        dummy_module_arguments = get_dummy_module_args(
+            resources=resources
+        )
+        set_module_args(dummy_module_arguments)
+        expected_msg = (
+            "One of your resources does not have the field, amount, set"
+        )
+        self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
+
+    def test_merge_simple(self):
+        a = {'a': 1, 'b': 2}
+        b = {'a': 3, 'c': 5}
+        expected = {'a': 4, 'b': 2, 'c': 5}
+        merged = merge(a, b, lambda x, y: x + y)
+        self.assertDictEqual(expected, merged)

tests/utils.py

@@ -0,0 +1,60 @@
+import json
+
+import unittest
+
+from ansible.module_utils import basic
+from ansible.module_utils._text import to_bytes
+
+# Python 2/3 compatibility.
+try:
+    from unittest.mock import MagicMock, patch
+except ImportError:
+    from mock import MagicMock, patch  # noqa
+
+
+def set_module_args(args):
+    if '_ansible_remote_tmp' not in args:
+        args['_ansible_remote_tmp'] = '/tmp'
+    if '_ansible_keep_remote_files' not in args:
+        args['_ansible_keep_remote_files'] = False
+
+    args = json.dumps({'ANSIBLE_MODULE_ARGS': args})
+    basic._ANSIBLE_ARGS = to_bytes(args)
+
+
+class AnsibleExitJson(Exception):
+    pass
+
+
+class AnsibleFailJson(Exception):
+    pass
+
+
+def exit_json(*args, **kwargs):
+    if 'changed' not in kwargs:
+        kwargs['changed'] = False
+    raise AnsibleExitJson(kwargs)
+
+
+def fail_json(*args, **kwargs):
+    kwargs['failed'] = True
+    raise AnsibleFailJson(kwargs)
+
+
+class ModuleTestCase(unittest.TestCase):
+
+    def __init__(self, *args, **kwargs):
+        super(ModuleTestCase, self).__init__(*args, **kwargs)
+        # Python 2 / 3 compatibility. assertRaisesRegexp was renamed to
+        # assertRaisesRegex in python 3.1.
+        if hasattr(self, 'assertRaisesRegexp') and \
+                not hasattr(self, 'assertRaisesRegex'):
+            self.assertRaisesRegex = self.assertRaisesRegexp
+
+    def setUp(self):
+        self.mock_module = patch.multiple(basic.AnsibleModule,
+                                          exit_json=exit_json,
+                                          fail_json=fail_json)
+        self.mock_module.start()
+        set_module_args({})
+        self.addCleanup(self.mock_module.stop)

tox.ini

@@ -41,7 +41,7 @@ commands = sphinx-build -W -b html doc/source doc/build/html
 basepython = python3
 setenv =
     VIRTUAL_ENV={envdir}
-    PYTHON=coverage run --source tenks,ansible/action_plugins --parallel-mode
+    PYTHON=coverage run --source tenks,ansible --parallel-mode
 commands =
     coverage erase
     stestr run {posargs}