From d4e7e58619fb84a1e1a184a64cd003e5b274b7c8 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Tue, 6 Nov 2018 16:35:06 +0000 Subject: [PATCH] Wait for resources to become available This will wait for any resources that are created to become available before exiting the script. This allows you to avoid a race condition where a server could be created before the resource tracker had been updated with the new resources; server creation would fail. Change-Id: I57f8c93cb1ebbc284b96ef1ced2c4edd59b27795 Story: 2004274 Task: 27823 Depends-On: https://review.openstack.org/617642 --- ansible/deploy.yml | 4 + ansible/resource_wait.yml | 46 ++ .../wait-for-resources/defaults/main.yml | 28 + .../wait-for-resources/files/requirements.txt | 4 + .../library/wait_for_resources.py | 383 ++++++++++ .../roles/wait-for-resources/tasks/main.yml | 38 + .../wait-for-resources/tasks/validate.yml | 17 + tests/test_tenks_wait_for_resources.py | 674 ++++++++++++++++++ tests/utils.py | 60 ++ tox.ini | 2 +- 10 files changed, 1255 insertions(+), 1 deletion(-) create mode 100644 ansible/resource_wait.yml create mode 100644 ansible/roles/wait-for-resources/defaults/main.yml create mode 100644 ansible/roles/wait-for-resources/files/requirements.txt create mode 100644 ansible/roles/wait-for-resources/library/wait_for_resources.py create mode 100644 ansible/roles/wait-for-resources/tasks/main.yml create mode 100644 ansible/roles/wait-for-resources/tasks/validate.yml create mode 100644 tests/test_tenks_wait_for_resources.py create mode 100644 tests/utils.py diff --git a/ansible/deploy.yml b/ansible/deploy.yml index 2a7f0ad..ad167ea 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -28,5 +28,9 @@ import_playbook: flavor_registration.yml tags: openstack +- name: Wait for resources to become available + import_playbook: resource_wait.yml + tags: openstack, resource + - name: Clean up Tenks state import_playbook: cleanup_state.yml diff --git a/ansible/resource_wait.yml b/ansible/resource_wait.yml new file mode 100644 index 0000000..ea6714b --- /dev/null +++ b/ansible/resource_wait.yml @@ -0,0 +1,46 @@ +--- +- hosts: localhost + tasks: + - name: Check that OpenStack credentials exist in the environment + fail: + msg: > + $OS_USERNAME was not found in the environment. Ensure the OpenStack + credentials exist in your environment, perhaps by sourcing your RC file. + when: not lookup('env', 'OS_USERNAME') + + - name: Gather list of OpenStack services + command: >- + {{ virtualenv_path }}/bin/openstack service list -f json + register: service_list_output + changed_when: false + + - block: + - name: Set default value for expected resources + set_fact: + tenks_expected_resources: [] + + - name: Build list of expected resources + # takes the form: [{ resource_class: CUSTOM_TEST_RC, amount: 2, traits: [] }, ] + vars: + resource: + amount: "{{ spec.count | int }}" # this gets converted back to a string + resource_class: "{{ 'CUSTOM_' ~ spec.ironic_config.resource_class | upper | replace('-', '_') }}" + traits: "{{ spec.ironic_config.traits | default([])}}" + set_fact: + tenks_expected_resources: >- + {{ tenks_expected_resources + [resource] }} + loop: "{{ specs }}" + when: "'ironic_config' in spec" + loop_control: + loop_var: spec + + - name: Include the wait-for-resources role + include_role: + name: wait-for-resources + vars: + wait_for_resources_required_resources: "{{ tenks_expected_resources }}" + wait_for_resources_venv: "{{ virtualenv_path }}" + wait_for_resources_python_upper_constraints_url: >- + {{ python_upper_constraints_url }} + # Only attempt to wait for resources when the placement service is running + when: service_list_output.stdout | from_json | selectattr('Type', 'equalto', 'placement') | list | length >= 1 diff --git a/ansible/roles/wait-for-resources/defaults/main.yml b/ansible/roles/wait-for-resources/defaults/main.yml new file mode 100644 index 0000000..0e638b6 --- /dev/null +++ b/ansible/roles/wait-for-resources/defaults/main.yml @@ -0,0 +1,28 @@ +--- + +# A list of dictionaries, each with the following keys: +# * resource_class - the name of the resource class that you want to be +# available +# * traits - a list of traits a provider requires before that provider is +# considered to provide that resource +# * amount - natural number representing the amount of the resource that +# should be available. This will be tested against the total resource count +# minus the amount that is reserved. +wait_for_resources_required_resources: + +# Virtualenv containing OpenStack client +wait_for_resources_venv: + +# The resources are polled this many times. The delay between each iteration +# is also configurable, but defaults to 10 seconds +wait_for_resources_retry_limit: 15 + +# The delay between each iteration. +wait_for_resources_delay: + +# The URL of the upper constraints file to pass to pip when installing Python +# packages. +wait_for_resources_python_upper_constraints_url: + +# Path on the remote machine to copy the python requirements file to +wait_for_resources_python_requirements_path: /tmp/wait-for-python-requirements.txt diff --git a/ansible/roles/wait-for-resources/files/requirements.txt b/ansible/roles/wait-for-resources/files/requirements.txt new file mode 100644 index 0000000..ee52fad --- /dev/null +++ b/ansible/roles/wait-for-resources/files/requirements.txt @@ -0,0 +1,4 @@ +# This file contains the Python packages that are needed in the +# what-for-resource virtual env. + +osc-placement>=1.3.0 # Apache License 2.0 diff --git a/ansible/roles/wait-for-resources/library/wait_for_resources.py b/ansible/roles/wait-for-resources/library/wait_for_resources.py new file mode 100644 index 0000000..aab4967 --- /dev/null +++ b/ansible/roles/wait-for-resources/library/wait_for_resources.py @@ -0,0 +1,383 @@ +#!/usr/bin/python + +# Copyright (c) 2018 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +DOCUMENTATION = """ +module: wait_for_resources +short_description: Waits for a set of resources to become available in +placement +author: Will Szumski(will@stackhpc.com) +options: + - option-name: resources + description: List of dictionaries describing the resources that should + be available. Each dictionary should contain the keys: resource_class, + amount, and optionally, a list of traits that the provider needs to provide + before it is considered to provide the resource. + required: True + type: list + - option-name: venv + description: Path to a virtualenv containing the OpenStack CLI. It is a + requirement for the placement api plugin (osc-placement) to be installed. + required: False + type: string + - option-name: maximum_retries + description: The maximum number of iterations to poll for the resources + required: False + type: int + - option-name: delay + description: Delay between each iteration of the polling loop in seconds + required: False + type: int +requirements: + - python-openstackclient + - osc-placement +""" + +EXAMPLES = """ +- name: Wait for resources to become available + wait_for_resources: + resources: {'resource_class: CUSTOM_TEST_RC, 'traits': [], 'amount': 2} + delay: 10 + maximum_retries: 15 + venv: /path/to/venv +""" + +RETURN = """ +iterations: + description: The Number of iterations required before the resource became + available + returned: success + type: int + sample: 9 +""" + +# Need to disable PEP8, as it wants all imports at top of file +from ansible.module_utils.basic import AnsibleModule # noqa +from collections import namedtuple # noqa +import six # noqa +import os # noqa +import time # noqa + +_RETRY_LIMIT_FAILURE_TEMPLATE = "exceeded retry limit of {max_retries} " \ + "whilst waiting for resources to become " \ + "available" + +Specifier = namedtuple("Specifier", "name traits") +Provider = namedtuple("Provider", "uuid inventory_list traits") + +# Store a list of import errors to report to the user. +IMPORT_ERRORS = [] +try: + import json +except Exception as e: + IMPORT_ERRORS.append(e) + + +def meets_criteria(actual, requested): + """For each resource, determines whether the total satisfies the amount + requested, ignoring any unrequested resource_classes""" + for specifier in requested.keys(): + if specifier not in actual: + return False + if actual[specifier] < requested[specifier]: + return False + return True + + +def get_openstack_binary_path(module): + """Returns the path to the openstack binary taking into account the + virtualenv that was specified (if available)""" + venv = module.params["venv"] + if venv: + return os.path.join(venv, "bin", "openstack") + # use openstack in PATH + return "openstack" + + +def get_inventory(module, provider_uuid): + """ + Gets inventory of resources for a give provider UUID + :param module: ansible module + :param provider_uuid: provider to query + :return: list of dictionaries of the form: + [ + { + "allocation_ratio": 16.0, + "total": 24, + "reserved": 0, + "resource_class": "VCPU", + "step_size": 1, + "min_unit": 1, + "max_unit": 24 + }, + ] + """ + cmd = "{openstack} resource provider inventory list {uuid} -f json" \ + .format(uuid=provider_uuid, + openstack=get_openstack_binary_path(module)) + + rc, out, err = module.run_command(cmd) + if rc != 0: + msg = "{} failed with return code: {}, stderr: {}".format(cmd, rc, err) + module.fail_json(msg=msg) + return json.loads(out) + + +def get_providers(module): + """ + Gets a list of resource providers + :param module: ansible module + :return: list of dictionaries of the form: + [ + { + "generation": 2, + "uuid": "657c4ab0-de82-4def-b7b0-d13ce672bfd0", + "name": "kayobe-will-master" + }, + ] + """ + cmd = "{openstack} resource provider list -f json".format( + openstack=get_openstack_binary_path(module) + ) + rc, out, err = module.run_command(cmd) + if rc != 0: + msg = "{} failed with return code: {}, stderr: {}".format(cmd, rc, err) + module.fail_json(msg=msg) + return json.loads(out) + + +def get_traits(module, provider_uuid): + """ + Gets a list of traits for a resource provider + :param provider_uuid: the uuid of the provider + :param module: ansible module + :return: set of traits of the form: + { + "HW_CPU_X86_SSE2", + } + """ + cmd = "{openstack} resource provider trait list {uuid} " \ + "--os-placement-api-version 1.6 -f json " \ + .format(uuid=provider_uuid, + openstack=get_openstack_binary_path(module)) + + rc, out, err = module.run_command(cmd) + if rc != 0: + msg = "{} failed with return code: {}, stderr: {}".format(cmd, rc, err) + module.fail_json(msg=msg) + raw = json.loads(out) + return set([x["name"] for x in raw]) + + +def merge(x, y, f): + """ Merges two dictionaries. If a key appears in both dictionaries, the + common values are merged using the function ``f``""" + # Start with symmetric difference; keys either in A or B, but not both + merged = {k: x.get(k, y.get(k)) for k in six.viewkeys(x) ^ six.viewkeys(y)} + # Update with `f()` applied to the intersection + merged.update( + {k: f(x[k], y[k]) for k in six.viewkeys(x) & six.viewkeys(y)}) + return merged + + +def collect(specifiers, provider): + """Given a specifier and a provider, gets the amount of resource that is + available for that given provider""" + inventory = {} + + # we want to be able to look up items by resource_class + for item in provider.inventory_list: + inventory[item["resource_class"]] = item + result = {} + for specifier in specifiers: + if specifier.traits != provider.traits: + continue + if specifier.name in inventory: + reserved = inventory[specifier.name]["reserved"] + total_available = inventory[specifier.name]["total"] + result[specifier] = total_available - reserved + break + return result + + +def get_totals(specifiers, providers): + """Loops over the providers adding up all of the resources that + are available""" + totals = {} + + for specifier in specifiers: + # initialise totals so the combine function does not blow up + totals[specifier] = 0 + + for provider in providers: + current = collect(specifiers, provider) + totals = merge(totals, current, lambda x, y: x + y) + + return totals + + +def are_resources_available(module, specifiers, expected): + """ + Determines whether or not a set of resources are available + :param module: Ansible module + :param specifiers: tuples of the form: (resource_class, traits) + :param expected: dictionary of the form: + {(resource_class, traits): amount } + :return: True if resource available, otherwise False + """ + providers_raw = get_providers(module) + providers = [] + for provider in providers_raw: + uuid = provider["uuid"] + traits = get_traits(module, uuid) + inventory = get_inventory(module, uuid) + provider = Provider( + uuid=uuid, + inventory_list=inventory, + traits=traits + ) + providers.append(provider) + actual = get_totals(specifiers, providers) + return meets_criteria(actual, expected) + + +def wait_for_resources(module): + """ + Waits for a set of resources to become available + :param module: Ansible module + :return: the number of attempts needed + """ + max_retries = module.params["maximum_retries"] + delay = module.params["delay"] + resources = module.params["resources"] + expected = {} + specifiers = [] + for resource in resources: + # default value for traits + traits = resource["traits"] if resource["traits"] else [] + specifier = Specifier(name=resource["resource_class"], + traits=frozenset(traits)) + specifiers.append(specifier) + expected[specifier] = resource["amount"] + + for i in range(max_retries): + if are_resources_available(module, specifiers, expected): + return i + 1 + time.sleep(delay) + + fail_msg = _RETRY_LIMIT_FAILURE_TEMPLATE.format( + max_retries=max_retries) + module.fail_json(msg=fail_msg) + + +def validate_spec(module, resource, field, type_): + check_resource_msg = ( + "Please check the dictionaries in your resources list meet the " + "required specification.") + unset_field_msg = ( + "One of your resources does not have the field, {field}, set." + ) + field_type_msg = ( + "The field, {field}, should be type {type}." + ) + if field not in resource: + msg = "{} {}".format(unset_field_msg, check_resource_msg) + module.fail_json(msg=msg.format(field=field)) + elif not isinstance(resource[field], type_): + msg = "{} {}".format(field_type_msg, check_resource_msg) + module.fail_json(msg=msg.format(field=field, type=type_.__name__)) + + +def validate_resources(module, specs): + resources = module.params["resources"] + for resource in resources: + for spec in specs: + validate_spec(module, resource, spec["field"], type_=spec["type"]) + + +def get_module(): + """Creates and returns an Ansible module""" + module = AnsibleModule( + argument_spec=dict( + resources=dict(required=True, type='list'), + maximum_retries=dict(required=False, type='int', default=15), + delay=dict(required=False, type='int', default=10), + venv=dict(required=False, type='str', default="") + ), + supports_check_mode=True, + ) + + # note(wszumski): amount seems to get converted back to a string + # https://github.com/ansible/ansible/issues/18095 + resources = module.params["resources"] + for resource in resources: + if 'amount' not in resource: + continue + amount = resource["amount"] + if isinstance(amount, str): + try: + amount_int = int(amount) + resource["amount"] = amount_int + except ValueError: + # this will get picked up in validate_resources + pass + + # Validate resources list + resources_specs = [ + { + 'field': 'resource_class', + 'type': str + }, + { + 'field': 'traits', + 'type': list + }, + { + 'field': 'amount', + 'type': int + }, + ] + + validate_resources(module, resources_specs) + + # Fail if there were any exceptions when importing modules. + if IMPORT_ERRORS: + module.fail_json(msg="Import errors: %s" % + ", ".join([repr(e) for e in IMPORT_ERRORS])) + + return module + + +def main(): + module = get_module() + + attempts_needed = 0 + + # In check mode, only perform validation of parameters + if not module.check_mode: + attempts_needed = wait_for_resources(module) + + # This module doesn't really change anything, it just waits for + # something to change externally + result = { + "changed": False, + "iterations": attempts_needed + } + + module.exit_json(**result) + + +if __name__ == '__main__': + main() diff --git a/ansible/roles/wait-for-resources/tasks/main.yml b/ansible/roles/wait-for-resources/tasks/main.yml new file mode 100644 index 0000000..401dbd0 --- /dev/null +++ b/ansible/roles/wait-for-resources/tasks/main.yml @@ -0,0 +1,38 @@ +--- + +# Waits for for the resource tracker to be updated with a given resource +# class + +- name: Validate that the virtualenv variable is set + fail: + msg: >- + You must set the variable, wait_for_resources_venv, to use this role. + when: wait_for_resources_venv is none + +- name: Validate resources + include_tasks: validate.yml + vars: + wait_for_resources_amount: "{{ item.amount }}" + wait_for_resources_resource_class: "{{ item.resource_class }}" + loop: "{{ wait_for_resources_required_resources }}" + +- name: Copy over requirements file + copy: + src: requirements.txt + dest: "{{ wait_for_resources_python_requirements_path }}" + +- name: Install dependencies in supplied venv + pip: + requirements: "{{ wait_for_resources_python_requirements_path }}" + extra_args: >- + {%- if wait_for_resources_python_upper_constraints_url -%} + -c {{ wait_for_resources_python_upper_constraints_url }} + {%- endif -%} + virtualenv: "{{ wait_for_resources_venv }}" + +- name: Call wait_for_resources module + wait_for_resources: + venv: "{{ wait_for_resources_venv }}" + resources: "{{ wait_for_resources_required_resources }}" + delay: "{{ wait_for_resources_delay | default(omit, true) }}" + maximum_retries: "{{ wait_for_resources_retry_limit }}" diff --git a/ansible/roles/wait-for-resources/tasks/validate.yml b/ansible/roles/wait-for-resources/tasks/validate.yml new file mode 100644 index 0000000..ee5a86b --- /dev/null +++ b/ansible/roles/wait-for-resources/tasks/validate.yml @@ -0,0 +1,17 @@ +--- + +- name: Validate that the target resource class is set + fail: + msg: >- + Each item in the dictionary wait_for_resources_required_resources must contain + the key, resource_class. + when: wait_for_resources_class is none + +- name: Validate expected resource count + vars: + count: "{{ wait_for_resources_amount | int(default=-1) }}" + fail: + msg: >- + Each item in the dictionary wait_for_resources_required_resources must contain + the key, amount. + when: wait_for_resources_amount is none or count | int < 0 diff --git a/tests/test_tenks_wait_for_resources.py b/tests/test_tenks_wait_for_resources.py new file mode 100644 index 0000000..c2ef8eb --- /dev/null +++ b/tests/test_tenks_wait_for_resources.py @@ -0,0 +1,674 @@ +# Copyright (c) 2018 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import +import copy +import imp +import json +import os +import random +from itertools import repeat, chain, cycle + +from ansible.module_utils import basic + +from tests.utils import ModuleTestCase, set_module_args, AnsibleExitJson, \ + AnsibleFailJson + +# Python 2/3 compatibility. +try: + from unittest.mock import MagicMock, patch +except ImportError: + from mock import MagicMock, patch # noqa + +# Import method lifted from kolla_ansible's test_merge_config.py +PROJECT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../')) +PLUGIN_FILE = os.path.join(PROJECT_DIR, + 'ansible/roles/wait-for-resources/library' + '/wait_for_resources.py') + +wait_for = imp.load_source('wait_for_resources', PLUGIN_FILE) + +meets_criteria = wait_for.meets_criteria +get_providers = wait_for.get_providers +get_inventory = wait_for.get_inventory +get_traits = wait_for.get_traits +merge = wait_for.merge +get_openstack_binary_path = wait_for.get_openstack_binary_path +Specifier = wait_for.Specifier + +inventory_list_out = """[ + { + "allocation_ratio": 1.0, + "total": 1, + "reserved": 0, + "resource_class": "CUSTOM_TEST_RC", + "step_size": 1, + "min_unit": 1, + "max_unit": 1 + } +]""" # noqa + +inventory_custom_b_out = """[ + { + "allocation_ratio": 1.0, + "total": 1, + "reserved": 0, + "resource_class": "CUSTOM_B", + "step_size": 1, + "min_unit": 1, + "max_unit": 1 + } +]""" # noqa + +inventory_reserved_out = """[ + { + "allocation_ratio": 1.0, + "total": 1, + "reserved": 1, + "resource_class": "CUSTOM_TEST_RC", + "step_size": 1, + "min_unit": 1, + "max_unit": 1 + } +]""" # noqa + +inventory_list = [{'allocation_ratio': 1.0, + 'max_unit': 1, + 'min_unit': 1, + 'reserved': 0, + 'resource_class': 'CUSTOM_TEST_RC', + 'step_size': 1, + 'total': 1}] + +resource_provider_list_out = """[ + { + "generation": 2, + "uuid": "657c4ab0-de82-4def-b7b0-d13ce672bfd0", + "name": "kayobe-will-master" + }, + { + "generation": 1, + "uuid": "e2e78f98-d3ec-466a-862b-42d7ef5dca7d", + "name": "e2e78f98-d3ec-466a-862b-42d7ef5dca7d" + }, + { + "generation": 1, + "uuid": "07072aea-cc2b-4135-a7e2-3a4dd3a9f629", + "name": "07072aea-cc2b-4135-a7e2-3a4dd3a9f629" + } +]""" # noqa + +resource_provider_list = [{'generation': 2, 'name': 'kayobe-will-master', + 'uuid': '657c4ab0-de82-4def-b7b0-d13ce672bfd0'}, + {'generation': 1, + 'name': 'e2e78f98-d3ec-466a-862b-42d7ef5dca7d', + 'uuid': 'e2e78f98-d3ec-466a-862b-42d7ef5dca7d'}, + {'generation': 1, + 'name': '07072aea-cc2b-4135-a7e2-3a4dd3a9f629', + 'uuid': '07072aea-cc2b-4135-a7e2-3a4dd3a9f629'}] + +resource_provider_traits_out = """[ + { + "name": "HW_CPU_X86_SSE2" + }, + { + "name": "HW_CPU_X86_CLMUL" + }, + { + "name": "HW_CPU_X86_SSE" + }, + { + "name": "HW_CPU_X86_ABM" + }, + { + "name": "HW_CPU_X86_MMX" + }, + { + "name": "HW_CPU_X86_AVX2" + }, + { + "name": "HW_CPU_X86_SSE41" + }, + { + "name": "HW_CPU_X86_SSE42" + }, + { + "name": "HW_CPU_X86_AESNI" + }, + { + "name": "HW_CPU_X86_AVX" + }, + { + "name": "HW_CPU_X86_VMX" + }, + { + "name": "HW_CPU_X86_BMI2" + }, + { + "name": "HW_CPU_X86_FMA3" + }, + { + "name": "HW_CPU_X86_SSSE3" + }, + { + "name": "HW_CPU_X86_F16C" + }, + { + "name": "HW_CPU_X86_BMI" + } +]""" # noqa + + +resource_provider_no_traits_out = """[] +""" + +resource_provider_traits = {'HW_CPU_X86_SSE2', + 'HW_CPU_X86_CLMUL', + 'HW_CPU_X86_SSE', + 'HW_CPU_X86_ABM', + 'HW_CPU_X86_MMX', + 'HW_CPU_X86_AVX2', + 'HW_CPU_X86_SSE41', + 'HW_CPU_X86_SSE42', + 'HW_CPU_X86_AESNI', + 'HW_CPU_X86_AVX', + 'HW_CPU_X86_VMX', + 'HW_CPU_X86_BMI2', + 'HW_CPU_X86_FMA3', + 'HW_CPU_X86_SSSE3', + 'HW_CPU_X86_F16C', + 'HW_CPU_X86_BMI'} + + +def traits_to_json(traits): + return json.dumps( + [{'name': x} for x in traits] + ) + + +def traits(*args): + # converts list to frozenset + return frozenset(*args) + + +resources_expected = { + Specifier('CUSTOM_B', traits()): 2, + Specifier('CUSTOM_A', traits('TRAIT_A')): 3 +} + +resources_not_enough = { + Specifier('CUSTOM_B', traits()): 1, + Specifier('CUSTOM_A', traits('TRAIT_A')): 2 +} + +resources_one_ok = { + Specifier('CUSTOM_B', traits()): 2, + Specifier('CUSTOM_A', traits('TRAIT_A')): 2 +} + +resources_both_ok = { + Specifier('CUSTOM_B', traits()): 2, + Specifier('CUSTOM_A', traits('TRAIT_A')): 3 +} + +resources_too_many = { + Specifier('CUSTOM_B', traits()): 5, + Specifier('CUSTOM_A', traits('TRAIT_A')): 6 +} + +resources_extra_keys = { + Specifier('CUSTOM_B', traits()): 2, + Specifier('CUSTOM_A', traits('TRAIT_A')): 3, + Specifier('CUSTOM_C', traits('TRAIT_A')): 3, +} + +resources_key_missing = { + 'CUSTOM_B': 2 +} + +dummy_resources = [ + { + "resource_class": "CUSTOM_TEST_RC", + "traits": [], + "amount": 3 + }, +] + +# Scenario definitions - we define these outside the test as we want to pass +# them into the 'patch' decorator + +# scenario - resource class is incorrect on first iteration + +inventory_resource_class_incorrect = chain( + repeat(inventory_custom_b_out, 3), + repeat(inventory_list_out, 3) +) + +# scenario - resource class is reserved on first iteration + +inventory_resource_class_reserved = chain( + repeat(inventory_reserved_out, 3), + repeat(inventory_list_out, 3) +) + +# scenario - same resource class, different traits - success + +# Arbitrary subset +resource_provider_traits_subset = random.sample(resource_provider_traits, 3) + +resource_provider_traits_subset_out = traits_to_json( + resource_provider_traits_subset +) + +inventory_same_resource_different_traits = chain( + # We are using a provider output with 3 providers, so we need three + # inventory strings - one for each + repeat(inventory_reserved_out, 1), repeat(inventory_list_out, 2) +) + +traits_same_resource_different_traits = chain( + # We are using a provider output with 3 providers, so we need three + # trait strings - one for each + repeat(resource_provider_no_traits_out, 1), + repeat(resource_provider_traits_out, 1), + repeat(resource_provider_traits_subset_out, 1) +) + + +def get_traits_mocked(start=0, stop=None): + def mocked_function(_module, _provider): + all_traits = list(resource_provider_traits) + stop_ = stop if stop else len(all_traits) + return frozenset(all_traits[start:stop_]) + return mocked_function + + +def get_dummy_resources(resource_class="CUSTOM_TEST_RC", amount=3, + traits=[]): + result = [ + { + "resource_class": resource_class, + "traits": traits, + "amount": amount + }, + ] + return result + + +def get_dummy_module_args(resources, + maximum_retries=10, + delay=10, + venv="/my/dummy/venv"): + result = { + 'resources': resources, + 'maximum_retries': maximum_retries, + 'delay': delay, + 'venv': venv + } + return result + + +def pop_output(a): + next_ = next(a, None) + if next_: + return 0, next_, "" + else: + return 1, "", "Ran out of input" + + +def noop(*_args, **_kwargs): + pass + + +def create_run_cmd(providers, inventories, traits): + def dummy_run_command(*args, **_kwargs): + print(args) + if "resource provider list" in args[1]: + return pop_output(providers) + elif "resource provider trait list" in args[1]: + return pop_output(traits) + elif "resource provider inventory list" in args[1]: + return pop_output(inventories) + else: + raise ValueError("{} not expected", args) + + return dummy_run_command + + +class TestTenksWaitForResource(ModuleTestCase): + + def setUp(self): + super(TestTenksWaitForResource, self).setUp() + + def test_meets_criteria_not_enough(self): + self.assertFalse(meets_criteria(actual=resources_not_enough, + requested=resources_expected)) + + def test_meets_criteria_one_ok(self): + self.assertFalse(meets_criteria(actual=resources_one_ok, + requested=resources_expected)) + + def test_meets_criteria_both_ok(self): + self.assertTrue(meets_criteria(actual=resources_both_ok, + requested=resources_expected)) + + def test_meets_criteria_too_many(self): + self.assertTrue(meets_criteria(actual=resources_too_many, + requested=resources_expected)) + + def test_meets_criteria_keys_we_dont_care_about(self): + self.assertTrue(meets_criteria(actual=resources_extra_keys, + requested=resources_expected)) + + def test_meets_criteria_missing_key(self): + self.assertFalse(meets_criteria(actual=resources_key_missing, + requested=resources_expected)) + + @patch.object(basic.AnsibleModule, 'run_command') + def test_resource_provider_list(self, run_command): + dummy_module_arguments = get_dummy_module_args( + resources=dummy_resources + ) + run_command.return_value = 0, resource_provider_list_out, '' + set_module_args(dummy_module_arguments) + module = wait_for.get_module() + providers = get_providers(module) + calls = [call for call in run_command.call_args_list if + "openstack resource provider list" in call[0][0]] + self.assertGreater(len(calls), 0) + self.assertListEqual(resource_provider_list, providers) + + @patch.object(basic.AnsibleModule, 'run_command') + def test_inventory_list(self, run_command): + dummy_module_arguments = get_dummy_module_args( + resources=dummy_resources + ) + run_command.return_value = 0, inventory_list_out, '' + set_module_args(dummy_module_arguments) + module = wait_for.get_module() + expected = get_inventory(module, "provider-uuid") + calls = [call for call in run_command.call_args_list if + "openstack resource provider inventory list" in call[0][0]] + self.assertGreater(len(calls), 0) + self.assertListEqual(expected, inventory_list) + + @patch.object(basic.AnsibleModule, 'run_command') + def test_traits(self, run_command): + dummy_module_arguments = get_dummy_module_args( + resources=dummy_resources + ) + run_command.return_value = 0, resource_provider_traits_out, '' + set_module_args(dummy_module_arguments) + module = wait_for.get_module() + expected = get_traits(module, "provider-uuid") + calls = [call for call in run_command.call_args_list if + "openstack resource provider trait list" in call[0][0]] + self.assertGreater(len(calls), 0) + self.assertSetEqual(expected, resource_provider_traits) + + def test_venv_path(self): + dummy_module_arguments = get_dummy_module_args( + resources=dummy_resources + ) + set_module_args(dummy_module_arguments) + module = wait_for.get_module() + openstack_binary_path = get_openstack_binary_path(module) + venv = dummy_module_arguments["venv"] + expected = os.path.join(venv, "bin", "openstack") + self.assertEqual(expected, openstack_binary_path) + + def test_venv_path_unset(self): + dummy_module_arguments = get_dummy_module_args( + resources=dummy_resources + ) + args = copy.copy(dummy_module_arguments) + del args["venv"] + set_module_args(args) + module = wait_for.get_module() + openstack_binary_path = get_openstack_binary_path(module) + expected = "openstack" + self.assertEqual(expected, openstack_binary_path) + + @patch.object(basic.AnsibleModule, 'run_command', + create_run_cmd( + providers=cycle(repeat(resource_provider_list_out, 1)), + inventories=cycle(repeat(inventory_reserved_out, 1)), + traits=cycle(repeat(resource_provider_no_traits_out, 3)) + )) + @patch('time.sleep', noop) + def test_main_failure_exhaust_retries(self): + dummy_module_arguments = get_dummy_module_args( + resources=dummy_resources + ) + set_module_args(dummy_module_arguments) + expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format( + max_retries=dummy_module_arguments["maximum_retries"]) + self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main) + + @patch.object(basic.AnsibleModule, 'run_command', + create_run_cmd( + providers=cycle(repeat(resource_provider_list_out, 1)), + inventories=cycle(repeat(inventory_list_out, 1)), + traits=cycle(repeat(resource_provider_traits_out, 3)) + )) + @patch('time.sleep', noop) + @patch.object(wait_for, 'get_traits', get_traits_mocked(0, 2)) + def test_main_failure_provider_does_not_provide_all_traits( + self): + expected_traits = list(resource_provider_traits) + resources = get_dummy_resources(traits=expected_traits) + dummy_module_arguments = get_dummy_module_args( + resources=resources, + ) + set_module_args(dummy_module_arguments) + expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format( + max_retries=dummy_module_arguments["maximum_retries"]) + self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main) + + @patch.object(basic.AnsibleModule, 'run_command', + create_run_cmd( + providers=cycle(repeat(resource_provider_list_out, 1)), + inventories=cycle(repeat(inventory_list_out, 1)), + traits=cycle(repeat(resource_provider_traits_out, 3)) + )) + @patch('time.sleep', noop) + def test_main_failure_request_subset_of_traits( + self): + # pick an arbitrary sub range of traits + expected_traits = list(resource_provider_traits)[3:6] + resources = get_dummy_resources(traits=expected_traits) + dummy_module_arguments = get_dummy_module_args( + resources=resources, + ) + set_module_args(dummy_module_arguments) + expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format( + max_retries=dummy_module_arguments["maximum_retries"]) + self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main) + + @patch.object(basic.AnsibleModule, 'run_command', + create_run_cmd( + providers=cycle(repeat(resource_provider_list_out, 1)), + inventories=cycle(repeat(inventory_reserved_out, 1)), + traits=cycle(repeat(resource_provider_no_traits_out, 3)) + )) + @patch('time.sleep', noop) + def test_main_failure_exhaust_retries_traits_not_matched(self): + resources = get_dummy_resources(traits=["WE_NEED_THIS"]) + dummy_module_arguments = get_dummy_module_args( + resources=resources + ) + set_module_args(dummy_module_arguments) + expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format( + max_retries=dummy_module_arguments["maximum_retries"]) + self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main) + + @patch.object(basic.AnsibleModule, 'run_command', + create_run_cmd( + providers=repeat(resource_provider_list_out, 1), + # one per provider + inventories=repeat(inventory_list_out, 3), + traits=repeat(resource_provider_no_traits_out, 3) + )) + def test_main_success_one_iteration(self): + dummy_module_arguments = get_dummy_module_args( + resources=dummy_resources + ) + set_module_args(dummy_module_arguments) + with self.assertRaises(AnsibleExitJson) as cm: + wait_for.main() + exception = cm.exception + self.assertFalse(exception.args[0]['changed']) + self.assertEqual(1, exception.args[0]["iterations"]) + + @patch.object(basic.AnsibleModule, 'run_command', + create_run_cmd( + providers=repeat(resource_provider_list_out, 1), + inventories=inventory_same_resource_different_traits, + traits=traits_same_resource_different_traits + )) + def test_main_success_same_resource_class_different_traits(self): + resource_a = get_dummy_resources( + traits=list(resource_provider_traits_subset), + amount=1 + ) + resource_b = get_dummy_resources( + traits=list(resource_provider_traits), + amount=1 + ) + dummy_module_arguments = get_dummy_module_args( + resources=resource_a + resource_b, + ) + set_module_args(dummy_module_arguments) + try: + wait_for.main() + except AnsibleExitJson as result: + self.assertFalse(result.args[0]['changed']) + self.assertEqual(1, result.args[0]["iterations"]) + else: + self.fail("Should have thrown AnsibleExitJson") + + @patch.object(basic.AnsibleModule, 'run_command', + create_run_cmd( + providers=repeat(resource_provider_list_out, 1), + # one per provider + inventories=repeat(inventory_list_out, 3), + traits=repeat(resource_provider_no_traits_out, 3) + )) + def test_main_success_no_traits(self): + # if the resource provider doesn't have any traits and no traits + # were requested, this should still pass + dummy_module_arguments = get_dummy_module_args( + resources=dummy_resources + ) + set_module_args(dummy_module_arguments) + try: + wait_for.main() + except AnsibleExitJson as result: + self.assertFalse(result.args[0]['changed']) + self.assertEqual(1, result.args[0]["iterations"]) + else: + self.fail("Should have thrown AnsibleExitJson") + + @patch.object(basic.AnsibleModule, 'run_command', + create_run_cmd( + providers=repeat(resource_provider_list_out, 2), + # one per provider * 2 iterations + inventories=inventory_resource_class_incorrect, + traits=repeat(resource_provider_no_traits_out, 6))) + @patch('time.sleep', noop) + def test_success_first_iteration_wrong_resource_class(self): + # different resource class to the one we are looking for on first + # iteration + dummy_module_arguments = get_dummy_module_args( + resources=dummy_resources + ) + set_module_args(dummy_module_arguments) + try: + wait_for.main() + except AnsibleExitJson as result: + self.assertFalse(result.args[0]['changed']) + self.assertEqual(2, result.args[0]["iterations"]) + else: + self.fail("Should have thrown AnsibleExitJson") + + @patch.object(basic.AnsibleModule, 'run_command', + create_run_cmd( + providers=repeat(resource_provider_list_out, 2), + inventories=inventory_resource_class_reserved, + # one per provider * 2 iterations + traits=repeat(resource_provider_no_traits_out, 6))) + @patch('time.sleep', noop) + def test_success_resource_class_reserved(self): + # resource class is reserved on the first iteration + dummy_module_arguments = get_dummy_module_args( + resources=dummy_resources + ) + set_module_args(dummy_module_arguments) + try: + wait_for.main() + except AnsibleExitJson as result: + self.assertFalse(result.args[0]['changed']) + self.assertEqual(2, result.args[0]["iterations"]) + else: + self.fail("Should have thrown AnsibleExitJson") + + @patch.object(basic.AnsibleModule, 'run_command', + create_run_cmd( + providers=repeat(resource_provider_list_out, 1), + # one per provider + inventories=repeat(inventory_list_out, 3), + traits=repeat(resource_provider_no_traits_out, 3) + )) + def test_validation_amount_is_str_repr_of_int(self): + dummy_module_arguments = get_dummy_module_args( + # amount should be an int + resources=get_dummy_resources(amount="3") + ) + set_module_args(dummy_module_arguments) + try: + wait_for.main() + except AnsibleExitJson as result: + self.assertFalse(result.args[0]['changed']) + self.assertEqual(1, result.args[0]["iterations"]) + else: + self.fail("Should have thrown AnsibleExitJson") + + def test_validation_amount_not_int(self): + dummy_module_arguments = get_dummy_module_args( + # amount should be an int + resources=get_dummy_resources(amount="not_a_number") + ) + set_module_args(dummy_module_arguments) + expected_msg = "amount, should be type int" + self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main) + + def test_validation_amount_missing(self): + resources = get_dummy_resources() + for resource in resources: + del resource["amount"] + dummy_module_arguments = get_dummy_module_args( + resources=resources + ) + set_module_args(dummy_module_arguments) + expected_msg = ( + "One of your resources does not have the field, amount, set" + ) + self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main) + + def test_merge_simple(self): + a = {'a': 1, 'b': 2} + b = {'a': 3, 'c': 5} + expected = {'a': 4, 'b': 2, 'c': 5} + merged = merge(a, b, lambda x, y: x + y) + self.assertDictEqual(expected, merged) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..2b10d07 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,60 @@ +import json + +import unittest + +from ansible.module_utils import basic +from ansible.module_utils._text import to_bytes + +# Python 2/3 compatibility. +try: + from unittest.mock import MagicMock, patch +except ImportError: + from mock import MagicMock, patch # noqa + + +def set_module_args(args): + if '_ansible_remote_tmp' not in args: + args['_ansible_remote_tmp'] = '/tmp' + if '_ansible_keep_remote_files' not in args: + args['_ansible_keep_remote_files'] = False + + args = json.dumps({'ANSIBLE_MODULE_ARGS': args}) + basic._ANSIBLE_ARGS = to_bytes(args) + + +class AnsibleExitJson(Exception): + pass + + +class AnsibleFailJson(Exception): + pass + + +def exit_json(*args, **kwargs): + if 'changed' not in kwargs: + kwargs['changed'] = False + raise AnsibleExitJson(kwargs) + + +def fail_json(*args, **kwargs): + kwargs['failed'] = True + raise AnsibleFailJson(kwargs) + + +class ModuleTestCase(unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(ModuleTestCase, self).__init__(*args, **kwargs) + # Python 2 / 3 compatibility. assertRaisesRegexp was renamed to + # assertRaisesRegex in python 3.1. + if hasattr(self, 'assertRaisesRegexp') and \ + not hasattr(self, 'assertRaisesRegex'): + self.assertRaisesRegex = self.assertRaisesRegexp + + def setUp(self): + self.mock_module = patch.multiple(basic.AnsibleModule, + exit_json=exit_json, + fail_json=fail_json) + self.mock_module.start() + set_module_args({}) + self.addCleanup(self.mock_module.stop) diff --git a/tox.ini b/tox.ini index e24c608..eb3609d 100644 --- a/tox.ini +++ b/tox.ini @@ -41,7 +41,7 @@ commands = sphinx-build -W -b html doc/source doc/build/html basepython = python3 setenv = VIRTUAL_ENV={envdir} - PYTHON=coverage run --source tenks,ansible/action_plugins --parallel-mode + PYTHON=coverage run --source tenks,ansible --parallel-mode commands = coverage erase stestr run {posargs}