openstack/tenks
Code Issues Proposed changes

Wait for resources to become available

Browse Source 
This will wait for any resources that are created to become available
before exiting the script. This allows you to avoid a race condition
where a server could be created before the resource tracker had been
updated with the new resources; server creation would fail.

Change-Id: I57f8c93cb1ebbc284b96ef1ced2c4edd59b27795
Story: 2004274
Task: 27823
Depends-On: https://review.openstack.org/617642
This commit is contained in:
Will Szumski 2018-11-06 16:35:06 +00:00
parent 4f20ed32f6
commit d4e7e58619
10 changed files with 1255 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ansible/deploy.yml
View File

@ -28,5 +28,9 @@
import_playbook: flavor_registration.yml
tags: openstack
- name: Wait for resources to become available
import_playbook: resource_wait.yml
tags: openstack, resource
- name: Clean up Tenks state
import_playbook: cleanup_state.yml

46
ansible/resource_wait.yml Normal file
View File

@ -0,0 +1,46 @@
---
- hosts: localhost
tasks:
- name: Check that OpenStack credentials exist in the environment
fail:
msg: >
$OS_USERNAME was not found in the environment. Ensure the OpenStack
credentials exist in your environment, perhaps by sourcing your RC file.
when: not lookup('env', 'OS_USERNAME')
- name: Gather list of OpenStack services
command: >-
{{ virtualenv_path }}/bin/openstack service list -f json
register: service_list_output
changed_when: false
- block:
- name: Set default value for expected resources
set_fact:
tenks_expected_resources: []
- name: Build list of expected resources
# takes the form: [{ resource_class: CUSTOM_TEST_RC, amount: 2, traits: [] }, ]
vars:
resource:
amount: "{{ spec.count | int }}" # this gets converted back to a string
resource_class: "{{ 'CUSTOM_' ~ spec.ironic_config.resource_class | upper | replace('-', '_') }}"
traits: "{{ spec.ironic_config.traits | default([])}}"
set_fact:
tenks_expected_resources: >-
{{ tenks_expected_resources + [resource] }}
loop: "{{ specs }}"
when: "'ironic_config' in spec"
loop_control:
loop_var: spec
- name: Include the wait-for-resources role
include_role:
name: wait-for-resources
vars:
wait_for_resources_required_resources: "{{ tenks_expected_resources }}"
wait_for_resources_venv: "{{ virtualenv_path }}"
wait_for_resources_python_upper_constraints_url: >-
{{ python_upper_constraints_url }}
# Only attempt to wait for resources when the placement service is running
when: service_list_output.stdout | from_json | selectattr('Type', 'equalto', 'placement') | list | length >= 1

28
ansible/roles/wait-for-resources/defaults/main.yml Normal file
View File

@ -0,0 +1,28 @@
---
# A list of dictionaries, each with the following keys:
# * resource_class - the name of the resource class that you want to be
# available
# * traits - a list of traits a provider requires before that provider is
# considered to provide that resource
# * amount - natural number representing the amount of the resource that
# should be available. This will be tested against the total resource count
# minus the amount that is reserved.
wait_for_resources_required_resources:
# Virtualenv containing OpenStack client
wait_for_resources_venv:
# The resources are polled this many times. The delay between each iteration
# is also configurable, but defaults to 10 seconds
wait_for_resources_retry_limit: 15
# The delay between each iteration.
wait_for_resources_delay:
# The URL of the upper constraints file to pass to pip when installing Python
# packages.
wait_for_resources_python_upper_constraints_url:
# Path on the remote machine to copy the python requirements file to
wait_for_resources_python_requirements_path: /tmp/wait-for-python-requirements.txt

4
ansible/roles/wait-for-resources/files/requirements.txt Normal file
View File

@ -0,0 +1,4 @@
# This file contains the Python packages that are needed in the
# what-for-resource virtual env.
osc-placement>=1.3.0 # Apache License 2.0

383
ansible/roles/wait-for-resources/library/wait_for_resources.py Normal file
View File

@ -0,0 +1,383 @@
#!/usr/bin/python
# Copyright (c) 2018 StackHPC Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
DOCUMENTATION = """
module: wait_for_resources
short_description: Waits for a set of resources to become available in
placement
author: Will Szumski(will@stackhpc.com)
options:
- option-name: resources
description: List of dictionaries describing the resources that should
be available. Each dictionary should contain the keys: resource_class,
amount, and optionally, a list of traits that the provider needs to provide
before it is considered to provide the resource.
required: True
type: list
- option-name: venv
description: Path to a virtualenv containing the OpenStack CLI. It is a
requirement for the placement api plugin (osc-placement) to be installed.
required: False
type: string
- option-name: maximum_retries
description: The maximum number of iterations to poll for the resources
required: False
type: int
- option-name: delay
description: Delay between each iteration of the polling loop in seconds
required: False
type: int
requirements:
- python-openstackclient
- osc-placement
"""
EXAMPLES = """
- name: Wait for resources to become available
wait_for_resources:
resources: {'resource_class: CUSTOM_TEST_RC, 'traits': [], 'amount': 2}
delay: 10
maximum_retries: 15
venv: /path/to/venv
"""
RETURN = """
iterations:
description: The Number of iterations required before the resource became
available
returned: success
type: int
sample: 9
"""
# Need to disable PEP8, as it wants all imports at top of file
from ansible.module_utils.basic import AnsibleModule # noqa
from collections import namedtuple # noqa
import six # noqa
import os # noqa
import time # noqa
_RETRY_LIMIT_FAILURE_TEMPLATE = "exceeded retry limit of {max_retries} " \
"whilst waiting for resources to become " \
"available"
Specifier = namedtuple("Specifier", "name traits")
Provider = namedtuple("Provider", "uuid inventory_list traits")
# Store a list of import errors to report to the user.
IMPORT_ERRORS = []
try:
import json
except Exception as e:
IMPORT_ERRORS.append(e)
def meets_criteria(actual, requested):
"""For each resource, determines whether the total satisfies the amount
requested, ignoring any unrequested resource_classes"""
for specifier in requested.keys():
if specifier not in actual:
return False
if actual[specifier] < requested[specifier]:
return False
return True
def get_openstack_binary_path(module):
"""Returns the path to the openstack binary taking into account the
virtualenv that was specified (if available)"""
venv = module.params["venv"]
if venv:
return os.path.join(venv, "bin", "openstack")
# use openstack in PATH
return "openstack"
def get_inventory(module, provider_uuid):
"""
Gets inventory of resources for a give provider UUID
:param module: ansible module
:param provider_uuid: provider to query
:return: list of dictionaries of the form:
[
{
"allocation_ratio": 16.0,
"total": 24,
"reserved": 0,
"resource_class": "VCPU",
"step_size": 1,
"min_unit": 1,
"max_unit": 24
},
]
"""
cmd = "{openstack} resource provider inventory list {uuid} -f json" \
.format(uuid=provider_uuid,
openstack=get_openstack_binary_path(module))
rc, out, err = module.run_command(cmd)
if rc != 0:
msg = "{} failed with return code: {}, stderr: {}".format(cmd, rc, err)
module.fail_json(msg=msg)
return json.loads(out)
def get_providers(module):
"""
Gets a list of resource providers
:param module: ansible module
:return: list of dictionaries of the form:
[
{
"generation": 2,
"uuid": "657c4ab0-de82-4def-b7b0-d13ce672bfd0",
"name": "kayobe-will-master"
},
]
"""
cmd = "{openstack} resource provider list -f json".format(
openstack=get_openstack_binary_path(module)
)
rc, out, err = module.run_command(cmd)
if rc != 0:
msg = "{} failed with return code: {}, stderr: {}".format(cmd, rc, err)
module.fail_json(msg=msg)
return json.loads(out)
def get_traits(module, provider_uuid):
"""
Gets a list of traits for a resource provider
:param provider_uuid: the uuid of the provider
:param module: ansible module
:return: set of traits of the form:
{
"HW_CPU_X86_SSE2",
}
"""
cmd = "{openstack} resource provider trait list {uuid} " \
"--os-placement-api-version 1.6 -f json " \
.format(uuid=provider_uuid,
openstack=get_openstack_binary_path(module))
rc, out, err = module.run_command(cmd)
if rc != 0:
msg = "{} failed with return code: {}, stderr: {}".format(cmd, rc, err)
module.fail_json(msg=msg)
raw = json.loads(out)
return set([x["name"] for x in raw])
def merge(x, y, f):
""" Merges two dictionaries. If a key appears in both dictionaries, the
common values are merged using the function ``f``"""
# Start with symmetric difference; keys either in A or B, but not both
merged = {k: x.get(k, y.get(k)) for k in six.viewkeys(x) ^ six.viewkeys(y)}
# Update with `f()` applied to the intersection
merged.update(
{k: f(x[k], y[k]) for k in six.viewkeys(x) & six.viewkeys(y)})
return merged
def collect(specifiers, provider):
"""Given a specifier and a provider, gets the amount of resource that is
available for that given provider"""
inventory = {}
# we want to be able to look up items by resource_class
for item in provider.inventory_list:
inventory[item["resource_class"]] = item
result = {}
for specifier in specifiers:
if specifier.traits != provider.traits:
continue
if specifier.name in inventory:
reserved = inventory[specifier.name]["reserved"]
total_available = inventory[specifier.name]["total"]
result[specifier] = total_available - reserved
break
return result
def get_totals(specifiers, providers):
"""Loops over the providers adding up all of the resources that
are available"""
totals = {}
for specifier in specifiers:
# initialise totals so the combine function does not blow up
totals[specifier] = 0
for provider in providers:
current = collect(specifiers, provider)
totals = merge(totals, current, lambda x, y: x + y)
return totals
def are_resources_available(module, specifiers, expected):
"""
Determines whether or not a set of resources are available
:param module: Ansible module
:param specifiers: tuples of the form: (resource_class, traits)
:param expected: dictionary of the form:
{(resource_class, traits): amount }
:return: True if resource available, otherwise False
"""
providers_raw = get_providers(module)
providers = []
for provider in providers_raw:
uuid = provider["uuid"]
traits = get_traits(module, uuid)
inventory = get_inventory(module, uuid)
provider = Provider(
uuid=uuid,
inventory_list=inventory,
traits=traits
)
providers.append(provider)
actual = get_totals(specifiers, providers)
return meets_criteria(actual, expected)
def wait_for_resources(module):
"""
Waits for a set of resources to become available
:param module: Ansible module
:return: the number of attempts needed
"""
max_retries = module.params["maximum_retries"]
delay = module.params["delay"]
resources = module.params["resources"]
expected = {}
specifiers = []
for resource in resources:
# default value for traits
traits = resource["traits"] if resource["traits"] else []
specifier = Specifier(name=resource["resource_class"],
traits=frozenset(traits))
specifiers.append(specifier)
expected[specifier] = resource["amount"]
for i in range(max_retries):
if are_resources_available(module, specifiers, expected):
return i + 1
time.sleep(delay)
fail_msg = _RETRY_LIMIT_FAILURE_TEMPLATE.format(
max_retries=max_retries)
module.fail_json(msg=fail_msg)
def validate_spec(module, resource, field, type_):
check_resource_msg = (
"Please check the dictionaries in your resources list meet the "
"required specification.")
unset_field_msg = (
"One of your resources does not have the field, {field}, set."
)
field_type_msg = (
"The field, {field}, should be type {type}."
)
if field not in resource:
msg = "{} {}".format(unset_field_msg, check_resource_msg)
module.fail_json(msg=msg.format(field=field))
elif not isinstance(resource[field], type_):
msg = "{} {}".format(field_type_msg, check_resource_msg)
module.fail_json(msg=msg.format(field=field, type=type_.__name__))
def validate_resources(module, specs):
resources = module.params["resources"]
for resource in resources:
for spec in specs:
validate_spec(module, resource, spec["field"], type_=spec["type"])
def get_module():
"""Creates and returns an Ansible module"""
module = AnsibleModule(
argument_spec=dict(
resources=dict(required=True, type='list'),
maximum_retries=dict(required=False, type='int', default=15),
delay=dict(required=False, type='int', default=10),
venv=dict(required=False, type='str', default="")
),
supports_check_mode=True,
)
# note(wszumski): amount seems to get converted back to a string
# https://github.com/ansible/ansible/issues/18095
resources = module.params["resources"]
for resource in resources:
if 'amount' not in resource:
continue
amount = resource["amount"]
if isinstance(amount, str):
try:
amount_int = int(amount)
resource["amount"] = amount_int
except ValueError:
# this will get picked up in validate_resources
pass
# Validate resources list
resources_specs = [
{
'field': 'resource_class',
'type': str
},
{
'field': 'traits',
'type': list
},
{
'field': 'amount',
'type': int
},
]
validate_resources(module, resources_specs)
# Fail if there were any exceptions when importing modules.
if IMPORT_ERRORS:
module.fail_json(msg="Import errors: %s" %
", ".join([repr(e) for e in IMPORT_ERRORS]))
return module
def main():
module = get_module()
attempts_needed = 0
# In check mode, only perform validation of parameters
if not module.check_mode:
attempts_needed = wait_for_resources(module)
# This module doesn't really change anything, it just waits for
# something to change externally
result = {
"changed": False,
"iterations": attempts_needed
}
module.exit_json(**result)
if __name__ == '__main__':
main()

38
ansible/roles/wait-for-resources/tasks/main.yml Normal file
View File

@ -0,0 +1,38 @@
---
# Waits for for the resource tracker to be updated with a given resource
# class
- name: Validate that the virtualenv variable is set
fail:
msg: >-
You must set the variable, wait_for_resources_venv, to use this role.
when: wait_for_resources_venv is none
- name: Validate resources
include_tasks: validate.yml
vars:
wait_for_resources_amount: "{{ item.amount }}"
wait_for_resources_resource_class: "{{ item.resource_class }}"
loop: "{{ wait_for_resources_required_resources }}"
- name: Copy over requirements file
copy:
src: requirements.txt
dest: "{{ wait_for_resources_python_requirements_path }}"
- name: Install dependencies in supplied venv
pip:
requirements: "{{ wait_for_resources_python_requirements_path }}"
extra_args: >-
{%- if wait_for_resources_python_upper_constraints_url -%}
-c {{ wait_for_resources_python_upper_constraints_url }}
{%- endif -%}
virtualenv: "{{ wait_for_resources_venv }}"
- name: Call wait_for_resources module
wait_for_resources:
venv: "{{ wait_for_resources_venv }}"
resources: "{{ wait_for_resources_required_resources }}"
delay: "{{ wait_for_resources_delay | default(omit, true) }}"
maximum_retries: "{{ wait_for_resources_retry_limit }}"

17
ansible/roles/wait-for-resources/tasks/validate.yml Normal file
View File

@ -0,0 +1,17 @@
---
- name: Validate that the target resource class is set
fail:
msg: >-
Each item in the dictionary wait_for_resources_required_resources must contain
the key, resource_class.
when: wait_for_resources_class is none
- name: Validate expected resource count
vars:
count: "{{ wait_for_resources_amount | int(default=-1) }}"
fail:
msg: >-
Each item in the dictionary wait_for_resources_required_resources must contain
the key, amount.
when: wait_for_resources_amount is none or count | int < 0

674
tests/test_tenks_wait_for_resources.py Normal file
View File

@ -0,0 +1,674 @@
# Copyright (c) 2018 StackHPC Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import copy
import imp
import json
import os
import random
from itertools import repeat, chain, cycle
from ansible.module_utils import basic
from tests.utils import ModuleTestCase, set_module_args, AnsibleExitJson, \
AnsibleFailJson
# Python 2/3 compatibility.
try:
from unittest.mock import MagicMock, patch
except ImportError:
from mock import MagicMock, patch # noqa
# Import method lifted from kolla_ansible's test_merge_config.py
PROJECT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
PLUGIN_FILE = os.path.join(PROJECT_DIR,
'ansible/roles/wait-for-resources/library'
'/wait_for_resources.py')
wait_for = imp.load_source('wait_for_resources', PLUGIN_FILE)
meets_criteria = wait_for.meets_criteria
get_providers = wait_for.get_providers
get_inventory = wait_for.get_inventory
get_traits = wait_for.get_traits
merge = wait_for.merge
get_openstack_binary_path = wait_for.get_openstack_binary_path
Specifier = wait_for.Specifier
inventory_list_out = """[
{
"allocation_ratio": 1.0,
"total": 1,
"reserved": 0,
"resource_class": "CUSTOM_TEST_RC",
"step_size": 1,
"min_unit": 1,
"max_unit": 1
}
]""" # noqa
inventory_custom_b_out = """[
{
"allocation_ratio": 1.0,
"total": 1,
"reserved": 0,
"resource_class": "CUSTOM_B",
"step_size": 1,
"min_unit": 1,
"max_unit": 1
}
]""" # noqa
inventory_reserved_out = """[
{
"allocation_ratio": 1.0,
"total": 1,
"reserved": 1,
"resource_class": "CUSTOM_TEST_RC",
"step_size": 1,
"min_unit": 1,
"max_unit": 1
}
]""" # noqa
inventory_list = [{'allocation_ratio': 1.0,
'max_unit': 1,
'min_unit': 1,
'reserved': 0,
'resource_class': 'CUSTOM_TEST_RC',
'step_size': 1,
'total': 1}]
resource_provider_list_out = """[
{
"generation": 2,
"uuid": "657c4ab0-de82-4def-b7b0-d13ce672bfd0",
"name": "kayobe-will-master"
},
{
"generation": 1,
"uuid": "e2e78f98-d3ec-466a-862b-42d7ef5dca7d",
"name": "e2e78f98-d3ec-466a-862b-42d7ef5dca7d"
},
{
"generation": 1,
"uuid": "07072aea-cc2b-4135-a7e2-3a4dd3a9f629",
"name": "07072aea-cc2b-4135-a7e2-3a4dd3a9f629"
}
]""" # noqa
resource_provider_list = [{'generation': 2, 'name': 'kayobe-will-master',
'uuid': '657c4ab0-de82-4def-b7b0-d13ce672bfd0'},
{'generation': 1,
'name': 'e2e78f98-d3ec-466a-862b-42d7ef5dca7d',
'uuid': 'e2e78f98-d3ec-466a-862b-42d7ef5dca7d'},
{'generation': 1,
'name': '07072aea-cc2b-4135-a7e2-3a4dd3a9f629',
'uuid': '07072aea-cc2b-4135-a7e2-3a4dd3a9f629'}]
resource_provider_traits_out = """[
{
"name": "HW_CPU_X86_SSE2"
},
{
"name": "HW_CPU_X86_CLMUL"
},
{
"name": "HW_CPU_X86_SSE"
},
{
"name": "HW_CPU_X86_ABM"
},
{
"name": "HW_CPU_X86_MMX"
},
{
"name": "HW_CPU_X86_AVX2"
},
{
"name": "HW_CPU_X86_SSE41"
},
{
"name": "HW_CPU_X86_SSE42"
},
{
"name": "HW_CPU_X86_AESNI"
},
{
"name": "HW_CPU_X86_AVX"
},
{
"name": "HW_CPU_X86_VMX"
},
{
"name": "HW_CPU_X86_BMI2"
},
{
"name": "HW_CPU_X86_FMA3"
},
{
"name": "HW_CPU_X86_SSSE3"
},
{
"name": "HW_CPU_X86_F16C"
},
{
"name": "HW_CPU_X86_BMI"
}
]""" # noqa
resource_provider_no_traits_out = """[]
"""
resource_provider_traits = {'HW_CPU_X86_SSE2',
'HW_CPU_X86_CLMUL',
'HW_CPU_X86_SSE',
'HW_CPU_X86_ABM',
'HW_CPU_X86_MMX',
'HW_CPU_X86_AVX2',
'HW_CPU_X86_SSE41',
'HW_CPU_X86_SSE42',
'HW_CPU_X86_AESNI',
'HW_CPU_X86_AVX',
'HW_CPU_X86_VMX',
'HW_CPU_X86_BMI2',
'HW_CPU_X86_FMA3',
'HW_CPU_X86_SSSE3',
'HW_CPU_X86_F16C',
'HW_CPU_X86_BMI'}
def traits_to_json(traits):
return json.dumps(
[{'name': x} for x in traits]
)
def traits(*args):
# converts list to frozenset
return frozenset(*args)
resources_expected = {
Specifier('CUSTOM_B', traits()): 2,
Specifier('CUSTOM_A', traits('TRAIT_A')): 3
}
resources_not_enough = {
Specifier('CUSTOM_B', traits()): 1,
Specifier('CUSTOM_A', traits('TRAIT_A')): 2
}
resources_one_ok = {
Specifier('CUSTOM_B', traits()): 2,
Specifier('CUSTOM_A', traits('TRAIT_A')): 2
}
resources_both_ok = {
Specifier('CUSTOM_B', traits()): 2,
Specifier('CUSTOM_A', traits('TRAIT_A')): 3
}
resources_too_many = {
Specifier('CUSTOM_B', traits()): 5,
Specifier('CUSTOM_A', traits('TRAIT_A')): 6
}
resources_extra_keys = {
Specifier('CUSTOM_B', traits()): 2,
Specifier('CUSTOM_A', traits('TRAIT_A')): 3,
Specifier('CUSTOM_C', traits('TRAIT_A')): 3,
}
resources_key_missing = {
'CUSTOM_B': 2
}
dummy_resources = [
{
"resource_class": "CUSTOM_TEST_RC",
"traits": [],
"amount": 3
},
]
# Scenario definitions - we define these outside the test as we want to pass
# them into the 'patch' decorator
# scenario - resource class is incorrect on first iteration
inventory_resource_class_incorrect = chain(
repeat(inventory_custom_b_out, 3),
repeat(inventory_list_out, 3)
)
# scenario - resource class is reserved on first iteration
inventory_resource_class_reserved = chain(
repeat(inventory_reserved_out, 3),
repeat(inventory_list_out, 3)
)
# scenario - same resource class, different traits - success
# Arbitrary subset
resource_provider_traits_subset = random.sample(resource_provider_traits, 3)
resource_provider_traits_subset_out = traits_to_json(
resource_provider_traits_subset
)
inventory_same_resource_different_traits = chain(
# We are using a provider output with 3 providers, so we need three
# inventory strings - one for each
repeat(inventory_reserved_out, 1), repeat(inventory_list_out, 2)
)
traits_same_resource_different_traits = chain(
# We are using a provider output with 3 providers, so we need three
# trait strings - one for each
repeat(resource_provider_no_traits_out, 1),
repeat(resource_provider_traits_out, 1),
repeat(resource_provider_traits_subset_out, 1)
)
def get_traits_mocked(start=0, stop=None):
def mocked_function(_module, _provider):
all_traits = list(resource_provider_traits)
stop_ = stop if stop else len(all_traits)
return frozenset(all_traits[start:stop_])
return mocked_function
def get_dummy_resources(resource_class="CUSTOM_TEST_RC", amount=3,
traits=[]):
result = [
{
"resource_class": resource_class,
"traits": traits,
"amount": amount
},
]
return result
def get_dummy_module_args(resources,
maximum_retries=10,
delay=10,
venv="/my/dummy/venv"):
result = {
'resources': resources,
'maximum_retries': maximum_retries,
'delay': delay,
'venv': venv
}
return result
def pop_output(a):
next_ = next(a, None)
if next_:
return 0, next_, ""
else:
return 1, "", "Ran out of input"
def noop(*_args, **_kwargs):
pass
def create_run_cmd(providers, inventories, traits):
def dummy_run_command(*args, **_kwargs):
print(args)
if "resource provider list" in args[1]:
return pop_output(providers)
elif "resource provider trait list" in args[1]:
return pop_output(traits)
elif "resource provider inventory list" in args[1]:
return pop_output(inventories)
else:
raise ValueError("{} not expected", args)
return dummy_run_command
class TestTenksWaitForResource(ModuleTestCase):
def setUp(self):
super(TestTenksWaitForResource, self).setUp()
def test_meets_criteria_not_enough(self):
self.assertFalse(meets_criteria(actual=resources_not_enough,
requested=resources_expected))
def test_meets_criteria_one_ok(self):
self.assertFalse(meets_criteria(actual=resources_one_ok,
requested=resources_expected))
def test_meets_criteria_both_ok(self):
self.assertTrue(meets_criteria(actual=resources_both_ok,
requested=resources_expected))
def test_meets_criteria_too_many(self):
self.assertTrue(meets_criteria(actual=resources_too_many,
requested=resources_expected))
def test_meets_criteria_keys_we_dont_care_about(self):
self.assertTrue(meets_criteria(actual=resources_extra_keys,
requested=resources_expected))
def test_meets_criteria_missing_key(self):
self.assertFalse(meets_criteria(actual=resources_key_missing,
requested=resources_expected))
@patch.object(basic.AnsibleModule, 'run_command')
def test_resource_provider_list(self, run_command):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
run_command.return_value = 0, resource_provider_list_out, ''
set_module_args(dummy_module_arguments)
module = wait_for.get_module()
providers = get_providers(module)
calls = [call for call in run_command.call_args_list if
"openstack resource provider list" in call[0][0]]
self.assertGreater(len(calls), 0)
self.assertListEqual(resource_provider_list, providers)
@patch.object(basic.AnsibleModule, 'run_command')
def test_inventory_list(self, run_command):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
run_command.return_value = 0, inventory_list_out, ''
set_module_args(dummy_module_arguments)
module = wait_for.get_module()
expected = get_inventory(module, "provider-uuid")
calls = [call for call in run_command.call_args_list if
"openstack resource provider inventory list" in call[0][0]]
self.assertGreater(len(calls), 0)
self.assertListEqual(expected, inventory_list)
@patch.object(basic.AnsibleModule, 'run_command')
def test_traits(self, run_command):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
run_command.return_value = 0, resource_provider_traits_out, ''
set_module_args(dummy_module_arguments)
module = wait_for.get_module()
expected = get_traits(module, "provider-uuid")
calls = [call for call in run_command.call_args_list if
"openstack resource provider trait list" in call[0][0]]
self.assertGreater(len(calls), 0)
self.assertSetEqual(expected, resource_provider_traits)
def test_venv_path(self):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
set_module_args(dummy_module_arguments)
module = wait_for.get_module()
openstack_binary_path = get_openstack_binary_path(module)
venv = dummy_module_arguments["venv"]
expected = os.path.join(venv, "bin", "openstack")
self.assertEqual(expected, openstack_binary_path)
def test_venv_path_unset(self):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
args = copy.copy(dummy_module_arguments)
del args["venv"]
set_module_args(args)
module = wait_for.get_module()
openstack_binary_path = get_openstack_binary_path(module)
expected = "openstack"
self.assertEqual(expected, openstack_binary_path)
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=cycle(repeat(resource_provider_list_out, 1)),
inventories=cycle(repeat(inventory_reserved_out, 1)),
traits=cycle(repeat(resource_provider_no_traits_out, 3))
))
@patch('time.sleep', noop)
def test_main_failure_exhaust_retries(self):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
set_module_args(dummy_module_arguments)
expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format(
max_retries=dummy_module_arguments["maximum_retries"])
self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=cycle(repeat(resource_provider_list_out, 1)),
inventories=cycle(repeat(inventory_list_out, 1)),
traits=cycle(repeat(resource_provider_traits_out, 3))
))
@patch('time.sleep', noop)
@patch.object(wait_for, 'get_traits', get_traits_mocked(0, 2))
def test_main_failure_provider_does_not_provide_all_traits(
self):
expected_traits = list(resource_provider_traits)
resources = get_dummy_resources(traits=expected_traits)
dummy_module_arguments = get_dummy_module_args(
resources=resources,
)
set_module_args(dummy_module_arguments)
expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format(
max_retries=dummy_module_arguments["maximum_retries"])
self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=cycle(repeat(resource_provider_list_out, 1)),
inventories=cycle(repeat(inventory_list_out, 1)),
traits=cycle(repeat(resource_provider_traits_out, 3))
))
@patch('time.sleep', noop)
def test_main_failure_request_subset_of_traits(
self):
# pick an arbitrary sub range of traits
expected_traits = list(resource_provider_traits)[3:6]
resources = get_dummy_resources(traits=expected_traits)
dummy_module_arguments = get_dummy_module_args(
resources=resources,
)
set_module_args(dummy_module_arguments)
expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format(
max_retries=dummy_module_arguments["maximum_retries"])
self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=cycle(repeat(resource_provider_list_out, 1)),
inventories=cycle(repeat(inventory_reserved_out, 1)),
traits=cycle(repeat(resource_provider_no_traits_out, 3))
))
@patch('time.sleep', noop)
def test_main_failure_exhaust_retries_traits_not_matched(self):
resources = get_dummy_resources(traits=["WE_NEED_THIS"])
dummy_module_arguments = get_dummy_module_args(
resources=resources
)
set_module_args(dummy_module_arguments)
expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format(
max_retries=dummy_module_arguments["maximum_retries"])
self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=repeat(resource_provider_list_out, 1),
# one per provider
inventories=repeat(inventory_list_out, 3),
traits=repeat(resource_provider_no_traits_out, 3)
))
def test_main_success_one_iteration(self):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
set_module_args(dummy_module_arguments)
with self.assertRaises(AnsibleExitJson) as cm:
wait_for.main()
exception = cm.exception
self.assertFalse(exception.args[0]['changed'])
self.assertEqual(1, exception.args[0]["iterations"])
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=repeat(resource_provider_list_out, 1),
inventories=inventory_same_resource_different_traits,
traits=traits_same_resource_different_traits
))
def test_main_success_same_resource_class_different_traits(self):
resource_a = get_dummy_resources(
traits=list(resource_provider_traits_subset),
amount=1
)
resource_b = get_dummy_resources(
traits=list(resource_provider_traits),
amount=1
)
dummy_module_arguments = get_dummy_module_args(
resources=resource_a + resource_b,
)
set_module_args(dummy_module_arguments)
try:
wait_for.main()
except AnsibleExitJson as result:
self.assertFalse(result.args[0]['changed'])
self.assertEqual(1, result.args[0]["iterations"])
else:
self.fail("Should have thrown AnsibleExitJson")
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=repeat(resource_provider_list_out, 1),
# one per provider
inventories=repeat(inventory_list_out, 3),
traits=repeat(resource_provider_no_traits_out, 3)
))
def test_main_success_no_traits(self):
# if the resource provider doesn't have any traits and no traits
# were requested, this should still pass
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
set_module_args(dummy_module_arguments)
try:
wait_for.main()
except AnsibleExitJson as result:
self.assertFalse(result.args[0]['changed'])
self.assertEqual(1, result.args[0]["iterations"])
else:
self.fail("Should have thrown AnsibleExitJson")
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=repeat(resource_provider_list_out, 2),
# one per provider * 2 iterations
inventories=inventory_resource_class_incorrect,
traits=repeat(resource_provider_no_traits_out, 6)))
@patch('time.sleep', noop)
def test_success_first_iteration_wrong_resource_class(self):
# different resource class to the one we are looking for on first
# iteration
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
set_module_args(dummy_module_arguments)
try:
wait_for.main()
except AnsibleExitJson as result:
self.assertFalse(result.args[0]['changed'])
self.assertEqual(2, result.args[0]["iterations"])
else:
self.fail("Should have thrown AnsibleExitJson")
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=repeat(resource_provider_list_out, 2),
inventories=inventory_resource_class_reserved,
# one per provider * 2 iterations
traits=repeat(resource_provider_no_traits_out, 6)))
@patch('time.sleep', noop)
def test_success_resource_class_reserved(self):
# resource class is reserved on the first iteration
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
set_module_args(dummy_module_arguments)
try:
wait_for.main()
except AnsibleExitJson as result:
self.assertFalse(result.args[0]['changed'])
self.assertEqual(2, result.args[0]["iterations"])
else:
self.fail("Should have thrown AnsibleExitJson")
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=repeat(resource_provider_list_out, 1),
# one per provider
inventories=repeat(inventory_list_out, 3),
traits=repeat(resource_provider_no_traits_out, 3)
))
def test_validation_amount_is_str_repr_of_int(self):
dummy_module_arguments = get_dummy_module_args(
# amount should be an int
resources=get_dummy_resources(amount="3")
)
set_module_args(dummy_module_arguments)
try:
wait_for.main()
except AnsibleExitJson as result:
self.assertFalse(result.args[0]['changed'])
self.assertEqual(1, result.args[0]["iterations"])
else:
self.fail("Should have thrown AnsibleExitJson")
def test_validation_amount_not_int(self):
dummy_module_arguments = get_dummy_module_args(
# amount should be an int
resources=get_dummy_resources(amount="not_a_number")
)
set_module_args(dummy_module_arguments)
expected_msg = "amount, should be type int"
self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
def test_validation_amount_missing(self):
resources = get_dummy_resources()
for resource in resources:
del resource["amount"]
dummy_module_arguments = get_dummy_module_args(
resources=resources
)
set_module_args(dummy_module_arguments)
expected_msg = (
"One of your resources does not have the field, amount, set"
)
self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
def test_merge_simple(self):
a = {'a': 1, 'b': 2}
b = {'a': 3, 'c': 5}
expected = {'a': 4, 'b': 2, 'c': 5}
merged = merge(a, b, lambda x, y: x + y)
self.assertDictEqual(expected, merged)

60
tests/utils.py Normal file
View File

@ -0,0 +1,60 @@
import json
import unittest
from ansible.module_utils import basic
from ansible.module_utils._text import to_bytes
# Python 2/3 compatibility.
try:
from unittest.mock import MagicMock, patch
except ImportError:
from mock import MagicMock, patch # noqa
def set_module_args(args):
if '_ansible_remote_tmp' not in args:
args['_ansible_remote_tmp'] = '/tmp'
if '_ansible_keep_remote_files' not in args:
args['_ansible_keep_remote_files'] = False
args = json.dumps({'ANSIBLE_MODULE_ARGS': args})
basic._ANSIBLE_ARGS = to_bytes(args)
class AnsibleExitJson(Exception):
pass
class AnsibleFailJson(Exception):
pass
def exit_json(*args, **kwargs):
if 'changed' not in kwargs:
kwargs['changed'] = False
raise AnsibleExitJson(kwargs)
def fail_json(*args, **kwargs):
kwargs['failed'] = True
raise AnsibleFailJson(kwargs)
class ModuleTestCase(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(ModuleTestCase, self).__init__(*args, **kwargs)
# Python 2 / 3 compatibility. assertRaisesRegexp was renamed to
# assertRaisesRegex in python 3.1.
if hasattr(self, 'assertRaisesRegexp') and \
not hasattr(self, 'assertRaisesRegex'):
self.assertRaisesRegex = self.assertRaisesRegexp
def setUp(self):
self.mock_module = patch.multiple(basic.AnsibleModule,
exit_json=exit_json,
fail_json=fail_json)
self.mock_module.start()
set_module_args({})
self.addCleanup(self.mock_module.stop)

2
tox.ini
View File

@ -41,7 +41,7 @@ commands = sphinx-build -W -b html doc/source doc/build/html
basepython = python3
setenv =
VIRTUAL_ENV={envdir}
PYTHON=coverage run --source tenks,ansible/action_plugins --parallel-mode
PYTHON=coverage run --source tenks,ansible --parallel-mode
commands =
coverage erase
stestr run {posargs}