111a38b4d6
Glance image import is asynchronous and may be configured to do image conversion. If image import is being used, it's possible that the tempest configuration code is executed before the import has completed and there may be no active images yet. In that case, we will poll glance every TEMPEST_GLANCE_IMPORT_POLL_INTERVAL seconds (default: 1) to see if there are TEMPEST_GLANCE_IMAGE_COUNT active images (default: 1) up to TEMPEST_GLANCE_IMPORT_POLL_LIMIT times (default: 12). You can see an example of the issue this patch addresses in real life: https://review.opendev.org/c/openstack/glance/+/841278/1#message-456096e48b28e5b866deb8bf53e9258ee08219a0 Change-Id: Ie99f12691d9062611a8930accfa14d9540970cc5
821 lines
36 KiB
Bash
821 lines
36 KiB
Bash
#!/bin/bash
|
|
#
|
|
# lib/tempest
|
|
# Install and configure Tempest
|
|
|
|
# Dependencies:
|
|
#
|
|
# - ``functions`` file
|
|
# - ``lib/nova`` service is running
|
|
# - Global vars that are assumed to be defined:
|
|
# - ``DEST``, ``FILES``
|
|
# - ``ADMIN_PASSWORD``
|
|
# - ``DEFAULT_IMAGE_NAME``
|
|
# - ``DEFAULT_IMAGE_FILE_NAME``
|
|
# - ``S3_SERVICE_PORT``
|
|
# - ``SERVICE_HOST``
|
|
# - ``BASE_SQL_CONN`` ``lib/database`` declares
|
|
# - ``PUBLIC_NETWORK_NAME``
|
|
# - ``VIRT_DRIVER``
|
|
# - ``LIBVIRT_TYPE``
|
|
# - ``KEYSTONE_SERVICE_URI``, ``KEYSTONE_SERVICE_URI_V3`` from lib/keystone
|
|
#
|
|
# Optional Dependencies:
|
|
#
|
|
# - ``ALT_*``
|
|
# - ``LIVE_MIGRATION_AVAILABLE``
|
|
# - ``USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION``
|
|
# - ``DEFAULT_INSTANCE_TYPE``
|
|
# - ``DEFAULT_INSTANCE_USER``
|
|
# - ``DEFAULT_INSTANCE_ALT_USER``
|
|
# - ``CINDER_ENABLED_BACKENDS``
|
|
# - ``NOVA_ALLOW_DUPLICATE_NETWORKS``
|
|
#
|
|
# ``stack.sh`` calls the entry points in this order:
|
|
#
|
|
# - install_tempest
|
|
# - configure_tempest
|
|
|
|
# Save trace setting
|
|
_XTRACE_TEMPEST=$(set +o | grep xtrace)
|
|
set +o xtrace
|
|
|
|
|
|
# Defaults
|
|
# --------
|
|
|
|
# Set up default directories
|
|
TEMPEST_DIR=$DEST/tempest
|
|
TEMPEST_CONFIG_DIR=${TEMPEST_CONFIG_DIR:-$TEMPEST_DIR/etc}
|
|
TEMPEST_CONFIG=$TEMPEST_CONFIG_DIR/tempest.conf
|
|
TEMPEST_STATE_PATH=${TEMPEST_STATE_PATH:=$DATA_DIR/tempest}
|
|
|
|
# This is the timeout that tempest will wait for a VM to change state,
|
|
# spawn, delete, etc.
|
|
# The default is set to 196 seconds.
|
|
BUILD_TIMEOUT=${BUILD_TIMEOUT:-196}
|
|
|
|
# This must be False on stable branches, as master tempest
|
|
# deps do not match stable branch deps. Set this to True to
|
|
# have tempest installed in DevStack by default.
|
|
INSTALL_TEMPEST=${INSTALL_TEMPEST:-"True"}
|
|
|
|
# This variable is passed directly to pip install inside the common tox venv
|
|
# that is created
|
|
TEMPEST_PLUGINS=${TEMPEST_PLUGINS:-0}
|
|
|
|
# Cinder/Volume variables
|
|
TEMPEST_VOLUME_DRIVER=${TEMPEST_VOLUME_DRIVER:-default}
|
|
TEMPEST_DEFAULT_VOLUME_VENDOR="Open Source"
|
|
TEMPEST_VOLUME_VENDOR=${TEMPEST_VOLUME_VENDOR:-$TEMPEST_DEFAULT_VOLUME_VENDOR}
|
|
TEMPEST_DEFAULT_STORAGE_PROTOCOL="iSCSI"
|
|
TEMPEST_STORAGE_PROTOCOL=${TEMPEST_STORAGE_PROTOCOL:-$TEMPEST_DEFAULT_STORAGE_PROTOCOL}
|
|
|
|
# Glance/Image variables
|
|
# When Glance image import is enabled, image creation is asynchronous and images
|
|
# may not yet be active when tempest looks for them. In that case, we poll
|
|
# Glance every TEMPEST_GLANCE_IMPORT_POLL_INTERVAL seconds for the number of
|
|
# times specified by TEMPEST_GLANCE_IMPORT_POLL_LIMIT. If you are importing
|
|
# multiple images, set TEMPEST_GLANCE_IMAGE_COUNT so the poller does not quit
|
|
# too early (though it will not exceed the polling limit).
|
|
TEMPEST_GLANCE_IMPORT_POLL_INTERVAL=${TEMPEST_GLANCE_IMPORT_POLL_INTERVAL:-1}
|
|
TEMPEST_GLANCE_IMPORT_POLL_LIMIT=${TEMPEST_GLANCE_IMPORT_POLL_LIMIT:-12}
|
|
TEMPEST_GLANCE_IMAGE_COUNT=${TEMPEST_GLANCE_IMAGE_COUNT:-1}
|
|
|
|
# Neutron/Network variables
|
|
IPV6_ENABLED=$(trueorfalse True IPV6_ENABLED)
|
|
IPV6_SUBNET_ATTRIBUTES_ENABLED=$(trueorfalse True IPV6_SUBNET_ATTRIBUTES_ENABLED)
|
|
|
|
# Do we want to make a configuration where Tempest has admin on
|
|
# the cloud. We don't always want to so that we can ensure Tempest
|
|
# would work on a public cloud.
|
|
TEMPEST_HAS_ADMIN=$(trueorfalse True TEMPEST_HAS_ADMIN)
|
|
|
|
# Credential provider configuration option variables
|
|
TEMPEST_ALLOW_TENANT_ISOLATION=${TEMPEST_ALLOW_TENANT_ISOLATION:-$TEMPEST_HAS_ADMIN}
|
|
TEMPEST_USE_TEST_ACCOUNTS=$(trueorfalse False TEMPEST_USE_TEST_ACCOUNTS)
|
|
|
|
# The number of workers tempest is expected to be run with. This is used for
|
|
# generating a accounts.yaml for running with test-accounts. This is also the
|
|
# same variable that devstack-gate uses to specify the number of workers that
|
|
# it will run tempest with
|
|
TEMPEST_CONCURRENCY=${TEMPEST_CONCURRENCY:-$(nproc)}
|
|
|
|
# Functions
|
|
# ---------
|
|
|
|
# remove_disabled_extension - removes disabled extensions from the list of extensions
|
|
# to test for a given service
|
|
function remove_disabled_extensions {
|
|
local extensions_list=$1
|
|
shift
|
|
local disabled_exts=$*
|
|
remove_disabled_services "$extensions_list" "$disabled_exts"
|
|
}
|
|
|
|
# image_size_in_gib - converts an image size from bytes to GiB, rounded up
|
|
# Takes an image ID parameter as input
|
|
function image_size_in_gib {
|
|
local size
|
|
size=$(openstack --os-cloud devstack-admin image show $1 -c size -f value)
|
|
echo $size | python3 -c "import math; print(int(math.ceil(float(int(input()) / 1024.0 ** 3))))"
|
|
}
|
|
|
|
function set_tempest_venv_constraints {
|
|
local tmp_c
|
|
tmp_c=$1
|
|
if [[ $TEMPEST_VENV_UPPER_CONSTRAINTS == "master" ]]; then
|
|
(cd $REQUIREMENTS_DIR &&
|
|
git show master:upper-constraints.txt 2>/dev/null ||
|
|
git show origin/master:upper-constraints.txt) > $tmp_c
|
|
else
|
|
echo "Using $TEMPEST_VENV_UPPER_CONSTRAINTS constraints in Tempest virtual env."
|
|
cat $TEMPEST_VENV_UPPER_CONSTRAINTS > $tmp_c
|
|
# NOTE: setting both tox env var and once Tempest start using new var
|
|
# TOX_CONSTRAINTS_FILE then we can remove the old one.
|
|
export UPPER_CONSTRAINTS_FILE=$TEMPEST_VENV_UPPER_CONSTRAINTS
|
|
export TOX_CONSTRAINTS_FILE=$TEMPEST_VENV_UPPER_CONSTRAINTS
|
|
fi
|
|
}
|
|
|
|
# Makes a call to glance to get a list of active images, ignoring
|
|
# ramdisk and kernel images. Takes 3 arguments, an array and two
|
|
# variables. The array will contain the list of active image UUIDs;
|
|
# if an image with ``DEFAULT_IMAGE_NAME`` is found, its UUID will be
|
|
# set as the value of *both* other parameters.
|
|
function get_active_images {
|
|
declare -n img_array=$1
|
|
declare -n img_id=$2
|
|
declare -n img_id_alt=$3
|
|
|
|
# start with a fresh array in case we are called multiple times
|
|
img_array=()
|
|
|
|
while read -r IMAGE_NAME IMAGE_UUID; do
|
|
if [ "$IMAGE_NAME" = "$DEFAULT_IMAGE_NAME" ]; then
|
|
img_id="$IMAGE_UUID"
|
|
img_id_alt="$IMAGE_UUID"
|
|
fi
|
|
img_array+=($IMAGE_UUID)
|
|
done < <(openstack --os-cloud devstack-admin image list --property status=active | awk -F'|' '!/^(+--)|ID|aki|ari/ { print $3,$2 }')
|
|
}
|
|
|
|
function poll_glance_images {
|
|
declare -n image_array=$1
|
|
declare -n image_id=$2
|
|
declare -n image_id_alt=$3
|
|
local -i poll_count
|
|
|
|
poll_count=$TEMPEST_GLANCE_IMPORT_POLL_LIMIT
|
|
while (( poll_count-- > 0 )) ; do
|
|
sleep $TEMPEST_GLANCE_IMPORT_POLL_INTERVAL
|
|
get_active_images image_array image_id image_id_alt
|
|
if (( ${#image_array[*]} >= $TEMPEST_GLANCE_IMAGE_COUNT )) ; then
|
|
return
|
|
fi
|
|
done
|
|
local msg
|
|
msg="Polling limit of $TEMPEST_GLANCE_IMPORT_POLL_LIMIT exceeded; "
|
|
msg+="poll interval was $TEMPEST_GLANCE_IMPORT_POLL_INTERVAL sec"
|
|
warn $LINENO "$msg"
|
|
}
|
|
|
|
# configure_tempest() - Set config files, create data dirs, etc
|
|
function configure_tempest {
|
|
if [[ "$INSTALL_TEMPEST" == "True" ]]; then
|
|
setup_develop $TEMPEST_DIR
|
|
else
|
|
# install testr since its used to process tempest logs
|
|
pip_install_gr testrepository
|
|
fi
|
|
|
|
local image_lines
|
|
local images
|
|
local num_images
|
|
local image_uuid
|
|
local image_uuid_alt
|
|
local password
|
|
local line
|
|
local flavors
|
|
local available_flavors
|
|
local flavors_ref
|
|
local flavor_lines
|
|
local flavor_ref_size
|
|
local flavor_ref_alt_size
|
|
local public_network_id
|
|
local public_router_id
|
|
local ssh_connect_method="floating"
|
|
local disk
|
|
|
|
# Save IFS
|
|
ifs=$IFS
|
|
|
|
# Glance should already contain images to be used in tempest
|
|
# testing. Here we simply look for images stored in Glance
|
|
# and set the appropriate variables for use in the tempest config
|
|
# We ignore ramdisk and kernel images, look for the default image
|
|
# ``DEFAULT_IMAGE_NAME``. If not found, we set the ``image_uuid`` to the
|
|
# first image returned and set ``image_uuid_alt`` to the second,
|
|
# if there is more than one returned...
|
|
# ... Also ensure we only take active images, so we don't get snapshots in process
|
|
declare -a images
|
|
|
|
if is_service_enabled glance; then
|
|
get_active_images images image_uuid image_uuid_alt
|
|
|
|
if (( ${#images[*]} < $TEMPEST_GLANCE_IMAGE_COUNT )); then
|
|
# Glance image import is asynchronous and may be configured
|
|
# to do image conversion. If image import is being used,
|
|
# it's possible that this code is being executed before the
|
|
# import has completed and there may be no active images yet.
|
|
if [[ "$GLANCE_USE_IMPORT_WORKFLOW" == "True" ]]; then
|
|
poll_glance_images images image_uuid image_uuid_alt
|
|
if (( ${#images[*]} < $TEMPEST_GLANCE_IMAGE_COUNT )); then
|
|
echo "Only found ${#images[*]} image(s), was looking for $TEMPEST_GLANCE_IMAGE_COUNT"
|
|
exit 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
case "${#images[*]}" in
|
|
0)
|
|
echo "Found no valid images to use!"
|
|
exit 1
|
|
;;
|
|
1)
|
|
if [ -z "$image_uuid" ]; then
|
|
image_uuid=${images[0]}
|
|
image_uuid_alt=${images[0]}
|
|
fi
|
|
;;
|
|
*)
|
|
if [ -z "$image_uuid" ]; then
|
|
image_uuid=${images[0]}
|
|
image_uuid_alt=${images[1]}
|
|
fi
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
# (Re)create ``tempest.conf``
|
|
# Create every time because the image UUIDS are going to change
|
|
sudo install -d -o $STACK_USER $TEMPEST_CONFIG_DIR
|
|
rm -f $TEMPEST_CONFIG
|
|
|
|
local password=${ADMIN_PASSWORD:-secret}
|
|
|
|
# See ``lib/keystone`` where these users and tenants are set up
|
|
local admin_username=${ADMIN_USERNAME:-admin}
|
|
local admin_project_name=${ADMIN_TENANT_NAME:-admin}
|
|
local admin_domain_name=${ADMIN_DOMAIN_NAME:-Default}
|
|
local alt_username=${ALT_USERNAME:-alt_demo}
|
|
local alt_project_name=${ALT_TENANT_NAME:-alt_demo}
|
|
local admin_project_id
|
|
admin_project_id=$(openstack --os-cloud devstack-admin project list | awk "/ admin / { print \$2 }")
|
|
|
|
if is_service_enabled nova; then
|
|
# If ``DEFAULT_INSTANCE_TYPE`` is not declared, use the new behavior
|
|
# Tempest creates its own instance types
|
|
available_flavors=$(openstack --os-cloud devstack-admin flavor list)
|
|
if [[ -z "$DEFAULT_INSTANCE_TYPE" ]]; then
|
|
if [[ ! ( $available_flavors =~ 'm1.nano' ) ]]; then
|
|
# Determine the flavor disk size based on the image size.
|
|
disk=$(image_size_in_gib $image_uuid)
|
|
openstack --os-cloud devstack-admin flavor create --id 42 --ram 128 --disk $disk --vcpus 1 --property hw_rng:allowed=True m1.nano
|
|
fi
|
|
flavor_ref=42
|
|
if [[ ! ( $available_flavors =~ 'm1.micro' ) ]]; then
|
|
# Determine the alt flavor disk size based on the alt image size.
|
|
disk=$(image_size_in_gib $image_uuid_alt)
|
|
openstack --os-cloud devstack-admin flavor create --id 84 --ram 192 --disk $disk --vcpus 1 --property hw_rng:allowed=True m1.micro
|
|
fi
|
|
flavor_ref_alt=84
|
|
else
|
|
# Check Nova for existing flavors, if ``DEFAULT_INSTANCE_TYPE`` is set use it.
|
|
IFS=$'\r\n'
|
|
flavors=""
|
|
for line in $available_flavors; do
|
|
f=$(echo $line | awk "/ $DEFAULT_INSTANCE_TYPE / { print \$2 }")
|
|
flavors="$flavors $f"
|
|
done
|
|
|
|
for line in $available_flavors; do
|
|
flavors="$flavors `echo $line | grep -v "^\(|\s*ID\|+--\)" | cut -d' ' -f2`"
|
|
done
|
|
|
|
IFS=" "
|
|
flavors=($flavors)
|
|
num_flavors=${#flavors[*]}
|
|
echo "Found $num_flavors flavors"
|
|
if [[ $num_flavors -eq 0 ]]; then
|
|
echo "Found no valid flavors to use!"
|
|
exit 1
|
|
fi
|
|
flavor_ref=${flavors[0]}
|
|
flavor_ref_alt=$flavor_ref
|
|
flavor_ref_size=$(openstack --os-cloud devstack-admin flavor show --format value --column disk "${flavor_ref}")
|
|
|
|
# Ensure ``flavor_ref`` and ``flavor_ref_alt`` have different values.
|
|
# Some resize instance in tempest tests depends on this.
|
|
for f in ${flavors[@]:1}; do
|
|
if [[ "$f" != "$flavor_ref" ]]; then
|
|
#
|
|
# NOTE(sdatko): Resize is only possible when target flavor
|
|
# is not smaller than the original one. For
|
|
# Tempest tests, in case there was a bigger
|
|
# flavor selected as default, e.g. m1.small,
|
|
# we need to perform additional check.
|
|
#
|
|
flavor_ref_alt_size=$(openstack --os-cloud devstack-admin flavor show --format value --column disk "${f}")
|
|
if [[ "${flavor_ref_alt_size}" -lt "${flavor_ref_size}" ]]; then
|
|
continue
|
|
fi
|
|
|
|
flavor_ref_alt=$f
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
fi
|
|
|
|
iniset $TEMPEST_CONFIG network project_network_cidr $FIXED_RANGE
|
|
|
|
ssh_connect_method=${TEMPEST_SSH_CONNECT_METHOD:-$ssh_connect_method}
|
|
|
|
# the public network (for floating ip access) is only available
|
|
# if the extension is enabled.
|
|
# If NEUTRON_CREATE_INITIAL_NETWORKS is not true, there is no network created
|
|
# and the public_network_id should not be set.
|
|
if [[ "$NEUTRON_CREATE_INITIAL_NETWORKS" == "True" ]] && is_networking_extension_supported 'external-net'; then
|
|
public_network_id=$(openstack --os-cloud devstack-admin network show -f value -c id $PUBLIC_NETWORK_NAME)
|
|
# make sure shared network presence does not confuses the tempest tests
|
|
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create --share shared
|
|
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create --description shared-subnet --subnet-range ${TEMPEST_SHARED_POOL:-192.168.233.0/24} --network shared shared-subnet
|
|
fi
|
|
|
|
iniset $TEMPEST_CONFIG DEFAULT use_syslog $SYSLOG
|
|
|
|
# Oslo
|
|
iniset $TEMPEST_CONFIG oslo_concurrency lock_path $TEMPEST_STATE_PATH
|
|
mkdir -p $TEMPEST_STATE_PATH
|
|
iniset $TEMPEST_CONFIG DEFAULT use_stderr False
|
|
iniset $TEMPEST_CONFIG DEFAULT log_file tempest.log
|
|
iniset $TEMPEST_CONFIG DEFAULT debug True
|
|
|
|
# Timeouts
|
|
iniset $TEMPEST_CONFIG compute build_timeout $BUILD_TIMEOUT
|
|
iniset $TEMPEST_CONFIG volume build_timeout $BUILD_TIMEOUT
|
|
|
|
# Identity
|
|
iniset $TEMPEST_CONFIG identity uri "$KEYSTONE_SERVICE_URI/v2.0/"
|
|
iniset $TEMPEST_CONFIG identity uri_v3 "$KEYSTONE_SERVICE_URI_V3"
|
|
iniset $TEMPEST_CONFIG identity user_lockout_failure_attempts $KEYSTONE_LOCKOUT_FAILURE_ATTEMPTS
|
|
iniset $TEMPEST_CONFIG identity user_lockout_duration $KEYSTONE_LOCKOUT_DURATION
|
|
iniset $TEMPEST_CONFIG identity user_unique_last_password_count $KEYSTONE_UNIQUE_LAST_PASSWORD_COUNT
|
|
if [[ "$TEMPEST_HAS_ADMIN" == "True" ]]; then
|
|
iniset $TEMPEST_CONFIG auth admin_username $admin_username
|
|
iniset $TEMPEST_CONFIG auth admin_password "$password"
|
|
iniset $TEMPEST_CONFIG auth admin_project_name $admin_project_name
|
|
iniset $TEMPEST_CONFIG auth admin_domain_name $admin_domain_name
|
|
fi
|
|
if [ "$ENABLE_IDENTITY_V2" == "True" ]; then
|
|
# Run Identity API v2 tests ONLY if needed
|
|
iniset $TEMPEST_CONFIG identity-feature-enabled api_v2 True
|
|
else
|
|
# Skip Identity API v2 tests by default
|
|
iniset $TEMPEST_CONFIG identity-feature-enabled api_v2 False
|
|
fi
|
|
iniset $TEMPEST_CONFIG identity auth_version ${TEMPEST_AUTH_VERSION:-v3}
|
|
if [[ "$TEMPEST_AUTH_VERSION" != "v2" ]]; then
|
|
# we're going to disable v2 admin unless we're using v2 by default.
|
|
iniset $TEMPEST_CONFIG identity-feature-enabled api_v2_admin False
|
|
fi
|
|
|
|
if is_service_enabled tls-proxy; then
|
|
iniset $TEMPEST_CONFIG identity ca_certificates_file $SSL_BUNDLE_FILE
|
|
fi
|
|
|
|
# Identity Features
|
|
if [[ "$KEYSTONE_SECURITY_COMPLIANCE_ENABLED" = True ]]; then
|
|
iniset $TEMPEST_CONFIG identity-feature-enabled security_compliance True
|
|
fi
|
|
|
|
# When LDAP is enabled domain specific drivers are also enabled and the users
|
|
# and groups identity tests must adapt to this scenario
|
|
if is_service_enabled ldap; then
|
|
iniset $TEMPEST_CONFIG identity-feature-enabled domain_specific_drivers True
|
|
fi
|
|
|
|
# TODO(felipemonteiro): Remove this once Tempest no longer supports Pike
|
|
# as this is supported in Queens and beyond.
|
|
iniset $TEMPEST_CONFIG identity-feature-enabled project_tags True
|
|
|
|
# In Queens and later, application credentials are enabled by default
|
|
# so remove this once Tempest no longer supports Pike.
|
|
iniset $TEMPEST_CONFIG identity-feature-enabled application_credentials True
|
|
|
|
# In Train and later, access rules for application credentials are enabled
|
|
# by default so remove this once Tempest no longer supports Stein.
|
|
iniset $TEMPEST_CONFIG identity-feature-enabled access_rules True
|
|
|
|
# Image
|
|
# We want to be able to override this variable in the gate to avoid
|
|
# doing an external HTTP fetch for this test.
|
|
if [[ ! -z "$TEMPEST_HTTP_IMAGE" ]]; then
|
|
iniset $TEMPEST_CONFIG image http_image $TEMPEST_HTTP_IMAGE
|
|
fi
|
|
iniset $TEMPEST_CONFIG image-feature-enabled import_image $GLANCE_USE_IMPORT_WORKFLOW
|
|
iniset $TEMPEST_CONFIG image-feature-enabled os_glance_reserved True
|
|
if is_service_enabled g-api-r; then
|
|
iniset $TEMPEST_CONFIG image alternate_image_endpoint image_remote
|
|
fi
|
|
|
|
# Compute
|
|
iniset $TEMPEST_CONFIG compute image_ref $image_uuid
|
|
iniset $TEMPEST_CONFIG compute image_ref_alt $image_uuid_alt
|
|
iniset $TEMPEST_CONFIG compute flavor_ref $flavor_ref
|
|
iniset $TEMPEST_CONFIG compute flavor_ref_alt $flavor_ref_alt
|
|
iniset $TEMPEST_CONFIG validation connect_method $ssh_connect_method
|
|
if ! is_service_enabled neutron; then
|
|
iniset $TEMPEST_CONFIG compute fixed_network_name $PRIVATE_NETWORK_NAME
|
|
fi
|
|
|
|
# Set the service catalog entry for Tempest to run on. Typically
|
|
# used to try different compute API version targets. The tempest
|
|
# default if 'compute', which is typically valid, so only set this
|
|
# if you want to change it.
|
|
if [[ -n "$TEMPEST_COMPUTE_TYPE" ]]; then
|
|
iniset $TEMPEST_CONFIG compute catalog_type $TEMPEST_COMPUTE_TYPE
|
|
fi
|
|
|
|
# Compute Features
|
|
# Set the microversion range for compute tests.
|
|
# This is used to run the Nova microversions tests.
|
|
# Setting [None, latest] range of microversion which allow Tempest to run all microversions tests.
|
|
# NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_compute_max_microversion"
|
|
# for stable branch on each release which should be changed from "latest" to max supported version of that release.
|
|
local tempest_compute_min_microversion=${TEMPEST_COMPUTE_MIN_MICROVERSION:-None}
|
|
local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"latest"}
|
|
# Reset microversions to None where v2.0 is running which does not support microversion.
|
|
# Both "None" means no microversion testing.
|
|
if [[ "$TEMPEST_COMPUTE_TYPE" == "compute_legacy" ]]; then
|
|
tempest_compute_min_microversion=None
|
|
tempest_compute_max_microversion=None
|
|
fi
|
|
if [ "$tempest_compute_min_microversion" == "None" ]; then
|
|
inicomment $TEMPEST_CONFIG compute min_microversion
|
|
else
|
|
iniset $TEMPEST_CONFIG compute min_microversion $tempest_compute_min_microversion
|
|
fi
|
|
if [ "$tempest_compute_max_microversion" == "None" ]; then
|
|
inicomment $TEMPEST_CONFIG compute max_microversion
|
|
else
|
|
iniset $TEMPEST_CONFIG compute max_microversion $tempest_compute_max_microversion
|
|
fi
|
|
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled personality ${ENABLE_FILE_INJECTION:-False}
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled resize True
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled live_migration ${LIVE_MIGRATION_AVAILABLE:-False}
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled change_password False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled block_migration_for_live_migration ${USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION:-False}
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled live_migrate_back_and_forth ${LIVE_MIGRATE_BACK_AND_FORTH:-False}
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled attach_encrypted_volume ${ATTACH_ENCRYPTED_VOLUME_AVAILABLE:-True}
|
|
|
|
# Starting Wallaby, nova sanitizes instance hostnames having freeform characters with dashes
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled hostname_fqdn_sanitization True
|
|
|
|
if [[ -n "$NOVA_FILTERS" ]]; then
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled scheduler_enabled_filters ${NOVA_FILTERS}
|
|
fi
|
|
|
|
if [[ $ENABLE_VOLUME_MULTIATTACH == "True" ]]; then
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled volume_multiattach True
|
|
fi
|
|
|
|
if is_service_enabled n-novnc; then
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled vnc_console True
|
|
fi
|
|
|
|
# Network
|
|
iniset $TEMPEST_CONFIG network project_networks_reachable false
|
|
iniset $TEMPEST_CONFIG network public_network_id "$public_network_id"
|
|
iniset $TEMPEST_CONFIG network public_router_id "$public_router_id"
|
|
iniset $TEMPEST_CONFIG network default_network "$FIXED_RANGE"
|
|
iniset $TEMPEST_CONFIG network-feature-enabled ipv6 "$IPV6_ENABLED"
|
|
iniset $TEMPEST_CONFIG network-feature-enabled ipv6_subnet_attributes "$IPV6_SUBNET_ATTRIBUTES_ENABLED"
|
|
iniset $TEMPEST_CONFIG network-feature-enabled port_security $NEUTRON_PORT_SECURITY
|
|
|
|
iniset $TEMPEST_CONFIG enforce_scope neutron "$NEUTRON_ENFORCE_SCOPE"
|
|
|
|
# Scenario
|
|
SCENARIO_IMAGE_DIR=${SCENARIO_IMAGE_DIR:-$FILES}
|
|
SCENARIO_IMAGE_FILE=$DEFAULT_IMAGE_FILE_NAME
|
|
iniset $TEMPEST_CONFIG scenario img_file $SCENARIO_IMAGE_DIR/$SCENARIO_IMAGE_FILE
|
|
|
|
# If using provider networking, use the physical network for validation rather than private
|
|
TEMPEST_SSH_NETWORK_NAME=$PRIVATE_NETWORK_NAME
|
|
if is_provider_network; then
|
|
TEMPEST_SSH_NETWORK_NAME=$PHYSICAL_NETWORK
|
|
fi
|
|
# Validation
|
|
iniset $TEMPEST_CONFIG validation run_validation ${TEMPEST_RUN_VALIDATION:-True}
|
|
iniset $TEMPEST_CONFIG validation ip_version_for_ssh 4
|
|
iniset $TEMPEST_CONFIG validation ssh_timeout $BUILD_TIMEOUT
|
|
iniset $TEMPEST_CONFIG validation image_ssh_user ${DEFAULT_INSTANCE_USER:=cirros}
|
|
iniset $TEMPEST_CONFIG validation image_alt_ssh_user ${DEFAULT_INSTANCE_ALT_USER:-$DEFAULT_INSTANCE_USER}
|
|
iniset $TEMPEST_CONFIG validation network_for_ssh $TEMPEST_SSH_NETWORK_NAME
|
|
|
|
# Volume
|
|
# Only turn on TEMPEST_VOLUME_MANAGE_SNAPSHOT by default for "lvm" backends
|
|
if [[ "$CINDER_ENABLED_BACKENDS" == *"lvm"* ]]; then
|
|
TEMPEST_VOLUME_MANAGE_SNAPSHOT=${TEMPEST_VOLUME_MANAGE_SNAPSHOT:-True}
|
|
fi
|
|
iniset $TEMPEST_CONFIG volume-feature-enabled manage_snapshot $(trueorfalse False TEMPEST_VOLUME_MANAGE_SNAPSHOT)
|
|
# Only turn on TEMPEST_VOLUME_MANAGE_VOLUME by default for "lvm" backends
|
|
if [[ "$CINDER_ENABLED_BACKENDS" == *"lvm"* ]]; then
|
|
TEMPEST_VOLUME_MANAGE_VOLUME=${TEMPEST_VOLUME_MANAGE_VOLUME:-True}
|
|
fi
|
|
iniset $TEMPEST_CONFIG volume-feature-enabled manage_volume $(trueorfalse False TEMPEST_VOLUME_MANAGE_VOLUME)
|
|
# Only turn on TEMPEST_EXTEND_ATTACHED_VOLUME by default for "lvm" backends
|
|
# in Cinder and the libvirt driver in Nova.
|
|
if [[ "$CINDER_ENABLED_BACKENDS" == *"lvm"* ]] && [ "$VIRT_DRIVER" = "libvirt" ]; then
|
|
TEMPEST_EXTEND_ATTACHED_VOLUME=${TEMPEST_EXTEND_ATTACHED_VOLUME:-True}
|
|
fi
|
|
iniset $TEMPEST_CONFIG volume-feature-enabled extend_attached_volume $(trueorfalse False TEMPEST_EXTEND_ATTACHED_VOLUME)
|
|
# Only turn on TEMPEST_VOLUME_REVERT_TO_SNAPSHOT by default for "lvm" backends
|
|
if [[ "$CINDER_ENABLED_BACKENDS" == *"lvm"* ]]; then
|
|
TEMPEST_VOLUME_REVERT_TO_SNAPSHOT=${TEMPEST_VOLUME_REVERT_TO_SNAPSHOT:-True}
|
|
fi
|
|
iniset $TEMPEST_CONFIG volume-feature-enabled volume_revert $(trueorfalse False TEMPEST_VOLUME_REVERT_TO_SNAPSHOT)
|
|
local tempest_volume_min_microversion=${TEMPEST_VOLUME_MIN_MICROVERSION:-None}
|
|
local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"latest"}
|
|
if [ "$tempest_volume_min_microversion" == "None" ]; then
|
|
inicomment $TEMPEST_CONFIG volume min_microversion
|
|
else
|
|
iniset $TEMPEST_CONFIG volume min_microversion $tempest_volume_min_microversion
|
|
fi
|
|
|
|
if [ "$tempest_volume_max_microversion" == "None" ]; then
|
|
inicomment $TEMPEST_CONFIG volume max_microversion
|
|
else
|
|
iniset $TEMPEST_CONFIG volume max_microversion $tempest_volume_max_microversion
|
|
fi
|
|
|
|
if ! is_service_enabled c-bak; then
|
|
iniset $TEMPEST_CONFIG volume-feature-enabled backup False
|
|
fi
|
|
|
|
# Using ``CINDER_ENABLED_BACKENDS``
|
|
# Cinder uses a comma separated list with "type:backend_name":
|
|
# CINDER_ENABLED_BACKENDS = ceph:cephBE1,lvm:lvmBE2,foo:my_foo
|
|
if [[ -n "$CINDER_ENABLED_BACKENDS" ]] && [[ $CINDER_ENABLED_BACKENDS =~ .*,.* ]]; then
|
|
# We have at least 2 backends
|
|
iniset $TEMPEST_CONFIG volume-feature-enabled multi_backend "True"
|
|
local add_comma_seperator=0
|
|
local backends_list=''
|
|
local be
|
|
# Tempest uses a comma separated list of backend_names:
|
|
# backend_names = BACKEND_1,BACKEND_2
|
|
for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
|
|
if [ "$add_comma_seperator" -eq "1" ]; then
|
|
backends_list+=,${be##*:}
|
|
else
|
|
# first element in the list
|
|
backends_list+=${be##*:}
|
|
add_comma_seperator=1
|
|
fi
|
|
done
|
|
iniset $TEMPEST_CONFIG volume "backend_names" "$backends_list"
|
|
fi
|
|
|
|
if [ $TEMPEST_VOLUME_DRIVER != "default" -o \
|
|
"$TEMPEST_VOLUME_VENDOR" != "$TEMPEST_DEFAULT_VOLUME_VENDOR" ]; then
|
|
iniset $TEMPEST_CONFIG volume vendor_name "$TEMPEST_VOLUME_VENDOR"
|
|
fi
|
|
if [ $TEMPEST_VOLUME_DRIVER != "default" -o \
|
|
"$TEMPEST_STORAGE_PROTOCOL" != "$TEMPEST_DEFAULT_STORAGE_PROTOCOL" ]; then
|
|
iniset $TEMPEST_CONFIG volume storage_protocol "$TEMPEST_STORAGE_PROTOCOL"
|
|
fi
|
|
|
|
# Placement Features
|
|
# Set the microversion range for placement.
|
|
# Setting [None, latest] range of microversion which allow Tempest to run all microversions tests.
|
|
# NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_placement_max_microversion"
|
|
# for stable branch on each release which should be changed from "latest" to max supported version of that release.
|
|
local tempest_placement_min_microversion=${TEMPEST_PLACEMENT_MIN_MICROVERSION:-None}
|
|
local tempest_placement_max_microversion=${TEMPEST_PLACEMENT_MAX_MICROVERSION:-"latest"}
|
|
if [ "$tempest_placement_min_microversion" == "None" ]; then
|
|
inicomment $TEMPEST_CONFIG placement min_microversion
|
|
else
|
|
iniset $TEMPEST_CONFIG placement min_microversion $tempest_placement_min_microversion
|
|
fi
|
|
if [ "$tempest_placement_max_microversion" == "None" ]; then
|
|
inicomment $TEMPEST_CONFIG placement max_microversion
|
|
else
|
|
iniset $TEMPEST_CONFIG placement max_microversion $tempest_placement_max_microversion
|
|
fi
|
|
|
|
# Baremetal
|
|
if [ "$VIRT_DRIVER" = "ironic" ] ; then
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled change_password False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled console_output False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled interface_attach False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled live_migration False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled pause False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled rescue False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled resize False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled shelve False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled snapshot False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled suspend False
|
|
fi
|
|
|
|
# Libvirt
|
|
if [ "$VIRT_DRIVER" = "libvirt" ]; then
|
|
# Libvirt-LXC
|
|
if [ "$LIBVIRT_TYPE" = "lxc" ]; then
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled rescue False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled resize False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled shelve False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled snapshot False
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled suspend False
|
|
else
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled shelve_migrate True
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled stable_rescue True
|
|
iniset $TEMPEST_CONFIG compute-feature-enabled swap_volume True
|
|
fi
|
|
fi
|
|
|
|
# ``service_available``
|
|
#
|
|
# this tempest service list needs to be the services that
|
|
# tempest own, otherwise we can have an erroneous set of
|
|
# defaults (something defaulting true in Tempest, but not listed here).
|
|
# services tested by tempest plugins needs to be set on service devstack
|
|
# plugin side as devstack cannot keep track of all the tempest plugins
|
|
# services. Refer Bug#1743688 for more details.
|
|
# 'horizon' is also kept here as no devtack plugin for horizon.
|
|
local service
|
|
local tempest_services="key,glance,nova,neutron,cinder,swift,horizon"
|
|
for service in ${tempest_services//,/ }; do
|
|
if is_service_enabled $service ; then
|
|
iniset $TEMPEST_CONFIG service_available $service "True"
|
|
else
|
|
iniset $TEMPEST_CONFIG service_available $service "False"
|
|
fi
|
|
done
|
|
|
|
# ``enforce_scope``
|
|
# If services enable the enforce_scope for their policy
|
|
# we need to enable the same on Tempest side so that
|
|
# test can be run with scoped token.
|
|
if [[ "$KEYSTONE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
|
|
iniset $TEMPEST_CONFIG enforce_scope keystone true
|
|
iniset $TEMPEST_CONFIG auth admin_system 'all'
|
|
iniset $TEMPEST_CONFIG auth admin_project_name ''
|
|
fi
|
|
|
|
if [[ "$GLANCE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
|
|
iniset $TEMPEST_CONFIG enforce_scope glance true
|
|
fi
|
|
|
|
if [[ "$CINDER_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
|
|
iniset $TEMPEST_CONFIG enforce_scope cinder true
|
|
fi
|
|
|
|
if [ "$VIRT_DRIVER" = "libvirt" ] && [ "$LIBVIRT_TYPE" = "lxc" ]; then
|
|
# libvirt-lxc does not support boot from volume or attaching volumes
|
|
# so basically anything with cinder is out of the question.
|
|
iniset $TEMPEST_CONFIG service_available cinder "False"
|
|
fi
|
|
|
|
# Run tempest configuration utilities. This must be done last during configuration to
|
|
# ensure as complete a config as possible already exists
|
|
|
|
# NOTE(mtreinish): Respect constraints on tempest verify-config venv
|
|
local tmp_cfg_file
|
|
tmp_cfg_file=$(mktemp)
|
|
cd $TEMPEST_DIR
|
|
if [[ "$OFFLINE" != "True" ]]; then
|
|
tox -revenv-tempest --notest
|
|
fi
|
|
|
|
local tmp_u_c_m
|
|
tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX)
|
|
set_tempest_venv_constraints $tmp_u_c_m
|
|
tox -evenv-tempest -- pip install -c $tmp_u_c_m -r requirements.txt
|
|
rm -f $tmp_u_c_m
|
|
|
|
# Auth:
|
|
if [[ $TEMPEST_USE_TEST_ACCOUNTS == "True" ]]; then
|
|
if [[ $TEMPEST_HAS_ADMIN == "True" ]]; then
|
|
tox -evenv-tempest -- tempest account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-project-name $admin_project_name -r $TEMPEST_CONCURRENCY --with-admin etc/accounts.yaml
|
|
else
|
|
tox -evenv-tempest -- tempest account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-project-name $admin_project_name -r $TEMPEST_CONCURRENCY etc/accounts.yaml
|
|
fi
|
|
iniset $TEMPEST_CONFIG auth use_dynamic_credentials False
|
|
iniset $TEMPEST_CONFIG auth test_accounts_file "etc/accounts.yaml"
|
|
elif [[ $TEMPEST_HAS_ADMIN == "False" ]]; then
|
|
iniset $TEMPEST_CONFIG auth use_dynamic_credentials ${TEMPEST_ALLOW_TENANT_ISOLATION:-False}
|
|
|
|
else
|
|
iniset $TEMPEST_CONFIG auth use_dynamic_credentials ${TEMPEST_ALLOW_TENANT_ISOLATION:-True}
|
|
fi
|
|
|
|
# API Extensions
|
|
# Run ``verify_tempest_config -ur`` to retrieve enabled extensions on API endpoints
|
|
# NOTE(mtreinish): This must be done after auth settings are added to the tempest config
|
|
tox -evenv -- tempest verify-config -uro $tmp_cfg_file
|
|
|
|
# Neutron API Extensions
|
|
|
|
# disable metering if we didn't enable the service
|
|
if ! is_service_enabled q-metering; then
|
|
DISABLE_NETWORK_API_EXTENSIONS+=", metering"
|
|
fi
|
|
|
|
# disable l3_agent_scheduler if we didn't enable L3 agent
|
|
if ! is_service_enabled q-l3; then
|
|
DISABLE_NETWORK_API_EXTENSIONS+=", l3_agent_scheduler"
|
|
fi
|
|
|
|
local network_api_extensions=${NETWORK_API_EXTENSIONS:-"all"}
|
|
if [[ ! -z "$DISABLE_NETWORK_API_EXTENSIONS" ]]; then
|
|
# Enabled extensions are either the ones explicitly specified or those available on the API endpoint
|
|
network_api_extensions=${NETWORK_API_EXTENSIONS:-$(iniget $tmp_cfg_file network-feature-enabled api_extensions | tr -d " ")}
|
|
# Remove disabled extensions
|
|
network_api_extensions=$(remove_disabled_extensions $network_api_extensions $DISABLE_NETWORK_API_EXTENSIONS)
|
|
fi
|
|
if [[ -n "$ADDITIONAL_NETWORK_API_EXTENSIONS" ]] && [[ "$network_api_extensions" != "all" ]]; then
|
|
network_api_extensions+=",$ADDITIONAL_NETWORK_API_EXTENSIONS"
|
|
fi
|
|
iniset $TEMPEST_CONFIG network-feature-enabled api_extensions $network_api_extensions
|
|
# Swift API Extensions
|
|
local object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-"all"}
|
|
if [[ ! -z "$DISABLE_OBJECT_STORAGE_API_EXTENSIONS" ]]; then
|
|
# Enabled extensions are either the ones explicitly specified or those available on the API endpoint
|
|
object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-$(iniget $tmp_cfg_file object-storage-feature-enabled discoverable_apis | tr -d " ")}
|
|
# Remove disabled extensions
|
|
object_storage_api_extensions=$(remove_disabled_extensions $object_storage_api_extensions $DISABLE_STORAGE_API_EXTENSIONS)
|
|
fi
|
|
iniset $TEMPEST_CONFIG object-storage-feature-enabled discoverable_apis $object_storage_api_extensions
|
|
# Cinder API Extensions
|
|
local volume_api_extensions=${VOLUME_API_EXTENSIONS:-"all"}
|
|
if [[ ! -z "$DISABLE_VOLUME_API_EXTENSIONS" ]]; then
|
|
# Enabled extensions are either the ones explicitly specified or those available on the API endpoint
|
|
volume_api_extensions=${VOLUME_API_EXTENSIONS:-$(iniget $tmp_cfg_file volume-feature-enabled api_extensions | tr -d " ")}
|
|
# Remove disabled extensions
|
|
volume_api_extensions=$(remove_disabled_extensions $volume_api_extensions $DISABLE_VOLUME_API_EXTENSIONS)
|
|
fi
|
|
iniset $TEMPEST_CONFIG volume-feature-enabled api_extensions $volume_api_extensions
|
|
|
|
# Restore IFS
|
|
IFS=$ifs
|
|
}
|
|
|
|
# install_tempest() - Collect source and prepare
|
|
function install_tempest {
|
|
git_clone $TEMPEST_REPO $TEMPEST_DIR $TEMPEST_BRANCH
|
|
pip_install 'tox!=2.8.0'
|
|
pushd $TEMPEST_DIR
|
|
# NOTE(gmann): checkout the TEMPEST_BRANCH in case TEMPEST_BRANCH
|
|
# is tag name not master. git_clone would not checkout tag because
|
|
# TEMPEST_DIR already exist until RECLONE is true.
|
|
git checkout $TEMPEST_BRANCH
|
|
|
|
local tmp_u_c_m
|
|
tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX)
|
|
set_tempest_venv_constraints $tmp_u_c_m
|
|
|
|
tox -r --notest -efull
|
|
# NOTE(mtreinish) Respect constraints in the tempest full venv, things that
|
|
# are using a tox job other than full will not be respecting constraints but
|
|
# running pip install -U on tempest requirements
|
|
$TEMPEST_DIR/.tox/tempest/bin/pip install -c $tmp_u_c_m -r requirements.txt
|
|
PROJECT_VENV["tempest"]=${TEMPEST_DIR}/.tox/tempest
|
|
rm -f $tmp_u_c_m
|
|
popd
|
|
}
|
|
|
|
# install_tempest_plugins() - Install any specified plugins into the tempest venv
|
|
function install_tempest_plugins {
|
|
pushd $TEMPEST_DIR
|
|
if [[ $TEMPEST_PLUGINS != 0 ]] ; then
|
|
local tmp_u_c_m
|
|
tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX)
|
|
set_tempest_venv_constraints $tmp_u_c_m
|
|
tox -evenv-tempest -- pip install -c $tmp_u_c_m $TEMPEST_PLUGINS
|
|
rm -f $tmp_u_c_m
|
|
echo "Checking installed Tempest plugins:"
|
|
tox -evenv-tempest -- tempest list-plugins
|
|
fi
|
|
popd
|
|
}
|
|
|
|
# Restore xtrace
|
|
$_XTRACE_TEMPEST
|
|
|
|
# Tell emacs to use shell-script-mode
|
|
## Local variables:
|
|
## mode: shell-script
|
|
## End:
|