Create script that finds candidate tests for refstack
This patch introduces script that finds candidate tests from
tempest (and tempest plugins) that can be used for refstack testing.
The script lists all test from tempest (and tempest plugins) except those
that:
- are/were part of interop repository or
- are mentioned in exclude file or
- are scenario tests or
- require admin access.
Change-Id: I7c087bca458943b3ff8b6fc098626af466210474
This commit is contained in:
lpiwowar
parent
f4c568cf94
commit
7a9916beed
254
tools/cross_check/cross_check.sh
Executable file
254
tools/cross_check/cross_check.sh
Executable file
@ -0,0 +1,254 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# This script is useful when it is time to find
|
||||||
|
# new candidate tests for refstack. It lists
|
||||||
|
# all API tests from tempest, designate-tempest-plugin,
|
||||||
|
# heat-tempest-plugin and manila tempest-plugin except
|
||||||
|
# those that:
|
||||||
|
# * are/were part of interop repository or
|
||||||
|
# * are mentioned in exclude file (see option --exclude-file) or
|
||||||
|
# * are scenario tests or
|
||||||
|
# * require admin access.
|
||||||
|
#
|
||||||
|
# The list of tests that is produced by this script is
|
||||||
|
# good start to find new tests but not all tests listed
|
||||||
|
# by the script must necessarily be added to the refstack.
|
||||||
|
|
||||||
|
function print_help() {
|
||||||
|
SCRIPT_NAME="basename ${BASH_SOURCE[0]}"
|
||||||
|
echo "Usage: ${SCRIPT_NAME} [OPTION] ..."
|
||||||
|
echo "Find missing test in interop repository."
|
||||||
|
echo ""
|
||||||
|
echo "--interop-url URL to download interop git repository"
|
||||||
|
echo "--interop-commit ID of interop commit that should be used"
|
||||||
|
echo "--tempest-url URL to download tempest git repository"
|
||||||
|
echo "--tempest-commit ID of tempest commit that should be used"
|
||||||
|
echo "--designate-url URL to download designate-tempest-plugin git repository"
|
||||||
|
echo "--designate-commit ID of designate-tempest-plugin commit that should be used"
|
||||||
|
echo "--heat-url URL to download heat-tempest-plugin"
|
||||||
|
echo "--heat-commit ID of heat-tempest-plugin commit that should be used"
|
||||||
|
echo "--manila-url URL to download manila-tempest-plugin"
|
||||||
|
echo "--manila-commit ID of manila-tempest-plugin commit that should be used"
|
||||||
|
echo "--exclude-file File that contains tests that should be ignored"
|
||||||
|
echo " (Default: ./exclude_file.txt)"
|
||||||
|
}
|
||||||
|
|
||||||
|
TEMPEST_GIT_URL=https://opendev.org/openstack/tempest.git
|
||||||
|
TEMPEST_COMMIT=e64c78dcf720202a0542bb1e1184f5229a11524f
|
||||||
|
|
||||||
|
DESIGNATE_GIT_URL=https://opendev.org/openstack/designate-tempest-plugin.git
|
||||||
|
DESIGNATE_COMMIT=master
|
||||||
|
|
||||||
|
HEAT_GIT_URL=https://opendev.org/openstack/heat-tempest-plugin.git
|
||||||
|
HEAT_COMMIT=master
|
||||||
|
|
||||||
|
MANILA_GIT_URL=https://opendev.org/openstack/manila-tempest-plugin.git
|
||||||
|
MANILA_COMMIT=master
|
||||||
|
|
||||||
|
INTEROP_GIT_URL=https://opendev.org/osf/interop.git
|
||||||
|
INTEROP_COMMIT=master
|
||||||
|
|
||||||
|
TMP_DIR=$(mktemp -d)
|
||||||
|
SCRIPTDIR=$(dirname $0)
|
||||||
|
EXCLUDE_FILE="${SCRIPTDIR}/exclude_file.txt"
|
||||||
|
|
||||||
|
while [[ ! -z "$1" ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--interop-url)
|
||||||
|
INTEROP_GIT_URL="$2";
|
||||||
|
shift 2;;
|
||||||
|
--interop-commit)
|
||||||
|
INTEROP_COMMIT="$2";
|
||||||
|
shift 2;;
|
||||||
|
--tempest-url)
|
||||||
|
TEMPEST_GIT_URL="$2";
|
||||||
|
shift 2;;
|
||||||
|
--tempest-commit)
|
||||||
|
TEMPEST_COMMIT="$2";
|
||||||
|
shift 2;;
|
||||||
|
--designate-url)
|
||||||
|
DESIGNATE_GIT_URL="$2";
|
||||||
|
shift 2;;
|
||||||
|
--designate-commit)
|
||||||
|
DESIGNATE_COMMIT="$2";
|
||||||
|
shift 2;;
|
||||||
|
--heat-url)
|
||||||
|
HEAT_URL="$2";
|
||||||
|
shift 2;;
|
||||||
|
--heat-commit)
|
||||||
|
HEAT_COMMIT="$2";
|
||||||
|
shift 2;;
|
||||||
|
--manila-url)
|
||||||
|
MANILA_URL="$2";
|
||||||
|
shift 2;;
|
||||||
|
--manila-commit)
|
||||||
|
MANILA_COMMIT="$2"
|
||||||
|
shift 2;;
|
||||||
|
--exclude-file)
|
||||||
|
EXCLUDE_FILE="$2";
|
||||||
|
shift 2;;
|
||||||
|
--help)
|
||||||
|
print_help
|
||||||
|
exit;;
|
||||||
|
*)
|
||||||
|
echo "ERROR: Not valid option used ($1)";
|
||||||
|
exit;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ ! -z ${EXCLUDE_FILE} ]] && [[ -f ${EXCLUDE_FILE} ]]; then
|
||||||
|
EXCLUDE_FILE=$(realpath ${EXCLUDE_FILE})
|
||||||
|
elif [[ ! -z ${EXCLUDE_FILE} ]] && [[ ! -f ${EXCLUDE_FILE} ]]; then
|
||||||
|
echo "ERROR: Not valid exclude file (${EXCLUDE_FILE})"
|
||||||
|
exit
|
||||||
|
else
|
||||||
|
touch ${TMP_DIR}/exclude_file.txt
|
||||||
|
EXCLUDE_FILE="${TMP_DIR}/exclude_file.txt"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Temporary directory used by cross_check.sh: ${TMP_DIR}"
|
||||||
|
|
||||||
|
# Clone repositories
|
||||||
|
echo "Cloning interop repository ..."
|
||||||
|
git clone --quiet ${INTEROP_GIT_URL} ${TMP_DIR}/interop
|
||||||
|
cd ${TMP_DIR}/interop
|
||||||
|
git checkout --quiet ${INTEROP_COMMIT}
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
echo "Cloning tempest repository ..."
|
||||||
|
git clone --quiet ${TEMPEST_GIT_URL} ${TMP_DIR}/tempest
|
||||||
|
cd ${TMP_DIR}/tempest
|
||||||
|
git checkout --quiet ${TEMPEST_COMMIT}
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
echo "Cloning designate-tempest-plugin repository ..."
|
||||||
|
git clone --quiet ${DESIGNATE_GIT_URL} ${TMP_DIR}/designate-tempest-plugin
|
||||||
|
cd ${TMP_DIR}/designate-tempest-plugin
|
||||||
|
git checkout --quiet ${DESIGNATE_COMMIT}
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
echo "Cloning heat-tempest-plugin repository ..."
|
||||||
|
git clone --quiet ${HEAT_GIT_URL} ${TMP_DIR}/heat-tempest-plugin
|
||||||
|
cd ${TMP_DIR}/heat-tempest-plugin
|
||||||
|
git checkout --quiet ${HEAT_COMMIT}
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
echo "Cloning manila-tempest-plugin repository ..."
|
||||||
|
git clone --quiet ${MANILA_GIT_URL} ${TMP_DIR}/manila-tempest-plugin
|
||||||
|
cd ${TMP_DIR}/manila-tempest-plugin
|
||||||
|
git checkout --quiet ${MANILA_COMMIT}
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
########################################################
|
||||||
|
# Find and print missing tests from interop repository.
|
||||||
|
#
|
||||||
|
# GLOBALS:
|
||||||
|
# TMP_DIR, EXCLUDE_FILE
|
||||||
|
# ARGUMENTS:
|
||||||
|
# 1 - Path to the repository in which the missing
|
||||||
|
# tests should be found.
|
||||||
|
# OUTPUTS:
|
||||||
|
# List of tests that are part of passed repository
|
||||||
|
# but not part of interop and not mentioned in
|
||||||
|
# exclude file.
|
||||||
|
#
|
||||||
|
#######################################################
|
||||||
|
function cross_check() {
|
||||||
|
|
||||||
|
REPO_DIR=$1
|
||||||
|
|
||||||
|
# Get every idempotent id from the repository
|
||||||
|
grep -hr --exclude-dir="doc" "@decorators.idempotent_id('" ${REPO_DIR} \
|
||||||
|
| sed --expression="s/@decorators.idempotent_id('//g" \
|
||||||
|
| sed --expression="s/')//g" \
|
||||||
|
| sed --expression="s/ *//g" \
|
||||||
|
| sort > ${TMP_DIR}/repo_idempotent_ids.txt
|
||||||
|
|
||||||
|
# Get every idempotent id from interop
|
||||||
|
cd ${TMP_DIR}/interop/guidelines
|
||||||
|
cat $(ls -1 | grep -E "[0-9]{4}\.[0-9]{2}\.json|next.json") ${TMP_DIR}/interop/add-ons/guidelines/* \
|
||||||
|
| grep -oE "id-[[:alnum:]]{8}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{12}" \
|
||||||
|
| sed --expression='s/id-//g' \
|
||||||
|
| sort > ${TMP_DIR}/interop_idempotent_ids.txt
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
|
||||||
|
# Get ids of missing tests in interop
|
||||||
|
comm -23 ${TMP_DIR}/repo_idempotent_ids.txt ${TMP_DIR}/interop_idempotent_ids.txt \
|
||||||
|
| sort > ${TMP_DIR}/missing_idempotent_ids.txt
|
||||||
|
|
||||||
|
# Print missing tests to stdout
|
||||||
|
cd ${REPO_DIR}
|
||||||
|
while read ID; do
|
||||||
|
FILE=$(grep -rl "${ID}" && cd - > /dev/null)
|
||||||
|
|
||||||
|
FUNC_NAME=$(sed -e "1,/${ID}/d" ${FILE} \
|
||||||
|
| grep -m1 "def" \
|
||||||
|
| sed "s/def//g; s/(.*//g; s/ //g")
|
||||||
|
|
||||||
|
CLASS_NAME=$(sed "/${ID}/q" ${FILE} \
|
||||||
|
| grep "^class " \
|
||||||
|
| tail -1 \
|
||||||
|
| sed "s/class//g; s/(.*//g; s/ //g")
|
||||||
|
|
||||||
|
FILE_ID=$(echo ${FILE} | sed "s/\.py//g; s/\.\///g; s/\//\./g")
|
||||||
|
TEST_NAME="${FILE_ID}.${CLASS_NAME}.${FUNC_NAME}"
|
||||||
|
|
||||||
|
# Check if test:
|
||||||
|
# * requires admin access or
|
||||||
|
# * is already present in interop next files
|
||||||
|
# (necessary for older next files as they do not
|
||||||
|
# use idempotent ids) or
|
||||||
|
# * is scenario test or
|
||||||
|
# * is present in exclude file.
|
||||||
|
ADMIN_TEST=$(echo ${FILE_ID} | grep "\.admin\.")
|
||||||
|
SCENARIO_TEST=$(echo ${FILE_ID} | grep "\.scenario\.")
|
||||||
|
IN_INTEROP=$(grep -r ${TEST_NAME} ${TMP_DIR}/interop)
|
||||||
|
IN_EXCLUDE=$(cat ${EXCLUDE_FILE} | grep "${TEST_NAME}")
|
||||||
|
|
||||||
|
if [[ ! ${IN_INTEROP} ]] && \
|
||||||
|
[[ ! ${ADMIN_TEST} ]] && \
|
||||||
|
[[ ! ${IN_EXCLUDE} ]] && \
|
||||||
|
[[ ! ${SCENARIO_TEST} ]]
|
||||||
|
then
|
||||||
|
echo "${TEST_NAME}[${ID}]"
|
||||||
|
fi
|
||||||
|
|
||||||
|
done < ${TMP_DIR}/missing_idempotent_ids.txt
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "Searching tempest repository ..."
|
||||||
|
cross_check ${TMP_DIR}/tempest > ${TMP_DIR}/tempest_tests.txt
|
||||||
|
|
||||||
|
echo "Searching designate-tempest-plugin repository ..."
|
||||||
|
cross_check ${TMP_DIR}/designate-tempest-plugin > ${TMP_DIR}/designate_tests.txt
|
||||||
|
|
||||||
|
echo "Searching heat-tempest-plugin repository ..."
|
||||||
|
cross_check ${TMP_DIR}/heat-tempest-plugin > ${TMP_DIR}/heat_tests.txt
|
||||||
|
|
||||||
|
echo "Searching manila-tempest-plugin repository ..."
|
||||||
|
cross_check ${TMP_DIR}/manila-tempest-plugin > ${TMP_DIR}/manila_tests.txt
|
||||||
|
|
||||||
|
echo "Tempest tests:"
|
||||||
|
echo "--------------"
|
||||||
|
sort ${TMP_DIR}/tempest_tests.txt
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
echo "Designate-tempest-plugin tests:"
|
||||||
|
echo "-------------------------------"
|
||||||
|
sort ${TMP_DIR}/designate_tests.txt
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
echo "Heat-tempest-plugin tests:"
|
||||||
|
echo "--------------------------"
|
||||||
|
sort ${TMP_DIR}/heat_tests.txt
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
echo "Manila-tempest-plugin tests:"
|
||||||
|
echo "----------------------------"
|
||||||
|
sort ${TMP_DIR}/manila_tests.txt
|
||||||
|
|
||||||
|
# Cleanup
|
||||||
|
rm -rf ${TMP_DIR}
|
||||||
|
|
||||||
20
tools/cross_check/exclude_file.txt
Normal file
20
tools/cross_check/exclude_file.txt
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
# This file is used by cross_check.sh script. It contains list of tests
|
||||||
|
# that should be ignored by the script. Each test (or group of tests) should
|
||||||
|
# be accompanied by a comment explaining why the test should be ignored.
|
||||||
|
|
||||||
|
# Tests marked as deprecated (see: https://review.opendev.org/c/osf/interop/+/800795/)
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_already_associated_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_disassociate_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_allocate_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_delete_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_with_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_without_passing_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_nonexistent_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_dissociate_nonexistent_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_allocate_floating_ip_from_nonexistent_pool
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_delete_nonexistent_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_get_floating_ip_details
|
||||||
|
tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ip_pools
|
||||||
|
tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ips
|
||||||
|
tempest.api.compute.floating_ips.test_list_floating_ips_negative.FloatingIPDetailsNegativeTestJSON.test_get_nonexistent_floating_ip_details
|
||||||
|
|
||||||
Loading…
Reference in New Issue
Block a user