Merge "Create script that finds candidate tests for refstack"
This commit is contained in:
Zuul
Gerrit Code Review
commit
a93487957e
254
tools/cross_check/cross_check.sh
Executable file
254
tools/cross_check/cross_check.sh
Executable file
@ -0,0 +1,254 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# This script is useful when it is time to find
|
||||||
|
# new candidate tests for refstack. It lists
|
||||||
|
# all API tests from tempest, designate-tempest-plugin,
|
||||||
|
# heat-tempest-plugin and manila tempest-plugin except
|
||||||
|
# those that:
|
||||||
|
# * are/were part of interop repository or
|
||||||
|
# * are mentioned in exclude file (see option --exclude-file) or
|
||||||
|
# * are scenario tests or
|
||||||
|
# * require admin access.
|
||||||
|
#
|
||||||
|
# The list of tests that is produced by this script is
|
||||||
|
# good start to find new tests but not all tests listed
|
||||||
|
# by the script must necessarily be added to the refstack.
|
||||||
|
|
||||||
|
function print_help() {
|
||||||
|
SCRIPT_NAME="basename ${BASH_SOURCE[0]}"
|
||||||
|
echo "Usage: ${SCRIPT_NAME} [OPTION] ..."
|
||||||
|
echo "Find missing test in interop repository."
|
||||||
|
echo ""
|
||||||
|
echo "--interop-url URL to download interop git repository"
|
||||||
|
echo "--interop-commit ID of interop commit that should be used"
|
||||||
|
echo "--tempest-url URL to download tempest git repository"
|
||||||
|
echo "--tempest-commit ID of tempest commit that should be used"
|
||||||
|
echo "--designate-url URL to download designate-tempest-plugin git repository"
|
||||||
|
echo "--designate-commit ID of designate-tempest-plugin commit that should be used"
|
||||||
|
echo "--heat-url URL to download heat-tempest-plugin"
|
||||||
|
echo "--heat-commit ID of heat-tempest-plugin commit that should be used"
|
||||||
|
echo "--manila-url URL to download manila-tempest-plugin"
|
||||||
|
echo "--manila-commit ID of manila-tempest-plugin commit that should be used"
|
||||||
|
echo "--exclude-file File that contains tests that should be ignored"
|
||||||
|
echo " (Default: ./exclude_file.txt)"
|
||||||
|
}
|
||||||
|
|
||||||
|
TEMPEST_GIT_URL=https://opendev.org/openstack/tempest.git
|
||||||
|
TEMPEST_COMMIT=e64c78dcf720202a0542bb1e1184f5229a11524f
|
||||||
|
|
||||||
|
DESIGNATE_GIT_URL=https://opendev.org/openstack/designate-tempest-plugin.git
|
||||||
|
DESIGNATE_COMMIT=master
|
||||||
|
|
||||||
|
HEAT_GIT_URL=https://opendev.org/openstack/heat-tempest-plugin.git
|
||||||
|
HEAT_COMMIT=master
|
||||||
|
|
||||||
|
MANILA_GIT_URL=https://opendev.org/openstack/manila-tempest-plugin.git
|
||||||
|
MANILA_COMMIT=master
|
||||||
|
|
||||||
|
INTEROP_GIT_URL=https://opendev.org/osf/interop.git
|
||||||
|
INTEROP_COMMIT=master
|
||||||
|
|
||||||
|
TMP_DIR=$(mktemp -d)
|
||||||
|
SCRIPTDIR=$(dirname $0)
|
||||||
|
EXCLUDE_FILE="${SCRIPTDIR}/exclude_file.txt"
|
||||||
|
|
||||||
|
while [[ ! -z "$1" ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--interop-url)
|
||||||
|
INTEROP_GIT_URL="$2";
|
||||||
|
shift 2;;
|
||||||
|
--interop-commit)
|
||||||
|
INTEROP_COMMIT="$2";
|
||||||
|
shift 2;;
|
||||||
|
--tempest-url)
|
||||||
|
TEMPEST_GIT_URL="$2";
|
||||||
|
shift 2;;
|
||||||
|
--tempest-commit)
|
||||||
|
TEMPEST_COMMIT="$2";
|
||||||
|
shift 2;;
|
||||||
|
--designate-url)
|
||||||
|
DESIGNATE_GIT_URL="$2";
|
||||||
|
shift 2;;
|
||||||
|
--designate-commit)
|
||||||
|
DESIGNATE_COMMIT="$2";
|
||||||
|
shift 2;;
|
||||||
|
--heat-url)
|
||||||
|
HEAT_URL="$2";
|
||||||
|
shift 2;;
|
||||||
|
--heat-commit)
|
||||||
|
HEAT_COMMIT="$2";
|
||||||
|
shift 2;;
|
||||||
|
--manila-url)
|
||||||
|
MANILA_URL="$2";
|
||||||
|
shift 2;;
|
||||||
|
--manila-commit)
|
||||||
|
MANILA_COMMIT="$2"
|
||||||
|
shift 2;;
|
||||||
|
--exclude-file)
|
||||||
|
EXCLUDE_FILE="$2";
|
||||||
|
shift 2;;
|
||||||
|
--help)
|
||||||
|
print_help
|
||||||
|
exit;;
|
||||||
|
*)
|
||||||
|
echo "ERROR: Not valid option used ($1)";
|
||||||
|
exit;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ ! -z ${EXCLUDE_FILE} ]] && [[ -f ${EXCLUDE_FILE} ]]; then
|
||||||
|
EXCLUDE_FILE=$(realpath ${EXCLUDE_FILE})
|
||||||
|
elif [[ ! -z ${EXCLUDE_FILE} ]] && [[ ! -f ${EXCLUDE_FILE} ]]; then
|
||||||
|
echo "ERROR: Not valid exclude file (${EXCLUDE_FILE})"
|
||||||
|
exit
|
||||||
|
else
|
||||||
|
touch ${TMP_DIR}/exclude_file.txt
|
||||||
|
EXCLUDE_FILE="${TMP_DIR}/exclude_file.txt"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Temporary directory used by cross_check.sh: ${TMP_DIR}"
|
||||||
|
|
||||||
|
# Clone repositories
|
||||||
|
echo "Cloning interop repository ..."
|
||||||
|
git clone --quiet ${INTEROP_GIT_URL} ${TMP_DIR}/interop
|
||||||
|
cd ${TMP_DIR}/interop
|
||||||
|
git checkout --quiet ${INTEROP_COMMIT}
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
echo "Cloning tempest repository ..."
|
||||||
|
git clone --quiet ${TEMPEST_GIT_URL} ${TMP_DIR}/tempest
|
||||||
|
cd ${TMP_DIR}/tempest
|
||||||
|
git checkout --quiet ${TEMPEST_COMMIT}
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
echo "Cloning designate-tempest-plugin repository ..."
|
||||||
|
git clone --quiet ${DESIGNATE_GIT_URL} ${TMP_DIR}/designate-tempest-plugin
|
||||||
|
cd ${TMP_DIR}/designate-tempest-plugin
|
||||||
|
git checkout --quiet ${DESIGNATE_COMMIT}
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
echo "Cloning heat-tempest-plugin repository ..."
|
||||||
|
git clone --quiet ${HEAT_GIT_URL} ${TMP_DIR}/heat-tempest-plugin
|
||||||
|
cd ${TMP_DIR}/heat-tempest-plugin
|
||||||
|
git checkout --quiet ${HEAT_COMMIT}
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
echo "Cloning manila-tempest-plugin repository ..."
|
||||||
|
git clone --quiet ${MANILA_GIT_URL} ${TMP_DIR}/manila-tempest-plugin
|
||||||
|
cd ${TMP_DIR}/manila-tempest-plugin
|
||||||
|
git checkout --quiet ${MANILA_COMMIT}
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
########################################################
|
||||||
|
# Find and print missing tests from interop repository.
|
||||||
|
#
|
||||||
|
# GLOBALS:
|
||||||
|
# TMP_DIR, EXCLUDE_FILE
|
||||||
|
# ARGUMENTS:
|
||||||
|
# 1 - Path to the repository in which the missing
|
||||||
|
# tests should be found.
|
||||||
|
# OUTPUTS:
|
||||||
|
# List of tests that are part of passed repository
|
||||||
|
# but not part of interop and not mentioned in
|
||||||
|
# exclude file.
|
||||||
|
#
|
||||||
|
#######################################################
|
||||||
|
function cross_check() {
|
||||||
|
|
||||||
|
REPO_DIR=$1
|
||||||
|
|
||||||
|
# Get every idempotent id from the repository
|
||||||
|
grep -hr --exclude-dir="doc" "@decorators.idempotent_id('" ${REPO_DIR} \
|
||||||
|
| sed --expression="s/@decorators.idempotent_id('//g" \
|
||||||
|
| sed --expression="s/')//g" \
|
||||||
|
| sed --expression="s/ *//g" \
|
||||||
|
| sort > ${TMP_DIR}/repo_idempotent_ids.txt
|
||||||
|
|
||||||
|
# Get every idempotent id from interop
|
||||||
|
cd ${TMP_DIR}/interop/guidelines
|
||||||
|
cat $(ls -1 | grep -E "[0-9]{4}\.[0-9]{2}\.json|next.json") ${TMP_DIR}/interop/add-ons/guidelines/* \
|
||||||
|
| grep -oE "id-[[:alnum:]]{8}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{12}" \
|
||||||
|
| sed --expression='s/id-//g' \
|
||||||
|
| sort > ${TMP_DIR}/interop_idempotent_ids.txt
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
|
||||||
|
# Get ids of missing tests in interop
|
||||||
|
comm -23 ${TMP_DIR}/repo_idempotent_ids.txt ${TMP_DIR}/interop_idempotent_ids.txt \
|
||||||
|
| sort > ${TMP_DIR}/missing_idempotent_ids.txt
|
||||||
|
|
||||||
|
# Print missing tests to stdout
|
||||||
|
cd ${REPO_DIR}
|
||||||
|
while read ID; do
|
||||||
|
FILE=$(grep -rl "${ID}" && cd - > /dev/null)
|
||||||
|
|
||||||
|
FUNC_NAME=$(sed -e "1,/${ID}/d" ${FILE} \
|
||||||
|
| grep -m1 "def" \
|
||||||
|
| sed "s/def//g; s/(.*//g; s/ //g")
|
||||||
|
|
||||||
|
CLASS_NAME=$(sed "/${ID}/q" ${FILE} \
|
||||||
|
| grep "^class " \
|
||||||
|
| tail -1 \
|
||||||
|
| sed "s/class//g; s/(.*//g; s/ //g")
|
||||||
|
|
||||||
|
FILE_ID=$(echo ${FILE} | sed "s/\.py//g; s/\.\///g; s/\//\./g")
|
||||||
|
TEST_NAME="${FILE_ID}.${CLASS_NAME}.${FUNC_NAME}"
|
||||||
|
|
||||||
|
# Check if test:
|
||||||
|
# * requires admin access or
|
||||||
|
# * is already present in interop next files
|
||||||
|
# (necessary for older next files as they do not
|
||||||
|
# use idempotent ids) or
|
||||||
|
# * is scenario test or
|
||||||
|
# * is present in exclude file.
|
||||||
|
ADMIN_TEST=$(echo ${FILE_ID} | grep "\.admin\.")
|
||||||
|
SCENARIO_TEST=$(echo ${FILE_ID} | grep "\.scenario\.")
|
||||||
|
IN_INTEROP=$(grep -r ${TEST_NAME} ${TMP_DIR}/interop)
|
||||||
|
IN_EXCLUDE=$(cat ${EXCLUDE_FILE} | grep "${TEST_NAME}")
|
||||||
|
|
||||||
|
if [[ ! ${IN_INTEROP} ]] && \
|
||||||
|
[[ ! ${ADMIN_TEST} ]] && \
|
||||||
|
[[ ! ${IN_EXCLUDE} ]] && \
|
||||||
|
[[ ! ${SCENARIO_TEST} ]]
|
||||||
|
then
|
||||||
|
echo "${TEST_NAME}[${ID}]"
|
||||||
|
fi
|
||||||
|
|
||||||
|
done < ${TMP_DIR}/missing_idempotent_ids.txt
|
||||||
|
cd - &> /dev/null
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "Searching tempest repository ..."
|
||||||
|
cross_check ${TMP_DIR}/tempest > ${TMP_DIR}/tempest_tests.txt
|
||||||
|
|
||||||
|
echo "Searching designate-tempest-plugin repository ..."
|
||||||
|
cross_check ${TMP_DIR}/designate-tempest-plugin > ${TMP_DIR}/designate_tests.txt
|
||||||
|
|
||||||
|
echo "Searching heat-tempest-plugin repository ..."
|
||||||
|
cross_check ${TMP_DIR}/heat-tempest-plugin > ${TMP_DIR}/heat_tests.txt
|
||||||
|
|
||||||
|
echo "Searching manila-tempest-plugin repository ..."
|
||||||
|
cross_check ${TMP_DIR}/manila-tempest-plugin > ${TMP_DIR}/manila_tests.txt
|
||||||
|
|
||||||
|
echo "Tempest tests:"
|
||||||
|
echo "--------------"
|
||||||
|
sort ${TMP_DIR}/tempest_tests.txt
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
echo "Designate-tempest-plugin tests:"
|
||||||
|
echo "-------------------------------"
|
||||||
|
sort ${TMP_DIR}/designate_tests.txt
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
echo "Heat-tempest-plugin tests:"
|
||||||
|
echo "--------------------------"
|
||||||
|
sort ${TMP_DIR}/heat_tests.txt
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
echo "Manila-tempest-plugin tests:"
|
||||||
|
echo "----------------------------"
|
||||||
|
sort ${TMP_DIR}/manila_tests.txt
|
||||||
|
|
||||||
|
# Cleanup
|
||||||
|
rm -rf ${TMP_DIR}
|
||||||
|
|
||||||
20
tools/cross_check/exclude_file.txt
Normal file
20
tools/cross_check/exclude_file.txt
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
# This file is used by cross_check.sh script. It contains list of tests
|
||||||
|
# that should be ignored by the script. Each test (or group of tests) should
|
||||||
|
# be accompanied by a comment explaining why the test should be ignored.
|
||||||
|
|
||||||
|
# Tests marked as deprecated (see: https://review.opendev.org/c/osf/interop/+/800795/)
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_already_associated_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_disassociate_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_allocate_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_delete_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_with_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_without_passing_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_nonexistent_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_dissociate_nonexistent_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_allocate_floating_ip_from_nonexistent_pool
|
||||||
|
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_delete_nonexistent_floating_ip
|
||||||
|
tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_get_floating_ip_details
|
||||||
|
tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ip_pools
|
||||||
|
tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ips
|
||||||
|
tempest.api.compute.floating_ips.test_list_floating_ips_negative.FloatingIPDetailsNegativeTestJSON.test_get_nonexistent_floating_ip_details
|
||||||
|
|
||||||
Loading…
x
Reference in New Issue
Block a user