Merge "Create script that finds candidate tests for refstack"
This commit is contained in:
Zuul
Gerrit Code Review
commit
a93487957e
254
tools/cross_check/cross_check.sh
Executable file
254
tools/cross_check/cross_check.sh
Executable file
@ -0,0 +1,254 @@
|
||||
#!/bin/bash
|
||||
# This script is useful when it is time to find
|
||||
# new candidate tests for refstack. It lists
|
||||
# all API tests from tempest, designate-tempest-plugin,
|
||||
# heat-tempest-plugin and manila tempest-plugin except
|
||||
# those that:
|
||||
# * are/were part of interop repository or
|
||||
# * are mentioned in exclude file (see option --exclude-file) or
|
||||
# * are scenario tests or
|
||||
# * require admin access.
|
||||
#
|
||||
# The list of tests that is produced by this script is
|
||||
# good start to find new tests but not all tests listed
|
||||
# by the script must necessarily be added to the refstack.
|
||||
|
||||
function print_help() {
|
||||
SCRIPT_NAME="basename ${BASH_SOURCE[0]}"
|
||||
echo "Usage: ${SCRIPT_NAME} [OPTION] ..."
|
||||
echo "Find missing test in interop repository."
|
||||
echo ""
|
||||
echo "--interop-url URL to download interop git repository"
|
||||
echo "--interop-commit ID of interop commit that should be used"
|
||||
echo "--tempest-url URL to download tempest git repository"
|
||||
echo "--tempest-commit ID of tempest commit that should be used"
|
||||
echo "--designate-url URL to download designate-tempest-plugin git repository"
|
||||
echo "--designate-commit ID of designate-tempest-plugin commit that should be used"
|
||||
echo "--heat-url URL to download heat-tempest-plugin"
|
||||
echo "--heat-commit ID of heat-tempest-plugin commit that should be used"
|
||||
echo "--manila-url URL to download manila-tempest-plugin"
|
||||
echo "--manila-commit ID of manila-tempest-plugin commit that should be used"
|
||||
echo "--exclude-file File that contains tests that should be ignored"
|
||||
echo " (Default: ./exclude_file.txt)"
|
||||
}
|
||||
|
||||
TEMPEST_GIT_URL=https://opendev.org/openstack/tempest.git
|
||||
TEMPEST_COMMIT=e64c78dcf720202a0542bb1e1184f5229a11524f
|
||||
|
||||
DESIGNATE_GIT_URL=https://opendev.org/openstack/designate-tempest-plugin.git
|
||||
DESIGNATE_COMMIT=master
|
||||
|
||||
HEAT_GIT_URL=https://opendev.org/openstack/heat-tempest-plugin.git
|
||||
HEAT_COMMIT=master
|
||||
|
||||
MANILA_GIT_URL=https://opendev.org/openstack/manila-tempest-plugin.git
|
||||
MANILA_COMMIT=master
|
||||
|
||||
INTEROP_GIT_URL=https://opendev.org/osf/interop.git
|
||||
INTEROP_COMMIT=master
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
SCRIPTDIR=$(dirname $0)
|
||||
EXCLUDE_FILE="${SCRIPTDIR}/exclude_file.txt"
|
||||
|
||||
while [[ ! -z "$1" ]]; do
|
||||
case "$1" in
|
||||
--interop-url)
|
||||
INTEROP_GIT_URL="$2";
|
||||
shift 2;;
|
||||
--interop-commit)
|
||||
INTEROP_COMMIT="$2";
|
||||
shift 2;;
|
||||
--tempest-url)
|
||||
TEMPEST_GIT_URL="$2";
|
||||
shift 2;;
|
||||
--tempest-commit)
|
||||
TEMPEST_COMMIT="$2";
|
||||
shift 2;;
|
||||
--designate-url)
|
||||
DESIGNATE_GIT_URL="$2";
|
||||
shift 2;;
|
||||
--designate-commit)
|
||||
DESIGNATE_COMMIT="$2";
|
||||
shift 2;;
|
||||
--heat-url)
|
||||
HEAT_URL="$2";
|
||||
shift 2;;
|
||||
--heat-commit)
|
||||
HEAT_COMMIT="$2";
|
||||
shift 2;;
|
||||
--manila-url)
|
||||
MANILA_URL="$2";
|
||||
shift 2;;
|
||||
--manila-commit)
|
||||
MANILA_COMMIT="$2"
|
||||
shift 2;;
|
||||
--exclude-file)
|
||||
EXCLUDE_FILE="$2";
|
||||
shift 2;;
|
||||
--help)
|
||||
print_help
|
||||
exit;;
|
||||
*)
|
||||
echo "ERROR: Not valid option used ($1)";
|
||||
exit;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ ! -z ${EXCLUDE_FILE} ]] && [[ -f ${EXCLUDE_FILE} ]]; then
|
||||
EXCLUDE_FILE=$(realpath ${EXCLUDE_FILE})
|
||||
elif [[ ! -z ${EXCLUDE_FILE} ]] && [[ ! -f ${EXCLUDE_FILE} ]]; then
|
||||
echo "ERROR: Not valid exclude file (${EXCLUDE_FILE})"
|
||||
exit
|
||||
else
|
||||
touch ${TMP_DIR}/exclude_file.txt
|
||||
EXCLUDE_FILE="${TMP_DIR}/exclude_file.txt"
|
||||
fi
|
||||
|
||||
echo "Temporary directory used by cross_check.sh: ${TMP_DIR}"
|
||||
|
||||
# Clone repositories
|
||||
echo "Cloning interop repository ..."
|
||||
git clone --quiet ${INTEROP_GIT_URL} ${TMP_DIR}/interop
|
||||
cd ${TMP_DIR}/interop
|
||||
git checkout --quiet ${INTEROP_COMMIT}
|
||||
cd - &> /dev/null
|
||||
|
||||
echo "Cloning tempest repository ..."
|
||||
git clone --quiet ${TEMPEST_GIT_URL} ${TMP_DIR}/tempest
|
||||
cd ${TMP_DIR}/tempest
|
||||
git checkout --quiet ${TEMPEST_COMMIT}
|
||||
cd - &> /dev/null
|
||||
|
||||
echo "Cloning designate-tempest-plugin repository ..."
|
||||
git clone --quiet ${DESIGNATE_GIT_URL} ${TMP_DIR}/designate-tempest-plugin
|
||||
cd ${TMP_DIR}/designate-tempest-plugin
|
||||
git checkout --quiet ${DESIGNATE_COMMIT}
|
||||
cd - &> /dev/null
|
||||
|
||||
echo "Cloning heat-tempest-plugin repository ..."
|
||||
git clone --quiet ${HEAT_GIT_URL} ${TMP_DIR}/heat-tempest-plugin
|
||||
cd ${TMP_DIR}/heat-tempest-plugin
|
||||
git checkout --quiet ${HEAT_COMMIT}
|
||||
cd - &> /dev/null
|
||||
|
||||
echo "Cloning manila-tempest-plugin repository ..."
|
||||
git clone --quiet ${MANILA_GIT_URL} ${TMP_DIR}/manila-tempest-plugin
|
||||
cd ${TMP_DIR}/manila-tempest-plugin
|
||||
git checkout --quiet ${MANILA_COMMIT}
|
||||
cd - &> /dev/null
|
||||
|
||||
########################################################
|
||||
# Find and print missing tests from interop repository.
|
||||
#
|
||||
# GLOBALS:
|
||||
# TMP_DIR, EXCLUDE_FILE
|
||||
# ARGUMENTS:
|
||||
# 1 - Path to the repository in which the missing
|
||||
# tests should be found.
|
||||
# OUTPUTS:
|
||||
# List of tests that are part of passed repository
|
||||
# but not part of interop and not mentioned in
|
||||
# exclude file.
|
||||
#
|
||||
#######################################################
|
||||
function cross_check() {
|
||||
|
||||
REPO_DIR=$1
|
||||
|
||||
# Get every idempotent id from the repository
|
||||
grep -hr --exclude-dir="doc" "@decorators.idempotent_id('" ${REPO_DIR} \
|
||||
| sed --expression="s/@decorators.idempotent_id('//g" \
|
||||
| sed --expression="s/')//g" \
|
||||
| sed --expression="s/ *//g" \
|
||||
| sort > ${TMP_DIR}/repo_idempotent_ids.txt
|
||||
|
||||
# Get every idempotent id from interop
|
||||
cd ${TMP_DIR}/interop/guidelines
|
||||
cat $(ls -1 | grep -E "[0-9]{4}\.[0-9]{2}\.json|next.json") ${TMP_DIR}/interop/add-ons/guidelines/* \
|
||||
| grep -oE "id-[[:alnum:]]{8}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{12}" \
|
||||
| sed --expression='s/id-//g' \
|
||||
| sort > ${TMP_DIR}/interop_idempotent_ids.txt
|
||||
cd - &> /dev/null
|
||||
|
||||
|
||||
# Get ids of missing tests in interop
|
||||
comm -23 ${TMP_DIR}/repo_idempotent_ids.txt ${TMP_DIR}/interop_idempotent_ids.txt \
|
||||
| sort > ${TMP_DIR}/missing_idempotent_ids.txt
|
||||
|
||||
# Print missing tests to stdout
|
||||
cd ${REPO_DIR}
|
||||
while read ID; do
|
||||
FILE=$(grep -rl "${ID}" && cd - > /dev/null)
|
||||
|
||||
FUNC_NAME=$(sed -e "1,/${ID}/d" ${FILE} \
|
||||
| grep -m1 "def" \
|
||||
| sed "s/def//g; s/(.*//g; s/ //g")
|
||||
|
||||
CLASS_NAME=$(sed "/${ID}/q" ${FILE} \
|
||||
| grep "^class " \
|
||||
| tail -1 \
|
||||
| sed "s/class//g; s/(.*//g; s/ //g")
|
||||
|
||||
FILE_ID=$(echo ${FILE} | sed "s/\.py//g; s/\.\///g; s/\//\./g")
|
||||
TEST_NAME="${FILE_ID}.${CLASS_NAME}.${FUNC_NAME}"
|
||||
|
||||
# Check if test:
|
||||
# * requires admin access or
|
||||
# * is already present in interop next files
|
||||
# (necessary for older next files as they do not
|
||||
# use idempotent ids) or
|
||||
# * is scenario test or
|
||||
# * is present in exclude file.
|
||||
ADMIN_TEST=$(echo ${FILE_ID} | grep "\.admin\.")
|
||||
SCENARIO_TEST=$(echo ${FILE_ID} | grep "\.scenario\.")
|
||||
IN_INTEROP=$(grep -r ${TEST_NAME} ${TMP_DIR}/interop)
|
||||
IN_EXCLUDE=$(cat ${EXCLUDE_FILE} | grep "${TEST_NAME}")
|
||||
|
||||
if [[ ! ${IN_INTEROP} ]] && \
|
||||
[[ ! ${ADMIN_TEST} ]] && \
|
||||
[[ ! ${IN_EXCLUDE} ]] && \
|
||||
[[ ! ${SCENARIO_TEST} ]]
|
||||
then
|
||||
echo "${TEST_NAME}[${ID}]"
|
||||
fi
|
||||
|
||||
done < ${TMP_DIR}/missing_idempotent_ids.txt
|
||||
cd - &> /dev/null
|
||||
|
||||
}
|
||||
|
||||
echo "Searching tempest repository ..."
|
||||
cross_check ${TMP_DIR}/tempest > ${TMP_DIR}/tempest_tests.txt
|
||||
|
||||
echo "Searching designate-tempest-plugin repository ..."
|
||||
cross_check ${TMP_DIR}/designate-tempest-plugin > ${TMP_DIR}/designate_tests.txt
|
||||
|
||||
echo "Searching heat-tempest-plugin repository ..."
|
||||
cross_check ${TMP_DIR}/heat-tempest-plugin > ${TMP_DIR}/heat_tests.txt
|
||||
|
||||
echo "Searching manila-tempest-plugin repository ..."
|
||||
cross_check ${TMP_DIR}/manila-tempest-plugin > ${TMP_DIR}/manila_tests.txt
|
||||
|
||||
echo "Tempest tests:"
|
||||
echo "--------------"
|
||||
sort ${TMP_DIR}/tempest_tests.txt
|
||||
echo ""
|
||||
|
||||
echo "Designate-tempest-plugin tests:"
|
||||
echo "-------------------------------"
|
||||
sort ${TMP_DIR}/designate_tests.txt
|
||||
echo ""
|
||||
|
||||
echo "Heat-tempest-plugin tests:"
|
||||
echo "--------------------------"
|
||||
sort ${TMP_DIR}/heat_tests.txt
|
||||
echo ""
|
||||
|
||||
echo "Manila-tempest-plugin tests:"
|
||||
echo "----------------------------"
|
||||
sort ${TMP_DIR}/manila_tests.txt
|
||||
|
||||
# Cleanup
|
||||
rm -rf ${TMP_DIR}
|
||||
|
||||
20
tools/cross_check/exclude_file.txt
Normal file
20
tools/cross_check/exclude_file.txt
Normal file
@ -0,0 +1,20 @@
|
||||
# This file is used by cross_check.sh script. It contains list of tests
|
||||
# that should be ignored by the script. Each test (or group of tests) should
|
||||
# be accompanied by a comment explaining why the test should be ignored.
|
||||
|
||||
# Tests marked as deprecated (see: https://review.opendev.org/c/osf/interop/+/800795/)
|
||||
tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_already_associated_floating_ip
|
||||
tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_disassociate_floating_ip
|
||||
tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_allocate_floating_ip
|
||||
tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_delete_floating_ip
|
||||
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_with_floating_ip
|
||||
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_without_passing_floating_ip
|
||||
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_nonexistent_floating_ip
|
||||
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_dissociate_nonexistent_floating_ip
|
||||
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_allocate_floating_ip_from_nonexistent_pool
|
||||
tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_delete_nonexistent_floating_ip
|
||||
tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_get_floating_ip_details
|
||||
tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ip_pools
|
||||
tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ips
|
||||
tempest.api.compute.floating_ips.test_list_floating_ips_negative.FloatingIPDetailsNegativeTestJSON.test_get_nonexistent_floating_ip_details
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user