255 lines
8.3 KiB
Bash
Executable File
255 lines
8.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# This script is useful when it is time to find
|
|
# new candidate tests for refstack. It lists
|
|
# all API tests from tempest, designate-tempest-plugin,
|
|
# heat-tempest-plugin and manila tempest-plugin except
|
|
# those that:
|
|
# * are/were part of interop repository or
|
|
# * are mentioned in exclude file (see option --exclude-file) or
|
|
# * are scenario tests or
|
|
# * require admin access.
|
|
#
|
|
# The list of tests that is produced by this script is
|
|
# good start to find new tests but not all tests listed
|
|
# by the script must necessarily be added to the refstack.
|
|
|
|
function print_help() {
|
|
SCRIPT_NAME="basename ${BASH_SOURCE[0]}"
|
|
echo "Usage: ${SCRIPT_NAME} [OPTION] ..."
|
|
echo "Find missing test in interop repository."
|
|
echo ""
|
|
echo "--interop-url URL to download interop git repository"
|
|
echo "--interop-commit ID of interop commit that should be used"
|
|
echo "--tempest-url URL to download tempest git repository"
|
|
echo "--tempest-commit ID of tempest commit that should be used"
|
|
echo "--designate-url URL to download designate-tempest-plugin git repository"
|
|
echo "--designate-commit ID of designate-tempest-plugin commit that should be used"
|
|
echo "--heat-url URL to download heat-tempest-plugin"
|
|
echo "--heat-commit ID of heat-tempest-plugin commit that should be used"
|
|
echo "--manila-url URL to download manila-tempest-plugin"
|
|
echo "--manila-commit ID of manila-tempest-plugin commit that should be used"
|
|
echo "--exclude-file File that contains tests that should be ignored"
|
|
echo " (Default: ./exclude_file.txt)"
|
|
}
|
|
|
|
TEMPEST_GIT_URL=https://opendev.org/openstack/tempest.git
|
|
TEMPEST_COMMIT=1d500e79156ada6bc6fdb628ed1da0efd4121f6a
|
|
|
|
DESIGNATE_GIT_URL=https://opendev.org/openstack/designate-tempest-plugin.git
|
|
DESIGNATE_COMMIT=master
|
|
|
|
HEAT_GIT_URL=https://opendev.org/openstack/heat-tempest-plugin.git
|
|
HEAT_COMMIT=master
|
|
|
|
MANILA_GIT_URL=https://opendev.org/openstack/manila-tempest-plugin.git
|
|
MANILA_COMMIT=master
|
|
|
|
INTEROP_GIT_URL=https://opendev.org/openinfra/interop.git
|
|
INTEROP_COMMIT=master
|
|
|
|
TMP_DIR=$(mktemp -d)
|
|
SCRIPTDIR=$(dirname $0)
|
|
EXCLUDE_FILE="${SCRIPTDIR}/exclude_file.txt"
|
|
|
|
while [[ ! -z "$1" ]]; do
|
|
case "$1" in
|
|
--interop-url)
|
|
INTEROP_GIT_URL="$2";
|
|
shift 2;;
|
|
--interop-commit)
|
|
INTEROP_COMMIT="$2";
|
|
shift 2;;
|
|
--tempest-url)
|
|
TEMPEST_GIT_URL="$2";
|
|
shift 2;;
|
|
--tempest-commit)
|
|
TEMPEST_COMMIT="$2";
|
|
shift 2;;
|
|
--designate-url)
|
|
DESIGNATE_GIT_URL="$2";
|
|
shift 2;;
|
|
--designate-commit)
|
|
DESIGNATE_COMMIT="$2";
|
|
shift 2;;
|
|
--heat-url)
|
|
HEAT_URL="$2";
|
|
shift 2;;
|
|
--heat-commit)
|
|
HEAT_COMMIT="$2";
|
|
shift 2;;
|
|
--manila-url)
|
|
MANILA_URL="$2";
|
|
shift 2;;
|
|
--manila-commit)
|
|
MANILA_COMMIT="$2"
|
|
shift 2;;
|
|
--exclude-file)
|
|
EXCLUDE_FILE="$2";
|
|
shift 2;;
|
|
--help)
|
|
print_help
|
|
exit;;
|
|
*)
|
|
echo "ERROR: Not valid option used ($1)";
|
|
exit;;
|
|
esac
|
|
done
|
|
|
|
if [[ ! -z ${EXCLUDE_FILE} ]] && [[ -f ${EXCLUDE_FILE} ]]; then
|
|
EXCLUDE_FILE=$(realpath ${EXCLUDE_FILE})
|
|
elif [[ ! -z ${EXCLUDE_FILE} ]] && [[ ! -f ${EXCLUDE_FILE} ]]; then
|
|
echo "ERROR: Not valid exclude file (${EXCLUDE_FILE})"
|
|
exit
|
|
else
|
|
touch ${TMP_DIR}/exclude_file.txt
|
|
EXCLUDE_FILE="${TMP_DIR}/exclude_file.txt"
|
|
fi
|
|
|
|
echo "Temporary directory used by cross_check.sh: ${TMP_DIR}"
|
|
|
|
# Clone repositories
|
|
echo "Cloning interop repository ..."
|
|
git clone --quiet ${INTEROP_GIT_URL} ${TMP_DIR}/interop
|
|
cd ${TMP_DIR}/interop
|
|
git checkout --quiet ${INTEROP_COMMIT}
|
|
cd - &> /dev/null
|
|
|
|
echo "Cloning tempest repository ..."
|
|
git clone --quiet ${TEMPEST_GIT_URL} ${TMP_DIR}/tempest
|
|
cd ${TMP_DIR}/tempest
|
|
git checkout --quiet ${TEMPEST_COMMIT}
|
|
cd - &> /dev/null
|
|
|
|
echo "Cloning designate-tempest-plugin repository ..."
|
|
git clone --quiet ${DESIGNATE_GIT_URL} ${TMP_DIR}/designate-tempest-plugin
|
|
cd ${TMP_DIR}/designate-tempest-plugin
|
|
git checkout --quiet ${DESIGNATE_COMMIT}
|
|
cd - &> /dev/null
|
|
|
|
echo "Cloning heat-tempest-plugin repository ..."
|
|
git clone --quiet ${HEAT_GIT_URL} ${TMP_DIR}/heat-tempest-plugin
|
|
cd ${TMP_DIR}/heat-tempest-plugin
|
|
git checkout --quiet ${HEAT_COMMIT}
|
|
cd - &> /dev/null
|
|
|
|
echo "Cloning manila-tempest-plugin repository ..."
|
|
git clone --quiet ${MANILA_GIT_URL} ${TMP_DIR}/manila-tempest-plugin
|
|
cd ${TMP_DIR}/manila-tempest-plugin
|
|
git checkout --quiet ${MANILA_COMMIT}
|
|
cd - &> /dev/null
|
|
|
|
########################################################
|
|
# Find and print missing tests from interop repository.
|
|
#
|
|
# GLOBALS:
|
|
# TMP_DIR, EXCLUDE_FILE
|
|
# ARGUMENTS:
|
|
# 1 - Path to the repository in which the missing
|
|
# tests should be found.
|
|
# OUTPUTS:
|
|
# List of tests that are part of passed repository
|
|
# but not part of interop and not mentioned in
|
|
# exclude file.
|
|
#
|
|
#######################################################
|
|
function cross_check() {
|
|
|
|
REPO_DIR=$1
|
|
|
|
# Get every idempotent id from the repository
|
|
grep -hr --exclude-dir="doc" "@decorators.idempotent_id('" ${REPO_DIR} \
|
|
| sed --expression="s/@decorators.idempotent_id('//g" \
|
|
| sed --expression="s/')//g" \
|
|
| sed --expression="s/ *//g" \
|
|
| sort > ${TMP_DIR}/repo_idempotent_ids.txt
|
|
|
|
# Get every idempotent id from interop
|
|
cd ${TMP_DIR}/interop/guidelines
|
|
cat $(ls -1 | grep -E "[0-9]{4}\.[0-9]{2}\.json|next.json") ${TMP_DIR}/interop/add-ons/guidelines/* \
|
|
| grep -oE "id-[[:alnum:]]{8}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{12}" \
|
|
| sed --expression='s/id-//g' \
|
|
| sort > ${TMP_DIR}/interop_idempotent_ids.txt
|
|
cd - &> /dev/null
|
|
|
|
|
|
# Get ids of missing tests in interop
|
|
comm -23 ${TMP_DIR}/repo_idempotent_ids.txt ${TMP_DIR}/interop_idempotent_ids.txt \
|
|
| sort > ${TMP_DIR}/missing_idempotent_ids.txt
|
|
|
|
# Print missing tests to stdout
|
|
cd ${REPO_DIR}
|
|
while read ID; do
|
|
FILE=$(grep -rl "${ID}" && cd - > /dev/null)
|
|
|
|
FUNC_NAME=$(sed -e "1,/${ID}/d" ${FILE} \
|
|
| grep -m1 "def" \
|
|
| sed "s/def//g; s/(.*//g; s/ //g")
|
|
|
|
CLASS_NAME=$(sed "/${ID}/q" ${FILE} \
|
|
| grep "^class " \
|
|
| tail -1 \
|
|
| sed "s/class//g; s/(.*//g; s/ //g")
|
|
|
|
FILE_ID=$(echo ${FILE} | sed "s/\.py//g; s/\.\///g; s/\//\./g")
|
|
TEST_NAME="${FILE_ID}.${CLASS_NAME}.${FUNC_NAME}"
|
|
|
|
# Check if test:
|
|
# * requires admin access or
|
|
# * is already present in interop next files
|
|
# (necessary for older next files as they do not
|
|
# use idempotent ids) or
|
|
# * is scenario test or
|
|
# * is present in exclude file.
|
|
ADMIN_TEST=$(echo ${FILE_ID} | grep "\.admin\.")
|
|
SCENARIO_TEST=$(echo ${FILE_ID} | grep "\.scenario\.")
|
|
IN_INTEROP=$(grep -r ${TEST_NAME} ${TMP_DIR}/interop)
|
|
IN_EXCLUDE=$(cat ${EXCLUDE_FILE} | grep "${TEST_NAME}")
|
|
|
|
if [[ ! ${IN_INTEROP} ]] && \
|
|
[[ ! ${ADMIN_TEST} ]] && \
|
|
[[ ! ${IN_EXCLUDE} ]] && \
|
|
[[ ! ${SCENARIO_TEST} ]]
|
|
then
|
|
echo "${TEST_NAME}[${ID}]"
|
|
fi
|
|
|
|
done < ${TMP_DIR}/missing_idempotent_ids.txt
|
|
cd - &> /dev/null
|
|
|
|
}
|
|
|
|
echo "Searching tempest repository ..."
|
|
cross_check ${TMP_DIR}/tempest > ${TMP_DIR}/tempest_tests.txt
|
|
|
|
echo "Searching designate-tempest-plugin repository ..."
|
|
cross_check ${TMP_DIR}/designate-tempest-plugin > ${TMP_DIR}/designate_tests.txt
|
|
|
|
echo "Searching heat-tempest-plugin repository ..."
|
|
cross_check ${TMP_DIR}/heat-tempest-plugin > ${TMP_DIR}/heat_tests.txt
|
|
|
|
echo "Searching manila-tempest-plugin repository ..."
|
|
cross_check ${TMP_DIR}/manila-tempest-plugin > ${TMP_DIR}/manila_tests.txt
|
|
|
|
echo "Tempest tests:"
|
|
echo "--------------"
|
|
sort ${TMP_DIR}/tempest_tests.txt
|
|
echo ""
|
|
|
|
echo "Designate-tempest-plugin tests:"
|
|
echo "-------------------------------"
|
|
sort ${TMP_DIR}/designate_tests.txt
|
|
echo ""
|
|
|
|
echo "Heat-tempest-plugin tests:"
|
|
echo "--------------------------"
|
|
sort ${TMP_DIR}/heat_tests.txt
|
|
echo ""
|
|
|
|
echo "Manila-tempest-plugin tests:"
|
|
echo "----------------------------"
|
|
sort ${TMP_DIR}/manila_tests.txt
|
|
|
|
# Cleanup
|
|
rm -rf ${TMP_DIR}
|
|
|