diff --git a/tools/cross_check/cross_check.sh b/tools/cross_check/cross_check.sh new file mode 100755 index 00000000..bbb09d00 --- /dev/null +++ b/tools/cross_check/cross_check.sh @@ -0,0 +1,254 @@ +#!/bin/bash +# This script is useful when it is time to find +# new candidate tests for refstack. It lists +# all API tests from tempest, designate-tempest-plugin, +# heat-tempest-plugin and manila tempest-plugin except +# those that: +# * are/were part of interop repository or +# * are mentioned in exclude file (see option --exclude-file) or +# * are scenario tests or +# * require admin access. +# +# The list of tests that is produced by this script is +# good start to find new tests but not all tests listed +# by the script must necessarily be added to the refstack. + +function print_help() { + SCRIPT_NAME="basename ${BASH_SOURCE[0]}" + echo "Usage: ${SCRIPT_NAME} [OPTION] ..." + echo "Find missing test in interop repository." + echo "" + echo "--interop-url URL to download interop git repository" + echo "--interop-commit ID of interop commit that should be used" + echo "--tempest-url URL to download tempest git repository" + echo "--tempest-commit ID of tempest commit that should be used" + echo "--designate-url URL to download designate-tempest-plugin git repository" + echo "--designate-commit ID of designate-tempest-plugin commit that should be used" + echo "--heat-url URL to download heat-tempest-plugin" + echo "--heat-commit ID of heat-tempest-plugin commit that should be used" + echo "--manila-url URL to download manila-tempest-plugin" + echo "--manila-commit ID of manila-tempest-plugin commit that should be used" + echo "--exclude-file File that contains tests that should be ignored" + echo " (Default: ./exclude_file.txt)" +} + +TEMPEST_GIT_URL=https://opendev.org/openstack/tempest.git +TEMPEST_COMMIT=e64c78dcf720202a0542bb1e1184f5229a11524f + +DESIGNATE_GIT_URL=https://opendev.org/openstack/designate-tempest-plugin.git +DESIGNATE_COMMIT=master + +HEAT_GIT_URL=https://opendev.org/openstack/heat-tempest-plugin.git +HEAT_COMMIT=master + +MANILA_GIT_URL=https://opendev.org/openstack/manila-tempest-plugin.git +MANILA_COMMIT=master + +INTEROP_GIT_URL=https://opendev.org/osf/interop.git +INTEROP_COMMIT=master + +TMP_DIR=$(mktemp -d) +SCRIPTDIR=$(dirname $0) +EXCLUDE_FILE="${SCRIPTDIR}/exclude_file.txt" + +while [[ ! -z "$1" ]]; do + case "$1" in + --interop-url) + INTEROP_GIT_URL="$2"; + shift 2;; + --interop-commit) + INTEROP_COMMIT="$2"; + shift 2;; + --tempest-url) + TEMPEST_GIT_URL="$2"; + shift 2;; + --tempest-commit) + TEMPEST_COMMIT="$2"; + shift 2;; + --designate-url) + DESIGNATE_GIT_URL="$2"; + shift 2;; + --designate-commit) + DESIGNATE_COMMIT="$2"; + shift 2;; + --heat-url) + HEAT_URL="$2"; + shift 2;; + --heat-commit) + HEAT_COMMIT="$2"; + shift 2;; + --manila-url) + MANILA_URL="$2"; + shift 2;; + --manila-commit) + MANILA_COMMIT="$2" + shift 2;; + --exclude-file) + EXCLUDE_FILE="$2"; + shift 2;; + --help) + print_help + exit;; + *) + echo "ERROR: Not valid option used ($1)"; + exit;; + esac +done + +if [[ ! -z ${EXCLUDE_FILE} ]] && [[ -f ${EXCLUDE_FILE} ]]; then + EXCLUDE_FILE=$(realpath ${EXCLUDE_FILE}) +elif [[ ! -z ${EXCLUDE_FILE} ]] && [[ ! -f ${EXCLUDE_FILE} ]]; then + echo "ERROR: Not valid exclude file (${EXCLUDE_FILE})" + exit +else + touch ${TMP_DIR}/exclude_file.txt + EXCLUDE_FILE="${TMP_DIR}/exclude_file.txt" +fi + +echo "Temporary directory used by cross_check.sh: ${TMP_DIR}" + +# Clone repositories +echo "Cloning interop repository ..." +git clone --quiet ${INTEROP_GIT_URL} ${TMP_DIR}/interop +cd ${TMP_DIR}/interop +git checkout --quiet ${INTEROP_COMMIT} +cd - &> /dev/null + +echo "Cloning tempest repository ..." +git clone --quiet ${TEMPEST_GIT_URL} ${TMP_DIR}/tempest +cd ${TMP_DIR}/tempest +git checkout --quiet ${TEMPEST_COMMIT} +cd - &> /dev/null + +echo "Cloning designate-tempest-plugin repository ..." +git clone --quiet ${DESIGNATE_GIT_URL} ${TMP_DIR}/designate-tempest-plugin +cd ${TMP_DIR}/designate-tempest-plugin +git checkout --quiet ${DESIGNATE_COMMIT} +cd - &> /dev/null + +echo "Cloning heat-tempest-plugin repository ..." +git clone --quiet ${HEAT_GIT_URL} ${TMP_DIR}/heat-tempest-plugin +cd ${TMP_DIR}/heat-tempest-plugin +git checkout --quiet ${HEAT_COMMIT} +cd - &> /dev/null + +echo "Cloning manila-tempest-plugin repository ..." +git clone --quiet ${MANILA_GIT_URL} ${TMP_DIR}/manila-tempest-plugin +cd ${TMP_DIR}/manila-tempest-plugin +git checkout --quiet ${MANILA_COMMIT} +cd - &> /dev/null + +######################################################## +# Find and print missing tests from interop repository. +# +# GLOBALS: +# TMP_DIR, EXCLUDE_FILE +# ARGUMENTS: +# 1 - Path to the repository in which the missing +# tests should be found. +# OUTPUTS: +# List of tests that are part of passed repository +# but not part of interop and not mentioned in +# exclude file. +# +####################################################### +function cross_check() { + + REPO_DIR=$1 + + # Get every idempotent id from the repository + grep -hr --exclude-dir="doc" "@decorators.idempotent_id('" ${REPO_DIR} \ + | sed --expression="s/@decorators.idempotent_id('//g" \ + | sed --expression="s/')//g" \ + | sed --expression="s/ *//g" \ + | sort > ${TMP_DIR}/repo_idempotent_ids.txt + + # Get every idempotent id from interop + cd ${TMP_DIR}/interop/guidelines + cat $(ls -1 | grep -E "[0-9]{4}\.[0-9]{2}\.json|next.json") ${TMP_DIR}/interop/add-ons/guidelines/* \ + | grep -oE "id-[[:alnum:]]{8}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{12}" \ + | sed --expression='s/id-//g' \ + | sort > ${TMP_DIR}/interop_idempotent_ids.txt + cd - &> /dev/null + + + # Get ids of missing tests in interop + comm -23 ${TMP_DIR}/repo_idempotent_ids.txt ${TMP_DIR}/interop_idempotent_ids.txt \ + | sort > ${TMP_DIR}/missing_idempotent_ids.txt + + # Print missing tests to stdout + cd ${REPO_DIR} + while read ID; do + FILE=$(grep -rl "${ID}" && cd - > /dev/null) + + FUNC_NAME=$(sed -e "1,/${ID}/d" ${FILE} \ + | grep -m1 "def" \ + | sed "s/def//g; s/(.*//g; s/ //g") + + CLASS_NAME=$(sed "/${ID}/q" ${FILE} \ + | grep "^class " \ + | tail -1 \ + | sed "s/class//g; s/(.*//g; s/ //g") + + FILE_ID=$(echo ${FILE} | sed "s/\.py//g; s/\.\///g; s/\//\./g") + TEST_NAME="${FILE_ID}.${CLASS_NAME}.${FUNC_NAME}" + + # Check if test: + # * requires admin access or + # * is already present in interop next files + # (necessary for older next files as they do not + # use idempotent ids) or + # * is scenario test or + # * is present in exclude file. + ADMIN_TEST=$(echo ${FILE_ID} | grep "\.admin\.") + SCENARIO_TEST=$(echo ${FILE_ID} | grep "\.scenario\.") + IN_INTEROP=$(grep -r ${TEST_NAME} ${TMP_DIR}/interop) + IN_EXCLUDE=$(cat ${EXCLUDE_FILE} | grep "${TEST_NAME}") + + if [[ ! ${IN_INTEROP} ]] && \ + [[ ! ${ADMIN_TEST} ]] && \ + [[ ! ${IN_EXCLUDE} ]] && \ + [[ ! ${SCENARIO_TEST} ]] + then + echo "${TEST_NAME}[${ID}]" + fi + + done < ${TMP_DIR}/missing_idempotent_ids.txt + cd - &> /dev/null + +} + +echo "Searching tempest repository ..." +cross_check ${TMP_DIR}/tempest > ${TMP_DIR}/tempest_tests.txt + +echo "Searching designate-tempest-plugin repository ..." +cross_check ${TMP_DIR}/designate-tempest-plugin > ${TMP_DIR}/designate_tests.txt + +echo "Searching heat-tempest-plugin repository ..." +cross_check ${TMP_DIR}/heat-tempest-plugin > ${TMP_DIR}/heat_tests.txt + +echo "Searching manila-tempest-plugin repository ..." +cross_check ${TMP_DIR}/manila-tempest-plugin > ${TMP_DIR}/manila_tests.txt + +echo "Tempest tests:" +echo "--------------" +sort ${TMP_DIR}/tempest_tests.txt +echo "" + +echo "Designate-tempest-plugin tests:" +echo "-------------------------------" +sort ${TMP_DIR}/designate_tests.txt +echo "" + +echo "Heat-tempest-plugin tests:" +echo "--------------------------" +sort ${TMP_DIR}/heat_tests.txt +echo "" + +echo "Manila-tempest-plugin tests:" +echo "----------------------------" +sort ${TMP_DIR}/manila_tests.txt + +# Cleanup +rm -rf ${TMP_DIR} + diff --git a/tools/cross_check/exclude_file.txt b/tools/cross_check/exclude_file.txt new file mode 100644 index 00000000..fe7a91bb --- /dev/null +++ b/tools/cross_check/exclude_file.txt @@ -0,0 +1,20 @@ +# This file is used by cross_check.sh script. It contains list of tests +# that should be ignored by the script. Each test (or group of tests) should +# be accompanied by a comment explaining why the test should be ignored. + +# Tests marked as deprecated (see: https://review.opendev.org/c/osf/interop/+/800795/) +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_already_associated_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_disassociate_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_allocate_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_delete_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_with_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_without_passing_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_nonexistent_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_dissociate_nonexistent_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_allocate_floating_ip_from_nonexistent_pool +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_delete_nonexistent_floating_ip +tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_get_floating_ip_details +tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ip_pools +tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ips +tempest.api.compute.floating_ips.test_list_floating_ips_negative.FloatingIPDetailsNegativeTestJSON.test_get_nonexistent_floating_ip_details +