From 7a9916beedf56241a77c730594e25e6ed97065b1 Mon Sep 17 00:00:00 2001 From: lpiwowar Date: Fri, 2 Jul 2021 15:03:35 +0200 Subject: [PATCH] Create script that finds candidate tests for refstack This patch introduces script that finds candidate tests from tempest (and tempest plugins) that can be used for refstack testing. The script lists all test from tempest (and tempest plugins) except those that: - are/were part of interop repository or - are mentioned in exclude file or - are scenario tests or - require admin access. Change-Id: I7c087bca458943b3ff8b6fc098626af466210474 --- tools/cross_check/cross_check.sh | 254 +++++++++++++++++++++++++++++ tools/cross_check/exclude_file.txt | 20 +++ 2 files changed, 274 insertions(+) create mode 100755 tools/cross_check/cross_check.sh create mode 100644 tools/cross_check/exclude_file.txt diff --git a/tools/cross_check/cross_check.sh b/tools/cross_check/cross_check.sh new file mode 100755 index 00000000..bbb09d00 --- /dev/null +++ b/tools/cross_check/cross_check.sh @@ -0,0 +1,254 @@ +#!/bin/bash +# This script is useful when it is time to find +# new candidate tests for refstack. It lists +# all API tests from tempest, designate-tempest-plugin, +# heat-tempest-plugin and manila tempest-plugin except +# those that: +# * are/were part of interop repository or +# * are mentioned in exclude file (see option --exclude-file) or +# * are scenario tests or +# * require admin access. +# +# The list of tests that is produced by this script is +# good start to find new tests but not all tests listed +# by the script must necessarily be added to the refstack. + +function print_help() { + SCRIPT_NAME="basename ${BASH_SOURCE[0]}" + echo "Usage: ${SCRIPT_NAME} [OPTION] ..." + echo "Find missing test in interop repository." + echo "" + echo "--interop-url URL to download interop git repository" + echo "--interop-commit ID of interop commit that should be used" + echo "--tempest-url URL to download tempest git repository" + echo "--tempest-commit ID of tempest commit that should be used" + echo "--designate-url URL to download designate-tempest-plugin git repository" + echo "--designate-commit ID of designate-tempest-plugin commit that should be used" + echo "--heat-url URL to download heat-tempest-plugin" + echo "--heat-commit ID of heat-tempest-plugin commit that should be used" + echo "--manila-url URL to download manila-tempest-plugin" + echo "--manila-commit ID of manila-tempest-plugin commit that should be used" + echo "--exclude-file File that contains tests that should be ignored" + echo " (Default: ./exclude_file.txt)" +} + +TEMPEST_GIT_URL=https://opendev.org/openstack/tempest.git +TEMPEST_COMMIT=e64c78dcf720202a0542bb1e1184f5229a11524f + +DESIGNATE_GIT_URL=https://opendev.org/openstack/designate-tempest-plugin.git +DESIGNATE_COMMIT=master + +HEAT_GIT_URL=https://opendev.org/openstack/heat-tempest-plugin.git +HEAT_COMMIT=master + +MANILA_GIT_URL=https://opendev.org/openstack/manila-tempest-plugin.git +MANILA_COMMIT=master + +INTEROP_GIT_URL=https://opendev.org/osf/interop.git +INTEROP_COMMIT=master + +TMP_DIR=$(mktemp -d) +SCRIPTDIR=$(dirname $0) +EXCLUDE_FILE="${SCRIPTDIR}/exclude_file.txt" + +while [[ ! -z "$1" ]]; do + case "$1" in + --interop-url) + INTEROP_GIT_URL="$2"; + shift 2;; + --interop-commit) + INTEROP_COMMIT="$2"; + shift 2;; + --tempest-url) + TEMPEST_GIT_URL="$2"; + shift 2;; + --tempest-commit) + TEMPEST_COMMIT="$2"; + shift 2;; + --designate-url) + DESIGNATE_GIT_URL="$2"; + shift 2;; + --designate-commit) + DESIGNATE_COMMIT="$2"; + shift 2;; + --heat-url) + HEAT_URL="$2"; + shift 2;; + --heat-commit) + HEAT_COMMIT="$2"; + shift 2;; + --manila-url) + MANILA_URL="$2"; + shift 2;; + --manila-commit) + MANILA_COMMIT="$2" + shift 2;; + --exclude-file) + EXCLUDE_FILE="$2"; + shift 2;; + --help) + print_help + exit;; + *) + echo "ERROR: Not valid option used ($1)"; + exit;; + esac +done + +if [[ ! -z ${EXCLUDE_FILE} ]] && [[ -f ${EXCLUDE_FILE} ]]; then + EXCLUDE_FILE=$(realpath ${EXCLUDE_FILE}) +elif [[ ! -z ${EXCLUDE_FILE} ]] && [[ ! -f ${EXCLUDE_FILE} ]]; then + echo "ERROR: Not valid exclude file (${EXCLUDE_FILE})" + exit +else + touch ${TMP_DIR}/exclude_file.txt + EXCLUDE_FILE="${TMP_DIR}/exclude_file.txt" +fi + +echo "Temporary directory used by cross_check.sh: ${TMP_DIR}" + +# Clone repositories +echo "Cloning interop repository ..." +git clone --quiet ${INTEROP_GIT_URL} ${TMP_DIR}/interop +cd ${TMP_DIR}/interop +git checkout --quiet ${INTEROP_COMMIT} +cd - &> /dev/null + +echo "Cloning tempest repository ..." +git clone --quiet ${TEMPEST_GIT_URL} ${TMP_DIR}/tempest +cd ${TMP_DIR}/tempest +git checkout --quiet ${TEMPEST_COMMIT} +cd - &> /dev/null + +echo "Cloning designate-tempest-plugin repository ..." +git clone --quiet ${DESIGNATE_GIT_URL} ${TMP_DIR}/designate-tempest-plugin +cd ${TMP_DIR}/designate-tempest-plugin +git checkout --quiet ${DESIGNATE_COMMIT} +cd - &> /dev/null + +echo "Cloning heat-tempest-plugin repository ..." +git clone --quiet ${HEAT_GIT_URL} ${TMP_DIR}/heat-tempest-plugin +cd ${TMP_DIR}/heat-tempest-plugin +git checkout --quiet ${HEAT_COMMIT} +cd - &> /dev/null + +echo "Cloning manila-tempest-plugin repository ..." +git clone --quiet ${MANILA_GIT_URL} ${TMP_DIR}/manila-tempest-plugin +cd ${TMP_DIR}/manila-tempest-plugin +git checkout --quiet ${MANILA_COMMIT} +cd - &> /dev/null + +######################################################## +# Find and print missing tests from interop repository. +# +# GLOBALS: +# TMP_DIR, EXCLUDE_FILE +# ARGUMENTS: +# 1 - Path to the repository in which the missing +# tests should be found. +# OUTPUTS: +# List of tests that are part of passed repository +# but not part of interop and not mentioned in +# exclude file. +# +####################################################### +function cross_check() { + + REPO_DIR=$1 + + # Get every idempotent id from the repository + grep -hr --exclude-dir="doc" "@decorators.idempotent_id('" ${REPO_DIR} \ + | sed --expression="s/@decorators.idempotent_id('//g" \ + | sed --expression="s/')//g" \ + | sed --expression="s/ *//g" \ + | sort > ${TMP_DIR}/repo_idempotent_ids.txt + + # Get every idempotent id from interop + cd ${TMP_DIR}/interop/guidelines + cat $(ls -1 | grep -E "[0-9]{4}\.[0-9]{2}\.json|next.json") ${TMP_DIR}/interop/add-ons/guidelines/* \ + | grep -oE "id-[[:alnum:]]{8}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{12}" \ + | sed --expression='s/id-//g' \ + | sort > ${TMP_DIR}/interop_idempotent_ids.txt + cd - &> /dev/null + + + # Get ids of missing tests in interop + comm -23 ${TMP_DIR}/repo_idempotent_ids.txt ${TMP_DIR}/interop_idempotent_ids.txt \ + | sort > ${TMP_DIR}/missing_idempotent_ids.txt + + # Print missing tests to stdout + cd ${REPO_DIR} + while read ID; do + FILE=$(grep -rl "${ID}" && cd - > /dev/null) + + FUNC_NAME=$(sed -e "1,/${ID}/d" ${FILE} \ + | grep -m1 "def" \ + | sed "s/def//g; s/(.*//g; s/ //g") + + CLASS_NAME=$(sed "/${ID}/q" ${FILE} \ + | grep "^class " \ + | tail -1 \ + | sed "s/class//g; s/(.*//g; s/ //g") + + FILE_ID=$(echo ${FILE} | sed "s/\.py//g; s/\.\///g; s/\//\./g") + TEST_NAME="${FILE_ID}.${CLASS_NAME}.${FUNC_NAME}" + + # Check if test: + # * requires admin access or + # * is already present in interop next files + # (necessary for older next files as they do not + # use idempotent ids) or + # * is scenario test or + # * is present in exclude file. + ADMIN_TEST=$(echo ${FILE_ID} | grep "\.admin\.") + SCENARIO_TEST=$(echo ${FILE_ID} | grep "\.scenario\.") + IN_INTEROP=$(grep -r ${TEST_NAME} ${TMP_DIR}/interop) + IN_EXCLUDE=$(cat ${EXCLUDE_FILE} | grep "${TEST_NAME}") + + if [[ ! ${IN_INTEROP} ]] && \ + [[ ! ${ADMIN_TEST} ]] && \ + [[ ! ${IN_EXCLUDE} ]] && \ + [[ ! ${SCENARIO_TEST} ]] + then + echo "${TEST_NAME}[${ID}]" + fi + + done < ${TMP_DIR}/missing_idempotent_ids.txt + cd - &> /dev/null + +} + +echo "Searching tempest repository ..." +cross_check ${TMP_DIR}/tempest > ${TMP_DIR}/tempest_tests.txt + +echo "Searching designate-tempest-plugin repository ..." +cross_check ${TMP_DIR}/designate-tempest-plugin > ${TMP_DIR}/designate_tests.txt + +echo "Searching heat-tempest-plugin repository ..." +cross_check ${TMP_DIR}/heat-tempest-plugin > ${TMP_DIR}/heat_tests.txt + +echo "Searching manila-tempest-plugin repository ..." +cross_check ${TMP_DIR}/manila-tempest-plugin > ${TMP_DIR}/manila_tests.txt + +echo "Tempest tests:" +echo "--------------" +sort ${TMP_DIR}/tempest_tests.txt +echo "" + +echo "Designate-tempest-plugin tests:" +echo "-------------------------------" +sort ${TMP_DIR}/designate_tests.txt +echo "" + +echo "Heat-tempest-plugin tests:" +echo "--------------------------" +sort ${TMP_DIR}/heat_tests.txt +echo "" + +echo "Manila-tempest-plugin tests:" +echo "----------------------------" +sort ${TMP_DIR}/manila_tests.txt + +# Cleanup +rm -rf ${TMP_DIR} + diff --git a/tools/cross_check/exclude_file.txt b/tools/cross_check/exclude_file.txt new file mode 100644 index 00000000..fe7a91bb --- /dev/null +++ b/tools/cross_check/exclude_file.txt @@ -0,0 +1,20 @@ +# This file is used by cross_check.sh script. It contains list of tests +# that should be ignored by the script. Each test (or group of tests) should +# be accompanied by a comment explaining why the test should be ignored. + +# Tests marked as deprecated (see: https://review.opendev.org/c/osf/interop/+/800795/) +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_already_associated_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_disassociate_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_allocate_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_delete_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_with_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_without_passing_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_nonexistent_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_dissociate_nonexistent_floating_ip +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_allocate_floating_ip_from_nonexistent_pool +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_delete_nonexistent_floating_ip +tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_get_floating_ip_details +tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ip_pools +tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ips +tempest.api.compute.floating_ips.test_list_floating_ips_negative.FloatingIPDetailsNegativeTestJSON.test_get_nonexistent_floating_ip_details +