Merge "Remove broken uptime tests during upgrade"
This commit is contained in:
Zuul
Gerrit Code Review
commit
0dd48bc716
@ -194,43 +194,11 @@ if [[ "${ACTION}" == "upgrade" ]]; then
|
||||
# Source the current scripts-library.sh functions
|
||||
source "${OSA_CLONE_DIR}/scripts/scripts-library.sh"
|
||||
|
||||
# Kick off the data plane tester
|
||||
bash ${OSA_CLONE_DIR}/tests/data-plane-test.sh &> /var/log/data-plane-error.log &
|
||||
|
||||
# Fetch script to execute API availability tests, then
|
||||
# background them while the upgrade runs.
|
||||
get_bowling_ball_tests
|
||||
start_bowling_ball_tests
|
||||
|
||||
# To execute the upgrade script we need to provide
|
||||
# an affirmative response to the warning that the
|
||||
# upgrade is irreversable.
|
||||
echo 'YES' | bash "${OSA_CLONE_DIR}/scripts/run-upgrade.sh"
|
||||
|
||||
# Terminate the API availability tests
|
||||
kill_bowling_ball_tests
|
||||
|
||||
# Terminate the data plane tester
|
||||
rm -f /var/run/data-plane-test.socket
|
||||
|
||||
# Wait 10s for the tests to complete
|
||||
sleep 10
|
||||
|
||||
# Output the API availability test results
|
||||
print_bowling_ball_results
|
||||
|
||||
# Check for any data plane failures, and fail if there are
|
||||
if ! egrep -q "^FAIL: 0$" /var/log/data-plane-test.log; then
|
||||
echo -e "\n\nFAIL: The L3 data plane check failed!\n\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check for any disk access failures, and fail if there are
|
||||
if ! egrep -q "^FAIL: 0$" /var/log/disk-access-test.log; then
|
||||
echo -e "\n\nFAIL: The disk access data plane check failed!\n\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
exit_success
|
||||
|
||||
@ -398,36 +398,6 @@ function get_instance_info {
|
||||
netstat -tulpn > "/openstack/log/instance-info/netstat_${TS}.log" || true
|
||||
}
|
||||
|
||||
function get_bowling_ball_tests {
|
||||
# Retrieve the latest bowling ball test script in case we don't already have it.
|
||||
if [ -f scripts/rolling_tests.py ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
curl --silent https://raw.githubusercontent.com/openstack/openstack-ansible-ops/master/bowling_ball/rolling_tests.py > scripts/rolling_tests.py
|
||||
}
|
||||
|
||||
function start_bowling_ball_tests {
|
||||
# The tests will pull Keystone information from the env vars defined in openrc
|
||||
source ~/openrc
|
||||
# Get the list of services to test for from the script, so we're not hard coding.
|
||||
for SERVICE in $(python ./scripts/rolling_tests.py list | cut -d - -f 1); do
|
||||
# Start the scripts in the background and wait between each invocation.
|
||||
# Without the wait, they do not all launch.
|
||||
python ./scripts/rolling_tests.py $SERVICE &
|
||||
sleep 1
|
||||
echo "Started $SERVICE test in background"
|
||||
done
|
||||
}
|
||||
|
||||
function kill_bowling_ball_tests {
|
||||
pkill -f rolling_tests
|
||||
}
|
||||
|
||||
function print_bowling_ball_results {
|
||||
grep "failure rate" /var/log/*_rolling.log
|
||||
}
|
||||
|
||||
## Signal traps --------------------------------------------------------------
|
||||
# Trap all Death Signals and Errors
|
||||
trap "exit_fail ${LINENO} $? 'Received STOP Signal'" SIGHUP SIGINT SIGTERM
|
||||
|
||||
@ -1,185 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2017, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
## Shell Opts ----------------------------------------------------------------
|
||||
set -e -x
|
||||
|
||||
## Vars ----------------------------------------------------------------------
|
||||
|
||||
# Log file location
|
||||
TEST_LOG_FILE="/var/log/data-plane-test.log"
|
||||
|
||||
# The test instance name
|
||||
INSTANCE_NAME="test1"
|
||||
|
||||
# Test script socket file location
|
||||
TEST_SOCKET_FILE="/var/run/data-plane-test.socket"
|
||||
|
||||
# Disk access test script
|
||||
DISK_ACCESS_SCRIPT="/opt/openstack-ansible/tests/disk-access-test.sh"
|
||||
|
||||
# Disk access log file
|
||||
DISK_ACCESS_LOG="~/disk-access-test.log"
|
||||
|
||||
# Setup counters
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
# SSH/SCP prefixes
|
||||
CMD_SCP_PREFIX="sshpass -p gocubsgo scp -o StrictHostKeyChecking=no"
|
||||
CMD_SSH_PREFIX="sshpass -p gocubsgo ssh -o StrictHostKeyChecking=no"
|
||||
|
||||
## Functions -----------------------------------------------------------------
|
||||
|
||||
# Create a demorc file with auth credentials
|
||||
# and other state tracking information.
|
||||
setup_demorc() {
|
||||
cp /root/openrc /root/demorc
|
||||
sed -i 's/OS_PROJECT_NAME=.*/OS_PROJECT_NAME=demo/' /root/demorc
|
||||
sed -i 's/OS_TENANT_NAME=.*/OS_TENANT_NAME=demo/' /root/demorc
|
||||
sed -i 's/OS_USERNAME=.*/OS_USERNAME=demo/' /root/demorc
|
||||
sed -i 's/OS_PASSWORD=.*/OS_PASSWORD=demo/' /root/demorc
|
||||
echo "INSTANCE_NAME=${INSTANCE_NAME}" >> /root/demorc
|
||||
echo "TEST_LOG_FILE=${TEST_LOG_FILE}" >> /root/demorc
|
||||
echo "TEST_SOCKET_FILE=${TEST_SOCKET_FILE}" >> /root/demorc
|
||||
}
|
||||
|
||||
# Log results
|
||||
result_log() {
|
||||
# We want the output format to be:
|
||||
# YYYY-MM-DD HH:MM:SS <result>
|
||||
STAMP=$(date -u "+%Y-%m-%d %H:%M:%S")
|
||||
echo "${STAMP} ${1}" >> ${TEST_LOG_FILE}
|
||||
}
|
||||
|
||||
# Tests to execute
|
||||
tests() {
|
||||
# A simple end-to-end test to verify that we can login via the floating
|
||||
# IP address and can read data from the disk.
|
||||
CMD_CONNECT="timeout 1s ${CMD_SSH_PREFIX} cirros@${INSTANCE_PUBLIC_ADDRESS}"
|
||||
if ${CMD_CONNECT} cat /etc/issue > /dev/null; then
|
||||
result_log PASS
|
||||
PASS=$((PASS+1))
|
||||
else
|
||||
result_log FAIL
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
}
|
||||
|
||||
# Steps to execute when finishing
|
||||
finish() {
|
||||
finish_disk_test
|
||||
rm -f ${TEST_SOCKET_FILE} > /dev/null
|
||||
echo "PASS: ${PASS}" >> ${TEST_LOG_FILE}
|
||||
echo "FAIL: ${FAIL}" >> ${TEST_LOG_FILE}
|
||||
}
|
||||
|
||||
# Setup the disk access test
|
||||
setup_disk_test() {
|
||||
# Copy the disk access test script to the instance
|
||||
${CMD_SCP_PREFIX} ${DISK_ACCESS_SCRIPT} cirros@${INSTANCE_PUBLIC_ADDRESS}:~
|
||||
|
||||
# Initiate the disk access test in the instance
|
||||
CMD_TO_START="sudo /bin/sh disk-access-test.sh > ${DISK_ACCESS_LOG} &"
|
||||
${CMD_SSH_PREFIX} cirros@${INSTANCE_PUBLIC_ADDRESS} "${CMD_TO_START}"
|
||||
}
|
||||
|
||||
# Finish the disk access test
|
||||
finish_disk_test() {
|
||||
# Remove the socket file to stop the test
|
||||
${CMD_SSH_PREFIX} cirros@${INSTANCE_PUBLIC_ADDRESS} sudo rm -f /var/run/disk-access-test.socket
|
||||
|
||||
# Wait 2s for test to finalise
|
||||
sleep 2
|
||||
|
||||
# Fetch the log file with the results
|
||||
${CMD_SCP_PREFIX} cirros@${INSTANCE_PUBLIC_ADDRESS}:${DISK_ACCESS_LOG} /var/log/disk-access-log.log
|
||||
}
|
||||
|
||||
# Setup the trap for the interrupt
|
||||
trap finish SIGHUP SIGINT SIGTERM
|
||||
|
||||
## Main ----------------------------------------------------------------------
|
||||
|
||||
# Create the demorc file if it doesn't exist
|
||||
if [[ ! -f /root/demorc ]]; then
|
||||
setup_demorc
|
||||
fi
|
||||
|
||||
# Fetch the environment variables to be used
|
||||
source /root/demorc
|
||||
|
||||
# Create a volume for the test instance to use for the disk access test
|
||||
if [ -z ${INSTANCE_VOLUME_UUID+x} ]; then
|
||||
INSTANCE_VOLUME_UUID=$(openstack volume create --size 1 ${INSTANCE_NAME} --column id --format value)
|
||||
echo "INSTANCE_VOLUME_UUID=${INSTANCE_VOLUME_UUID}" >> /root/demorc
|
||||
fi
|
||||
|
||||
# Register the private network UUID
|
||||
if [ -z ${INSTANCE_NETWORK_UUID+x} ]; then
|
||||
INSTANCE_NETWORK_UUID=$(openstack network show private --column id --format value)
|
||||
echo "INSTANCE_NETWORK_UUID=${INSTANCE_NETWORK_UUID}" >> /root/demorc
|
||||
fi
|
||||
|
||||
# If a test instance does not exist, create it
|
||||
if [ -z ${INSTANCE_UUID+x} ]; then
|
||||
INSTANCE_UUID=$(openstack server create --flavor tempest1 --image cirros ${INSTANCE_NAME} --nic net-id=${INSTANCE_NETWORK_UUID} --column id --format value)
|
||||
echo "INSTANCE_UUID=${INSTANCE_UUID}" >> /root/demorc
|
||||
fi
|
||||
|
||||
# If a floating IP address has not been allocated, do so
|
||||
if [ -z ${INSTANCE_PUBLIC_ADDRESS+x} ]; then
|
||||
INSTANCE_PUBLIC_ADDRESS=$(openstack floating ip create public --column floating_ip_address --format value)
|
||||
echo "INSTANCE_PUBLIC_ADDRESS=${INSTANCE_PUBLIC_ADDRESS}" >> /root/demorc
|
||||
fi
|
||||
|
||||
# Wait for the server to be ready
|
||||
while [[ "$(openstack server show ${INSTANCE_UUID} --column status --format value)" != "ACTIVE" ]]; do
|
||||
sleep 4
|
||||
done
|
||||
|
||||
# If the floating IP is not associated with the test instance, associate it
|
||||
if ! openstack server show ${INSTANCE_UUID} --column addresses --format value | grep -q ${INSTANCE_PUBLIC_ADDRESS}; then
|
||||
openstack server add floating ip ${INSTANCE_UUID} ${INSTANCE_PUBLIC_ADDRESS}
|
||||
fi
|
||||
|
||||
# Wait for the volume to be ready
|
||||
while [[ "$(openstack volume show ${INSTANCE_VOLUME_UUID} --column status --format value)" != "available" ]]; do
|
||||
sleep 4
|
||||
done
|
||||
|
||||
# Attach the volume to the test instance
|
||||
openstack server add volume ${INSTANCE_UUID} ${INSTANCE_VOLUME_UUID}
|
||||
|
||||
# Wait for the volume to show as in-use
|
||||
while [[ "$(openstack volume show ${INSTANCE_VOLUME_UUID} --column status --format value)" != "in-use" ]]; do
|
||||
sleep 4
|
||||
done
|
||||
|
||||
# Start the disk access test in the instance
|
||||
setup_disk_test
|
||||
|
||||
# Setup the socket file to allow termination later
|
||||
echo $$ > ${TEST_SOCKET_FILE}
|
||||
|
||||
# Execute the test loop
|
||||
while [ -f "${TEST_SOCKET_FILE}" ]; do
|
||||
tests
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# This point will only be reached if the
|
||||
# socket file is removed
|
||||
finish
|
||||
@ -1,82 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2017, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
## Shell Opts ----------------------------------------------------------------
|
||||
set -e
|
||||
|
||||
## Vars ----------------------------------------------------------------------
|
||||
|
||||
# Test script socket file location
|
||||
TEST_SOCKET_FILE="/var/run/disk-access-test.socket"
|
||||
|
||||
# The location to write to
|
||||
TEST_DATA_FILE="/mnt/test"
|
||||
|
||||
# Setup counters
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
## Functions -----------------------------------------------------------------
|
||||
|
||||
# Tests to execute
|
||||
tests() {
|
||||
# We want the output format to be:
|
||||
# YYYY-MM-DD HH:MM:SS <result>
|
||||
echo -n "$(date -u '+%Y-%m-%d %H:%M:%S') "
|
||||
# A simple disk write test to validate whether
|
||||
# we are able to write to disk.
|
||||
CMD_WRITE="timeout 1s dd bs=1M count=50 if=/dev/zero of=${TEST_DATA_FILE} conv=fdatasync"
|
||||
if ${CMD_WRITE}; then
|
||||
echo "PASS"
|
||||
PASS=$((PASS+1))
|
||||
else
|
||||
echo "FAIL"
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
}
|
||||
|
||||
# Steps to execute when finishing
|
||||
finish() {
|
||||
rm -f ${TEST_SOCKET_FILE} > /dev/null
|
||||
echo "PASS: ${PASS}"
|
||||
echo "FAIL: ${FAIL}"
|
||||
}
|
||||
|
||||
# Setup the trap for the interrupt
|
||||
trap finish SIGHUP SIGINT SIGTERM
|
||||
|
||||
## Main ----------------------------------------------------------------------
|
||||
|
||||
# Partition the volume
|
||||
echo ';' | sfdisk --quiet /dev/vdb > /dev/null
|
||||
|
||||
# Format the volume
|
||||
mkfs /dev/vdb1 > /dev/null
|
||||
|
||||
# Mount the volume
|
||||
mount /dev/vdb1 /mnt
|
||||
|
||||
# Setup the socket file to allow termination later
|
||||
echo $$ > ${TEST_SOCKET_FILE}
|
||||
|
||||
# Execute the test loop
|
||||
while [ -f "${TEST_SOCKET_FILE}" ]; do
|
||||
tests
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# This point will only be reached if the
|
||||
# socket file is removed
|
||||
finish
|
||||
Loading…
Reference in New Issue
Block a user