Add tool for updating datasets
Crude copy of migration script for updating datasets for testing. Change-Id: Ic92809c2b9f963af94ca43872835a7e43a9c6d35
This commit is contained in:
Joshua Hesketh
parent
e4c3c390d9
commit
648f74b006
278
tools/update_datasets.sh
Normal file
278
tools/update_datasets.sh
Normal file
@ -0,0 +1,278 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Rackspace Australia
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# A tool to update a given dataset to a given version. Used to keep datasets
|
||||
# somewhat fresh rather than over-exercising old migrations.
|
||||
|
||||
|
||||
# Usage: ./update_datasets.sh VENV_NAME WORKING_DIR GIT_PATH DB_USER DB_PASS DB_NAME SEED_DATA OUTPUT_DATA
|
||||
# apt-get install git virtualenvwrapper python-pip mysql-server python-lxml build-essential libffi-dev
|
||||
|
||||
# $1 is the unique job id
|
||||
# $2 is the working dir path
|
||||
# $3 is the path to the git repo path
|
||||
# $4 is the nova db user
|
||||
# $5 is the nova db password
|
||||
# $6 is the nova db name
|
||||
# $7 is the path to the seed dataset to test against
|
||||
# $8 is the logging.conf for openstack
|
||||
# $9 is the pip cache dir
|
||||
|
||||
UNIQUE_ID=$1
|
||||
WORKING_DIR_PATH=`realpath $2`
|
||||
GIT_REPO_PATH=`realpath $3`
|
||||
DB_USER=$4
|
||||
DB_PASS=$5
|
||||
DB_NAME=$6
|
||||
DATASET_SEED_SQL=`realpath $7`
|
||||
DATASET_OUTPUT_SQL=$8
|
||||
|
||||
SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
|
||||
|
||||
# We also support the following environment variables to tweak our behavour:
|
||||
# NOCLEANUP: if set to anything, don't cleanup at the end of the run
|
||||
|
||||
pip_requires() {
|
||||
pip install -q mysql-python
|
||||
pip install -q eventlet
|
||||
requires="tools/pip-requires"
|
||||
if [ ! -e $requires ]
|
||||
then
|
||||
requires="requirements.txt"
|
||||
fi
|
||||
echo "Install pip requirements from $requires"
|
||||
pip install -q -r $requires
|
||||
echo "Requirements installed"
|
||||
}
|
||||
|
||||
db_sync() {
|
||||
# $1 is the test target (ie branch name)
|
||||
# $2 is an (optional) destination version number
|
||||
|
||||
# Create a nova.conf file
|
||||
cat - > $WORKING_DIR_PATH/nova-$1.conf <<EOF
|
||||
[DEFAULT]
|
||||
sql_connection = mysql://$DB_USER:$DB_PASS@localhost/$DB_NAME?charset=utf8
|
||||
#log_config = $LOG_CONF_FILE
|
||||
EOF
|
||||
|
||||
# Silently return git to a known good state (delete untracked files)
|
||||
git clean -xfdq
|
||||
|
||||
echo "***** Start DB upgrade to state of $1 *****"
|
||||
echo "HEAD of branch under test is:"
|
||||
git log -n 1
|
||||
|
||||
echo "Setting up the nova-manage entry point"
|
||||
python setup.py -q clean
|
||||
python setup.py -q develop
|
||||
python setup.py -q install
|
||||
|
||||
# Log the migrations present
|
||||
echo "Migrations present:"
|
||||
ls $GIT_REPO_PATH/nova/db/sqlalchemy/migrate_repo/versions/*.py | sed 's/.*\///' | egrep "^[0-9]+_"
|
||||
|
||||
# Flush innodb's caches
|
||||
echo "Restarting mysql"
|
||||
sudo service mysql stop
|
||||
sudo service mysql start
|
||||
|
||||
start_version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
|
||||
|
||||
if [ "%$2%" == "%%" ]
|
||||
then
|
||||
end_version=`ls $GIT_REPO_PATH/nova/db/sqlalchemy/migrate_repo/versions/*.py | sed 's/.*\///' | egrep "^[0-9]+_" | tail -1 | cut -f 1 -d "_"`
|
||||
else
|
||||
end_version=$2
|
||||
fi
|
||||
|
||||
echo "Test will migrate from $start_version to $end_version"
|
||||
if [ $end_version -lt $start_version ]
|
||||
then
|
||||
increment=-1
|
||||
end_version=$(( $end_version + 1 ))
|
||||
else
|
||||
increment=1
|
||||
start_version=$(( $start_version + 1))
|
||||
fi
|
||||
|
||||
for i in `seq $start_version $increment $end_version`
|
||||
do
|
||||
set -x
|
||||
$SCRIPT_DIR/nova-manage-wrapper.sh $VENV_PATH --config-file $WORKING_DIR_PATH/nova-$1.conf --verbose db sync --version $i
|
||||
manage_exit=$?
|
||||
set +x
|
||||
|
||||
echo "nova-manage returned exit code $manage_exit"
|
||||
if [ $manage_exit -gt 0 ]
|
||||
then
|
||||
echo "Aborting early"
|
||||
exit $manage_exit
|
||||
fi
|
||||
done
|
||||
|
||||
echo "***** Finished DB upgrade to state of $1 *****"
|
||||
}
|
||||
|
||||
stable_release_db_sync() {
|
||||
version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
|
||||
|
||||
# Some databases are from Folsom
|
||||
echo "Schema version is $version"
|
||||
if [ $version -lt "161" ]
|
||||
then
|
||||
echo "Database is from Folsom! Upgrade via Grizzly"
|
||||
git branch -D eol/grizzly || true
|
||||
git remote update
|
||||
git checkout -b eol/grizzly
|
||||
# Use tag
|
||||
git reset --hard grizzly-eol
|
||||
pip_requires
|
||||
db_sync "grizzly"
|
||||
fi
|
||||
|
||||
version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
|
||||
# Some databases are from Grizzly
|
||||
echo "Schema version is $version"
|
||||
if [ $version -lt "216" ]
|
||||
then
|
||||
echo "Database is from Grizzly! Upgrade via Havana"
|
||||
git branch -D eol/havana || true
|
||||
git remote update
|
||||
git checkout -b eol/havana
|
||||
# Use tag
|
||||
git reset --hard havana-eol
|
||||
pip_requires
|
||||
db_sync "havana"
|
||||
fi
|
||||
|
||||
version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
|
||||
# Some databases are from Havana
|
||||
echo "Schema version is $version"
|
||||
if [ $version -lt "234" ]
|
||||
then
|
||||
echo "Database is from Havana! Upgrade via Icehouse"
|
||||
git branch -D eol/icehouse || true
|
||||
git remote update
|
||||
git checkout -b eol/icehouse
|
||||
# Use tag
|
||||
git reset --hard icehouse-eol
|
||||
pip_requires
|
||||
db_sync "icehouse"
|
||||
fi
|
||||
|
||||
version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
|
||||
# Some databases are from Icehouse
|
||||
echo "Schema version is $version"
|
||||
if [ $version -lt "254" ]
|
||||
then
|
||||
echo "Database is from Icehouse! Upgrade via Juno"
|
||||
git branch -D stable/juno || true
|
||||
git remote update
|
||||
git checkout -b stable/juno
|
||||
git reset --hard remotes/origin/stable/juno
|
||||
pip_requires
|
||||
db_sync "juno"
|
||||
fi
|
||||
|
||||
version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
|
||||
# Some databases are from Juno
|
||||
echo "Schema version is $version"
|
||||
if [ $version -lt "280" ]
|
||||
then
|
||||
echo "Database is from Juno! Upgrade via Kilo"
|
||||
git branch -D stable/kilo || true
|
||||
git remote update
|
||||
git checkout -b stable/kilo
|
||||
git reset --hard remotes/origin/stable/kilo
|
||||
pip_requires
|
||||
db_sync "kilo"
|
||||
|
||||
# TODO(jhesketh): This is a bit of a hack until we update our datasets to
|
||||
# have the flavour data migrated. We have to do this before upgrading from
|
||||
set -x
|
||||
$SCRIPT_DIR/nova-manage-wrapper.sh $VENV_PATH --config-file $WORKING_DIR_PATH/nova-kilo.conf --verbose db migrate_flavor_data --force
|
||||
set +x
|
||||
fi
|
||||
|
||||
# TODO(jhesketh): Add in Liberty here once released
|
||||
|
||||
# TODO(jhesketh): Make this more DRY and/or automatically match migration
|
||||
# numbers to releases.
|
||||
}
|
||||
|
||||
echo "Test running on "`hostname`" as "`whoami`" ("`echo ~`", $HOME)"
|
||||
echo "To execute this script manually, run this:"
|
||||
echo "$0 $@"
|
||||
|
||||
# Setup the environment
|
||||
set -x
|
||||
export PATH=/usr/lib/ccache:$PATH
|
||||
#export PIP_INDEX_URL="http://www.rcbops.com/pypi/mirror"
|
||||
export PIP_INDEX_URL="http://pypi.openstack.org/simple/"
|
||||
export PIP_EXTRA_INDEX_URL="https://pypi.python.org/simple/"
|
||||
which pip
|
||||
pip --version
|
||||
which virtualenv
|
||||
virtualenv --version
|
||||
which mkvirtualenv
|
||||
set +x
|
||||
|
||||
# Restore database to known good state
|
||||
echo "Restoring test database $DB_NAME"
|
||||
set -x
|
||||
mysql -u $DB_USER --password=$DB_PASS -e "drop database $DB_NAME"
|
||||
mysql -u $DB_USER --password=$DB_PASS -e "create database $DB_NAME"
|
||||
mysql -u $DB_USER --password=$DB_PASS $DB_NAME < $DATASET_SEED_SQL
|
||||
set +x
|
||||
|
||||
echo "Build test environment"
|
||||
cd $GIT_REPO_PATH
|
||||
|
||||
echo "Setting up virtual env"
|
||||
source ~/.bashrc
|
||||
export WORKON_HOME=`pwd`/envs
|
||||
mkdir -p $WORKON_HOME
|
||||
VENV_PATH=$WORKON_HOME/$UNIQUE_ID
|
||||
rm -rf $VENV_PATH
|
||||
source /usr/local/bin/virtualenvwrapper.sh
|
||||
source /etc/bash_completion.d/virtualenvwrapper
|
||||
mkvirtualenv --no-site-packages $UNIQUE_ID
|
||||
#toggleglobalsitepackages
|
||||
export PYTHONPATH=$PYTHONPATH:$GIT_REPO_PATH
|
||||
|
||||
if [ ! -e $VENV_PATH ]
|
||||
then
|
||||
echo "Error: making the virtual env failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
stable_release_db_sync
|
||||
|
||||
# Determine the final schema version
|
||||
version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
|
||||
echo "Final schema version is $version"
|
||||
|
||||
if [ "%$NOCLEANUP%" == "%%" ]
|
||||
then
|
||||
# Cleanup virtual env
|
||||
echo "Cleaning up virtual env"
|
||||
deactivate
|
||||
rmvirtualenv $UNIQUE_ID
|
||||
fi
|
||||
|
||||
cd $SCRIPT_DIR
|
||||
mysqldump -u $DB_USER --password=$DB_PASS $DB_NAME > $DATASET_OUTPUT_SQL
|
||||
Loading…
Reference in New Issue
Block a user