From 1278d067ef6d36ca73efb7fb0e9371ad2fe73b79 Mon Sep 17 00:00:00 2001 From: "Joe H. Rahme" Date: Fri, 29 Aug 2014 13:33:30 +0200 Subject: [PATCH] Run unit tests against MySQL - Creates a new tox environment for testing against mysql - Adds a script set-test-env-mysql.sh to set up mysql - Adds a new file `functions.sh` for common shell functions - Creates a new DB manager for tests - Adds a new scenario for mysql in the MixinTestsWithBackendScenarios Related to blueprint sql-unit-tests-on-real-backend Co-Authored-By: Ala Rezmerita Change-Id: I2c7378c79b1a0bffae5b2489b04c60e509f62f85 --- ceilometer/tests/db.py | 45 ++++++++++++++++--- functions.sh | 28 ++++++++++++ ...p-test-env.sh => setup-test-env-mongodb.sh | 34 +++++--------- setup-test-env-mysql.sh | 30 +++++++++++++ tox.ini | 8 +++- 5 files changed, 114 insertions(+), 31 deletions(-) create mode 100644 functions.sh rename setup-test-env.sh => setup-test-env-mongodb.sh (60%) create mode 100755 setup-test-env-mysql.sh diff --git a/ceilometer/tests/db.py b/ceilometer/tests/db.py index 68c869b58..2cba7f2f7 100644 --- a/ceilometer/tests/db.py +++ b/ceilometer/tests/db.py @@ -63,6 +63,23 @@ class MongoDbManager(fixtures.Fixture): } +class MySQLDbManager(fixtures.Fixture): + + def __init__(self, url): + self._url = url + + def setUp(self): + super(MySQLDbManager, self).setUp() + self.connection = storage.get_connection( + self.url, 'ceilometer.metering.storage') + self.alarm_connection = storage.get_connection( + self.url, 'ceilometer.alarm.storage') + + @property + def url(self): + return self._url + + class HBaseManager(fixtures.Fixture): def __init__(self, url): self._url = url @@ -118,6 +135,7 @@ class TestBase(testscenarios.testcase.WithScenarios, test_base.BaseTestCase): DRIVER_MANAGERS = { 'mongodb': MongoDbManager, + 'mysql': MySQLDbManager, 'db2': MongoDbManager, 'sqlite': SQLiteManager, 'hbase': HBaseManager, @@ -202,11 +220,24 @@ class MixinTestsWithBackendScenarios(object): scenarios = [ ('sqlite', {'db_url': 'sqlite://'}), - ('mongodb', {'db_url': os.environ.get('CEILOMETER_TEST_MONGODB_URL')}), - ('hbase', {'db_url': os.environ.get('CEILOMETER_TEST_HBASE_URL', - 'hbase://__test__')}), - ('db2', {'db_url': (os.environ.get('CEILOMETER_TEST_DB2_URL') or - os.environ.get('CEILOMETER_TEST_MONGODB_URL', - '').replace('mongodb://', - 'db2://'))}) ] + + for db in ('MONGODB', 'MYSQL', 'HBASE', 'DB2'): + if os.environ.get('CEILOMETER_TEST_%s_URL' % db): + scenarios.append( + (db.lower(), {'db_url': os.environ.get( + 'CEILOMETER_TEST_%s_URL' % db)})) + + scenarios_db = [db for db, _ in scenarios] + + # Insert default value for hbase test + if 'hbase' not in scenarios_db: + scenarios.append( + ('hbase', {'db_url': 'hbase://__test__'})) + + # Insert default value for db2 test + if 'mongodb' in scenarios_db and 'db2' not in scenarios_db: + scenarios.append( + ('db2', {'db_url': os.environ.get('CEILOMETER_TEST_MONGODB_URL', + '').replace('mongodb://', + 'db2://')})) diff --git a/functions.sh b/functions.sh new file mode 100644 index 000000000..633d30f84 --- /dev/null +++ b/functions.sh @@ -0,0 +1,28 @@ +function clean_exit(){ + local error_code="$?" + if test -n "$CEILOMETER_TEST_HBASE_URL" + then + python tools/test_hbase_table_utils.py --clear + fi + rm -rf "$1" + kill $(jobs -p) + return $error_code +} + +check_for_cmd () { + if ! which "$1" >/dev/null 2>&1 + then + echo "Could not find $1 command" 1>&2 + exit 1 + fi +} + +wait_for_line () { + while read line + do + echo "$line" | grep -q "$1" && break + done < "$2" + # Read the fifo for ever otherwise process would block + cat "$2" >/dev/null & +} + diff --git a/setup-test-env.sh b/setup-test-env-mongodb.sh similarity index 60% rename from setup-test-env.sh rename to setup-test-env-mongodb.sh index 09fc9b4e8..3dcad1e90 100755 --- a/setup-test-env.sh +++ b/setup-test-env-mongodb.sh @@ -1,34 +1,24 @@ #!/bin/bash set -e -function clean_exit(){ - local error_code="$?" - rm -rf ${MONGO_DATA} - if test -n "$CEILOMETER_TEST_HBASE_URL" - then - python tools/test_hbase_table_utils.py --clear - fi - kill $(jobs -p) - return $error_code -} +source functions.sh -# Setup MongoDB test server +if [ "$1" = "--coverage" ]; then + COVERAGE_ARG="$1" + shift +fi + +export PATH=${PATH:+$PATH:}/sbin:/usr/sbin +check_for_cmd mongod + +# Start MongoDB process for tests MONGO_DATA=`mktemp -d CEILO-MONGODB-XXXXX` MONGO_PORT=29000 -trap "clean_exit" EXIT +trap "clean_exit ${MONGO_DATA}" EXIT mkfifo ${MONGO_DATA}/out -export PATH=${PATH:+$PATH:}/sbin:/usr/sbin -if ! which mongod >/dev/null 2>&1 -then - echo "Could not find mongod command" 1>&2 - exit 1 -fi mongod --maxConns 32 --nojournal --noprealloc --smallfiles --quiet --noauth --port ${MONGO_PORT} --dbpath "${MONGO_DATA}" --bind_ip localhost --config /dev/null &>${MONGO_DATA}/out & # Wait for Mongo to start listening to connections -while read line -do - echo "$line" | grep -q "waiting for connections on port ${MONGO_PORT}" && break -done < ${MONGO_DATA}/out +wait_for_line "waiting for connections on port ${MONGO_PORT}" ${MONGO_DATA}/out # Read the fifo for ever otherwise mongod would block cat ${MONGO_DATA}/out > /dev/null & export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:${MONGO_PORT}/ceilometer" diff --git a/setup-test-env-mysql.sh b/setup-test-env-mysql.sh new file mode 100755 index 000000000..301ab446b --- /dev/null +++ b/setup-test-env-mysql.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e + +source functions.sh + +if [ "$1" = "--coverage" ]; then + COVERAGE_ARG="$1" + shift +fi + +export PATH=${PATH:+$PATH:}/sbin:/usr/sbin + +# On systems like Fedora here's where mysqld can be found +export PATH=$PATH:/usr/libexec + +check_for_cmd mysqld + +# Start MySQL process for tests +MYSQL_DATA=`mktemp -d /tmp/CEILO-MYSQL-XXXXX` +trap "clean_exit ${MYSQL_DATA}" EXIT +mkfifo ${MYSQL_DATA}/out +mysqld --datadir=${MYSQL_DATA} --pid-file=${MYSQL_DATA}/mysql.pid --socket=${MYSQL_DATA}/mysql.socket --skip-networking --skip-grant-tables &> ${MYSQL_DATA}/out & +# Wait for MySQL to start listening to connections +wait_for_line "mysqld: ready for connections." ${MYSQL_DATA}/out +export CEILOMETER_TEST_MYSQL_URL="mysql://root@localhost/ceilometer?unix_socket=${MYSQL_DATA}/mysql.socket&charset=utf8" + +mysql -S ${MYSQL_DATA}/mysql.socket -e 'CREATE DATABASE ceilometer;' + +# Yield execution to venv command +$* diff --git a/tox.ini b/tox.ini index ec05ceee8..6e2ef3fe4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] minversion = 1.6 skipsdist = True -envlist = py26,py27,py33,pep8 +envlist = py26,py27,py27-mysql,py33,pep8 [testenv] deps = -r{toxinidir}/requirements.txt @@ -12,10 +12,14 @@ setenv = VIRTUAL_ENV={envdir} EVENTLET_NO_GREENDNS=yes PYTHONHASHSEED=0 commands = - bash -x {toxinidir}/setup-test-env.sh python setup.py testr --slowest --testr-args="{posargs}" + bash -x {toxinidir}/setup-test-env-mongodb.sh python setup.py testr --slowest --testr-args="{posargs}" downloadcache = {toxworkdir}/_download whitelist_externals = bash +[testenv:py27-mysql] +commands = + bash -x {toxinidir}/setup-test-env-mysql.sh python setup.py testr --slowest --testr-args="{posargs}" + [testenv:py33] deps = -r{toxinidir}/requirements-py3.txt -r{toxinidir}/test-requirements-py3.txt