Run unit tests against MySQL

- Creates a new tox environment for testing against mysql
- Adds a script set-test-env-mysql.sh to set up mysql
- Adds a new file `functions.sh` for common shell functions
- Creates a new DB manager for tests
- Adds a new scenario for mysql in the MixinTestsWithBackendScenarios

Related to blueprint sql-unit-tests-on-real-backend
Co-Authored-By: Ala Rezmerita <ala.rezmerita@cloudwatt.com>

Change-Id: I2c7378c79b1a0bffae5b2489b04c60e509f62f85

ceilometer/tests/db.py

@@ -63,6 +63,23 @@ class MongoDbManager(fixtures.Fixture):
         }
 
 
+class MySQLDbManager(fixtures.Fixture):
+
+    def __init__(self, url):
+        self._url = url
+
+    def setUp(self):
+        super(MySQLDbManager, self).setUp()
+        self.connection = storage.get_connection(
+            self.url, 'ceilometer.metering.storage')
+        self.alarm_connection = storage.get_connection(
+            self.url, 'ceilometer.alarm.storage')
+
+    @property
+    def url(self):
+        return self._url
+
+
 class HBaseManager(fixtures.Fixture):
     def __init__(self, url):
         self._url = url
@@ -118,6 +135,7 @@ class TestBase(testscenarios.testcase.WithScenarios, test_base.BaseTestCase):
 
     DRIVER_MANAGERS = {
         'mongodb': MongoDbManager,
+        'mysql': MySQLDbManager,
         'db2': MongoDbManager,
         'sqlite': SQLiteManager,
         'hbase': HBaseManager,
@@ -202,11 +220,24 @@ class MixinTestsWithBackendScenarios(object):
 
     scenarios = [
         ('sqlite', {'db_url': 'sqlite://'}),
-        ('mongodb', {'db_url': os.environ.get('CEILOMETER_TEST_MONGODB_URL')}),
-        ('hbase', {'db_url': os.environ.get('CEILOMETER_TEST_HBASE_URL',
-                                            'hbase://__test__')}),
-        ('db2', {'db_url': (os.environ.get('CEILOMETER_TEST_DB2_URL') or
-                            os.environ.get('CEILOMETER_TEST_MONGODB_URL',
-                                           '').replace('mongodb://',
-                                                       'db2://'))})
     ]
+
+    for db in ('MONGODB', 'MYSQL', 'HBASE', 'DB2'):
+        if os.environ.get('CEILOMETER_TEST_%s_URL' % db):
+            scenarios.append(
+                (db.lower(), {'db_url': os.environ.get(
+                    'CEILOMETER_TEST_%s_URL' % db)}))
+
+    scenarios_db = [db for db, _ in scenarios]
+
+    # Insert default value for hbase test
+    if 'hbase' not in scenarios_db:
+        scenarios.append(
+            ('hbase', {'db_url': 'hbase://__test__'}))
+
+    # Insert default value for db2 test
+    if 'mongodb' in scenarios_db and 'db2' not in scenarios_db:
+        scenarios.append(
+            ('db2', {'db_url': os.environ.get('CEILOMETER_TEST_MONGODB_URL',
+                                              '').replace('mongodb://',
+                                                          'db2://')}))

functions.sh

@@ -0,0 +1,28 @@
+function clean_exit(){
+    local error_code="$?"
+    if test -n "$CEILOMETER_TEST_HBASE_URL"
+    then
+            python tools/test_hbase_table_utils.py --clear
+    fi
+    rm -rf "$1"
+    kill $(jobs -p)
+    return $error_code
+}
+
+check_for_cmd () {
+    if ! which "$1" >/dev/null 2>&1
+    then
+        echo "Could not find $1 command" 1>&2
+        exit 1
+    fi
+}
+
+wait_for_line () {
+    while read line
+    do
+        echo "$line" | grep -q "$1" && break
+    done < "$2"
+    # Read the fifo for ever otherwise process would block
+    cat "$2" >/dev/null &
+}
+

setup-test-env-mongodb.sh

@@ -1,34 +1,24 @@
 #!/bin/bash
 set -e
 
-function clean_exit(){
+source functions.sh
-    local error_code="$?"
-    rm -rf ${MONGO_DATA}
-    if test -n "$CEILOMETER_TEST_HBASE_URL"
-    then
-            python tools/test_hbase_table_utils.py --clear
-    fi
-    kill $(jobs -p)
-    return $error_code
-}
 
-# Setup MongoDB test server
+if [ "$1" = "--coverage" ]; then
+	COVERAGE_ARG="$1"
+	shift
+fi
+
+export PATH=${PATH:+$PATH:}/sbin:/usr/sbin
+check_for_cmd mongod
+
+# Start MongoDB process for tests
 MONGO_DATA=`mktemp -d CEILO-MONGODB-XXXXX`
 MONGO_PORT=29000
-trap "clean_exit" EXIT
+trap "clean_exit ${MONGO_DATA}" EXIT
 mkfifo ${MONGO_DATA}/out
-export PATH=${PATH:+$PATH:}/sbin:/usr/sbin
-if ! which mongod >/dev/null 2>&1
-then
-    echo "Could not find mongod command" 1>&2
-    exit 1
-fi
 mongod --maxConns 32 --nojournal --noprealloc --smallfiles --quiet --noauth --port ${MONGO_PORT} --dbpath "${MONGO_DATA}" --bind_ip localhost --config /dev/null &>${MONGO_DATA}/out &
 # Wait for Mongo to start listening to connections
-while read line
+wait_for_line "waiting for connections on port ${MONGO_PORT}" ${MONGO_DATA}/out
-do
-    echo "$line" | grep -q "waiting for connections on port ${MONGO_PORT}" && break
-done < ${MONGO_DATA}/out
 # Read the fifo for ever otherwise mongod would block
 cat ${MONGO_DATA}/out > /dev/null &
 export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:${MONGO_PORT}/ceilometer"

setup-test-env-mysql.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+
+source functions.sh
+
+if [ "$1" = "--coverage" ]; then
+	COVERAGE_ARG="$1"
+	shift
+fi
+
+export PATH=${PATH:+$PATH:}/sbin:/usr/sbin
+
+# On systems like Fedora here's where mysqld can be found
+export PATH=$PATH:/usr/libexec
+
+check_for_cmd mysqld
+
+# Start MySQL process for tests
+MYSQL_DATA=`mktemp -d /tmp/CEILO-MYSQL-XXXXX`
+trap "clean_exit ${MYSQL_DATA}" EXIT
+mkfifo ${MYSQL_DATA}/out
+mysqld --datadir=${MYSQL_DATA} --pid-file=${MYSQL_DATA}/mysql.pid --socket=${MYSQL_DATA}/mysql.socket --skip-networking --skip-grant-tables &> ${MYSQL_DATA}/out &
+# Wait for MySQL to start listening to connections
+wait_for_line "mysqld: ready for connections." ${MYSQL_DATA}/out
+export CEILOMETER_TEST_MYSQL_URL="mysql://root@localhost/ceilometer?unix_socket=${MYSQL_DATA}/mysql.socket&charset=utf8"
+
+mysql -S ${MYSQL_DATA}/mysql.socket -e 'CREATE DATABASE ceilometer;'
+
+# Yield execution to venv command
+$*

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 minversion = 1.6
 skipsdist = True
-envlist = py26,py27,py33,pep8
+envlist = py26,py27,py27-mysql,py33,pep8
 
 [testenv]
 deps = -r{toxinidir}/requirements.txt
@@ -12,10 +12,14 @@ setenv = VIRTUAL_ENV={envdir}
          EVENTLET_NO_GREENDNS=yes
          PYTHONHASHSEED=0
 commands =
-   bash -x {toxinidir}/setup-test-env.sh python setup.py testr --slowest --testr-args="{posargs}"
+   bash -x {toxinidir}/setup-test-env-mongodb.sh python setup.py testr --slowest --testr-args="{posargs}"
 downloadcache = {toxworkdir}/_download
 whitelist_externals = bash
 
+[testenv:py27-mysql]
+commands =
+   bash -x {toxinidir}/setup-test-env-mysql.sh python setup.py testr --slowest --testr-args="{posargs}"
+
 [testenv:py33]
 deps = -r{toxinidir}/requirements-py3.txt
        -r{toxinidir}/test-requirements-py3.txt