Correct database functional tests

The database related functional tests would not work. Several
changes were required to fix them:

* Clean up permissions in function post_test_hook

  Without this tox can't create its .tox directories. We want to be
  running as user stack against a directory owned by user stack.

  We need to check for a directory named 'devstack' not a file in
  order to get the permissions changes to happen.

* Call pg_ctl initdb, not initdb to create the postgresql database.

* Add a DRIVER_MANAGER for mysql+pymysql (without this all mysql
  tests are skipped, even in the unit tests).

* Change get_connection so it doesn't split on + when looking up
  storage extensions, as we do want to use pymysql.

* Make sure AODH_BACKEND is set and exported in gate_hook.sh

* Mysql requires a database exist when we name it in a sql
  create_engine, replace the placeholder with '' when in mysql.

* Move keystonemiddleware config to the correct section of the paste
  file, temporarily.

Change-Id: I001e34e28353a35148e2101ab90cb3d238bd53cb

aodh/storage/__init__.py

@@ -79,13 +79,9 @@ def get_connection_from_config(conf):
     retries = conf.database.max_retries
     url = conf.database.connection
     connection_scheme = urlparse.urlparse(url).scheme
-    # SQLAlchemy connections specify may specify a 'dialect' or
-    # 'dialect+driver'. Handle the case where driver is specified.
-    engine_name = connection_scheme.split('+')[0]
-    # NOTE: translation not applied bug #1446983
     LOG.debug('looking for %(name)r driver in %(namespace)r',
-              {'name': engine_name, 'namespace': _NAMESPACE})
+              {'name': connection_scheme, 'namespace': _NAMESPACE})
-    mgr = driver.DriverManager(_NAMESPACE, engine_name)
+    mgr = driver.DriverManager(_NAMESPACE, connection_scheme)
 
     # Convert retry_interval secs to msecs for retry decorator
     @retrying.retry(wait_fixed=conf.database.retry_interval * 1000,

aodh/tests/db.py

@@ -51,7 +51,9 @@ class SQLManager(fixtures.Fixture):
     def __init__(self, conf):
         self.conf = conf
         db_name = 'aodh_%s' % uuid.uuid4().hex
-        self._engine = sqlalchemy.create_engine(conf.database.connection)
+        self._engine = sqlalchemy.create_engine(
+            conf.database.connection.replace(self.url_dbname_placeholder,
+                                             self.url_dbname_createstring))
         self._conn = self._engine.connect()
         self._create_db(self._conn, db_name)
         self._conn.close()
@@ -63,6 +65,7 @@ class SQLManager(fixtures.Fixture):
 class PgSQLManager(SQLManager):
 
     url_dbname_placeholder = 'template1'
+    url_dbname_createstring = url_dbname_placeholder
 
     @staticmethod
     def _create_db(conn, db_name):
@@ -74,6 +77,7 @@ class PgSQLManager(SQLManager):
 class MySQLManager(SQLManager):
 
     url_dbname_placeholder = 'test'
+    url_dbname_createstring = ''
 
     @staticmethod
     def _create_db(conn, db_name):
@@ -121,6 +125,7 @@ class TestBase(testscenarios.testcase.WithScenarios, test_base.BaseTestCase):
     DRIVER_MANAGERS = {
         'mongodb': MongoDbManager,
         'mysql': MySQLManager,
+        'mysql+pymysql': MySQLManager,
         'postgresql': PgSQLManager,
         'db2': MongoDbManager,
         'sqlite': SQLiteManager,

aodh/tests/functional/hooks/post_test_hook.sh

@@ -31,10 +31,10 @@ function generate_testr_results {
 # If we're running in the gate find our keystone endpoint to give to
 # gabbi tests and do a chown. Otherwise the existing environment
 # should provide URL and TOKEN.
-if [ -f $BASE/new/devstack ]; then
+if [ -d $BASE/new/devstack ]; then
     export AODH_DIR="$BASE/new/aodh"
-    JENKINS_USER=jenkins
+    STACK_USER=stack
-    sudo chown -R jenkins:stack $AODH_DIR
+    sudo chown -R $STACK_USER:stack $AODH_DIR
     source $BASE/new/devstack/openrc admin admin
     openstack catalog list
     export AODH_SERVICE_URL=$(openstack catalog show alarming -c endpoints -f value | awk '/publicURL/{print $2}')
@@ -48,7 +48,7 @@ echo "Running aodh functional test suite"
 set +e
 
 # NOTE(ityaptin) Expect a script param which contains at least one backend name
-AODH_TEST_BACKEND="${1:?test backend required}" sudo -E -H -u ${JENKINS_USER:-${USER}} tox -efunctional
+AODH_TEST_BACKEND="${1:?test backend required}" sudo -E -H -u ${STACK_USER:-${USER}} tox -efunctional
 EXIT_CODE=$?
 set -e
 

devstack/gate/gate_hook.sh

@@ -14,13 +14,12 @@
 
 # This script is executed inside gate_hook function in devstack gate.
 
-# A space separated lists of storage backends.
-STORAGE_DRIVERS="$1"
-
 ENABLED_SERVICES="key,aodi-api,aodh-notifier,aodh-evaluator"
 ENABLED_SERVICES+="ceilometer-acompute,ceilometer-acentral,ceilometer-anotification,"
 ENABLED_SERVICES+="ceilometer-collector,ceilometer-api,"
 
+# The backend is passed in by the job as the first and only argument
+export AODH_BACKEND="${1}"
 export DEVSTACK_GATE_INSTALL_TESTONLY=1
 export DEVSTACK_GATE_NO_SERVICES=1
 export DEVSTACK_GATE_TEMPEST=0
@@ -28,12 +27,14 @@ export DEVSTACK_GATE_EXERCISES=0
 export KEEP_LOCALRC=1
 
 # default to mysql
-case $STORAGE_DRIVER in
+case $AODH_BACKEND in
-    *postgresql*)
+    postgresql)
         export DEVSTACK_GATE_POSTGRES=1
         ;;
 esac
 
+DEVSTACK_LOCAL_CONFIG+=$'\n'"export AODH_BACKEND=${AODH_BACKEND}"
+
 export ENABLED_SERVICES
 
 $BASE/new/devstack-gate/devstack-vm-gate.sh

devstack/plugin.sh

@@ -190,7 +190,14 @@ function configure_aodh {
     iniset $AODH_CONF service_credentials os_region_name $REGION_NAME
     iniset $AODH_CONF service_credentials os_auth_url $KEYSTONE_SERVICE_URI/v2.0
 
-    configure_auth_token_middleware $AODH_CONF aodh $AODH_AUTH_CACHE_DIR
+    # TODO(chdent): Until
+    # https://bugs.launchpad.net/keystonemiddleware/+bug/1482078
+    # and
+    # https://bugs.launchpad.net/keystonemiddleware/+bug/1406218
+    # are resolved the easiest way to deal with the auth_token
+    # middleware when using a non-global conf is to put it in the
+    # paste file.
+    configure_auth_token_middleware $AODH_CONF_DIR/api_paste.ini aodh $AODH_AUTH_CACHE_DIR filter:authtoken
 
     iniset $AODH_CONF notification store_events $AODH_EVENTS
 

setup-test-env-postgresql.sh

@@ -24,10 +24,10 @@ check_for_cmd pg_config
 PGSQL_DATA=`mktemp -d /tmp/AODH-PGSQL-XXXXX`
 PGSQL_PATH=`pg_config --bindir`
 PGSQL_PORT=9823
-${PGSQL_PATH}/initdb -E UTF8 ${PGSQL_DATA}
+${PGSQL_PATH}/pg_ctl initdb -D ${PGSQL_DATA}
 trap "clean_exit_pgsql ${PGSQL_PATH} ${PGSQL_DATA} ${PGSQL_PORT}" EXIT
 
-LANGUAGE=C ${PGSQL_PATH}/pg_ctl -w -D ${PGSQL_DATA} -o "-F -k ${PGSQL_DATA} -p ${PGSQL_PORT}" start
+LANGUAGE=C ${PGSQL_PATH}/pg_ctl -w -D ${PGSQL_DATA} -o "-k ${PGSQL_DATA} -p ${PGSQL_PORT}" start
 export AODH_TEST_PGSQL_URL="postgresql:///?host=${PGSQL_DATA}&port=${PGSQL_PORT}&dbname=template1"
 
 # Yield execution to venv command

setup.cfg

@@ -30,6 +30,7 @@ aodh.storage =
     log = aodh.storage.impl_log:Connection
     mongodb = aodh.storage.impl_mongodb:Connection
     mysql = aodh.storage.impl_sqlalchemy:Connection
+    mysql+pymysql = aodh.storage.impl_sqlalchemy:Connection
     postgresql = aodh.storage.impl_sqlalchemy:Connection
     sqlite = aodh.storage.impl_sqlalchemy:Connection
     hbase = aodh.storage.impl_hbase:Connection