Run some APIv1 tests on different backends

This is a first change towards implementing bug #1131638. It clears a lot of
stuff in the various DB mocking stuff, and I plan to do more after.

In the same process, this implements a unit test and a fix for bug #1138137

Change-Id: I36a6741393455c764913e0388d73d834db06da8c
Signed-off-by: Julien Danjou <julien@danjou.info>

ceilometer/storage/base.py

@@ -182,3 +182,7 @@ class Connection(object):
           }
 
         """
+
+    @abc.abstractmethod
+    def clear(self):
+        """Clear database."""

ceilometer/storage/impl_log.py

@@ -48,6 +48,9 @@ class Connection(base.Connection):
     def upgrade(self, version=None):
         pass
 
+    def clear(self):
+        pass
+
     def record_metering_data(self, data):
         """Write the data to the backend storage system.
 

ceilometer/storage/impl_mongodb.py

@@ -268,6 +268,9 @@ class Connection(base.Connection):
     def upgrade(self, version=None):
         pass
 
+    def clear(self):
+        self.conn.drop_database(self.db)
+
     def _get_connection(self, opts):
         """Return a connection to the database.
 

ceilometer/storage/impl_sqlalchemy.py

@@ -27,7 +27,7 @@ from ceilometer.openstack.common import log
 from ceilometer.openstack.common import timeutils
 from ceilometer.storage import base
 from ceilometer.storage.sqlalchemy.models import Meter, Project, Resource
-from ceilometer.storage.sqlalchemy.models import Source, User
+from ceilometer.storage.sqlalchemy.models import Source, User, Base
 import ceilometer.storage.sqlalchemy.session as sqlalchemy_session
 from ceilometer.storage.sqlalchemy import migration
 
@@ -86,7 +86,8 @@ class SQLAlchemyStorage(base.StorageEngine):
         """
         conf.register_opts(self.OPTIONS)
 
-    def get_connection(self, conf):
+    @staticmethod
+    def get_connection(conf):
         """Return a Connection instance based on the configuration settings.
         """
         return Connection(conf)
@@ -114,7 +115,7 @@ def make_query_from_filter(query, event_filter, require_meter=True):
         query = query.filter(Meter.timestamp < ts_end)
     if event_filter.user:
         query = query.filter_by(user_id=event_filter.user)
-    elif event_filter.project:
+    if event_filter.project:
         query = query.filter_by(project_id=event_filter.project)
     if event_filter.resource:
         query = query.filter_by(resource_id=event_filter.resource)
@@ -137,6 +138,11 @@ class Connection(base.Connection):
     def upgrade(self, version=None):
         migration.db_sync(self.session.get_bind(), version=version)
 
+    def clear(self):
+        engine = self.session.get_bind()
+        for table in reversed(Base.metadata.sorted_tables):
+            engine.execute(table.delete())
+
     def _get_connection(self, conf):
         """Return a connection to the database.
         """

ceilometer/storage/impl_test.py

@@ -79,14 +79,14 @@ class TestConnection(impl_mongodb.Connection):
     _mim_instance = None
     FORCE_MONGO = bool(int(os.environ.get('CEILOMETER_TEST_LIVE', 0)))
 
-    def drop_database(self, database):
+    def clear(self):
         if TestConnection._mim_instance is not None:
-            # Don't want to use drop_database() because we
+            # Don't want to use drop_database() because
             # may end up running out of spidermonkey instances.
             # http://davisp.lighthouseapp.com/projects/26898/tickets/22
-            self.conn[database].clear()
+            self.db.clear()
         else:
-            self.conn.drop_database(database)
+            super(TestConnection, self).clear()
 
     def _get_connection(self, conf):
         # Use a real MongoDB server if we can connect, but fall back

ceilometer/tests/api.py

@@ -66,25 +66,19 @@ class TestBase(db_test_base.TestBase):
         return rv
 
 
-class FunctionalTest(base.TestCase):
+class FunctionalTest(db_test_base.TestBase):
     """
     Used for functional tests of Pecan controllers where you need to
     test your literal application and its integration with the
     framework.
     """
 
-    DBNAME = 'testdb'
-
     PATH_PREFIX = ''
 
     SOURCE_DATA = {'test_source': {'somekey': '666'}}
 
     def setUp(self):
         super(FunctionalTest, self).setUp()
-
-        cfg.CONF.database_connection = 'test://localhost/%s' % self.DBNAME
-        self.conn = storage.get_connection(cfg.CONF)
-        self.conn.drop_database(self.DBNAME)
         self.app = self._make_app()
 
     def _make_app(self, enable_acl=False):

ceilometer/tests/db.py

@@ -1,8 +1,10 @@
 # -*- encoding: utf-8 -*-
 #
 # Copyright © 2012 New Dream Network, LLC (DreamHost)
+# Copyright © 2013 eNovance
 #
 # Author: Doug Hellmann <doug.hellmann@dreamhost.com>
+#         Julien Danjou <julien@danjou.info>
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -19,33 +21,24 @@
 """
 
 from ming import mim
-
-import mock
-
 from nose.plugins import skip
+from oslo.config import cfg
 
-from ceilometer.openstack.common import log as logging
-from ceilometer.storage.impl_test import TestConnection
+from ceilometer import storage
 from ceilometer.tests import base as test_base
 
-LOG = logging.getLogger(__name__)
-
 
 class TestBase(test_base.TestCase):
 
-    DBNAME = 'testdb'
+    # Default tests use test:// (MIM)
+    database_connection = 'test://'
 
     def setUp(self):
         super(TestBase, self).setUp()
-        self.conf = mock.Mock()
-        self.conf.database_connection = 'mongodb://localhost/%s' % self.DBNAME
-        self.conn = TestConnection(self.conf)
-        self.conn.drop_database(self.DBNAME)
-        self.conn.conn[self.DBNAME]
-
-    def tearDown(self):
-        self.conn.drop_database(self.DBNAME)
-        super(TestBase, self).tearDown()
+        cfg.CONF.database_connection = self.database_connection
+        self.conn = storage.get_connection(cfg.CONF)
+        self.conn.upgrade()
+        self.conn.clear()
 
 
 def require_map_reduce(conn):

tests/api/v1/list_events.py

@@ -20,7 +20,6 @@
 """
 
 import datetime
-import logging
 
 from oslo.config import cfg
 
@@ -29,8 +28,6 @@ from ceilometer import counter
 
 from ceilometer.tests import api as tests_api
 
-LOG = logging.getLogger(__name__)
-
 
 class TestListEvents(tests_api.TestBase):
 
@@ -169,6 +166,15 @@ class TestListEvents(tests_api.TestBase):
                         end_timestamp=datetime.datetime(2012, 7, 2, 10, 42))
         self.assertEquals(1, len(data['events']))
 
+    def test_template_list_event(self):
+        rv = self.get('/resources/resource-id/meters/instance',
+                      headers={"Accept": "text/html"})
+        self.assertEqual(200, rv.status_code)
+        self.assertTrue("text/html" in rv.content_type)
+
+
+class TestListEventsMetaquery(TestListEvents):
+
     def test_metaquery1(self):
         q = '/sources/source1/meters/instance'
         data = self.get('%s?metadata.tag=self.counter2' % q)
@@ -204,9 +210,3 @@ class TestListEvents(tests_api.TestBase):
                         headers={"X-Roles": "Member",
                                  "X-Tenant-Id": "project2"})
         self.assertEquals(1, len(data['events']))
-
-    def test_template_list_event(self):
-        rv = self.get('/resources/resource-id/meters/instance',
-                      headers={"Accept": "text/html"})
-        self.assertEqual(200, rv.status_code)
-        self.assertTrue("text/html" in rv.content_type)

tests/api/v1/test_impl_mongodb.py

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2013 eNovance
+#
+# Author: Julien Danjou <julien@danjou.info>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Test listing raw events.
+"""
+from . import list_events
+
+
+class TestListEvents(list_events.TestListEvents):
+    database_connection = 'test://'
+
+
+class TestListEventsMetaQuery(list_events.TestListEventsMetaquery):
+    database_connection = 'test://'

tests/api/v1/test_impl_sqlalchemy.py

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2013 eNovance
+#
+# Author: Julien Danjou <julien@danjou.info>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Test listing raw events.
+"""
+from . import list_events
+
+
+class TestListEvents(list_events.TestListEvents):
+    database_connection = 'sqlite://'

tests/storage/base.py

@@ -77,6 +77,7 @@ class DBTestBase(test_base.TestCase):
 
     def setUp(self):
         super(DBTestBase, self).setUp()
+        # TODO(jd) remove, use test_base.TestCase setUp to do that
         self.engine = self.get_engine()
         self.conn = self.engine.get_connection()
         self.prepare_data()

tests/storage/test_impl_mongodb.py

@@ -69,7 +69,7 @@ class MongoDBEngine(base.DBEngineBase):
         return self.conn
 
     def clean_up(self):
-        self.conn.drop_database(self.DBNAME)
+        self.conn.clear()
 
     def get_sources_by_project_id(self, id):
         project = self.db.project.find_one({'_id': id})