Merge "[HBase] Improves speed of unit tests on real HBase backend"

ceilometer/tests/db.py

@@ -23,6 +23,7 @@ import os
 import uuid
 import warnings
 
+import mock
 from oslo.config import fixture as fixture_config
 from oslotest import mockpatch
 import six
@@ -32,6 +33,7 @@ from testtools import testcase
 
 from ceilometer import storage
 from ceilometer.tests import base as test_base
+from ceilometer.tests import mocks
 
 
 class MongoDbManager(fixtures.Fixture):
@@ -71,12 +73,31 @@ class HBaseManager(fixtures.Fixture):
             self.url, 'ceilometer.metering.storage')
         self.alarm_connection = storage.get_connection(
             self.url, 'ceilometer.alarm.storage')
+        # Unique prefix for each test to keep data is distinguished because
+        # all test data is stored in one table
+        data_prefix = str(uuid.uuid4().hex)
+
+        def table(conn, name):
+            return mocks.MockHBaseTable(name, conn, data_prefix)
+
+        # Mock only real HBase connection, MConnection "table" method
+        # stays origin.
+        mock.patch('happybase.Connection.table', new=table).start()
+        # We shouldn't delete data and tables after each test,
+        # because it last for too long.
+        # All tests tables will be deleted in setup-test-env.sh
+        mock.patch("happybase.Connection.disable_table",
+                   new=mock.MagicMock()).start()
+        mock.patch("happybase.Connection.delete_table",
+                   new=mock.MagicMock()).start()
+        mock.patch("happybase.Connection.create_table",
+                   new=mock.MagicMock()).start()
 
     @property
     def url(self):
         return '%s?table_prefix=%s' % (
             self._url,
-            uuid.uuid4().hex
+            os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", "test")
         )
 
 
@@ -106,7 +127,6 @@ class TestBase(testscenarios.testcase.WithScenarios, test_base.BaseTestCase):
 
     def setUp(self):
         super(TestBase, self).setUp()
-
         engine = urlparse.urlparse(self.db_url).scheme
 
         # NOTE(Alexei_987) Shortcut to skip expensive db setUp

ceilometer/tests/mocks.py

@@ -0,0 +1,79 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import happybase
+
+
+class MockHBaseTable(happybase.Table):
+
+    def __init__(self, name, connection, data_prefix):
+        # data_prefix is added to all rows which are written
+        # in this test. It allows to divide data from different tests
+        self.data_prefix = data_prefix
+        # We create happybase Table with prefix from
+        # CEILOMETER_TEST_HBASE_TABLE_PREFIX
+        prefix = os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", 'test')
+        super(MockHBaseTable, self).__init__(
+            "%s_%s" % (prefix, name),
+            connection)
+
+    def put(self, row, *args, **kwargs):
+        row = self.data_prefix + row
+        return super(MockHBaseTable, self).put(row, *args,
+                                               **kwargs)
+
+    def scan(self, row_start=None, row_stop=None, row_prefix=None,
+             columns=None, filter=None, timestamp=None,
+             include_timestamp=False, batch_size=10, scan_batching=None,
+             limit=None, sorted_columns=False):
+        # Add data prefix for row parameters
+        # row_prefix could not be combined with row_start or row_stop
+        if not row_start and not row_stop:
+            row_prefix = self.data_prefix + (row_prefix or "")
+            row_start = None
+            row_stop = None
+        elif row_start and not row_stop:
+            # Adding data_prefix to row_start and row_stop does not work
+            # if it looks like row_start = %data_prefix%foo,
+            # row_stop = %data_prefix, because row_start > row_stop
+            filter = self._update_filter_row(filter)
+            row_start = self.data_prefix + row_start
+        else:
+            row_start = self.data_prefix + (row_start or "")
+            row_stop = self.data_prefix + (row_stop or "")
+        gen = super(MockHBaseTable, self).scan(row_start, row_stop,
+                                               row_prefix, columns,
+                                               filter, timestamp,
+                                               include_timestamp, batch_size,
+                                               scan_batching, limit,
+                                               sorted_columns)
+        data_prefix_len = len(self.data_prefix)
+        # Restore original row format
+        for row, data in gen:
+            yield (row[data_prefix_len:], data)
+
+    def row(self, row, *args, **kwargs):
+        row = self.data_prefix + row
+        return super(MockHBaseTable, self).row(row, *args, **kwargs)
+
+    def delete(self, row, *args, **kwargs):
+        row = self.data_prefix + row
+        return super(MockHBaseTable, self).delete(row, *args, **kwargs)
+
+    def _update_filter_row(self, filter):
+        if filter:
+            return "PrefixFilter(%s) AND %s" % (self.data_prefix, filter)
+        else:
+            return "PrefixFilter(%s)" % self.data_prefix

ceilometer/tests/storage/test_impl_hbase.py

@@ -26,7 +26,6 @@
 import mock
 
 from ceilometer.alarm.storage import impl_hbase as hbase_alarm
-from ceilometer.storage.hbase import inmemory as hbase_inmemory
 from ceilometer.storage import impl_hbase as hbase
 from ceilometer.tests import base as test_base
 from ceilometer.tests import db as tests_db
@@ -37,9 +36,6 @@ class ConnectionTest(tests_db.TestBase,
 
     @tests_db.run_with('hbase')
     def test_hbase_connection(self):
-        conn = hbase.Connection(self.db_manager.url)
-        self.assertIsInstance(conn.conn_pool.connection(),
-                              hbase_inmemory.MConnection)
 
         class TestConn(object):
             def __init__(self, host, port):

setup-test-env.sh

@@ -4,6 +4,10 @@ set -e
 function clean_exit(){
     local error_code="$?"
     rm -rf ${MONGO_DATA}
+    if test -n "$CEILOMETER_TEST_HBASE_URL"
+    then
+            python tools/test_hbase_table_utils.py --clear
+    fi
     kill $(jobs -p)
     return $error_code
 }
@@ -28,6 +32,11 @@ done < ${MONGO_DATA}/out
 # Read the fifo for ever otherwise mongod would block
 cat ${MONGO_DATA}/out > /dev/null &
 export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:${MONGO_PORT}/ceilometer"
+if test -n "$CEILOMETER_TEST_HBASE_URL"
+then
+    export CEILOMETER_TEST_HBASE_TABLE_PREFIX=$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom)
+    python tools/test_hbase_table_utils.py --upgrade
+fi
 
 # Yield execution to venv command
 $*

tools/test_hbase_table_utils.py

@@ -0,0 +1,38 @@
+    # Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import sys
+
+from oslo.config import cfg
+
+from ceilometer import storage
+
+def main(argv):
+    cfg.CONF([], project='ceilometer')
+    if os.getenv("CEILOMETER_TEST_HBASE_URL"):
+        url = ("%s?table_prefix=%s" %
+               (os.getenv("CEILOMETER_TEST_HBASE_URL"),
+                os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", "test")))
+        conn = storage.get_connection(url, 'ceilometer.metering.storage')
+        alarm_conn = storage.get_connection(url, 'ceilometer.alarm.storage')
+        for arg in argv:
+            if arg == "--upgrade":
+                conn.upgrade()
+                alarm_conn.upgrade()
+            if arg == "--clear":
+                conn.clear()
+                alarm_conn.clear()
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])