openstack/aodh
Code Issues Proposed changes

Merge "[HBase] Improves speed of unit tests on real HBase backend"

Browse Source
This commit is contained in:
Jenkins 2014-09-29 13:16:04 +00:00 committed by Gerrit Code Review
commit 8b61fdd2db
5 changed files with 148 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
ceilometer/tests/db.py
View File

@ -23,6 +23,7 @@ import os
import uuid
import warnings
import mock
from oslo.config import fixture as fixture_config
from oslotest import mockpatch
import six
@ -32,6 +33,7 @@ from testtools import testcase
from ceilometer import storage
from ceilometer.tests import base as test_base
from ceilometer.tests import mocks
class MongoDbManager(fixtures.Fixture):
@ -71,12 +73,31 @@ class HBaseManager(fixtures.Fixture):
self.url, 'ceilometer.metering.storage')
self.alarm_connection = storage.get_connection(
self.url, 'ceilometer.alarm.storage')
# Unique prefix for each test to keep data is distinguished because
# all test data is stored in one table
data_prefix = str(uuid.uuid4().hex)
def table(conn, name):
return mocks.MockHBaseTable(name, conn, data_prefix)
# Mock only real HBase connection, MConnection "table" method
# stays origin.
mock.patch('happybase.Connection.table', new=table).start()
# We shouldn't delete data and tables after each test,
# because it last for too long.
# All tests tables will be deleted in setup-test-env.sh
mock.patch("happybase.Connection.disable_table",
new=mock.MagicMock()).start()
mock.patch("happybase.Connection.delete_table",
new=mock.MagicMock()).start()
mock.patch("happybase.Connection.create_table",
new=mock.MagicMock()).start()
@property
def url(self):
return '%s?table_prefix=%s' % (
self._url,
uuid.uuid4().hex
os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", "test")
)
@ -106,7 +127,6 @@ class TestBase(testscenarios.testcase.WithScenarios, test_base.BaseTestCase):
def setUp(self):
super(TestBase, self).setUp()
engine = urlparse.urlparse(self.db_url).scheme
# NOTE(Alexei_987) Shortcut to skip expensive db setUp

79
ceilometer/tests/mocks.py Normal file
View File

@ -0,0 +1,79 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import happybase
class MockHBaseTable(happybase.Table):
def __init__(self, name, connection, data_prefix):
# data_prefix is added to all rows which are written
# in this test. It allows to divide data from different tests
self.data_prefix = data_prefix
# We create happybase Table with prefix from
# CEILOMETER_TEST_HBASE_TABLE_PREFIX
prefix = os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", 'test')
super(MockHBaseTable, self).__init__(
"%s_%s" % (prefix, name),
connection)
def put(self, row, *args, **kwargs):
row = self.data_prefix + row
return super(MockHBaseTable, self).put(row, *args,
**kwargs)
def scan(self, row_start=None, row_stop=None, row_prefix=None,
columns=None, filter=None, timestamp=None,
include_timestamp=False, batch_size=10, scan_batching=None,
limit=None, sorted_columns=False):
# Add data prefix for row parameters
# row_prefix could not be combined with row_start or row_stop
if not row_start and not row_stop:
row_prefix = self.data_prefix + (row_prefix or "")
row_start = None
row_stop = None
elif row_start and not row_stop:
# Adding data_prefix to row_start and row_stop does not work
# if it looks like row_start = %data_prefix%foo,
# row_stop = %data_prefix, because row_start > row_stop
filter = self._update_filter_row(filter)
row_start = self.data_prefix + row_start
else:
row_start = self.data_prefix + (row_start or "")
row_stop = self.data_prefix + (row_stop or "")
gen = super(MockHBaseTable, self).scan(row_start, row_stop,
row_prefix, columns,
filter, timestamp,
include_timestamp, batch_size,
scan_batching, limit,
sorted_columns)
data_prefix_len = len(self.data_prefix)
# Restore original row format
for row, data in gen:
yield (row[data_prefix_len:], data)
def row(self, row, *args, **kwargs):
row = self.data_prefix + row
return super(MockHBaseTable, self).row(row, *args, **kwargs)
def delete(self, row, *args, **kwargs):
row = self.data_prefix + row
return super(MockHBaseTable, self).delete(row, *args, **kwargs)
def _update_filter_row(self, filter):
if filter:
return "PrefixFilter(%s) AND %s" % (self.data_prefix, filter)
else:
return "PrefixFilter(%s)" % self.data_prefix

4
ceilometer/tests/storage/test_impl_hbase.py
View File

@ -26,7 +26,6 @@
import mock
from ceilometer.alarm.storage import impl_hbase as hbase_alarm
from ceilometer.storage.hbase import inmemory as hbase_inmemory
from ceilometer.storage import impl_hbase as hbase
from ceilometer.tests import base as test_base
from ceilometer.tests import db as tests_db
@ -37,9 +36,6 @@ class ConnectionTest(tests_db.TestBase,
@tests_db.run_with('hbase')
def test_hbase_connection(self):
conn = hbase.Connection(self.db_manager.url)
self.assertIsInstance(conn.conn_pool.connection(),
hbase_inmemory.MConnection)
class TestConn(object):
def __init__(self, host, port):

9
setup-test-env.sh
View File

@ -4,6 +4,10 @@ set -e
function clean_exit(){
local error_code="$?"
rm -rf ${MONGO_DATA}
if test -n "$CEILOMETER_TEST_HBASE_URL"
then
python tools/test_hbase_table_utils.py --clear
fi
kill $(jobs -p)
return $error_code
}
@ -28,6 +32,11 @@ done < ${MONGO_DATA}/out
# Read the fifo for ever otherwise mongod would block
cat ${MONGO_DATA}/out > /dev/null &
export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:${MONGO_PORT}/ceilometer"
if test -n "$CEILOMETER_TEST_HBASE_URL"
then
export CEILOMETER_TEST_HBASE_TABLE_PREFIX=$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom)
python tools/test_hbase_table_utils.py --upgrade
fi
# Yield execution to venv command
$*

38
tools/test_hbase_table_utils.py Normal file
View File

@ -0,0 +1,38 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import sys
from oslo.config import cfg
from ceilometer import storage
def main(argv):
cfg.CONF([], project='ceilometer')
if os.getenv("CEILOMETER_TEST_HBASE_URL"):
url = ("%s?table_prefix=%s" %
(os.getenv("CEILOMETER_TEST_HBASE_URL"),
os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", "test")))
conn = storage.get_connection(url, 'ceilometer.metering.storage')
alarm_conn = storage.get_connection(url, 'ceilometer.alarm.storage')
for arg in argv:
if arg == "--upgrade":
conn.upgrade()
alarm_conn.upgrade()
if arg == "--clear":
conn.clear()
alarm_conn.clear()
if __name__ == '__main__':
main(sys.argv[1:])