From 9a8cb2228efbfaf22149fa040a6dc96fa3937bab Mon Sep 17 00:00:00 2001 From: Petr Malik Date: Wed, 17 Aug 2016 15:58:41 -0400 Subject: [PATCH] Add Couchbase helper client methods Implement the helper client methods for data operations in scenario tests. Fixed two other test-related issues discovered while testing Couchbase: * The code that builds helper user json definition generates invalid payload when no database is specified (i.e. it uses None as database name which is wrong). * Another issue is that the flavor for cluster grow tests is retrieved from the instance info which would not be initialized when running standalone (i.e. --group=cluster) cluster tests. Change-Id: Iab0c9b4b98f9c428f2ea7461f5d5834461b66fa4 --- .../scenario/helpers/couchbase_helper.py | 100 ++++++++++++++++++ trove/tests/scenario/runners/test_runners.py | 6 +- trove/tests/unittests/common/test_utils.py | 48 +++++++++ trove/tests/util/utils.py | 57 ++++++++++ 4 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 trove/tests/scenario/helpers/couchbase_helper.py create mode 100644 trove/tests/util/utils.py diff --git a/trove/tests/scenario/helpers/couchbase_helper.py b/trove/tests/scenario/helpers/couchbase_helper.py new file mode 100644 index 0000000000..9b4c91eb70 --- /dev/null +++ b/trove/tests/scenario/helpers/couchbase_helper.py @@ -0,0 +1,100 @@ +# Copyright 2016 Tesora Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from couchbase.bucket import Bucket +from couchbase import exceptions as cb_except + +from trove.tests.scenario.helpers.test_helper import TestHelper +from trove.tests.scenario.runners.test_runners import TestRunner +from trove.tests.util import utils + + +class CouchbaseHelper(TestHelper): + + def __init__(self, expected_override_name, report): + super(CouchbaseHelper, self).__init__(expected_override_name, report) + + self._data_cache = dict() + + def get_helper_credentials(self): + return {'name': 'lite', 'password': 'litepass'} + + def create_client(self, host, *args, **kwargs): + user = self.get_helper_credentials() + return self._create_test_bucket(host, user['name'], user['password']) + + def _create_test_bucket(self, host, bucket_name, password): + return Bucket('couchbase://%s/%s' % (host, bucket_name), + password=password) + + # Add data overrides + def add_actual_data(self, data_label, data_start, data_size, host, + *args, **kwargs): + client = self.get_client(host, *args, **kwargs) + if not self._key_exists(client, data_label, *args, **kwargs): + self._set_data_point(client, data_label, + self._get_dataset(data_start, data_size)) + + @utils.retry((cb_except.TemporaryFailError, cb_except.BusyError)) + def _key_exists(self, client, key, *args, **kwargs): + return client.get(key, quiet=True).success + + @utils.retry((cb_except.TemporaryFailError, cb_except.BusyError)) + def _set_data_point(self, client, key, value, *args, **kwargs): + client.insert(key, value) + + def _get_dataset(self, data_start, data_size): + cache_key = str(data_size) + if cache_key in self._data_cache: + return self._data_cache.get(cache_key) + + data = range(data_start, data_start + data_size) + self._data_cache[cache_key] = data + return data + + # Remove data overrides + def remove_actual_data(self, data_label, data_start, data_size, host, + *args, **kwargs): + client = self.get_client(host, *args, **kwargs) + if self._key_exists(client, data_label, *args, **kwargs): + self._remove_data_point(client, data_label, *args, **kwargs) + + @utils.retry((cb_except.TemporaryFailError, cb_except.BusyError)) + def _remove_data_point(self, client, key, *args, **kwargs): + client.remove(key) + + # Verify data overrides + def verify_actual_data(self, data_label, data_start, data_size, host, + *args, **kwargs): + client = self.get_client(host, *args, **kwargs) + expected_value = self._get_dataset(data_start, data_size) + self._verify_data_point(client, data_label, expected_value) + + def _verify_data_point(self, client, key, expected_value, *args, **kwargs): + value = self._get_data_point(client, key, *args, **kwargs) + TestRunner.assert_equal(expected_value, value, + "Unexpected value '%s' returned from " + "Couchbase key '%s'" % (value, key)) + + @utils.retry((cb_except.TemporaryFailError, cb_except.BusyError)) + def _get_data_point(self, client, key, *args, **kwargs): + return client.get(key).value + + def ping(self, host, *args, **kwargs): + try: + self.create_client(host, *args, **kwargs) + return True + except Exception: + return False diff --git a/trove/tests/scenario/runners/test_runners.py b/trove/tests/scenario/runners/test_runners.py index 3699b087be..a1f2fa1b05 100644 --- a/trove/tests/scenario/runners/test_runners.py +++ b/trove/tests/scenario/runners/test_runners.py @@ -846,8 +846,12 @@ class TestRunner(object): username = creds.get('name') if username: password = creds.get('password', '') + databases = [] + if database_def: + databases.append(database_def) + return {'name': username, 'password': password, - 'databases': [{'name': database}]} + 'databases': databases} return None credentials = self.test_helper.get_helper_credentials() diff --git a/trove/tests/unittests/common/test_utils.py b/trove/tests/unittests/common/test_utils.py index 5980c859c3..c469a5f0e3 100644 --- a/trove/tests/unittests/common/test_utils.py +++ b/trove/tests/unittests/common/test_utils.py @@ -15,11 +15,13 @@ # from mock import Mock +from mock import patch from testtools import ExpectedException from trove.common import exception from trove.common import utils from trove.tests.unittests import trove_testtools +from trove.tests.util import utils as test_utils class TestUtils(trove_testtools.TestCase): @@ -123,3 +125,49 @@ class TestUtils(trove_testtools.TestCase): def test_to_mb_zero(self): result = utils.to_mb(0) self.assertEqual(0.0, result) + + @patch('trove.common.utils.LOG') + def test_retry_decorator(self, _): + + class TestEx1(Exception): + pass + + class TestEx2(Exception): + pass + + class TestEx3(Exception): + pass + + class TestExecutor(object): + + def _test_foo(self, arg): + return arg + + @test_utils.retry(TestEx1, retries=5, delay_fun=lambda n: 0.2) + def test_foo_1(self, arg): + return self._test_foo(arg) + + @test_utils.retry((TestEx1, TestEx2), delay_fun=lambda n: 0.2) + def test_foo_2(self, arg): + return self._test_foo(arg) + + def assert_retry(fun, side_effect, exp_call_num, exp_exception): + with patch.object(te, '_test_foo', side_effect=side_effect) as f: + mock_arg = Mock() + if exp_exception: + self.assertRaises(exp_exception, fun, mock_arg) + else: + fun(mock_arg) + + f.assert_called_with(mock_arg) + self.assertEqual(exp_call_num, f.call_count) + + te = TestExecutor() + assert_retry(te.test_foo_1, [TestEx1, None], 2, None) + assert_retry(te.test_foo_1, TestEx3, 1, TestEx3) + assert_retry(te.test_foo_1, TestEx1, 5, TestEx1) + assert_retry(te.test_foo_1, [TestEx1, TestEx3], 2, TestEx3) + assert_retry(te.test_foo_2, [TestEx1, TestEx2, None], 3, None) + assert_retry(te.test_foo_2, TestEx3, 1, TestEx3) + assert_retry(te.test_foo_2, TestEx2, 3, TestEx2) + assert_retry(te.test_foo_2, [TestEx1, TestEx3, TestEx2], 2, TestEx3) diff --git a/trove/tests/util/utils.py b/trove/tests/util/utils.py new file mode 100644 index 0000000000..ade0ff95d0 --- /dev/null +++ b/trove/tests/util/utils.py @@ -0,0 +1,57 @@ +# Copyright 2016 Tesora Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import time + +from functools import wraps +from oslo_log import log as logging + +from trove.common.i18n import _ + + +LOG = logging.getLogger(__name__) + + +def retry(expected_exception_cls, retries=3, delay_fun=lambda n: 3 * n): + """Retry decorator. + Executes the decorated function N times with a variable timeout + on a given exception(s). + + :param expected_exception_cls: Handled exception classes. + :type expected_exception_cls: class or tuple of classes + + :param delay_fun: The time delay in sec as a function of the + number of attempts (n) already executed. + :type delay_fun: callable + """ + def retry_deco(f): + @wraps(f) + def wrapper(*args, **kwargs): + remaining_attempts = retries + while remaining_attempts > 1: + try: + return f(*args, **kwargs) + except expected_exception_cls: + remaining_attempts -= 1 + delay = delay_fun(retries - remaining_attempts) + LOG.exception(_( + "Retrying in %(delay)d seconds " + "(remaining attempts: %(remaining)d)...") % + {'delay': delay, 'remaining': remaining_attempts}) + time.sleep(delay) + return f(*args, **kwargs) + return wrapper + return retry_deco