diff --git a/.zuul.yaml b/.zuul.yaml index f7ba1d78..5678d472 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -11,8 +11,10 @@ parent: oslo.cache-functional-tox-py38 vars: tox_environment: - OSLO_CACHE_BACKEND: etcd3gw - OSLO_CACHE_BACKEND_DAEMON: etcd + PIFPAF_DAEMON: etcd + # The next env var correspond to the functional test + # directory to pass to STESTR_TEST_PATH + OSLO_BACKEND: etcd3gw - project: templates: diff --git a/oslo_cache/_memcache_pool.py b/oslo_cache/_memcache_pool.py index 3dfece97..150469c1 100644 --- a/oslo_cache/_memcache_pool.py +++ b/oslo_cache/_memcache_pool.py @@ -18,9 +18,9 @@ import collections import contextlib import itertools +import queue import threading import time -import queue try: import eventlet diff --git a/oslo_cache/tests/functional/etcd3gw/__init__.py b/oslo_cache/tests/functional/etcd3gw/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oslo_cache/tests/functional/etcd3gw/test_cache_backend.py b/oslo_cache/tests/functional/etcd3gw/test_cache_backend.py new file mode 100644 index 00000000..4a6fa7c4 --- /dev/null +++ b/oslo_cache/tests/functional/etcd3gw/test_cache_backend.py @@ -0,0 +1,33 @@ +# Copyright 2020 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import fixture as config_fixture + +from oslo_cache.tests.functional import test_base + + +class TestEtcdCacheBackend(test_base.BaseTestCaseCacheBackend): + def setUp(self): + self.config_fixture = self.useFixture(config_fixture.Config()) + self.config_fixture.config( + group='cache', + backend='oslo_cache.etcd3gw', + enabled=True, + proxies=['oslo_cache.testing.CacheIsolatingProxy'], + backend_argument=['host:127.0.0.1', 'port:2379'] + ) + # NOTE(hberaud): super must be called after all to ensure that + # config fixture is properly initialized with value related to + # the current backend in use. + super(TestEtcdCacheBackend, self).setUp() diff --git a/oslo_cache/tests/functional/test_base.py b/oslo_cache/tests/functional/test_base.py new file mode 100644 index 00000000..17bf3c06 --- /dev/null +++ b/oslo_cache/tests/functional/test_base.py @@ -0,0 +1,201 @@ +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_utils import uuidutils +from oslotest import base + +from oslo_cache import core as cache + + +NO_VALUE = cache.NO_VALUE + + +class BaseTestCaseCacheBackend(base.BaseTestCase): + + def setUp(self): + super(BaseTestCaseCacheBackend, self).setUp() + if not hasattr(self, 'config_fixture'): + raise Exception("Functional tests base class can't be used " + "directly first you should define a test class " + "backend wrapper to init the related " + "config fixture") + self.region = cache.create_region() + cache.configure_cache_region(self.config_fixture.conf, self.region) + self.region_kwargs = cache.create_region( + function=cache.kwarg_function_key_generator + ) + cache.configure_cache_region( + self.config_fixture.conf, + self.region_kwargs + ) + + def test_backend_get_missing_data(self): + random_key = uuidutils.generate_uuid(dashed=False) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, self.region.get(random_key)) + + def test_backend_set_data(self): + random_key = uuidutils.generate_uuid(dashed=False) + self.region.set(random_key, "dummyValue") + self.assertEqual("dummyValue", self.region.get(random_key)) + + def test_backend_set_none_as_data(self): + random_key = uuidutils.generate_uuid(dashed=False) + self.region.set(random_key, None) + self.assertIsNone(self.region.get(random_key)) + + def test_backend_set_blank_as_data(self): + random_key = uuidutils.generate_uuid(dashed=False) + self.region.set(random_key, "") + self.assertEqual("", self.region.get(random_key)) + + def test_backend_set_same_key_multiple_times(self): + random_key = uuidutils.generate_uuid(dashed=False) + self.region.set(random_key, "dummyValue") + self.assertEqual("dummyValue", self.region.get(random_key)) + + dict_value = {'key1': 'value1'} + self.region.set(random_key, dict_value) + self.assertEqual(dict_value, self.region.get(random_key)) + + self.region.set(random_key, "dummyValue2") + self.assertEqual("dummyValue2", self.region.get(random_key)) + + def test_backend_multi_set_data(self): + random_key = uuidutils.generate_uuid(dashed=False) + random_key1 = uuidutils.generate_uuid(dashed=False) + random_key2 = uuidutils.generate_uuid(dashed=False) + random_key3 = uuidutils.generate_uuid(dashed=False) + mapping = {random_key1: 'dummyValue1', + random_key2: 'dummyValue2', + random_key3: 'dummyValue3'} + self.region.set_multi(mapping) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, self.region.get(random_key)) + self.assertFalse(self.region.get(random_key)) + self.assertEqual("dummyValue1", self.region.get(random_key1)) + self.assertEqual("dummyValue2", self.region.get(random_key2)) + self.assertEqual("dummyValue3", self.region.get(random_key3)) + + def test_backend_multi_get_data(self): + random_key = uuidutils.generate_uuid(dashed=False) + random_key1 = uuidutils.generate_uuid(dashed=False) + random_key2 = uuidutils.generate_uuid(dashed=False) + random_key3 = uuidutils.generate_uuid(dashed=False) + mapping = {random_key1: 'dummyValue1', + random_key2: '', + random_key3: 'dummyValue3'} + self.region.set_multi(mapping) + + keys = [random_key, random_key1, random_key2, random_key3] + results = self.region.get_multi(keys) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, results[0]) + self.assertEqual("dummyValue1", results[1]) + self.assertEqual("", results[2]) + self.assertEqual("dummyValue3", results[3]) + + def test_backend_multi_set_should_update_existing(self): + random_key = uuidutils.generate_uuid(dashed=False) + random_key1 = uuidutils.generate_uuid(dashed=False) + random_key2 = uuidutils.generate_uuid(dashed=False) + random_key3 = uuidutils.generate_uuid(dashed=False) + mapping = {random_key1: 'dummyValue1', + random_key2: 'dummyValue2', + random_key3: 'dummyValue3'} + self.region.set_multi(mapping) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, self.region.get(random_key)) + self.assertEqual("dummyValue1", self.region.get(random_key1)) + self.assertEqual("dummyValue2", self.region.get(random_key2)) + self.assertEqual("dummyValue3", self.region.get(random_key3)) + + mapping = {random_key1: 'dummyValue4', + random_key2: 'dummyValue5'} + self.region.set_multi(mapping) + self.assertEqual(NO_VALUE, self.region.get(random_key)) + self.assertEqual("dummyValue4", self.region.get(random_key1)) + self.assertEqual("dummyValue5", self.region.get(random_key2)) + self.assertEqual("dummyValue3", self.region.get(random_key3)) + + def test_backend_multi_set_get_with_blanks_none(self): + random_key = uuidutils.generate_uuid(dashed=False) + random_key1 = uuidutils.generate_uuid(dashed=False) + random_key2 = uuidutils.generate_uuid(dashed=False) + random_key3 = uuidutils.generate_uuid(dashed=False) + random_key4 = uuidutils.generate_uuid(dashed=False) + mapping = {random_key1: 'dummyValue1', + random_key2: None, + random_key3: '', + random_key4: 'dummyValue4'} + self.region.set_multi(mapping) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, self.region.get(random_key)) + self.assertEqual("dummyValue1", self.region.get(random_key1)) + self.assertIsNone(self.region.get(random_key2)) + self.assertEqual("", self.region.get(random_key3)) + self.assertEqual("dummyValue4", self.region.get(random_key4)) + + keys = [random_key, random_key1, random_key2, random_key3, random_key4] + results = self.region.get_multi(keys) + + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, results[0]) + self.assertEqual("dummyValue1", results[1]) + self.assertIsNone(results[2]) + self.assertEqual("", results[3]) + self.assertEqual("dummyValue4", results[4]) + + mapping = {random_key1: 'dummyValue5', + random_key2: 'dummyValue6'} + self.region.set_multi(mapping) + self.assertEqual(NO_VALUE, self.region.get(random_key)) + self.assertEqual("dummyValue5", self.region.get(random_key1)) + self.assertEqual("dummyValue6", self.region.get(random_key2)) + self.assertEqual("", self.region.get(random_key3)) + + def test_backend_delete_data(self): + random_key = uuidutils.generate_uuid(dashed=False) + self.region.set(random_key, "dummyValue") + self.assertEqual("dummyValue", self.region.get(random_key)) + + self.region.delete(random_key) + # should return NO_VALUE as key no longer exists in cache + self.assertEqual(NO_VALUE, self.region.get(random_key)) + + def test_backend_multi_delete_data(self): + random_key = uuidutils.generate_uuid(dashed=False) + random_key1 = uuidutils.generate_uuid(dashed=False) + random_key2 = uuidutils.generate_uuid(dashed=False) + random_key3 = uuidutils.generate_uuid(dashed=False) + mapping = {random_key1: 'dummyValue1', + random_key2: 'dummyValue2', + random_key3: 'dummyValue3'} + self.region.set_multi(mapping) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, self.region.get(random_key)) + self.assertEqual("dummyValue1", self.region.get(random_key1)) + self.assertEqual("dummyValue2", self.region.get(random_key2)) + self.assertEqual("dummyValue3", self.region.get(random_key3)) + self.assertEqual(NO_VALUE, self.region.get("InvalidKey")) + + keys = mapping.keys() + + self.region.delete_multi(keys) + + self.assertEqual(NO_VALUE, self.region.get("InvalidKey")) + # should return NO_VALUE as keys no longer exist in cache + self.assertEqual(NO_VALUE, self.region.get(random_key1)) + self.assertEqual(NO_VALUE, self.region.get(random_key2)) + self.assertEqual(NO_VALUE, self.region.get(random_key3)) diff --git a/oslo_cache/tests/functional/test_cache_backend.py b/oslo_cache/tests/functional/test_cache_backend.py deleted file mode 100644 index edc19919..00000000 --- a/oslo_cache/tests/functional/test_cache_backend.py +++ /dev/null @@ -1,282 +0,0 @@ -# Copyright 2014 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - -from dogpile.cache import region as dp_region -from oslo_utils import uuidutils - -from oslotest import base - -from oslo_cache import core as cache - - -NO_VALUE = cache.NO_VALUE - -TESTABLE_BACKENDS = [ - 'etcd3gw', -] - - -class TestCacheBackend(base.BaseTestCase): - arguments = {} - - def setUp(self): - super(TestCacheBackend, self).setUp() - self.backend = os.getenv('OSLO_CACHE_BACKEND') - if self.backend not in TESTABLE_BACKENDS: - raise Exception( - "Backend (%s) not supported in tests" % self.backend) - if self.backend == 'etcd3gw': - self.backend = "oslo_cache.etcd3gw" - # TODO(hberaud): Ideally functional tests should be similar - # to the usage examples given in the oslo.cache documentation [1]. - # Config [2] should be used to pass arguments to the backend - # rather than using direct usage like below. - # It could help us to made the tests interoperable between - # the different backends in use in tests. - # [1] https://docs.openstack.org/oslo.cache/latest/user/usage.html - # [2] https://docs.openstack.org/oslo.cache/latest/configuration/index.html # noqa - self.arguments = { - 'host': '127.0.0.1', - 'port': 2379, - } - - def test_typical_configuration(self): - - dp_region.make_region().configure( - self.backend, - arguments=self.arguments - ) - self.assertTrue(True) # reached here means no initialization error - - def test_backend_get_missing_data(self): - - region = dp_region.make_region().configure( - self.backend, - arguments=self.arguments - ) - - random_key = uuidutils.generate_uuid(dashed=False) - # should return NO_VALUE as key does not exist in cache - self.assertEqual(NO_VALUE, region.get(random_key)) - - def test_backend_set_data(self): - region = dp_region.make_region().configure( - self.backend, - arguments=self.arguments - ) - - random_key = uuidutils.generate_uuid(dashed=False) - region.set(random_key, "dummyValue") - self.assertEqual("dummyValue", region.get(random_key)) - - def test_backend_set_none_as_data(self): - - region = dp_region.make_region().configure( - self.backend, - arguments=self.arguments - ) - - random_key = uuidutils.generate_uuid(dashed=False) - region.set(random_key, None) - self.assertIsNone(region.get(random_key)) - - def test_backend_set_blank_as_data(self): - - region = dp_region.make_region().configure( - self.backend, - arguments=self.arguments - ) - - random_key = uuidutils.generate_uuid(dashed=False) - region.set(random_key, "") - self.assertEqual("", region.get(random_key)) - - def test_backend_set_same_key_multiple_times(self): - - region = dp_region.make_region().configure( - self.backend, - arguments=self.arguments - ) - - random_key = uuidutils.generate_uuid(dashed=False) - region.set(random_key, "dummyValue") - self.assertEqual("dummyValue", region.get(random_key)) - - dict_value = {'key1': 'value1'} - region.set(random_key, dict_value) - self.assertEqual(dict_value, region.get(random_key)) - - region.set(random_key, "dummyValue2") - self.assertEqual("dummyValue2", region.get(random_key)) - - def test_backend_multi_set_data(self): - - region = dp_region.make_region().configure( - self.backend, - arguments=self.arguments - ) - random_key = uuidutils.generate_uuid(dashed=False) - random_key1 = uuidutils.generate_uuid(dashed=False) - random_key2 = uuidutils.generate_uuid(dashed=False) - random_key3 = uuidutils.generate_uuid(dashed=False) - mapping = {random_key1: 'dummyValue1', - random_key2: 'dummyValue2', - random_key3: 'dummyValue3'} - region.set_multi(mapping) - # should return NO_VALUE as key does not exist in cache - self.assertEqual(NO_VALUE, region.get(random_key)) - self.assertFalse(region.get(random_key)) - self.assertEqual("dummyValue1", region.get(random_key1)) - self.assertEqual("dummyValue2", region.get(random_key2)) - self.assertEqual("dummyValue3", region.get(random_key3)) - - def test_backend_multi_get_data(self): - - region = dp_region.make_region().configure( - self.backend, - arguments=self.arguments - ) - random_key = uuidutils.generate_uuid(dashed=False) - random_key1 = uuidutils.generate_uuid(dashed=False) - random_key2 = uuidutils.generate_uuid(dashed=False) - random_key3 = uuidutils.generate_uuid(dashed=False) - mapping = {random_key1: 'dummyValue1', - random_key2: '', - random_key3: 'dummyValue3'} - region.set_multi(mapping) - - keys = [random_key, random_key1, random_key2, random_key3] - results = region.get_multi(keys) - # should return NO_VALUE as key does not exist in cache - self.assertEqual(NO_VALUE, results[0]) - self.assertEqual("dummyValue1", results[1]) - self.assertEqual("", results[2]) - self.assertEqual("dummyValue3", results[3]) - - def test_backend_multi_set_should_update_existing(self): - - region = dp_region.make_region().configure( - self.backend, - arguments=self.arguments - ) - random_key = uuidutils.generate_uuid(dashed=False) - random_key1 = uuidutils.generate_uuid(dashed=False) - random_key2 = uuidutils.generate_uuid(dashed=False) - random_key3 = uuidutils.generate_uuid(dashed=False) - mapping = {random_key1: 'dummyValue1', - random_key2: 'dummyValue2', - random_key3: 'dummyValue3'} - region.set_multi(mapping) - # should return NO_VALUE as key does not exist in cache - self.assertEqual(NO_VALUE, region.get(random_key)) - self.assertEqual("dummyValue1", region.get(random_key1)) - self.assertEqual("dummyValue2", region.get(random_key2)) - self.assertEqual("dummyValue3", region.get(random_key3)) - - mapping = {random_key1: 'dummyValue4', - random_key2: 'dummyValue5'} - region.set_multi(mapping) - self.assertEqual(NO_VALUE, region.get(random_key)) - self.assertEqual("dummyValue4", region.get(random_key1)) - self.assertEqual("dummyValue5", region.get(random_key2)) - self.assertEqual("dummyValue3", region.get(random_key3)) - - def test_backend_multi_set_get_with_blanks_none(self): - - region = dp_region.make_region().configure( - self.backend, - arguments=self.arguments - ) - random_key = uuidutils.generate_uuid(dashed=False) - random_key1 = uuidutils.generate_uuid(dashed=False) - random_key2 = uuidutils.generate_uuid(dashed=False) - random_key3 = uuidutils.generate_uuid(dashed=False) - random_key4 = uuidutils.generate_uuid(dashed=False) - mapping = {random_key1: 'dummyValue1', - random_key2: None, - random_key3: '', - random_key4: 'dummyValue4'} - region.set_multi(mapping) - # should return NO_VALUE as key does not exist in cache - self.assertEqual(NO_VALUE, region.get(random_key)) - self.assertEqual("dummyValue1", region.get(random_key1)) - self.assertIsNone(region.get(random_key2)) - self.assertEqual("", region.get(random_key3)) - self.assertEqual("dummyValue4", region.get(random_key4)) - - keys = [random_key, random_key1, random_key2, random_key3, random_key4] - results = region.get_multi(keys) - - # should return NO_VALUE as key does not exist in cache - self.assertEqual(NO_VALUE, results[0]) - self.assertEqual("dummyValue1", results[1]) - self.assertIsNone(results[2]) - self.assertEqual("", results[3]) - self.assertEqual("dummyValue4", results[4]) - - mapping = {random_key1: 'dummyValue5', - random_key2: 'dummyValue6'} - region.set_multi(mapping) - self.assertEqual(NO_VALUE, region.get(random_key)) - self.assertEqual("dummyValue5", region.get(random_key1)) - self.assertEqual("dummyValue6", region.get(random_key2)) - self.assertEqual("", region.get(random_key3)) - - def test_backend_delete_data(self): - - region = dp_region.make_region().configure( - self.backend, - arguments=self.arguments - ) - - random_key = uuidutils.generate_uuid(dashed=False) - region.set(random_key, "dummyValue") - self.assertEqual("dummyValue", region.get(random_key)) - - region.delete(random_key) - # should return NO_VALUE as key no longer exists in cache - self.assertEqual(NO_VALUE, region.get(random_key)) - - def test_backend_multi_delete_data(self): - - region = dp_region.make_region().configure( - self.backend, - arguments=self.arguments - ) - random_key = uuidutils.generate_uuid(dashed=False) - random_key1 = uuidutils.generate_uuid(dashed=False) - random_key2 = uuidutils.generate_uuid(dashed=False) - random_key3 = uuidutils.generate_uuid(dashed=False) - mapping = {random_key1: 'dummyValue1', - random_key2: 'dummyValue2', - random_key3: 'dummyValue3'} - region.set_multi(mapping) - # should return NO_VALUE as key does not exist in cache - self.assertEqual(NO_VALUE, region.get(random_key)) - self.assertEqual("dummyValue1", region.get(random_key1)) - self.assertEqual("dummyValue2", region.get(random_key2)) - self.assertEqual("dummyValue3", region.get(random_key3)) - self.assertEqual(NO_VALUE, region.get("InvalidKey")) - - keys = mapping.keys() - - region.delete_multi(keys) - - self.assertEqual(NO_VALUE, region.get("InvalidKey")) - # should return NO_VALUE as keys no longer exist in cache - self.assertEqual(NO_VALUE, region.get(random_key1)) - self.assertEqual(NO_VALUE, region.get(random_key2)) - self.assertEqual(NO_VALUE, region.get(random_key3)) diff --git a/playbooks/tests/functional/pre.yml b/playbooks/tests/functional/pre.yml index 3ec145b1..e3c928f4 100644 --- a/playbooks/tests/functional/pre.yml +++ b/playbooks/tests/functional/pre.yml @@ -1,6 +1,6 @@ - hosts: all vars: - oslo_cache_backend_daemon: "{{ tox_environment.OSLO_CACHE_BACKEND_DAEMON | default(tox_environment.OSLO_CACHE_BACKEND) }}" + oslo_cache_backend_daemon: "{{ tox_environment.PIFPAF_DAEMON }}" roles: - role: bindep bindep_profile: "tests-functional-{{ oslo_cache_backend_daemon }}" diff --git a/tox.ini b/tox.ini index 7d769ab3..20103d07 100644 --- a/tox.ini +++ b/tox.ini @@ -16,11 +16,10 @@ commands = [testenv:functional] setenv = - STESTR_TEST_PATH=./oslo_cache/tests/functional -passenv = OSLO_CACHE_BACKEND + STESTR_TEST_PATH=./oslo_cache/tests/functional/{env:OSLO_BACKEND} commands = find . -type f -name "*.pyc" -delete - pifpaf -e OSLO_CACHE_TEST run {env:OSLO_CACHE_BACKEND_DAEMON:{env:OSLO_CACHE_BACKEND}} -- stestr run --slowest + pifpaf -e OSLO_CACHE_TEST run {env:PIFPAF_DAEMON} -- stestr run --slowest [testenv:pep8] deps = {[testenv]deps}