etcd3 driver for oslo.cache

Support for oslo.cache to use the etcd3 grpc gateway based
HTTP API (/v3alpha) using the python-etd3gw library

Change-Id: I41ed9f6ea72641dc1db5fec45920dc41de5088bf

oslo_cache/backends/etcd3gw.py

@@ -0,0 +1,75 @@
+# Copyright 2015 Mirantis Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""dogpile.cache backend that uses etcd 3.x for storage"""
+
+from __future__ import absolute_import
+import ast
+from dogpile.cache import api
+import etcd3gw
+
+from oslo_cache import core
+
+__all__ = [
+    'Etcd3gwCacheBackend'
+]
+
+_NO_VALUE = core.NO_VALUE
+
+
+class Etcd3gwCacheBackend(api.CacheBackend):
+    #: Default socket/lock/member/leader timeout used when none is provided.
+    DEFAULT_TIMEOUT = 30
+
+    #: Default hostname used when none is provided.
+    DEFAULT_HOST = "localhost"
+
+    #: Default port used if none provided (4001 or 2379 are the common ones).
+    DEFAULT_PORT = 2379
+
+    def __init__(self, arguments):
+        self.host = arguments.get('host', self.DEFAULT_HOST)
+        self.port = arguments.get('port', self.DEFAULT_PORT)
+        self.timeout = int(arguments.get('timeout', self.DEFAULT_TIMEOUT))
+        self._client = etcd3gw.client(host=self.host,
+                                      port=self.port,
+                                      timeout=self.timeout)
+
+    def get(self, key):
+        values = self._client.get(key, False)
+        if not values:
+            return core.NO_VALUE
+        (value, metadata) = ast.literal_eval(values[0])
+        return api.CachedValue(value, metadata)
+
+    def get_multi(self, keys):
+        """Retrieves the value for a list of keys."""
+        return [self.get(key) for key in keys]
+
+    def set(self, key, value):
+        self.set_multi({key: value})
+
+    def set_multi(self, mapping):
+        lease = None
+        if self.timeout:
+            lease = self._client.lease(ttl=self.timeout)
+        for key, value in mapping.items():
+            self._client.put(key, value, lease)
+
+    def delete(self, key):
+        self._client.delete(key)
+
+    def delete_multi(self, keys):
+        for key in keys:
+            self._client.delete(key)

oslo_cache/tests/functional/test_cache_backend_etcd3gw.py

@@ -0,0 +1,269 @@
+# Copyright 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from dogpile.cache import region as dp_region
+from oslo_utils import uuidutils
+import urllib3
+
+from oslo_cache import core
+from oslo_cache.tests import test_cache
+
+
+NO_VALUE = core.NO_VALUE
+
+
+class Etcd3gwCache(test_cache.BaseTestCase):
+    arguments = {
+        'host': '127.0.0.1',
+        'port': 2379,
+    }
+
+    def setUp(self):
+        test_cache.BaseTestCase.setUp(self)
+        try:
+            urllib3.PoolManager().request(
+                'GET',
+                '%s:%d' % (self.arguments['host'], self.arguments['port'])
+            )
+            return True
+        except urllib3.exceptions.HTTPError:
+            self.skipTest("skipping this test")
+
+    def test_typical_configuration(self):
+
+        dp_region.make_region().configure(
+            'oslo_cache.etcd3gw',
+            arguments=self.arguments
+        )
+        self.assertTrue(True)  # reached here means no initialization error
+
+    def test_backend_get_missing_data(self):
+
+        region = dp_region.make_region().configure(
+            'oslo_cache.etcd3gw',
+            arguments=self.arguments
+        )
+
+        random_key = uuidutils.generate_uuid(dashed=False)
+        # should return NO_VALUE as key does not exist in cache
+        self.assertEqual(NO_VALUE, region.get(random_key))
+
+    def test_backend_set_data(self):
+        region = dp_region.make_region().configure(
+            'oslo_cache.etcd3gw',
+            arguments=self.arguments
+        )
+
+        random_key = uuidutils.generate_uuid(dashed=False)
+        region.set(random_key, "dummyValue")
+        self.assertEqual("dummyValue", region.get(random_key))
+
+    def test_backend_set_none_as_data(self):
+
+        region = dp_region.make_region().configure(
+            'oslo_cache.etcd3gw',
+            arguments=self.arguments
+        )
+
+        random_key = uuidutils.generate_uuid(dashed=False)
+        region.set(random_key, None)
+        self.assertIsNone(region.get(random_key))
+
+    def test_backend_set_blank_as_data(self):
+
+        region = dp_region.make_region().configure(
+            'oslo_cache.etcd3gw',
+            arguments=self.arguments
+        )
+
+        random_key = uuidutils.generate_uuid(dashed=False)
+        region.set(random_key, "")
+        self.assertEqual("", region.get(random_key))
+
+    def test_backend_set_same_key_multiple_times(self):
+
+        region = dp_region.make_region().configure(
+            'oslo_cache.etcd3gw',
+            arguments=self.arguments
+        )
+
+        random_key = uuidutils.generate_uuid(dashed=False)
+        region.set(random_key, "dummyValue")
+        self.assertEqual("dummyValue", region.get(random_key))
+
+        dict_value = {'key1': 'value1'}
+        region.set(random_key, dict_value)
+        self.assertEqual(dict_value, region.get(random_key))
+
+        region.set(random_key, "dummyValue2")
+        self.assertEqual("dummyValue2", region.get(random_key))
+
+    def test_backend_multi_set_data(self):
+
+        region = dp_region.make_region().configure(
+            'oslo_cache.etcd3gw',
+            arguments=self.arguments
+        )
+        random_key = uuidutils.generate_uuid(dashed=False)
+        random_key1 = uuidutils.generate_uuid(dashed=False)
+        random_key2 = uuidutils.generate_uuid(dashed=False)
+        random_key3 = uuidutils.generate_uuid(dashed=False)
+        mapping = {random_key1: 'dummyValue1',
+                   random_key2: 'dummyValue2',
+                   random_key3: 'dummyValue3'}
+        region.set_multi(mapping)
+        # should return NO_VALUE as key does not exist in cache
+        self.assertEqual(NO_VALUE, region.get(random_key))
+        self.assertFalse(region.get(random_key))
+        self.assertEqual("dummyValue1", region.get(random_key1))
+        self.assertEqual("dummyValue2", region.get(random_key2))
+        self.assertEqual("dummyValue3", region.get(random_key3))
+
+    def test_backend_multi_get_data(self):
+
+        region = dp_region.make_region().configure(
+            'oslo_cache.etcd3gw',
+            arguments=self.arguments
+        )
+        random_key = uuidutils.generate_uuid(dashed=False)
+        random_key1 = uuidutils.generate_uuid(dashed=False)
+        random_key2 = uuidutils.generate_uuid(dashed=False)
+        random_key3 = uuidutils.generate_uuid(dashed=False)
+        mapping = {random_key1: 'dummyValue1',
+                   random_key2: '',
+                   random_key3: 'dummyValue3'}
+        region.set_multi(mapping)
+
+        keys = [random_key, random_key1, random_key2, random_key3]
+        results = region.get_multi(keys)
+        # should return NO_VALUE as key does not exist in cache
+        self.assertEqual(NO_VALUE, results[0])
+        self.assertEqual("dummyValue1", results[1])
+        self.assertEqual("", results[2])
+        self.assertEqual("dummyValue3", results[3])
+
+    def test_backend_multi_set_should_update_existing(self):
+
+        region = dp_region.make_region().configure(
+            'oslo_cache.etcd3gw',
+            arguments=self.arguments
+        )
+        random_key = uuidutils.generate_uuid(dashed=False)
+        random_key1 = uuidutils.generate_uuid(dashed=False)
+        random_key2 = uuidutils.generate_uuid(dashed=False)
+        random_key3 = uuidutils.generate_uuid(dashed=False)
+        mapping = {random_key1: 'dummyValue1',
+                   random_key2: 'dummyValue2',
+                   random_key3: 'dummyValue3'}
+        region.set_multi(mapping)
+        # should return NO_VALUE as key does not exist in cache
+        self.assertEqual(NO_VALUE, region.get(random_key))
+        self.assertEqual("dummyValue1", region.get(random_key1))
+        self.assertEqual("dummyValue2", region.get(random_key2))
+        self.assertEqual("dummyValue3", region.get(random_key3))
+
+        mapping = {random_key1: 'dummyValue4',
+                   random_key2: 'dummyValue5'}
+        region.set_multi(mapping)
+        self.assertEqual(NO_VALUE, region.get(random_key))
+        self.assertEqual("dummyValue4", region.get(random_key1))
+        self.assertEqual("dummyValue5", region.get(random_key2))
+        self.assertEqual("dummyValue3", region.get(random_key3))
+
+    def test_backend_multi_set_get_with_blanks_none(self):
+
+        region = dp_region.make_region().configure(
+            'oslo_cache.etcd3gw',
+            arguments=self.arguments
+        )
+        random_key = uuidutils.generate_uuid(dashed=False)
+        random_key1 = uuidutils.generate_uuid(dashed=False)
+        random_key2 = uuidutils.generate_uuid(dashed=False)
+        random_key3 = uuidutils.generate_uuid(dashed=False)
+        random_key4 = uuidutils.generate_uuid(dashed=False)
+        mapping = {random_key1: 'dummyValue1',
+                   random_key2: None,
+                   random_key3: '',
+                   random_key4: 'dummyValue4'}
+        region.set_multi(mapping)
+        # should return NO_VALUE as key does not exist in cache
+        self.assertEqual(NO_VALUE, region.get(random_key))
+        self.assertEqual("dummyValue1", region.get(random_key1))
+        self.assertIsNone(region.get(random_key2))
+        self.assertEqual("", region.get(random_key3))
+        self.assertEqual("dummyValue4", region.get(random_key4))
+
+        keys = [random_key, random_key1, random_key2, random_key3, random_key4]
+        results = region.get_multi(keys)
+
+        # should return NO_VALUE as key does not exist in cache
+        self.assertEqual(NO_VALUE, results[0])
+        self.assertEqual("dummyValue1", results[1])
+        self.assertIsNone(results[2])
+        self.assertEqual("", results[3])
+        self.assertEqual("dummyValue4", results[4])
+
+        mapping = {random_key1: 'dummyValue5',
+                   random_key2: 'dummyValue6'}
+        region.set_multi(mapping)
+        self.assertEqual(NO_VALUE, region.get(random_key))
+        self.assertEqual("dummyValue5", region.get(random_key1))
+        self.assertEqual("dummyValue6", region.get(random_key2))
+        self.assertEqual("", region.get(random_key3))
+
+    def test_backend_delete_data(self):
+
+        region = dp_region.make_region().configure(
+            'oslo_cache.etcd3gw',
+            arguments=self.arguments
+        )
+
+        random_key = uuidutils.generate_uuid(dashed=False)
+        region.set(random_key, "dummyValue")
+        self.assertEqual("dummyValue", region.get(random_key))
+
+        region.delete(random_key)
+        # should return NO_VALUE as key no longer exists in cache
+        self.assertEqual(NO_VALUE, region.get(random_key))
+
+    def test_backend_multi_delete_data(self):
+
+        region = dp_region.make_region().configure(
+            'oslo_cache.etcd3gw',
+            arguments=self.arguments
+        )
+        random_key = uuidutils.generate_uuid(dashed=False)
+        random_key1 = uuidutils.generate_uuid(dashed=False)
+        random_key2 = uuidutils.generate_uuid(dashed=False)
+        random_key3 = uuidutils.generate_uuid(dashed=False)
+        mapping = {random_key1: 'dummyValue1',
+                   random_key2: 'dummyValue2',
+                   random_key3: 'dummyValue3'}
+        region.set_multi(mapping)
+        # should return NO_VALUE as key does not exist in cache
+        self.assertEqual(NO_VALUE, region.get(random_key))
+        self.assertEqual("dummyValue1", region.get(random_key1))
+        self.assertEqual("dummyValue2", region.get(random_key2))
+        self.assertEqual("dummyValue3", region.get(random_key3))
+        self.assertEqual(NO_VALUE, region.get("InvalidKey"))
+
+        keys = mapping.keys()
+
+        region.delete_multi(keys)
+
+        self.assertEqual(NO_VALUE, region.get("InvalidKey"))
+        # should return NO_VALUE as keys no longer exist in cache
+        self.assertEqual(NO_VALUE, region.get(random_key1))
+        self.assertEqual(NO_VALUE, region.get(random_key2))
+        self.assertEqual(NO_VALUE, region.get(random_key3))

releasenotes/notes/etcd3gw_driver-8ba4511ae9553a91.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Added a new etcd3gw driver that uses the etcd 3.x grpc-gateway
+    HTTP "/v3alpha" API to cache key/value pairs.

setup.cfg

@@ -30,12 +30,15 @@ dogpile.cache =
     oslo_cache.mongo = oslo_cache.backends.mongo:MongoCacheBackend
     oslo_cache.memcache_pool = oslo_cache.backends.memcache_pool:PooledMemcachedBackend
     oslo_cache.dict = oslo_cache.backends.dictionary:DictCacheBackend
+    oslo_cache.etcd3gw = oslo_cache.backends.etcd3gw:Etcd3gwCacheBackend
 
 [extras]
 dogpile =
   python-memcached>=1.56 # PSF
 mongo =
   pymongo!=3.1,>=3.0.2 # Apache-2.0
+etcd3gw =
+  etcd3gw>=0.1.0 # Apache-2.0
 
 [pbr]
 warnerrors = true

test-requirements.txt

@@ -5,5 +5,6 @@ hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
 mock>=2.0 # BSD
 oslotest>=1.10.0 # Apache-2.0
 oslosphinx>=4.7.0 # Apache-2.0
+pifpaf>=0.10.0 # Apache-2.0
 sphinx!=1.6.1,>=1.5.1 # BSD
 reno>=1.8.0 # Apache-2.0

tools/setup-etcd-env.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -eux
+if [ -z "$(which etcd)" ]; then
+    ETCD_VERSION=3.1.3
+    case `uname -s` in
+        Darwin)
+            OS=darwin
+            SUFFIX=zip
+            ;;
+        Linux)
+            OS=linux
+            SUFFIX=tar.gz
+            ;;
+        *)
+            echo "Unsupported OS"
+            exit 1
+    esac
+    case `uname -m` in
+         x86_64)
+             MACHINE=amd64
+             ;;
+         *)
+            echo "Unsupported machine"
+            exit 1
+    esac
+    TARBALL_NAME=etcd-v${ETCD_VERSION}-$OS-$MACHINE
+    test ! -d "$TARBALL_NAME" && curl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/${TARBALL_NAME}.${SUFFIX} | tar xz
+    export PATH=$PATH:$TARBALL_NAME
+fi
+
+$*

tox.ini

@@ -10,11 +10,22 @@ setenv =
 install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
 deps = .[dogpile]
        .[mongo]
+       .[etcd3gw]
        -r{toxinidir}/test-requirements.txt
 commands =
     find . -type f -name "*.pyc" -delete
     python setup.py testr --slowest --testr-args='{posargs}'
 
+[testenv:py27-functional-etcd3gw]
+commands =
+    find . -type f -name "*.pyc" -delete
+    {toxinidir}/tools/setup-etcd-env.sh pifpaf -e OSLO_CACHE_TEST run etcd --  python setup.py testr --slowest --testr-args='functional.*'
+
+[testenv:py35-functional-etcd3gw]
+commands =
+    find . -type f -name "*.pyc" -delete
+    {toxinidir}/tools/setup-etcd-env.sh pifpaf -e OSLO_CACHE_TEST run etcd --  python setup.py testr --slowest --testr-args='functional.*'
+
 [testenv:pep8]
 commands = flake8