chore: Track the up-and-coming oslo.cache module

While we wait for oslo.cache to land, I'd like to maintain a
verbatim copy under marconi.common so that we can start using
it immediately.

When oslo.cache is approved upstream, this copy of the code
will go away and we will add cache to marconi's
openstack-commong.conf

Change-Id: I608a2891ca06caec47ce18eda4edacd07bec40e9

marconi/common/cache/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2013 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

marconi/common/cache/_backends/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2013 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

marconi/common/cache/_backends/memcached.py

@@ -0,0 +1,104 @@
+# Copyright 2013 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import time
+
+import memcache
+from oslo.config import cfg
+
+from marconi.common.cache import backends
+
+_memcache_opts = [
+    cfg.ListOpt('memcached_servers',
+                default=['127.0.0.1:11211'],
+                help='Memcached servers or None for in process cache.'),
+]
+
+
+class MemcachedBackend(backends.BaseCache):
+
+    def __init__(self, conf, group, cache_namespace):
+        conf.register_opts(_memcache_opts, group=group)
+        super(MemcachedBackend, self).__init__(conf, group, cache_namespace)
+        self._client = None
+
+    @property
+    def _cache(self):
+        if not self._client:
+            self._client = memcache.Client(self.conf.memcached_servers)
+        return self._client
+
+    def _get_ttl(self, ttl):
+        """Correct ttl for memcached."""
+
+        if ttl > 2592000:
+            # NOTE(flaper87): If ttl is bigger than 30 days,
+            # it needs to be translated to timestamp.
+            #
+            # See http://code.google.com/p/memcached/wiki/FAQ
+            # "You can set expire times up to 30 days in the
+            # future. After that memcached interprets it as a
+            # date, and will expire the item after said date.
+            # This is a simple (but obscure) mechanic."
+            return ttl + int(time.time())
+        return ttl
+
+    def set(self, key, value, ttl=0):
+        key = self._prepare_key(key)
+        self._cache.set(key, value, self._get_ttl(ttl))
+
+    def unset(self, key):
+        self._cache.delete(self._prepare_key(key))
+
+    def get(self, key, default=None):
+        key = self._prepare_key(key)
+        value = self._cache.get(key)
+        return value is None and default or value
+
+    def get_many(self, keys, default=None):
+        new_keys = map(self._prepare_key, keys)
+        ret = self._cache.get_multi(new_keys)
+
+        m = dict(zip(new_keys, keys))
+        for cache_key, key in m.items():
+            yield (key, ret.get(cache_key, default))
+
+    def set_many(self, data, ttl=0):
+        safe_data = {}
+        for key, value in data.items():
+            key = self._prepare_key(key)
+            safe_data[key] = value
+        self._cache.set_multi(safe_data, self._get_ttl(ttl))
+
+    def unset_many(self, keys, version=None):
+        self._cache.delete_multi(map(self._prepare_key, keys))
+
+    def incr(self, key, delta=1):
+        key = self._prepare_key(key)
+
+        try:
+            if delta < 0:
+                #NOTE(flaper87): memcached doesn't support a negative delta
+                return self._cache.decr(key, -delta)
+
+            return self._cache.incr(key, delta)
+        except ValueError:
+            return None
+
+    def add(self, key, value, ttl=0):
+        key = self._prepare_key(key)
+        return self._cache.add(key, value, self._get_ttl(ttl))
+
+    def flush(self):
+        self._cache.flush_all()

marconi/common/cache/_backends/memory.py

@@ -0,0 +1,87 @@
+# Copyright 2013 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from marconi.common.cache import backends
+from marconi.openstack.common import lockutils
+from marconi.openstack.common import timeutils
+
+
+class MemoryBackend(backends.BaseCache):
+
+    def __init__(self, conf, group, cache_namespace):
+        super(MemoryBackend, self).__init__(conf, group, cache_namespace)
+        self._cache = {}
+        self._keys_expires = {}
+
+    def set(self, key, value, ttl=0):
+        key = self._prepare_key(key)
+        with lockutils.lock(key):
+            expires_at = 0
+            if ttl != 0:
+                expires_at = timeutils.utcnow_ts() + ttl
+
+            self._cache[key] = (expires_at, value)
+
+            if expires_at:
+                self._keys_expires.setdefault(expires_at, set()).add(key)
+
+            return True
+
+    def get(self, key, default=None):
+        key = self._prepare_key(key)
+        with lockutils.lock(key):
+            now = timeutils.utcnow_ts()
+
+            try:
+                timeout, value = self._cache[key]
+
+                if timeout and now >= timeout:
+                    del self._cache[key]
+                    return default
+
+                return value
+            except KeyError:
+                return default
+
+    def _purge_expired(self):
+        """Removes expired keys from the cache."""
+
+        now = timeutils.utcnow_ts()
+        for timeout in sorted(self._keys_expires.keys()):
+
+            # NOTE(flaper87): If timeout is greater
+            # than `now`, stop the iteration, remaining
+            # keys have not expired.
+            if now < timeout:
+                break
+
+            # NOTE(flaper87): Unset every key in
+            # this set from the cache if its timeout
+            # is equal to `timeout`. (They key might
+            # have been updated)
+            for subkey in self._keys_expires.pop(timeout):
+                if self._cache[subkey][0] == timeout:
+                    del self._cache[subkey]
+
+    def unset(self, key):
+        self._purge_expired()
+
+        # NOTE(flaper87): Delete the key. Using pop
+        # since it could have been deleted already
+        self._cache.pop(self._prepare_key(key), None)
+
+    def flush(self):
+        self._cache = {}
+        self._keys_expires = {}

marconi/common/cache/backends.py

@@ -0,0 +1,179 @@
+# Copyright 2013 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import abc
+
+
+class BaseCache(object):
+
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, conf, group, cache_namespace):
+        self.conf = conf[group]
+        self._cache_namespace = cache_namespace
+
+    @abc.abstractmethod
+    def set(self, key, value, ttl=0):
+        """Sets or updates a cache entry
+
+        :params key: Item key as string.
+        :params value: Value to assign to the key. This
+                       can be anything that is handled
+                       by the current backend.
+        :params ttl: Key's timeout in seconds.
+
+        :returns: True if the operation succeeds.
+        """
+
+    @abc.abstractmethod
+    def get(self, key, default=None):
+        """Gets one item from the cache
+
+        :params key: Key for the item to retrieve
+                     from the cache.
+        :params default: The default value to return.
+
+        :returns: `key`'s value in the cache if it exists,
+                  otherwise `default` should be returned.
+        """
+
+    @abc.abstractmethod
+    def unset(self, key):
+        """Removes an item from cache.
+
+        :params key: The key to remove.
+
+        :returns: The key value if there's one
+        """
+
+    def _prepare_key(self, key):
+        """Prepares the key
+
+        This method concatenates the cache_namespace
+        and the key so it can be used in the cache.
+
+        NOTE: All cache backends have to call it
+        explicitly where needed.
+
+        :param key: The key to be prefixed
+        """
+        if self._cache_namespace:
+            return ("%(prefix)s-%(key)s" %
+                    {'prefix': self._cache_namespace, 'key': key})
+        return key
+
+    def add(self, key, value, ttl=0):
+        """Sets the value for a key if it doesn't exist
+
+        :params key: Key to create as string.
+        :params value: Value to assign to the key. This
+                       can be anything that is handled
+                       by current backend.
+        :params ttl: Key's timeout in seconds.
+
+        :returns: False if the key exists, otherwise,
+                  `set`'s result will be returned.
+        """
+
+        if self.get(key) is not None:
+            return False
+        return self.set(key, value, ttl)
+
+    def get_many(self, keys, default=None):
+        """Gets key's value from cache
+
+        :params keys: List of keys to retrieve.
+        :params default: The default value to return
+                         for each key that is not in
+                         the cache.
+
+        :returns: A  generator of (key, value)
+        """
+        for k in keys:
+            val = self.get(k, default=default)
+            if val is not None:
+                yield (k, val)
+
+    def has_key(self, key):
+        """Verifies that a key exists.
+
+        :params key: The key to verify.
+
+        :returns: True if the key exists, otherwise
+                  False.
+        """
+        return self.get(key) is not None
+
+    def set_many(self, data, ttl=0):
+        """Puts several items into the cache at once
+
+        Depending on the backend, this operation may or may
+        not be efficient. The default implementation calls
+        set for each (key, value) pair passed, other backends
+        support set_many operations as part of their protocols.
+
+        :params data: A dictionary like {key: val} to store
+                      in the cache.
+        :params ttl: Key's timeout in seconds.
+        """
+        for key, value in data.items():
+            self.set(key, value, ttl=ttl)
+
+    def unset_many(self, keys):
+        """Removes several keys from the cache at once
+
+        :params keys: List of keys to retrieve.
+        """
+        for key in keys:
+            self.unset(key)
+
+    def incr(self, key, delta=1):
+        """Increments the value for a key
+
+        NOTE: This method is not synchronized because
+        get and set are.
+
+        :params key: The key for the value to be incremented
+        :params delta: Number of units by which to increment
+                       the value. Pass a negative number to
+                       decrement the value.
+
+        :returns: The new value
+        """
+        value = self.get(key)
+        if value is None:
+            return None
+        new_value = value + delta
+        self.set(key, new_value)
+        return new_value
+
+    def append(self, key, tail):
+        """Appends `value` to `key`'s value.
+
+        :params key: The key of the value to which
+                     `tail` should be appended.
+        :params tail: The value to append to the
+                      original.
+
+        :returns: The new value
+        """
+        value = self.get(key)
+        if value is None:
+            return None
+        new_value = value + tail
+        self.set(key, new_value)
+        return new_value
+
+    def flush(self):
+        """Flushes all items from the cache."""

marconi/common/cache/cache.py

@@ -0,0 +1,60 @@
+# Copyright 2013 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Cache library.
+
+Supported configuration options:
+
+`cache_backend`: Name of the cache backend to use.
+"""
+
+from oslo.config import cfg
+from stevedore import driver
+
+_cache_options = [
+    cfg.StrOpt('cache_backend',
+               default='memory',
+               help='The cache driver to use, default value is `memory`.'),
+    cfg.StrOpt('cache_prefix',
+               default=None,
+               help='Prefix to use in every cache key'),
+]
+
+
+def get_cache(conf):
+    """Loads the cache backend
+
+    This function loads the cache backend
+    specified in the given configuration.
+
+    :param conf: Configuration instance to use
+    """
+
+    # NOTE(flaper87): oslo.config checks if options
+    # exist before registering them. The code bellow
+    # should be safe.
+    cache_group = cfg.OptGroup(name='oslo_cache',
+                               title='Cache options')
+
+    conf.register_group(cache_group)
+    conf.register_opts(_cache_options, group=cache_group)
+
+    kwargs = dict(cache_namespace=conf.oslo_cache.cache_prefix)
+
+    backend = conf.oslo_cache.cache_backend
+    mgr = driver.DriverManager('marconi.common.cache.backends', backend,
+                               invoke_on_load=True,
+                               invoke_args=[conf, cache_group.name],
+                               invoke_kwds=kwargs)
+    return mgr.driver

marconi/tests/common/test_cache.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2013 Rackspace, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import traceback
+
+from marconi.tests import util as testing
+
+
+class TestCache(testing.TestBase):
+
+    def test_import(self):
+        try:
+            from marconi.common.cache._backends import memcached
+            from marconi.common.cache._backends import memory
+            from marconi.common.cache import backends
+            from marconi.common.cache import cache
+
+        except ImportError as ex:
+            self.fail(traceback.format_exc(ex))
+
+        # Avoid pyflakes warnings
+        cache = cache
+        backends = backends
+        memory = memory
+        memcached = memcached

requirements.txt

@@ -8,6 +8,7 @@ iso8601>=0.1.4
 msgpack-python
 pymongo
 python-keystoneclient
+python-memcached
 simplejson
 WebOb
 stevedore>=0.9

setup.cfg

@@ -37,6 +37,10 @@ marconi.storage =
 marconi.transport =
     wsgi = marconi.transport.wsgi.driver:Driver
 
+marconi.common.cache.backends =
+    memory = marconi.common.cache._backends.memory:MemoryBackend
+    memcached = marconi.common.cache._backends.memcached:MemcachedBackend
+
 [nosetests]
 where=marconi/tests
 verbosity=2