openstack/keystonemiddleware
Code Issues Proposed changes

Revert "Disable memory caching of tokens"

Browse Source 
This reverts commit f27d7f776e8556d976f75d07c99373455106de52.

This change broke the gate due to causing timeouts. The
functionality needs to go through the normal deprecation before
being removed.

Change-Id: I4ab07f6e2bd5bd084bd16707126728929a4ba0f7
This commit is contained in:
Brant Knudson 2016-01-22 09:50:06 -06:00
parent f57a839909
commit 70a9754ae6
8 changed files with 131 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
keystonemiddleware/auth_token/__init__.py
View File

@ -802,6 +802,22 @@ class AuthProtocol(BaseAuthProtocol):
else:
return [token]
def _cache_get_hashes(self, token_hashes):
"""Check if the token is cached already.
Functions takes a list of hashes that might be in the cache and matches
the first one that is present. If nothing is found in the cache it
returns None.
:returns: token data if found else None.
"""
for token in token_hashes:
cached = self._token_cache.get(token)
if cached:
return cached
def fetch_token(self, token):
"""Retrieve a token from either a PKI bundle or the identity server.
@ -814,7 +830,7 @@ class AuthProtocol(BaseAuthProtocol):
try:
token_hashes = self._token_hashes(token)
cached = self._token_cache.get_first(*token_hashes)
cached = self._cache_get_hashes(token_hashes)
if cached:
data = cached
@ -1077,18 +1093,12 @@ class AuthProtocol(BaseAuthProtocol):
requested_auth_version=auth_version)
def _token_cache_factory(self):
memcached_servers = self._conf_get('memcached_servers')
env_cache_name = self._conf_get('cache')
if not (memcached_servers or env_cache_name):
return _cache.NoOpCache()
security_strategy = self._conf_get('memcache_security_strategy')
cache_kwargs = dict(
cache_time=int(self._conf_get('token_cache_time')),
env_cache_name=env_cache_name,
memcached_servers=memcached_servers,
env_cache_name=self._conf_get('cache'),
memcached_servers=self._conf_get('memcached_servers'),
use_advanced_pool=self._conf_get('memcache_use_advanced_pool'),
dead_retry=self._conf_get('memcache_pool_dead_retry'),
maxsize=self._conf_get('memcache_pool_maxsize'),

81
keystonemiddleware/auth_token/_cache.py
View File

@ -10,7 +10,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import abc
import contextlib
import hashlib
@ -21,22 +20,7 @@ from keystonemiddleware.auth_token import _exceptions as exc
from keystonemiddleware.auth_token import _memcache_crypt as memcache_crypt
from keystonemiddleware.auth_token import _memcache_pool as memcache_pool
from keystonemiddleware.i18n import _, _LE
memcache = None # module will be loaded on demand to avoid dependency
def _create_memcache_client(*args, **kwargs):
"""Create a new memcache client object.
This handles the lazy loaded import but also provides a point to mock out
in testing.
"""
global memcache
if not memcache:
import memcache
return memcache.Client(*args, **kwargs)
from keystonemiddleware.openstack.common import memorycache
def _hash_key(key):
@ -79,7 +63,8 @@ class _CachePool(list):
try:
c = self.pop()
except IndexError:
c = _create_memcache_client(self._memcached_servers, debug=0)
# the pool is empty, so we need to create a new client
c = memorycache.get_client(self._memcached_servers)
try:
yield c
@ -87,57 +72,6 @@ class _CachePool(list):
self.append(c)
@six.add_metaclass(abc.ABCMeta)
class _CacheInterface(object):
def initialize(self, *args, **kwargs):
pass
@abc.abstractmethod
def store(self, key, value):
pass
@abc.abstractmethod
def store_invalid(self, key):
pass
@abc.abstractmethod
def get(self, key):
pass
def get_first(self, *args):
"""Get the first cached value from many options.
:returns: token data if found else None.
"""
for a in args:
value = self.get(a)
if value:
return value
return None
class NoOpCache(_CacheInterface):
def store(self, key, value):
# Don't store anything
return None
def store_invalid(self, key):
# Don't store anything
return None
def get(self, key):
# Nothing to fetch from
return None
def get_first(self, *args):
# short circuit because calling get() multiple times wont help
return None
class _MemcacheClientPool(object):
"""An advanced memcached client pool that is eventlet safe."""
def __init__(self, memcache_servers, **kwargs):
@ -150,7 +84,7 @@ class _MemcacheClientPool(object):
yield client
class TokenCache(_CacheInterface):
class TokenCache(object):
"""Encapsulates the auth_token token cache functionality.
auth_token caches tokens that it's seen so that when a token is re-used the
@ -190,13 +124,8 @@ class TokenCache(_CacheInterface):
return _MemcacheClientPool(self._memcached_servers,
**self._memcache_pool_options)
elif self._memcached_servers:
return _CachePool(self._memcached_servers)
else:
raise RuntimeError('Trying to configure a memcache cache without '
'passing any servers. This should have been '
'caught.')
return _CachePool(self._memcached_servers)
def initialize(self, env):
if self._initialized:

0
keystonemiddleware/openstack/__init__.py Normal file
View File

0
keystonemiddleware/openstack/common/__init__.py Normal file
View File

97
keystonemiddleware/openstack/common/memorycache.py Normal file
View File

@ -0,0 +1,97 @@
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Super simple fake memcache client."""
import copy
from oslo_config import cfg
from oslo_utils import timeutils
memcache_opts = [
cfg.ListOpt('memcached_servers',
help='Memcached servers or None for in process cache.'),
]
CONF = cfg.CONF
CONF.register_opts(memcache_opts)
def list_opts():
"""Entry point for oslo-config-generator."""
return [(None, copy.deepcopy(memcache_opts))]
def get_client(memcached_servers=None):
client_cls = Client
if not memcached_servers:
memcached_servers = CONF.memcached_servers
if memcached_servers:
import memcache
client_cls = memcache.Client
return client_cls(memcached_servers, debug=0)
class Client(object):
"""Replicates a tiny subset of memcached client interface."""
def __init__(self, *args, **kwargs):
"""Ignores the passed in args."""
self.cache = {}
def get(self, key):
"""Retrieves the value for a key or None.
This expunges expired keys during each get.
"""
now = timeutils.utcnow_ts()
for k in list(self.cache):
(timeout, _value) = self.cache[k]
if timeout and now >= timeout:
del self.cache[k]
return self.cache.get(key, (0, None))[1]
def set(self, key, value, time=0, min_compress_len=0):
"""Sets the value for a key."""
timeout = 0
if time != 0:
timeout = timeutils.utcnow_ts() + time
self.cache[key] = (timeout, value)
return True
def add(self, key, value, time=0, min_compress_len=0):
"""Sets the value for a key if it doesn't exist."""
if self.get(key) is not None:
return False
return self.set(key, value, time, min_compress_len)
def incr(self, key, delta=1):
"""Increments the value for a key."""
value = self.get(key)
if value is None:
return None
new_value = int(value) + delta
self.cache[key] = (self.cache[key][0], str(new_value))
return new_value
def delete(self, key, time=0):
"""Deletes the value associated with a key."""
if key in self.cache:
del self.cache[key]

44
keystonemiddleware/tests/unit/auth_token/test_auth_token_middleware.py
View File

@ -34,7 +34,6 @@ from oslo_config import cfg
from oslo_serialization import jsonutils
from oslo_utils import timeutils
from oslotest import createfile
from oslotest import mockpatch
import six
import testresources
import testtools
@ -46,9 +45,9 @@ from keystonemiddleware import auth_token
from keystonemiddleware.auth_token import _base
from keystonemiddleware.auth_token import _exceptions as ksm_exceptions
from keystonemiddleware.auth_token import _revocations
from keystonemiddleware.openstack.common import memorycache
from keystonemiddleware.tests.unit.auth_token import base
from keystonemiddleware.tests.unit import client_fixtures
from keystonemiddleware.tests.unit import utils
EXPECTED_V2_DEFAULT_ENV_RESPONSE = {
@ -338,11 +337,6 @@ class BaseAuthTokenMiddlewareTest(base.BaseAuthTokenTestCase):
else:
self.assertIsNone(self.requests_mock.last_request)
def mock_memcache(self):
return self.useFixture(mockpatch.Patch(
'keystonemiddleware.auth_token._cache._create_memcache_client',
return_value=utils.FakeMemcache()))
class DiabloAuthTokenMiddlewareTest(BaseAuthTokenMiddlewareTest,
testresources.ResourcedTestCase):
@ -398,16 +392,14 @@ class CachePoolTest(BaseAuthTokenMiddlewareTest):
def test_not_use_cache_from_env(self):
# If `swift.cache` is set in the environment but `cache` isn't set
# initialize the config then the env cache isn't used.
self.mock_memcache()
self.set_middleware(conf={'memcached_servers': ['localhost:4444']})
self.set_middleware()
env = {'swift.cache': 'CACHE_TEST'}
self.middleware._token_cache.initialize(env)
with self.middleware._token_cache._cache_pool.reserve() as cache:
self.assertNotEqual(cache, 'CACHE_TEST')
def test_multiple_context_managers_share_single_client(self):
self.set_middleware(conf={'memcached_servers': ['localhost:4444']})
self.set_middleware()
token_cache = self.middleware._token_cache
env = {}
token_cache.initialize(env)
@ -424,8 +416,7 @@ class CachePoolTest(BaseAuthTokenMiddlewareTest):
self.assertEqual(set(caches), set(token_cache._cache_pool))
def test_nested_context_managers_create_multiple_clients(self):
self.set_middleware(conf={'memcached_servers': ['localhost:4444']})
self.set_middleware()
env = {}
self.middleware._token_cache.initialize(env)
token_cache = self.middleware._token_cache
@ -470,8 +461,7 @@ class GeneralAuthTokenMiddlewareTest(BaseAuthTokenMiddlewareTest,
self.assertTrue(auth_token._token_is_v3(token_response))
def test_fixed_cache_key_length(self):
self.set_middleware(conf={'memcached_servers': ['localhost:4444']})
self.set_middleware()
short_string = uuid.uuid4().hex
long_string = 8 * uuid.uuid4().hex
@ -646,9 +636,6 @@ class CommonAuthTokenMiddlewareTest(object):
def _test_cache_revoked(self, token, revoked_form=None):
# When the token is cached and revoked, 401 is returned.
self.mock_memcache()
self.set_middleware(conf={'memcached_servers': ['127.0.0.1:4444']})
self.middleware._check_revocations_for_cached = True
# Token should be cached as ok after this.
@ -662,9 +649,6 @@ class CommonAuthTokenMiddlewareTest(object):
expected_status=401)
def test_cached_revoked_error(self):
self.mock_memcache()
self.set_middleware(conf={'memcached_servers': ['127.0.0.1:4444']})
# When the token is cached and revocation list retrieval fails,
# 503 is returned
token = self.token_dict['uuid_token_default']
@ -1013,8 +997,6 @@ class CommonAuthTokenMiddlewareTest(object):
return self.middleware._token_cache.get(token_id)
def test_memcache(self):
self.mock_memcache()
self.set_middleware(conf={'memcached_servers': ['127.0.0.1:4444']})
token = self.token_dict['signed_token_scoped']
self.call_middleware(headers={'X-Auth-Token': token})
self.assertIsNotNone(self._get_cached_token(token))
@ -1025,9 +1007,6 @@ class CommonAuthTokenMiddlewareTest(object):
expected_status=401)
def test_memcache_set_invalid_uuid(self):
self.mock_memcache()
self.set_middleware(conf={'memcached_servers': ['127.0.0.1:4444']})
invalid_uri = "%s/v2.0/tokens/invalid-token" % BASE_URI
self.requests_mock.get(invalid_uri, status_code=404)
@ -1038,11 +1017,8 @@ class CommonAuthTokenMiddlewareTest(object):
self._get_cached_token, token)
def test_memcache_set_expired(self, extra_conf={}, extra_environ={}):
self.mock_memcache()
token_cache_time = 10
conf = {
'memcached_servers': ['127.0.0.1:4444'],
'token_cache_time': '%s' % token_cache_time,
}
conf.update(extra_conf)
@ -1065,7 +1041,7 @@ class CommonAuthTokenMiddlewareTest(object):
def test_swift_memcache_set_expired(self):
extra_conf = {'cache': 'swift.cache'}
extra_environ = {'swift.cache': utils.FakeMemcache()}
extra_environ = {'swift.cache': memorycache.Client()}
self.test_memcache_set_expired(extra_conf, extra_environ)
def test_http_error_not_cached_token(self):
@ -1267,8 +1243,8 @@ class CommonAuthTokenMiddlewareTest(object):
# When the token is cached it isn't cached again when it's verified.
# The token cache has to be initialized with our cache instance.
self.set_middleware(conf={'cache': 'cache'})
cache = utils.FakeMemcache()
self.middleware._token_cache._env_cache_name = 'cache'
cache = memorycache.Client()
self.middleware._token_cache.initialize(env={'cache': cache})
# Mock cache.set since then the test can verify call_count.
@ -1849,11 +1825,9 @@ class v3AuthTokenMiddlewareTest(BaseAuthTokenMiddlewareTest,
def test_expire_stored_in_cache(self):
# tests the upgrade path from storing a tuple vs just the data in the
# cache. Can be removed in the future.
self.mock_memcache()
token = 'mytoken'
data = 'this_data'
self.set_middleware(conf={'memcached_servers': ['localhost:4444']})
self.set_middleware()
self.middleware._token_cache.initialize({})
now = datetime.datetime.utcnow()
delta = datetime.timedelta(hours=1)

51
keystonemiddleware/tests/unit/utils.py
View File

@ -17,7 +17,6 @@ import warnings
import fixtures
import mock
from oslo_utils import timeutils
import oslotest.base as oslotest
import requests
import uuid
@ -151,53 +150,3 @@ class NoModuleFinder(object):
def find_module(self, fullname, path):
if fullname == self.module or fullname.startswith(self.module + '.'):
raise ImportError
class FakeMemcache(object):
"""Replicates a tiny subset of memcached client interface."""
def __init__(self, *args, **kwargs):
"""Ignores the passed in args."""
self.cache = {}
def get(self, key):
"""Retrieves the value for a key or None.
This expunges expired keys during each get.
"""
now = timeutils.utcnow_ts()
for k in list(self.cache):
(timeout, _value) = self.cache[k]
if timeout and now >= timeout:
del self.cache[k]
return self.cache.get(key, (0, None))[1]
def set(self, key, value, time=0, min_compress_len=0):
"""Sets the value for a key."""
timeout = 0
if time != 0:
timeout = timeutils.utcnow_ts() + time
self.cache[key] = (timeout, value)
return True
def add(self, key, value, time=0, min_compress_len=0):
"""Sets the value for a key if it doesn't exist."""
if self.get(key) is not None:
return False
return self.set(key, value, time, min_compress_len)
def incr(self, key, delta=1):
"""Increments the value for a key."""
value = self.get(key)
if value is None:
return None
new_value = int(value) + delta
self.cache[key] = (self.cache[key][0], str(new_value))
return new_value
def delete(self, key, time=0):
"""Deletes the value associated with a key."""
if key in self.cache:
del self.cache[key]

1
openstack-common.conf
View File

@ -1,6 +1,7 @@
[DEFAULT]
# The list of modules to copy from oslo-incubator
module=memorycache
# The base module to hold the copy of openstack.common
base=keystonemiddleware