Merge "proxy: Add a chance to skip memcache for get_*_info calls"
This commit is contained in:
Zuul
Gerrit Code Review
commit
8ab6af27c5
@ -156,9 +156,9 @@ ionice_priority None I/O scheduling p
|
||||
[proxy-server]
|
||||
**************
|
||||
|
||||
====================================== =============== =====================================
|
||||
============================================== =============== =====================================
|
||||
Option Default Description
|
||||
-------------------------------------- --------------- -------------------------------------
|
||||
---------------------------------------------- --------------- -------------------------------------
|
||||
use Entry point for paste.deploy for
|
||||
the proxy server. For most
|
||||
cases, this should be
|
||||
@ -177,6 +177,34 @@ recheck_account_existence 60 Cache timeout in second
|
||||
recheck_container_existence 60 Cache timeout in seconds to
|
||||
send memcached for container
|
||||
existence
|
||||
account_existence_skip_cache_pct 0.0 Periodically, bypass the cache
|
||||
for account info requests and
|
||||
goto disk to refresh the data
|
||||
in the cache. This is a percentage
|
||||
of requests should randomly skip.
|
||||
Values around 0.0 - 0.1 (1 in every
|
||||
1000) are recommended.
|
||||
container_existence_skip_cache_pct 0.0 Periodically, bypass the cache
|
||||
for container info requests and
|
||||
goto disk to refresh the data
|
||||
in the cache. This is a percentage
|
||||
of requests should randomly skip.
|
||||
Values around 0.0 - 0.1 (1 in every
|
||||
1000) are recommended.
|
||||
container_updating_shard_ranges_skip_cache_pct 0.0 Periodically, bypass the cache
|
||||
for shard_range update requests and
|
||||
goto disk to refresh the data
|
||||
in the cache. This is a percentage
|
||||
of requests should randomly skip.
|
||||
Values around 0.0 - 0.1 (1 in every
|
||||
1000) are recommended.
|
||||
container_listing_shard_ranges_skip_cache_pct 0.0 Periodically, bypass the cache
|
||||
for shard_range listing info requests
|
||||
and goto disk to refresh the data
|
||||
in the cache. This is a percentage
|
||||
of requests should randomly skip.
|
||||
Values around 0.0 - 0.1 (1 in every
|
||||
1000) are recommended.
|
||||
object_chunk_size 65536 Chunk size to read from
|
||||
object servers
|
||||
client_chunk_size 65536 Chunk size to read from
|
||||
@ -355,4 +383,4 @@ write_affinity_handoff_delete_count auto The number of local (as
|
||||
(replicas - len(local_primary_nodes)).
|
||||
This option may be overridden in a
|
||||
per-policy configuration section.
|
||||
====================================== =============== =====================================
|
||||
============================================== =============== =====================================
|
||||
|
||||
@ -153,8 +153,10 @@ use = egg:swift#proxy
|
||||
# data is present in memcache, we can periodically refresh the data in memcache
|
||||
# without causing a thundering herd. Values around 0.0 - 0.1 (i.e., one in
|
||||
# every thousand requests skips cache, or fewer) are recommended.
|
||||
# container_existence_skip_cache_pct = 0.0
|
||||
# container_updating_shard_ranges_skip_cache_pct = 0.0
|
||||
# container_listing_shard_ranges_skip_cache_pct = 0.0
|
||||
# account_existence_skip_cache_pct = 0.0
|
||||
#
|
||||
# object_chunk_size = 65536
|
||||
# client_chunk_size = 65536
|
||||
|
||||
@ -167,6 +167,9 @@ from swift.common.registry import register_swift_info, \
|
||||
class ListingEtagMiddleware(object):
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
# Pass this along so get_container_info will have the configured
|
||||
# odds to skip cache
|
||||
self._pipeline_final_app = app._pipeline_final_app
|
||||
|
||||
def __call__(self, env, start_response):
|
||||
# a lot of this is cribbed from listing_formats / swob.Request
|
||||
|
||||
@ -47,5 +47,8 @@ def filter_factory(global_conf, **local_conf):
|
||||
if 'symlink' not in get_swift_info():
|
||||
raise ValueError('object versioning requires symlinks')
|
||||
app = ObjectVersioningMiddleware(app, conf)
|
||||
# Pass this along so get_container_info will have the configured
|
||||
# odds to skip cache
|
||||
app._pipeline_final_app = app.app._pipeline_final_app
|
||||
return VersionedWritesMiddleware(app, conf)
|
||||
return versioning_filter
|
||||
|
||||
@ -3716,7 +3716,11 @@ class StreamingPile(GreenAsyncPile):
|
||||
|
||||
# Keep populating the pile as greenthreads become available
|
||||
for args in args_iter:
|
||||
yield next(self)
|
||||
try:
|
||||
to_yield = next(self)
|
||||
except StopIteration:
|
||||
break
|
||||
yield to_yield
|
||||
self.spawn(func, *args)
|
||||
|
||||
# Drain the pile
|
||||
|
||||
@ -750,7 +750,30 @@ def _get_info_from_memcache(app, env, account, container=None):
|
||||
cache_key = get_cache_key(account, container)
|
||||
memcache = cache_from_env(env, True)
|
||||
if memcache:
|
||||
try:
|
||||
proxy_app = app._pipeline_final_app
|
||||
except AttributeError:
|
||||
# Only the middleware entry-points get a reference to the
|
||||
# proxy-server app; if a middleware composes itself as multiple
|
||||
# filters, we'll just have to choose a reasonable default
|
||||
skip_chance = 0.0
|
||||
logger = None
|
||||
else:
|
||||
if container:
|
||||
skip_chance = proxy_app.container_existence_skip_cache
|
||||
else:
|
||||
skip_chance = proxy_app.account_existence_skip_cache
|
||||
logger = proxy_app.logger
|
||||
info_type = 'container' if container else 'account'
|
||||
if skip_chance and random.random() < skip_chance:
|
||||
info = None
|
||||
if logger:
|
||||
logger.increment('%s.info.cache.skip' % info_type)
|
||||
else:
|
||||
info = memcache.get(cache_key)
|
||||
if logger:
|
||||
logger.increment('%s.info.cache.%s' % (
|
||||
info_type, 'hit' if info else 'miss'))
|
||||
if info and six.PY2:
|
||||
# Get back to native strings
|
||||
new_info = {}
|
||||
|
||||
@ -193,6 +193,10 @@ class Application(object):
|
||||
|
||||
def __init__(self, conf, logger=None, account_ring=None,
|
||||
container_ring=None):
|
||||
# This is for the sake of tests which instantiate an Application
|
||||
# directly rather than via loadapp().
|
||||
self._pipeline_final_app = self
|
||||
|
||||
if conf is None:
|
||||
conf = {}
|
||||
if logger is None:
|
||||
@ -230,12 +234,16 @@ class Application(object):
|
||||
self.recheck_account_existence = \
|
||||
int(conf.get('recheck_account_existence',
|
||||
DEFAULT_RECHECK_ACCOUNT_EXISTENCE))
|
||||
self.container_existence_skip_cache = config_percent_value(
|
||||
conf.get('container_existence_skip_cache_pct', 0))
|
||||
self.container_updating_shard_ranges_skip_cache = \
|
||||
config_percent_value(conf.get(
|
||||
'container_updating_shard_ranges_skip_cache_pct', 0))
|
||||
self.container_listing_shard_ranges_skip_cache = \
|
||||
config_percent_value(conf.get(
|
||||
'container_listing_shard_ranges_skip_cache_pct', 0))
|
||||
self.account_existence_skip_cache = config_percent_value(
|
||||
conf.get('account_existence_skip_cache_pct', 0))
|
||||
self.allow_account_management = \
|
||||
config_true_value(conf.get('allow_account_management', 'no'))
|
||||
self.container_ring = container_ring or Ring(swift_dir,
|
||||
|
||||
@ -77,6 +77,8 @@ class FakeSwift(object):
|
||||
ALLOWED_METHODS = [
|
||||
'PUT', 'POST', 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'REPLICATE',
|
||||
'SSYNC', 'UPDATE']
|
||||
container_existence_skip_cache = 0.0
|
||||
account_existence_skip_cache = 0.0
|
||||
|
||||
def __init__(self):
|
||||
self._calls = []
|
||||
|
||||
@ -29,8 +29,13 @@ from test.unit.common.middleware.s3api.helpers import FakeSwift
|
||||
|
||||
|
||||
class FakeApp(object):
|
||||
container_existence_skip_cache = 0.0
|
||||
account_existence_skip_cache = 0.0
|
||||
|
||||
def __init__(self):
|
||||
self._pipeline_final_app = self
|
||||
self.swift = FakeSwift()
|
||||
self.logger = debug_logger()
|
||||
|
||||
def _update_s3_path_info(self, env):
|
||||
"""
|
||||
|
||||
@ -2207,7 +2207,8 @@ class TestContainerController(TestRingBase):
|
||||
req.environ['swift.infocache']['shard-listing/a/c'])
|
||||
self.assertEqual(
|
||||
[x[0][0] for x in self.logger.logger.log_dict['increment']],
|
||||
['container.shard_listing.backend.200'])
|
||||
['container.info.cache.miss',
|
||||
'container.shard_listing.backend.200'])
|
||||
|
||||
# container is sharded and proxy has that state cached, but
|
||||
# no shard ranges cached; expect a cache miss and write-back
|
||||
@ -2244,7 +2245,8 @@ class TestContainerController(TestRingBase):
|
||||
req.environ['swift.infocache']['shard-listing/a/c'])
|
||||
self.assertEqual(
|
||||
[x[0][0] for x in self.logger.logger.log_dict['increment']],
|
||||
['container.shard_listing.cache.miss',
|
||||
['container.info.cache.hit',
|
||||
'container.shard_listing.cache.miss',
|
||||
'container.shard_listing.backend.200'])
|
||||
|
||||
# container is sharded and proxy does have that state cached and
|
||||
@ -2268,7 +2270,8 @@ class TestContainerController(TestRingBase):
|
||||
req.environ['swift.infocache']['shard-listing/a/c'])
|
||||
self.assertEqual(
|
||||
[x[0][0] for x in self.logger.logger.log_dict['increment']],
|
||||
['container.shard_listing.cache.hit'])
|
||||
['container.info.cache.hit',
|
||||
'container.shard_listing.cache.hit'])
|
||||
|
||||
# if there's a chance to skip cache, maybe we go to disk again...
|
||||
self.memcache.clear_calls()
|
||||
@ -2304,7 +2307,8 @@ class TestContainerController(TestRingBase):
|
||||
req.environ['swift.infocache']['shard-listing/a/c'])
|
||||
self.assertEqual(
|
||||
[x[0][0] for x in self.logger.logger.log_dict['increment']],
|
||||
['container.shard_listing.cache.skip',
|
||||
['container.info.cache.hit',
|
||||
'container.shard_listing.cache.skip',
|
||||
'container.shard_listing.backend.200'])
|
||||
|
||||
# ... or maybe we serve from cache
|
||||
@ -2328,8 +2332,8 @@ class TestContainerController(TestRingBase):
|
||||
req.environ['swift.infocache']['shard-listing/a/c'])
|
||||
self.assertEqual(
|
||||
[x[0][0] for x in self.logger.logger.log_dict['increment']],
|
||||
['container.shard_listing.cache.hit'])
|
||||
|
||||
['container.info.cache.hit',
|
||||
'container.shard_listing.cache.hit'])
|
||||
# put this back the way we found it for later subtests
|
||||
self.app.container_listing_shard_ranges_skip_cache = 0.0
|
||||
|
||||
@ -2479,7 +2483,8 @@ class TestContainerController(TestRingBase):
|
||||
self.assertEqual(404, self.memcache.calls[2][1][1]['status'])
|
||||
self.assertEqual(b'', resp.body)
|
||||
self.assertEqual(404, resp.status_int)
|
||||
self.assertEqual({'container.shard_listing.cache.miss': 1,
|
||||
self.assertEqual({'container.info.cache.hit': 1,
|
||||
'container.shard_listing.cache.miss': 1,
|
||||
'container.shard_listing.backend.404': 1},
|
||||
self.logger.get_increment_counts())
|
||||
|
||||
@ -2512,7 +2517,8 @@ class TestContainerController(TestRingBase):
|
||||
self.assertEqual(404, self.memcache.calls[2][1][1]['status'])
|
||||
self.assertEqual(b'', resp.body)
|
||||
self.assertEqual(404, resp.status_int)
|
||||
self.assertEqual({'container.shard_listing.cache.error': 1,
|
||||
self.assertEqual({'container.info.cache.hit': 1,
|
||||
'container.shard_listing.cache.error': 1,
|
||||
'container.shard_listing.backend.404': 1},
|
||||
self.logger.get_increment_counts())
|
||||
|
||||
@ -2535,7 +2541,8 @@ class TestContainerController(TestRingBase):
|
||||
[mock.call.get('container/a/c'),
|
||||
mock.call.get('shard-listing/a/c', raise_on_error=True)],
|
||||
self.memcache.calls)
|
||||
self.assertEqual({'container.shard_listing.cache.hit': 1},
|
||||
self.assertEqual({'container.info.cache.hit': 1,
|
||||
'container.shard_listing.cache.hit': 1},
|
||||
self.logger.get_increment_counts())
|
||||
return resp
|
||||
|
||||
@ -2625,7 +2632,8 @@ class TestContainerController(TestRingBase):
|
||||
# shards were cached
|
||||
self.assertEqual('sharded',
|
||||
self.memcache.calls[2][1][1]['sharding_state'])
|
||||
self.assertEqual({'container.shard_listing.backend.200': 1},
|
||||
self.assertEqual({'container.info.cache.miss': 1,
|
||||
'container.shard_listing.backend.200': 1},
|
||||
self.logger.get_increment_counts())
|
||||
return resp
|
||||
|
||||
@ -2718,7 +2726,8 @@ class TestContainerController(TestRingBase):
|
||||
self.memcache.calls)
|
||||
self.assertEqual('sharded',
|
||||
self.memcache.calls[2][1][1]['sharding_state'])
|
||||
self.assertEqual({'container.shard_listing.backend.200': 1},
|
||||
self.assertEqual({'container.info.cache.miss': 1,
|
||||
'container.shard_listing.backend.200': 1},
|
||||
self.logger.get_increment_counts())
|
||||
|
||||
def _do_test_GET_shard_ranges_no_cache_write(self, resp_hdrs):
|
||||
@ -2890,7 +2899,8 @@ class TestContainerController(TestRingBase):
|
||||
self.memcache.calls)
|
||||
self.assertEqual(resp.headers.get('X-Backend-Sharding-State'),
|
||||
self.memcache.calls[1][1][1]['sharding_state'])
|
||||
self.assertEqual({'container.shard_listing.backend.200': 1},
|
||||
self.assertEqual({'container.info.cache.miss': 1,
|
||||
'container.shard_listing.backend.200': 1},
|
||||
self.logger.get_increment_counts())
|
||||
self.memcache.delete_all()
|
||||
|
||||
|
||||
@ -508,6 +508,7 @@ class TestController(unittest.TestCase):
|
||||
|
||||
def test_get_account_info_returns_values_as_strings(self):
|
||||
app = mock.MagicMock()
|
||||
app._pipeline_final_app.account_existence_skip_cache = 0.0
|
||||
memcache = mock.MagicMock()
|
||||
memcache.get = mock.MagicMock()
|
||||
memcache.get.return_value = {
|
||||
@ -533,6 +534,7 @@ class TestController(unittest.TestCase):
|
||||
|
||||
def test_get_container_info_returns_values_as_strings(self):
|
||||
app = mock.MagicMock()
|
||||
app._pipeline_final_app.container_existence_skip_cache = 0.0
|
||||
memcache = mock.MagicMock()
|
||||
memcache.get = mock.MagicMock()
|
||||
memcache.get.return_value = {
|
||||
@ -4134,9 +4136,10 @@ class TestReplicatedObjectController(
|
||||
|
||||
self.assertEqual(resp.status_int, 202)
|
||||
stats = self.app.logger.get_increment_counts()
|
||||
self.assertEqual({'object.shard_updating.cache.miss': 1,
|
||||
self.assertEqual({'account.info.cache.miss': 1,
|
||||
'container.info.cache.miss': 1,
|
||||
'object.shard_updating.cache.miss': 1,
|
||||
'object.shard_updating.backend.200': 1}, stats)
|
||||
# verify statsd prefix is not mutated
|
||||
self.assertEqual([], self.app.logger.log_dict['set_statsd_prefix'])
|
||||
|
||||
backend_requests = fake_conn.requests
|
||||
@ -4234,7 +4237,9 @@ class TestReplicatedObjectController(
|
||||
|
||||
self.assertEqual(resp.status_int, 202)
|
||||
stats = self.app.logger.get_increment_counts()
|
||||
self.assertEqual({'object.shard_updating.cache.hit': 1}, stats)
|
||||
self.assertEqual({'account.info.cache.miss': 1,
|
||||
'container.info.cache.miss': 1,
|
||||
'object.shard_updating.cache.hit': 1}, stats)
|
||||
# verify statsd prefix is not mutated
|
||||
self.assertEqual([], self.app.logger.log_dict['set_statsd_prefix'])
|
||||
|
||||
@ -4328,7 +4333,9 @@ class TestReplicatedObjectController(
|
||||
|
||||
self.assertEqual(resp.status_int, 202)
|
||||
stats = self.app.logger.get_increment_counts()
|
||||
self.assertEqual({'object.shard_updating.cache.hit': 1}, stats)
|
||||
self.assertEqual({'account.info.cache.miss': 1,
|
||||
'container.info.cache.miss': 1,
|
||||
'object.shard_updating.cache.hit': 1}, stats)
|
||||
|
||||
# cached shard ranges are still there
|
||||
cache_key = 'shard-updating/a/c'
|
||||
@ -4366,7 +4373,11 @@ class TestReplicatedObjectController(
|
||||
|
||||
self.assertEqual(resp.status_int, 202)
|
||||
stats = self.app.logger.get_increment_counts()
|
||||
self.assertEqual({'object.shard_updating.cache.skip': 1,
|
||||
self.assertEqual({'account.info.cache.miss': 1,
|
||||
'account.info.cache.hit': 1,
|
||||
'container.info.cache.miss': 1,
|
||||
'container.info.cache.hit': 1,
|
||||
'object.shard_updating.cache.skip': 1,
|
||||
'object.shard_updating.cache.hit': 1,
|
||||
'object.shard_updating.backend.200': 1}, stats)
|
||||
# verify statsd prefix is not mutated
|
||||
@ -4425,10 +4436,15 @@ class TestReplicatedObjectController(
|
||||
|
||||
self.assertEqual(resp.status_int, 202)
|
||||
stats = self.app.logger.get_increment_counts()
|
||||
self.assertEqual({'object.shard_updating.cache.skip': 1,
|
||||
self.assertEqual(stats, {
|
||||
'account.info.cache.hit': 2,
|
||||
'account.info.cache.miss': 1,
|
||||
'container.info.cache.hit': 2,
|
||||
'container.info.cache.miss': 1,
|
||||
'object.shard_updating.cache.skip': 1,
|
||||
'object.shard_updating.cache.hit': 1,
|
||||
'object.shard_updating.cache.error': 1,
|
||||
'object.shard_updating.backend.200': 2}, stats)
|
||||
'object.shard_updating.backend.200': 2})
|
||||
|
||||
do_test('POST', 'sharding')
|
||||
do_test('POST', 'sharded')
|
||||
|
||||
Loading…
Reference in New Issue
Block a user