From 45be775ebf62af67e74face216ab59a367784fd0 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Sun, 14 Jun 2020 16:57:08 -0700 Subject: [PATCH] Get rid of memcache attr on proxy-server app In a running proxy server, this only ever comes from one place: a key in the wsgi env, populated by the memcache middleware. By littering proxy controller code with things like memcache = getattr(app, 'memcache', None) or env.get('swift.cache') we cause ourselves to second-guess that. Change-Id: Ia652b1381990b61194978c4efaebac7d2a602ca3 --- swift/proxy/controllers/base.py | 19 +- swift/proxy/server.py | 7 +- test/unit/common/middleware/test_copy.py | 4 +- test/unit/proxy/controllers/test_account.py | 8 +- test/unit/proxy/controllers/test_base.py | 6 +- test/unit/proxy/controllers/test_container.py | 18 +- test/unit/proxy/controllers/test_obj.py | 6 +- test/unit/proxy/test_server.py | 239 +++++++----------- test/unit/proxy/test_sysmeta.py | 82 +++--- 9 files changed, 171 insertions(+), 218 deletions(-) diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index bf2f3d1ea8..95ff8e7653 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -45,7 +45,8 @@ from swift.common.wsgi import make_pre_authed_env, make_pre_authed_request from swift.common.utils import Timestamp, WatchdogTimeout, config_true_value, \ public, split_path, list_from_csv, GreenthreadSafeIterator, \ GreenAsyncPile, quorum_size, parse_content_type, drain_and_close, \ - document_iters_to_http_response_body, ShardRange, find_shard_range + document_iters_to_http_response_body, ShardRange, find_shard_range, \ + cache_from_env from swift.common.bufferedhttp import http_connect from swift.common import constraints from swift.common.exceptions import ChunkReadTimeout, ChunkWriteTimeout, \ @@ -541,8 +542,7 @@ def set_info_cache(app, env, account, container, resp): """ cache_key = get_cache_key(account, container) infocache = env.setdefault('swift.infocache', {}) - memcache = getattr(app, 'memcache', None) or env.get('swift.cache') - + memcache = cache_from_env(env, True) if resp is None: infocache.pop(cache_key, None) if memcache: @@ -645,7 +645,7 @@ def _get_info_from_memcache(app, env, account, container=None): returns None if memcache is not in use. """ cache_key = get_cache_key(account, container) - memcache = getattr(app, 'memcache', None) or env.get('swift.cache') + memcache = cache_from_env(env, True) if memcache: info = memcache.get(cache_key) if info and six.PY2: @@ -1678,12 +1678,12 @@ class Controller(object): headers['referer'] = referer return headers - def account_info(self, account, req=None): + def account_info(self, account, req): """ Get account information, and also verify that the account exists. :param account: native str name of the account to get the info for - :param req: caller's HTTP request context object (optional) + :param req: caller's HTTP request context object :returns: tuple of (account partition, account nodes, container_count) or (None, None, None) if it does not exist """ @@ -1704,14 +1704,14 @@ class Controller(object): partition, nodes = self.app.account_ring.get_nodes(account) return partition, nodes, container_count - def container_info(self, account, container, req=None): + def container_info(self, account, container, req): """ Get container information and thusly verify container existence. This will also verify account existence. :param account: native-str account name for the container :param container: native-str container name to look up - :param req: caller's HTTP request context object (optional) + :param req: caller's HTTP request context object :returns: dict containing at least container partition ('partition'), container nodes ('containers'), container read acl ('read_acl'), container write acl ('write_acl'), @@ -2268,8 +2268,7 @@ class Controller(object): cache_key = get_cache_key(account, container, shard='updating') infocache = req.environ.setdefault('swift.infocache', {}) - memcache = getattr(self.app, 'memcache', None) or req.environ.get( - 'swift.cache') + memcache = cache_from_env(req.environ, True) cached_ranges = infocache.get(cache_key) if cached_ranges is None and memcache: diff --git a/swift/proxy/server.py b/swift/proxy/server.py index c79251c69c..b130852413 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -32,7 +32,7 @@ from swift.common import constraints from swift.common.http import is_server_error from swift.common.storage_policy import POLICIES from swift.common.ring import Ring -from swift.common.utils import Watchdog, cache_from_env, get_logger, \ +from swift.common.utils import Watchdog, get_logger, \ get_remote_client, split_path, config_true_value, generate_trans_id, \ affinity_key_function, affinity_locality_predicate, list_from_csv, \ register_swift_info, readconf, config_auto_int_value @@ -170,7 +170,7 @@ class ProxyOverrideOptions(object): class Application(object): """WSGI application for the proxy server.""" - def __init__(self, conf, memcache=None, logger=None, account_ring=None, + def __init__(self, conf, logger=None, account_ring=None, container_ring=None): if conf is None: conf = {} @@ -218,7 +218,6 @@ class Application(object): for policy in POLICIES: policy.load_ring(swift_dir) self.obj_controller_router = ObjectControllerRouter() - self.memcache = memcache mimetypes.init(mimetypes.knownfiles + [os.path.join(swift_dir, 'mime.types')]) self.account_autocreate = \ @@ -454,8 +453,6 @@ class Application(object): :param start_response: WSGI callable """ try: - if self.memcache is None: - self.memcache = cache_from_env(env, True) req = self.update_request(Request(env)) return self.handle_request(req)(env, start_response) except UnicodeError: diff --git a/test/unit/common/middleware/test_copy.py b/test/unit/common/middleware/test_copy.py index 128e047350..e1e5317cde 100644 --- a/test/unit/common/middleware/test_copy.py +++ b/test/unit/common/middleware/test_copy.py @@ -24,7 +24,7 @@ from swift.common.middleware import copy from swift.common.storage_policy import POLICIES from swift.common.swob import Request, HTTPException from swift.common.utils import closing_if_possible -from test.unit import patch_policies, debug_logger, FakeMemcache, FakeRing +from test.unit import patch_policies, debug_logger, FakeRing from test.unit.common.middleware.helpers import FakeSwift from test.unit.proxy.controllers.test_obj import set_http_connect, \ PatchedObjControllerApp @@ -1316,7 +1316,7 @@ class TestServerSideCopyMiddlewareWithEC(unittest.TestCase): self.logger = debug_logger('proxy-server') self.logger.thread_locals = ('txn1', '127.0.0.2') self.app = PatchedObjControllerApp( - None, FakeMemcache(), account_ring=FakeRing(), + None, account_ring=FakeRing(), container_ring=FakeRing(), logger=self.logger) self.ssc = copy.filter_factory({})(self.app) self.ssc.logger = self.app.logger diff --git a/test/unit/proxy/controllers/test_account.py b/test/unit/proxy/controllers/test_account.py index e768c1245c..eac9c7f954 100644 --- a/test/unit/proxy/controllers/test_account.py +++ b/test/unit/proxy/controllers/test_account.py @@ -21,8 +21,7 @@ from swift.common.middleware.acl import format_acl from swift.proxy import server as proxy_server from swift.proxy.controllers.base import headers_to_account_info from swift.common import constraints -from test.unit import fake_http_connect, FakeRing, FakeMemcache, \ - mocked_http_conn +from test.unit import fake_http_connect, FakeRing, mocked_http_conn from swift.common.storage_policy import StoragePolicy from swift.common.request_helpers import get_sys_meta_prefix import swift.proxy.controllers.base @@ -38,7 +37,7 @@ class TestAccountController(unittest.TestCase): def setUp(self): self.app = proxy_server.Application( - None, FakeMemcache(), + None, account_ring=FakeRing(), container_ring=FakeRing()) def _make_callback_func(self, context): @@ -344,7 +343,6 @@ class TestAccountController4Replicas(TestAccountController): def setUp(self): self.app = proxy_server.Application( None, - FakeMemcache(), account_ring=FakeRing(replicas=4), container_ring=FakeRing(replicas=4)) @@ -413,7 +411,7 @@ class TestAccountController4Replicas(TestAccountController): class TestGetAccountInfo(unittest.TestCase): def setUp(self): self.app = proxy_server.Application( - None, FakeMemcache(), + None, account_ring=FakeRing(), container_ring=FakeRing()) def test_get_deleted_account_410(self): diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index cbff2d23ad..6464dbd6df 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -182,7 +182,7 @@ class FakeCache(FakeMemcache): @patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) class TestFuncs(unittest.TestCase): def setUp(self): - self.app = proxy_server.Application(None, FakeMemcache(), + self.app = proxy_server.Application(None, account_ring=FakeRing(), container_ring=FakeRing(), logger=FakeLogger()) @@ -710,7 +710,7 @@ class TestFuncs(unittest.TestCase): resp, headers_to_container_info(headers.items(), 200)) - def test_container_info_without_req(self): + def test_container_info_needs_req(self): base = Controller(self.app) base.account_name = 'a' base.container_name = 'c' @@ -719,7 +719,7 @@ class TestFuncs(unittest.TestCase): 'http_connect', fake_http_connect(200)): container_info = \ base.container_info(base.account_name, - base.container_name) + base.container_name, Request.blank('/')) self.assertEqual(container_info['status'], 0) def test_headers_to_account_info_missing(self): diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py index 6ca34662e1..b6d02937b0 100644 --- a/test/unit/proxy/controllers/test_container.py +++ b/test/unit/proxy/controllers/test_container.py @@ -27,7 +27,7 @@ from swift.common.swob import Request, bytes_to_wsgi, str_to_wsgi, wsgi_quote from swift.common.utils import ShardRange, Timestamp from swift.proxy import server as proxy_server from swift.proxy.controllers.base import headers_to_container_info, \ - Controller, get_container_info + Controller, get_container_info, get_cache_key from test import annotate_failure from test.unit import fake_http_connect, FakeRing, FakeMemcache, \ make_timestamp_iter @@ -49,7 +49,7 @@ class TestContainerController(TestRingBase): self.logger = debug_logger() self.container_ring = FakeRing(replicas=self.CONTAINER_REPLICAS, max_more_nodes=9) - self.app = proxy_server.Application(None, FakeMemcache(), + self.app = proxy_server.Application(None, logger=self.logger, account_ring=FakeRing(), container_ring=self.container_ring) @@ -97,7 +97,9 @@ class TestContainerController(TestRingBase): with mock.patch( 'swift.proxy.controllers.base.http_connect', fake_http_connect(*responses)): - req = Request.blank('/v1/a/c') + cache = FakeMemcache() + cache.set(get_cache_key('a'), {'status': 204}) + req = Request.blank('/v1/a/c', environ={'swift.cache': cache}) resp = getattr(controller, method)(req) self.assertEqual(expected, @@ -106,9 +108,10 @@ class TestContainerController(TestRingBase): (expected, resp.status_int, str(responses))) def test_container_info_got_cached(self): + memcache = FakeMemcache() controller = proxy_server.ContainerController(self.app, 'a', 'c') with mocked_http_conn(200, 200) as mock_conn: - req = Request.blank('/v1/a/c') + req = Request.blank('/v1/a/c', {'swift.cache': memcache}) resp = controller.HEAD(req) self.assertEqual(2, resp.status_int // 100) self.assertEqual(['/a', '/a/c'], @@ -119,12 +122,13 @@ class TestContainerController(TestRingBase): info_cache = resp.environ['swift.infocache'] self.assertIn("container/a/c", resp.environ['swift.infocache']) self.assertEqual(header_info, info_cache['container/a/c']) - self.assertEqual(header_info, self.app.memcache.get('container/a/c')) + self.assertEqual(header_info, memcache.get('container/a/c')) # The failure doesn't lead to cache eviction errors = [500] * self.CONTAINER_REPLICAS * 2 with mocked_http_conn(*errors) as mock_conn: - req = Request.blank('/v1/a/c', {'swift.infocache': info_cache}) + req = Request.blank('/v1/a/c', {'swift.infocache': info_cache, + 'swift.cache': memcache}) resp = controller.HEAD(req) self.assertEqual(5, resp.status_int // 100) self.assertEqual(['/a/c'] * self.CONTAINER_REPLICAS * 2, @@ -134,7 +138,7 @@ class TestContainerController(TestRingBase): self.assertIn("container/a/c", resp.environ['swift.infocache']) # NB: this is the *old* header_info, from the good req self.assertEqual(header_info, info_cache['container/a/c']) - self.assertEqual(header_info, self.app.memcache.get('container/a/c')) + self.assertEqual(header_info, memcache.get('container/a/c')) @mock.patch('swift.proxy.controllers.container.clear_info_cache') @mock.patch.object(Controller, 'make_requests') diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 9b862f5d3b..c859954a5f 100644 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -48,7 +48,7 @@ from swift.proxy.controllers.base import \ from swift.common.storage_policy import POLICIES, ECDriverError, \ StoragePolicy, ECStoragePolicy -from test.unit import FakeRing, FakeMemcache, fake_http_connect, \ +from test.unit import FakeRing, fake_http_connect, \ debug_logger, patch_policies, SlowBody, FakeStatus, \ DEFAULT_TEST_EC_TYPE, encode_frag_archive_bodies, make_ec_object_stub, \ fake_ec_node_response, StubResponse, mocked_http_conn, \ @@ -179,7 +179,7 @@ class BaseObjectControllerMixin(object): # increase connection timeout to avoid intermittent failures conf = {'conn_timeout': 1.0} self.app = PatchedObjControllerApp( - conf, FakeMemcache(), account_ring=FakeRing(), + conf, account_ring=FakeRing(), container_ring=FakeRing(), logger=self.logger) # you can over-ride the container_info just by setting it on the app @@ -969,7 +969,7 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin): self, conf, policy_conf, policy, affinity_regions, affinity_count): conf['policy_config'] = policy_conf app = PatchedObjControllerApp( - conf, FakeMemcache(), account_ring=FakeRing(), + conf, account_ring=FakeRing(), container_ring=FakeRing(), logger=self.logger) controller = self.controller_cls(app, 'a', 'c', 'o') diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 0a4d9de77b..db41ced574 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -173,14 +173,6 @@ def set_node_errors(proxy_app, ring_node, value, last_error): stats['last_error'] = last_error -class FakeMemcacheReturnsNone(FakeMemcache): - - def get(self, key): - # Returns None as the timestamp of the container; assumes we're only - # using the FakeMemcache for container existence checks. - return None - - @contextmanager def save_globals(): orig_http_connect = getattr(swift.proxy.controllers.base, 'http_connect', @@ -251,22 +243,23 @@ class TestController(unittest.TestCase): self.account_ring = FakeRing() self.container_ring = FakeRing() self.memcache = FakeMemcache() - app = proxy_server.Application(None, self.memcache, + app = proxy_server.Application(None, account_ring=self.account_ring, container_ring=self.container_ring) self.controller = swift.proxy.controllers.Controller(app) class FakeReq(object): - def __init__(self): + def __init__(self, memcache): self.url = "/foo/bar" self.method = "METHOD" + self.environ = {"swift.cache": memcache} def as_referer(self): return self.method + ' ' + self.url self.account = 'some_account' self.container = 'some_container' - self.request = FakeReq() + self.request = FakeReq(self.memcache) self.read_acl = 'read_acl' self.write_acl = 'write_acl' @@ -290,29 +283,31 @@ class TestController(unittest.TestCase): self.assertEqual(n, nodes) def test_account_info_container_count(self): + cache_key = get_cache_key(self.account) with save_globals(): set_http_connect(200, count=123) partition, nodes, count = \ - self.controller.account_info(self.account) + self.controller.account_info(self.account, self.request) self.assertEqual(count, 123) with save_globals(): set_http_connect(200, count='123') + del self.request.environ['swift.infocache'][cache_key] partition, nodes, count = \ - self.controller.account_info(self.account) + self.controller.account_info(self.account, self.request) self.assertEqual(count, 123) with save_globals(): - cache_key = get_cache_key(self.account) account_info = {'status': 200, 'container_count': 1234} self.memcache.set(cache_key, account_info) + del self.request.environ['swift.infocache'][cache_key] partition, nodes, count = \ - self.controller.account_info(self.account) + self.controller.account_info(self.account, self.request) self.assertEqual(count, 1234) with save_globals(): - cache_key = get_cache_key(self.account) account_info = {'status': 200, 'container_count': '1234'} self.memcache.set(cache_key, account_info) + del self.request.environ['swift.infocache'][cache_key] partition, nodes, count = \ - self.controller.account_info(self.account) + self.controller.account_info(self.account, self.request) self.assertEqual(count, 1234) def test_make_requests(self): @@ -407,7 +402,6 @@ class TestController(unittest.TestCase): def test_account_info_no_account(self): with save_globals(): - self.memcache.store = {} set_http_connect(404, 404, 404) partition, nodes, count = \ self.controller.account_info(self.account, self.request) @@ -514,13 +508,13 @@ class TestController(unittest.TestCase): def test_get_account_info_returns_values_as_strings(self): app = mock.MagicMock() - app.memcache = mock.MagicMock() - app.memcache.get = mock.MagicMock() - app.memcache.get.return_value = { + memcache = mock.MagicMock() + memcache.get = mock.MagicMock() + memcache.get.return_value = { u'foo': u'\u2603', u'meta': {u'bar': u'\u2603'}, u'sysmeta': {u'baz': u'\u2603'}} - env = {'PATH_INFO': '/v1/a'} + env = {'PATH_INFO': '/v1/a', 'swift.cache': memcache} ai = get_account_info(env, app) # Test info is returned as strings @@ -539,14 +533,14 @@ class TestController(unittest.TestCase): def test_get_container_info_returns_values_as_strings(self): app = mock.MagicMock() - app.memcache = mock.MagicMock() - app.memcache.get = mock.MagicMock() - app.memcache.get.return_value = { + memcache = mock.MagicMock() + memcache.get = mock.MagicMock() + memcache.get.return_value = { u'foo': u'\u2603', u'meta': {u'bar': u'\u2603'}, u'sysmeta': {u'baz': u'\u2603'}, u'cors': {u'expose_headers': u'\u2603'}} - env = {'PATH_INFO': '/v1/a/c'} + env = {'PATH_INFO': '/v1/a/c', 'swift.cache': memcache} ci = get_container_info(env, app) # Test info is returned as strings @@ -578,7 +572,7 @@ class TestProxyServerConfiguration(unittest.TestCase): def _make_app(self, conf): self.logger.clear() # helper function to instantiate a proxy server instance - return proxy_server.Application(conf, FakeMemcache(), + return proxy_server.Application(conf, container_ring=FakeRing(), account_ring=FakeRing(), logger=self.logger) @@ -634,7 +628,6 @@ class TestProxyServer(unittest.TestCase): def test_get_object_ring(self): baseapp = proxy_server.Application({}, - FakeMemcache(), container_ring=FakeRing(), account_ring=FakeRing()) with patch_policies([ @@ -664,7 +657,7 @@ class TestProxyServer(unittest.TestCase): def get_controller(self, path): raise Exception('this shouldn\'t be caught') - app = MyApp(None, FakeMemcache(), account_ring=FakeRing(), + app = MyApp(None, account_ring=FakeRing(), container_ring=FakeRing()) req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'}) app.update_request(req) @@ -673,7 +666,6 @@ class TestProxyServer(unittest.TestCase): def test_internal_method_request(self): baseapp = proxy_server.Application({}, - FakeMemcache(), container_ring=FakeRing(), account_ring=FakeRing()) resp = baseapp.handle_request( @@ -682,7 +674,6 @@ class TestProxyServer(unittest.TestCase): def test_inexistent_method_request(self): baseapp = proxy_server.Application({}, - FakeMemcache(), container_ring=FakeRing(), account_ring=FakeRing()) resp = baseapp.handle_request( @@ -691,7 +682,6 @@ class TestProxyServer(unittest.TestCase): def test_private_method_request(self): baseapp = proxy_server.Application({}, - FakeMemcache(), container_ring=FakeRing(), account_ring=FakeRing()) baseapp.logger = debug_logger() @@ -722,12 +712,13 @@ class TestProxyServer(unittest.TestCase): # set account info fake_cache = FakeMemcache() fake_cache.store[get_cache_key('a')] = {'status': 200} - app = proxy_server.Application({}, fake_cache, + app = proxy_server.Application({}, container_ring=FakeRing(), account_ring=FakeRing()) # build internal container request container = get_reserved_name('c') - req = Request.blank('/v1/a/%s' % container) + req = Request.blank('/v1/a/%s' % container, + {'swift.cache': fake_cache}) app.update_request(req) # try client request to reserved name @@ -748,7 +739,7 @@ class TestProxyServer(unittest.TestCase): called[0] = True with save_globals(): set_http_connect(200) - app = proxy_server.Application(None, FakeMemcache(), + app = proxy_server.Application(None, account_ring=FakeRing(), container_ring=FakeRing()) req = Request.blank('/v1/a') @@ -763,7 +754,7 @@ class TestProxyServer(unittest.TestCase): def authorize(req): called[0] = True return HTTPUnauthorized(request=req) - app = proxy_server.Application(None, FakeMemcache(), + app = proxy_server.Application(None, account_ring=FakeRing(), container_ring=FakeRing()) req = Request.blank('/v1/a') @@ -776,7 +767,7 @@ class TestProxyServer(unittest.TestCase): swift_dir = mkdtemp() try: baseapp = proxy_server.Application({'swift_dir': swift_dir}, - FakeMemcache(), FakeLogger(), + FakeLogger(), FakeRing(), FakeRing()) resp = baseapp.handle_request( Request.blank('/', environ={'CONTENT_LENGTH': '-1'})) @@ -794,7 +785,7 @@ class TestProxyServer(unittest.TestCase): try: logger = FakeLogger() baseapp = proxy_server.Application({'swift_dir': swift_dir}, - FakeMemcache(), logger, + logger, container_ring=FakeLogger(), account_ring=FakeRing()) baseapp.handle_request( @@ -814,7 +805,7 @@ class TestProxyServer(unittest.TestCase): try: logger = FakeLogger() baseapp = proxy_server.Application({'swift_dir': swift_dir}, - FakeMemcache(), logger, + logger, container_ring=FakeLogger(), account_ring=FakeRing()) baseapp.handle_request( @@ -832,7 +823,6 @@ class TestProxyServer(unittest.TestCase): baseapp = proxy_server.Application({'swift_dir': swift_dir, 'deny_host_headers': 'invalid_host.com'}, - FakeMemcache(), container_ring=FakeLogger(), account_ring=FakeRing()) resp = baseapp.handle_request( @@ -844,7 +834,6 @@ class TestProxyServer(unittest.TestCase): def test_node_timing(self): baseapp = proxy_server.Application({'sorting_method': 'timing'}, - FakeMemcache(), container_ring=FakeRing(), account_ring=FakeRing()) self.assertEqual(baseapp.node_timings, {}) @@ -876,7 +865,6 @@ class TestProxyServer(unittest.TestCase): nodes = list(nodes) conf = dict(conf, policy_config=policy_conf) baseapp = proxy_server.Application(conf, - FakeMemcache(), logger=FakeLogger(), container_ring=FakeRing(), account_ring=FakeRing()) @@ -1070,7 +1058,6 @@ class TestProxyServer(unittest.TestCase): app_conf = {'concurrent_gets': 'on', 'concurrency_timeout': 0} baseapp = proxy_server.Application(app_conf, - FakeMemcache(), container_ring=FakeRing(), account_ring=FakeRing()) self.assertTrue(baseapp.concurrent_gets) @@ -1095,7 +1082,6 @@ class TestProxyServer(unittest.TestCase): # Now lets set the concurrency_timeout app_conf['concurrency_timeout'] = 2 baseapp = proxy_server.Application(app_conf, - FakeMemcache(), container_ring=FakeRing(), account_ring=FakeRing()) self.assertEqual(baseapp.concurrency_timeout, 2) @@ -1106,7 +1092,7 @@ class TestProxyServer(unittest.TestCase): self.assertEqual(resp.body, b'Response from 127.0.0.2') def test_info_defaults(self): - app = proxy_server.Application({}, FakeMemcache(), + app = proxy_server.Application({}, account_ring=FakeRing(), container_ring=FakeRing()) @@ -1121,7 +1107,7 @@ class TestProxyServer(unittest.TestCase): def test_get_info_controller(self): req = Request.blank('/info') - app = proxy_server.Application({}, FakeMemcache(), + app = proxy_server.Application({}, account_ring=FakeRing(), container_ring=FakeRing()) @@ -1138,7 +1124,7 @@ class TestProxyServer(unittest.TestCase): def test_exception_occurred(self): def do_test(additional_info): logger = debug_logger('test') - app = proxy_server.Application({}, FakeMemcache(), + app = proxy_server.Application({}, account_ring=FakeRing(), container_ring=FakeRing(), logger=logger) @@ -1172,7 +1158,7 @@ class TestProxyServer(unittest.TestCase): def test_error_occurred(self): def do_test(msg): logger = debug_logger('test') - app = proxy_server.Application({}, FakeMemcache(), + app = proxy_server.Application({}, account_ring=FakeRing(), container_ring=FakeRing(), logger=logger) @@ -1198,7 +1184,7 @@ class TestProxyServer(unittest.TestCase): def test_error_limit_methods(self): logger = debug_logger('test') - app = proxy_server.Application({}, FakeMemcache(), + app = proxy_server.Application({}, account_ring=FakeRing(), container_ring=FakeRing(), logger=logger) @@ -1260,7 +1246,7 @@ class TestProxyServer(unittest.TestCase): self.assertEqual(4, node_error_count(app, node)) def test_valid_api_version(self): - app = proxy_server.Application({}, FakeMemcache(), + app = proxy_server.Application({}, account_ring=FakeRing(), container_ring=FakeRing()) @@ -1341,7 +1327,7 @@ class TestProxyServerLoading(unittest.TestCase): } for policy in POLICIES: policy.object_ring = FakeRing() - app = proxy_server.Application(conf, FakeMemcache(), debug_logger(), + app = proxy_server.Application(conf, debug_logger(), FakeRing(), FakeRing()) self.assertEqual(app.node_timeout, 2.3) self.assertEqual(app.recoverable_node_timeout, 1.4) @@ -2023,7 +2009,6 @@ class BaseTestObjectController(object): kwargs['raise_exc'] = raise_exc set_http_connect(*statuses, **kwargs) - self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) @@ -2036,7 +2021,6 @@ class BaseTestObjectController(object): # repeat test set_http_connect(*statuses, **kwargs) - self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) @@ -2171,7 +2155,7 @@ class TestReplicatedObjectController( def setUp(self): skip_if_no_xattrs() self.app = proxy_server.Application( - None, FakeMemcache(), + None, logger=debug_logger('proxy-ut'), account_ring=FakeRing(), container_ring=FakeRing()) @@ -3023,7 +3007,6 @@ class TestReplicatedObjectController( req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 0 self.app.update_request(req) - self.app.memcache.store = {} res = controller.PUT(req) self.assertEqual(test_errors, []) self.assertTrue(res.status.startswith('201 '), res.status) @@ -3052,7 +3035,6 @@ class TestReplicatedObjectController( req.content_length = 1 req.body = 'a' self.app.update_request(req) - self.app.memcache.store = {} res = controller.PUT(req) self.assertEqual(test_errors, []) self.assertTrue(res.status.startswith('201 ')) @@ -3070,7 +3052,7 @@ class TestReplicatedObjectController( # not be used in ring order is if affinity is respected. with mock.patch('swift.proxy.server.shuffle', lambda x: x): app = proxy_server.Application( - conf, FakeMemcache(), + conf, logger=debug_logger('proxy-ut'), account_ring=FakeRing(), container_ring=FakeRing()) @@ -3086,7 +3068,6 @@ class TestReplicatedObjectController( req = Request.blank( '/v1/a/c/o.jpg', method='PUT', body='a', headers={'X-Backend-Storage-Policy-Index': str(policy)}) - app.memcache.store = {} res = controller.PUT(req) self.assertTrue(res.status.startswith('201 ')) self.assertEqual(3, len(written_to)) @@ -3155,7 +3136,6 @@ class TestReplicatedObjectController( req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 1 req.body = 'a' - self.app.memcache.store = {} res = controller.PUT(req) self.assertTrue(res.status.startswith('201 ')) @@ -3554,7 +3534,7 @@ class TestReplicatedObjectController( with open(os.path.join(swift_dir, 'mime.types'), 'w') as fp: fp.write('foo/bar foo\n') proxy_server.Application({'swift_dir': swift_dir}, - FakeMemcache(), FakeLogger(), + FakeLogger(), FakeRing(), FakeRing()) self.assertEqual(proxy_server.mimetypes.guess_type('blah.foo')[0], 'foo/bar') @@ -3573,7 +3553,6 @@ class TestReplicatedObjectController( req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 0 self.app.update_request(req) - self.app.memcache.store = {} res = controller.PUT(req) expected = str(expected) self.assertEqual(res.status[:len(expected)], expected) @@ -3590,7 +3569,6 @@ class TestReplicatedObjectController( def test_status_map(statuses, expected): set_http_connect(*statuses) - self.app.memcache.store = {} req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 0 self.app.update_request(req) @@ -3620,7 +3598,6 @@ class TestReplicatedObjectController( self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): - self.app.memcache.store = {} set_http_connect(*statuses) req = Request.blank('/v1/a/c/o.jpg', environ={'REQUEST_METHOD': 'PUT'}, @@ -3666,7 +3643,6 @@ class TestReplicatedObjectController( self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): - self.app.memcache.store = {} set_http_connect(*statuses) req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 0 @@ -3686,7 +3662,6 @@ class TestReplicatedObjectController( with save_globals(): def test_status_map(statuses, expected): set_http_connect(*statuses) - self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {}, method='POST', headers={'Content-Type': 'foo/bar'}) self.app.update_request(req) @@ -3711,7 +3686,6 @@ class TestReplicatedObjectController( self.app.sort_nodes = lambda nodes, *args, **kwargs: nodes def do_test(resp_headers): - self.app.memcache.store = {} backend_requests = [] def capture_requests(ip, port, method, path, headers, *args, @@ -3779,7 +3753,6 @@ class TestReplicatedObjectController( self.assertEqual(container_headers, expected) # and again with policy override - self.app.memcache.store = {} backend_requests = [] req = Request.blank('/v1/a/c/o', {}, method='POST', headers={'X-Object-Meta-Color': 'Blue', @@ -3823,7 +3796,6 @@ class TestReplicatedObjectController( self.app.recheck_updating_shard_ranges = 0 def do_test(method, sharding_state): - self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {}, method=method, body='', headers={'Content-Type': 'text/plain'}) @@ -3918,9 +3890,9 @@ class TestReplicatedObjectController( self.app.recheck_updating_shard_ranges = 3600 def do_test(method, sharding_state): - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', {}, method=method, body='', - headers={'Content-Type': 'text/plain'}) + req = Request.blank( + '/v1/a/c/o', {'swift.cache': FakeMemcache()}, + method=method, body='', headers={'Content-Type': 'text/plain'}) # we want the container_info response to say policy index of 1 and # sharding state @@ -3977,8 +3949,8 @@ class TestReplicatedObjectController( headers={'X-Backend-Record-Type': 'shard'}) cache_key = 'shard-updating/a/c' - self.assertIn(cache_key, self.app.memcache.store) - self.assertEqual(self.app.memcache.store[cache_key], + self.assertIn(cache_key, req.environ['swift.cache'].store) + self.assertEqual(req.environ['swift.cache'].store[cache_key], [dict(sr) for sr in shard_ranges]) self.assertIn(cache_key, req.environ.get('swift.infocache')) self.assertEqual(req.environ['swift.infocache'][cache_key], @@ -4037,9 +4009,11 @@ class TestReplicatedObjectController( utils.ShardRange( '.shards_a/c_nope', utils.Timestamp.now(), 'u', ''), ] - self.app.memcache.store = {'shard-updating/a/c': tuple( - dict(shard_range) for shard_range in shard_ranges)} - req = Request.blank('/v1/a/c/o', {}, method=method, body='', + cache = FakeMemcache() + cache.set('shard-updating/a/c', tuple( + dict(shard_range) for shard_range in shard_ranges)) + req = Request.blank('/v1/a/c/o', {'swift.cache': cache}, + method=method, body='', headers={'Content-Type': 'text/plain'}) # we want the container_info response to say policy index of 1 and @@ -4122,7 +4096,6 @@ class TestReplicatedObjectController( with save_globals(): def test_status_map(statuses, expected): set_http_connect(*statuses) - self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'DELETE'}) self.app.update_request(req) res = req.get_response(self.app) @@ -4139,7 +4112,6 @@ class TestReplicatedObjectController( with save_globals(): def test_status_map(statuses, expected): set_http_connect(*statuses) - self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) res = req.get_response(self.app) @@ -4170,7 +4142,6 @@ class TestReplicatedObjectController( def test_status_map(statuses, expected, timestamps, expected_timestamp): set_http_connect(*statuses, timestamps=timestamps) - self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}, headers={'x-newest': 'true'}) self.app.update_request(req) @@ -4201,7 +4172,6 @@ class TestReplicatedObjectController( def test_status_map(statuses, expected, timestamps, expected_timestamp): set_http_connect(*statuses, timestamps=timestamps) - self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'GET'}, headers={'x-newest': 'true'}) self.app.update_request(req) @@ -4228,7 +4198,6 @@ class TestReplicatedObjectController( def test_status_map(statuses, expected, timestamps, expected_timestamp): set_http_connect(*statuses, timestamps=timestamps) - self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) res = req.get_response(self.app) @@ -4410,6 +4379,7 @@ class TestReplicatedObjectController( req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': FakeMemcache(), 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) @@ -4419,11 +4389,8 @@ class TestReplicatedObjectController( resp = req.get_response(self.app) self.assertEqual(resp.status_int, 201) self.app.client_timeout = 0.05 - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'PUT', - 'wsgi.input': SlowBody()}, - headers={'Content-Length': '4', - 'Content-Type': 'text/plain'}) + + req.environ['wsgi.input'] = SlowBody() # Need a fresh instance self.app.update_request(req) set_http_connect(201, 201, 201) # obj obj obj @@ -4537,7 +4504,8 @@ class TestReplicatedObjectController( for dev in object_ring.devs: dev['ip'] = '127.0.0.1' dev['port'] = 1 - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + req = Request.blank('/v1/a/c/o', environ={ + 'REQUEST_METHOD': 'GET', 'swift.cache': FakeMemcache()}) self.app.update_request(req) self.app.recoverable_node_timeout = 0.1 @@ -4561,7 +4529,6 @@ class TestReplicatedObjectController( resp = req.get_response(self.app) self.assertEqual(resp.body, b'lalala') - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) set_http_connect(200, 200, 200, body=b'lalala', slow=[1.0, 1.0], etags=['a', 'b', 'b']) resp = req.get_response(self.app) @@ -4605,7 +4572,6 @@ class TestReplicatedObjectController( def test_node_request_setting(self): baseapp = proxy_server.Application({'request_node_count': '3'}, - FakeMemcache(), container_ring=FakeRing(), account_ring=FakeRing()) self.assertEqual(baseapp.request_node_count(3), 3) @@ -4939,7 +4905,6 @@ class TestReplicatedObjectController( def test_acc_or_con_missing_returns_404(self): with save_globals(): - self.app.memcache = FakeMemcacheReturnsNone() self.app._error_limiting = {} controller = ReplicatedObjectController( self.app, 'account', 'container', 'object') @@ -5036,7 +5001,6 @@ class TestReplicatedObjectController( def test_PUT_POST_requires_container_exist(self): with save_globals(): - self.app.memcache = FakeMemcacheReturnsNone() controller = ReplicatedObjectController( self.app, 'account', 'container', 'object') @@ -5194,17 +5158,21 @@ class TestReplicatedObjectController( with save_globals(): controller = ReplicatedObjectController( self.app, 'account', 'container', 'object') + cache = FakeMemcache() set_http_connect(200, 200, 201, 201, 201) # acct cont obj obj obj - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0'}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, + headers={'Content-Length': '0'}) self.app.update_request(req) resp = controller.PUT(req) self.assertEqual(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank( - '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, headers={'Content-Length': '0', 'X-Object-Meta-' + ( 'a' * constraints.MAX_META_NAME_LENGTH): 'v'}) @@ -5213,7 +5181,8 @@ class TestReplicatedObjectController( self.assertEqual(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank( - '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, headers={ 'Content-Length': '0', 'X-Object-Meta-' + ( @@ -5223,16 +5192,19 @@ class TestReplicatedObjectController( self.assertEqual(resp.status_int, 400) set_http_connect(201, 201, 201) - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Object-Meta-Too-Long': 'a' * - constraints.MAX_META_VALUE_LENGTH}) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, + headers={'Content-Length': '0', + 'X-Object-Meta-Too-Long': 'a' * + constraints.MAX_META_VALUE_LENGTH}) self.app.update_request(req) resp = controller.PUT(req) self.assertEqual(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank( - '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, headers={'Content-Length': '0', 'X-Object-Meta-Too-Long': 'a' * (constraints.MAX_META_VALUE_LENGTH + 1)}) @@ -5244,8 +5216,10 @@ class TestReplicatedObjectController( headers = {'Content-Length': '0'} for x in range(constraints.MAX_META_COUNT): headers['X-Object-Meta-%d' % x] = 'v' - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers=headers) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, + headers=headers) self.app.update_request(req) resp = controller.PUT(req) self.assertEqual(resp.status_int, 201) @@ -5253,8 +5227,10 @@ class TestReplicatedObjectController( headers = {'Content-Length': '0'} for x in range(constraints.MAX_META_COUNT + 1): headers['X-Object-Meta-%d' % x] = 'v' - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers=headers) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, + headers=headers) self.app.update_request(req) resp = controller.PUT(req) self.assertEqual(resp.status_int, 400) @@ -5272,16 +5248,20 @@ class TestReplicatedObjectController( if constraints.MAX_META_OVERALL_SIZE - size > 1: headers['X-Object-Meta-a'] = \ 'a' * (constraints.MAX_META_OVERALL_SIZE - size - 1) - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers=headers) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, + headers=headers) self.app.update_request(req) resp = controller.PUT(req) self.assertEqual(resp.status_int, 201) set_http_connect(201, 201, 201) headers['X-Object-Meta-a'] = \ 'a' * (constraints.MAX_META_OVERALL_SIZE - size) - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers=headers) + req = Request.blank( + '/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, + headers=headers) self.app.update_request(req) resp = controller.PUT(req) self.assertEqual(resp.status_int, 400) @@ -5292,7 +5272,6 @@ class TestReplicatedObjectController( controller = ReplicatedObjectController( self.app, account, container, obj) self.app.update_request(req) - self.app.memcache.store = {} with save_globals(): new_connect = set_http_connect(*args, **kwargs) yield controller @@ -5702,7 +5681,7 @@ class TestReplicatedObjectController( # not be used in ring order is if affinity is respected. with mock.patch('swift.proxy.server.shuffle', lambda x: x): app = proxy_server.Application( - conf, FakeMemcache(), + conf, logger=debug_logger('proxy-ut'), account_ring=FakeRing(), container_ring=FakeRing()) @@ -5718,7 +5697,6 @@ class TestReplicatedObjectController( req = Request.blank( '/v1/a/c/o.jpg', headers={'X-Backend-Storage-Policy-Index': str(policy)}) - app.memcache.store = {} res = controller.GET(req) self.assertTrue(res.status.startswith('200 ')) self.assertEqual(3, len(actual_nodes)) @@ -5818,7 +5796,6 @@ class TestReplicatedObjectController( controller = ReplicatedObjectController( self.app, 'account', 'container', 'object') set_http_connect(200, 200, 202, 202, 202) - self.app.memcache.store = {} orig_time = time.time try: t = time.time() @@ -8089,7 +8066,7 @@ class TestObjectECRangedGET(unittest.TestCase): def setUp(self): _test_servers[0].logger._clear() self.app = proxy_server.Application( - None, FakeMemcache(), + None, logger=debug_logger('proxy-ut'), account_ring=FakeRing(), container_ring=FakeRing()) @@ -8674,7 +8651,7 @@ class TestContainerController(unittest.TestCase): def setUp(self): self.app = proxy_server.Application( - None, FakeMemcache(), + None, account_ring=FakeRing(), container_ring=FakeRing(base_port=2000), logger=debug_logger()) @@ -8774,14 +8751,12 @@ class TestContainerController(unittest.TestCase): if raise_exc: kwargs['raise_exc'] = raise_exc set_http_connect(*statuses, **kwargs) - self.app.memcache.store = {} req = Request.blank('/v1/a/c', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) self.app.update_request(req) res = method(req) self.assertEqual(res.status_int, expected) set_http_connect(*statuses, **kwargs) - self.app.memcache.store = {} req = Request.blank('/v1/a/c/', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) self.app.update_request(req) @@ -8795,7 +8770,6 @@ class TestContainerController(unittest.TestCase): def test_status_map(statuses, expected, c_expected=None, a_expected=None, **kwargs): set_http_connect(*statuses, **kwargs) - self.app.memcache.store = {} req = Request.blank('/v1/a/c', {}) self.app.update_request(req) res = controller.HEAD(req) @@ -8823,7 +8797,6 @@ class TestContainerController(unittest.TestCase): self.assertNotIn('account/a', res.environ) set_http_connect(*statuses, **kwargs) - self.app.memcache.store = {} req = Request.blank('/v1/a/c', {}) self.app.update_request(req) res = controller.GET(req) @@ -8882,7 +8855,6 @@ class TestContainerController(unittest.TestCase): with save_globals(): mock_conn = set_http_connect(200, 201, 201, 201, give_connect=capture_requests) - self.app.memcache.store = {} req = Request.blank('/v1/a/test', method='PUT', headers={'Content-Length': 0}) if requested_policy: @@ -8934,7 +8906,6 @@ class TestContainerController(unittest.TestCase): def test_status_map(statuses, expected, **kwargs): set_http_connect(*statuses, **kwargs) - self.app.memcache.store = {} req = Request.blank('/v1/a/c', {}) req.content_length = 0 self.app.update_request(req) @@ -8994,7 +8965,6 @@ class TestContainerController(unittest.TestCase): def test_status_map(statuses, expected, headers=None, **kwargs): set_http_connect(*statuses, **kwargs) - self.app.memcache.store = {} req = Request.blank('/v1/a/c', {}, headers=headers) req.content_length = 0 self.app.update_request(req) @@ -9033,7 +9003,6 @@ class TestContainerController(unittest.TestCase): def test_status_map(statuses, expected, **kwargs): set_http_connect(*statuses, **kwargs) - self.app.memcache.store = {} req = Request.blank('/v1/a/c', {}) req.content_length = 0 self.app.update_request(req) @@ -9126,7 +9095,6 @@ class TestContainerController(unittest.TestCase): def test_acc_missing_returns_404(self): for meth in ('DELETE', 'PUT'): with save_globals(): - self.app.memcache = FakeMemcacheReturnsNone() self.app._error_limiting = {} controller = proxy_server.ContainerController(self.app, 'account', @@ -9136,7 +9104,6 @@ class TestContainerController(unittest.TestCase): missing_container=True) else: set_http_connect(200, 200, 200, 200) - self.app.memcache.store = {} req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': meth}) self.app.update_request(req) @@ -9251,7 +9218,6 @@ class TestContainerController(unittest.TestCase): self.assert_status_map(controller.DELETE, (200, 204, 503, 404), 503) - self.app.memcache = FakeMemcacheReturnsNone() # 200: Account check, 404x3: Container check self.assert_status_map(controller.DELETE, (200, 404, 404, 404), 404) @@ -9556,7 +9522,7 @@ class TestContainerController(unittest.TestCase): def _do_test(method): with save_globals(): swift.proxy.controllers.Controller.account_info = account_info - app = proxy_server.Application(None, FakeMemcache(), + app = proxy_server.Application(None, account_ring=FakeRing(), container_ring=FakeRing()) set_http_connect(201, 201, 201) @@ -9595,7 +9561,7 @@ class TestContainerController(unittest.TestCase): def _do_test(method): with save_globals(): swift.proxy.controllers.Controller.account_info = account_info - app = proxy_server.Application(None, FakeMemcache(), + app = proxy_server.Application(None, account_ring=FakeRing(), container_ring=FakeRing()) set_http_connect(201, 201, 201) @@ -9982,7 +9948,7 @@ class TestAccountController(unittest.TestCase): def setUp(self): conf = {'error_suppression_interval': 0} - self.app = proxy_server.Application(conf, FakeMemcache(), + self.app = proxy_server.Application(conf, account_ring=FakeRing(), container_ring=FakeRing()) @@ -10065,13 +10031,11 @@ class TestAccountController(unittest.TestCase): self.assert_status_map(controller.GET, (404, 404, 503), 404, 404) self.assert_status_map(controller.GET, (404, 503, 503), 503) - self.app.memcache = FakeMemcacheReturnsNone() self.assert_status_map(controller.GET, (404, 404, 404), 404, 404) def test_GET_autocreate(self): with save_globals(): controller = proxy_server.AccountController(self.app, 'a') - self.app.memcache = FakeMemcacheReturnsNone() self.assertFalse(self.app.account_autocreate) # Repeat the test for autocreate = False and 404 by all self.assert_status_map(controller.GET, @@ -10138,7 +10102,6 @@ class TestAccountController(unittest.TestCase): # Same behaviour as GET with save_globals(): controller = proxy_server.AccountController(self.app, 'a') - self.app.memcache = FakeMemcacheReturnsNone() self.assertFalse(self.app.account_autocreate) self.assert_status_map(controller.HEAD, (404, 404, 404), 404) @@ -10154,7 +10117,6 @@ class TestAccountController(unittest.TestCase): def test_POST_autocreate(self): with save_globals(): controller = proxy_server.AccountController(self.app, 'a') - self.app.memcache = FakeMemcacheReturnsNone() # first test with autocreate being False self.assertFalse(self.app.account_autocreate) self.assert_status_map(controller.POST, @@ -10176,7 +10138,6 @@ class TestAccountController(unittest.TestCase): def test_POST_autocreate_with_sysmeta(self): with save_globals(): controller = proxy_server.AccountController(self.app, 'a') - self.app.memcache = FakeMemcacheReturnsNone() # first test with autocreate being False self.assertFalse(self.app.account_autocreate) self.assert_status_map(controller.POST, @@ -10249,7 +10210,6 @@ class TestAccountController(unittest.TestCase): def test_status_map(statuses, expected, **kwargs): set_http_connect(*statuses, **kwargs) - self.app.memcache.store = {} req = Request.blank('/v1/a', {}) req.content_length = 0 self.app.update_request(req) @@ -10452,7 +10412,6 @@ class TestAccountController(unittest.TestCase): def test_status_map(statuses, expected, **kwargs): set_http_connect(*statuses, **kwargs) - self.app.memcache.store = {} req = Request.blank('/v1/a', {'REQUEST_METHOD': 'DELETE'}) req.content_length = 0 self.app.update_request(req) @@ -10475,7 +10434,6 @@ class TestAccountController(unittest.TestCase): def test_status_map(statuses, expected, **kwargs): set_http_connect(*statuses, **kwargs) - self.app.memcache.store = {} req = Request.blank('/v1/a?whoops', environ={'REQUEST_METHOD': 'DELETE'}) req.content_length = 0 @@ -10500,11 +10458,10 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase): def setUp(self): conf = {'account_autocreate': 'yes'} self.app = listing_formats.ListingFilter( - proxy_server.Application(conf, FakeMemcache(), + proxy_server.Application(conf, account_ring=FakeRing(), container_ring=FakeRing()), {}) - self.app.app.memcache = FakeMemcacheReturnsNone() def test_GET_autocreate_accept_json(self): with save_globals(): @@ -10611,7 +10568,7 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase): version=2, acl_dict=acl)} app = proxy_server.Application( - None, FakeMemcache(), account_ring=FakeRing(), + None, account_ring=FakeRing(), container_ring=FakeRing()) with save_globals(): @@ -10686,7 +10643,7 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase): resps_to_send.append(canned_resp) app = proxy_server.Application( - None, FakeMemcache(), account_ring=FakeRing(), + None, account_ring=FakeRing(), container_ring=FakeRing()) app.allow_account_management = True @@ -10854,7 +10811,7 @@ class TestSwiftInfo(unittest.TestCase): utils._swift_admin_info = {} def test_registered_defaults(self): - app = proxy_server.Application({}, FakeMemcache(), + app = proxy_server.Application({}, account_ring=FakeRing(), container_ring=FakeRing()) req = Request.blank('/info') diff --git a/test/unit/proxy/test_sysmeta.py b/test/unit/proxy/test_sysmeta.py index 3570bb105c..7ec722237f 100644 --- a/test/unit/proxy/test_sysmeta.py +++ b/test/unit/proxy/test_sysmeta.py @@ -132,7 +132,7 @@ class TestObjectSysmeta(unittest.TestCase): def setUp(self): skip_if_no_xattrs() - self.app = proxy.Application(None, FakeMemcache(), + self.app = proxy.Application(None, logger=debug_logger('proxy-ut'), account_ring=FakeRing(replicas=1), container_ring=FakeRing(replicas=1)) @@ -227,8 +227,9 @@ class TestObjectSysmeta(unittest.TestCase): def test_sysmeta_replaced_by_PUT(self): path = '/v1/a/c/o' + cache = FakeMemcache() - env = {'REQUEST_METHOD': 'PUT'} + env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache} hdrs = dict(self.original_sysmeta_headers_1) hdrs.update(self.original_sysmeta_headers_2) hdrs.update(self.original_meta_headers_1) @@ -237,7 +238,7 @@ class TestObjectSysmeta(unittest.TestCase): resp = req.get_response(self.app) self._assertStatus(resp, 201) - env = {'REQUEST_METHOD': 'PUT'} + env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache} hdrs = dict(self.changed_sysmeta_headers) hdrs.update(self.new_sysmeta_headers) hdrs.update(self.changed_meta_headers) @@ -247,7 +248,7 @@ class TestObjectSysmeta(unittest.TestCase): resp = req.get_response(self.app) self._assertStatus(resp, 201) - req = Request.blank(path, environ={}) + req = Request.blank(path, environ={'swift.cache': cache}) resp = req.get_response(self.app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.changed_sysmeta_headers) @@ -257,29 +258,30 @@ class TestObjectSysmeta(unittest.TestCase): self._assertInHeaders(resp, self.new_meta_headers) self._assertNotInHeaders(resp, self.original_meta_headers_2) - def _test_sysmeta_not_updated_by_POST(self, app): + def test_sysmeta_not_updated_by_POST(self): # check sysmeta is not changed by a POST but user meta is replaced path = '/v1/a/c/o' + cache = FakeMemcache() - env = {'REQUEST_METHOD': 'PUT'} + env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache} hdrs = dict(self.original_sysmeta_headers_1) hdrs.update(self.original_meta_headers_1) req = Request.blank(path, environ=env, headers=hdrs, body=b'x') - resp = req.get_response(app) + resp = req.get_response(self.app) self._assertStatus(resp, 201) - env = {'REQUEST_METHOD': 'POST'} + env = {'REQUEST_METHOD': 'POST', 'swift.cache': cache} hdrs = dict(self.changed_sysmeta_headers) hdrs.update(self.new_sysmeta_headers) hdrs.update(self.changed_meta_headers) hdrs.update(self.new_meta_headers) hdrs.update(self.bad_headers) req = Request.blank(path, environ=env, headers=hdrs) - resp = req.get_response(app) + resp = req.get_response(self.app) self._assertStatus(resp, 202) - req = Request.blank(path, environ={}) - resp = req.get_response(app) + req = Request.blank(path, environ={'swift.cache': cache}) + resp = req.get_response(self.app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.original_sysmeta_headers_1) self._assertNotInHeaders(resp, self.new_sysmeta_headers) @@ -287,31 +289,28 @@ class TestObjectSysmeta(unittest.TestCase): self._assertInHeaders(resp, self.new_meta_headers) self._assertNotInHeaders(resp, self.bad_headers) - env = {'REQUEST_METHOD': 'PUT'} + env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache} hdrs = dict(self.changed_sysmeta_headers) hdrs.update(self.new_sysmeta_headers) hdrs.update(self.bad_headers) req = Request.blank(path, environ=env, headers=hdrs, body=b'x') - resp = req.get_response(app) + resp = req.get_response(self.app) self._assertStatus(resp, 201) - req = Request.blank(path, environ={}) - resp = req.get_response(app) + req = Request.blank(path, environ={'swift.cache': cache}) + resp = req.get_response(self.app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.changed_sysmeta_headers) self._assertInHeaders(resp, self.new_sysmeta_headers) self._assertNotInHeaders(resp, self.original_sysmeta_headers_2) - def test_sysmeta_not_updated_by_POST(self): - # test fast-post by issuing requests to the proxy app - self._test_sysmeta_not_updated_by_POST(self.app) - def test_sysmeta_updated_by_COPY(self): # check sysmeta is updated by a COPY in same way as user meta by # issuing requests to the copy middleware app path = '/v1/a/c/o' dest = '/c/o2' - env = {'REQUEST_METHOD': 'PUT'} + cache = FakeMemcache() + env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache} hdrs = dict(self.original_sysmeta_headers_1) hdrs.update(self.original_sysmeta_headers_2) hdrs.update(self.original_meta_headers_1) @@ -322,7 +321,7 @@ class TestObjectSysmeta(unittest.TestCase): resp = req.get_response(self.copy_app) self._assertStatus(resp, 201) - env = {'REQUEST_METHOD': 'COPY'} + env = {'REQUEST_METHOD': 'COPY', 'swift.cache': cache} hdrs = dict(self.changed_sysmeta_headers) hdrs.update(self.new_sysmeta_headers) hdrs.update(self.changed_meta_headers) @@ -345,7 +344,7 @@ class TestObjectSysmeta(unittest.TestCase): self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2) self._assertNotInHeaders(resp, self.bad_headers) - req = Request.blank('/v1/a/c/o2', environ={}) + req = Request.blank('/v1/a/c/o2', environ={'swift.cache': cache}) resp = req.get_response(self.copy_app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.changed_sysmeta_headers) @@ -363,7 +362,8 @@ class TestObjectSysmeta(unittest.TestCase): # check sysmeta is updated by a PUT with x-copy-from in same way as # user meta by issuing requests to the copy middleware app path = '/v1/a/c/o' - env = {'REQUEST_METHOD': 'PUT'} + cache = FakeMemcache() + env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache} hdrs = dict(self.original_sysmeta_headers_1) hdrs.update(self.original_sysmeta_headers_2) hdrs.update(self.original_meta_headers_1) @@ -372,7 +372,7 @@ class TestObjectSysmeta(unittest.TestCase): resp = req.get_response(self.copy_app) self._assertStatus(resp, 201) - env = {'REQUEST_METHOD': 'PUT'} + env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache} hdrs = dict(self.changed_sysmeta_headers) hdrs.update(self.new_sysmeta_headers) hdrs.update(self.changed_meta_headers) @@ -390,7 +390,7 @@ class TestObjectSysmeta(unittest.TestCase): self._assertInHeaders(resp, self.original_meta_headers_2) self._assertNotInHeaders(resp, self.bad_headers) - req = Request.blank('/v1/a/c/o2', environ={}) + req = Request.blank('/v1/a/c/o2', environ={'swift.cache': cache}) resp = req.get_response(self.copy_app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.changed_sysmeta_headers) @@ -401,40 +401,41 @@ class TestObjectSysmeta(unittest.TestCase): self._assertInHeaders(resp, self.original_meta_headers_2) self._assertNotInHeaders(resp, self.bad_headers) - def _test_transient_sysmeta_replaced_by_PUT_or_POST(self, app): + def test_transient_sysmeta_replaced_by_PUT_or_POST(self): # check transient_sysmeta is replaced en-masse by a POST path = '/v1/a/c/o' + cache = FakeMemcache() - env = {'REQUEST_METHOD': 'PUT'} + env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache} hdrs = dict(self.original_transient_sysmeta_headers_1) hdrs.update(self.original_transient_sysmeta_headers_2) hdrs.update(self.original_meta_headers_1) req = Request.blank(path, environ=env, headers=hdrs, body=b'x') - resp = req.get_response(app) + resp = req.get_response(self.app) self._assertStatus(resp, 201) req = Request.blank(path, environ={}) - resp = req.get_response(app) + resp = req.get_response(self.app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.original_transient_sysmeta_headers_1) self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2) self._assertInHeaders(resp, self.original_meta_headers_1) - info = get_object_info(req.environ, app) + info = get_object_info(req.environ, self.app) self.assertEqual(2, len(info.get('transient_sysmeta', ()))) self.assertEqual({'testa': 'A', 'testb': 'B'}, info['transient_sysmeta']) # POST will replace all existing transient_sysmeta and usermeta values - env = {'REQUEST_METHOD': 'POST'} + env = {'REQUEST_METHOD': 'POST', 'swift.cache': cache} hdrs = dict(self.changed_transient_sysmeta_headers) hdrs.update(self.new_transient_sysmeta_headers_1) req = Request.blank(path, environ=env, headers=hdrs) - resp = req.get_response(app) + resp = req.get_response(self.app) self._assertStatus(resp, 202) req = Request.blank(path, environ={}) - resp = req.get_response(app) + resp = req.get_response(self.app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.changed_transient_sysmeta_headers) self._assertInHeaders(resp, self.new_transient_sysmeta_headers_1) @@ -442,21 +443,21 @@ class TestObjectSysmeta(unittest.TestCase): self._assertNotInHeaders(resp, self.original_transient_sysmeta_headers_2) - info = get_object_info(req.environ, app) + info = get_object_info(req.environ, self.app) self.assertEqual(2, len(info.get('transient_sysmeta', ()))) self.assertEqual({'testa': 'changed_A', 'testc': 'C'}, info['transient_sysmeta']) # subsequent PUT replaces all transient_sysmeta and usermeta values - env = {'REQUEST_METHOD': 'PUT'} + env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache} hdrs = dict(self.new_transient_sysmeta_headers_2) hdrs.update(self.original_meta_headers_2) req = Request.blank(path, environ=env, headers=hdrs, body=b'x') - resp = req.get_response(app) + resp = req.get_response(self.app) self._assertStatus(resp, 201) - req = Request.blank(path, environ={}) - resp = req.get_response(app) + req = Request.blank(path, environ={'swift.cache': cache}) + resp = req.get_response(self.app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.original_meta_headers_2) self._assertInHeaders(resp, self.new_transient_sysmeta_headers_2) @@ -470,9 +471,6 @@ class TestObjectSysmeta(unittest.TestCase): self._assertNotInHeaders(resp, self.original_transient_sysmeta_headers_2) - info = get_object_info(req.environ, app) + info = get_object_info(req.environ, self.app) self.assertEqual(1, len(info.get('transient_sysmeta', ()))) self.assertEqual({'testd': 'D'}, info['transient_sysmeta']) - - def test_transient_sysmeta_replaced_by_PUT_or_POST(self): - self._test_transient_sysmeta_replaced_by_PUT_or_POST(self.app)