openstack/swift
Code Issues Proposed changes

Merge "Get rid of memcache attr on proxy-server app"

Browse Source
This commit is contained in:
Zuul 2020-07-14 18:53:24 +00:00 committed by Gerrit Code Review
commit d287854c8a
9 changed files with 171 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
swift/proxy/controllers/base.py
View File

@ -45,7 +45,8 @@ from swift.common.wsgi import make_pre_authed_env, make_pre_authed_request
from swift.common.utils import Timestamp, WatchdogTimeout, config_true_value, \ from swift.common.utils import Timestamp, WatchdogTimeout, config_true_value, \
public, split_path, list_from_csv, GreenthreadSafeIterator, \ public, split_path, list_from_csv, GreenthreadSafeIterator, \
GreenAsyncPile, quorum_size, parse_content_type, drain_and_close, \ GreenAsyncPile, quorum_size, parse_content_type, drain_and_close, \
document_iters_to_http_response_body, ShardRange, find_shard_range document_iters_to_http_response_body, ShardRange, find_shard_range, \
cache_from_env
from swift.common.bufferedhttp import http_connect from swift.common.bufferedhttp import http_connect
from swift.common import constraints from swift.common import constraints
from swift.common.exceptions import ChunkReadTimeout, ChunkWriteTimeout, \ from swift.common.exceptions import ChunkReadTimeout, ChunkWriteTimeout, \
@ -541,8 +542,7 @@ def set_info_cache(app, env, account, container, resp):
""" """
cache_key = get_cache_key(account, container) cache_key = get_cache_key(account, container)
infocache = env.setdefault('swift.infocache', {}) infocache = env.setdefault('swift.infocache', {})
memcache = getattr(app, 'memcache', None) or env.get('swift.cache') memcache = cache_from_env(env, True)
if resp is None: if resp is None:
infocache.pop(cache_key, None) infocache.pop(cache_key, None)
if memcache: if memcache:
@ -645,7 +645,7 @@ def _get_info_from_memcache(app, env, account, container=None):
returns None if memcache is not in use. returns None if memcache is not in use.
""" """
cache_key = get_cache_key(account, container) cache_key = get_cache_key(account, container)
memcache = getattr(app, 'memcache', None) or env.get('swift.cache') memcache = cache_from_env(env, True)
if memcache: if memcache:
info = memcache.get(cache_key) info = memcache.get(cache_key)
if info and six.PY2: if info and six.PY2:
@ -1678,12 +1678,12 @@ class Controller(object):
headers['referer'] = referer headers['referer'] = referer
return headers return headers
def account_info(self, account, req=None): def account_info(self, account, req):
""" """
Get account information, and also verify that the account exists. Get account information, and also verify that the account exists.
:param account: native str name of the account to get the info for :param account: native str name of the account to get the info for
:param req: caller's HTTP request context object (optional) :param req: caller's HTTP request context object
:returns: tuple of (account partition, account nodes, container_count) :returns: tuple of (account partition, account nodes, container_count)
or (None, None, None) if it does not exist or (None, None, None) if it does not exist
""" """
@ -1704,14 +1704,14 @@ class Controller(object):
partition, nodes = self.app.account_ring.get_nodes(account) partition, nodes = self.app.account_ring.get_nodes(account)
return partition, nodes, container_count return partition, nodes, container_count
def container_info(self, account, container, req=None): def container_info(self, account, container, req):
""" """
Get container information and thusly verify container existence. Get container information and thusly verify container existence.
This will also verify account existence. This will also verify account existence.
:param account: native-str account name for the container :param account: native-str account name for the container
:param container: native-str container name to look up :param container: native-str container name to look up
:param req: caller's HTTP request context object (optional) :param req: caller's HTTP request context object
:returns: dict containing at least container partition ('partition'), :returns: dict containing at least container partition ('partition'),
container nodes ('containers'), container read container nodes ('containers'), container read
acl ('read_acl'), container write acl ('write_acl'), acl ('read_acl'), container write acl ('write_acl'),
@ -2268,8 +2268,7 @@ class Controller(object):
cache_key = get_cache_key(account, container, shard='updating') cache_key = get_cache_key(account, container, shard='updating')
infocache = req.environ.setdefault('swift.infocache', {}) infocache = req.environ.setdefault('swift.infocache', {})
memcache = getattr(self.app, 'memcache', None) or req.environ.get( memcache = cache_from_env(req.environ, True)
'swift.cache')
cached_ranges = infocache.get(cache_key) cached_ranges = infocache.get(cache_key)
if cached_ranges is None and memcache: if cached_ranges is None and memcache:

7
swift/proxy/server.py
View File

@ -32,7 +32,7 @@ from swift.common import constraints
from swift.common.http import is_server_error from swift.common.http import is_server_error
from swift.common.storage_policy import POLICIES from swift.common.storage_policy import POLICIES
from swift.common.ring import Ring from swift.common.ring import Ring
from swift.common.utils import Watchdog, cache_from_env, get_logger, \ from swift.common.utils import Watchdog, get_logger, \
get_remote_client, split_path, config_true_value, generate_trans_id, \ get_remote_client, split_path, config_true_value, generate_trans_id, \
affinity_key_function, affinity_locality_predicate, list_from_csv, \ affinity_key_function, affinity_locality_predicate, list_from_csv, \
register_swift_info, readconf, config_auto_int_value register_swift_info, readconf, config_auto_int_value
@ -170,7 +170,7 @@ class ProxyOverrideOptions(object):
class Application(object): class Application(object):
"""WSGI application for the proxy server.""" """WSGI application for the proxy server."""
def __init__(self, conf, memcache=None, logger=None, account_ring=None, def __init__(self, conf, logger=None, account_ring=None,
container_ring=None): container_ring=None):
if conf is None: if conf is None:
conf = {} conf = {}
@ -218,7 +218,6 @@ class Application(object):
for policy in POLICIES: for policy in POLICIES:
policy.load_ring(swift_dir) policy.load_ring(swift_dir)
self.obj_controller_router = ObjectControllerRouter() self.obj_controller_router = ObjectControllerRouter()
self.memcache = memcache
mimetypes.init(mimetypes.knownfiles + mimetypes.init(mimetypes.knownfiles +
[os.path.join(swift_dir, 'mime.types')]) [os.path.join(swift_dir, 'mime.types')])
self.account_autocreate = \ self.account_autocreate = \
@ -454,8 +453,6 @@ class Application(object):
:param start_response: WSGI callable :param start_response: WSGI callable
""" """
try: try:
if self.memcache is None:
self.memcache = cache_from_env(env, True)
req = self.update_request(Request(env)) req = self.update_request(Request(env))
return self.handle_request(req)(env, start_response) return self.handle_request(req)(env, start_response)
except UnicodeError: except UnicodeError:

4
test/unit/common/middleware/test_copy.py
View File

@ -24,7 +24,7 @@ from swift.common.middleware import copy
from swift.common.storage_policy import POLICIES from swift.common.storage_policy import POLICIES
from swift.common.swob import Request, HTTPException from swift.common.swob import Request, HTTPException
from swift.common.utils import closing_if_possible from swift.common.utils import closing_if_possible
from test.unit import patch_policies, debug_logger, FakeMemcache, FakeRing from test.unit import patch_policies, debug_logger, FakeRing
from test.unit.common.middleware.helpers import FakeSwift from test.unit.common.middleware.helpers import FakeSwift
from test.unit.proxy.controllers.test_obj import set_http_connect, \ from test.unit.proxy.controllers.test_obj import set_http_connect, \
PatchedObjControllerApp PatchedObjControllerApp
@ -1316,7 +1316,7 @@ class TestServerSideCopyMiddlewareWithEC(unittest.TestCase):
self.logger = debug_logger('proxy-server') self.logger = debug_logger('proxy-server')
self.logger.thread_locals = ('txn1', '127.0.0.2') self.logger.thread_locals = ('txn1', '127.0.0.2')
self.app = PatchedObjControllerApp( self.app = PatchedObjControllerApp(
None, FakeMemcache(), account_ring=FakeRing(), None, account_ring=FakeRing(),
container_ring=FakeRing(), logger=self.logger) container_ring=FakeRing(), logger=self.logger)
self.ssc = copy.filter_factory({})(self.app) self.ssc = copy.filter_factory({})(self.app)
self.ssc.logger = self.app.logger self.ssc.logger = self.app.logger

8
test/unit/proxy/controllers/test_account.py
View File

@ -21,8 +21,7 @@ from swift.common.middleware.acl import format_acl
from swift.proxy import server as proxy_server from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_account_info from swift.proxy.controllers.base import headers_to_account_info
from swift.common import constraints from swift.common import constraints
from test.unit import fake_http_connect, FakeRing, FakeMemcache, \ from test.unit import fake_http_connect, FakeRing, mocked_http_conn
mocked_http_conn
from swift.common.storage_policy import StoragePolicy from swift.common.storage_policy import StoragePolicy
from swift.common.request_helpers import get_sys_meta_prefix from swift.common.request_helpers import get_sys_meta_prefix
import swift.proxy.controllers.base import swift.proxy.controllers.base
@ -38,7 +37,7 @@ class TestAccountController(unittest.TestCase):
def setUp(self): def setUp(self):
self.app = proxy_server.Application( self.app = proxy_server.Application(
None, FakeMemcache(), None,
account_ring=FakeRing(), container_ring=FakeRing()) account_ring=FakeRing(), container_ring=FakeRing())
def _make_callback_func(self, context): def _make_callback_func(self, context):
@ -344,7 +343,6 @@ class TestAccountController4Replicas(TestAccountController):
def setUp(self): def setUp(self):
self.app = proxy_server.Application( self.app = proxy_server.Application(
None, None,
FakeMemcache(),
account_ring=FakeRing(replicas=4), account_ring=FakeRing(replicas=4),
container_ring=FakeRing(replicas=4)) container_ring=FakeRing(replicas=4))
@ -413,7 +411,7 @@ class TestAccountController4Replicas(TestAccountController):
class TestGetAccountInfo(unittest.TestCase): class TestGetAccountInfo(unittest.TestCase):
def setUp(self): def setUp(self):
self.app = proxy_server.Application( self.app = proxy_server.Application(
None, FakeMemcache(), None,
account_ring=FakeRing(), container_ring=FakeRing()) account_ring=FakeRing(), container_ring=FakeRing())
def test_get_deleted_account_410(self): def test_get_deleted_account_410(self):

6
test/unit/proxy/controllers/test_base.py
View File

@ -182,7 +182,7 @@ class FakeCache(FakeMemcache):
@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) @patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())])
class TestFuncs(unittest.TestCase): class TestFuncs(unittest.TestCase):
def setUp(self): def setUp(self):
self.app = proxy_server.Application(None, FakeMemcache(), self.app = proxy_server.Application(None,
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing(), container_ring=FakeRing(),
logger=FakeLogger()) logger=FakeLogger())
@ -710,7 +710,7 @@ class TestFuncs(unittest.TestCase):
resp, resp,
headers_to_container_info(headers.items(), 200)) headers_to_container_info(headers.items(), 200))
def test_container_info_without_req(self): def test_container_info_needs_req(self):
base = Controller(self.app) base = Controller(self.app)
base.account_name = 'a' base.account_name = 'a'
base.container_name = 'c' base.container_name = 'c'
@ -719,7 +719,7 @@ class TestFuncs(unittest.TestCase):
'http_connect', fake_http_connect(200)): 'http_connect', fake_http_connect(200)):
container_info = \ container_info = \
base.container_info(base.account_name, base.container_info(base.account_name,
base.container_name) base.container_name, Request.blank('/'))
self.assertEqual(container_info['status'], 0) self.assertEqual(container_info['status'], 0)
def test_headers_to_account_info_missing(self): def test_headers_to_account_info_missing(self):

18
test/unit/proxy/controllers/test_container.py
View File

@ -27,7 +27,7 @@ from swift.common.swob import Request, bytes_to_wsgi, str_to_wsgi, wsgi_quote
from swift.common.utils import ShardRange, Timestamp from swift.common.utils import ShardRange, Timestamp
from swift.proxy import server as proxy_server from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_container_info, \ from swift.proxy.controllers.base import headers_to_container_info, \
Controller, get_container_info Controller, get_container_info, get_cache_key
from test import annotate_failure from test import annotate_failure
from test.unit import fake_http_connect, FakeRing, FakeMemcache, \ from test.unit import fake_http_connect, FakeRing, FakeMemcache, \
make_timestamp_iter make_timestamp_iter
@ -49,7 +49,7 @@ class TestContainerController(TestRingBase):
self.logger = debug_logger() self.logger = debug_logger()
self.container_ring = FakeRing(replicas=self.CONTAINER_REPLICAS, self.container_ring = FakeRing(replicas=self.CONTAINER_REPLICAS,
max_more_nodes=9) max_more_nodes=9)
self.app = proxy_server.Application(None, FakeMemcache(), self.app = proxy_server.Application(None,
logger=self.logger, logger=self.logger,
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=self.container_ring) container_ring=self.container_ring)
@ -97,7 +97,9 @@ class TestContainerController(TestRingBase):
with mock.patch( with mock.patch(
'swift.proxy.controllers.base.http_connect', 'swift.proxy.controllers.base.http_connect',
fake_http_connect(*responses)): fake_http_connect(*responses)):
req = Request.blank('/v1/a/c') cache = FakeMemcache()
cache.set(get_cache_key('a'), {'status': 204})
req = Request.blank('/v1/a/c', environ={'swift.cache': cache})
resp = getattr(controller, method)(req) resp = getattr(controller, method)(req)
self.assertEqual(expected, self.assertEqual(expected,
@ -106,9 +108,10 @@ class TestContainerController(TestRingBase):
(expected, resp.status_int, str(responses))) (expected, resp.status_int, str(responses)))
def test_container_info_got_cached(self): def test_container_info_got_cached(self):
memcache = FakeMemcache()
controller = proxy_server.ContainerController(self.app, 'a', 'c') controller = proxy_server.ContainerController(self.app, 'a', 'c')
with mocked_http_conn(200, 200) as mock_conn: with mocked_http_conn(200, 200) as mock_conn:
req = Request.blank('/v1/a/c') req = Request.blank('/v1/a/c', {'swift.cache': memcache})
resp = controller.HEAD(req) resp = controller.HEAD(req)
self.assertEqual(2, resp.status_int // 100) self.assertEqual(2, resp.status_int // 100)
self.assertEqual(['/a', '/a/c'], self.assertEqual(['/a', '/a/c'],
@ -119,12 +122,13 @@ class TestContainerController(TestRingBase):
info_cache = resp.environ['swift.infocache'] info_cache = resp.environ['swift.infocache']
self.assertIn("container/a/c", resp.environ['swift.infocache']) self.assertIn("container/a/c", resp.environ['swift.infocache'])
self.assertEqual(header_info, info_cache['container/a/c']) self.assertEqual(header_info, info_cache['container/a/c'])
self.assertEqual(header_info, self.app.memcache.get('container/a/c')) self.assertEqual(header_info, memcache.get('container/a/c'))
# The failure doesn't lead to cache eviction # The failure doesn't lead to cache eviction
errors = [500] * self.CONTAINER_REPLICAS * 2 errors = [500] * self.CONTAINER_REPLICAS * 2
with mocked_http_conn(*errors) as mock_conn: with mocked_http_conn(*errors) as mock_conn:
req = Request.blank('/v1/a/c', {'swift.infocache': info_cache}) req = Request.blank('/v1/a/c', {'swift.infocache': info_cache,
'swift.cache': memcache})
resp = controller.HEAD(req) resp = controller.HEAD(req)
self.assertEqual(5, resp.status_int // 100) self.assertEqual(5, resp.status_int // 100)
self.assertEqual(['/a/c'] * self.CONTAINER_REPLICAS * 2, self.assertEqual(['/a/c'] * self.CONTAINER_REPLICAS * 2,
@ -134,7 +138,7 @@ class TestContainerController(TestRingBase):
self.assertIn("container/a/c", resp.environ['swift.infocache']) self.assertIn("container/a/c", resp.environ['swift.infocache'])
# NB: this is the *old* header_info, from the good req # NB: this is the *old* header_info, from the good req
self.assertEqual(header_info, info_cache['container/a/c']) self.assertEqual(header_info, info_cache['container/a/c'])
self.assertEqual(header_info, self.app.memcache.get('container/a/c')) self.assertEqual(header_info, memcache.get('container/a/c'))
@mock.patch('swift.proxy.controllers.container.clear_info_cache') @mock.patch('swift.proxy.controllers.container.clear_info_cache')
@mock.patch.object(Controller, 'make_requests') @mock.patch.object(Controller, 'make_requests')

6
test/unit/proxy/controllers/test_obj.py
View File

@ -48,7 +48,7 @@ from swift.proxy.controllers.base import \
from swift.common.storage_policy import POLICIES, ECDriverError, \ from swift.common.storage_policy import POLICIES, ECDriverError, \
StoragePolicy, ECStoragePolicy StoragePolicy, ECStoragePolicy
from test.unit import FakeRing, FakeMemcache, fake_http_connect, \ from test.unit import FakeRing, fake_http_connect, \
debug_logger, patch_policies, SlowBody, FakeStatus, \ debug_logger, patch_policies, SlowBody, FakeStatus, \
DEFAULT_TEST_EC_TYPE, encode_frag_archive_bodies, make_ec_object_stub, \ DEFAULT_TEST_EC_TYPE, encode_frag_archive_bodies, make_ec_object_stub, \
fake_ec_node_response, StubResponse, mocked_http_conn, \ fake_ec_node_response, StubResponse, mocked_http_conn, \
@ -179,7 +179,7 @@ class BaseObjectControllerMixin(object):
# increase connection timeout to avoid intermittent failures # increase connection timeout to avoid intermittent failures
conf = {'conn_timeout': 1.0} conf = {'conn_timeout': 1.0}
self.app = PatchedObjControllerApp( self.app = PatchedObjControllerApp(
conf, FakeMemcache(), account_ring=FakeRing(), conf, account_ring=FakeRing(),
container_ring=FakeRing(), logger=self.logger) container_ring=FakeRing(), logger=self.logger)
# you can over-ride the container_info just by setting it on the app # you can over-ride the container_info just by setting it on the app
@ -969,7 +969,7 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin):
self, conf, policy_conf, policy, affinity_regions, affinity_count): self, conf, policy_conf, policy, affinity_regions, affinity_count):
conf['policy_config'] = policy_conf conf['policy_config'] = policy_conf
app = PatchedObjControllerApp( app = PatchedObjControllerApp(
conf, FakeMemcache(), account_ring=FakeRing(), conf, account_ring=FakeRing(),
container_ring=FakeRing(), logger=self.logger) container_ring=FakeRing(), logger=self.logger)
controller = self.controller_cls(app, 'a', 'c', 'o') controller = self.controller_cls(app, 'a', 'c', 'o')

239
test/unit/proxy/test_server.py
View File

@ -173,14 +173,6 @@ def set_node_errors(proxy_app, ring_node, value, last_error):
stats['last_error'] = last_error stats['last_error'] = last_error
class FakeMemcacheReturnsNone(FakeMemcache):
def get(self, key):
# Returns None as the timestamp of the container; assumes we're only
# using the FakeMemcache for container existence checks.
return None
@contextmanager @contextmanager
def save_globals(): def save_globals():
orig_http_connect = getattr(swift.proxy.controllers.base, 'http_connect', orig_http_connect = getattr(swift.proxy.controllers.base, 'http_connect',
@ -251,22 +243,23 @@ class TestController(unittest.TestCase):
self.account_ring = FakeRing() self.account_ring = FakeRing()
self.container_ring = FakeRing() self.container_ring = FakeRing()
self.memcache = FakeMemcache() self.memcache = FakeMemcache()
app = proxy_server.Application(None, self.memcache, app = proxy_server.Application(None,
account_ring=self.account_ring, account_ring=self.account_ring,
container_ring=self.container_ring) container_ring=self.container_ring)
self.controller = swift.proxy.controllers.Controller(app) self.controller = swift.proxy.controllers.Controller(app)
class FakeReq(object): class FakeReq(object):
def __init__(self): def __init__(self, memcache):
self.url = "/foo/bar" self.url = "/foo/bar"
self.method = "METHOD" self.method = "METHOD"
self.environ = {"swift.cache": memcache}
def as_referer(self): def as_referer(self):
return self.method + ' ' + self.url return self.method + ' ' + self.url
self.account = 'some_account' self.account = 'some_account'
self.container = 'some_container' self.container = 'some_container'
self.request = FakeReq() self.request = FakeReq(self.memcache)
self.read_acl = 'read_acl' self.read_acl = 'read_acl'
self.write_acl = 'write_acl' self.write_acl = 'write_acl'
@ -290,29 +283,31 @@ class TestController(unittest.TestCase):
self.assertEqual(n, nodes) self.assertEqual(n, nodes)
def test_account_info_container_count(self): def test_account_info_container_count(self):
cache_key = get_cache_key(self.account)
with save_globals(): with save_globals():
set_http_connect(200, count=123) set_http_connect(200, count=123)
partition, nodes, count = \ partition, nodes, count = \
self.controller.account_info(self.account) self.controller.account_info(self.account, self.request)
self.assertEqual(count, 123) self.assertEqual(count, 123)
with save_globals(): with save_globals():
set_http_connect(200, count='123') set_http_connect(200, count='123')
del self.request.environ['swift.infocache'][cache_key]
partition, nodes, count = \ partition, nodes, count = \
self.controller.account_info(self.account) self.controller.account_info(self.account, self.request)
self.assertEqual(count, 123) self.assertEqual(count, 123)
with save_globals(): with save_globals():
cache_key = get_cache_key(self.account)
account_info = {'status': 200, 'container_count': 1234} account_info = {'status': 200, 'container_count': 1234}
self.memcache.set(cache_key, account_info) self.memcache.set(cache_key, account_info)
del self.request.environ['swift.infocache'][cache_key]
partition, nodes, count = \ partition, nodes, count = \
self.controller.account_info(self.account) self.controller.account_info(self.account, self.request)
self.assertEqual(count, 1234) self.assertEqual(count, 1234)
with save_globals(): with save_globals():
cache_key = get_cache_key(self.account)
account_info = {'status': 200, 'container_count': '1234'} account_info = {'status': 200, 'container_count': '1234'}
self.memcache.set(cache_key, account_info) self.memcache.set(cache_key, account_info)
del self.request.environ['swift.infocache'][cache_key]
partition, nodes, count = \ partition, nodes, count = \
self.controller.account_info(self.account) self.controller.account_info(self.account, self.request)
self.assertEqual(count, 1234) self.assertEqual(count, 1234)
def test_make_requests(self): def test_make_requests(self):
@ -407,7 +402,6 @@ class TestController(unittest.TestCase):
def test_account_info_no_account(self): def test_account_info_no_account(self):
with save_globals(): with save_globals():
self.memcache.store = {}
set_http_connect(404, 404, 404) set_http_connect(404, 404, 404)
partition, nodes, count = \ partition, nodes, count = \
self.controller.account_info(self.account, self.request) self.controller.account_info(self.account, self.request)
@ -514,13 +508,13 @@ class TestController(unittest.TestCase):
def test_get_account_info_returns_values_as_strings(self): def test_get_account_info_returns_values_as_strings(self):
app = mock.MagicMock() app = mock.MagicMock()
app.memcache = mock.MagicMock() memcache = mock.MagicMock()
app.memcache.get = mock.MagicMock() memcache.get = mock.MagicMock()
app.memcache.get.return_value = { memcache.get.return_value = {
u'foo': u'\u2603', u'foo': u'\u2603',
u'meta': {u'bar': u'\u2603'}, u'meta': {u'bar': u'\u2603'},
u'sysmeta': {u'baz': u'\u2603'}} u'sysmeta': {u'baz': u'\u2603'}}
env = {'PATH_INFO': '/v1/a'} env = {'PATH_INFO': '/v1/a', 'swift.cache': memcache}
ai = get_account_info(env, app) ai = get_account_info(env, app)
# Test info is returned as strings # Test info is returned as strings
@ -539,14 +533,14 @@ class TestController(unittest.TestCase):
def test_get_container_info_returns_values_as_strings(self): def test_get_container_info_returns_values_as_strings(self):
app = mock.MagicMock() app = mock.MagicMock()
app.memcache = mock.MagicMock() memcache = mock.MagicMock()
app.memcache.get = mock.MagicMock() memcache.get = mock.MagicMock()
app.memcache.get.return_value = { memcache.get.return_value = {
u'foo': u'\u2603', u'foo': u'\u2603',
u'meta': {u'bar': u'\u2603'}, u'meta': {u'bar': u'\u2603'},
u'sysmeta': {u'baz': u'\u2603'}, u'sysmeta': {u'baz': u'\u2603'},
u'cors': {u'expose_headers': u'\u2603'}} u'cors': {u'expose_headers': u'\u2603'}}
env = {'PATH_INFO': '/v1/a/c'} env = {'PATH_INFO': '/v1/a/c', 'swift.cache': memcache}
ci = get_container_info(env, app) ci = get_container_info(env, app)
# Test info is returned as strings # Test info is returned as strings
@ -578,7 +572,7 @@ class TestProxyServerConfiguration(unittest.TestCase):
def _make_app(self, conf): def _make_app(self, conf):
self.logger.clear() self.logger.clear()
# helper function to instantiate a proxy server instance # helper function to instantiate a proxy server instance
return proxy_server.Application(conf, FakeMemcache(), return proxy_server.Application(conf,
container_ring=FakeRing(), container_ring=FakeRing(),
account_ring=FakeRing(), account_ring=FakeRing(),
logger=self.logger) logger=self.logger)
@ -634,7 +628,6 @@ class TestProxyServer(unittest.TestCase):
def test_get_object_ring(self): def test_get_object_ring(self):
baseapp = proxy_server.Application({}, baseapp = proxy_server.Application({},
FakeMemcache(),
container_ring=FakeRing(), container_ring=FakeRing(),
account_ring=FakeRing()) account_ring=FakeRing())
with patch_policies([ with patch_policies([
@ -664,7 +657,7 @@ class TestProxyServer(unittest.TestCase):
def get_controller(self, path): def get_controller(self, path):
raise Exception('this shouldn\'t be caught') raise Exception('this shouldn\'t be caught')
app = MyApp(None, FakeMemcache(), account_ring=FakeRing(), app = MyApp(None, account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'}) req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'})
app.update_request(req) app.update_request(req)
@ -673,7 +666,6 @@ class TestProxyServer(unittest.TestCase):
def test_internal_method_request(self): def test_internal_method_request(self):
baseapp = proxy_server.Application({}, baseapp = proxy_server.Application({},
FakeMemcache(),
container_ring=FakeRing(), container_ring=FakeRing(),
account_ring=FakeRing()) account_ring=FakeRing())
resp = baseapp.handle_request( resp = baseapp.handle_request(
@ -682,7 +674,6 @@ class TestProxyServer(unittest.TestCase):
def test_inexistent_method_request(self): def test_inexistent_method_request(self):
baseapp = proxy_server.Application({}, baseapp = proxy_server.Application({},
FakeMemcache(),
container_ring=FakeRing(), container_ring=FakeRing(),
account_ring=FakeRing()) account_ring=FakeRing())
resp = baseapp.handle_request( resp = baseapp.handle_request(
@ -691,7 +682,6 @@ class TestProxyServer(unittest.TestCase):
def test_private_method_request(self): def test_private_method_request(self):
baseapp = proxy_server.Application({}, baseapp = proxy_server.Application({},
FakeMemcache(),
container_ring=FakeRing(), container_ring=FakeRing(),
account_ring=FakeRing()) account_ring=FakeRing())
baseapp.logger = debug_logger() baseapp.logger = debug_logger()
@ -722,12 +712,13 @@ class TestProxyServer(unittest.TestCase):
# set account info # set account info
fake_cache = FakeMemcache() fake_cache = FakeMemcache()
fake_cache.store[get_cache_key('a')] = {'status': 200} fake_cache.store[get_cache_key('a')] = {'status': 200}
app = proxy_server.Application({}, fake_cache, app = proxy_server.Application({},
container_ring=FakeRing(), container_ring=FakeRing(),
account_ring=FakeRing()) account_ring=FakeRing())
# build internal container request # build internal container request
container = get_reserved_name('c') container = get_reserved_name('c')
req = Request.blank('/v1/a/%s' % container) req = Request.blank('/v1/a/%s' % container,
{'swift.cache': fake_cache})
app.update_request(req) app.update_request(req)
# try client request to reserved name # try client request to reserved name
@ -748,7 +739,7 @@ class TestProxyServer(unittest.TestCase):
called[0] = True called[0] = True
with save_globals(): with save_globals():
set_http_connect(200) set_http_connect(200)
app = proxy_server.Application(None, FakeMemcache(), app = proxy_server.Application(None,
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
req = Request.blank('/v1/a') req = Request.blank('/v1/a')
@ -763,7 +754,7 @@ class TestProxyServer(unittest.TestCase):
def authorize(req): def authorize(req):
called[0] = True called[0] = True
return HTTPUnauthorized(request=req) return HTTPUnauthorized(request=req)
app = proxy_server.Application(None, FakeMemcache(), app = proxy_server.Application(None,
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
req = Request.blank('/v1/a') req = Request.blank('/v1/a')
@ -776,7 +767,7 @@ class TestProxyServer(unittest.TestCase):
swift_dir = mkdtemp() swift_dir = mkdtemp()
try: try:
baseapp = proxy_server.Application({'swift_dir': swift_dir}, baseapp = proxy_server.Application({'swift_dir': swift_dir},
FakeMemcache(), FakeLogger(), FakeLogger(),
FakeRing(), FakeRing()) FakeRing(), FakeRing())
resp = baseapp.handle_request( resp = baseapp.handle_request(
Request.blank('/', environ={'CONTENT_LENGTH': '-1'})) Request.blank('/', environ={'CONTENT_LENGTH': '-1'}))
@ -794,7 +785,7 @@ class TestProxyServer(unittest.TestCase):
try: try:
logger = FakeLogger() logger = FakeLogger()
baseapp = proxy_server.Application({'swift_dir': swift_dir}, baseapp = proxy_server.Application({'swift_dir': swift_dir},
FakeMemcache(), logger, logger,
container_ring=FakeLogger(), container_ring=FakeLogger(),
account_ring=FakeRing()) account_ring=FakeRing())
baseapp.handle_request( baseapp.handle_request(
@ -814,7 +805,7 @@ class TestProxyServer(unittest.TestCase):
try: try:
logger = FakeLogger() logger = FakeLogger()
baseapp = proxy_server.Application({'swift_dir': swift_dir}, baseapp = proxy_server.Application({'swift_dir': swift_dir},
FakeMemcache(), logger, logger,
container_ring=FakeLogger(), container_ring=FakeLogger(),
account_ring=FakeRing()) account_ring=FakeRing())
baseapp.handle_request( baseapp.handle_request(
@ -832,7 +823,6 @@ class TestProxyServer(unittest.TestCase):
baseapp = proxy_server.Application({'swift_dir': swift_dir, baseapp = proxy_server.Application({'swift_dir': swift_dir,
'deny_host_headers': 'deny_host_headers':
'invalid_host.com'}, 'invalid_host.com'},
FakeMemcache(),
container_ring=FakeLogger(), container_ring=FakeLogger(),
account_ring=FakeRing()) account_ring=FakeRing())
resp = baseapp.handle_request( resp = baseapp.handle_request(
@ -844,7 +834,6 @@ class TestProxyServer(unittest.TestCase):
def test_node_timing(self): def test_node_timing(self):
baseapp = proxy_server.Application({'sorting_method': 'timing'}, baseapp = proxy_server.Application({'sorting_method': 'timing'},
FakeMemcache(),
container_ring=FakeRing(), container_ring=FakeRing(),
account_ring=FakeRing()) account_ring=FakeRing())
self.assertEqual(baseapp.node_timings, {}) self.assertEqual(baseapp.node_timings, {})
@ -876,7 +865,6 @@ class TestProxyServer(unittest.TestCase):
nodes = list(nodes) nodes = list(nodes)
conf = dict(conf, policy_config=policy_conf) conf = dict(conf, policy_config=policy_conf)
baseapp = proxy_server.Application(conf, baseapp = proxy_server.Application(conf,
FakeMemcache(),
logger=FakeLogger(), logger=FakeLogger(),
container_ring=FakeRing(), container_ring=FakeRing(),
account_ring=FakeRing()) account_ring=FakeRing())
@ -1070,7 +1058,6 @@ class TestProxyServer(unittest.TestCase):
app_conf = {'concurrent_gets': 'on', app_conf = {'concurrent_gets': 'on',
'concurrency_timeout': 0} 'concurrency_timeout': 0}
baseapp = proxy_server.Application(app_conf, baseapp = proxy_server.Application(app_conf,
FakeMemcache(),
container_ring=FakeRing(), container_ring=FakeRing(),
account_ring=FakeRing()) account_ring=FakeRing())
self.assertTrue(baseapp.concurrent_gets) self.assertTrue(baseapp.concurrent_gets)
@ -1095,7 +1082,6 @@ class TestProxyServer(unittest.TestCase):
# Now lets set the concurrency_timeout # Now lets set the concurrency_timeout
app_conf['concurrency_timeout'] = 2 app_conf['concurrency_timeout'] = 2
baseapp = proxy_server.Application(app_conf, baseapp = proxy_server.Application(app_conf,
FakeMemcache(),
container_ring=FakeRing(), container_ring=FakeRing(),
account_ring=FakeRing()) account_ring=FakeRing())
self.assertEqual(baseapp.concurrency_timeout, 2) self.assertEqual(baseapp.concurrency_timeout, 2)
@ -1106,7 +1092,7 @@ class TestProxyServer(unittest.TestCase):
self.assertEqual(resp.body, b'Response from 127.0.0.2') self.assertEqual(resp.body, b'Response from 127.0.0.2')
def test_info_defaults(self): def test_info_defaults(self):
app = proxy_server.Application({}, FakeMemcache(), app = proxy_server.Application({},
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
@ -1121,7 +1107,7 @@ class TestProxyServer(unittest.TestCase):
def test_get_info_controller(self): def test_get_info_controller(self):
req = Request.blank('/info') req = Request.blank('/info')
app = proxy_server.Application({}, FakeMemcache(), app = proxy_server.Application({},
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
@ -1138,7 +1124,7 @@ class TestProxyServer(unittest.TestCase):
def test_exception_occurred(self): def test_exception_occurred(self):
def do_test(additional_info): def do_test(additional_info):
logger = debug_logger('test') logger = debug_logger('test')
app = proxy_server.Application({}, FakeMemcache(), app = proxy_server.Application({},
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing(), container_ring=FakeRing(),
logger=logger) logger=logger)
@ -1172,7 +1158,7 @@ class TestProxyServer(unittest.TestCase):
def test_error_occurred(self): def test_error_occurred(self):
def do_test(msg): def do_test(msg):
logger = debug_logger('test') logger = debug_logger('test')
app = proxy_server.Application({}, FakeMemcache(), app = proxy_server.Application({},
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing(), container_ring=FakeRing(),
logger=logger) logger=logger)
@ -1198,7 +1184,7 @@ class TestProxyServer(unittest.TestCase):
def test_error_limit_methods(self): def test_error_limit_methods(self):
logger = debug_logger('test') logger = debug_logger('test')
app = proxy_server.Application({}, FakeMemcache(), app = proxy_server.Application({},
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing(), container_ring=FakeRing(),
logger=logger) logger=logger)
@ -1260,7 +1246,7 @@ class TestProxyServer(unittest.TestCase):
self.assertEqual(4, node_error_count(app, node)) self.assertEqual(4, node_error_count(app, node))
def test_valid_api_version(self): def test_valid_api_version(self):
app = proxy_server.Application({}, FakeMemcache(), app = proxy_server.Application({},
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
@ -1341,7 +1327,7 @@ class TestProxyServerLoading(unittest.TestCase):
} }
for policy in POLICIES: for policy in POLICIES:
policy.object_ring = FakeRing() policy.object_ring = FakeRing()
app = proxy_server.Application(conf, FakeMemcache(), debug_logger(), app = proxy_server.Application(conf, debug_logger(),
FakeRing(), FakeRing()) FakeRing(), FakeRing())
self.assertEqual(app.node_timeout, 2.3) self.assertEqual(app.node_timeout, 2.3)
self.assertEqual(app.recoverable_node_timeout, 1.4) self.assertEqual(app.recoverable_node_timeout, 1.4)
@ -2023,7 +2009,6 @@ class BaseTestObjectController(object):
kwargs['raise_exc'] = raise_exc kwargs['raise_exc'] = raise_exc
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', req = Request.blank('/v1/a/c/o',
headers={'Content-Length': '0', headers={'Content-Length': '0',
'Content-Type': 'text/plain'}) 'Content-Type': 'text/plain'})
@ -2036,7 +2021,6 @@ class BaseTestObjectController(object):
# repeat test # repeat test
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', req = Request.blank('/v1/a/c/o',
headers={'Content-Length': '0', headers={'Content-Length': '0',
'Content-Type': 'text/plain'}) 'Content-Type': 'text/plain'})
@ -2171,7 +2155,7 @@ class TestReplicatedObjectController(
def setUp(self): def setUp(self):
skip_if_no_xattrs() skip_if_no_xattrs()
self.app = proxy_server.Application( self.app = proxy_server.Application(
None, FakeMemcache(), None,
logger=debug_logger('proxy-ut'), logger=debug_logger('proxy-ut'),
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
@ -3023,7 +3007,6 @@ class TestReplicatedObjectController(
req = Request.blank('/v1/a/c/o.jpg', {}) req = Request.blank('/v1/a/c/o.jpg', {})
req.content_length = 0 req.content_length = 0
self.app.update_request(req) self.app.update_request(req)
self.app.memcache.store = {}
res = controller.PUT(req) res = controller.PUT(req)
self.assertEqual(test_errors, []) self.assertEqual(test_errors, [])
self.assertTrue(res.status.startswith('201 '), res.status) self.assertTrue(res.status.startswith('201 '), res.status)
@ -3052,7 +3035,6 @@ class TestReplicatedObjectController(
req.content_length = 1 req.content_length = 1
req.body = 'a' req.body = 'a'
self.app.update_request(req) self.app.update_request(req)
self.app.memcache.store = {}
res = controller.PUT(req) res = controller.PUT(req)
self.assertEqual(test_errors, []) self.assertEqual(test_errors, [])
self.assertTrue(res.status.startswith('201 ')) self.assertTrue(res.status.startswith('201 '))
@ -3070,7 +3052,7 @@ class TestReplicatedObjectController(
# not be used in ring order is if affinity is respected. # not be used in ring order is if affinity is respected.
with mock.patch('swift.proxy.server.shuffle', lambda x: x): with mock.patch('swift.proxy.server.shuffle', lambda x: x):
app = proxy_server.Application( app = proxy_server.Application(
conf, FakeMemcache(), conf,
logger=debug_logger('proxy-ut'), logger=debug_logger('proxy-ut'),
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
@ -3086,7 +3068,6 @@ class TestReplicatedObjectController(
req = Request.blank( req = Request.blank(
'/v1/a/c/o.jpg', method='PUT', body='a', '/v1/a/c/o.jpg', method='PUT', body='a',
headers={'X-Backend-Storage-Policy-Index': str(policy)}) headers={'X-Backend-Storage-Policy-Index': str(policy)})
app.memcache.store = {}
res = controller.PUT(req) res = controller.PUT(req)
self.assertTrue(res.status.startswith('201 ')) self.assertTrue(res.status.startswith('201 '))
self.assertEqual(3, len(written_to)) self.assertEqual(3, len(written_to))
@ -3155,7 +3136,6 @@ class TestReplicatedObjectController(
req = Request.blank('/v1/a/c/o.jpg', {}) req = Request.blank('/v1/a/c/o.jpg', {})
req.content_length = 1 req.content_length = 1
req.body = 'a' req.body = 'a'
self.app.memcache.store = {}
res = controller.PUT(req) res = controller.PUT(req)
self.assertTrue(res.status.startswith('201 ')) self.assertTrue(res.status.startswith('201 '))
@ -3554,7 +3534,7 @@ class TestReplicatedObjectController(
with open(os.path.join(swift_dir, 'mime.types'), 'w') as fp: with open(os.path.join(swift_dir, 'mime.types'), 'w') as fp:
fp.write('foo/bar foo\n') fp.write('foo/bar foo\n')
proxy_server.Application({'swift_dir': swift_dir}, proxy_server.Application({'swift_dir': swift_dir},
FakeMemcache(), FakeLogger(), FakeLogger(),
FakeRing(), FakeRing()) FakeRing(), FakeRing())
self.assertEqual(proxy_server.mimetypes.guess_type('blah.foo')[0], self.assertEqual(proxy_server.mimetypes.guess_type('blah.foo')[0],
'foo/bar') 'foo/bar')
@ -3573,7 +3553,6 @@ class TestReplicatedObjectController(
req = Request.blank('/v1/a/c/o.jpg', {}) req = Request.blank('/v1/a/c/o.jpg', {})
req.content_length = 0 req.content_length = 0
self.app.update_request(req) self.app.update_request(req)
self.app.memcache.store = {}
res = controller.PUT(req) res = controller.PUT(req)
expected = str(expected) expected = str(expected)
self.assertEqual(res.status[:len(expected)], expected) self.assertEqual(res.status[:len(expected)], expected)
@ -3590,7 +3569,6 @@ class TestReplicatedObjectController(
def test_status_map(statuses, expected): def test_status_map(statuses, expected):
set_http_connect(*statuses) set_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o.jpg', {}) req = Request.blank('/v1/a/c/o.jpg', {})
req.content_length = 0 req.content_length = 0
self.app.update_request(req) self.app.update_request(req)
@ -3620,7 +3598,6 @@ class TestReplicatedObjectController(
self.app, 'account', 'container', 'object') self.app, 'account', 'container', 'object')
def test_status_map(statuses, expected): def test_status_map(statuses, expected):
self.app.memcache.store = {}
set_http_connect(*statuses) set_http_connect(*statuses)
req = Request.blank('/v1/a/c/o.jpg', req = Request.blank('/v1/a/c/o.jpg',
environ={'REQUEST_METHOD': 'PUT'}, environ={'REQUEST_METHOD': 'PUT'},
@ -3666,7 +3643,6 @@ class TestReplicatedObjectController(
self.app, 'account', 'container', 'object') self.app, 'account', 'container', 'object')
def test_status_map(statuses, expected): def test_status_map(statuses, expected):
self.app.memcache.store = {}
set_http_connect(*statuses) set_http_connect(*statuses)
req = Request.blank('/v1/a/c/o.jpg', {}) req = Request.blank('/v1/a/c/o.jpg', {})
req.content_length = 0 req.content_length = 0
@ -3686,7 +3662,6 @@ class TestReplicatedObjectController(
with save_globals(): with save_globals():
def test_status_map(statuses, expected): def test_status_map(statuses, expected):
set_http_connect(*statuses) set_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {}, method='POST', req = Request.blank('/v1/a/c/o', {}, method='POST',
headers={'Content-Type': 'foo/bar'}) headers={'Content-Type': 'foo/bar'})
self.app.update_request(req) self.app.update_request(req)
@ -3711,7 +3686,6 @@ class TestReplicatedObjectController(
self.app.sort_nodes = lambda nodes, *args, **kwargs: nodes self.app.sort_nodes = lambda nodes, *args, **kwargs: nodes
def do_test(resp_headers): def do_test(resp_headers):
self.app.memcache.store = {}
backend_requests = [] backend_requests = []
def capture_requests(ip, port, method, path, headers, *args, def capture_requests(ip, port, method, path, headers, *args,
@ -3779,7 +3753,6 @@ class TestReplicatedObjectController(
self.assertEqual(container_headers, expected) self.assertEqual(container_headers, expected)
# and again with policy override # and again with policy override
self.app.memcache.store = {}
backend_requests = [] backend_requests = []
req = Request.blank('/v1/a/c/o', {}, method='POST', req = Request.blank('/v1/a/c/o', {}, method='POST',
headers={'X-Object-Meta-Color': 'Blue', headers={'X-Object-Meta-Color': 'Blue',
@ -3823,7 +3796,6 @@ class TestReplicatedObjectController(
self.app.recheck_updating_shard_ranges = 0 self.app.recheck_updating_shard_ranges = 0
def do_test(method, sharding_state): def do_test(method, sharding_state):
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {}, method=method, body='', req = Request.blank('/v1/a/c/o', {}, method=method, body='',
headers={'Content-Type': 'text/plain'}) headers={'Content-Type': 'text/plain'})
@ -3918,9 +3890,9 @@ class TestReplicatedObjectController(
self.app.recheck_updating_shard_ranges = 3600 self.app.recheck_updating_shard_ranges = 3600
def do_test(method, sharding_state): def do_test(method, sharding_state):
self.app.memcache.store = {} req = Request.blank(
req = Request.blank('/v1/a/c/o', {}, method=method, body='', '/v1/a/c/o', {'swift.cache': FakeMemcache()},
headers={'Content-Type': 'text/plain'}) method=method, body='', headers={'Content-Type': 'text/plain'})
# we want the container_info response to say policy index of 1 and # we want the container_info response to say policy index of 1 and
# sharding state # sharding state
@ -3977,8 +3949,8 @@ class TestReplicatedObjectController(
headers={'X-Backend-Record-Type': 'shard'}) headers={'X-Backend-Record-Type': 'shard'})
cache_key = 'shard-updating/a/c' cache_key = 'shard-updating/a/c'
self.assertIn(cache_key, self.app.memcache.store) self.assertIn(cache_key, req.environ['swift.cache'].store)
self.assertEqual(self.app.memcache.store[cache_key], self.assertEqual(req.environ['swift.cache'].store[cache_key],
[dict(sr) for sr in shard_ranges]) [dict(sr) for sr in shard_ranges])
self.assertIn(cache_key, req.environ.get('swift.infocache')) self.assertIn(cache_key, req.environ.get('swift.infocache'))
self.assertEqual(req.environ['swift.infocache'][cache_key], self.assertEqual(req.environ['swift.infocache'][cache_key],
@ -4037,9 +4009,11 @@ class TestReplicatedObjectController(
utils.ShardRange( utils.ShardRange(
'.shards_a/c_nope', utils.Timestamp.now(), 'u', ''), '.shards_a/c_nope', utils.Timestamp.now(), 'u', ''),
] ]
self.app.memcache.store = {'shard-updating/a/c': tuple( cache = FakeMemcache()
dict(shard_range) for shard_range in shard_ranges)} cache.set('shard-updating/a/c', tuple(
req = Request.blank('/v1/a/c/o', {}, method=method, body='', dict(shard_range) for shard_range in shard_ranges))
req = Request.blank('/v1/a/c/o', {'swift.cache': cache},
method=method, body='',
headers={'Content-Type': 'text/plain'}) headers={'Content-Type': 'text/plain'})
# we want the container_info response to say policy index of 1 and # we want the container_info response to say policy index of 1 and
@ -4122,7 +4096,6 @@ class TestReplicatedObjectController(
with save_globals(): with save_globals():
def test_status_map(statuses, expected): def test_status_map(statuses, expected):
set_http_connect(*statuses) set_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'DELETE'}) req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'DELETE'})
self.app.update_request(req) self.app.update_request(req)
res = req.get_response(self.app) res = req.get_response(self.app)
@ -4139,7 +4112,6 @@ class TestReplicatedObjectController(
with save_globals(): with save_globals():
def test_status_map(statuses, expected): def test_status_map(statuses, expected):
set_http_connect(*statuses) set_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}) req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'})
self.app.update_request(req) self.app.update_request(req)
res = req.get_response(self.app) res = req.get_response(self.app)
@ -4170,7 +4142,6 @@ class TestReplicatedObjectController(
def test_status_map(statuses, expected, timestamps, def test_status_map(statuses, expected, timestamps,
expected_timestamp): expected_timestamp):
set_http_connect(*statuses, timestamps=timestamps) set_http_connect(*statuses, timestamps=timestamps)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}, req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'},
headers={'x-newest': 'true'}) headers={'x-newest': 'true'})
self.app.update_request(req) self.app.update_request(req)
@ -4201,7 +4172,6 @@ class TestReplicatedObjectController(
def test_status_map(statuses, expected, timestamps, def test_status_map(statuses, expected, timestamps,
expected_timestamp): expected_timestamp):
set_http_connect(*statuses, timestamps=timestamps) set_http_connect(*statuses, timestamps=timestamps)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'GET'}, req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'GET'},
headers={'x-newest': 'true'}) headers={'x-newest': 'true'})
self.app.update_request(req) self.app.update_request(req)
@ -4228,7 +4198,6 @@ class TestReplicatedObjectController(
def test_status_map(statuses, expected, timestamps, def test_status_map(statuses, expected, timestamps,
expected_timestamp): expected_timestamp):
set_http_connect(*statuses, timestamps=timestamps) set_http_connect(*statuses, timestamps=timestamps)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}) req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'})
self.app.update_request(req) self.app.update_request(req)
res = req.get_response(self.app) res = req.get_response(self.app)
@ -4410,6 +4379,7 @@ class TestReplicatedObjectController(
req = Request.blank('/v1/a/c/o', req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', environ={'REQUEST_METHOD': 'PUT',
'swift.cache': FakeMemcache(),
'wsgi.input': SlowBody()}, 'wsgi.input': SlowBody()},
headers={'Content-Length': '4', headers={'Content-Length': '4',
'Content-Type': 'text/plain'}) 'Content-Type': 'text/plain'})
@ -4419,11 +4389,8 @@ class TestReplicatedObjectController(
resp = req.get_response(self.app) resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201) self.assertEqual(resp.status_int, 201)
self.app.client_timeout = 0.05 self.app.client_timeout = 0.05
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', req.environ['wsgi.input'] = SlowBody() # Need a fresh instance
'wsgi.input': SlowBody()},
headers={'Content-Length': '4',
'Content-Type': 'text/plain'})
self.app.update_request(req) self.app.update_request(req)
set_http_connect(201, 201, 201) set_http_connect(201, 201, 201)
# obj obj obj # obj obj obj
@ -4537,7 +4504,8 @@ class TestReplicatedObjectController(
for dev in object_ring.devs: for dev in object_ring.devs:
dev['ip'] = '127.0.0.1' dev['ip'] = '127.0.0.1'
dev['port'] = 1 dev['port'] = 1
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) req = Request.blank('/v1/a/c/o', environ={
'REQUEST_METHOD': 'GET', 'swift.cache': FakeMemcache()})
self.app.update_request(req) self.app.update_request(req)
self.app.recoverable_node_timeout = 0.1 self.app.recoverable_node_timeout = 0.1
@ -4561,7 +4529,6 @@ class TestReplicatedObjectController(
resp = req.get_response(self.app) resp = req.get_response(self.app)
self.assertEqual(resp.body, b'lalala') self.assertEqual(resp.body, b'lalala')
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
set_http_connect(200, 200, 200, body=b'lalala', set_http_connect(200, 200, 200, body=b'lalala',
slow=[1.0, 1.0], etags=['a', 'b', 'b']) slow=[1.0, 1.0], etags=['a', 'b', 'b'])
resp = req.get_response(self.app) resp = req.get_response(self.app)
@ -4605,7 +4572,6 @@ class TestReplicatedObjectController(
def test_node_request_setting(self): def test_node_request_setting(self):
baseapp = proxy_server.Application({'request_node_count': '3'}, baseapp = proxy_server.Application({'request_node_count': '3'},
FakeMemcache(),
container_ring=FakeRing(), container_ring=FakeRing(),
account_ring=FakeRing()) account_ring=FakeRing())
self.assertEqual(baseapp.request_node_count(3), 3) self.assertEqual(baseapp.request_node_count(3), 3)
@ -4939,7 +4905,6 @@ class TestReplicatedObjectController(
def test_acc_or_con_missing_returns_404(self): def test_acc_or_con_missing_returns_404(self):
with save_globals(): with save_globals():
self.app.memcache = FakeMemcacheReturnsNone()
self.app._error_limiting = {} self.app._error_limiting = {}
controller = ReplicatedObjectController( controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object') self.app, 'account', 'container', 'object')
@ -5036,7 +5001,6 @@ class TestReplicatedObjectController(
def test_PUT_POST_requires_container_exist(self): def test_PUT_POST_requires_container_exist(self):
with save_globals(): with save_globals():
self.app.memcache = FakeMemcacheReturnsNone()
controller = ReplicatedObjectController( controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object') self.app, 'account', 'container', 'object')
@ -5194,17 +5158,21 @@ class TestReplicatedObjectController(
with save_globals(): with save_globals():
controller = ReplicatedObjectController( controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object') self.app, 'account', 'container', 'object')
cache = FakeMemcache()
set_http_connect(200, 200, 201, 201, 201) set_http_connect(200, 200, 201, 201, 201)
# acct cont obj obj obj # acct cont obj obj obj
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, req = Request.blank(
headers={'Content-Length': '0'}) '/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers={'Content-Length': '0'})
self.app.update_request(req) self.app.update_request(req)
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEqual(resp.status_int, 201) self.assertEqual(resp.status_int, 201)
set_http_connect(201, 201, 201) set_http_connect(201, 201, 201)
req = Request.blank( req = Request.blank(
'/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, '/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers={'Content-Length': '0', headers={'Content-Length': '0',
'X-Object-Meta-' + ( 'X-Object-Meta-' + (
'a' * constraints.MAX_META_NAME_LENGTH): 'v'}) 'a' * constraints.MAX_META_NAME_LENGTH): 'v'})
@ -5213,7 +5181,8 @@ class TestReplicatedObjectController(
self.assertEqual(resp.status_int, 201) self.assertEqual(resp.status_int, 201)
set_http_connect(201, 201, 201) set_http_connect(201, 201, 201)
req = Request.blank( req = Request.blank(
'/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, '/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers={ headers={
'Content-Length': '0', 'Content-Length': '0',
'X-Object-Meta-' + ( 'X-Object-Meta-' + (
@ -5223,16 +5192,19 @@ class TestReplicatedObjectController(
self.assertEqual(resp.status_int, 400) self.assertEqual(resp.status_int, 400)
set_http_connect(201, 201, 201) set_http_connect(201, 201, 201)
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, req = Request.blank(
headers={'Content-Length': '0', '/v1/a/c/o',
'X-Object-Meta-Too-Long': 'a' * environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
constraints.MAX_META_VALUE_LENGTH}) headers={'Content-Length': '0',
'X-Object-Meta-Too-Long': 'a' *
constraints.MAX_META_VALUE_LENGTH})
self.app.update_request(req) self.app.update_request(req)
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEqual(resp.status_int, 201) self.assertEqual(resp.status_int, 201)
set_http_connect(201, 201, 201) set_http_connect(201, 201, 201)
req = Request.blank( req = Request.blank(
'/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, '/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers={'Content-Length': '0', headers={'Content-Length': '0',
'X-Object-Meta-Too-Long': 'a' * 'X-Object-Meta-Too-Long': 'a' *
(constraints.MAX_META_VALUE_LENGTH + 1)}) (constraints.MAX_META_VALUE_LENGTH + 1)})
@ -5244,8 +5216,10 @@ class TestReplicatedObjectController(
headers = {'Content-Length': '0'} headers = {'Content-Length': '0'}
for x in range(constraints.MAX_META_COUNT): for x in range(constraints.MAX_META_COUNT):
headers['X-Object-Meta-%d' % x] = 'v' headers['X-Object-Meta-%d' % x] = 'v'
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, req = Request.blank(
headers=headers) '/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers=headers)
self.app.update_request(req) self.app.update_request(req)
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEqual(resp.status_int, 201) self.assertEqual(resp.status_int, 201)
@ -5253,8 +5227,10 @@ class TestReplicatedObjectController(
headers = {'Content-Length': '0'} headers = {'Content-Length': '0'}
for x in range(constraints.MAX_META_COUNT + 1): for x in range(constraints.MAX_META_COUNT + 1):
headers['X-Object-Meta-%d' % x] = 'v' headers['X-Object-Meta-%d' % x] = 'v'
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, req = Request.blank(
headers=headers) '/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers=headers)
self.app.update_request(req) self.app.update_request(req)
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEqual(resp.status_int, 400) self.assertEqual(resp.status_int, 400)
@ -5272,16 +5248,20 @@ class TestReplicatedObjectController(
if constraints.MAX_META_OVERALL_SIZE - size > 1: if constraints.MAX_META_OVERALL_SIZE - size > 1:
headers['X-Object-Meta-a'] = \ headers['X-Object-Meta-a'] = \
'a' * (constraints.MAX_META_OVERALL_SIZE - size - 1) 'a' * (constraints.MAX_META_OVERALL_SIZE - size - 1)
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, req = Request.blank(
headers=headers) '/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers=headers)
self.app.update_request(req) self.app.update_request(req)
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEqual(resp.status_int, 201) self.assertEqual(resp.status_int, 201)
set_http_connect(201, 201, 201) set_http_connect(201, 201, 201)
headers['X-Object-Meta-a'] = \ headers['X-Object-Meta-a'] = \
'a' * (constraints.MAX_META_OVERALL_SIZE - size) 'a' * (constraints.MAX_META_OVERALL_SIZE - size)
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, req = Request.blank(
headers=headers) '/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers=headers)
self.app.update_request(req) self.app.update_request(req)
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEqual(resp.status_int, 400) self.assertEqual(resp.status_int, 400)
@ -5292,7 +5272,6 @@ class TestReplicatedObjectController(
controller = ReplicatedObjectController( controller = ReplicatedObjectController(
self.app, account, container, obj) self.app, account, container, obj)
self.app.update_request(req) self.app.update_request(req)
self.app.memcache.store = {}
with save_globals(): with save_globals():
new_connect = set_http_connect(*args, **kwargs) new_connect = set_http_connect(*args, **kwargs)
yield controller yield controller
@ -5702,7 +5681,7 @@ class TestReplicatedObjectController(
# not be used in ring order is if affinity is respected. # not be used in ring order is if affinity is respected.
with mock.patch('swift.proxy.server.shuffle', lambda x: x): with mock.patch('swift.proxy.server.shuffle', lambda x: x):
app = proxy_server.Application( app = proxy_server.Application(
conf, FakeMemcache(), conf,
logger=debug_logger('proxy-ut'), logger=debug_logger('proxy-ut'),
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
@ -5718,7 +5697,6 @@ class TestReplicatedObjectController(
req = Request.blank( req = Request.blank(
'/v1/a/c/o.jpg', '/v1/a/c/o.jpg',
headers={'X-Backend-Storage-Policy-Index': str(policy)}) headers={'X-Backend-Storage-Policy-Index': str(policy)})
app.memcache.store = {}
res = controller.GET(req) res = controller.GET(req)
self.assertTrue(res.status.startswith('200 ')) self.assertTrue(res.status.startswith('200 '))
self.assertEqual(3, len(actual_nodes)) self.assertEqual(3, len(actual_nodes))
@ -5818,7 +5796,6 @@ class TestReplicatedObjectController(
controller = ReplicatedObjectController( controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object') self.app, 'account', 'container', 'object')
set_http_connect(200, 200, 202, 202, 202) set_http_connect(200, 200, 202, 202, 202)
self.app.memcache.store = {}
orig_time = time.time orig_time = time.time
try: try:
t = time.time() t = time.time()
@ -8089,7 +8066,7 @@ class TestObjectECRangedGET(unittest.TestCase):
def setUp(self): def setUp(self):
_test_servers[0].logger._clear() _test_servers[0].logger._clear()
self.app = proxy_server.Application( self.app = proxy_server.Application(
None, FakeMemcache(), None,
logger=debug_logger('proxy-ut'), logger=debug_logger('proxy-ut'),
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
@ -8674,7 +8651,7 @@ class TestContainerController(unittest.TestCase):
def setUp(self): def setUp(self):
self.app = proxy_server.Application( self.app = proxy_server.Application(
None, FakeMemcache(), None,
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing(base_port=2000), container_ring=FakeRing(base_port=2000),
logger=debug_logger()) logger=debug_logger())
@ -8774,14 +8751,12 @@ class TestContainerController(unittest.TestCase):
if raise_exc: if raise_exc:
kwargs['raise_exc'] = raise_exc kwargs['raise_exc'] = raise_exc
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', headers={'Content-Length': '0', req = Request.blank('/v1/a/c', headers={'Content-Length': '0',
'Content-Type': 'text/plain'}) 'Content-Type': 'text/plain'})
self.app.update_request(req) self.app.update_request(req)
res = method(req) res = method(req)
self.assertEqual(res.status_int, expected) self.assertEqual(res.status_int, expected)
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/', headers={'Content-Length': '0', req = Request.blank('/v1/a/c/', headers={'Content-Length': '0',
'Content-Type': 'text/plain'}) 'Content-Type': 'text/plain'})
self.app.update_request(req) self.app.update_request(req)
@ -8795,7 +8770,6 @@ class TestContainerController(unittest.TestCase):
def test_status_map(statuses, expected, def test_status_map(statuses, expected,
c_expected=None, a_expected=None, **kwargs): c_expected=None, a_expected=None, **kwargs):
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', {}) req = Request.blank('/v1/a/c', {})
self.app.update_request(req) self.app.update_request(req)
res = controller.HEAD(req) res = controller.HEAD(req)
@ -8823,7 +8797,6 @@ class TestContainerController(unittest.TestCase):
self.assertNotIn('account/a', res.environ) self.assertNotIn('account/a', res.environ)
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', {}) req = Request.blank('/v1/a/c', {})
self.app.update_request(req) self.app.update_request(req)
res = controller.GET(req) res = controller.GET(req)
@ -8882,7 +8855,6 @@ class TestContainerController(unittest.TestCase):
with save_globals(): with save_globals():
mock_conn = set_http_connect(200, 201, 201, 201, mock_conn = set_http_connect(200, 201, 201, 201,
give_connect=capture_requests) give_connect=capture_requests)
self.app.memcache.store = {}
req = Request.blank('/v1/a/test', method='PUT', req = Request.blank('/v1/a/test', method='PUT',
headers={'Content-Length': 0}) headers={'Content-Length': 0})
if requested_policy: if requested_policy:
@ -8934,7 +8906,6 @@ class TestContainerController(unittest.TestCase):
def test_status_map(statuses, expected, **kwargs): def test_status_map(statuses, expected, **kwargs):
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', {}) req = Request.blank('/v1/a/c', {})
req.content_length = 0 req.content_length = 0
self.app.update_request(req) self.app.update_request(req)
@ -8994,7 +8965,6 @@ class TestContainerController(unittest.TestCase):
def test_status_map(statuses, expected, headers=None, **kwargs): def test_status_map(statuses, expected, headers=None, **kwargs):
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', {}, headers=headers) req = Request.blank('/v1/a/c', {}, headers=headers)
req.content_length = 0 req.content_length = 0
self.app.update_request(req) self.app.update_request(req)
@ -9033,7 +9003,6 @@ class TestContainerController(unittest.TestCase):
def test_status_map(statuses, expected, **kwargs): def test_status_map(statuses, expected, **kwargs):
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', {}) req = Request.blank('/v1/a/c', {})
req.content_length = 0 req.content_length = 0
self.app.update_request(req) self.app.update_request(req)
@ -9126,7 +9095,6 @@ class TestContainerController(unittest.TestCase):
def test_acc_missing_returns_404(self): def test_acc_missing_returns_404(self):
for meth in ('DELETE', 'PUT'): for meth in ('DELETE', 'PUT'):
with save_globals(): with save_globals():
self.app.memcache = FakeMemcacheReturnsNone()
self.app._error_limiting = {} self.app._error_limiting = {}
controller = proxy_server.ContainerController(self.app, controller = proxy_server.ContainerController(self.app,
'account', 'account',
@ -9136,7 +9104,6 @@ class TestContainerController(unittest.TestCase):
missing_container=True) missing_container=True)
else: else:
set_http_connect(200, 200, 200, 200) set_http_connect(200, 200, 200, 200)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', req = Request.blank('/v1/a/c',
environ={'REQUEST_METHOD': meth}) environ={'REQUEST_METHOD': meth})
self.app.update_request(req) self.app.update_request(req)
@ -9251,7 +9218,6 @@ class TestContainerController(unittest.TestCase):
self.assert_status_map(controller.DELETE, self.assert_status_map(controller.DELETE,
(200, 204, 503, 404), 503) (200, 204, 503, 404), 503)
self.app.memcache = FakeMemcacheReturnsNone()
# 200: Account check, 404x3: Container check # 200: Account check, 404x3: Container check
self.assert_status_map(controller.DELETE, self.assert_status_map(controller.DELETE,
(200, 404, 404, 404), 404) (200, 404, 404, 404), 404)
@ -9556,7 +9522,7 @@ class TestContainerController(unittest.TestCase):
def _do_test(method): def _do_test(method):
with save_globals(): with save_globals():
swift.proxy.controllers.Controller.account_info = account_info swift.proxy.controllers.Controller.account_info = account_info
app = proxy_server.Application(None, FakeMemcache(), app = proxy_server.Application(None,
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
set_http_connect(201, 201, 201) set_http_connect(201, 201, 201)
@ -9595,7 +9561,7 @@ class TestContainerController(unittest.TestCase):
def _do_test(method): def _do_test(method):
with save_globals(): with save_globals():
swift.proxy.controllers.Controller.account_info = account_info swift.proxy.controllers.Controller.account_info = account_info
app = proxy_server.Application(None, FakeMemcache(), app = proxy_server.Application(None,
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
set_http_connect(201, 201, 201) set_http_connect(201, 201, 201)
@ -9982,7 +9948,7 @@ class TestAccountController(unittest.TestCase):
def setUp(self): def setUp(self):
conf = {'error_suppression_interval': 0} conf = {'error_suppression_interval': 0}
self.app = proxy_server.Application(conf, FakeMemcache(), self.app = proxy_server.Application(conf,
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
@ -10065,13 +10031,11 @@ class TestAccountController(unittest.TestCase):
self.assert_status_map(controller.GET, (404, 404, 503), 404, 404) self.assert_status_map(controller.GET, (404, 404, 503), 404, 404)
self.assert_status_map(controller.GET, (404, 503, 503), 503) self.assert_status_map(controller.GET, (404, 503, 503), 503)
self.app.memcache = FakeMemcacheReturnsNone()
self.assert_status_map(controller.GET, (404, 404, 404), 404, 404) self.assert_status_map(controller.GET, (404, 404, 404), 404, 404)
def test_GET_autocreate(self): def test_GET_autocreate(self):
with save_globals(): with save_globals():
controller = proxy_server.AccountController(self.app, 'a') controller = proxy_server.AccountController(self.app, 'a')
self.app.memcache = FakeMemcacheReturnsNone()
self.assertFalse(self.app.account_autocreate) self.assertFalse(self.app.account_autocreate)
# Repeat the test for autocreate = False and 404 by all # Repeat the test for autocreate = False and 404 by all
self.assert_status_map(controller.GET, self.assert_status_map(controller.GET,
@ -10138,7 +10102,6 @@ class TestAccountController(unittest.TestCase):
# Same behaviour as GET # Same behaviour as GET
with save_globals(): with save_globals():
controller = proxy_server.AccountController(self.app, 'a') controller = proxy_server.AccountController(self.app, 'a')
self.app.memcache = FakeMemcacheReturnsNone()
self.assertFalse(self.app.account_autocreate) self.assertFalse(self.app.account_autocreate)
self.assert_status_map(controller.HEAD, self.assert_status_map(controller.HEAD,
(404, 404, 404), 404) (404, 404, 404), 404)
@ -10154,7 +10117,6 @@ class TestAccountController(unittest.TestCase):
def test_POST_autocreate(self): def test_POST_autocreate(self):
with save_globals(): with save_globals():
controller = proxy_server.AccountController(self.app, 'a') controller = proxy_server.AccountController(self.app, 'a')
self.app.memcache = FakeMemcacheReturnsNone()
# first test with autocreate being False # first test with autocreate being False
self.assertFalse(self.app.account_autocreate) self.assertFalse(self.app.account_autocreate)
self.assert_status_map(controller.POST, self.assert_status_map(controller.POST,
@ -10176,7 +10138,6 @@ class TestAccountController(unittest.TestCase):
def test_POST_autocreate_with_sysmeta(self): def test_POST_autocreate_with_sysmeta(self):
with save_globals(): with save_globals():
controller = proxy_server.AccountController(self.app, 'a') controller = proxy_server.AccountController(self.app, 'a')
self.app.memcache = FakeMemcacheReturnsNone()
# first test with autocreate being False # first test with autocreate being False
self.assertFalse(self.app.account_autocreate) self.assertFalse(self.app.account_autocreate)
self.assert_status_map(controller.POST, self.assert_status_map(controller.POST,
@ -10249,7 +10210,6 @@ class TestAccountController(unittest.TestCase):
def test_status_map(statuses, expected, **kwargs): def test_status_map(statuses, expected, **kwargs):
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a', {}) req = Request.blank('/v1/a', {})
req.content_length = 0 req.content_length = 0
self.app.update_request(req) self.app.update_request(req)
@ -10452,7 +10412,6 @@ class TestAccountController(unittest.TestCase):
def test_status_map(statuses, expected, **kwargs): def test_status_map(statuses, expected, **kwargs):
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a', {'REQUEST_METHOD': 'DELETE'}) req = Request.blank('/v1/a', {'REQUEST_METHOD': 'DELETE'})
req.content_length = 0 req.content_length = 0
self.app.update_request(req) self.app.update_request(req)
@ -10475,7 +10434,6 @@ class TestAccountController(unittest.TestCase):
def test_status_map(statuses, expected, **kwargs): def test_status_map(statuses, expected, **kwargs):
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a?whoops', req = Request.blank('/v1/a?whoops',
environ={'REQUEST_METHOD': 'DELETE'}) environ={'REQUEST_METHOD': 'DELETE'})
req.content_length = 0 req.content_length = 0
@ -10500,11 +10458,10 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase):
def setUp(self): def setUp(self):
conf = {'account_autocreate': 'yes'} conf = {'account_autocreate': 'yes'}
self.app = listing_formats.ListingFilter( self.app = listing_formats.ListingFilter(
proxy_server.Application(conf, FakeMemcache(), proxy_server.Application(conf,
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()), container_ring=FakeRing()),
{}) {})
self.app.app.memcache = FakeMemcacheReturnsNone()
def test_GET_autocreate_accept_json(self): def test_GET_autocreate_accept_json(self):
with save_globals(): with save_globals():
@ -10611,7 +10568,7 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase):
version=2, acl_dict=acl)} version=2, acl_dict=acl)}
app = proxy_server.Application( app = proxy_server.Application(
None, FakeMemcache(), account_ring=FakeRing(), None, account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
with save_globals(): with save_globals():
@ -10686,7 +10643,7 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase):
resps_to_send.append(canned_resp) resps_to_send.append(canned_resp)
app = proxy_server.Application( app = proxy_server.Application(
None, FakeMemcache(), account_ring=FakeRing(), None, account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
app.allow_account_management = True app.allow_account_management = True
@ -10854,7 +10811,7 @@ class TestSwiftInfo(unittest.TestCase):
utils._swift_admin_info = {} utils._swift_admin_info = {}
def test_registered_defaults(self): def test_registered_defaults(self):
app = proxy_server.Application({}, FakeMemcache(), app = proxy_server.Application({},
account_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing()) container_ring=FakeRing())
req = Request.blank('/info') req = Request.blank('/info')

82
test/unit/proxy/test_sysmeta.py
View File

@ -132,7 +132,7 @@ class TestObjectSysmeta(unittest.TestCase):
def setUp(self): def setUp(self):
skip_if_no_xattrs() skip_if_no_xattrs()
self.app = proxy.Application(None, FakeMemcache(), self.app = proxy.Application(None,
logger=debug_logger('proxy-ut'), logger=debug_logger('proxy-ut'),
account_ring=FakeRing(replicas=1), account_ring=FakeRing(replicas=1),
container_ring=FakeRing(replicas=1)) container_ring=FakeRing(replicas=1))
@ -227,8 +227,9 @@ class TestObjectSysmeta(unittest.TestCase):
def test_sysmeta_replaced_by_PUT(self): def test_sysmeta_replaced_by_PUT(self):
path = '/v1/a/c/o' path = '/v1/a/c/o'
cache = FakeMemcache()
env = {'REQUEST_METHOD': 'PUT'} env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.original_sysmeta_headers_1) hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_sysmeta_headers_2) hdrs.update(self.original_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1) hdrs.update(self.original_meta_headers_1)
@ -237,7 +238,7 @@ class TestObjectSysmeta(unittest.TestCase):
resp = req.get_response(self.app) resp = req.get_response(self.app)
self._assertStatus(resp, 201) self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'PUT'} env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.changed_sysmeta_headers) hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers) hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.changed_meta_headers) hdrs.update(self.changed_meta_headers)
@ -247,7 +248,7 @@ class TestObjectSysmeta(unittest.TestCase):
resp = req.get_response(self.app) resp = req.get_response(self.app)
self._assertStatus(resp, 201) self._assertStatus(resp, 201)
req = Request.blank(path, environ={}) req = Request.blank(path, environ={'swift.cache': cache})
resp = req.get_response(self.app) resp = req.get_response(self.app)
self._assertStatus(resp, 200) self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers) self._assertInHeaders(resp, self.changed_sysmeta_headers)
@ -257,29 +258,30 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertInHeaders(resp, self.new_meta_headers) self._assertInHeaders(resp, self.new_meta_headers)
self._assertNotInHeaders(resp, self.original_meta_headers_2) self._assertNotInHeaders(resp, self.original_meta_headers_2)
def _test_sysmeta_not_updated_by_POST(self, app): def test_sysmeta_not_updated_by_POST(self):
# check sysmeta is not changed by a POST but user meta is replaced # check sysmeta is not changed by a POST but user meta is replaced
path = '/v1/a/c/o' path = '/v1/a/c/o'
cache = FakeMemcache()
env = {'REQUEST_METHOD': 'PUT'} env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.original_sysmeta_headers_1) hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_meta_headers_1) hdrs.update(self.original_meta_headers_1)
req = Request.blank(path, environ=env, headers=hdrs, body=b'x') req = Request.blank(path, environ=env, headers=hdrs, body=b'x')
resp = req.get_response(app) resp = req.get_response(self.app)
self._assertStatus(resp, 201) self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'POST'} env = {'REQUEST_METHOD': 'POST', 'swift.cache': cache}
hdrs = dict(self.changed_sysmeta_headers) hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers) hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.changed_meta_headers) hdrs.update(self.changed_meta_headers)
hdrs.update(self.new_meta_headers) hdrs.update(self.new_meta_headers)
hdrs.update(self.bad_headers) hdrs.update(self.bad_headers)
req = Request.blank(path, environ=env, headers=hdrs) req = Request.blank(path, environ=env, headers=hdrs)
resp = req.get_response(app) resp = req.get_response(self.app)
self._assertStatus(resp, 202) self._assertStatus(resp, 202)
req = Request.blank(path, environ={}) req = Request.blank(path, environ={'swift.cache': cache})
resp = req.get_response(app) resp = req.get_response(self.app)
self._assertStatus(resp, 200) self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.original_sysmeta_headers_1) self._assertInHeaders(resp, self.original_sysmeta_headers_1)
self._assertNotInHeaders(resp, self.new_sysmeta_headers) self._assertNotInHeaders(resp, self.new_sysmeta_headers)
@ -287,31 +289,28 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertInHeaders(resp, self.new_meta_headers) self._assertInHeaders(resp, self.new_meta_headers)
self._assertNotInHeaders(resp, self.bad_headers) self._assertNotInHeaders(resp, self.bad_headers)
env = {'REQUEST_METHOD': 'PUT'} env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.changed_sysmeta_headers) hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers) hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.bad_headers) hdrs.update(self.bad_headers)
req = Request.blank(path, environ=env, headers=hdrs, body=b'x') req = Request.blank(path, environ=env, headers=hdrs, body=b'x')
resp = req.get_response(app) resp = req.get_response(self.app)
self._assertStatus(resp, 201) self._assertStatus(resp, 201)
req = Request.blank(path, environ={}) req = Request.blank(path, environ={'swift.cache': cache})
resp = req.get_response(app) resp = req.get_response(self.app)
self._assertStatus(resp, 200) self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers) self._assertInHeaders(resp, self.changed_sysmeta_headers)
self._assertInHeaders(resp, self.new_sysmeta_headers) self._assertInHeaders(resp, self.new_sysmeta_headers)
self._assertNotInHeaders(resp, self.original_sysmeta_headers_2) self._assertNotInHeaders(resp, self.original_sysmeta_headers_2)
def test_sysmeta_not_updated_by_POST(self):
# test fast-post by issuing requests to the proxy app
self._test_sysmeta_not_updated_by_POST(self.app)
def test_sysmeta_updated_by_COPY(self): def test_sysmeta_updated_by_COPY(self):
# check sysmeta is updated by a COPY in same way as user meta by # check sysmeta is updated by a COPY in same way as user meta by
# issuing requests to the copy middleware app # issuing requests to the copy middleware app
path = '/v1/a/c/o' path = '/v1/a/c/o'
dest = '/c/o2' dest = '/c/o2'
env = {'REQUEST_METHOD': 'PUT'} cache = FakeMemcache()
env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.original_sysmeta_headers_1) hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_sysmeta_headers_2) hdrs.update(self.original_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1) hdrs.update(self.original_meta_headers_1)
@ -322,7 +321,7 @@ class TestObjectSysmeta(unittest.TestCase):
resp = req.get_response(self.copy_app) resp = req.get_response(self.copy_app)
self._assertStatus(resp, 201) self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'COPY'} env = {'REQUEST_METHOD': 'COPY', 'swift.cache': cache}
hdrs = dict(self.changed_sysmeta_headers) hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers) hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.changed_meta_headers) hdrs.update(self.changed_meta_headers)
@ -345,7 +344,7 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2) self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers) self._assertNotInHeaders(resp, self.bad_headers)
req = Request.blank('/v1/a/c/o2', environ={}) req = Request.blank('/v1/a/c/o2', environ={'swift.cache': cache})
resp = req.get_response(self.copy_app) resp = req.get_response(self.copy_app)
self._assertStatus(resp, 200) self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers) self._assertInHeaders(resp, self.changed_sysmeta_headers)
@ -363,7 +362,8 @@ class TestObjectSysmeta(unittest.TestCase):
# check sysmeta is updated by a PUT with x-copy-from in same way as # check sysmeta is updated by a PUT with x-copy-from in same way as
# user meta by issuing requests to the copy middleware app # user meta by issuing requests to the copy middleware app
path = '/v1/a/c/o' path = '/v1/a/c/o'
env = {'REQUEST_METHOD': 'PUT'} cache = FakeMemcache()
env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.original_sysmeta_headers_1) hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_sysmeta_headers_2) hdrs.update(self.original_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1) hdrs.update(self.original_meta_headers_1)
@ -372,7 +372,7 @@ class TestObjectSysmeta(unittest.TestCase):
resp = req.get_response(self.copy_app) resp = req.get_response(self.copy_app)
self._assertStatus(resp, 201) self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'PUT'} env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.changed_sysmeta_headers) hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers) hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.changed_meta_headers) hdrs.update(self.changed_meta_headers)
@ -390,7 +390,7 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertInHeaders(resp, self.original_meta_headers_2) self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers) self._assertNotInHeaders(resp, self.bad_headers)
req = Request.blank('/v1/a/c/o2', environ={}) req = Request.blank('/v1/a/c/o2', environ={'swift.cache': cache})
resp = req.get_response(self.copy_app) resp = req.get_response(self.copy_app)
self._assertStatus(resp, 200) self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers) self._assertInHeaders(resp, self.changed_sysmeta_headers)
@ -401,40 +401,41 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertInHeaders(resp, self.original_meta_headers_2) self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers) self._assertNotInHeaders(resp, self.bad_headers)
def _test_transient_sysmeta_replaced_by_PUT_or_POST(self, app): def test_transient_sysmeta_replaced_by_PUT_or_POST(self):
# check transient_sysmeta is replaced en-masse by a POST # check transient_sysmeta is replaced en-masse by a POST
path = '/v1/a/c/o' path = '/v1/a/c/o'
cache = FakeMemcache()
env = {'REQUEST_METHOD': 'PUT'} env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.original_transient_sysmeta_headers_1) hdrs = dict(self.original_transient_sysmeta_headers_1)
hdrs.update(self.original_transient_sysmeta_headers_2) hdrs.update(self.original_transient_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1) hdrs.update(self.original_meta_headers_1)
req = Request.blank(path, environ=env, headers=hdrs, body=b'x') req = Request.blank(path, environ=env, headers=hdrs, body=b'x')
resp = req.get_response(app) resp = req.get_response(self.app)
self._assertStatus(resp, 201) self._assertStatus(resp, 201)
req = Request.blank(path, environ={}) req = Request.blank(path, environ={})
resp = req.get_response(app) resp = req.get_response(self.app)
self._assertStatus(resp, 200) self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.original_transient_sysmeta_headers_1) self._assertInHeaders(resp, self.original_transient_sysmeta_headers_1)
self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2) self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2)
self._assertInHeaders(resp, self.original_meta_headers_1) self._assertInHeaders(resp, self.original_meta_headers_1)
info = get_object_info(req.environ, app) info = get_object_info(req.environ, self.app)
self.assertEqual(2, len(info.get('transient_sysmeta', ()))) self.assertEqual(2, len(info.get('transient_sysmeta', ())))
self.assertEqual({'testa': 'A', 'testb': 'B'}, self.assertEqual({'testa': 'A', 'testb': 'B'},
info['transient_sysmeta']) info['transient_sysmeta'])
# POST will replace all existing transient_sysmeta and usermeta values # POST will replace all existing transient_sysmeta and usermeta values
env = {'REQUEST_METHOD': 'POST'} env = {'REQUEST_METHOD': 'POST', 'swift.cache': cache}
hdrs = dict(self.changed_transient_sysmeta_headers) hdrs = dict(self.changed_transient_sysmeta_headers)
hdrs.update(self.new_transient_sysmeta_headers_1) hdrs.update(self.new_transient_sysmeta_headers_1)
req = Request.blank(path, environ=env, headers=hdrs) req = Request.blank(path, environ=env, headers=hdrs)
resp = req.get_response(app) resp = req.get_response(self.app)
self._assertStatus(resp, 202) self._assertStatus(resp, 202)
req = Request.blank(path, environ={}) req = Request.blank(path, environ={})
resp = req.get_response(app) resp = req.get_response(self.app)
self._assertStatus(resp, 200) self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_transient_sysmeta_headers) self._assertInHeaders(resp, self.changed_transient_sysmeta_headers)
self._assertInHeaders(resp, self.new_transient_sysmeta_headers_1) self._assertInHeaders(resp, self.new_transient_sysmeta_headers_1)
@ -442,21 +443,21 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertNotInHeaders(resp, self._assertNotInHeaders(resp,
self.original_transient_sysmeta_headers_2) self.original_transient_sysmeta_headers_2)
info = get_object_info(req.environ, app) info = get_object_info(req.environ, self.app)
self.assertEqual(2, len(info.get('transient_sysmeta', ()))) self.assertEqual(2, len(info.get('transient_sysmeta', ())))
self.assertEqual({'testa': 'changed_A', 'testc': 'C'}, self.assertEqual({'testa': 'changed_A', 'testc': 'C'},
info['transient_sysmeta']) info['transient_sysmeta'])
# subsequent PUT replaces all transient_sysmeta and usermeta values # subsequent PUT replaces all transient_sysmeta and usermeta values
env = {'REQUEST_METHOD': 'PUT'} env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.new_transient_sysmeta_headers_2) hdrs = dict(self.new_transient_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_2) hdrs.update(self.original_meta_headers_2)
req = Request.blank(path, environ=env, headers=hdrs, body=b'x') req = Request.blank(path, environ=env, headers=hdrs, body=b'x')
resp = req.get_response(app) resp = req.get_response(self.app)
self._assertStatus(resp, 201) self._assertStatus(resp, 201)
req = Request.blank(path, environ={}) req = Request.blank(path, environ={'swift.cache': cache})
resp = req.get_response(app) resp = req.get_response(self.app)
self._assertStatus(resp, 200) self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.original_meta_headers_2) self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertInHeaders(resp, self.new_transient_sysmeta_headers_2) self._assertInHeaders(resp, self.new_transient_sysmeta_headers_2)
@ -470,9 +471,6 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertNotInHeaders(resp, self._assertNotInHeaders(resp,
self.original_transient_sysmeta_headers_2) self.original_transient_sysmeta_headers_2)
info = get_object_info(req.environ, app) info = get_object_info(req.environ, self.app)
self.assertEqual(1, len(info.get('transient_sysmeta', ()))) self.assertEqual(1, len(info.get('transient_sysmeta', ())))
self.assertEqual({'testd': 'D'}, info['transient_sysmeta']) self.assertEqual({'testd': 'D'}, info['transient_sysmeta'])
def test_transient_sysmeta_replaced_by_PUT_or_POST(self):
self._test_transient_sysmeta_replaced_by_PUT_or_POST(self.app)