openstack/swift
Code Issues Proposed changes

Get rid of memcache attr on proxy-server app

Browse Source 
In a running proxy server, this only ever comes from one place:
a key in the wsgi env, populated by the memcache middleware. By
littering proxy controller code with things like

   memcache = getattr(app, 'memcache', None) or env.get('swift.cache')

we cause ourselves to second-guess that.

Change-Id: Ia652b1381990b61194978c4efaebac7d2a602ca3
This commit is contained in:
Tim Burke 2020-06-14 16:57:08 -07:00
parent 5c087ad7b9
commit 45be775ebf
9 changed files with 171 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
swift/proxy/controllers/base.py
View File

@ -45,7 +45,8 @@ from swift.common.wsgi import make_pre_authed_env, make_pre_authed_request
from swift.common.utils import Timestamp, WatchdogTimeout, config_true_value, \
public, split_path, list_from_csv, GreenthreadSafeIterator, \
GreenAsyncPile, quorum_size, parse_content_type, drain_and_close, \
document_iters_to_http_response_body, ShardRange, find_shard_range
document_iters_to_http_response_body, ShardRange, find_shard_range, \
cache_from_env
from swift.common.bufferedhttp import http_connect
from swift.common import constraints
from swift.common.exceptions import ChunkReadTimeout, ChunkWriteTimeout, \
@ -541,8 +542,7 @@ def set_info_cache(app, env, account, container, resp):
"""
cache_key = get_cache_key(account, container)
infocache = env.setdefault('swift.infocache', {})
memcache = getattr(app, 'memcache', None) or env.get('swift.cache')
memcache = cache_from_env(env, True)
if resp is None:
infocache.pop(cache_key, None)
if memcache:
@ -645,7 +645,7 @@ def _get_info_from_memcache(app, env, account, container=None):
returns None if memcache is not in use.
"""
cache_key = get_cache_key(account, container)
memcache = getattr(app, 'memcache', None) or env.get('swift.cache')
memcache = cache_from_env(env, True)
if memcache:
info = memcache.get(cache_key)
if info and six.PY2:
@ -1678,12 +1678,12 @@ class Controller(object):
headers['referer'] = referer
return headers
def account_info(self, account, req=None):
def account_info(self, account, req):
"""
Get account information, and also verify that the account exists.
:param account: native str name of the account to get the info for
:param req: caller's HTTP request context object (optional)
:param req: caller's HTTP request context object
:returns: tuple of (account partition, account nodes, container_count)
or (None, None, None) if it does not exist
"""
@ -1704,14 +1704,14 @@ class Controller(object):
partition, nodes = self.app.account_ring.get_nodes(account)
return partition, nodes, container_count
def container_info(self, account, container, req=None):
def container_info(self, account, container, req):
"""
Get container information and thusly verify container existence.
This will also verify account existence.
:param account: native-str account name for the container
:param container: native-str container name to look up
:param req: caller's HTTP request context object (optional)
:param req: caller's HTTP request context object
:returns: dict containing at least container partition ('partition'),
container nodes ('containers'), container read
acl ('read_acl'), container write acl ('write_acl'),
@ -2268,8 +2268,7 @@ class Controller(object):
cache_key = get_cache_key(account, container, shard='updating')
infocache = req.environ.setdefault('swift.infocache', {})
memcache = getattr(self.app, 'memcache', None) or req.environ.get(
'swift.cache')
memcache = cache_from_env(req.environ, True)
cached_ranges = infocache.get(cache_key)
if cached_ranges is None and memcache:

7
swift/proxy/server.py
View File

@ -32,7 +32,7 @@ from swift.common import constraints
from swift.common.http import is_server_error
from swift.common.storage_policy import POLICIES
from swift.common.ring import Ring
from swift.common.utils import Watchdog, cache_from_env, get_logger, \
from swift.common.utils import Watchdog, get_logger, \
get_remote_client, split_path, config_true_value, generate_trans_id, \
affinity_key_function, affinity_locality_predicate, list_from_csv, \
register_swift_info, readconf, config_auto_int_value
@ -170,7 +170,7 @@ class ProxyOverrideOptions(object):
class Application(object):
"""WSGI application for the proxy server."""
def __init__(self, conf, memcache=None, logger=None, account_ring=None,
def __init__(self, conf, logger=None, account_ring=None,
container_ring=None):
if conf is None:
conf = {}
@ -218,7 +218,6 @@ class Application(object):
for policy in POLICIES:
policy.load_ring(swift_dir)
self.obj_controller_router = ObjectControllerRouter()
self.memcache = memcache
mimetypes.init(mimetypes.knownfiles +
[os.path.join(swift_dir, 'mime.types')])
self.account_autocreate = \
@ -454,8 +453,6 @@ class Application(object):
:param start_response: WSGI callable
"""
try:
if self.memcache is None:
self.memcache = cache_from_env(env, True)
req = self.update_request(Request(env))
return self.handle_request(req)(env, start_response)
except UnicodeError:

4
test/unit/common/middleware/test_copy.py
View File

@ -24,7 +24,7 @@ from swift.common.middleware import copy
from swift.common.storage_policy import POLICIES
from swift.common.swob import Request, HTTPException
from swift.common.utils import closing_if_possible
from test.unit import patch_policies, debug_logger, FakeMemcache, FakeRing
from test.unit import patch_policies, debug_logger, FakeRing
from test.unit.common.middleware.helpers import FakeSwift
from test.unit.proxy.controllers.test_obj import set_http_connect, \
PatchedObjControllerApp
@ -1316,7 +1316,7 @@ class TestServerSideCopyMiddlewareWithEC(unittest.TestCase):
self.logger = debug_logger('proxy-server')
self.logger.thread_locals = ('txn1', '127.0.0.2')
self.app = PatchedObjControllerApp(
None, FakeMemcache(), account_ring=FakeRing(),
None, account_ring=FakeRing(),
container_ring=FakeRing(), logger=self.logger)
self.ssc = copy.filter_factory({})(self.app)
self.ssc.logger = self.app.logger

8
test/unit/proxy/controllers/test_account.py
View File

@ -21,8 +21,7 @@ from swift.common.middleware.acl import format_acl
from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_account_info
from swift.common import constraints
from test.unit import fake_http_connect, FakeRing, FakeMemcache, \
mocked_http_conn
from test.unit import fake_http_connect, FakeRing, mocked_http_conn
from swift.common.storage_policy import StoragePolicy
from swift.common.request_helpers import get_sys_meta_prefix
import swift.proxy.controllers.base
@ -38,7 +37,7 @@ class TestAccountController(unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(
None, FakeMemcache(),
None,
account_ring=FakeRing(), container_ring=FakeRing())
def _make_callback_func(self, context):
@ -344,7 +343,6 @@ class TestAccountController4Replicas(TestAccountController):
def setUp(self):
self.app = proxy_server.Application(
None,
FakeMemcache(),
account_ring=FakeRing(replicas=4),
container_ring=FakeRing(replicas=4))
@ -413,7 +411,7 @@ class TestAccountController4Replicas(TestAccountController):
class TestGetAccountInfo(unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(
None, FakeMemcache(),
None,
account_ring=FakeRing(), container_ring=FakeRing())
def test_get_deleted_account_410(self):

6
test/unit/proxy/controllers/test_base.py
View File

@ -182,7 +182,7 @@ class FakeCache(FakeMemcache):
@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())])
class TestFuncs(unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(None, FakeMemcache(),
self.app = proxy_server.Application(None,
account_ring=FakeRing(),
container_ring=FakeRing(),
logger=FakeLogger())
@ -710,7 +710,7 @@ class TestFuncs(unittest.TestCase):
resp,
headers_to_container_info(headers.items(), 200))
def test_container_info_without_req(self):
def test_container_info_needs_req(self):
base = Controller(self.app)
base.account_name = 'a'
base.container_name = 'c'
@ -719,7 +719,7 @@ class TestFuncs(unittest.TestCase):
'http_connect', fake_http_connect(200)):
container_info = \
base.container_info(base.account_name,
base.container_name)
base.container_name, Request.blank('/'))
self.assertEqual(container_info['status'], 0)
def test_headers_to_account_info_missing(self):

18
test/unit/proxy/controllers/test_container.py
View File

@ -27,7 +27,7 @@ from swift.common.swob import Request, bytes_to_wsgi, str_to_wsgi, wsgi_quote
from swift.common.utils import ShardRange, Timestamp
from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_container_info, \
Controller, get_container_info
Controller, get_container_info, get_cache_key
from test import annotate_failure
from test.unit import fake_http_connect, FakeRing, FakeMemcache, \
make_timestamp_iter
@ -49,7 +49,7 @@ class TestContainerController(TestRingBase):
self.logger = debug_logger()
self.container_ring = FakeRing(replicas=self.CONTAINER_REPLICAS,
max_more_nodes=9)
self.app = proxy_server.Application(None, FakeMemcache(),
self.app = proxy_server.Application(None,
logger=self.logger,
account_ring=FakeRing(),
container_ring=self.container_ring)
@ -97,7 +97,9 @@ class TestContainerController(TestRingBase):
with mock.patch(
'swift.proxy.controllers.base.http_connect',
fake_http_connect(*responses)):
req = Request.blank('/v1/a/c')
cache = FakeMemcache()
cache.set(get_cache_key('a'), {'status': 204})
req = Request.blank('/v1/a/c', environ={'swift.cache': cache})
resp = getattr(controller, method)(req)
self.assertEqual(expected,
@ -106,9 +108,10 @@ class TestContainerController(TestRingBase):
(expected, resp.status_int, str(responses)))
def test_container_info_got_cached(self):
memcache = FakeMemcache()
controller = proxy_server.ContainerController(self.app, 'a', 'c')
with mocked_http_conn(200, 200) as mock_conn:
req = Request.blank('/v1/a/c')
req = Request.blank('/v1/a/c', {'swift.cache': memcache})
resp = controller.HEAD(req)
self.assertEqual(2, resp.status_int // 100)
self.assertEqual(['/a', '/a/c'],
@ -119,12 +122,13 @@ class TestContainerController(TestRingBase):
info_cache = resp.environ['swift.infocache']
self.assertIn("container/a/c", resp.environ['swift.infocache'])
self.assertEqual(header_info, info_cache['container/a/c'])
self.assertEqual(header_info, self.app.memcache.get('container/a/c'))
self.assertEqual(header_info, memcache.get('container/a/c'))
# The failure doesn't lead to cache eviction
errors = [500] * self.CONTAINER_REPLICAS * 2
with mocked_http_conn(*errors) as mock_conn:
req = Request.blank('/v1/a/c', {'swift.infocache': info_cache})
req = Request.blank('/v1/a/c', {'swift.infocache': info_cache,
'swift.cache': memcache})
resp = controller.HEAD(req)
self.assertEqual(5, resp.status_int // 100)
self.assertEqual(['/a/c'] * self.CONTAINER_REPLICAS * 2,
@ -134,7 +138,7 @@ class TestContainerController(TestRingBase):
self.assertIn("container/a/c", resp.environ['swift.infocache'])
# NB: this is the *old* header_info, from the good req
self.assertEqual(header_info, info_cache['container/a/c'])
self.assertEqual(header_info, self.app.memcache.get('container/a/c'))
self.assertEqual(header_info, memcache.get('container/a/c'))
@mock.patch('swift.proxy.controllers.container.clear_info_cache')
@mock.patch.object(Controller, 'make_requests')

6
test/unit/proxy/controllers/test_obj.py
View File

@ -48,7 +48,7 @@ from swift.proxy.controllers.base import \
from swift.common.storage_policy import POLICIES, ECDriverError, \
StoragePolicy, ECStoragePolicy
from test.unit import FakeRing, FakeMemcache, fake_http_connect, \
from test.unit import FakeRing, fake_http_connect, \
debug_logger, patch_policies, SlowBody, FakeStatus, \
DEFAULT_TEST_EC_TYPE, encode_frag_archive_bodies, make_ec_object_stub, \
fake_ec_node_response, StubResponse, mocked_http_conn, \
@ -179,7 +179,7 @@ class BaseObjectControllerMixin(object):
# increase connection timeout to avoid intermittent failures
conf = {'conn_timeout': 1.0}
self.app = PatchedObjControllerApp(
conf, FakeMemcache(), account_ring=FakeRing(),
conf, account_ring=FakeRing(),
container_ring=FakeRing(), logger=self.logger)
# you can over-ride the container_info just by setting it on the app
@ -969,7 +969,7 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin):
self, conf, policy_conf, policy, affinity_regions, affinity_count):
conf['policy_config'] = policy_conf
app = PatchedObjControllerApp(
conf, FakeMemcache(), account_ring=FakeRing(),
conf, account_ring=FakeRing(),
container_ring=FakeRing(), logger=self.logger)
controller = self.controller_cls(app, 'a', 'c', 'o')

239
test/unit/proxy/test_server.py
View File

@ -173,14 +173,6 @@ def set_node_errors(proxy_app, ring_node, value, last_error):
stats['last_error'] = last_error
class FakeMemcacheReturnsNone(FakeMemcache):
def get(self, key):
# Returns None as the timestamp of the container; assumes we're only
# using the FakeMemcache for container existence checks.
return None
@contextmanager
def save_globals():
orig_http_connect = getattr(swift.proxy.controllers.base, 'http_connect',
@ -251,22 +243,23 @@ class TestController(unittest.TestCase):
self.account_ring = FakeRing()
self.container_ring = FakeRing()
self.memcache = FakeMemcache()
app = proxy_server.Application(None, self.memcache,
app = proxy_server.Application(None,
account_ring=self.account_ring,
container_ring=self.container_ring)
self.controller = swift.proxy.controllers.Controller(app)
class FakeReq(object):
def __init__(self):
def __init__(self, memcache):
self.url = "/foo/bar"
self.method = "METHOD"
self.environ = {"swift.cache": memcache}
def as_referer(self):
return self.method + ' ' + self.url
self.account = 'some_account'
self.container = 'some_container'
self.request = FakeReq()
self.request = FakeReq(self.memcache)
self.read_acl = 'read_acl'
self.write_acl = 'write_acl'
@ -290,29 +283,31 @@ class TestController(unittest.TestCase):
self.assertEqual(n, nodes)
def test_account_info_container_count(self):
cache_key = get_cache_key(self.account)
with save_globals():
set_http_connect(200, count=123)
partition, nodes, count = \
self.controller.account_info(self.account)
self.controller.account_info(self.account, self.request)
self.assertEqual(count, 123)
with save_globals():
set_http_connect(200, count='123')
del self.request.environ['swift.infocache'][cache_key]
partition, nodes, count = \
self.controller.account_info(self.account)
self.controller.account_info(self.account, self.request)
self.assertEqual(count, 123)
with save_globals():
cache_key = get_cache_key(self.account)
account_info = {'status': 200, 'container_count': 1234}
self.memcache.set(cache_key, account_info)
del self.request.environ['swift.infocache'][cache_key]
partition, nodes, count = \
self.controller.account_info(self.account)
self.controller.account_info(self.account, self.request)
self.assertEqual(count, 1234)
with save_globals():
cache_key = get_cache_key(self.account)
account_info = {'status': 200, 'container_count': '1234'}
self.memcache.set(cache_key, account_info)
del self.request.environ['swift.infocache'][cache_key]
partition, nodes, count = \
self.controller.account_info(self.account)
self.controller.account_info(self.account, self.request)
self.assertEqual(count, 1234)
def test_make_requests(self):
@ -407,7 +402,6 @@ class TestController(unittest.TestCase):
def test_account_info_no_account(self):
with save_globals():
self.memcache.store = {}
set_http_connect(404, 404, 404)
partition, nodes, count = \
self.controller.account_info(self.account, self.request)
@ -514,13 +508,13 @@ class TestController(unittest.TestCase):
def test_get_account_info_returns_values_as_strings(self):
app = mock.MagicMock()
app.memcache = mock.MagicMock()
app.memcache.get = mock.MagicMock()
app.memcache.get.return_value = {
memcache = mock.MagicMock()
memcache.get = mock.MagicMock()
memcache.get.return_value = {
u'foo': u'\u2603',
u'meta': {u'bar': u'\u2603'},
u'sysmeta': {u'baz': u'\u2603'}}
env = {'PATH_INFO': '/v1/a'}
env = {'PATH_INFO': '/v1/a', 'swift.cache': memcache}
ai = get_account_info(env, app)
# Test info is returned as strings
@ -539,14 +533,14 @@ class TestController(unittest.TestCase):
def test_get_container_info_returns_values_as_strings(self):
app = mock.MagicMock()
app.memcache = mock.MagicMock()
app.memcache.get = mock.MagicMock()
app.memcache.get.return_value = {
memcache = mock.MagicMock()
memcache.get = mock.MagicMock()
memcache.get.return_value = {
u'foo': u'\u2603',
u'meta': {u'bar': u'\u2603'},
u'sysmeta': {u'baz': u'\u2603'},
u'cors': {u'expose_headers': u'\u2603'}}
env = {'PATH_INFO': '/v1/a/c'}
env = {'PATH_INFO': '/v1/a/c', 'swift.cache': memcache}
ci = get_container_info(env, app)
# Test info is returned as strings
@ -578,7 +572,7 @@ class TestProxyServerConfiguration(unittest.TestCase):
def _make_app(self, conf):
self.logger.clear()
# helper function to instantiate a proxy server instance
return proxy_server.Application(conf, FakeMemcache(),
return proxy_server.Application(conf,
container_ring=FakeRing(),
account_ring=FakeRing(),
logger=self.logger)
@ -634,7 +628,6 @@ class TestProxyServer(unittest.TestCase):
def test_get_object_ring(self):
baseapp = proxy_server.Application({},
FakeMemcache(),
container_ring=FakeRing(),
account_ring=FakeRing())
with patch_policies([
@ -664,7 +657,7 @@ class TestProxyServer(unittest.TestCase):
def get_controller(self, path):
raise Exception('this shouldn\'t be caught')
app = MyApp(None, FakeMemcache(), account_ring=FakeRing(),
app = MyApp(None, account_ring=FakeRing(),
container_ring=FakeRing())
req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'})
app.update_request(req)
@ -673,7 +666,6 @@ class TestProxyServer(unittest.TestCase):
def test_internal_method_request(self):
baseapp = proxy_server.Application({},
FakeMemcache(),
container_ring=FakeRing(),
account_ring=FakeRing())
resp = baseapp.handle_request(
@ -682,7 +674,6 @@ class TestProxyServer(unittest.TestCase):
def test_inexistent_method_request(self):
baseapp = proxy_server.Application({},
FakeMemcache(),
container_ring=FakeRing(),
account_ring=FakeRing())
resp = baseapp.handle_request(
@ -691,7 +682,6 @@ class TestProxyServer(unittest.TestCase):
def test_private_method_request(self):
baseapp = proxy_server.Application({},
FakeMemcache(),
container_ring=FakeRing(),
account_ring=FakeRing())
baseapp.logger = debug_logger()
@ -722,12 +712,13 @@ class TestProxyServer(unittest.TestCase):
# set account info
fake_cache = FakeMemcache()
fake_cache.store[get_cache_key('a')] = {'status': 200}
app = proxy_server.Application({}, fake_cache,
app = proxy_server.Application({},
container_ring=FakeRing(),
account_ring=FakeRing())
# build internal container request
container = get_reserved_name('c')
req = Request.blank('/v1/a/%s' % container)
req = Request.blank('/v1/a/%s' % container,
{'swift.cache': fake_cache})
app.update_request(req)
# try client request to reserved name
@ -748,7 +739,7 @@ class TestProxyServer(unittest.TestCase):
called[0] = True
with save_globals():
set_http_connect(200)
app = proxy_server.Application(None, FakeMemcache(),
app = proxy_server.Application(None,
account_ring=FakeRing(),
container_ring=FakeRing())
req = Request.blank('/v1/a')
@ -763,7 +754,7 @@ class TestProxyServer(unittest.TestCase):
def authorize(req):
called[0] = True
return HTTPUnauthorized(request=req)
app = proxy_server.Application(None, FakeMemcache(),
app = proxy_server.Application(None,
account_ring=FakeRing(),
container_ring=FakeRing())
req = Request.blank('/v1/a')
@ -776,7 +767,7 @@ class TestProxyServer(unittest.TestCase):
swift_dir = mkdtemp()
try:
baseapp = proxy_server.Application({'swift_dir': swift_dir},
FakeMemcache(), FakeLogger(),
FakeLogger(),
FakeRing(), FakeRing())
resp = baseapp.handle_request(
Request.blank('/', environ={'CONTENT_LENGTH': '-1'}))
@ -794,7 +785,7 @@ class TestProxyServer(unittest.TestCase):
try:
logger = FakeLogger()
baseapp = proxy_server.Application({'swift_dir': swift_dir},
FakeMemcache(), logger,
logger,
container_ring=FakeLogger(),
account_ring=FakeRing())
baseapp.handle_request(
@ -814,7 +805,7 @@ class TestProxyServer(unittest.TestCase):
try:
logger = FakeLogger()
baseapp = proxy_server.Application({'swift_dir': swift_dir},
FakeMemcache(), logger,
logger,
container_ring=FakeLogger(),
account_ring=FakeRing())
baseapp.handle_request(
@ -832,7 +823,6 @@ class TestProxyServer(unittest.TestCase):
baseapp = proxy_server.Application({'swift_dir': swift_dir,
'deny_host_headers':
'invalid_host.com'},
FakeMemcache(),
container_ring=FakeLogger(),
account_ring=FakeRing())
resp = baseapp.handle_request(
@ -844,7 +834,6 @@ class TestProxyServer(unittest.TestCase):
def test_node_timing(self):
baseapp = proxy_server.Application({'sorting_method': 'timing'},
FakeMemcache(),
container_ring=FakeRing(),
account_ring=FakeRing())
self.assertEqual(baseapp.node_timings, {})
@ -876,7 +865,6 @@ class TestProxyServer(unittest.TestCase):
nodes = list(nodes)
conf = dict(conf, policy_config=policy_conf)
baseapp = proxy_server.Application(conf,
FakeMemcache(),
logger=FakeLogger(),
container_ring=FakeRing(),
account_ring=FakeRing())
@ -1070,7 +1058,6 @@ class TestProxyServer(unittest.TestCase):
app_conf = {'concurrent_gets': 'on',
'concurrency_timeout': 0}
baseapp = proxy_server.Application(app_conf,
FakeMemcache(),
container_ring=FakeRing(),
account_ring=FakeRing())
self.assertTrue(baseapp.concurrent_gets)
@ -1095,7 +1082,6 @@ class TestProxyServer(unittest.TestCase):
# Now lets set the concurrency_timeout
app_conf['concurrency_timeout'] = 2
baseapp = proxy_server.Application(app_conf,
FakeMemcache(),
container_ring=FakeRing(),
account_ring=FakeRing())
self.assertEqual(baseapp.concurrency_timeout, 2)
@ -1106,7 +1092,7 @@ class TestProxyServer(unittest.TestCase):
self.assertEqual(resp.body, b'Response from 127.0.0.2')
def test_info_defaults(self):
app = proxy_server.Application({}, FakeMemcache(),
app = proxy_server.Application({},
account_ring=FakeRing(),
container_ring=FakeRing())
@ -1121,7 +1107,7 @@ class TestProxyServer(unittest.TestCase):
def test_get_info_controller(self):
req = Request.blank('/info')
app = proxy_server.Application({}, FakeMemcache(),
app = proxy_server.Application({},
account_ring=FakeRing(),
container_ring=FakeRing())
@ -1138,7 +1124,7 @@ class TestProxyServer(unittest.TestCase):
def test_exception_occurred(self):
def do_test(additional_info):
logger = debug_logger('test')
app = proxy_server.Application({}, FakeMemcache(),
app = proxy_server.Application({},
account_ring=FakeRing(),
container_ring=FakeRing(),
logger=logger)
@ -1172,7 +1158,7 @@ class TestProxyServer(unittest.TestCase):
def test_error_occurred(self):
def do_test(msg):
logger = debug_logger('test')
app = proxy_server.Application({}, FakeMemcache(),
app = proxy_server.Application({},
account_ring=FakeRing(),
container_ring=FakeRing(),
logger=logger)
@ -1198,7 +1184,7 @@ class TestProxyServer(unittest.TestCase):
def test_error_limit_methods(self):
logger = debug_logger('test')
app = proxy_server.Application({}, FakeMemcache(),
app = proxy_server.Application({},
account_ring=FakeRing(),
container_ring=FakeRing(),
logger=logger)
@ -1260,7 +1246,7 @@ class TestProxyServer(unittest.TestCase):
self.assertEqual(4, node_error_count(app, node))
def test_valid_api_version(self):
app = proxy_server.Application({}, FakeMemcache(),
app = proxy_server.Application({},
account_ring=FakeRing(),
container_ring=FakeRing())
@ -1341,7 +1327,7 @@ class TestProxyServerLoading(unittest.TestCase):
}
for policy in POLICIES:
policy.object_ring = FakeRing()
app = proxy_server.Application(conf, FakeMemcache(), debug_logger(),
app = proxy_server.Application(conf, debug_logger(),
FakeRing(), FakeRing())
self.assertEqual(app.node_timeout, 2.3)
self.assertEqual(app.recoverable_node_timeout, 1.4)
@ -2023,7 +2009,6 @@ class BaseTestObjectController(object):
kwargs['raise_exc'] = raise_exc
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o',
headers={'Content-Length': '0',
'Content-Type': 'text/plain'})
@ -2036,7 +2021,6 @@ class BaseTestObjectController(object):
# repeat test
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o',
headers={'Content-Length': '0',
'Content-Type': 'text/plain'})
@ -2171,7 +2155,7 @@ class TestReplicatedObjectController(
def setUp(self):
skip_if_no_xattrs()
self.app = proxy_server.Application(
None, FakeMemcache(),
None,
logger=debug_logger('proxy-ut'),
account_ring=FakeRing(),
container_ring=FakeRing())
@ -3023,7 +3007,6 @@ class TestReplicatedObjectController(
req = Request.blank('/v1/a/c/o.jpg', {})
req.content_length = 0
self.app.update_request(req)
self.app.memcache.store = {}
res = controller.PUT(req)
self.assertEqual(test_errors, [])
self.assertTrue(res.status.startswith('201 '), res.status)
@ -3052,7 +3035,6 @@ class TestReplicatedObjectController(
req.content_length = 1
req.body = 'a'
self.app.update_request(req)
self.app.memcache.store = {}
res = controller.PUT(req)
self.assertEqual(test_errors, [])
self.assertTrue(res.status.startswith('201 '))
@ -3070,7 +3052,7 @@ class TestReplicatedObjectController(
# not be used in ring order is if affinity is respected.
with mock.patch('swift.proxy.server.shuffle', lambda x: x):
app = proxy_server.Application(
conf, FakeMemcache(),
conf,
logger=debug_logger('proxy-ut'),
account_ring=FakeRing(),
container_ring=FakeRing())
@ -3086,7 +3068,6 @@ class TestReplicatedObjectController(
req = Request.blank(
'/v1/a/c/o.jpg', method='PUT', body='a',
headers={'X-Backend-Storage-Policy-Index': str(policy)})
app.memcache.store = {}
res = controller.PUT(req)
self.assertTrue(res.status.startswith('201 '))
self.assertEqual(3, len(written_to))
@ -3155,7 +3136,6 @@ class TestReplicatedObjectController(
req = Request.blank('/v1/a/c/o.jpg', {})
req.content_length = 1
req.body = 'a'
self.app.memcache.store = {}
res = controller.PUT(req)
self.assertTrue(res.status.startswith('201 '))
@ -3554,7 +3534,7 @@ class TestReplicatedObjectController(
with open(os.path.join(swift_dir, 'mime.types'), 'w') as fp:
fp.write('foo/bar foo\n')
proxy_server.Application({'swift_dir': swift_dir},
FakeMemcache(), FakeLogger(),
FakeLogger(),
FakeRing(), FakeRing())
self.assertEqual(proxy_server.mimetypes.guess_type('blah.foo')[0],
'foo/bar')
@ -3573,7 +3553,6 @@ class TestReplicatedObjectController(
req = Request.blank('/v1/a/c/o.jpg', {})
req.content_length = 0
self.app.update_request(req)
self.app.memcache.store = {}
res = controller.PUT(req)
expected = str(expected)
self.assertEqual(res.status[:len(expected)], expected)
@ -3590,7 +3569,6 @@ class TestReplicatedObjectController(
def test_status_map(statuses, expected):
set_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o.jpg', {})
req.content_length = 0
self.app.update_request(req)
@ -3620,7 +3598,6 @@ class TestReplicatedObjectController(
self.app, 'account', 'container', 'object')
def test_status_map(statuses, expected):
self.app.memcache.store = {}
set_http_connect(*statuses)
req = Request.blank('/v1/a/c/o.jpg',
environ={'REQUEST_METHOD': 'PUT'},
@ -3666,7 +3643,6 @@ class TestReplicatedObjectController(
self.app, 'account', 'container', 'object')
def test_status_map(statuses, expected):
self.app.memcache.store = {}
set_http_connect(*statuses)
req = Request.blank('/v1/a/c/o.jpg', {})
req.content_length = 0
@ -3686,7 +3662,6 @@ class TestReplicatedObjectController(
with save_globals():
def test_status_map(statuses, expected):
set_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {}, method='POST',
headers={'Content-Type': 'foo/bar'})
self.app.update_request(req)
@ -3711,7 +3686,6 @@ class TestReplicatedObjectController(
self.app.sort_nodes = lambda nodes, *args, **kwargs: nodes
def do_test(resp_headers):
self.app.memcache.store = {}
backend_requests = []
def capture_requests(ip, port, method, path, headers, *args,
@ -3779,7 +3753,6 @@ class TestReplicatedObjectController(
self.assertEqual(container_headers, expected)
# and again with policy override
self.app.memcache.store = {}
backend_requests = []
req = Request.blank('/v1/a/c/o', {}, method='POST',
headers={'X-Object-Meta-Color': 'Blue',
@ -3823,7 +3796,6 @@ class TestReplicatedObjectController(
self.app.recheck_updating_shard_ranges = 0
def do_test(method, sharding_state):
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {}, method=method, body='',
headers={'Content-Type': 'text/plain'})
@ -3918,9 +3890,9 @@ class TestReplicatedObjectController(
self.app.recheck_updating_shard_ranges = 3600
def do_test(method, sharding_state):
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {}, method=method, body='',
headers={'Content-Type': 'text/plain'})
req = Request.blank(
'/v1/a/c/o', {'swift.cache': FakeMemcache()},
method=method, body='', headers={'Content-Type': 'text/plain'})
# we want the container_info response to say policy index of 1 and
# sharding state
@ -3977,8 +3949,8 @@ class TestReplicatedObjectController(
headers={'X-Backend-Record-Type': 'shard'})
cache_key = 'shard-updating/a/c'
self.assertIn(cache_key, self.app.memcache.store)
self.assertEqual(self.app.memcache.store[cache_key],
self.assertIn(cache_key, req.environ['swift.cache'].store)
self.assertEqual(req.environ['swift.cache'].store[cache_key],
[dict(sr) for sr in shard_ranges])
self.assertIn(cache_key, req.environ.get('swift.infocache'))
self.assertEqual(req.environ['swift.infocache'][cache_key],
@ -4037,9 +4009,11 @@ class TestReplicatedObjectController(
utils.ShardRange(
'.shards_a/c_nope', utils.Timestamp.now(), 'u', ''),
]
self.app.memcache.store = {'shard-updating/a/c': tuple(
dict(shard_range) for shard_range in shard_ranges)}
req = Request.blank('/v1/a/c/o', {}, method=method, body='',
cache = FakeMemcache()
cache.set('shard-updating/a/c', tuple(
dict(shard_range) for shard_range in shard_ranges))
req = Request.blank('/v1/a/c/o', {'swift.cache': cache},
method=method, body='',
headers={'Content-Type': 'text/plain'})
# we want the container_info response to say policy index of 1 and
@ -4122,7 +4096,6 @@ class TestReplicatedObjectController(
with save_globals():
def test_status_map(statuses, expected):
set_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'DELETE'})
self.app.update_request(req)
res = req.get_response(self.app)
@ -4139,7 +4112,6 @@ class TestReplicatedObjectController(
with save_globals():
def test_status_map(statuses, expected):
set_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'})
self.app.update_request(req)
res = req.get_response(self.app)
@ -4170,7 +4142,6 @@ class TestReplicatedObjectController(
def test_status_map(statuses, expected, timestamps,
expected_timestamp):
set_http_connect(*statuses, timestamps=timestamps)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'},
headers={'x-newest': 'true'})
self.app.update_request(req)
@ -4201,7 +4172,6 @@ class TestReplicatedObjectController(
def test_status_map(statuses, expected, timestamps,
expected_timestamp):
set_http_connect(*statuses, timestamps=timestamps)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'GET'},
headers={'x-newest': 'true'})
self.app.update_request(req)
@ -4228,7 +4198,6 @@ class TestReplicatedObjectController(
def test_status_map(statuses, expected, timestamps,
expected_timestamp):
set_http_connect(*statuses, timestamps=timestamps)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'})
self.app.update_request(req)
res = req.get_response(self.app)
@ -4410,6 +4379,7 @@ class TestReplicatedObjectController(
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': FakeMemcache(),
'wsgi.input': SlowBody()},
headers={'Content-Length': '4',
'Content-Type': 'text/plain'})
@ -4419,11 +4389,8 @@ class TestReplicatedObjectController(
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
self.app.client_timeout = 0.05
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'wsgi.input': SlowBody()},
headers={'Content-Length': '4',
'Content-Type': 'text/plain'})
req.environ['wsgi.input'] = SlowBody() # Need a fresh instance
self.app.update_request(req)
set_http_connect(201, 201, 201)
# obj obj obj
@ -4537,7 +4504,8 @@ class TestReplicatedObjectController(
for dev in object_ring.devs:
dev['ip'] = '127.0.0.1'
dev['port'] = 1
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
req = Request.blank('/v1/a/c/o', environ={
'REQUEST_METHOD': 'GET', 'swift.cache': FakeMemcache()})
self.app.update_request(req)
self.app.recoverable_node_timeout = 0.1
@ -4561,7 +4529,6 @@ class TestReplicatedObjectController(
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'lalala')
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
set_http_connect(200, 200, 200, body=b'lalala',
slow=[1.0, 1.0], etags=['a', 'b', 'b'])
resp = req.get_response(self.app)
@ -4605,7 +4572,6 @@ class TestReplicatedObjectController(
def test_node_request_setting(self):
baseapp = proxy_server.Application({'request_node_count': '3'},
FakeMemcache(),
container_ring=FakeRing(),
account_ring=FakeRing())
self.assertEqual(baseapp.request_node_count(3), 3)
@ -4939,7 +4905,6 @@ class TestReplicatedObjectController(
def test_acc_or_con_missing_returns_404(self):
with save_globals():
self.app.memcache = FakeMemcacheReturnsNone()
self.app._error_limiting = {}
controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object')
@ -5036,7 +5001,6 @@ class TestReplicatedObjectController(
def test_PUT_POST_requires_container_exist(self):
with save_globals():
self.app.memcache = FakeMemcacheReturnsNone()
controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object')
@ -5194,17 +5158,21 @@ class TestReplicatedObjectController(
with save_globals():
controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object')
cache = FakeMemcache()
set_http_connect(200, 200, 201, 201, 201)
# acct cont obj obj obj
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0'})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers={'Content-Length': '0'})
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEqual(resp.status_int, 201)
set_http_connect(201, 201, 201)
req = Request.blank(
'/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers={'Content-Length': '0',
'X-Object-Meta-' + (
'a' * constraints.MAX_META_NAME_LENGTH): 'v'})
@ -5213,7 +5181,8 @@ class TestReplicatedObjectController(
self.assertEqual(resp.status_int, 201)
set_http_connect(201, 201, 201)
req = Request.blank(
'/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers={
'Content-Length': '0',
'X-Object-Meta-' + (
@ -5223,16 +5192,19 @@ class TestReplicatedObjectController(
self.assertEqual(resp.status_int, 400)
set_http_connect(201, 201, 201)
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Object-Meta-Too-Long': 'a' *
constraints.MAX_META_VALUE_LENGTH})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers={'Content-Length': '0',
'X-Object-Meta-Too-Long': 'a' *
constraints.MAX_META_VALUE_LENGTH})
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEqual(resp.status_int, 201)
set_http_connect(201, 201, 201)
req = Request.blank(
'/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers={'Content-Length': '0',
'X-Object-Meta-Too-Long': 'a' *
(constraints.MAX_META_VALUE_LENGTH + 1)})
@ -5244,8 +5216,10 @@ class TestReplicatedObjectController(
headers = {'Content-Length': '0'}
for x in range(constraints.MAX_META_COUNT):
headers['X-Object-Meta-%d' % x] = 'v'
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers=headers)
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers=headers)
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEqual(resp.status_int, 201)
@ -5253,8 +5227,10 @@ class TestReplicatedObjectController(
headers = {'Content-Length': '0'}
for x in range(constraints.MAX_META_COUNT + 1):
headers['X-Object-Meta-%d' % x] = 'v'
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers=headers)
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers=headers)
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEqual(resp.status_int, 400)
@ -5272,16 +5248,20 @@ class TestReplicatedObjectController(
if constraints.MAX_META_OVERALL_SIZE - size > 1:
headers['X-Object-Meta-a'] = \
'a' * (constraints.MAX_META_OVERALL_SIZE - size - 1)
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers=headers)
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers=headers)
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEqual(resp.status_int, 201)
set_http_connect(201, 201, 201)
headers['X-Object-Meta-a'] = \
'a' * (constraints.MAX_META_OVERALL_SIZE - size)
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers=headers)
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache},
headers=headers)
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEqual(resp.status_int, 400)
@ -5292,7 +5272,6 @@ class TestReplicatedObjectController(
controller = ReplicatedObjectController(
self.app, account, container, obj)
self.app.update_request(req)
self.app.memcache.store = {}
with save_globals():
new_connect = set_http_connect(*args, **kwargs)
yield controller
@ -5702,7 +5681,7 @@ class TestReplicatedObjectController(
# not be used in ring order is if affinity is respected.
with mock.patch('swift.proxy.server.shuffle', lambda x: x):
app = proxy_server.Application(
conf, FakeMemcache(),
conf,
logger=debug_logger('proxy-ut'),
account_ring=FakeRing(),
container_ring=FakeRing())
@ -5718,7 +5697,6 @@ class TestReplicatedObjectController(
req = Request.blank(
'/v1/a/c/o.jpg',
headers={'X-Backend-Storage-Policy-Index': str(policy)})
app.memcache.store = {}
res = controller.GET(req)
self.assertTrue(res.status.startswith('200 '))
self.assertEqual(3, len(actual_nodes))
@ -5818,7 +5796,6 @@ class TestReplicatedObjectController(
controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object')
set_http_connect(200, 200, 202, 202, 202)
self.app.memcache.store = {}
orig_time = time.time
try:
t = time.time()
@ -8089,7 +8066,7 @@ class TestObjectECRangedGET(unittest.TestCase):
def setUp(self):
_test_servers[0].logger._clear()
self.app = proxy_server.Application(
None, FakeMemcache(),
None,
logger=debug_logger('proxy-ut'),
account_ring=FakeRing(),
container_ring=FakeRing())
@ -8674,7 +8651,7 @@ class TestContainerController(unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(
None, FakeMemcache(),
None,
account_ring=FakeRing(),
container_ring=FakeRing(base_port=2000),
logger=debug_logger())
@ -8774,14 +8751,12 @@ class TestContainerController(unittest.TestCase):
if raise_exc:
kwargs['raise_exc'] = raise_exc
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', headers={'Content-Length': '0',
'Content-Type': 'text/plain'})
self.app.update_request(req)
res = method(req)
self.assertEqual(res.status_int, expected)
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c/', headers={'Content-Length': '0',
'Content-Type': 'text/plain'})
self.app.update_request(req)
@ -8795,7 +8770,6 @@ class TestContainerController(unittest.TestCase):
def test_status_map(statuses, expected,
c_expected=None, a_expected=None, **kwargs):
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', {})
self.app.update_request(req)
res = controller.HEAD(req)
@ -8823,7 +8797,6 @@ class TestContainerController(unittest.TestCase):
self.assertNotIn('account/a', res.environ)
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', {})
self.app.update_request(req)
res = controller.GET(req)
@ -8882,7 +8855,6 @@ class TestContainerController(unittest.TestCase):
with save_globals():
mock_conn = set_http_connect(200, 201, 201, 201,
give_connect=capture_requests)
self.app.memcache.store = {}
req = Request.blank('/v1/a/test', method='PUT',
headers={'Content-Length': 0})
if requested_policy:
@ -8934,7 +8906,6 @@ class TestContainerController(unittest.TestCase):
def test_status_map(statuses, expected, **kwargs):
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', {})
req.content_length = 0
self.app.update_request(req)
@ -8994,7 +8965,6 @@ class TestContainerController(unittest.TestCase):
def test_status_map(statuses, expected, headers=None, **kwargs):
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', {}, headers=headers)
req.content_length = 0
self.app.update_request(req)
@ -9033,7 +9003,6 @@ class TestContainerController(unittest.TestCase):
def test_status_map(statuses, expected, **kwargs):
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c', {})
req.content_length = 0
self.app.update_request(req)
@ -9126,7 +9095,6 @@ class TestContainerController(unittest.TestCase):
def test_acc_missing_returns_404(self):
for meth in ('DELETE', 'PUT'):
with save_globals():
self.app.memcache = FakeMemcacheReturnsNone()
self.app._error_limiting = {}
controller = proxy_server.ContainerController(self.app,
'account',
@ -9136,7 +9104,6 @@ class TestContainerController(unittest.TestCase):
missing_container=True)
else:
set_http_connect(200, 200, 200, 200)
self.app.memcache.store = {}
req = Request.blank('/v1/a/c',
environ={'REQUEST_METHOD': meth})
self.app.update_request(req)
@ -9251,7 +9218,6 @@ class TestContainerController(unittest.TestCase):
self.assert_status_map(controller.DELETE,
(200, 204, 503, 404), 503)
self.app.memcache = FakeMemcacheReturnsNone()
# 200: Account check, 404x3: Container check
self.assert_status_map(controller.DELETE,
(200, 404, 404, 404), 404)
@ -9556,7 +9522,7 @@ class TestContainerController(unittest.TestCase):
def _do_test(method):
with save_globals():
swift.proxy.controllers.Controller.account_info = account_info
app = proxy_server.Application(None, FakeMemcache(),
app = proxy_server.Application(None,
account_ring=FakeRing(),
container_ring=FakeRing())
set_http_connect(201, 201, 201)
@ -9595,7 +9561,7 @@ class TestContainerController(unittest.TestCase):
def _do_test(method):
with save_globals():
swift.proxy.controllers.Controller.account_info = account_info
app = proxy_server.Application(None, FakeMemcache(),
app = proxy_server.Application(None,
account_ring=FakeRing(),
container_ring=FakeRing())
set_http_connect(201, 201, 201)
@ -9982,7 +9948,7 @@ class TestAccountController(unittest.TestCase):
def setUp(self):
conf = {'error_suppression_interval': 0}
self.app = proxy_server.Application(conf, FakeMemcache(),
self.app = proxy_server.Application(conf,
account_ring=FakeRing(),
container_ring=FakeRing())
@ -10065,13 +10031,11 @@ class TestAccountController(unittest.TestCase):
self.assert_status_map(controller.GET, (404, 404, 503), 404, 404)
self.assert_status_map(controller.GET, (404, 503, 503), 503)
self.app.memcache = FakeMemcacheReturnsNone()
self.assert_status_map(controller.GET, (404, 404, 404), 404, 404)
def test_GET_autocreate(self):
with save_globals():
controller = proxy_server.AccountController(self.app, 'a')
self.app.memcache = FakeMemcacheReturnsNone()
self.assertFalse(self.app.account_autocreate)
# Repeat the test for autocreate = False and 404 by all
self.assert_status_map(controller.GET,
@ -10138,7 +10102,6 @@ class TestAccountController(unittest.TestCase):
# Same behaviour as GET
with save_globals():
controller = proxy_server.AccountController(self.app, 'a')
self.app.memcache = FakeMemcacheReturnsNone()
self.assertFalse(self.app.account_autocreate)
self.assert_status_map(controller.HEAD,
(404, 404, 404), 404)
@ -10154,7 +10117,6 @@ class TestAccountController(unittest.TestCase):
def test_POST_autocreate(self):
with save_globals():
controller = proxy_server.AccountController(self.app, 'a')
self.app.memcache = FakeMemcacheReturnsNone()
# first test with autocreate being False
self.assertFalse(self.app.account_autocreate)
self.assert_status_map(controller.POST,
@ -10176,7 +10138,6 @@ class TestAccountController(unittest.TestCase):
def test_POST_autocreate_with_sysmeta(self):
with save_globals():
controller = proxy_server.AccountController(self.app, 'a')
self.app.memcache = FakeMemcacheReturnsNone()
# first test with autocreate being False
self.assertFalse(self.app.account_autocreate)
self.assert_status_map(controller.POST,
@ -10249,7 +10210,6 @@ class TestAccountController(unittest.TestCase):
def test_status_map(statuses, expected, **kwargs):
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a', {})
req.content_length = 0
self.app.update_request(req)
@ -10452,7 +10412,6 @@ class TestAccountController(unittest.TestCase):
def test_status_map(statuses, expected, **kwargs):
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a', {'REQUEST_METHOD': 'DELETE'})
req.content_length = 0
self.app.update_request(req)
@ -10475,7 +10434,6 @@ class TestAccountController(unittest.TestCase):
def test_status_map(statuses, expected, **kwargs):
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/v1/a?whoops',
environ={'REQUEST_METHOD': 'DELETE'})
req.content_length = 0
@ -10500,11 +10458,10 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase):
def setUp(self):
conf = {'account_autocreate': 'yes'}
self.app = listing_formats.ListingFilter(
proxy_server.Application(conf, FakeMemcache(),
proxy_server.Application(conf,
account_ring=FakeRing(),
container_ring=FakeRing()),
{})
self.app.app.memcache = FakeMemcacheReturnsNone()
def test_GET_autocreate_accept_json(self):
with save_globals():
@ -10611,7 +10568,7 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase):
version=2, acl_dict=acl)}
app = proxy_server.Application(
None, FakeMemcache(), account_ring=FakeRing(),
None, account_ring=FakeRing(),
container_ring=FakeRing())
with save_globals():
@ -10686,7 +10643,7 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase):
resps_to_send.append(canned_resp)
app = proxy_server.Application(
None, FakeMemcache(), account_ring=FakeRing(),
None, account_ring=FakeRing(),
container_ring=FakeRing())
app.allow_account_management = True
@ -10854,7 +10811,7 @@ class TestSwiftInfo(unittest.TestCase):
utils._swift_admin_info = {}
def test_registered_defaults(self):
app = proxy_server.Application({}, FakeMemcache(),
app = proxy_server.Application({},
account_ring=FakeRing(),
container_ring=FakeRing())
req = Request.blank('/info')

82
test/unit/proxy/test_sysmeta.py
View File

@ -132,7 +132,7 @@ class TestObjectSysmeta(unittest.TestCase):
def setUp(self):
skip_if_no_xattrs()
self.app = proxy.Application(None, FakeMemcache(),
self.app = proxy.Application(None,
logger=debug_logger('proxy-ut'),
account_ring=FakeRing(replicas=1),
container_ring=FakeRing(replicas=1))
@ -227,8 +227,9 @@ class TestObjectSysmeta(unittest.TestCase):
def test_sysmeta_replaced_by_PUT(self):
path = '/v1/a/c/o'
cache = FakeMemcache()
env = {'REQUEST_METHOD': 'PUT'}
env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1)
@ -237,7 +238,7 @@ class TestObjectSysmeta(unittest.TestCase):
resp = req.get_response(self.app)
self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'PUT'}
env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.changed_meta_headers)
@ -247,7 +248,7 @@ class TestObjectSysmeta(unittest.TestCase):
resp = req.get_response(self.app)
self._assertStatus(resp, 201)
req = Request.blank(path, environ={})
req = Request.blank(path, environ={'swift.cache': cache})
resp = req.get_response(self.app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
@ -257,29 +258,30 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertInHeaders(resp, self.new_meta_headers)
self._assertNotInHeaders(resp, self.original_meta_headers_2)
def _test_sysmeta_not_updated_by_POST(self, app):
def test_sysmeta_not_updated_by_POST(self):
# check sysmeta is not changed by a POST but user meta is replaced
path = '/v1/a/c/o'
cache = FakeMemcache()
env = {'REQUEST_METHOD': 'PUT'}
env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_meta_headers_1)
req = Request.blank(path, environ=env, headers=hdrs, body=b'x')
resp = req.get_response(app)
resp = req.get_response(self.app)
self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'POST'}
env = {'REQUEST_METHOD': 'POST', 'swift.cache': cache}
hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.changed_meta_headers)
hdrs.update(self.new_meta_headers)
hdrs.update(self.bad_headers)
req = Request.blank(path, environ=env, headers=hdrs)
resp = req.get_response(app)
resp = req.get_response(self.app)
self._assertStatus(resp, 202)
req = Request.blank(path, environ={})
resp = req.get_response(app)
req = Request.blank(path, environ={'swift.cache': cache})
resp = req.get_response(self.app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.original_sysmeta_headers_1)
self._assertNotInHeaders(resp, self.new_sysmeta_headers)
@ -287,31 +289,28 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertInHeaders(resp, self.new_meta_headers)
self._assertNotInHeaders(resp, self.bad_headers)
env = {'REQUEST_METHOD': 'PUT'}
env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.bad_headers)
req = Request.blank(path, environ=env, headers=hdrs, body=b'x')
resp = req.get_response(app)
resp = req.get_response(self.app)
self._assertStatus(resp, 201)
req = Request.blank(path, environ={})
resp = req.get_response(app)
req = Request.blank(path, environ={'swift.cache': cache})
resp = req.get_response(self.app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
self._assertInHeaders(resp, self.new_sysmeta_headers)
self._assertNotInHeaders(resp, self.original_sysmeta_headers_2)
def test_sysmeta_not_updated_by_POST(self):
# test fast-post by issuing requests to the proxy app
self._test_sysmeta_not_updated_by_POST(self.app)
def test_sysmeta_updated_by_COPY(self):
# check sysmeta is updated by a COPY in same way as user meta by
# issuing requests to the copy middleware app
path = '/v1/a/c/o'
dest = '/c/o2'
env = {'REQUEST_METHOD': 'PUT'}
cache = FakeMemcache()
env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1)
@ -322,7 +321,7 @@ class TestObjectSysmeta(unittest.TestCase):
resp = req.get_response(self.copy_app)
self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'COPY'}
env = {'REQUEST_METHOD': 'COPY', 'swift.cache': cache}
hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.changed_meta_headers)
@ -345,7 +344,7 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers)
req = Request.blank('/v1/a/c/o2', environ={})
req = Request.blank('/v1/a/c/o2', environ={'swift.cache': cache})
resp = req.get_response(self.copy_app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
@ -363,7 +362,8 @@ class TestObjectSysmeta(unittest.TestCase):
# check sysmeta is updated by a PUT with x-copy-from in same way as
# user meta by issuing requests to the copy middleware app
path = '/v1/a/c/o'
env = {'REQUEST_METHOD': 'PUT'}
cache = FakeMemcache()
env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1)
@ -372,7 +372,7 @@ class TestObjectSysmeta(unittest.TestCase):
resp = req.get_response(self.copy_app)
self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'PUT'}
env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.changed_meta_headers)
@ -390,7 +390,7 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers)
req = Request.blank('/v1/a/c/o2', environ={})
req = Request.blank('/v1/a/c/o2', environ={'swift.cache': cache})
resp = req.get_response(self.copy_app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
@ -401,40 +401,41 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers)
def _test_transient_sysmeta_replaced_by_PUT_or_POST(self, app):
def test_transient_sysmeta_replaced_by_PUT_or_POST(self):
# check transient_sysmeta is replaced en-masse by a POST
path = '/v1/a/c/o'
cache = FakeMemcache()
env = {'REQUEST_METHOD': 'PUT'}
env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.original_transient_sysmeta_headers_1)
hdrs.update(self.original_transient_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1)
req = Request.blank(path, environ=env, headers=hdrs, body=b'x')
resp = req.get_response(app)
resp = req.get_response(self.app)
self._assertStatus(resp, 201)
req = Request.blank(path, environ={})
resp = req.get_response(app)
resp = req.get_response(self.app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.original_transient_sysmeta_headers_1)
self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2)
self._assertInHeaders(resp, self.original_meta_headers_1)
info = get_object_info(req.environ, app)
info = get_object_info(req.environ, self.app)
self.assertEqual(2, len(info.get('transient_sysmeta', ())))
self.assertEqual({'testa': 'A', 'testb': 'B'},
info['transient_sysmeta'])
# POST will replace all existing transient_sysmeta and usermeta values
env = {'REQUEST_METHOD': 'POST'}
env = {'REQUEST_METHOD': 'POST', 'swift.cache': cache}
hdrs = dict(self.changed_transient_sysmeta_headers)
hdrs.update(self.new_transient_sysmeta_headers_1)
req = Request.blank(path, environ=env, headers=hdrs)
resp = req.get_response(app)
resp = req.get_response(self.app)
self._assertStatus(resp, 202)
req = Request.blank(path, environ={})
resp = req.get_response(app)
resp = req.get_response(self.app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_transient_sysmeta_headers)
self._assertInHeaders(resp, self.new_transient_sysmeta_headers_1)
@ -442,21 +443,21 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertNotInHeaders(resp,
self.original_transient_sysmeta_headers_2)
info = get_object_info(req.environ, app)
info = get_object_info(req.environ, self.app)
self.assertEqual(2, len(info.get('transient_sysmeta', ())))
self.assertEqual({'testa': 'changed_A', 'testc': 'C'},
info['transient_sysmeta'])
# subsequent PUT replaces all transient_sysmeta and usermeta values
env = {'REQUEST_METHOD': 'PUT'}
env = {'REQUEST_METHOD': 'PUT', 'swift.cache': cache}
hdrs = dict(self.new_transient_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_2)
req = Request.blank(path, environ=env, headers=hdrs, body=b'x')
resp = req.get_response(app)
resp = req.get_response(self.app)
self._assertStatus(resp, 201)
req = Request.blank(path, environ={})
resp = req.get_response(app)
req = Request.blank(path, environ={'swift.cache': cache})
resp = req.get_response(self.app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertInHeaders(resp, self.new_transient_sysmeta_headers_2)
@ -470,9 +471,6 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertNotInHeaders(resp,
self.original_transient_sysmeta_headers_2)
info = get_object_info(req.environ, app)
info = get_object_info(req.environ, self.app)
self.assertEqual(1, len(info.get('transient_sysmeta', ())))
self.assertEqual({'testd': 'D'}, info['transient_sysmeta'])
def test_transient_sysmeta_replaced_by_PUT_or_POST(self):
self._test_transient_sysmeta_replaced_by_PUT_or_POST(self.app)