openstack/swift
Code Issues Proposed changes

Merge "get_info - removes duplicate code (Take 3)"

Browse Source
This commit is contained in:
Jenkins 2013-06-12 13:15:06 +00:00 committed by Gerrit Code Review
commit abef11f89c
10 changed files with 809 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
swift/proxy/controllers/account.py
View File

@ -31,7 +31,7 @@ from swift.account.utils import account_listing_response, \
from swift.common.utils import public
from swift.common.constraints import check_metadata, MAX_ACCOUNT_NAME_LENGTH
from swift.common.http import HTTP_NOT_FOUND
from swift.proxy.controllers.base import Controller, get_account_memcache_key
from swift.proxy.controllers.base import Controller, clear_info_cache
from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed
@ -84,9 +84,7 @@ class AccountController(Controller):
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
clear_info_cache(self.app, req.environ, self.account_name)
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'PUT',
req.path_info, [headers] * len(accounts))
@ -106,14 +104,12 @@ class AccountController(Controller):
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
clear_info_cache(self.app, req.environ, self.account_name)
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'POST',
req.path_info, [headers] * len(accounts))
if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate:
self.autocreate_account(self.account_name)
self.autocreate_account(req.environ, self.account_name)
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'POST',
req.path_info, [headers] * len(accounts))
@ -134,9 +130,7 @@ class AccountController(Controller):
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = self.generate_request_headers(req)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
clear_info_cache(self.app, req.environ, self.account_name)
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'DELETE',
req.path_info, [headers] * len(accounts))

499
swift/proxy/controllers/base.py
View File

@ -28,15 +28,15 @@ import os
import time
import functools
import inspect
from urllib import quote
from eventlet import spawn_n, GreenPile
from eventlet.queue import Queue, Empty, Full
from eventlet.timeout import Timeout
from swift.common.wsgi import make_pre_authed_request
from swift.common.wsgi import make_pre_authed_env
from swift.common.utils import normalize_timestamp, config_true_value, \
public, split_path, cache_from_env, list_from_csv, \
GreenthreadSafeIterator
public, split_path, list_from_csv, GreenthreadSafeIterator
from swift.common.bufferedhttp import http_connect
from swift.common.exceptions import ChunkReadTimeout, ConnectionTimeout
from swift.common.http import is_informational, is_success, is_redirection, \
@ -91,11 +91,15 @@ def delay_denial(func):
def get_account_memcache_key(account):
return 'account/%s' % account
cache_key, env_key = _get_cache_key(account, None)
return cache_key
def get_container_memcache_key(account, container):
return 'container/%s/%s' % (account, container)
if not container:
raise ValueError("container not provided")
cache_key, env_key = _get_cache_key(account, container)
return cache_key
def headers_to_account_info(headers, status_int=HTTP_OK):
@ -105,6 +109,10 @@ def headers_to_account_info(headers, status_int=HTTP_OK):
headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
return {
'status': status_int,
# 'container_count' anomaly:
# Previous code sometimes expects an int sometimes a string
# Current code aligns to str and None, yet translates to int in
# deprecated functions as needed
'container_count': headers.get('x-account-container-count'),
'total_object_count': headers.get('x-account-object-count'),
'bytes': headers.get('x-account-bytes-used'),
@ -162,7 +170,7 @@ def cors_validation(func):
# Yes, this is a CORS request so test if the origin is allowed
container_info = \
controller.container_info(controller.account_name,
controller.container_name)
controller.container_name, req)
cors_info = container_info.get('cors', {})
# Call through to the decorated method
@ -207,28 +215,14 @@ def get_container_info(env, app, swift_source=None):
Get the info structure for a container, based on env and app.
This is useful to middlewares.
Note: This call bypasses auth. Success does not imply that the
request has authorization to the container_info.
request has authorization to the account.
"""
cache = cache_from_env(env)
if not cache:
return None
(version, account, container, _) = \
split_path(env['PATH_INFO'], 3, 4, True)
cache_key = get_container_memcache_key(account, container)
# Use a unique environment cache key per container. If you copy this env
# to make a new request, it won't accidentally reuse the old container info
env_key = 'swift.%s' % cache_key
if env_key not in env:
container_info = cache.get(cache_key)
if not container_info:
resp = make_pre_authed_request(
env, 'HEAD', '/%s/%s/%s' % (version, account, container),
swift_source=swift_source,
).get_response(app)
container_info = headers_to_container_info(
resp.headers, resp.status_int)
env[env_key] = container_info
return env[env_key]
info = get_info(app, env, account, container, ret_not_found=True)
if not info:
info = headers_to_container_info({}, 0)
return info
def get_account_info(env, app, swift_source=None):
@ -236,28 +230,177 @@ def get_account_info(env, app, swift_source=None):
Get the info structure for an account, based on env and app.
This is useful to middlewares.
Note: This call bypasses auth. Success does not imply that the
request has authorization to the account_info.
request has authorization to the container.
"""
cache = cache_from_env(env)
if not cache:
return None
(version, account, _junk, _junk) = \
split_path(env['PATH_INFO'], 2, 4, True)
cache_key = get_account_memcache_key(account)
# Use a unique environment cache key per account. If you copy this env
# to make a new request, it won't accidentally reuse the old account info
info = get_info(app, env, account, ret_not_found=True)
if not info:
info = headers_to_account_info({}, 0)
if info.get('container_count') is None:
info['container_count'] = 0
else:
info['container_count'] = int(info['container_count'])
return info
def _get_cache_key(account, container):
"""
Get the keys for both memcache (cache_key) and env (env_key)
where info about accounts and containers is cached
:param account: The name of the account
:param container: The name of the container (or None if account)
:returns a tuple of (cache_key, env_key)
"""
if container:
cache_key = 'container/%s/%s' % (account, container)
else:
cache_key = 'account/%s' % account
# Use a unique environment cache key per account and one container.
# This allows caching both account and container and ensures that when we
# copy this env to form a new request, it won't accidentally reuse the
# old container or account info
env_key = 'swift.%s' % cache_key
if env_key not in env:
account_info = cache.get(cache_key)
if not account_info:
resp = make_pre_authed_request(
env, 'HEAD', '/%s/%s' % (version, account),
swift_source=swift_source,
).get_response(app)
account_info = headers_to_account_info(
resp.headers, resp.status_int)
env[env_key] = account_info
return env[env_key]
return cache_key, env_key
def _set_info_cache(app, env, account, container, resp):
"""
Cache info in both memcache and env.
Caching is used to avoid unnecessary calls to account & container servers.
This is a private function that is being called by GETorHEAD_base and
by clear_info_cache.
Any attempt to GET or HEAD from the container/account server should use
the GETorHEAD_base interface which would than set the cache.
:param app: the application object
:param account: the unquoted account name
:param container: the unquoted containr name or None
:param resp: the response received or None if info cache should be cleared
"""
if container:
cache_time = app.recheck_container_existence
else:
cache_time = app.recheck_account_existence
cache_key, env_key = _get_cache_key(account, container)
if resp:
if resp.status_int == HTTP_NOT_FOUND:
cache_time *= 0.1
elif not is_success(resp.status_int):
cache_time = None
else:
cache_time = None
# Next actually set both memcache and the env chache
memcache = getattr(app, 'memcache', None) or env.get('swift.cache')
if not cache_time:
env.pop(env_key, None)
if memcache:
memcache.delete(cache_key)
return
if container:
info = headers_to_container_info(resp.headers, resp.status_int)
else:
info = headers_to_account_info(resp.headers, resp.status_int)
if memcache:
memcache.set(cache_key, info, cache_time)
env[env_key] = info
def clear_info_cache(app, env, account, container=None):
"""
Clear the cached info in both memcache and env
:param app: the application object
:param account: the account name
:param container: the containr name or None if setting info for containers
"""
_set_info_cache(app, env, account, container, None)
def _get_info_cache(app, env, account, container=None):
"""
Get the cached info from env or memcache (if used) in that order
Used for both account and container info
A private function used by get_info
:param app: the application object
:param env: the environment used by the current request
:returns the cached info or None if not cached
"""
cache_key, env_key = _get_cache_key(account, container)
if env_key in env:
return env[env_key]
memcache = getattr(app, 'memcache', None) or env.get('swift.cache')
if memcache:
info = memcache.get(cache_key)
if info:
env[env_key] = info
return info
return None
def _prepare_pre_auth_info_request(env, path):
"""
Prepares a pre authed request to obtain info using a HEAD.
:param env: the environment used by the current request
:param path: The unquoted request path
:returns: the pre authed request
"""
# Set the env for the pre_authed call without a query string
newenv = make_pre_authed_env(env, 'HEAD', path, agent='Swift',
query_string='', swift_source='GET_INFO')
# Note that Request.blank expects quoted path
return Request.blank(quote(path), environ=newenv)
def get_info(app, env, account, container=None, ret_not_found=False):
"""
Get the info about accounts or containers
Note: This call bypasses auth. Success does not imply that the
request has authorization to the info.
:param app: the application object
:param env: the environment used by the current request
:param account: The unquoted name of the account
:param container: The unquoted name of the container (or None if account)
:returns: the cached info or None if cannot be retrieved
"""
info = _get_info_cache(app, env, account, container)
if info:
if ret_not_found or is_success(info['status']):
return info
return None
# Not in cached, let's try the account servers
path = '/v1/%s' % account
if container:
# Stop and check if we have an account?
if not get_info(app, env, account):
return None
path += '/' + container
req = _prepare_pre_auth_info_request(env, path)
# Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in
# the environment under environ[env_key] and in memcache. We will
# pick the one from environ[env_key] and use it to set the caller env
resp = req.get_response(app)
cache_key, env_key = _get_cache_key(account, container)
try:
info = resp.environ[env_key]
env[env_key] = info
if ret_not_found or is_success(info['status']):
return info
except (KeyError, AttributeError):
pass
return None
class Controller(object):
@ -268,6 +411,11 @@ class Controller(object):
pass_through_headers = []
def __init__(self, app):
"""
Creates a controller attached to an application instance
:param app: the application instance
"""
self.account_name = None
self.app = app
self.trans_id = '-'
@ -278,9 +426,21 @@ class Controller(object):
self.allowed_methods.add(name)
def _x_remove_headers(self):
"""
Returns a list of headers that must not be sent to the backend
:returns: a list of header
"""
return []
def transfer_headers(self, src_headers, dst_headers):
"""
Transfer legal headers from an original client request to dictionary
that will be used as headers by the backend request
:param src_headers: A dictionary of the original client request headers
:param dst_headers: A dictionary of the backend request headers
"""
st = self.server_type.lower()
x_remove = 'x-remove-%s-meta-' % st
@ -297,6 +457,14 @@ class Controller(object):
def generate_request_headers(self, orig_req=None, additional=None,
transfer=False):
"""
Create a list of headers to be used in backend requets
:param orig_req: the original request sent by the client to the proxy
:param additional: additional headers to send to the backend
:param transfer: If True, transfer headers from original client request
:returns: a dictionary of headers
"""
# Use the additional headers first so they don't overwrite the headers
# we require.
headers = HeaderKeyDict(additional) if additional else HeaderKeyDict()
@ -385,101 +553,31 @@ class Controller(object):
:param account: name of the account to get the info for
:param req: caller's HTTP request context object (optional)
:param autocreate: whether or not to automatically create the given
account or not (optional, default: False)
:returns: tuple of (account partition, account nodes, container_count)
or (None, None, None) if it does not exist
"""
partition, nodes = self.app.account_ring.get_nodes(account)
account_info = {'status': 0,
'container_count': 0,
'total_object_count': None,
'bytes': None,
'meta': {}}
# 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
if self.app.memcache:
cache_key = get_account_memcache_key(account)
cache_value = self.app.memcache.get(cache_key)
if not isinstance(cache_value, dict):
result_code = cache_value
container_count = 0
else:
result_code = cache_value['status']
try:
container_count = int(cache_value['container_count'])
except ValueError:
container_count = 0
if result_code == HTTP_OK:
return partition, nodes, container_count
elif result_code == HTTP_NOT_FOUND:
return None, None, None
result_code = 0
path = '/%s' % account
headers = self.generate_request_headers(req)
for node in self.iter_nodes(self.app.account_ring, partition):
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], partition, 'HEAD',
path, headers)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
body = resp.read()
if is_success(resp.status):
result_code = HTTP_OK
account_info.update(
headers_to_account_info(resp.getheaders()))
break
elif resp.status == HTTP_NOT_FOUND:
if result_code == 0:
result_code = HTTP_NOT_FOUND
elif result_code != HTTP_NOT_FOUND:
result_code = -1
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
continue
else:
result_code = -1
if is_server_error(resp.status):
self.error_occurred(
node,
_('ERROR %(status)d %(body)s From Account '
'Server') %
{'status': resp.status, 'body': body[:1024]})
except (Exception, Timeout):
self.exception_occurred(node, _('Account'),
_('Trying to get account info for %s')
% path)
if self.app.memcache and result_code in (HTTP_OK, HTTP_NOT_FOUND):
if result_code == HTTP_OK:
cache_timeout = self.app.recheck_account_existence
else:
cache_timeout = self.app.recheck_account_existence * 0.1
account_info.update(status=result_code)
self.app.memcache.set(cache_key,
account_info,
time=cache_timeout)
if result_code == HTTP_OK:
try:
container_count = int(account_info['container_count'])
except ValueError:
container_count = 0
return partition, nodes, container_count
return None, None, None
if req:
env = getattr(req, 'environ', {})
else:
env = {}
info = get_info(self.app, env, account)
if not info:
return None, None, None
if info.get('container_count') is None:
container_count = 0
else:
container_count = int(info['container_count'])
return partition, nodes, container_count
def container_info(self, account, container, req=None):
"""
Get container information and thusly verify container existence.
This will also make a call to account_info to verify that the
account exists.
This will also verify account existence.
:param account: account name for the container
:param container: container name to look up
:param req: caller's HTTP request context object (optional)
:param account_autocreate: whether or not to automatically create the
given account or not (optional, default: False)
:returns: dict containing at least container partition ('partition'),
container nodes ('containers'), container read
acl ('read_acl'), container write acl ('write_acl'),
@ -487,69 +585,19 @@ class Controller(object):
Values are set to None if the container does not exist.
"""
part, nodes = self.app.container_ring.get_nodes(account, container)
path = '/%s/%s' % (account, container)
container_info = {'status': 0, 'read_acl': None,
'write_acl': None, 'sync_key': None,
'count': None, 'bytes': None,
'versions': None, 'partition': None,
'nodes': None}
if self.app.memcache:
cache_key = get_container_memcache_key(account, container)
cache_value = self.app.memcache.get(cache_key)
if isinstance(cache_value, dict):
if 'container_size' in cache_value:
cache_value['count'] = cache_value['container_size']
if is_success(cache_value['status']):
container_info.update(cache_value)
container_info['partition'] = part
container_info['nodes'] = nodes
return container_info
if not self.account_info(account, req)[1]:
return container_info
headers = self.generate_request_headers(req)
for node in self.iter_nodes(self.app.container_ring, part):
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], part, 'HEAD',
path, headers)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
body = resp.read()
if is_success(resp.status):
container_info.update(
headers_to_container_info(resp.getheaders()))
break
elif resp.status == HTTP_NOT_FOUND:
container_info['status'] = HTTP_NOT_FOUND
else:
container_info['status'] = -1
if resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(resp.status):
self.error_occurred(node, _(
'ERROR %(status)d %(body)s From Container '
'Server') %
{'status': resp.status, 'body': body[:1024]})
except (Exception, Timeout):
self.exception_occurred(
node, _('Container'),
_('Trying to get container info for %s') % path)
if self.app.memcache:
if container_info['status'] == HTTP_OK:
self.app.memcache.set(
cache_key, container_info,
time=self.app.recheck_container_existence)
elif container_info['status'] == HTTP_NOT_FOUND:
self.app.memcache.set(
cache_key, container_info,
time=self.app.recheck_container_existence * 0.1)
if container_info['status'] == HTTP_OK:
container_info['partition'] = part
container_info['nodes'] = nodes
return container_info
if req:
env = getattr(req, 'environ', {})
else:
env = {}
info = get_info(self.app, env, account, container)
if not info:
info = headers_to_container_info({}, 0)
info['partition'] = None
info['nodes'] = None
else:
info['partition'] = part
info['nodes'] = nodes
return info
def iter_nodes(self, ring, partition):
"""
@ -595,6 +643,23 @@ class Controller(object):
def _make_request(self, nodes, part, method, path, headers, query,
logger_thread_locals):
"""
Sends an HTTP request to a single node and aggregates the result.
It attempts the primary node, then iterates over the handoff nodes
as needed.
:param nodes: an iterator of the backend server and handoff servers
:param part: the partition number
:param method: the method to send to the backend
:param path: the path to send to the backend
:param headers: a list of dicts, where each dict represents one
backend request that should be made.
:param query: query string to send to the backend.
:param logger_thread_locals: The thread local values to be set on the
self.app.logger to retain transaction
logging information.
:returns: a swob.Response object
"""
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
try:
@ -624,8 +689,14 @@ class Controller(object):
It attempts the primary nodes concurrently, then iterates over the
handoff nodes as needed.
:param req: a request sent by the client
:param ring: the ring used for finding backend servers
:param part: the partition number
:param method: the method to send to the backend
:param path: the path to send to the backend
:param headers: a list of dicts, where each dict represents one
backend request that should be made.
:param query_string: optional query string to send to the backend
:returns: a swob.Response object
"""
start_nodes = ring.get_part_nodes(part)
@ -675,12 +746,22 @@ class Controller(object):
@public
def GET(self, req):
"""Handler for HTTP GET requests."""
"""
Handler for HTTP GET requests.
:param req: The client request
:returns: the response to the client
"""
return self.GETorHEAD(req)
@public
def HEAD(self, req):
"""Handler for HTTP HEAD requests."""
"""
Handler for HTTP HEAD requests.
:param req: The client request
:returns: the response to the client
"""
return self.GETorHEAD(req)
def _make_app_iter_reader(self, node, source, queue, logger_thread_locals):
@ -761,6 +842,11 @@ class Controller(object):
raise
def close_swift_conn(self, src):
"""
Force close the http connection to the backend.
:param src: the response from the backend
"""
try:
src.swift_conn.close()
except Exception:
@ -780,10 +866,19 @@ class Controller(object):
"""
Indicates whether or not the request made to the backend found
what it was looking for.
:param src: the response from the backend
:returns: True if found, False if not
"""
return is_success(src.status) or is_redirection(src.status)
def autocreate_account(self, account):
def autocreate_account(self, env, account):
"""
Autocreate an account
:param env: the environment of the request leading to this autocreate
:param account: the unquoted account name
"""
partition, nodes = self.app.account_ring.get_nodes(account)
path = '/%s' % account
headers = {'X-Timestamp': normalize_timestamp(time.time()),
@ -794,9 +889,7 @@ class Controller(object):
path, [headers] * len(nodes))
if is_success(resp.status_int):
self.app.logger.info('autocreate account %r' % path)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(account))
clear_info_cache(self.app, env, account)
else:
self.app.logger.warning('Could not autocreate account %r' % path)
@ -862,6 +955,7 @@ class Controller(object):
{'status': possible_source.status,
'body': bodies[-1][:1024],
'type': server_type})
res = None
if sources:
sources.sort(key=lambda s: source_key(s[0]))
source, node = sources.pop()
@ -884,9 +978,15 @@ class Controller(object):
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
return self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method))
if not res:
res = self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method))
try:
(account, container) = split_path(req.path_info, 1, 2)
_set_info_cache(self.app, req.environ, account, container, res)
except ValueError:
pass
return res
def is_origin_allowed(self, cors_info, origin):
"""
@ -926,7 +1026,8 @@ class Controller(object):
# This is a CORS preflight request so check it's allowed
try:
container_info = \
self.container_info(self.account_name, self.container_name)
self.container_info(self.account_name,
self.container_name, req)
except AttributeError:
# This should only happen for requests to the Account. A future
# change could allow CORS requests to the Account level as well.

36
swift/proxy/controllers/container.py
View File

@ -30,7 +30,7 @@ from swift.common.utils import public, csv_append
from swift.common.constraints import check_metadata, MAX_CONTAINER_NAME_LENGTH
from swift.common.http import HTTP_ACCEPTED
from swift.proxy.controllers.base import Controller, delay_denial, \
get_container_memcache_key, headers_to_container_info, cors_validation
cors_validation, clear_info_cache
from swift.common.swob import HTTPBadRequest, HTTPForbidden, \
HTTPNotFound
@ -75,15 +75,6 @@ class ContainerController(Controller):
self.account_name, self.container_name)
resp = self.GETorHEAD_base(
req, _('Container'), self.app.container_ring, part, req.path_info)
if self.app.memcache:
# set the memcache container size for ratelimiting
cache_key = get_container_memcache_key(self.account_name,
self.container_name)
self.app.memcache.set(
cache_key,
headers_to_container_info(resp.headers, resp.status_int),
time=self.app.recheck_container_existence)
if 'swift.authorize' in req.environ:
req.acl = resp.headers.get('x-container-read')
aresp = req.environ['swift.authorize'](req)
@ -124,11 +115,11 @@ class ContainerController(Controller):
(len(self.container_name), MAX_CONTAINER_NAME_LENGTH)
return resp
account_partition, accounts, container_count = \
self.account_info(self.account_name)
self.account_info(self.account_name, req)
if not accounts and self.app.account_autocreate:
self.autocreate_account(self.account_name)
self.autocreate_account(req.environ, self.account_name)
account_partition, accounts, container_count = \
self.account_info(self.account_name)
self.account_info(self.account_name, req)
if not accounts:
return HTTPNotFound(request=req)
if self.app.max_containers_per_account > 0 and \
@ -142,10 +133,8 @@ class ContainerController(Controller):
self.account_name, self.container_name)
headers = self._backend_requests(req, len(containers),
account_partition, accounts)
if self.app.memcache:
cache_key = get_container_memcache_key(self.account_name,
self.container_name)
self.app.memcache.delete(cache_key)
clear_info_cache(self.app, req.environ,
self.account_name, self.container_name)
resp = self.make_requests(
req, self.app.container_ring,
container_partition, 'PUT', req.path_info, headers)
@ -160,15 +149,14 @@ class ContainerController(Controller):
if error_response:
return error_response
account_partition, accounts, container_count = \
self.account_info(self.account_name)
self.account_info(self.account_name, req)
if not accounts:
return HTTPNotFound(request=req)
container_partition, containers = self.app.container_ring.get_nodes(
self.account_name, self.container_name)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(get_container_memcache_key(
self.account_name, self.container_name))
clear_info_cache(self.app, req.environ,
self.account_name, self.container_name)
resp = self.make_requests(
req, self.app.container_ring, container_partition, 'POST',
req.path_info, [headers] * len(containers))
@ -186,10 +174,8 @@ class ContainerController(Controller):
self.account_name, self.container_name)
headers = self._backend_requests(req, len(containers),
account_partition, accounts)
if self.app.memcache:
cache_key = get_container_memcache_key(self.account_name,
self.container_name)
self.app.memcache.delete(cache_key)
clear_info_cache(self.app, req.environ,
self.account_name, self.container_name)
resp = self.make_requests(
req, self.app.container_ring, container_partition, 'DELETE',
req.path_info, headers)

4
swift/proxy/controllers/obj.py
View File

@ -568,7 +568,7 @@ class ObjectController(Controller):
if error_response:
return error_response
container_info = self.container_info(
self.account_name, self.container_name)
self.account_name, self.container_name, req)
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']
@ -693,7 +693,7 @@ class ObjectController(Controller):
def PUT(self, req):
"""HTTP PUT request handler."""
container_info = self.container_info(
self.account_name, self.container_name)
self.account_name, self.container_name, req)
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']

73
test/unit/__init__.py
View File

@ -17,6 +17,78 @@ from eventlet import sleep, Timeout
import logging.handlers
from httplib import HTTPException
class FakeRing(object):
def __init__(self, replicas=3):
# 9 total nodes (6 more past the initial 3) is the cap, no matter if
# this is set higher, or R^2 for R replicas
self.replicas = replicas
self.max_more_nodes = 0
self.devs = {}
def set_replicas(self, replicas):
self.replicas = replicas
self.devs = {}
@property
def replica_count(self):
return self.replicas
def get_part(self, account, container=None, obj=None):
return 1
def get_nodes(self, account, container=None, obj=None):
devs = []
for x in xrange(self.replicas):
devs.append(self.devs.get(x))
if devs[x] is None:
self.devs[x] = devs[x] = \
{'ip': '10.0.0.%s' % x,
'port': 1000 + x,
'device': 'sd' + (chr(ord('a') + x)),
'id': x}
return 1, devs
def get_part_nodes(self, part):
return self.get_nodes('blah')[1]
def get_more_nodes(self, nodes):
# replicas^2 is the true cap
for x in xrange(self.replicas, min(self.replicas + self.max_more_nodes,
self.replicas * self.replicas)):
yield {'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda'}
class FakeMemcache(object):
def __init__(self):
self.store = {}
def get(self, key):
return self.store.get(key)
def keys(self):
return self.store.keys()
def set(self, key, value, time=0):
self.store[key] = value
return True
def incr(self, key, time=0):
self.store[key] = self.store.setdefault(key, 0) + 1
return self.store[key]
@contextmanager
def soft_lock(self, key, timeout=0, retries=5):
yield True
def delete(self, key):
try:
del self.store[key]
except Exception:
pass
return True
def readuntil2crlfs(fd):
rv = ''
@ -133,6 +205,7 @@ class FakeLogger(object):
def exception(self, *args, **kwargs):
self.log_dict['exception'].append((args, kwargs, str(exc_info()[1])))
print 'FakeLogger Exception: %s' % self.log_dict
# mock out the StatsD logging methods:
increment = _store_in('increment')

6
test/unit/common/middleware/test_account_quotas.py
View File

@ -17,6 +17,9 @@ from swift.common.swob import Request
from swift.common.middleware import account_quotas
from swift.proxy.controllers.base import _get_cache_key, \
headers_to_account_info
class FakeCache(object):
def __init__(self, val):
@ -43,6 +46,9 @@ class FakeApp(object):
self.headers = headers
def __call__(self, env, start_response):
# Cache the account_info (same as a real application)
cache_key, env_key = _get_cache_key('a', None)
env[env_key] = headers_to_account_info(self.headers, 200)
start_response('200 OK', self.headers)
return []

47
test/unit/proxy/controllers/test_account.py Normal file
View File

@ -0,0 +1,47 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import unittest
import swift.proxy.controllers.base
from contextlib import contextmanager
from swift.common.swob import Request
from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_account_info
from test.unit import fake_http_connect, FakeRing, FakeMemcache
class TestAccountController(unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(),
container_ring=FakeRing(),
object_ring=FakeRing)
def test_account_info_in_response_env(self):
controller = proxy_server.AccountController(self.app, 'AUTH_bob')
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, body='')):
req = Request.blank('/AUTH_bob', {'PATH_INFO': '/AUTH_bob'})
resp = controller.HEAD(req)
self.assertEqual(2, resp.status_int // 100)
self.assertTrue('swift.account/AUTH_bob' in resp.environ)
self.assertEqual(headers_to_account_info(resp.headers),
resp.environ['swift.account/AUTH_bob'])
if __name__ == '__main__':
unittest.main()

197
test/unit/proxy/controllers/test_base.py
View File

@ -14,31 +14,45 @@
# limitations under the License.
import unittest
import swift.proxy.controllers.base
from mock import patch
from swift.proxy.controllers.base import headers_to_container_info, \
headers_to_account_info, get_container_info, get_container_memcache_key, \
get_account_info, get_account_memcache_key
get_account_info, get_account_memcache_key, _get_cache_key, get_info, \
Controller
from swift.common.swob import Request
from swift.common.utils import split_path
import swift.proxy.controllers.base
from test.unit import FakeLogger, fake_http_connect, FakeRing, FakeMemcache
from swift.proxy import server as proxy_server
FakeResponse_status_int = 201
class FakeResponse(object):
def __init__(self, headers):
def __init__(self, headers, env, account, container):
self.headers = headers
self.status_int = 201
self.status_int = FakeResponse_status_int
self.environ = env
cache_key, env_key = _get_cache_key(account, container)
if container:
info = headers_to_container_info(headers, FakeResponse_status_int)
else:
info = headers_to_account_info(headers, FakeResponse_status_int)
env[env_key] = info
class FakeRequest(object):
def __init__(self, env, method, path, swift_source=None):
(version, account,
container, obj) = split_path(env['PATH_INFO'], 2, 4, True)
def __init__(self, env, path):
self.environ = env
(version, account, container, obj) = split_path(path, 2, 4, True)
self.account = account
self.container = container
stype = container and 'container' or 'account'
self.headers = {'x-%s-object-count' % (stype): 1000,
'x-%s-bytes-used' % (stype): 6666}
def get_response(self, app):
return FakeResponse(self.headers)
return FakeResponse(self.headers, self.environ, self.account,
self.container)
class FakeCache(object):
@ -50,22 +64,150 @@ class FakeCache(object):
class TestFuncs(unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(),
container_ring=FakeRing(),
object_ring=FakeRing)
def test_GETorHEAD_base(self):
base = Controller(self.app)
req = Request.blank('/a/c')
with patch('swift.proxy.controllers.base.'
'http_connect', fake_http_connect(200)):
resp = base.GETorHEAD_base(req, 'container', FakeRing(), 'part',
'/a/c')
self.assertTrue('swift.container/a/c' in resp.environ)
self.assertEqual(resp.environ['swift.container/a/c']['status'], 200)
req = Request.blank('/a')
with patch('swift.proxy.controllers.base.'
'http_connect', fake_http_connect(200)):
resp = base.GETorHEAD_base(req, 'account', FakeRing(), 'part',
'/a')
self.assertTrue('swift.account/a' in resp.environ)
self.assertEqual(resp.environ['swift.account/a']['status'], 200)
def test_get_info(self):
global FakeResponse_status_int
# Do a non cached call to account
env = {}
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
info_a = get_info(None, env, 'a')
# Check that you got proper info
self.assertEquals(info_a['status'], 201)
self.assertEquals(info_a['bytes'], 6666)
self.assertEquals(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env, {'swift.account/a': info_a})
# Do an env cached call to account
info_a = get_info(None, env, 'a')
# Check that you got proper info
self.assertEquals(info_a['status'], 201)
self.assertEquals(info_a['bytes'], 6666)
self.assertEquals(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env, {'swift.account/a': info_a})
# This time do env cached call to account and non cached to container
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
info_c = get_info(None, env, 'a', 'c')
# Check that you got proper info
self.assertEquals(info_a['status'], 201)
self.assertEquals(info_c['bytes'], 6666)
self.assertEquals(info_c['object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env['swift.account/a'], info_a)
self.assertEquals(env['swift.container/a/c'], info_c)
# This time do a non cached call to account than non cached to container
env = {} # abandon previous call to env
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
info_c = get_info(None, env, 'a', 'c')
# Check that you got proper info
self.assertEquals(info_a['status'], 201)
self.assertEquals(info_c['bytes'], 6666)
self.assertEquals(info_c['object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env['swift.account/a'], info_a)
self.assertEquals(env['swift.container/a/c'], info_c)
# This time do an env cached call to container while account is not cached
del(env['swift.account/a'])
info_c = get_info(None, env, 'a', 'c')
# Check that you got proper info
self.assertEquals(info_a['status'], 201)
self.assertEquals(info_c['bytes'], 6666)
self.assertEquals(info_c['object_count'], 1000)
# Make sure the env cache is set and account still not cached
self.assertEquals(env, {'swift.container/a/c': info_c})
# Do a non cached call to account not found with ret_not_found
env = {}
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
try:
FakeResponse_status_int = 404
info_a = get_info(None, env, 'a', ret_not_found=True)
finally:
FakeResponse_status_int = 201
# Check that you got proper info
self.assertEquals(info_a['status'], 404)
self.assertEquals(info_a['bytes'], 6666)
self.assertEquals(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env, {'swift.account/a': info_a})
# Do a cached call to account not found with ret_not_found
info_a = get_info(None, env, 'a', ret_not_found=True)
# Check that you got proper info
self.assertEquals(info_a['status'], 404)
self.assertEquals(info_a['bytes'], 6666)
self.assertEquals(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env, {'swift.account/a': info_a})
# Do a non cached call to account not found without ret_not_found
env = {}
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
try:
FakeResponse_status_int = 404
info_a = get_info(None, env, 'a')
finally:
FakeResponse_status_int = 201
# Check that you got proper info
self.assertEquals(info_a, None)
self.assertEquals(env['swift.account/a']['status'], 404)
# Do a cached call to account not found without ret_not_found
info_a = get_info(None, env, 'a')
# Check that you got proper info
self.assertEquals(info_a, None)
self.assertEquals(env['swift.account/a']['status'], 404)
def test_get_container_info_no_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
req = Request.blank("/v1/AUTH_account/cont",
environ={'swift.cache': FakeCache({})})
resp = get_container_info(req.environ, 'xxx')
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_container_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 6666)
self.assertEquals(resp['object_count'], 1000)
def test_get_container_info_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
cached = {'status': 404,
'bytes': 3333,
'object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_container_info(req.environ, 'xxx')
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_container_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3333)
self.assertEquals(resp['object_count'], 10)
self.assertEquals(resp['status'], 404)
@ -80,26 +222,45 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(resp['bytes'], 3867)
def test_get_account_info_no_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
req = Request.blank("/v1/AUTH_account",
environ={'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
print resp
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 6666)
self.assertEquals(resp['total_object_count'], 1000)
def test_get_account_info_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
# The original test that we prefer to preserve
cached = {'status': 404,
'bytes': 3333,
'total_object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, 'xxx')
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3333)
self.assertEquals(resp['total_object_count'], 10)
self.assertEquals(resp['status'], 404)
# Here is a more realistic test
cached = {'status': 404,
'bytes': '3333',
'container_count': '234',
'total_object_count': '10',
'meta': {}}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['status'], 404)
self.assertEquals(resp['bytes'], '3333')
self.assertEquals(resp['container_count'], 234)
self.assertEquals(resp['meta'], {})
self.assertEquals(resp['total_object_count'], '10')
def test_get_account_info_env(self):
cache_key = get_account_memcache_key("account")
env_key = 'swift.%s' % cache_key

47
test/unit/proxy/controllers/test_container.py Normal file
View File

@ -0,0 +1,47 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import unittest
import swift.proxy.controllers.base
from contextlib import contextmanager
from swift.common.swob import Request
from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_container_info
from test.unit import fake_http_connect, FakeRing, FakeMemcache
class TestContainerController(unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(),
container_ring=FakeRing(),
object_ring=FakeRing)
def test_container_info_in_response_env(self):
controller = proxy_server.ContainerController(self.app, 'a', 'c')
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, body='')):
req = Request.blank('/a/c', {'PATH_INFO': '/a/c'})
resp = controller.HEAD(req)
self.assertEqual(2, resp.status_int // 100)
self.assertTrue("swift.container/a/c" in resp.environ)
self.assertEqual(headers_to_container_info(resp.headers),
resp.environ['swift.container/a/c'])
if __name__ == '__main__':
unittest.main()

264
test/unit/proxy/test_server.py
View File

@ -34,7 +34,8 @@ import mock
from eventlet import sleep, spawn, wsgi, listen
import simplejson
from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect
from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, \
fake_http_connect, FakeRing, FakeMemcache
from swift.proxy import server as proxy_server
from swift.account import server as account_server
from swift.container import server as container_server
@ -196,79 +197,6 @@ def sortHeaderNames(headerNames):
return ', '.join(headers)
class FakeRing(object):
def __init__(self, replicas=3):
# 9 total nodes (6 more past the initial 3) is the cap, no matter if
# this is set higher, or R^2 for R replicas
self.replicas = replicas
self.max_more_nodes = 0
self.devs = {}
def set_replicas(self, replicas):
self.replicas = replicas
self.devs = {}
@property
def replica_count(self):
return self.replicas
def get_part(self, account, container=None, obj=None):
return 1
def get_nodes(self, account, container=None, obj=None):
devs = []
for x in xrange(self.replicas):
devs.append(self.devs.get(x))
if devs[x] is None:
self.devs[x] = devs[x] = \
{'ip': '10.0.0.%s' % x,
'port': 1000 + x,
'device': 'sd' + (chr(ord('a') + x)),
'id': x}
return 1, devs
def get_part_nodes(self, part):
return self.get_nodes('blah')[1]
def get_more_nodes(self, nodes):
# replicas^2 is the true cap
for x in xrange(self.replicas, min(self.replicas + self.max_more_nodes,
self.replicas * self.replicas)):
yield {'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda'}
class FakeMemcache(object):
def __init__(self):
self.store = {}
def get(self, key):
return self.store.get(key)
def keys(self):
return self.store.keys()
def set(self, key, value, time=0):
self.store[key] = value
return True
def incr(self, key, time=0):
self.store[key] = self.store.setdefault(key, 0) + 1
return self.store[key]
@contextmanager
def soft_lock(self, key, timeout=0, retries=5):
yield True
def delete(self, key):
try:
del self.store[key]
except Exception:
pass
return True
class FakeMemcacheReturnsNone(FakeMemcache):
def get(self, key):
@ -393,9 +321,11 @@ class TestController(unittest.TestCase):
self.check_account_info_return(partition, nodes)
self.assertEquals(count, 12345)
# Test the internal representation in memcache
# 'container_count' changed from int to str
cache_key = get_account_memcache_key(self.account)
container_info = {'status': 200,
'container_count': 12345,
'container_count': '12345',
'total_object_count': None,
'bytes': None,
'meta': {}}
@ -417,9 +347,11 @@ class TestController(unittest.TestCase):
self.check_account_info_return(partition, nodes, True)
self.assertEquals(count, None)
# Test the internal representation in memcache
# 'container_count' changed from 0 to None
cache_key = get_account_memcache_key(self.account)
container_info = {'status': 404,
'container_count': 0,
'container_count': None, # internally keep None
'total_object_count': None,
'bytes': None,
'meta': {}}
@ -443,8 +375,9 @@ class TestController(unittest.TestCase):
self.assertEquals(count, None)
with save_globals():
test(503, 404, 404)
test(404, 404, 503)
# We cache if we have two 404 responses - fail if only one
test(503, 503, 404)
test(504, 404, 503)
test(404, 507, 503)
test(503, 503, 503)
@ -482,14 +415,12 @@ class TestController(unittest.TestCase):
# tests if 200 is cached and used
def test_container_info_200(self):
def account_info(self, account, request, autocreate=False):
return True, True, 0
with save_globals():
headers = {'x-container-read': self.read_acl,
'x-container-write': self.write_acl}
swift.proxy.controllers.Controller.account_info = account_info
set_http_connect(200, headers=headers)
set_http_connect(200, # account_info is found
200, headers=headers) # container_info is found
ret = self.controller.container_info(
self.account, self.container, self.request)
self.check_container_info_return(ret)
@ -507,12 +438,28 @@ class TestController(unittest.TestCase):
# tests if 404 is cached and used
def test_container_info_404(self):
def account_info(self, account, request, autocreate=False):
def account_info(self, account, request):
return True, True, 0
with save_globals():
swift.proxy.controllers.Controller.account_info = account_info
set_http_connect(404, 404, 404)
set_http_connect(503, 204, # account_info found
504, 404, 404) # container_info 'NotFound'
ret = self.controller.container_info(
self.account, self.container, self.request)
self.check_container_info_return(ret, True)
cache_key = get_container_memcache_key(self.account,
self.container)
cache_value = self.memcache.get(cache_key)
self.assertTrue(isinstance(cache_value, dict))
self.assertEquals(404, cache_value.get('status'))
set_http_connect()
ret = self.controller.container_info(
self.account, self.container, self.request)
self.check_container_info_return(ret, True)
set_http_connect(503, 404, 404)# account_info 'NotFound'
ret = self.controller.container_info(
self.account, self.container, self.request)
self.check_container_info_return(ret, True)
@ -538,8 +485,9 @@ class TestController(unittest.TestCase):
self.check_container_info_return(ret, True)
with save_globals():
test(503, 404, 404)
test(404, 404, 503)
# We cache if we have two 404 responses - fail if only one
test(503, 503, 404)
test(504, 404, 503)
test(404, 507, 503)
test(503, 503, 503)
@ -2338,36 +2286,50 @@ class TestObjectController(unittest.TestCase):
set_http_connect(404, 404, 404)
# acct acct acct
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(503, 404, 404)
# acct acct acct
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(503, 503, 404)
# acct acct acct
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(503, 503, 503)
# acct acct acct
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(200, 200, 204, 204, 204)
# acct cont obj obj obj
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 204)
set_http_connect(200, 404, 404, 404)
# acct cont cont cont
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(200, 503, 503, 503)
# acct cont cont cont
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
@ -2377,6 +2339,8 @@ class TestObjectController(unittest.TestCase):
set_http_connect(200)
# acct [isn't actually called since everything
# is error limited]
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
@ -2388,6 +2352,8 @@ class TestObjectController(unittest.TestCase):
set_http_connect(200, 200)
# acct cont [isn't actually called since
# everything is error limited]
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
@ -4664,10 +4630,10 @@ class TestContainerController(unittest.TestCase):
def test_HEAD_GET(self):
with save_globals():
controller = proxy_server.ContainerController(self.app, 'account',
'container')
controller = proxy_server.ContainerController(self.app, 'a', 'c')
def test_status_map(statuses, expected, **kwargs):
def test_status_map(statuses, expected,
c_expected=None, a_expected=None, **kwargs):
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/a/c', {})
@ -4678,6 +4644,18 @@ class TestContainerController(unittest.TestCase):
if expected < 400:
self.assert_('x-works' in res.headers)
self.assertEquals(res.headers['x-works'], 'yes')
if c_expected:
self.assertTrue('swift.container/a/c' in res.environ)
self.assertEquals(res.environ['swift.container/a/c']['status'],
c_expected)
else:
self.assertTrue('swift.container/a/c' not in res.environ)
if a_expected:
self.assertTrue('swift.account/a' in res.environ)
self.assertEquals(res.environ['swift.account/a']['status'],
a_expected)
else:
self.assertTrue('swift.account/a' not in res.environ)
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
@ -4689,17 +4667,37 @@ class TestContainerController(unittest.TestCase):
if expected < 400:
self.assert_('x-works' in res.headers)
self.assertEquals(res.headers['x-works'], 'yes')
test_status_map((200, 200, 404, 404), 200)
test_status_map((200, 200, 500, 404), 200)
test_status_map((200, 304, 500, 404), 304)
test_status_map((200, 404, 404, 404), 404)
test_status_map((200, 404, 404, 500), 404)
test_status_map((200, 500, 500, 500), 503)
if c_expected:
self.assertTrue('swift.container/a/c' in res.environ)
self.assertEquals(res.environ['swift.container/a/c']['status'],
c_expected)
else:
self.assertTrue('swift.container/a/c' not in res.environ)
if a_expected:
self.assertTrue('swift.account/a' in res.environ)
self.assertEquals(res.environ['swift.account/a']['status'],
a_expected)
else:
self.assertTrue('swift.account/a' not in res.environ)
# In all the following tests cache 200 for account
# return and ache vary for container
# return 200 and cache 200 for and container
test_status_map((200, 200, 404, 404), 200, 200, 200)
test_status_map((200, 200, 500, 404), 200, 200, 200)
# return 304 dont cache container
test_status_map((200, 304, 500, 404), 304, None, 200)
# return 404 and cache 404 for container
test_status_map((200, 404, 404, 404), 404, 404, 200)
test_status_map((200, 404, 404, 500), 404, 404, 200)
# return 503, dont cache container
test_status_map((200, 500, 500, 500), 503, None, 200)
self.assertFalse(self.app.account_autocreate)
test_status_map((404, 404, 404), 404)
self.app.account_autocreate = True
test_status_map((404, 404, 404), 404)
# In all the following tests cache 404 for account
# return 404 (as account is not found) and dont cache container
test_status_map((404, 404, 404), 404, None, 404)
self.app.account_autocreate = True # This should make no difference
test_status_map((404, 404, 404), 404, None, 404)
def test_PUT(self):
with save_globals():
@ -4852,14 +4850,20 @@ class TestContainerController(unittest.TestCase):
self.assertEquals(resp.status_int, 200)
set_http_connect(404, 404, 404, 200, 200, 200)
# Make sure it is a blank request wthout env caching
req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth})
resp = getattr(controller, meth)(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(503, 404, 404)
# Make sure it is a blank request wthout env caching
req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth})
resp = getattr(controller, meth)(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(503, 404, raise_exc=True)
# Make sure it is a blank request wthout env caching
req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth})
resp = getattr(controller, meth)(req)
self.assertEquals(resp.status_int, 404)
@ -4867,6 +4871,8 @@ class TestContainerController(unittest.TestCase):
dev['errors'] = self.app.error_suppression_limit + 1
dev['last_error'] = time.time()
set_http_connect(200, 200, 200, 200, 200, 200)
# Make sure it is a blank request wthout env caching
req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth})
resp = getattr(controller, meth)(req)
self.assertEquals(resp.status_int, 404)
@ -5163,6 +5169,7 @@ class TestContainerController(unittest.TestCase):
req = Request.blank('/a/c')
self.app.update_request(req)
res = controller.GET(req)
self.assertEquals(res.environ['swift.container/a/c']['status'], 204)
self.assertEquals(res.content_length, 0)
self.assertTrue('transfer-encoding' not in res.headers)
@ -5180,6 +5187,7 @@ class TestContainerController(unittest.TestCase):
req.environ['swift.authorize'] = authorize
self.app.update_request(req)
res = controller.GET(req)
self.assertEquals(res.environ['swift.container/a/c']['status'], 201)
self.assert_(called[0])
def test_HEAD_calls_authorize(self):
@ -5475,18 +5483,24 @@ class TestAccountController(unittest.TestCase):
container_ring=FakeRing(),
object_ring=FakeRing)
def assert_status_map(self, method, statuses, expected):
def assert_status_map(self, method, statuses, expected, env_expected=None):
with save_globals():
set_http_connect(*statuses)
req = Request.blank('/a', {})
self.app.update_request(req)
res = method(req)
self.assertEquals(res.status_int, expected)
if env_expected:
self.assertEquals(res.environ['swift.account/a']['status'],
env_expected)
set_http_connect(*statuses)
req = Request.blank('/a/', {})
self.app.update_request(req)
res = method(req)
self.assertEquals(res.status_int, expected)
if env_expected:
self.assertEquals(res.environ['swift.account/a']['status'],
env_expected)
def test_OPTIONS(self):
with save_globals():
@ -5532,23 +5546,23 @@ class TestAccountController(unittest.TestCase):
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
# GET returns after the first successful call to an Account Server
self.assert_status_map(controller.GET, (200,), 200)
self.assert_status_map(controller.GET, (503, 200), 200)
self.assert_status_map(controller.GET, (503, 503, 200), 200)
self.assert_status_map(controller.GET, (204,), 204)
self.assert_status_map(controller.GET, (503, 204), 204)
self.assert_status_map(controller.GET, (503, 503, 204), 204)
self.assert_status_map(controller.GET, (404, 200), 200)
self.assert_status_map(controller.GET, (404, 404, 200), 200)
self.assert_status_map(controller.GET, (404, 503, 204), 204)
self.assert_status_map(controller.GET, (200,), 200, 200)
self.assert_status_map(controller.GET, (503, 200), 200, 200)
self.assert_status_map(controller.GET, (503, 503, 200), 200, 200)
self.assert_status_map(controller.GET, (204,), 204, 204)
self.assert_status_map(controller.GET, (503, 204), 204, 204)
self.assert_status_map(controller.GET, (503, 503, 204), 204, 204)
self.assert_status_map(controller.GET, (404, 200), 200, 200)
self.assert_status_map(controller.GET, (404, 404, 200), 200, 200)
self.assert_status_map(controller.GET, (404, 503, 204), 204, 204)
# If Account servers fail, if autocreate = False, return majority
# response
self.assert_status_map(controller.GET, (404, 404, 404), 404)
self.assert_status_map(controller.GET, (404, 404, 503), 404)
self.assert_status_map(controller.GET, (404, 404, 404), 404, 404)
self.assert_status_map(controller.GET, (404, 404, 503), 404, 404)
self.assert_status_map(controller.GET, (404, 503, 503), 503)
self.app.memcache = FakeMemcacheReturnsNone()
self.assert_status_map(controller.GET, (404, 404, 404), 404)
self.assert_status_map(controller.GET, (404, 404, 404), 404, 404)
def test_GET_autocreate(self):
@ -5579,19 +5593,19 @@ class TestAccountController(unittest.TestCase):
# Same behaviour as GET
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
self.assert_status_map(controller.HEAD, (200,), 200)
self.assert_status_map(controller.HEAD, (503, 200), 200)
self.assert_status_map(controller.HEAD, (503, 503, 200), 200)
self.assert_status_map(controller.HEAD, (204,), 204)
self.assert_status_map(controller.HEAD, (503, 204), 204)
self.assert_status_map(controller.HEAD, (204, 503, 503), 204)
self.assert_status_map(controller.HEAD, (204,), 204)
self.assert_status_map(controller.HEAD, (404, 404, 404), 404)
self.assert_status_map(controller.HEAD, (404, 404, 200), 200)
self.assert_status_map(controller.HEAD, (404, 200), 200)
self.assert_status_map(controller.HEAD, (404, 404, 503), 404)
self.assert_status_map(controller.HEAD, (200,), 200, 200)
self.assert_status_map(controller.HEAD, (503, 200), 200, 200)
self.assert_status_map(controller.HEAD, (503, 503, 200), 200, 200)
self.assert_status_map(controller.HEAD, (204,), 204, 204)
self.assert_status_map(controller.HEAD, (503, 204), 204, 204)
self.assert_status_map(controller.HEAD, (204, 503, 503), 204, 204)
self.assert_status_map(controller.HEAD, (204,), 204, 204)
self.assert_status_map(controller.HEAD, (404, 404, 404), 404, 404)
self.assert_status_map(controller.HEAD, (404, 404, 200), 200, 200)
self.assert_status_map(controller.HEAD, (404, 200), 200, 200)
self.assert_status_map(controller.HEAD, (404, 404, 503), 404, 404)
self.assert_status_map(controller.HEAD, (404, 503, 503), 503)
self.assert_status_map(controller.HEAD, (404, 503, 204), 204)
self.assert_status_map(controller.HEAD, (404, 503, 204), 204, 204)
def test_HEAD_autocreate(self):
# Same behaviour as GET