openstack/swift
Code Issues Proposed changes

Allow internal clients to use reserved namespace

Browse Source 
Reserve the namespace starting with the NULL byte for internal
use-cases.  Backend services will allow path names to include the NULL
byte in urls and validate names in the reserved namespace.  Database
services will filter all names starting with the NULL byte from
responses unless the request includes the header:

    X-Backend-Allow-Reserved-Names: true

The proxy server will not allow path names to include the NULL byte in
urls unless a middlware has set the X-Backend-Allow-Reserved-Names
header.  Middlewares can use the reserved namespace to create objects
and containers that can not be directly manipulated by clients.  Any
objects and bytes created in the reserved namespace will be aggregated
to the user's account totals.

When deploying internal proxys developers and operators may configure
the gatekeeper middleware to translate the X-Allow-Reserved-Names header
to the Backend header so they can manipulate the reserved namespace
directly through the normal API.

UpgradeImpact: it's not safe to rollback from this change

Change-Id: If912f71d8b0d03369680374e8233da85d8d38f85
This commit is contained in:
Clay Gerrard 2019-09-13 12:25:24 -05:00
parent a2aaf59852
commit 698717d886
40 changed files with 2005 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
etc/proxy-server.conf-sample
View File

@ -1008,6 +1008,11 @@ use = egg:swift#gatekeeper
# difficult-to-delete data.
# shunt_inbound_x_timestamp = true
#
# Set this to true if you want to allow clients to access and manipulate the
# (normally internal-to-swift) null namespace by including a header like
# X-Allow-Reserved-Names: true
# allow_reserved_names_header = false
#
# You can override the default log routing for this filter here:
# set log_name = gatekeeper
# set log_facility = LOG_LOCAL0

10
swift/account/backend.py
View File

@ -22,7 +22,7 @@ import sqlite3
import six
from swift.common.utils import Timestamp
from swift.common.utils import Timestamp, RESERVED_BYTE
from swift.common.db import DatabaseBroker, utf8encode, zero_like
DATADIR = 'accounts'
@ -355,7 +355,7 @@ class AccountBroker(DatabaseBroker):
''').fetchone())
def list_containers_iter(self, limit, marker, end_marker, prefix,
delimiter, reverse=False):
delimiter, reverse=False, allow_reserved=False):
"""
Get a list of containers sorted by name starting at marker onward, up
to limit entries. Entries will begin with the prefix and will not have
@ -367,6 +367,7 @@ class AccountBroker(DatabaseBroker):
:param prefix: prefix query
:param delimiter: delimiter for query
:param reverse: reverse the result order.
:param allow_reserved: exclude names with reserved-byte by default
:returns: list of tuples of (name, object_count, bytes_used,
put_timestamp, 0)
@ -410,6 +411,9 @@ class AccountBroker(DatabaseBroker):
elif prefix:
query += ' name >= ? AND'
query_args.append(prefix)
if not allow_reserved:
query += ' name >= ? AND'
query_args.append(chr(ord(RESERVED_BYTE) + 1))
if self.get_db_version(conn) < 1:
query += ' +deleted = 0'
else:
@ -441,7 +445,7 @@ class AccountBroker(DatabaseBroker):
curs.close()
return results
end = name.find(delimiter, len(prefix))
if end > 0:
if end >= 0:
if reverse:
end_marker = name[:end + len(delimiter)]
else:

5
swift/account/reaper.py
View File

@ -262,7 +262,7 @@ class AccountReaper(Daemon):
container_limit *= len(nodes)
try:
containers = list(broker.list_containers_iter(
container_limit, '', None, None, None))
container_limit, '', None, None, None, allow_reserved=True))
while containers:
try:
for (container, _junk, _junk, _junk, _junk) in containers:
@ -282,7 +282,8 @@ class AccountReaper(Daemon):
self.logger.exception(
'Exception with containers for account %s', account)
containers = list(broker.list_containers_iter(
container_limit, containers[-1][0], None, None, None))
container_limit, containers[-1][0], None, None, None,
allow_reserved=True))
log_buf = ['Completed pass on account %s' % account]
except (Exception, Timeout):
self.logger.exception('Exception with account %s', account)

43
swift/account/server.py
View File

@ -26,7 +26,8 @@ from swift.account.backend import AccountBroker, DATADIR
from swift.account.utils import account_listing_response, get_response_headers
from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists
from swift.common.request_helpers import get_param, \
split_and_validate_path
split_and_validate_path, validate_internal_account, \
validate_internal_container
from swift.common.utils import get_logger, hash_path, public, \
Timestamp, storage_directory, config_true_value, \
timing_stats, replication, get_log_line, \
@ -44,6 +45,32 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, \
from swift.common.request_helpers import is_sys_or_user_meta
def get_account_name_and_placement(req):
"""
Split and validate path for an account.
:param req: a swob request
:returns: a tuple of path parts as strings
"""
drive, part, account = split_and_validate_path(req, 3)
validate_internal_account(account)
return drive, part, account
def get_container_name_and_placement(req):
"""
Split and validate path for a container.
:param req: a swob request
:returns: a tuple of path parts as strings
"""
drive, part, account, container = split_and_validate_path(req, 3, 4)
validate_internal_container(account, container)
return drive, part, account, container
class AccountController(BaseStorageServer):
"""WSGI controller for the account server."""
@ -96,7 +123,7 @@ class AccountController(BaseStorageServer):
@timing_stats()
def DELETE(self, req):
"""Handle HTTP DELETE request."""
drive, part, account = split_and_validate_path(req, 3)
drive, part, account = get_account_name_and_placement(req)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
@ -120,7 +147,7 @@ class AccountController(BaseStorageServer):
@timing_stats()
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container = split_and_validate_path(req, 3, 4)
drive, part, account, container = get_container_name_and_placement(req)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
@ -185,7 +212,7 @@ class AccountController(BaseStorageServer):
@timing_stats()
def HEAD(self, req):
"""Handle HTTP HEAD request."""
drive, part, account = split_and_validate_path(req, 3)
drive, part, account = get_account_name_and_placement(req)
out_content_type = listing_formats.get_listing_content_type(req)
try:
check_drive(self.root, drive, self.mount_check)
@ -204,7 +231,7 @@ class AccountController(BaseStorageServer):
@timing_stats()
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account = split_and_validate_path(req, 3)
drive, part, account = get_account_name_and_placement(req)
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
@ -262,7 +289,7 @@ class AccountController(BaseStorageServer):
@timing_stats()
def POST(self, req):
"""Handle HTTP POST request."""
drive, part, account = split_and_validate_path(req, 3)
drive, part, account = get_account_name_and_placement(req)
req_timestamp = valid_timestamp(req)
try:
check_drive(self.root, drive, self.mount_check)
@ -280,8 +307,8 @@ class AccountController(BaseStorageServer):
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(wsgi_to_str(req.path_info)):
res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
if not check_utf8(wsgi_to_str(req.path_info), internal=True):
res = HTTPPreconditionFailed(body='Invalid UTF8')
else:
try:
# disallow methods which are not publicly accessible

9
swift/account/utils.py
View File

@ -17,6 +17,7 @@ import json
import six
from swift.common import constraints
from swift.common.middleware import listing_formats
from swift.common.swob import HTTPOk, HTTPNoContent, str_to_wsgi
from swift.common.utils import Timestamp
@ -71,15 +72,17 @@ def get_response_headers(broker):
def account_listing_response(account, req, response_content_type, broker=None,
limit='', marker='', end_marker='', prefix='',
delimiter='', reverse=False):
limit=constraints.ACCOUNT_LISTING_LIMIT,
marker='', end_marker='', prefix='', delimiter='',
reverse=False):
if broker is None:
broker = FakeAccountBroker()
resp_headers = get_response_headers(broker)
account_list = broker.list_containers_iter(limit, marker, end_marker,
prefix, delimiter, reverse)
prefix, delimiter, reverse,
req.allow_reserved_names)
data = []
for (name, object_count, bytes_used, put_timestamp, is_subdir) \
in account_list:

10
swift/common/constraints.py
View File

@ -346,12 +346,13 @@ def check_delete_headers(request):
return request
def check_utf8(string):
def check_utf8(string, internal=False):
"""
Validate if a string is valid UTF-8 str or unicode and that it
does not contain any null character.
does not contain any reserved characters.
:param string: string to be validated
:param internal: boolean, allows reserved characters if True
:returns: True if the string is valid utf-8 str or unicode and
contains no null characters, False otherwise
"""
@ -382,7 +383,9 @@ def check_utf8(string):
if any(0xD800 <= ord(codepoint) <= 0xDFFF
for codepoint in decoded):
return False
return b'\x00' not in encoded
if b'\x00' != utils.RESERVED_BYTE and b'\x00' in encoded:
return False
return True if internal else utils.RESERVED_BYTE not in encoded
# If string is unicode, decode() will raise UnicodeEncodeError
# So, we should catch both UnicodeDecodeError & UnicodeEncodeError
except UnicodeError:
@ -413,6 +416,7 @@ def check_name_format(req, name, target_type):
body='%s name cannot contain slashes' % target_type)
return name
check_account_format = functools.partial(check_name_format,
target_type='Account')
check_container_format = functools.partial(check_name_format,

2
swift/common/direct_client.py
View File

@ -100,6 +100,7 @@ def _make_req(node, part, method, path, headers, stype,
if content_length is None:
headers['Transfer-Encoding'] = 'chunked'
headers.setdefault('X-Backend-Allow-Reserved-Names', 'true')
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
method, path, headers=headers)
@ -193,6 +194,7 @@ def gen_headers(hdrs_in=None, add_ts=True):
hdrs_out['X-Timestamp'] = Timestamp.now().internal
if 'user-agent' not in hdrs_out:
hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid()
hdrs_out.setdefault('X-Backend-Allow-Reserved-Names', 'true')
return hdrs_out

27
swift/common/internal_client.py
View File

@ -184,6 +184,7 @@ class InternalClient(object):
headers = dict(headers)
headers['user-agent'] = self.user_agent
headers.setdefault('x-backend-allow-reserved-names', 'true')
for attempt in range(self.request_tries):
resp = exc_type = exc_value = exc_traceback = None
req = Request.blank(
@ -384,6 +385,19 @@ class InternalClient(object):
return self._iter_items(path, marker, end_marker, prefix,
acceptable_statuses)
def create_account(self, account):
"""
Creates an account.
:param account: Account to create.
:raises UnexpectedResponse: Exception raised when requests fail
to get a response with an acceptable status
:raises Exception: Exception is raised when code fails in an
unexpected way.
"""
path = self.make_path(account)
self.make_request('PUT', path, {}, (201, 202))
def delete_account(self, account, acceptable_statuses=(2, HTTP_NOT_FOUND)):
"""
Deletes an account.
@ -514,7 +528,8 @@ class InternalClient(object):
self.make_request('PUT', path, headers, acceptable_statuses)
def delete_container(
self, account, container, acceptable_statuses=(2, HTTP_NOT_FOUND)):
self, account, container, headers=None,
acceptable_statuses=(2, HTTP_NOT_FOUND)):
"""
Deletes a container.
@ -529,8 +544,9 @@ class InternalClient(object):
unexpected way.
"""
headers = headers or {}
path = self.make_path(account, container)
self.make_request('DELETE', path, {}, acceptable_statuses)
self.make_request('DELETE', path, headers, acceptable_statuses)
def get_container_metadata(
self, account, container, metadata_prefix='',
@ -669,7 +685,7 @@ class InternalClient(object):
return self._get_metadata(path, metadata_prefix, acceptable_statuses,
headers=headers, params=params)
def get_object(self, account, container, obj, headers,
def get_object(self, account, container, obj, headers=None,
acceptable_statuses=(2,), params=None):
"""
Gets an object.
@ -771,7 +787,8 @@ class InternalClient(object):
path, metadata, metadata_prefix, acceptable_statuses)
def upload_object(
self, fobj, account, container, obj, headers=None):
self, fobj, account, container, obj, headers=None,
acceptable_statuses=(2,)):
"""
:param fobj: File object to read object's content from.
:param account: The object's account.
@ -789,7 +806,7 @@ class InternalClient(object):
if 'Content-Length' not in headers:
headers['Transfer-Encoding'] = 'chunked'
path = self.make_path(account, container, obj)
self.make_request('PUT', path, headers, (2,), fobj)
self.make_request('PUT', path, headers, acceptable_statuses, fobj)
def get_auth(url, user, key, auth_version='1.0', **kwargs):

7
swift/common/middleware/gatekeeper.py
View File

@ -75,6 +75,8 @@ class GatekeeperMiddleware(object):
self.outbound_condition = make_exclusion_test(outbound_exclusions)
self.shunt_x_timestamp = config_true_value(
conf.get('shunt_inbound_x_timestamp', 'true'))
self.allow_reserved_names_header = config_true_value(
conf.get('allow_reserved_names_header', 'false'))
def __call__(self, env, start_response):
req = Request(env)
@ -89,6 +91,11 @@ class GatekeeperMiddleware(object):
self.logger.debug('shunted request headers: %s' %
[('X-Timestamp', ts)])
if 'X-Allow-Reserved-Names' in req.headers \
and self.allow_reserved_names_header:
req.headers['X-Backend-Allow-Reserved-Names'] = \
req.headers.pop('X-Allow-Reserved-Names')
def gatekeeper_response(status, response_headers, exc_info=None):
def fixed_response_headers():
def relative_path(value):

59
swift/common/middleware/listing_formats.py
View File

@ -21,7 +21,8 @@ from swift.common.constraints import valid_api_version
from swift.common.http import HTTP_NO_CONTENT
from swift.common.request_helpers import get_param
from swift.common.swob import HTTPException, HTTPNotAcceptable, Request, \
RESPONSE_REASONS, HTTPBadRequest
RESPONSE_REASONS, HTTPBadRequest, wsgi_quote, wsgi_to_bytes
from swift.common.utils import RESERVED, get_logger
#: Mapping of query string ``format=`` values to their corresponding
@ -73,8 +74,6 @@ def to_xml(document_element):
def account_to_xml(listing, account_name):
if isinstance(account_name, bytes):
account_name = account_name.decode('utf-8')
doc = Element('account', name=account_name)
doc.text = '\n'
for record in listing:
@ -91,8 +90,6 @@ def account_to_xml(listing, account_name):
def container_to_xml(listing, base_name):
if isinstance(base_name, bytes):
base_name = base_name.decode('utf-8')
doc = Element('container', name=base_name)
for record in listing:
if 'subdir' in record:
@ -119,8 +116,33 @@ def listing_to_text(listing):
class ListingFilter(object):
def __init__(self, app):
def __init__(self, app, conf, logger=None):
self.app = app
self.logger = logger or get_logger(conf, log_route='listing-filter')
def filter_reserved(self, listing, account, container):
new_listing = []
for entry in list(listing):
for key in ('name', 'subdir'):
value = entry.get(key, '')
if six.PY2:
value = value.encode('utf-8')
if RESERVED in value:
if container:
self.logger.warning(
'Container listing for %s/%s had '
'reserved byte in %s: %r',
wsgi_quote(account), wsgi_quote(container),
key, value)
else:
self.logger.warning(
'Account listing for %s had '
'reserved byte in %s: %r',
wsgi_quote(account), key, value)
break # out of the *key* loop; check next entry
else:
new_listing.append(entry)
return new_listing
def __call__(self, env, start_response):
req = Request(env)
@ -128,10 +150,10 @@ class ListingFilter(object):
# account and container only
version, acct, cont = req.split_path(2, 3)
except ValueError:
is_container_req = False
is_account_or_container_req = False
else:
is_container_req = True
if not is_container_req:
is_account_or_container_req = True
if not is_account_or_container_req:
return self.app(env, start_response)
if not valid_api_version(version) or req.method not in ('GET', 'HEAD'):
@ -201,15 +223,21 @@ class ListingFilter(object):
start_response(status, headers)
return [body]
if not req.allow_reserved_names:
listing = self.filter_reserved(listing, acct, cont)
try:
if out_content_type.endswith('/xml'):
if cont:
body = container_to_xml(listing, cont)
body = container_to_xml(
listing, wsgi_to_bytes(cont).decode('utf-8'))
else:
body = account_to_xml(listing, acct)
body = account_to_xml(
listing, wsgi_to_bytes(acct).decode('utf-8'))
elif out_content_type == 'text/plain':
body = listing_to_text(listing)
# else, json -- we continue down here to be sure we set charset
else:
body = json.dumps(listing).encode('ascii')
except KeyError:
# listing was in a bad format -- funky static web listing??
start_response(status, headers)
@ -226,4 +254,9 @@ class ListingFilter(object):
def filter_factory(global_conf, **local_conf):
return ListingFilter
conf = global_conf.copy()
conf.update(local_conf)
def listing_filter(app):
return ListingFilter(app, conf)
return listing_filter

74
swift/common/request_helpers.py
View File

@ -38,8 +38,8 @@ from swift.common.swob import HTTPBadRequest, \
from swift.common.utils import split_path, validate_device_partition, \
close_if_possible, maybe_multipart_byteranges_to_document_iters, \
multipart_byteranges_to_document_iters, parse_content_type, \
parse_content_range, csv_append, list_from_csv, Spliterator, quote
parse_content_range, csv_append, list_from_csv, Spliterator, quote, \
RESERVED
from swift.common.wsgi import make_subrequest
@ -83,6 +83,54 @@ def get_param(req, name, default=None):
return value
def _validate_internal_name(name, type_='name'):
if RESERVED in name and not name.startswith(RESERVED):
raise HTTPBadRequest(body='Invalid reserved-namespace %s' % (type_))
def validate_internal_account(account):
"""
Validate internal account name.
:raises: HTTPBadRequest
"""
_validate_internal_name(account, 'account')
def validate_internal_container(account, container):
"""
Validate internal account and container names.
:raises: HTTPBadRequest
"""
if not account:
raise ValueError('Account is required')
validate_internal_account(account)
if container:
_validate_internal_name(container, 'container')
def validate_internal_obj(account, container, obj):
"""
Validate internal account, container and object names.
:raises: HTTPBadRequest
"""
if not account:
raise ValueError('Account is required')
if not container:
raise ValueError('Container is required')
validate_internal_container(account, container)
if obj:
_validate_internal_name(obj, 'object')
if container.startswith(RESERVED) and not obj.startswith(RESERVED):
raise HTTPBadRequest(body='Invalid user-namespace object '
'in reserved-namespace container')
elif obj.startswith(RESERVED) and not container.startswith(RESERVED):
raise HTTPBadRequest(body='Invalid reserved-namespace object '
'in user-namespace container')
def get_name_and_placement(request, minsegs=1, maxsegs=None,
rest_with_last=False):
"""
@ -273,6 +321,28 @@ def get_container_update_override_key(key):
return header.title()
def get_reserved_name(*parts):
"""
Generate a valid reserved name that joins the component parts.
:returns: a string
"""
if any(RESERVED in p for p in parts):
raise ValueError('Invalid reserved part in components')
return RESERVED + RESERVED.join(parts)
def split_reserved_name(name):
"""
Seperate a valid reserved name into the component parts.
:returns: a list of strings
"""
if not name.startswith(RESERVED):
raise ValueError('Invalid reserved name')
return name.split(RESERVED)[1:]
def remove_items(headers, condition):
"""
Removes items from a dict whose keys satisfy

7
swift/common/swob.py
View File

@ -52,7 +52,7 @@ from six.moves import urllib
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.utils import UTC, reiterate, split_path, Timestamp, pairs, \
close_if_possible, closing_if_possible
close_if_possible, closing_if_possible, config_true_value
from swift.common.exceptions import InvalidTimestamp
@ -1063,6 +1063,11 @@ class Request(object):
"Provides the full url of the request"
return self.host_url + self.path_qs
@property
def allow_reserved_names(self):
return config_true_value(self.environ.get(
'HTTP_X_BACKEND_ALLOW_RESERVED_NAMES'))
def as_referer(self):
return self.method + ' ' + self.url

4
swift/common/utils.py
View File

@ -186,6 +186,10 @@ O_TMPFILE = getattr(os, 'O_TMPFILE', 0o20000000 | os.O_DIRECTORY)
IPV6_RE = re.compile("^\[(?P<address>.*)\](:(?P<port>[0-9]+))?$")
MD5_OF_EMPTY_STRING = 'd41d8cd98f00b204e9800998ecf8427e'
RESERVED_BYTE = b'\x00'
RESERVED_STR = u'\x00'
RESERVED = '\x00'
LOG_LINE_DEFAULT_FORMAT = '{remote_addr} - - [{time.d}/{time.b}/{time.Y}' \
':{time.H}:{time.M}:{time.S} +0000] ' \

11
swift/container/backend.py
View File

@ -30,7 +30,8 @@ from swift.common.exceptions import LockTimeout
from swift.common.utils import Timestamp, encode_timestamps, \
decode_timestamps, extract_swift_bytes, storage_directory, hash_path, \
ShardRange, renamer, find_shard_range, MD5_OF_EMPTY_STRING, mkdirs, \
get_db_files, parse_db_filename, make_db_file_path, split_path
get_db_files, parse_db_filename, make_db_file_path, split_path, \
RESERVED_BYTE
from swift.common.db import DatabaseBroker, utf8encode, BROKER_TIMEOUT, \
zero_like, DatabaseAlreadyExists
@ -1028,7 +1029,8 @@ class ContainerBroker(DatabaseBroker):
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
path=None, storage_policy_index=0, reverse=False,
include_deleted=False, since_row=None,
transform_func=None, all_policies=False):
transform_func=None, all_policies=False,
allow_reserved=False):
"""
Get a list of objects sorted by name starting at marker onward, up
to limit entries. Entries will begin with the prefix and will not
@ -1054,6 +1056,8 @@ class ContainerBroker(DatabaseBroker):
:meth:`~_transform_record`; defaults to :meth:`~_transform_record`.
:param all_policies: if True, include objects for all storage policies
ignoring any value given for ``storage_policy_index``
:param allow_reserved: exclude names with reserved-byte by default
:returns: list of tuples of (name, created_at, size, content_type,
etag, deleted)
"""
@ -1110,6 +1114,9 @@ class ContainerBroker(DatabaseBroker):
elif prefix:
query_conditions.append('name >= ?')
query_args.append(prefix)
if not allow_reserved:
query_conditions.append('name >= ?')
query_args.append(chr(ord(RESERVED_BYTE) + 1))
query_conditions.append(deleted_key + deleted_arg)
if since_row:
query_conditions.append('ROWID > ?')

50
swift/container/server.py
View File

@ -32,7 +32,8 @@ from swift.container.replicator import ContainerReplicatorRpc
from swift.common.db import DatabaseAlreadyExists
from swift.common.container_sync_realms import ContainerSyncRealms
from swift.common.request_helpers import get_param, \
split_and_validate_path, is_sys_or_user_meta
split_and_validate_path, is_sys_or_user_meta, \
validate_internal_container, validate_internal_obj
from swift.common.utils import get_logger, hash_path, public, \
Timestamp, storage_directory, validate_sync_to, \
config_true_value, timing_stats, replication, \
@ -83,6 +84,33 @@ def gen_resp_headers(info, is_deleted=False):
return headers
def get_container_name_and_placement(req):
"""
Split and validate path for a container.
:param req: a swob request
:returns: a tuple of path parts as strings
"""
drive, part, account, container = split_and_validate_path(req, 4)
validate_internal_container(account, container)
return drive, part, account, container
def get_obj_name_and_placement(req):
"""
Split and validate path for an object.
:param req: a swob request
:returns: a tuple of path parts as strings
"""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
validate_internal_obj(account, container, obj)
return drive, part, account, container, obj
class ContainerController(BaseStorageServer):
"""WSGI Controller for the container server."""
@ -311,8 +339,7 @@ class ContainerController(BaseStorageServer):
@timing_stats()
def DELETE(self, req):
"""Handle HTTP DELETE request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
drive, part, account, container, obj = get_obj_name_and_placement(req)
req_timestamp = valid_timestamp(req)
try:
check_drive(self.root, drive, self.mount_check)
@ -433,8 +460,7 @@ class ContainerController(BaseStorageServer):
@timing_stats()
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
drive, part, account, container, obj = get_obj_name_and_placement(req)
req_timestamp = valid_timestamp(req)
if 'x-container-sync-to' in req.headers:
err, sync_to, realm, realm_key = validate_sync_to(
@ -514,8 +540,7 @@ class ContainerController(BaseStorageServer):
@timing_stats(sample_rate=0.1)
def HEAD(self, req):
"""Handle HTTP HEAD request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
drive, part, account, container, obj = get_obj_name_and_placement(req)
out_content_type = listing_formats.get_listing_content_type(req)
try:
check_drive(self.root, drive, self.mount_check)
@ -632,8 +657,7 @@ class ContainerController(BaseStorageServer):
:param req: an instance of :class:`swift.common.swob.Request`
:returns: an instance of :class:`swift.common.swob.Response`
"""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
drive, part, account, container, obj = get_obj_name_and_placement(req)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
@ -696,7 +720,7 @@ class ContainerController(BaseStorageServer):
container_list = src_broker.list_objects_iter(
limit, marker, end_marker, prefix, delimiter, path,
storage_policy_index=info['storage_policy_index'],
reverse=reverse)
reverse=reverse, allow_reserved=req.allow_reserved_names)
return self.create_listing(req, out_content_type, info, resp_headers,
broker.metadata, container_list, container)
@ -751,7 +775,7 @@ class ContainerController(BaseStorageServer):
"""
Handle HTTP UPDATE request (merge_items RPCs coming from the proxy.)
"""
drive, part, account, container = split_and_validate_path(req, 4)
drive, part, account, container = get_container_name_and_placement(req)
req_timestamp = valid_timestamp(req)
try:
check_drive(self.root, drive, self.mount_check)
@ -775,7 +799,7 @@ class ContainerController(BaseStorageServer):
@timing_stats()
def POST(self, req):
"""Handle HTTP POST request."""
drive, part, account, container = split_and_validate_path(req, 4)
drive, part, account, container = get_container_name_and_placement(req)
req_timestamp = valid_timestamp(req)
if 'x-container-sync-to' in req.headers:
err, sync_to, realm, realm_key = validate_sync_to(
@ -800,7 +824,7 @@ class ContainerController(BaseStorageServer):
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(wsgi_to_str(req.path_info)):
if not check_utf8(wsgi_to_str(req.path_info), internal=True):
res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
else:
try:

32
swift/obj/server.py
View File

@ -35,7 +35,8 @@ from swift.common.utils import public, get_logger, \
normalize_delete_at_timestamp, get_log_line, Timestamp, \
get_expirer_container, parse_mime_headers, \
iter_multipart_mime_documents, extract_swift_bytes, safe_json_loads, \
config_auto_int_value, split_path, get_redirect_data, normalize_timestamp
config_auto_int_value, split_path, get_redirect_data, \
normalize_timestamp
from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_object_creation, \
valid_timestamp, check_utf8
@ -51,7 +52,7 @@ from swift.common.base_storage_server import BaseStorageServer
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.request_helpers import get_name_and_placement, \
is_user_meta, is_sys_or_user_meta, is_object_transient_sysmeta, \
resolve_etag_is_at_header, is_sys_meta
resolve_etag_is_at_header, is_sys_meta, validate_internal_obj
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
@ -89,6 +90,20 @@ def drain(file_like, read_size, timeout):
break
def get_obj_name_and_placement(request):
"""
Split and validate path for an object.
:param request: a swob request
:returns: a tuple of path parts and storage policy
"""
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
validate_internal_obj(account, container, obj)
return device, partition, account, container, obj, policy
def _make_backend_fragments_header(fragments):
if fragments:
result = {}
@ -603,7 +618,8 @@ class ObjectController(BaseStorageServer):
def POST(self, request):
"""Handle HTTP POST requests for the Swift Object Server."""
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
get_obj_name_and_placement(request)
req_timestamp = valid_timestamp(request)
new_delete_at = int(request.headers.get('X-Delete-At') or 0)
if new_delete_at and new_delete_at < req_timestamp:
@ -995,7 +1011,7 @@ class ObjectController(BaseStorageServer):
def PUT(self, request):
"""Handle HTTP PUT requests for the Swift Object Server."""
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
get_obj_name_and_placement(request)
disk_file, fsize, orig_metadata = self._pre_create_checks(
request, device, partition, account, container, obj, policy)
writer = disk_file.writer(size=fsize)
@ -1037,7 +1053,7 @@ class ObjectController(BaseStorageServer):
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
get_obj_name_and_placement(request)
request.headers.setdefault('X-Timestamp',
normalize_timestamp(time.time()))
req_timestamp = valid_timestamp(request)
@ -1104,7 +1120,7 @@ class ObjectController(BaseStorageServer):
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
get_obj_name_and_placement(request)
request.headers.setdefault('X-Timestamp',
normalize_timestamp(time.time()))
req_timestamp = valid_timestamp(request)
@ -1163,7 +1179,7 @@ class ObjectController(BaseStorageServer):
def DELETE(self, request):
"""Handle HTTP DELETE requests for the Swift Object Server."""
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
get_obj_name_and_placement(request)
req_timestamp = valid_timestamp(request)
next_part_power = request.headers.get('X-Backend-Next-Part-Power')
try:
@ -1275,7 +1291,7 @@ class ObjectController(BaseStorageServer):
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(wsgi_to_str(req.path_info)):
if not check_utf8(wsgi_to_str(req.path_info), internal=True):
res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
else:
try:

9
swift/proxy/controllers/base.py
View File

@ -356,6 +356,9 @@ def get_container_info(env, app, swift_source=None):
req = _prepare_pre_auth_info_request(
env, ("/%s/%s/%s" % (version, wsgi_account, wsgi_container)),
(swift_source or 'GET_CONTAINER_INFO'))
# *Always* allow reserved names for get-info requests -- it's on the
# caller to keep the result private-ish
req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
resp = req.get_response(app)
close_if_possible(resp.app_iter)
# Check in infocache to see if the proxy (or anyone else) already
@ -412,6 +415,9 @@ def get_account_info(env, app, swift_source=None):
req = _prepare_pre_auth_info_request(
env, "/%s/%s" % (version, wsgi_account),
(swift_source or 'GET_ACCOUNT_INFO'))
# *Always* allow reserved names for get-info requests -- it's on the
# caller to keep the result private-ish
req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
resp = req.get_response(app)
close_if_possible(resp.app_iter)
# Check in infocache to see if the proxy (or anyone else) already
@ -739,6 +745,9 @@ def _get_object_info(app, env, account, container, obj, swift_source=None):
# Not in cache, let's try the object servers
path = '/v1/%s/%s/%s' % (account, container, obj)
req = _prepare_pre_auth_info_request(env, path, swift_source)
# *Always* allow reserved names for get-info requests -- it's on the
# caller to keep the result private-ish
req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
resp = req.get_response(app)
# Unlike get_account_info() and get_container_info(), we don't save
# things in memcache, so we can store the info without network traffic,

17
swift/proxy/controllers/obj.py
View File

@ -24,7 +24,7 @@
# These shenanigans are to ensure all related objects can be garbage
# collected. We've seen objects hang around forever otherwise.
from six.moves.urllib.parse import unquote
from six.moves.urllib.parse import quote, unquote
from six.moves import zip
import collections
@ -72,7 +72,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
HTTPUnprocessableEntity, Response, HTTPException, \
HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError
from swift.common.request_helpers import update_etag_is_at_header, \
resolve_etag_is_at_header
resolve_etag_is_at_header, validate_internal_obj
def check_content_type(req):
@ -168,6 +168,8 @@ class BaseObjectController(Controller):
self.account_name = unquote(account_name)
self.container_name = unquote(container_name)
self.object_name = unquote(object_name)
validate_internal_obj(
self.account_name, self.container_name, self.object_name)
def iter_nodes_local_first(self, ring, partition, policy=None,
local_handoffs_first=False):
@ -637,7 +639,8 @@ class BaseObjectController(Controller):
except (Exception, Timeout):
self.app.exception_occurred(
node, _('Object'),
_('Expect: 100-continue on %s') % req.swift_entity_path)
_('Expect: 100-continue on %s') %
quote(req.swift_entity_path))
def _get_put_connections(self, req, nodes, partition, outgoing_headers,
policy):
@ -1399,10 +1402,10 @@ class ECAppIter(object):
except ChunkReadTimeout:
# unable to resume in GetOrHeadHandler
self.logger.exception(_("Timeout fetching fragments for %r"),
self.path)
quote(self.path))
except: # noqa
self.logger.exception(_("Exception fetching fragments for"
" %r"), self.path)
" %r"), quote(self.path))
finally:
queue.resize(2) # ensure there's room
queue.put(None)
@ -1431,7 +1434,7 @@ class ECAppIter(object):
segment = self.policy.pyeclib_driver.decode(fragments)
except ECDriverError:
self.logger.exception(_("Error decoding fragments for"
" %r"), self.path)
" %r"), quote(self.path))
raise
yield segment
@ -1670,7 +1673,7 @@ class Putter(object):
self.failed = True
self.send_exception_handler(self.node, _('Object'),
_('Trying to write to %s')
% self.path)
% quote(self.path))
def close(self):
# release reference to response to ensure connection really does close,

3
swift/proxy/server.py
View File

@ -480,7 +480,8 @@ class Application(object):
body='Invalid Content-Length')
try:
if not check_utf8(wsgi_to_str(req.path_info)):
if not check_utf8(wsgi_to_str(req.path_info),
internal=req.allow_reserved_names):
self.logger.increment('errors')
return HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')

1
test/probe/common.py
View File

@ -487,6 +487,7 @@ class ProbeTest(unittest.TestCase):
[app:proxy-server]
use = egg:swift#proxy
allow_account_management = True
[filter:copy]
use = egg:swift#copy

23
test/probe/test_account_reaper.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from io import BytesIO
from time import sleep
import uuid
import unittest
@ -23,6 +24,7 @@ from swift.common import utils
from swift.common.manager import Manager
from swift.common.direct_client import direct_delete_account, \
direct_get_object, direct_head_container, ClientException
from swift.common.request_helpers import get_reserved_name
from test.probe.common import ReplProbeTest, ENABLED_POLICIES
@ -30,8 +32,9 @@ class TestAccountReaper(ReplProbeTest):
def setUp(self):
super(TestAccountReaper, self).setUp()
self.all_objects = []
int_client = self.make_internal_client()
# upload some containers
body = 'test-body'
body = b'test-body'
for policy in ENABLED_POLICIES:
container = 'container-%s-%s' % (policy.name, uuid.uuid4())
client.put_container(self.url, self.token, container,
@ -39,6 +42,18 @@ class TestAccountReaper(ReplProbeTest):
obj = 'object-%s' % uuid.uuid4()
client.put_object(self.url, self.token, container, obj, body)
self.all_objects.append((policy, container, obj))
# Also create some reserved names
container = get_reserved_name(
'reserved', policy.name, str(uuid.uuid4()))
int_client.create_container(
self.account, container,
headers={'X-Storage-Policy': policy.name})
obj = get_reserved_name('object', str(uuid.uuid4()))
int_client.upload_object(
BytesIO(body), self.account, container, obj)
self.all_objects.append((policy, container, obj))
policy.load_ring('/etc/swift')
Manager(['container-updater']).once()
@ -46,11 +61,11 @@ class TestAccountReaper(ReplProbeTest):
headers = client.head_account(self.url, self.token)
self.assertEqual(int(headers['x-account-container-count']),
len(ENABLED_POLICIES))
len(self.all_objects))
self.assertEqual(int(headers['x-account-object-count']),
len(ENABLED_POLICIES))
len(self.all_objects))
self.assertEqual(int(headers['x-account-bytes-used']),
len(ENABLED_POLICIES) * len(body))
len(self.all_objects) * len(body))
part, nodes = self.account_ring.get_nodes(self.account)

33
test/probe/test_replication_servers_working.py
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from io import BytesIO
from unittest import main
from uuid import uuid4
import os
@ -25,6 +26,7 @@ from swiftclient import client
from swift.obj.diskfile import get_data_dir
from test.probe.common import ReplProbeTest
from swift.common.request_helpers import get_reserved_name
from swift.common.utils import readconf
EXCLUDE_FILES = re.compile('^(hashes\.(pkl|invalid)|lock(-\d+)?)$')
@ -80,6 +82,15 @@ class TestReplicatorFunctions(ReplProbeTest):
different port values.
"""
def put_data(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy':
self.policy.name})
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, 'VERIFY')
def test_main(self):
# Create one account, container and object file.
# Find node with account, container and object replicas.
@ -102,13 +113,7 @@ class TestReplicatorFunctions(ReplProbeTest):
path_list.append(os.path.join(device_path, device))
# Put data to storage nodes
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy':
self.policy.name})
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, 'VERIFY')
self.put_data()
# Get all data file information
(files_list, dir_list) = collect_info(path_list)
@ -200,5 +205,19 @@ class TestReplicatorFunctions(ReplProbeTest):
self.replicators.stop()
class TestReplicatorFunctionsReservedNames(TestReplicatorFunctions):
def put_data(self):
int_client = self.make_internal_client()
int_client.create_account(self.account)
container = get_reserved_name('container', str(uuid4()))
int_client.create_container(self.account, container,
headers={'X-Storage-Policy':
self.policy.name})
obj = get_reserved_name('object', str(uuid4()))
int_client.upload_object(
BytesIO(b'VERIFY'), self.account, container, obj)
if __name__ == '__main__':
main()

131
test/probe/test_reserved_name.py Normal file
View File

@ -0,0 +1,131 @@
#!/usr/bin/python -u
# Copyright (c) 2019 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from io import BytesIO
from uuid import uuid4
from swift.common.request_helpers import get_reserved_name
from test.probe.common import ReplProbeTest
from swiftclient import client, ClientException
class TestReservedNames(ReplProbeTest):
def test_simple_crud(self):
int_client = self.make_internal_client()
# Create reserve named container
user_cont = 'container-%s' % uuid4()
reserved_cont = get_reserved_name('container-%s' % uuid4())
client.put_container(self.url, self.token, user_cont)
int_client.create_container(self.account, reserved_cont)
# Check that we can list both reserved and non-reserved containers
self.assertEqual([reserved_cont, user_cont], [
c['name'] for c in int_client.iter_containers(self.account)])
# sanity, user can't get to reserved name
with self.assertRaises(ClientException) as cm:
client.head_container(self.url, self.token, reserved_cont)
self.assertEqual(412, cm.exception.http_status)
user_obj = 'obj-%s' % uuid4()
reserved_obj = get_reserved_name('obj-%s' % uuid4())
# InternalClient can write & read reserved names fine
int_client.upload_object(
BytesIO(b'data'), self.account, reserved_cont, reserved_obj)
int_client.get_object_metadata(
self.account, reserved_cont, reserved_obj)
_, _, app_iter = int_client.get_object(
self.account, reserved_cont, reserved_obj)
self.assertEqual(b''.join(app_iter), b'data')
self.assertEqual([reserved_obj], [
o['name']
for o in int_client.iter_objects(self.account, reserved_cont)])
# But reserved objects must be in reserved containers, and
# user objects must be in user containers (at least for now)
int_client.upload_object(
BytesIO(b'data'), self.account, reserved_cont, user_obj,
acceptable_statuses=(400,))
int_client.upload_object(
BytesIO(b'data'), self.account, user_cont, reserved_obj,
acceptable_statuses=(400,))
# Make sure we can clean up, too
int_client.delete_object(self.account, reserved_cont, reserved_obj)
int_client.delete_container(self.account, reserved_cont)
def test_symlink_target(self):
if 'symlink' not in self.cluster_info:
raise unittest.SkipTest(
"Symlink not enabled in proxy; can't test "
"symlink to reserved name")
int_client = self.make_internal_client()
# create link container first, ensure account gets created too
client.put_container(self.url, self.token, 'c1')
# Create reserve named container
tgt_cont = get_reserved_name('container-%s' % uuid4())
int_client.create_container(self.account, tgt_cont)
# sanity, user can't get to reserved name
with self.assertRaises(ClientException) as cm:
client.head_container(self.url, self.token, tgt_cont)
self.assertEqual(412, cm.exception.http_status)
tgt_obj = get_reserved_name('obj-%s' % uuid4())
int_client.upload_object(
BytesIO(b'target object'), self.account, tgt_cont, tgt_obj)
metadata = int_client.get_object_metadata(
self.account, tgt_cont, tgt_obj)
etag = metadata['etag']
# users can write a dynamic symlink that targets a reserved
# name object
client.put_object(
self.url, self.token, 'c1', 'symlink',
headers={
'X-Symlink-Target': '%s/%s' % (tgt_cont, tgt_obj),
'Content-Type': 'application/symlink',
})
# but can't read the symlink
with self.assertRaises(ClientException) as cm:
client.get_object(self.url, self.token, 'c1', 'symlink')
self.assertEqual(412, cm.exception.http_status)
# user's can't create static symlink to reserved name
with self.assertRaises(ClientException) as cm:
client.put_object(
self.url, self.token, 'c1', 'static-symlink',
headers={
'X-Symlink-Target': '%s/%s' % (tgt_cont, tgt_obj),
'X-Symlink-Target-Etag': etag,
'Content-Type': 'application/symlink',
})
self.assertEqual(412, cm.exception.http_status)
# clean-up
client.delete_object(self.url, self.token, 'c1', 'symlink')
int_client.delete_object(self.account, tgt_cont, tgt_obj)
int_client.delete_container(self.account, tgt_cont)

30
test/unit/account/test_backend.py
View File

@ -38,6 +38,7 @@ from swift.account.backend import AccountBroker
from swift.common.utils import Timestamp
from test.unit import patch_policies, with_tempdir, make_timestamp_iter
from swift.common.db import DatabaseConnectionError
from swift.common.request_helpers import get_reserved_name
from swift.common.storage_policy import StoragePolicy, POLICIES
from test.unit.common import test_db
@ -545,6 +546,35 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual([row[0] for row in listing],
['c10', 'c1'])
def test_list_container_iter_with_reserved_name(self):
# Test ContainerBroker.list_objects_iter
broker = AccountBroker(':memory:', account='a')
broker.initialize(next(self.ts).internal, 0)
broker.put_container(
'foo', next(self.ts).internal, 0, 0, 0, POLICIES.default.idx)
broker.put_container(
get_reserved_name('foo'), next(self.ts).internal, 0, 0, 0,
POLICIES.default.idx)
listing = broker.list_containers_iter(100, None, None, '', '')
self.assertEqual([row[0] for row in listing], ['foo'])
listing = broker.list_containers_iter(100, None, None, '', '',
reverse=True)
self.assertEqual([row[0] for row in listing], ['foo'])
listing = broker.list_containers_iter(100, None, None, '', '',
allow_reserved=True)
self.assertEqual([row[0] for row in listing],
[get_reserved_name('foo'), 'foo'])
listing = broker.list_containers_iter(100, None, None, '', '',
reverse=True,
allow_reserved=True)
self.assertEqual([row[0] for row in listing],
['foo', get_reserved_name('foo')])
def test_reverse_prefix_delim(self):
expectations = [
{

16
test/unit/account/test_reaper.py
View File

@ -50,7 +50,12 @@ class FakeAccountBroker(object):
'delete_timestamp': time.time() - 10}
return info
def list_containers_iter(self, limit, marker, *args):
def list_containers_iter(self, limit, marker, *args, **kwargs):
if not kwargs.pop('allow_reserved'):
raise RuntimeError('Expected allow_reserved to be True!')
if kwargs:
raise RuntimeError('Got unexpected keyword arguments: %r' % (
kwargs, ))
for cont in self.containers:
if cont > marker:
yield cont, None, None, None, None
@ -710,11 +715,16 @@ class TestReaper(unittest.TestCase):
devices = self.prepare_data_dir()
self.called_amount = 0
conf = {'devices': devices}
r = self.init_reaper(conf, myips=['10.10.10.2'])
r = self.init_reaper(conf, myips=['10.10.10.2'], fakelogger=True)
container_reaped = [0]
def fake_list_containers_iter(self, *args):
def fake_list_containers_iter(self, *args, **kwargs):
if not kwargs.pop('allow_reserved'):
raise RuntimeError('Expected allow_reserved to be True!')
if kwargs:
raise RuntimeError('Got unexpected keyword arguments: %r' % (
kwargs, ))
for container in self.containers:
if container in self.containers_yielded:
continue

450
test/unit/account/test_server.py
View File

@ -27,17 +27,19 @@ from io import BytesIO
import json
from six import StringIO
from six.moves.urllib.parse import quote
import xml.dom.minidom
from swift import __version__ as swift_version
from swift.common.swob import (Request, WsgiBytesIO, HTTPNoContent)
from swift.common import constraints
from swift.common.constraints import ACCOUNT_LISTING_LIMIT
from swift.account.backend import AccountBroker
from swift.account.server import AccountController
from swift.common.utils import (normalize_timestamp, replication, public,
mkdirs, storage_directory, Timestamp)
from swift.common.request_helpers import get_sys_meta_prefix
from test.unit import patch_policies, debug_logger, mock_check_drive
from swift.common.request_helpers import get_sys_meta_prefix, get_reserved_name
from test.unit import patch_policies, debug_logger, mock_check_drive, \
make_timestamp_iter
from swift.common.storage_policy import StoragePolicy, POLICIES
@ -52,6 +54,7 @@ class TestAccountController(unittest.TestCase):
self.controller = AccountController(
{'devices': self.testdir, 'mount_check': 'false'},
logger=debug_logger())
self.ts = make_timestamp_iter()
def tearDown(self):
"""Tear down for testing swift.account.server.AccountController"""
@ -501,6 +504,65 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(resp.body, b'Recently deleted')
self.assertEqual(resp.headers['X-Account-Status'], 'Deleted')
def test_create_reserved_namespace_account(self):
path = '/sda1/p/%s' % get_reserved_name('a')
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller)
self.assertEqual(resp.status, '201 Created')
path = '/sda1/p/%s' % get_reserved_name('foo', 'bar')
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller)
self.assertEqual(resp.status, '201 Created')
def test_create_invalid_reserved_namespace_account(self):
account_name = get_reserved_name('foo', 'bar')[1:]
path = '/sda1/p/%s' % account_name
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller)
self.assertEqual(resp.status, '400 Bad Request')
def test_create_reserved_container_in_account(self):
# create account
path = '/sda1/p/a'
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
# put null container in it
path += '/%s' % get_reserved_name('c', 'stuff')
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal,
'X-Put-Timestamp': next(self.ts).internal,
'X-Delete-Timestamp': 0,
'X-Object-Count': 0,
'X-Bytes-Used': 0,
})
resp = req.get_response(self.controller)
self.assertEqual(resp.status, '201 Created')
def test_create_invalid_reserved_container_in_account(self):
# create account
path = '/sda1/p/a'
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
# put invalid container in it
path += '/%s' % get_reserved_name('c', 'stuff')[1:]
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal,
'X-Put-Timestamp': next(self.ts).internal,
'X-Delete-Timestamp': 0,
'X-Object-Count': 0,
'X-Bytes-Used': 0,
})
resp = req.get_response(self.controller)
self.assertEqual(resp.status, '400 Bad Request')
def test_PUT_non_utf8_metadata(self):
# Set metadata header
req = Request.blank(
@ -706,7 +768,7 @@ class TestAccountController(unittest.TestCase):
headers={'X-Timestamp': normalize_timestamp(1),
'X-Account-Meta-Test': 'Value'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
self.assertEqual(resp.status_int, 204, resp.body)
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
@ -935,7 +997,7 @@ class TestAccountController(unittest.TestCase):
def test_GET_over_limit(self):
req = Request.blank(
'/sda1/p/a?limit=%d' % (constraints.ACCOUNT_LISTING_LIMIT + 1),
'/sda1/p/a?limit=%d' % (ACCOUNT_LISTING_LIMIT + 1),
environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 412)
@ -1752,6 +1814,382 @@ class TestAccountController(unittest.TestCase):
for item in json.loads(resp.body)],
[{"name": "US~~UT~~~B"}])
def _expected_listing(self, containers):
return [dict(
last_modified=c['timestamp'].isoformat, **{
k: v for k, v in c.items()
if k != 'timestamp'
}) for c in sorted(containers, key=lambda c: c['name'])]
def _report_containers(self, containers, account='a'):
req = Request.blank('/sda1/p/%s' % account, method='PUT', headers={
'x-timestamp': next(self.ts).internal})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int // 100, 2, resp.body)
for container in containers:
path = '/sda1/p/%s/%s' % (account, container['name'])
req = Request.blank(path, method='PUT', headers={
'X-Put-Timestamp': container['timestamp'].internal,
'X-Delete-Timestamp': container.get(
'deleted', Timestamp(0)).internal,
'X-Object-Count': container['count'],
'X-Bytes-Used': container['bytes'],
})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int // 100, 2, resp.body)
def test_delimiter_with_reserved_and_no_public(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
}]
self._report_containers(containers)
req = Request.blank('/sda1/p/a', headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [])
req = Request.blank('/sda1/p/a', headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers))
req = Request.blank('/sda1/p/a?prefix=%s&delimiter=l' %
get_reserved_name('nul'), headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [])
req = Request.blank('/sda1/p/a?prefix=%s&delimiter=l' %
get_reserved_name('nul'), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [{
'subdir': '%s' % get_reserved_name('null')}])
def test_delimiter_with_reserved_and_public(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
}, {
'name': 'nullish',
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
}]
self._report_containers(containers)
req = Request.blank('/sda1/p/a?prefix=nul&delimiter=l', headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [{'subdir': 'null'}])
# allow-reserved header doesn't really make a difference
req = Request.blank('/sda1/p/a?prefix=nul&delimiter=l', headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [{'subdir': 'null'}])
req = Request.blank('/sda1/p/a?prefix=%s&delimiter=l' %
get_reserved_name('nul'), headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [])
req = Request.blank('/sda1/p/a?prefix=%s&delimiter=l' %
get_reserved_name('nul'), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [{
'subdir': '%s' % get_reserved_name('null')}])
req = Request.blank('/sda1/p/a?delimiter=%00', headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers)[1:])
req = Request.blank('/sda1/p/a?delimiter=%00', headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
[{'subdir': '\x00'}] +
self._expected_listing(containers)[1:])
def test_markers_with_reserved(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
}]
self._report_containers(containers)
req = Request.blank('/sda1/p/a?marker=%s' %
get_reserved_name('null', ''), headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [])
req = Request.blank('/sda1/p/a?marker=%s' %
get_reserved_name('null', ''), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers))
req = Request.blank('/sda1/p/a?marker=%s' % quote(
self._expected_listing(containers)[0]['name']), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers)[1:])
containers.append({
'name': get_reserved_name('null', 'test03'),
'bytes': 300,
'count': 30,
'timestamp': next(self.ts),
})
self._report_containers(containers)
req = Request.blank('/sda1/p/a?marker=%s' % quote(
self._expected_listing(containers)[0]['name']), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers)[1:])
req = Request.blank('/sda1/p/a?marker=%s' % quote(
self._expected_listing(containers)[1]['name']), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers)[-1:])
def test_prefix_with_reserved(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'foo'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('nullish'),
'bytes': 300,
'count': 32,
'timestamp': next(self.ts),
}]
self._report_containers(containers)
req = Request.blank('/sda1/p/a?prefix=%s' %
get_reserved_name('null', 'test'), headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [])
req = Request.blank('/sda1/p/a?prefix=%s' %
get_reserved_name('null', 'test'), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers[:2]))
def test_prefix_and_delim_with_reserved(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'foo'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('nullish'),
'bytes': 300,
'count': 32,
'timestamp': next(self.ts),
}]
self._report_containers(containers)
req = Request.blank('/sda1/p/a?prefix=%s&delimiter=%s' % (
get_reserved_name('null'), get_reserved_name()), headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [])
req = Request.blank('/sda1/p/a?prefix=%s&delimiter=%s' % (
get_reserved_name('null'), get_reserved_name()), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
expected = [{'subdir': get_reserved_name('null', '')}] + \
self._expected_listing(containers[-1:])
self.assertEqual(json.loads(resp.body), expected)
def test_reserved_markers_with_non_reserved(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
}, {
'name': 'nullish',
'bytes': 300,
'count': 32,
'timestamp': next(self.ts),
}]
self._report_containers(containers)
req = Request.blank('/sda1/p/a?marker=%s' %
get_reserved_name('null', ''), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers))
req = Request.blank('/sda1/p/a?marker=%s' %
get_reserved_name('null', ''), headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
[c for c in self._expected_listing(containers)
if get_reserved_name() not in c['name']])
req = Request.blank('/sda1/p/a?marker=%s' %
get_reserved_name('null', ''), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers))
req = Request.blank('/sda1/p/a?marker=%s' % quote(
self._expected_listing(containers)[0]['name']), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers)[1:])
def test_null_markers(self):
containers = [{
'name': get_reserved_name('null', ''),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
}, {
'name': 'null',
'bytes': 300,
'count': 32,
'timestamp': next(self.ts),
}]
self._report_containers(containers)
req = Request.blank('/sda1/p/a?marker=%s' % get_reserved_name('null'),
headers={'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers)[-1:])
req = Request.blank('/sda1/p/a?marker=%s' % get_reserved_name('null'),
headers={'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers))
req = Request.blank('/sda1/p/a?marker=%s' %
get_reserved_name('null', ''), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers)[1:])
req = Request.blank('/sda1/p/a?marker=%s' %
get_reserved_name('null', 'test00'), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers)[1:])
def test_through_call(self):
inbuf = BytesIO()
errbuf = StringIO()
@ -1814,7 +2252,7 @@ class TestAccountController(unittest.TestCase):
self.controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '\x00',
'PATH_INFO': '/sda1/p/a/c\xd8\x3e%20',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',

84
test/unit/account/test_utils.py
View File

@ -14,15 +14,18 @@
import itertools
import time
import unittest
import json
import mock
from swift.account import utils, backend
from swift.common.storage_policy import POLICIES
from swift.common.storage_policy import POLICIES, StoragePolicy
from swift.common.swob import Request
from swift.common.utils import Timestamp
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.request_helpers import get_reserved_name
from test.unit import patch_policies
from test.unit import patch_policies, make_timestamp_iter
class TestFakeAccountBroker(unittest.TestCase):
@ -57,6 +60,9 @@ class TestFakeAccountBroker(unittest.TestCase):
class TestAccountUtils(unittest.TestCase):
def setUp(self):
self.ts = make_timestamp_iter()
def test_get_response_headers_fake_broker(self):
broker = utils.FakeAccountBroker()
now = time.time()
@ -187,3 +193,77 @@ class TestAccountUtils(unittest.TestCase):
'value for %r was %r not %r' % (
key, value, expected_value))
self.assertFalse(expected)
def test_account_listing_response(self):
req = Request.blank('')
now = time.time()
with mock.patch('time.time', new=lambda: now):
resp = utils.account_listing_response('a', req, 'text/plain')
self.assertEqual(resp.status_int, 204)
expected = HeaderKeyDict({
'Content-Type': 'text/plain; charset=utf-8',
'X-Account-Container-Count': 0,
'X-Account-Object-Count': 0,
'X-Account-Bytes-Used': 0,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': Timestamp(now).normal,
})
self.assertEqual(expected, resp.headers)
self.assertEqual(b'', resp.body)
@patch_policies([StoragePolicy(0, 'zero', is_default=True)])
def test_account_listing_reserved_names(self):
broker = backend.AccountBroker(':memory:', account='a')
put_timestamp = next(self.ts)
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(put_timestamp.internal)
container_timestamp = next(self.ts)
broker.put_container(get_reserved_name('foo'),
container_timestamp.internal, 0, 10, 100, 0)
req = Request.blank('')
resp = utils.account_listing_response(
'a', req, 'application/json', broker)
self.assertEqual(resp.status_int, 200)
expected = HeaderKeyDict({
'Content-Type': 'application/json; charset=utf-8',
'Content-Length': 2,
'X-Account-Container-Count': 1,
'X-Account-Object-Count': 10,
'X-Account-Bytes-Used': 100,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': put_timestamp.normal,
'X-Account-Storage-Policy-Zero-Container-Count': 1,
'X-Account-Storage-Policy-Zero-Object-Count': 10,
'X-Account-Storage-Policy-Zero-Bytes-Used': 100,
})
self.assertEqual(expected, resp.headers)
self.assertEqual(b'[]', resp.body)
req = Request.blank('', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = utils.account_listing_response(
'a', req, 'application/json', broker)
self.assertEqual(resp.status_int, 200)
expected = HeaderKeyDict({
'Content-Type': 'application/json; charset=utf-8',
'Content-Length': 97,
'X-Account-Container-Count': 1,
'X-Account-Object-Count': 10,
'X-Account-Bytes-Used': 100,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': put_timestamp.normal,
'X-Account-Storage-Policy-Zero-Container-Count': 1,
'X-Account-Storage-Policy-Zero-Object-Count': 10,
'X-Account-Storage-Policy-Zero-Bytes-Used': 100,
})
self.assertEqual(expected, resp.headers)
expected = [{
"last_modified": container_timestamp.isoformat,
"count": 10,
"bytes": 100,
"name": get_reserved_name('foo'),
}]
self.assertEqual(sorted(json.dumps(expected).encode('ascii')),
sorted(resp.body))

33
test/unit/common/middleware/test_gatekeeper.py
View File

@ -245,6 +245,39 @@ class TestGatekeeper(unittest.TestCase):
self._test_location_header('/v/a/c/o2?query=path#test')
self._test_location_header('/v/a/c/o2;whatisparam?query=path#test')
def test_allow_reserved_names(self):
fake_app = FakeApp()
app = self.get_app(fake_app, {})
headers = {
'X-Allow-Reserved-Names': 'some-value'
}
req = Request.blank('/v/a/c/o', method='GET', headers=headers)
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertNotIn('X-Backend-Allow-Reserved-Names',
fake_app.req.headers)
self.assertIn('X-Allow-Reserved-Names',
fake_app.req.headers)
self.assertEqual(
'some-value',
fake_app.req.headers['X-Allow-Reserved-Names'])
app.allow_reserved_names_header = True
req = Request.blank('/v/a/c/o', method='GET', headers=headers)
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertIn('X-Backend-Allow-Reserved-Names',
fake_app.req.headers)
self.assertEqual(
'some-value',
fake_app.req.headers['X-Backend-Allow-Reserved-Names'])
self.assertEqual(
'some-value',
req.headers['X-Backend-Allow-Reserved-Names'])
self.assertNotIn('X-Allow-Reserved-Names', fake_app.req.headers)
self.assertNotIn('X-Allow-Reserved-Names', req.headers)
if __name__ == '__main__':
unittest.main()

264
test/unit/common/middleware/test_listing_formats.py
View File

@ -18,13 +18,17 @@ import unittest
from swift.common.swob import Request, HTTPOk
from swift.common.middleware import listing_formats
from swift.common.request_helpers import get_reserved_name
from test.unit import debug_logger
from test.unit.common.middleware.helpers import FakeSwift
class TestListingFormats(unittest.TestCase):
def setUp(self):
self.fake_swift = FakeSwift()
self.app = listing_formats.ListingFilter(self.fake_swift)
self.logger = debug_logger('test-listing')
self.app = listing_formats.ListingFilter(self.fake_swift, {},
logger=self.logger)
self.fake_account_listing = json.dumps([
{'name': 'bar', 'bytes': 0, 'count': 0,
'last_modified': '1970-01-01T00:00:00.000000'},
@ -37,6 +41,25 @@ class TestListingFormats(unittest.TestCase):
{'subdir': 'foo/'},
]).encode('ascii')
self.fake_account_listing_with_reserved = json.dumps([
{'name': 'bar', 'bytes': 0, 'count': 0,
'last_modified': '1970-01-01T00:00:00.000000'},
{'name': get_reserved_name('bar', 'versions'), 'bytes': 0,
'count': 0, 'last_modified': '1970-01-01T00:00:00.000000'},
{'subdir': 'foo_'},
{'subdir': get_reserved_name('foo_')},
]).encode('ascii')
self.fake_container_listing_with_reserved = json.dumps([
{'name': 'bar', 'hash': 'etag', 'bytes': 0,
'content_type': 'text/plain',
'last_modified': '1970-01-01T00:00:00.000000'},
{'name': get_reserved_name('bar', 'extra data'), 'hash': 'etag',
'bytes': 0, 'content_type': 'text/plain',
'last_modified': '1970-01-01T00:00:00.000000'},
{'subdir': 'foo/'},
{'subdir': get_reserved_name('foo/')},
]).encode('ascii')
def test_valid_account(self):
self.fake_swift.register('GET', '/v1/a', HTTPOk, {
'Content-Length': str(len(self.fake_account_listing)),
@ -60,7 +83,8 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a?format=json')
resp = req.get_response(self.app)
self.assertEqual(resp.body, self.fake_account_listing)
self.assertEqual(json.loads(resp.body),
json.loads(self.fake_account_listing))
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
@ -82,6 +106,119 @@ class TestListingFormats(unittest.TestCase):
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a?format=json'))
def test_valid_account_with_reserved(self):
body_len = len(self.fake_account_listing_with_reserved)
self.fake_swift.register(
'GET', '/v1/a\xe2\x98\x83', HTTPOk, {
'Content-Length': str(body_len),
'Content-Type': 'application/json',
}, self.fake_account_listing_with_reserved)
req = Request.blank('/v1/a\xe2\x98\x83')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo_\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a\xe2\x98\x83?format=json'))
self.assertEqual(self.logger.get_lines_for_level('warning'), [
"Account listing for a%E2%98%83 had reserved byte in name: "
"'\\x00bar\\x00versions'",
"Account listing for a%E2%98%83 had reserved byte in subdir: "
"'\\x00foo_'",
])
req = Request.blank('/v1/a\xe2\x98\x83', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\n' % (
get_reserved_name('bar', 'versions').encode('ascii'),
get_reserved_name('foo_').encode('ascii'),
))
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a\xe2\x98\x83?format=json'))
req = Request.blank('/v1/a\xe2\x98\x83?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo_\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a\xe2\x98\x83?format=json'))
req = Request.blank('/v1/a\xe2\x98\x83?format=txt', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\n' % (
get_reserved_name('bar', 'versions').encode('ascii'),
get_reserved_name('foo_').encode('ascii'),
))
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a\xe2\x98\x83?format=json'))
req = Request.blank('/v1/a\xe2\x98\x83?format=json')
resp = req.get_response(self.app)
self.assertEqual(json.loads(resp.body),
json.loads(self.fake_account_listing))
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a\xe2\x98\x83?format=json'))
req = Request.blank('/v1/a\xe2\x98\x83?format=json', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(json.loads(resp.body),
json.loads(self.fake_account_listing_with_reserved))
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a\xe2\x98\x83?format=json'))
req = Request.blank('/v1/a\xe2\x98\x83?format=xml')
resp = req.get_response(self.app)
self.assertEqual(resp.body.split(b'\n'), [
b'<?xml version="1.0" encoding="UTF-8"?>',
b'<account name="a\xe2\x98\x83">',
b'<container><name>bar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</container>',
b'<subdir name="foo_" />',
b'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a\xe2\x98\x83?format=json'))
req = Request.blank('/v1/a\xe2\x98\x83?format=xml', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(resp.body.split(b'\n'), [
b'<?xml version="1.0" encoding="UTF-8"?>',
b'<account name="a\xe2\x98\x83">',
b'<container><name>bar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</container>',
b'<container><name>%s</name>'
b'<count>0</count><bytes>0</bytes>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</container>' % get_reserved_name(
'bar', 'versions').encode('ascii'),
b'<subdir name="foo_" />',
b'<subdir name="%s" />' % get_reserved_name(
'foo_').encode('ascii'),
b'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a\xe2\x98\x83?format=json'))
def test_valid_container(self):
self.fake_swift.register('GET', '/v1/a/c', HTTPOk, {
'Content-Length': str(len(self.fake_container_listing)),
@ -105,7 +242,8 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a/c?format=json')
resp = req.get_response(self.app)
self.assertEqual(resp.body, self.fake_container_listing)
self.assertEqual(json.loads(resp.body),
json.loads(self.fake_container_listing))
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
@ -129,6 +267,126 @@ class TestListingFormats(unittest.TestCase):
self.assertEqual(self.fake_swift.calls[-1], (
'GET', '/v1/a/c?format=json'))
def test_valid_container_with_reserved(self):
path = '/v1/a\xe2\x98\x83/c\xf0\x9f\x8c\xb4'
body_len = len(self.fake_container_listing_with_reserved)
self.fake_swift.register(
'GET', path, HTTPOk, {
'Content-Length': str(body_len),
'Content-Type': 'application/json',
}, self.fake_container_listing_with_reserved)
req = Request.blank(path)
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo/\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', path + '?format=json'))
self.assertEqual(self.logger.get_lines_for_level('warning'), [
"Container listing for a%E2%98%83/c%F0%9F%8C%B4 had reserved byte "
"in name: '\\x00bar\\x00extra data'",
"Container listing for a%E2%98%83/c%F0%9F%8C%B4 had reserved byte "
"in subdir: '\\x00foo/'",
])
req = Request.blank(path, headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\n%s\nfoo/\n%s\n' % (
get_reserved_name('bar', 'extra data').encode('ascii'),
get_reserved_name('foo/').encode('ascii'),
))
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', path + '?format=json'))
req = Request.blank(path + '?format=txt')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo/\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', path + '?format=json'))
req = Request.blank(path + '?format=txt', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\n%s\nfoo/\n%s\n' % (
get_reserved_name('bar', 'extra data').encode('ascii'),
get_reserved_name('foo/').encode('ascii'),
))
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', path + '?format=json'))
req = Request.blank(path + '?format=json')
resp = req.get_response(self.app)
self.assertEqual(json.loads(resp.body),
json.loads(self.fake_container_listing))
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', path + '?format=json'))
req = Request.blank(path + '?format=json', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(json.loads(resp.body),
json.loads(self.fake_container_listing_with_reserved))
self.assertEqual(resp.headers['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', path + '?format=json'))
req = Request.blank(path + '?format=xml')
resp = req.get_response(self.app)
self.assertEqual(
resp.body,
b'<?xml version="1.0" encoding="UTF-8"?>\n'
b'<container name="c\xf0\x9f\x8c\xb4">'
b'<object><name>bar</name><hash>etag</hash><bytes>0</bytes>'
b'<content_type>text/plain</content_type>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</object>'
b'<subdir name="foo/"><name>foo/</name></subdir>'
b'</container>'
)
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', path + '?format=json'))
req = Request.blank(path + '?format=xml', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(
resp.body,
b'<?xml version="1.0" encoding="UTF-8"?>\n'
b'<container name="c\xf0\x9f\x8c\xb4">'
b'<object><name>bar</name><hash>etag</hash><bytes>0</bytes>'
b'<content_type>text/plain</content_type>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</object>'
b'<object><name>%s</name>'
b'<hash>etag</hash><bytes>0</bytes>'
b'<content_type>text/plain</content_type>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</object>'
b'<subdir name="foo/"><name>foo/</name></subdir>'
b'<subdir name="%s"><name>%s</name></subdir>'
b'</container>' % (
get_reserved_name('bar', 'extra data').encode('ascii'),
get_reserved_name('foo/').encode('ascii'),
get_reserved_name('foo/').encode('ascii'),
))
self.assertEqual(resp.headers['Content-Type'],
'application/xml; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
'GET', path + '?format=json'))
def test_blank_account(self):
self.fake_swift.register('GET', '/v1/a', HTTPOk, {
'Content-Length': '2', 'Content-Type': 'application/json'}, b'[]')

15
test/unit/common/test_constraints.py
View File

@ -506,7 +506,7 @@ class TestConstraints(unittest.TestCase):
def test_check_utf8(self):
unicode_sample = u'\uc77c\uc601'
unicode_with_null = u'abc\u0000def'
unicode_with_reserved = u'abc%sdef' % utils.RESERVED_STR
# Some false-y values
self.assertFalse(constraints.check_utf8(None))
@ -518,15 +518,24 @@ class TestConstraints(unittest.TestCase):
self.assertFalse(constraints.check_utf8(
unicode_sample.encode('utf-8')[::-1]))
# unicode with null
self.assertFalse(constraints.check_utf8(unicode_with_null))
self.assertFalse(constraints.check_utf8(unicode_with_reserved))
# utf8 bytes with null
self.assertFalse(constraints.check_utf8(
unicode_with_null.encode('utf8')))
unicode_with_reserved.encode('utf8')))
self.assertTrue(constraints.check_utf8('this is ascii and utf-8, too'))
self.assertTrue(constraints.check_utf8(unicode_sample))
self.assertTrue(constraints.check_utf8(unicode_sample.encode('utf8')))
def test_check_utf8_internal(self):
unicode_with_reserved = u'abc%sdef' % utils.RESERVED_STR
# sanity
self.assertFalse(constraints.check_utf8(unicode_with_reserved))
self.assertTrue(constraints.check_utf8('foobar', internal=True))
# internal allows reserved names
self.assertTrue(constraints.check_utf8(unicode_with_reserved,
internal=True))
def test_check_utf8_non_canonical(self):
self.assertFalse(constraints.check_utf8(b'\xed\xa0\xbc\xed\xbc\xb8'))
self.assertTrue(constraints.check_utf8(u'\U0001f338'))

100
test/unit/common/test_direct_client.py
View File

@ -132,66 +132,76 @@ class TestDirectClient(unittest.TestCase):
stub_user_agent = 'direct-client %s' % os.getpid()
headers = direct_client.gen_headers(add_ts=False)
self.assertEqual(headers['user-agent'], stub_user_agent)
self.assertEqual(1, len(headers))
self.assertEqual(dict(headers), {
'User-Agent': stub_user_agent,
'X-Backend-Allow-Reserved-Names': 'true',
})
now = time.time()
headers = direct_client.gen_headers()
self.assertEqual(headers['user-agent'], stub_user_agent)
self.assertTrue(now - 1 < Timestamp(headers['x-timestamp']) < now + 1)
self.assertEqual(headers['x-timestamp'],
Timestamp(headers['x-timestamp']).internal)
self.assertEqual(2, len(headers))
with mock.patch('swift.common.utils.Timestamp.now',
return_value=Timestamp('123.45')):
headers = direct_client.gen_headers()
self.assertEqual(dict(headers), {
'User-Agent': stub_user_agent,
'X-Backend-Allow-Reserved-Names': 'true',
'X-Timestamp': '0000000123.45000',
})
headers = direct_client.gen_headers(hdrs_in={'x-timestamp': '15'})
self.assertEqual(headers['x-timestamp'], '15')
self.assertEqual(headers['user-agent'], stub_user_agent)
self.assertEqual(2, len(headers))
self.assertEqual(dict(headers), {
'User-Agent': stub_user_agent,
'X-Backend-Allow-Reserved-Names': 'true',
'X-Timestamp': '15',
})
headers = direct_client.gen_headers(hdrs_in={'foo-bar': '63'})
self.assertEqual(headers['user-agent'], stub_user_agent)
self.assertEqual(headers['foo-bar'], '63')
self.assertTrue(now - 1 < Timestamp(headers['x-timestamp']) < now + 1)
self.assertEqual(headers['x-timestamp'],
Timestamp(headers['x-timestamp']).internal)
self.assertEqual(3, len(headers))
with mock.patch('swift.common.utils.Timestamp.now',
return_value=Timestamp('12345.6789')):
headers = direct_client.gen_headers(hdrs_in={'foo-bar': '63'})
self.assertEqual(dict(headers), {
'User-Agent': stub_user_agent,
'Foo-Bar': '63',
'X-Backend-Allow-Reserved-Names': 'true',
'X-Timestamp': '0000012345.67890',
})
hdrs_in = {'foo-bar': '55'}
headers = direct_client.gen_headers(hdrs_in, add_ts=False)
self.assertEqual(headers['user-agent'], stub_user_agent)
self.assertEqual(headers['foo-bar'], '55')
self.assertEqual(2, len(headers))
self.assertEqual(dict(headers), {
'User-Agent': stub_user_agent,
'Foo-Bar': '55',
'X-Backend-Allow-Reserved-Names': 'true',
})
headers = direct_client.gen_headers(hdrs_in={'user-agent': '32'})
self.assertEqual(headers['user-agent'], '32')
self.assertTrue(now - 1 < Timestamp(headers['x-timestamp']) < now + 1)
self.assertEqual(headers['x-timestamp'],
Timestamp(headers['x-timestamp']).internal)
self.assertEqual(2, len(headers))
with mock.patch('swift.common.utils.Timestamp.now',
return_value=Timestamp('12345')):
headers = direct_client.gen_headers(hdrs_in={'user-agent': '32'})
self.assertEqual(dict(headers), {
'User-Agent': '32',
'X-Backend-Allow-Reserved-Names': 'true',
'X-Timestamp': '0000012345.00000',
})
hdrs_in = {'user-agent': '47'}
headers = direct_client.gen_headers(hdrs_in, add_ts=False)
self.assertEqual(headers['user-agent'], '47')
self.assertEqual(1, len(headers))
self.assertEqual(dict(headers), {
'User-Agent': '47',
'X-Backend-Allow-Reserved-Names': 'true',
})
for policy in POLICIES:
for add_ts in (True, False):
now = time.time()
headers = direct_client.gen_headers(
{'X-Backend-Storage-Policy-Index': policy.idx},
add_ts=add_ts)
self.assertEqual(headers['user-agent'], stub_user_agent)
self.assertEqual(headers['X-Backend-Storage-Policy-Index'],
str(policy.idx))
expected_header_count = 2
with mock.patch('swift.common.utils.Timestamp.now',
return_value=Timestamp('123456789')):
headers = direct_client.gen_headers(
{'X-Backend-Storage-Policy-Index': policy.idx},
add_ts=add_ts)
expected = {
'User-Agent': stub_user_agent,
'X-Backend-Storage-Policy-Index': str(policy.idx),
'X-Backend-Allow-Reserved-Names': 'true',
}
if add_ts:
expected_header_count += 1
self.assertEqual(
headers['x-timestamp'],
Timestamp(headers['x-timestamp']).internal)
self.assertTrue(
now - 1 < Timestamp(headers['x-timestamp']) < now + 1)
self.assertEqual(expected_header_count, len(headers))
expected['X-Timestamp'] = '0123456789.00000'
self.assertEqual(dict(headers), expected)
def test_direct_get_account(self):
def do_test(req_params):

3
test/unit/common/test_internal_client.py
View File

@ -1233,7 +1233,8 @@ class TestInternalClient(unittest.TestCase):
self.assertEqual(app.call_count, 1)
req_headers.update({
'host': 'localhost:80', # from swob.Request.blank
'user-agent': 'test', # from InternalClient.make_request
'user-agent': 'test', # from InternalClient.make_request
'x-backend-allow-reserved-names': 'true', # also from IC
})
self.assertEqual(app.calls_with_headers, [(
'GET', path_info + '?symlink=get', HeaderKeyDict(req_headers))])

168
test/unit/common/test_request_helpers.py
View File

@ -181,6 +181,174 @@ class TestRequestHelpers(unittest.TestCase):
self.assertEqual(policy, POLICIES[1])
self.assertEqual(policy.policy_type, REPL_POLICY)
def test_validate_internal_name(self):
self.assertIsNone(rh._validate_internal_name('foo'))
self.assertIsNone(rh._validate_internal_name(
rh.get_reserved_name('foo')))
self.assertIsNone(rh._validate_internal_name(
rh.get_reserved_name('foo', 'bar')))
self.assertIsNone(rh._validate_internal_name(''))
self.assertIsNone(rh._validate_internal_name(rh.RESERVED))
def test_invalid_reserved_name(self):
with self.assertRaises(HTTPException) as raised:
rh._validate_internal_name('foo' + rh.RESERVED)
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace name")
def test_validate_internal_account(self):
self.assertIsNone(rh.validate_internal_account('AUTH_foo'))
self.assertIsNone(rh.validate_internal_account(
rh.get_reserved_name('AUTH_foo')))
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_account('AUTH_foo' + rh.RESERVED)
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace account")
def test_validate_internal_container(self):
self.assertIsNone(rh.validate_internal_container('AUTH_foo', 'bar'))
self.assertIsNone(rh.validate_internal_container(
rh.get_reserved_name('AUTH_foo'), 'bar'))
self.assertIsNone(rh.validate_internal_container(
'foo', rh.get_reserved_name('bar')))
self.assertIsNone(rh.validate_internal_container(
rh.get_reserved_name('AUTH_foo'), rh.get_reserved_name('bar')))
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_container('AUTH_foo' + rh.RESERVED, 'bar')
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace account")
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_container('AUTH_foo', 'bar' + rh.RESERVED)
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace container")
# These should always be operating on split_path outputs so this
# shouldn't really be an issue, but just in case...
for acct in ('', None):
with self.assertRaises(ValueError) as raised:
rh.validate_internal_container(
acct, 'bar')
self.assertEqual(raised.exception.args[0], 'Account is required')
def test_validate_internal_object(self):
self.assertIsNone(rh.validate_internal_obj('AUTH_foo', 'bar', 'baz'))
self.assertIsNone(rh.validate_internal_obj(
rh.get_reserved_name('AUTH_foo'), 'bar', 'baz'))
for acct in ('AUTH_foo', rh.get_reserved_name('AUTH_foo')):
self.assertIsNone(rh.validate_internal_obj(
acct,
rh.get_reserved_name('bar'),
rh.get_reserved_name('baz')))
for acct in ('AUTH_foo', rh.get_reserved_name('AUTH_foo')):
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_obj(
acct, 'bar', rh.get_reserved_name('baz'))
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace object "
b"in user-namespace container")
for acct in ('AUTH_foo', rh.get_reserved_name('AUTH_foo')):
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_obj(
acct, rh.get_reserved_name('bar'), 'baz')
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid user-namespace object "
b"in reserved-namespace container")
# These should always be operating on split_path outputs so this
# shouldn't really be an issue, but just in case...
for acct in ('', None):
with self.assertRaises(ValueError) as raised:
rh.validate_internal_obj(
acct, 'bar', 'baz')
self.assertEqual(raised.exception.args[0], 'Account is required')
for cont in ('', None):
with self.assertRaises(ValueError) as raised:
rh.validate_internal_obj(
'AUTH_foo', cont, 'baz')
self.assertEqual(raised.exception.args[0], 'Container is required')
def test_invalid_reserved_names(self):
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_obj('AUTH_foo' + rh.RESERVED, 'bar', 'baz')
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace account")
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_obj('AUTH_foo', 'bar' + rh.RESERVED, 'baz')
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace container")
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_obj('AUTH_foo', 'bar', 'baz' + rh.RESERVED)
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace object")
def test_get_reserved_name(self):
expectations = {
tuple(): rh.RESERVED,
('',): rh.RESERVED,
('foo',): rh.RESERVED + 'foo',
('foo', 'bar'): rh.RESERVED + 'foo' + rh.RESERVED + 'bar',
('foo', ''): rh.RESERVED + 'foo' + rh.RESERVED,
('', ''): rh.RESERVED * 2,
}
failures = []
for parts, expected in expectations.items():
name = rh.get_reserved_name(*parts)
if name != expected:
failures.append('get given %r expected %r != %r' % (
parts, expected, name))
if failures:
self.fail('Unexpected reults:\n' + '\n'.join(failures))
def test_invalid_get_reserved_name(self):
self.assertRaises(ValueError)
with self.assertRaises(ValueError) as ctx:
rh.get_reserved_name('foo', rh.RESERVED + 'bar', 'baz')
self.assertEqual(str(ctx.exception),
'Invalid reserved part in components')
def test_split_reserved_name(self):
expectations = {
rh.RESERVED: ('',),
rh.RESERVED + 'foo': ('foo',),
rh.RESERVED + 'foo' + rh.RESERVED + 'bar': ('foo', 'bar'),
rh.RESERVED + 'foo' + rh.RESERVED: ('foo', ''),
rh.RESERVED * 2: ('', ''),
}
failures = []
for name, expected in expectations.items():
parts = rh.split_reserved_name(name)
if tuple(parts) != expected:
failures.append('split given %r expected %r != %r' % (
name, expected, parts))
if failures:
self.fail('Unexpected reults:\n' + '\n'.join(failures))
def test_invalid_split_reserved_name(self):
self.assertRaises(ValueError)
with self.assertRaises(ValueError) as ctx:
rh.split_reserved_name('foo')
self.assertEqual(str(ctx.exception),
'Invalid reserved name')
class TestHTTPResponseToDocumentIters(unittest.TestCase):
def test_200(self):

13
test/unit/common/test_swob.py
View File

@ -1115,6 +1115,19 @@ class TestRequest(unittest.TestCase):
else:
self.fail("Expected an AttributeError raised for 'gzip,identity'")
def test_allow_reserved_names(self):
req = swift.common.swob.Request.blank('', headers={})
self.assertFalse(req.allow_reserved_names)
req = swift.common.swob.Request.blank('', headers={
'X-Allow-Reserved-Names': 'true'})
self.assertFalse(req.allow_reserved_names)
req = swift.common.swob.Request.blank('', headers={
'X-Backend-Allow-Reserved-Names': 'false'})
self.assertFalse(req.allow_reserved_names)
req = swift.common.swob.Request.blank('', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
self.assertTrue(req.allow_reserved_names)
class TestStatusMap(unittest.TestCase):
def test_status_map(self):

28
test/unit/container/test_backend.py
View File

@ -36,6 +36,7 @@ from swift.container.backend import ContainerBroker, \
update_new_item_from_existing, UNSHARDED, SHARDING, SHARDED, \
COLLAPSED, SHARD_LISTING_STATES, SHARD_UPDATE_STATES
from swift.common.db import DatabaseAlreadyExists, GreenDBConnection
from swift.common.request_helpers import get_reserved_name
from swift.common.utils import Timestamp, encode_timestamps, hash_path, \
ShardRange, make_db_file_path
from swift.common.storage_policy import POLICIES
@ -2365,6 +2366,33 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(len(listing), 2)
self.assertEqual([row[0] for row in listing], ['3/0000', '3/0001'])
def test_list_objects_iter_with_reserved_name(self):
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(next(self.ts).internal, 0)
broker.put_object(
'foo', next(self.ts).internal, 0, 0, 0, POLICIES.default.idx)
broker.put_object(
get_reserved_name('foo'), next(self.ts).internal, 0, 0, 0,
POLICIES.default.idx)
listing = broker.list_objects_iter(100, None, None, '', '')
self.assertEqual([row[0] for row in listing], ['foo'])
listing = broker.list_objects_iter(100, None, None, '', '',
reverse=True)
self.assertEqual([row[0] for row in listing], ['foo'])
listing = broker.list_objects_iter(100, None, None, '', '',
allow_reserved=True)
self.assertEqual([row[0] for row in listing],
[get_reserved_name('foo'), 'foo'])
listing = broker.list_objects_iter(100, None, None, '', '',
reverse=True, allow_reserved=True)
self.assertEqual([row[0] for row in listing],
['foo', get_reserved_name('foo')])
def test_reverse_prefix_delim(self):
expectations = [
{

314
test/unit/container/test_server.py
View File

@ -26,13 +26,13 @@ from contextlib import contextmanager
from io import BytesIO
from shutil import rmtree
from tempfile import mkdtemp
from test.unit import make_timestamp_iter, mock_timestamp_now
from xml.dom import minidom
from eventlet import spawn, Timeout
import json
import six
from six import StringIO
from six.moves.urllib.parse import quote
from swift import __version__ as swift_version
from swift.common.header_key_dict import HeaderKeyDict
@ -43,13 +43,13 @@ from swift.container import server as container_server
from swift.common import constraints
from swift.common.utils import (Timestamp, mkdirs, public, replication,
storage_directory, lock_parent_directory,
ShardRange)
ShardRange, RESERVED_STR)
from test.unit import fake_http_connect, debug_logger, mock_check_drive
from swift.common.storage_policy import (POLICIES, StoragePolicy)
from swift.common.request_helpers import get_sys_meta_prefix
from swift.common.request_helpers import get_sys_meta_prefix, get_reserved_name
from test import listen_zero, annotate_failure
from test.unit import patch_policies
from test.unit import patch_policies, make_timestamp_iter, mock_timestamp_now
@contextmanager
@ -78,6 +78,7 @@ class TestContainerController(unittest.TestCase):
logger=self.logger)
# some of the policy tests want at least two policies
self.assertTrue(len(POLICIES) > 1)
self.ts = make_timestamp_iter()
def tearDown(self):
rmtree(os.path.dirname(self.testdir), ignore_errors=1)
@ -282,11 +283,9 @@ class TestContainerController(unittest.TestCase):
self.assertIsNone(resp.headers[header])
def test_deleted_headers(self):
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
request_method_times = {
'PUT': next(ts),
'DELETE': next(ts),
'PUT': next(self.ts).internal,
'DELETE': next(self.ts).internal,
}
# setup a deleted container
for method in ('PUT', 'DELETE'):
@ -547,11 +546,10 @@ class TestContainerController(unittest.TestCase):
self.assertFalse('X-Backend-Storage-Policy-Index' in resp.headers)
def test_PUT_no_policy_change(self):
ts = (Timestamp(t).internal for t in itertools.count(time.time()))
policy = random.choice(list(POLICIES))
# Set metadata header
req = Request.blank('/sda1/p/a/c', method='PUT', headers={
'X-Timestamp': next(ts),
'X-Timestamp': next(self.ts).internal,
'X-Backend-Storage-Policy-Index': policy.idx})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
@ -565,7 +563,7 @@ class TestContainerController(unittest.TestCase):
# now try to update w/o changing the policy
for method in ('POST', 'PUT'):
req = Request.blank('/sda1/p/a/c', method=method, headers={
'X-Timestamp': next(ts),
'X-Timestamp': next(self.ts).internal,
'X-Backend-Storage-Policy-Index': policy.idx
})
resp = req.get_response(self.controller)
@ -578,11 +576,10 @@ class TestContainerController(unittest.TestCase):
str(policy.idx))
def test_PUT_bad_policy_change(self):
ts = (Timestamp(t).internal for t in itertools.count(time.time()))
policy = random.choice(list(POLICIES))
# Set metadata header
req = Request.blank('/sda1/p/a/c', method='PUT', headers={
'X-Timestamp': next(ts),
'X-Timestamp': next(self.ts).internal,
'X-Backend-Storage-Policy-Index': policy.idx})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
@ -597,7 +594,7 @@ class TestContainerController(unittest.TestCase):
for other_policy in other_policies:
# now try to change it and make sure we get a conflict
req = Request.blank('/sda1/p/a/c', method='PUT', headers={
'X-Timestamp': next(ts),
'X-Timestamp': next(self.ts).internal,
'X-Backend-Storage-Policy-Index': other_policy.idx
})
resp = req.get_response(self.controller)
@ -615,10 +612,9 @@ class TestContainerController(unittest.TestCase):
str(policy.idx))
def test_POST_ignores_policy_change(self):
ts = (Timestamp(t).internal for t in itertools.count(time.time()))
policy = random.choice(list(POLICIES))
req = Request.blank('/sda1/p/a/c', method='PUT', headers={
'X-Timestamp': next(ts),
'X-Timestamp': next(self.ts).internal,
'X-Backend-Storage-Policy-Index': policy.idx})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
@ -633,7 +629,7 @@ class TestContainerController(unittest.TestCase):
for other_policy in other_policies:
# now try to change it and make sure we get a conflict
req = Request.blank('/sda1/p/a/c', method='POST', headers={
'X-Timestamp': next(ts),
'X-Timestamp': next(self.ts).internal,
'X-Backend-Storage-Policy-Index': other_policy.idx
})
resp = req.get_response(self.controller)
@ -650,11 +646,9 @@ class TestContainerController(unittest.TestCase):
str(policy.idx))
def test_PUT_no_policy_for_existing_default(self):
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
# create a container with the default storage policy
req = Request.blank('/sda1/p/a/c', method='PUT', headers={
'X-Timestamp': next(ts),
'X-Timestamp': next(self.ts).internal,
})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201) # sanity check
@ -668,7 +662,7 @@ class TestContainerController(unittest.TestCase):
# put again without specifying the storage policy
req = Request.blank('/sda1/p/a/c', method='PUT', headers={
'X-Timestamp': next(ts),
'X-Timestamp': next(self.ts).internal,
})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 202) # sanity check
@ -685,11 +679,9 @@ class TestContainerController(unittest.TestCase):
# during a config change restart across a multi node cluster.
proxy_default = random.choice([p for p in POLICIES if not
p.is_default])
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
# create a container with the default storage policy
req = Request.blank('/sda1/p/a/c', method='PUT', headers={
'X-Timestamp': next(ts),
'X-Timestamp': next(self.ts).internal,
'X-Backend-Storage-Policy-Default': int(proxy_default),
})
resp = req.get_response(self.controller)
@ -704,7 +696,7 @@ class TestContainerController(unittest.TestCase):
# put again without proxy specifying the different default
req = Request.blank('/sda1/p/a/c', method='PUT', headers={
'X-Timestamp': next(ts),
'X-Timestamp': next(self.ts).internal,
'X-Backend-Storage-Policy-Default': int(POLICIES.default),
})
resp = req.get_response(self.controller)
@ -718,11 +710,10 @@ class TestContainerController(unittest.TestCase):
int(proxy_default))
def test_PUT_no_policy_for_existing_non_default(self):
ts = (Timestamp(t).internal for t in itertools.count(time.time()))
non_default_policy = [p for p in POLICIES if not p.is_default][0]
# create a container with the non-default storage policy
req = Request.blank('/sda1/p/a/c', method='PUT', headers={
'X-Timestamp': next(ts),
'X-Timestamp': next(self.ts).internal,
'X-Backend-Storage-Policy-Index': non_default_policy.idx,
})
resp = req.get_response(self.controller)
@ -737,7 +728,7 @@ class TestContainerController(unittest.TestCase):
# put again without specifying the storage policy
req = Request.blank('/sda1/p/a/c', method='PUT', headers={
'X-Timestamp': next(ts),
'X-Timestamp': next(self.ts).internal,
})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 202) # sanity check
@ -749,6 +740,39 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(resp.headers['X-Backend-Storage-Policy-Index'],
str(non_default_policy.idx))
def test_create_reserved_namespace_container(self):
path = '/sda1/p/a/%sc' % RESERVED_STR
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller)
self.assertEqual(resp.status, '201 Created', resp.body)
path = '/sda1/p/a/%sc%stest' % (RESERVED_STR, RESERVED_STR)
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller)
self.assertEqual(resp.status, '201 Created', resp.body)
def test_create_reserved_object_in_container(self):
# create container
path = '/sda1/p/a/c/'
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
# put null object in it
path += '%so' % RESERVED_STR
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal,
'X-Size': 0,
'X-Content-Type': 'application/x-test',
'X-Etag': 'x',
})
resp = req.get_response(self.controller)
self.assertEqual(resp.status, '400 Bad Request')
self.assertEqual(resp.body, b'Invalid reserved-namespace object '
b'in user-namespace container')
def test_PUT_non_utf8_metadata(self):
# Set metadata header
req = Request.blank(
@ -4106,6 +4130,238 @@ class TestContainerController(unittest.TestCase):
for item in json.loads(resp.body)],
[{"name": "US~~UT~~~B"}])
def _report_objects(self, path, objects):
req = Request.blank(path, method='PUT', headers={
'x-timestamp': next(self.ts).internal})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int // 100, 2, resp.body)
for obj in objects:
obj_path = path + '/%s' % obj['name']
req = Request.blank(obj_path, method='PUT', headers={
'X-Timestamp': obj['timestamp'].internal,
'X-Size': obj['bytes'],
'X-Content-Type': obj['content_type'],
'X-Etag': obj['hash'],
})
self._update_object_put_headers(req)
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int // 100, 2, resp.body)
def _expected_listing(self, objects):
return [dict(
last_modified=o['timestamp'].isoformat, **{
k: v for k, v in o.items()
if k != 'timestamp'
}) for o in sorted(objects, key=lambda o: o['name'])]
def test_listing_with_reserved(self):
objects = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 8,
'content_type': 'application/octet-stream',
'hash': '70c1db56f301c9e337b0099bd4174b28',
'timestamp': next(self.ts),
}]
path = '/sda1/p/a/%s' % get_reserved_name('null')
self._report_objects(path, objects)
req = Request.blank(path, headers={'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [])
req = Request.blank(path, headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(objects))
def test_delimiter_with_reserved(self):
objects = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 8,
'content_type': 'application/octet-stream',
'hash': '70c1db56f301c9e337b0099bd4174b28',
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 8,
'content_type': 'application/octet-stream',
'hash': '70c1db56f301c9e337b0099bd4174b28',
'timestamp': next(self.ts),
}]
path = '/sda1/p/a/%s' % get_reserved_name('null')
self._report_objects(path, objects)
req = Request.blank(path + '?prefix=%s&delimiter=l' %
get_reserved_name('nul'), headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [])
req = Request.blank(path + '?prefix=%s&delimiter=l' %
get_reserved_name('nul'), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [{
'subdir': '%s' % get_reserved_name('null')}])
req = Request.blank(path + '?prefix=%s&delimiter=%s' % (
get_reserved_name('nul'), get_reserved_name('')),
headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [{
'subdir': '%s' % get_reserved_name('null', '')}])
def test_markers_with_reserved(self):
objects = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 8,
'content_type': 'application/octet-stream',
'hash': '70c1db56f301c9e337b0099bd4174b28',
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'content_type': 'application/octet-stream',
'hash': '912ec803b2ce49e4a541068d495ab570',
'timestamp': next(self.ts),
}]
path = '/sda1/p/a/%s' % get_reserved_name('null')
self._report_objects(path, objects)
req = Request.blank(path + '?marker=%s' %
get_reserved_name('null', ''), headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [])
req = Request.blank(path + '?marker=%s' %
get_reserved_name('null', ''), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(objects))
req = Request.blank(path + '?marker=%s' %
quote(json.loads(resp.body)[0]['name']), headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [])
req = Request.blank(path + '?marker=%s' %
quote(self._expected_listing(objects)[0]['name']),
headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(objects)[1:])
def test_prefix_with_reserved(self):
objects = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 8,
'content_type': 'application/octet-stream',
'hash': '70c1db56f301c9e337b0099bd4174b28',
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'content_type': 'application/octet-stream',
'hash': '912ec803b2ce49e4a541068d495ab570',
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'foo'),
'bytes': 12,
'content_type': 'application/octet-stream',
'hash': 'acbd18db4cc2f85cedef654fccc4a4d8',
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('nullish'),
'bytes': 13,
'content_type': 'application/octet-stream',
'hash': '37b51d194a7513e45b56f6524f2d51f2',
'timestamp': next(self.ts),
}]
path = '/sda1/p/a/%s' % get_reserved_name('null')
self._report_objects(path, objects)
req = Request.blank(path + '?prefix=%s' %
get_reserved_name('null', 'test'), headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [])
req = Request.blank(path + '?prefix=%s' %
get_reserved_name('null', 'test'), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body),
self._expected_listing(objects[:2]))
def test_prefix_and_delim_with_reserved(self):
objects = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 8,
'content_type': 'application/octet-stream',
'hash': '70c1db56f301c9e337b0099bd4174b28',
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'content_type': 'application/octet-stream',
'hash': '912ec803b2ce49e4a541068d495ab570',
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('null', 'foo'),
'bytes': 12,
'content_type': 'application/octet-stream',
'hash': 'acbd18db4cc2f85cedef654fccc4a4d8',
'timestamp': next(self.ts),
}, {
'name': get_reserved_name('nullish'),
'bytes': 13,
'content_type': 'application/octet-stream',
'hash': '37b51d194a7513e45b56f6524f2d51f2',
'timestamp': next(self.ts),
}]
path = '/sda1/p/a/%s' % get_reserved_name('null')
self._report_objects(path, objects)
req = Request.blank(path + '?prefix=%s&delimiter=%s' % (
get_reserved_name('null'), get_reserved_name()), headers={
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
self.assertEqual(json.loads(resp.body), [])
req = Request.blank(path + '?prefix=%s&delimiter=%s' % (
get_reserved_name('null'), get_reserved_name()), headers={
'X-Backend-Allow-Reserved-Names': 'true',
'Accept': 'application/json'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200, resp.body)
expected = [{'subdir': get_reserved_name('null', '')}] + \
self._expected_listing(objects)[-1:]
self.assertEqual(json.loads(resp.body), expected)
def test_GET_delimiter_non_ascii(self):
req = Request.blank(
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
@ -4309,7 +4565,7 @@ class TestContainerController(unittest.TestCase):
self.controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '\x00',
'PATH_INFO': '/sda1/p/a/c\xd8\x3e%20/%',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',

3
test/unit/helpers.py
View File

@ -212,7 +212,8 @@ def setup_servers(the_object_server=object_server, extra_conf=None):
obj4srv, obj5srv, obj6srv)
nl = NullLogger()
logging_prosv = proxy_logging.ProxyLoggingMiddleware(
listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger)
listing_formats.ListingFilter(prosrv, {}, logger=prosrv.logger),
conf, logger=prosrv.logger)
prospa = spawn(wsgi.server, prolis, logging_prosv, nl,
protocol=SwiftHttpProtocol,
capitalize_response_headers=False)

57
test/unit/obj/test_server.py
View File

@ -55,6 +55,7 @@ from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \
NullLogger, storage_directory, public, replication, encode_timestamps, \
Timestamp
from swift.common import constraints
from swift.common.request_helpers import get_reserved_name
from swift.common.swob import Request, WsgiBytesIO
from swift.common.splice import splice
from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy,
@ -7147,6 +7148,60 @@ class TestObjectController(unittest.TestCase):
self.assertEqual(errbuf.getvalue(), '')
self.assertEqual(outbuf.getvalue()[:4], '405 ')
def test_create_reserved_namespace_object(self):
path = '/sda1/p/a/%sc/%so' % (utils.RESERVED_STR, utils.RESERVED_STR)
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal,
'Content-Type': 'application/x-test',
'Content-Length': 0,
})
resp = req.get_response(self.object_controller)
self.assertEqual(resp.status, '201 Created')
def test_create_reserved_namespace_object_in_user_container(self):
path = '/sda1/p/a/c/%so' % utils.RESERVED_STR
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal,
'Content-Type': 'application/x-test',
'Content-Length': 0,
})
resp = req.get_response(self.object_controller)
self.assertEqual(resp.status, '400 Bad Request', resp.body)
self.assertEqual(resp.body, b'Invalid reserved-namespace object in '
b'user-namespace container')
def test_other_methods_reserved_namespace_object(self):
container = get_reserved_name('c')
obj = get_reserved_name('o', 'v1')
path = '/sda1/p/a/%s/%s' % (container, obj)
req = Request.blank(path, method='PUT', headers={
'X-Timestamp': next(self.ts).internal,
'Content-Type': 'application/x-test',
'Content-Length': 0,
})
resp = req.get_response(self.object_controller)
self.assertEqual(resp.status, '201 Created')
bad_req = Request.blank('/sda1/p/a/c/%s' % obj, method='PUT', headers={
'X-Timestamp': next(self.ts).internal})
resp = bad_req.get_response(self.object_controller)
self.assertEqual(resp.status, '400 Bad Request')
self.assertEqual(resp.body, b'Invalid reserved-namespace object '
b'in user-namespace container')
for method in ('GET', 'POST', 'DELETE'):
req.method = method
req.headers['X-Timestamp'] = next(self.ts).internal
resp = req.get_response(self.object_controller)
self.assertEqual(resp.status_int // 100, 2)
bad_req.method = method
req.headers['X-Timestamp'] = next(self.ts).internal
resp = bad_req.get_response(self.object_controller)
self.assertEqual(resp.status, '400 Bad Request')
self.assertEqual(resp.body, b'Invalid reserved-namespace object '
b'in user-namespace container')
def test_not_utf8_and_not_logging_requests(self):
inbuf = WsgiBytesIO()
errbuf = StringIO()
@ -7164,7 +7219,7 @@ class TestObjectController(unittest.TestCase):
env = {'REQUEST_METHOD': method,
'SCRIPT_NAME': '',
'PATH_INFO': '/sda1/p/a/c/\x00%20/%',
'PATH_INFO': '/sda1/p/a/c/\xd8\x3e%20/%',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',

12
test/unit/proxy/controllers/test_obj.py
View File

@ -31,6 +31,7 @@ from eventlet import Timeout
import six
from six import StringIO
from six.moves import range
from six.moves.urllib.parse import quote
if six.PY2:
from email.parser import FeedParser as EmailFeedParser
else:
@ -1258,10 +1259,7 @@ class TestReplicatedObjController(CommonObjectControllerMixin,
log_lines = self.app.logger.get_lines_for_level('error')
self.assertFalse(log_lines[1:])
self.assertIn('ERROR with Object server', log_lines[0])
if six.PY3:
self.assertIn(req.swift_entity_path, log_lines[0])
else:
self.assertIn(req.swift_entity_path.decode('utf-8'), log_lines[0])
self.assertIn(quote(req.swift_entity_path), log_lines[0])
self.assertIn('re: Expect: 100-continue', log_lines[0])
def test_PUT_get_expect_errors_with_unicode_path(self):
@ -1305,11 +1303,7 @@ class TestReplicatedObjController(CommonObjectControllerMixin,
log_lines = self.app.logger.get_lines_for_level('error')
self.assertFalse(log_lines[1:])
self.assertIn('ERROR with Object server', log_lines[0])
if six.PY3:
self.assertIn(req.swift_entity_path, log_lines[0])
else:
self.assertIn(req.swift_entity_path.decode('utf-8'),
log_lines[0])
self.assertIn(quote(req.swift_entity_path), log_lines[0])
self.assertIn('Trying to write to', log_lines[0])
do_test(Exception('Exception while sending data on connection'))

31
test/unit/proxy/test_server.py
View File

@ -83,7 +83,7 @@ from swift.common.swob import Request, Response, HTTPUnauthorized, \
HTTPException, HTTPBadRequest, wsgi_to_str
from swift.common.storage_policy import StoragePolicy, POLICIES
import swift.common.request_helpers
from swift.common.request_helpers import get_sys_meta_prefix
from swift.common.request_helpers import get_sys_meta_prefix, get_reserved_name
# mocks
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
@ -698,6 +698,29 @@ class TestProxyServer(unittest.TestCase):
self.assertEqual(sorted(resp.headers['Allow'].split(', ')), [
'DELETE', 'GET', 'HEAD', 'OPTIONS', 'POST', 'PUT', 'UPDATE'])
def test_internal_reserved_name_request(self):
# set account info
fake_cache = FakeMemcache()
fake_cache.store[get_cache_key('a')] = {'status': 200}
app = proxy_server.Application({}, fake_cache,
container_ring=FakeRing(),
account_ring=FakeRing())
# build internal container request
container = get_reserved_name('c')
req = Request.blank('/v1/a/%s' % container)
app.update_request(req)
# try client request to reserved name
resp = app.handle_request(req)
self.assertEqual(resp.status_int, 412)
self.assertEqual(resp.body, b'Invalid UTF8 or contains NULL')
# set backend header
req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
with mocked_http_conn(200):
resp = app.handle_request(req)
self.assertEqual(resp.status_int, 200)
def test_calls_authorize_allow(self):
called = [False]
@ -10405,7 +10428,8 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase):
self.app = listing_formats.ListingFilter(
proxy_server.Application(conf, FakeMemcache(),
account_ring=FakeRing(),
container_ring=FakeRing()))
container_ring=FakeRing()),
{})
self.app.app.memcache = FakeMemcacheReturnsNone()
def test_GET_autocreate_accept_json(self):
@ -10823,7 +10847,8 @@ class TestSocketObjectVersions(unittest.TestCase):
_test_servers[0], conf,
logger=_test_servers[0].logger), {}),
{}
)
),
{}, logger=_test_servers[0].logger
)
self.coro = spawn(wsgi.server, prolis, prosrv, NullLogger(),
protocol=SwiftHttpProtocol)