diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 030d0d93d8..6e4d369f39 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -1008,6 +1008,11 @@ use = egg:swift#gatekeeper # difficult-to-delete data. # shunt_inbound_x_timestamp = true # +# Set this to true if you want to allow clients to access and manipulate the +# (normally internal-to-swift) null namespace by including a header like +# X-Allow-Reserved-Names: true +# allow_reserved_names_header = false +# # You can override the default log routing for this filter here: # set log_name = gatekeeper # set log_facility = LOG_LOCAL0 diff --git a/swift/account/backend.py b/swift/account/backend.py index 7e65fa054e..cb1eeb5863 100644 --- a/swift/account/backend.py +++ b/swift/account/backend.py @@ -22,7 +22,7 @@ import sqlite3 import six -from swift.common.utils import Timestamp +from swift.common.utils import Timestamp, RESERVED_BYTE from swift.common.db import DatabaseBroker, utf8encode, zero_like DATADIR = 'accounts' @@ -355,7 +355,7 @@ class AccountBroker(DatabaseBroker): ''').fetchone()) def list_containers_iter(self, limit, marker, end_marker, prefix, - delimiter, reverse=False): + delimiter, reverse=False, allow_reserved=False): """ Get a list of containers sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not have @@ -367,6 +367,7 @@ class AccountBroker(DatabaseBroker): :param prefix: prefix query :param delimiter: delimiter for query :param reverse: reverse the result order. + :param allow_reserved: exclude names with reserved-byte by default :returns: list of tuples of (name, object_count, bytes_used, put_timestamp, 0) @@ -410,6 +411,9 @@ class AccountBroker(DatabaseBroker): elif prefix: query += ' name >= ? AND' query_args.append(prefix) + if not allow_reserved: + query += ' name >= ? AND' + query_args.append(chr(ord(RESERVED_BYTE) + 1)) if self.get_db_version(conn) < 1: query += ' +deleted = 0' else: @@ -441,7 +445,7 @@ class AccountBroker(DatabaseBroker): curs.close() return results end = name.find(delimiter, len(prefix)) - if end > 0: + if end >= 0: if reverse: end_marker = name[:end + len(delimiter)] else: diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 885b0099cf..fc47d776aa 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -262,7 +262,7 @@ class AccountReaper(Daemon): container_limit *= len(nodes) try: containers = list(broker.list_containers_iter( - container_limit, '', None, None, None)) + container_limit, '', None, None, None, allow_reserved=True)) while containers: try: for (container, _junk, _junk, _junk, _junk) in containers: @@ -282,7 +282,8 @@ class AccountReaper(Daemon): self.logger.exception( 'Exception with containers for account %s', account) containers = list(broker.list_containers_iter( - container_limit, containers[-1][0], None, None, None)) + container_limit, containers[-1][0], None, None, None, + allow_reserved=True)) log_buf = ['Completed pass on account %s' % account] except (Exception, Timeout): self.logger.exception('Exception with account %s', account) diff --git a/swift/account/server.py b/swift/account/server.py index bda89f82e9..2bfbd93166 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -26,7 +26,8 @@ from swift.account.backend import AccountBroker, DATADIR from swift.account.utils import account_listing_response, get_response_headers from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists from swift.common.request_helpers import get_param, \ - split_and_validate_path + split_and_validate_path, validate_internal_account, \ + validate_internal_container from swift.common.utils import get_logger, hash_path, public, \ Timestamp, storage_directory, config_true_value, \ timing_stats, replication, get_log_line, \ @@ -44,6 +45,32 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, \ from swift.common.request_helpers import is_sys_or_user_meta +def get_account_name_and_placement(req): + """ + Split and validate path for an account. + + :param req: a swob request + + :returns: a tuple of path parts as strings + """ + drive, part, account = split_and_validate_path(req, 3) + validate_internal_account(account) + return drive, part, account + + +def get_container_name_and_placement(req): + """ + Split and validate path for a container. + + :param req: a swob request + + :returns: a tuple of path parts as strings + """ + drive, part, account, container = split_and_validate_path(req, 3, 4) + validate_internal_container(account, container) + return drive, part, account, container + + class AccountController(BaseStorageServer): """WSGI controller for the account server.""" @@ -96,7 +123,7 @@ class AccountController(BaseStorageServer): @timing_stats() def DELETE(self, req): """Handle HTTP DELETE request.""" - drive, part, account = split_and_validate_path(req, 3) + drive, part, account = get_account_name_and_placement(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: @@ -120,7 +147,7 @@ class AccountController(BaseStorageServer): @timing_stats() def PUT(self, req): """Handle HTTP PUT request.""" - drive, part, account, container = split_and_validate_path(req, 3, 4) + drive, part, account, container = get_container_name_and_placement(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: @@ -185,7 +212,7 @@ class AccountController(BaseStorageServer): @timing_stats() def HEAD(self, req): """Handle HTTP HEAD request.""" - drive, part, account = split_and_validate_path(req, 3) + drive, part, account = get_account_name_and_placement(req) out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) @@ -204,7 +231,7 @@ class AccountController(BaseStorageServer): @timing_stats() def GET(self, req): """Handle HTTP GET request.""" - drive, part, account = split_and_validate_path(req, 3) + drive, part, account = get_account_name_and_placement(req) prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') limit = constraints.ACCOUNT_LISTING_LIMIT @@ -262,7 +289,7 @@ class AccountController(BaseStorageServer): @timing_stats() def POST(self, req): """Handle HTTP POST request.""" - drive, part, account = split_and_validate_path(req, 3) + drive, part, account = get_account_name_and_placement(req) req_timestamp = valid_timestamp(req) try: check_drive(self.root, drive, self.mount_check) @@ -280,8 +307,8 @@ class AccountController(BaseStorageServer): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) - if not check_utf8(wsgi_to_str(req.path_info)): - res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') + if not check_utf8(wsgi_to_str(req.path_info), internal=True): + res = HTTPPreconditionFailed(body='Invalid UTF8') else: try: # disallow methods which are not publicly accessible diff --git a/swift/account/utils.py b/swift/account/utils.py index 4e5ebe1add..89fb26f841 100644 --- a/swift/account/utils.py +++ b/swift/account/utils.py @@ -17,6 +17,7 @@ import json import six +from swift.common import constraints from swift.common.middleware import listing_formats from swift.common.swob import HTTPOk, HTTPNoContent, str_to_wsgi from swift.common.utils import Timestamp @@ -71,15 +72,17 @@ def get_response_headers(broker): def account_listing_response(account, req, response_content_type, broker=None, - limit='', marker='', end_marker='', prefix='', - delimiter='', reverse=False): + limit=constraints.ACCOUNT_LISTING_LIMIT, + marker='', end_marker='', prefix='', delimiter='', + reverse=False): if broker is None: broker = FakeAccountBroker() resp_headers = get_response_headers(broker) account_list = broker.list_containers_iter(limit, marker, end_marker, - prefix, delimiter, reverse) + prefix, delimiter, reverse, + req.allow_reserved_names) data = [] for (name, object_count, bytes_used, put_timestamp, is_subdir) \ in account_list: diff --git a/swift/common/constraints.py b/swift/common/constraints.py index c7263f7b90..dd2e75a8f0 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -346,12 +346,13 @@ def check_delete_headers(request): return request -def check_utf8(string): +def check_utf8(string, internal=False): """ Validate if a string is valid UTF-8 str or unicode and that it - does not contain any null character. + does not contain any reserved characters. :param string: string to be validated + :param internal: boolean, allows reserved characters if True :returns: True if the string is valid utf-8 str or unicode and contains no null characters, False otherwise """ @@ -382,7 +383,9 @@ def check_utf8(string): if any(0xD800 <= ord(codepoint) <= 0xDFFF for codepoint in decoded): return False - return b'\x00' not in encoded + if b'\x00' != utils.RESERVED_BYTE and b'\x00' in encoded: + return False + return True if internal else utils.RESERVED_BYTE not in encoded # If string is unicode, decode() will raise UnicodeEncodeError # So, we should catch both UnicodeDecodeError & UnicodeEncodeError except UnicodeError: @@ -413,6 +416,7 @@ def check_name_format(req, name, target_type): body='%s name cannot contain slashes' % target_type) return name + check_account_format = functools.partial(check_name_format, target_type='Account') check_container_format = functools.partial(check_name_format, diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py index 174b61601f..3c9f140e67 100644 --- a/swift/common/direct_client.py +++ b/swift/common/direct_client.py @@ -100,6 +100,7 @@ def _make_req(node, part, method, path, headers, stype, if content_length is None: headers['Transfer-Encoding'] = 'chunked' + headers.setdefault('X-Backend-Allow-Reserved-Names', 'true') with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, method, path, headers=headers) @@ -193,6 +194,7 @@ def gen_headers(hdrs_in=None, add_ts=True): hdrs_out['X-Timestamp'] = Timestamp.now().internal if 'user-agent' not in hdrs_out: hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid() + hdrs_out.setdefault('X-Backend-Allow-Reserved-Names', 'true') return hdrs_out diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py index 0e2c4ab449..851282bd6f 100644 --- a/swift/common/internal_client.py +++ b/swift/common/internal_client.py @@ -184,6 +184,7 @@ class InternalClient(object): headers = dict(headers) headers['user-agent'] = self.user_agent + headers.setdefault('x-backend-allow-reserved-names', 'true') for attempt in range(self.request_tries): resp = exc_type = exc_value = exc_traceback = None req = Request.blank( @@ -384,6 +385,19 @@ class InternalClient(object): return self._iter_items(path, marker, end_marker, prefix, acceptable_statuses) + def create_account(self, account): + """ + Creates an account. + + :param account: Account to create. + :raises UnexpectedResponse: Exception raised when requests fail + to get a response with an acceptable status + :raises Exception: Exception is raised when code fails in an + unexpected way. + """ + path = self.make_path(account) + self.make_request('PUT', path, {}, (201, 202)) + def delete_account(self, account, acceptable_statuses=(2, HTTP_NOT_FOUND)): """ Deletes an account. @@ -514,7 +528,8 @@ class InternalClient(object): self.make_request('PUT', path, headers, acceptable_statuses) def delete_container( - self, account, container, acceptable_statuses=(2, HTTP_NOT_FOUND)): + self, account, container, headers=None, + acceptable_statuses=(2, HTTP_NOT_FOUND)): """ Deletes a container. @@ -529,8 +544,9 @@ class InternalClient(object): unexpected way. """ + headers = headers or {} path = self.make_path(account, container) - self.make_request('DELETE', path, {}, acceptable_statuses) + self.make_request('DELETE', path, headers, acceptable_statuses) def get_container_metadata( self, account, container, metadata_prefix='', @@ -669,7 +685,7 @@ class InternalClient(object): return self._get_metadata(path, metadata_prefix, acceptable_statuses, headers=headers, params=params) - def get_object(self, account, container, obj, headers, + def get_object(self, account, container, obj, headers=None, acceptable_statuses=(2,), params=None): """ Gets an object. @@ -771,7 +787,8 @@ class InternalClient(object): path, metadata, metadata_prefix, acceptable_statuses) def upload_object( - self, fobj, account, container, obj, headers=None): + self, fobj, account, container, obj, headers=None, + acceptable_statuses=(2,)): """ :param fobj: File object to read object's content from. :param account: The object's account. @@ -789,7 +806,7 @@ class InternalClient(object): if 'Content-Length' not in headers: headers['Transfer-Encoding'] = 'chunked' path = self.make_path(account, container, obj) - self.make_request('PUT', path, headers, (2,), fobj) + self.make_request('PUT', path, headers, acceptable_statuses, fobj) def get_auth(url, user, key, auth_version='1.0', **kwargs): diff --git a/swift/common/middleware/gatekeeper.py b/swift/common/middleware/gatekeeper.py index 511394691a..0254fef98b 100644 --- a/swift/common/middleware/gatekeeper.py +++ b/swift/common/middleware/gatekeeper.py @@ -75,6 +75,8 @@ class GatekeeperMiddleware(object): self.outbound_condition = make_exclusion_test(outbound_exclusions) self.shunt_x_timestamp = config_true_value( conf.get('shunt_inbound_x_timestamp', 'true')) + self.allow_reserved_names_header = config_true_value( + conf.get('allow_reserved_names_header', 'false')) def __call__(self, env, start_response): req = Request(env) @@ -89,6 +91,11 @@ class GatekeeperMiddleware(object): self.logger.debug('shunted request headers: %s' % [('X-Timestamp', ts)]) + if 'X-Allow-Reserved-Names' in req.headers \ + and self.allow_reserved_names_header: + req.headers['X-Backend-Allow-Reserved-Names'] = \ + req.headers.pop('X-Allow-Reserved-Names') + def gatekeeper_response(status, response_headers, exc_info=None): def fixed_response_headers(): def relative_path(value): diff --git a/swift/common/middleware/listing_formats.py b/swift/common/middleware/listing_formats.py index 1924facd8f..8c07965af6 100644 --- a/swift/common/middleware/listing_formats.py +++ b/swift/common/middleware/listing_formats.py @@ -21,7 +21,8 @@ from swift.common.constraints import valid_api_version from swift.common.http import HTTP_NO_CONTENT from swift.common.request_helpers import get_param from swift.common.swob import HTTPException, HTTPNotAcceptable, Request, \ - RESPONSE_REASONS, HTTPBadRequest + RESPONSE_REASONS, HTTPBadRequest, wsgi_quote, wsgi_to_bytes +from swift.common.utils import RESERVED, get_logger #: Mapping of query string ``format=`` values to their corresponding @@ -73,8 +74,6 @@ def to_xml(document_element): def account_to_xml(listing, account_name): - if isinstance(account_name, bytes): - account_name = account_name.decode('utf-8') doc = Element('account', name=account_name) doc.text = '\n' for record in listing: @@ -91,8 +90,6 @@ def account_to_xml(listing, account_name): def container_to_xml(listing, base_name): - if isinstance(base_name, bytes): - base_name = base_name.decode('utf-8') doc = Element('container', name=base_name) for record in listing: if 'subdir' in record: @@ -119,8 +116,33 @@ def listing_to_text(listing): class ListingFilter(object): - def __init__(self, app): + def __init__(self, app, conf, logger=None): self.app = app + self.logger = logger or get_logger(conf, log_route='listing-filter') + + def filter_reserved(self, listing, account, container): + new_listing = [] + for entry in list(listing): + for key in ('name', 'subdir'): + value = entry.get(key, '') + if six.PY2: + value = value.encode('utf-8') + if RESERVED in value: + if container: + self.logger.warning( + 'Container listing for %s/%s had ' + 'reserved byte in %s: %r', + wsgi_quote(account), wsgi_quote(container), + key, value) + else: + self.logger.warning( + 'Account listing for %s had ' + 'reserved byte in %s: %r', + wsgi_quote(account), key, value) + break # out of the *key* loop; check next entry + else: + new_listing.append(entry) + return new_listing def __call__(self, env, start_response): req = Request(env) @@ -128,10 +150,10 @@ class ListingFilter(object): # account and container only version, acct, cont = req.split_path(2, 3) except ValueError: - is_container_req = False + is_account_or_container_req = False else: - is_container_req = True - if not is_container_req: + is_account_or_container_req = True + if not is_account_or_container_req: return self.app(env, start_response) if not valid_api_version(version) or req.method not in ('GET', 'HEAD'): @@ -201,15 +223,21 @@ class ListingFilter(object): start_response(status, headers) return [body] + if not req.allow_reserved_names: + listing = self.filter_reserved(listing, acct, cont) + try: if out_content_type.endswith('/xml'): if cont: - body = container_to_xml(listing, cont) + body = container_to_xml( + listing, wsgi_to_bytes(cont).decode('utf-8')) else: - body = account_to_xml(listing, acct) + body = account_to_xml( + listing, wsgi_to_bytes(acct).decode('utf-8')) elif out_content_type == 'text/plain': body = listing_to_text(listing) - # else, json -- we continue down here to be sure we set charset + else: + body = json.dumps(listing).encode('ascii') except KeyError: # listing was in a bad format -- funky static web listing?? start_response(status, headers) @@ -226,4 +254,9 @@ class ListingFilter(object): def filter_factory(global_conf, **local_conf): - return ListingFilter + conf = global_conf.copy() + conf.update(local_conf) + + def listing_filter(app): + return ListingFilter(app, conf) + return listing_filter diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index 03d571f1f8..bdc234c9ea 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -38,8 +38,8 @@ from swift.common.swob import HTTPBadRequest, \ from swift.common.utils import split_path, validate_device_partition, \ close_if_possible, maybe_multipart_byteranges_to_document_iters, \ multipart_byteranges_to_document_iters, parse_content_type, \ - parse_content_range, csv_append, list_from_csv, Spliterator, quote - + parse_content_range, csv_append, list_from_csv, Spliterator, quote, \ + RESERVED from swift.common.wsgi import make_subrequest @@ -83,6 +83,54 @@ def get_param(req, name, default=None): return value +def _validate_internal_name(name, type_='name'): + if RESERVED in name and not name.startswith(RESERVED): + raise HTTPBadRequest(body='Invalid reserved-namespace %s' % (type_)) + + +def validate_internal_account(account): + """ + Validate internal account name. + + :raises: HTTPBadRequest + """ + _validate_internal_name(account, 'account') + + +def validate_internal_container(account, container): + """ + Validate internal account and container names. + + :raises: HTTPBadRequest + """ + if not account: + raise ValueError('Account is required') + validate_internal_account(account) + if container: + _validate_internal_name(container, 'container') + + +def validate_internal_obj(account, container, obj): + """ + Validate internal account, container and object names. + + :raises: HTTPBadRequest + """ + if not account: + raise ValueError('Account is required') + if not container: + raise ValueError('Container is required') + validate_internal_container(account, container) + if obj: + _validate_internal_name(obj, 'object') + if container.startswith(RESERVED) and not obj.startswith(RESERVED): + raise HTTPBadRequest(body='Invalid user-namespace object ' + 'in reserved-namespace container') + elif obj.startswith(RESERVED) and not container.startswith(RESERVED): + raise HTTPBadRequest(body='Invalid reserved-namespace object ' + 'in user-namespace container') + + def get_name_and_placement(request, minsegs=1, maxsegs=None, rest_with_last=False): """ @@ -273,6 +321,28 @@ def get_container_update_override_key(key): return header.title() +def get_reserved_name(*parts): + """ + Generate a valid reserved name that joins the component parts. + + :returns: a string + """ + if any(RESERVED in p for p in parts): + raise ValueError('Invalid reserved part in components') + return RESERVED + RESERVED.join(parts) + + +def split_reserved_name(name): + """ + Seperate a valid reserved name into the component parts. + + :returns: a list of strings + """ + if not name.startswith(RESERVED): + raise ValueError('Invalid reserved name') + return name.split(RESERVED)[1:] + + def remove_items(headers, condition): """ Removes items from a dict whose keys satisfy diff --git a/swift/common/swob.py b/swift/common/swob.py index e864a39a5a..c5bb978389 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -52,7 +52,7 @@ from six.moves import urllib from swift.common.header_key_dict import HeaderKeyDict from swift.common.utils import UTC, reiterate, split_path, Timestamp, pairs, \ - close_if_possible, closing_if_possible + close_if_possible, closing_if_possible, config_true_value from swift.common.exceptions import InvalidTimestamp @@ -1063,6 +1063,11 @@ class Request(object): "Provides the full url of the request" return self.host_url + self.path_qs + @property + def allow_reserved_names(self): + return config_true_value(self.environ.get( + 'HTTP_X_BACKEND_ALLOW_RESERVED_NAMES')) + def as_referer(self): return self.method + ' ' + self.url diff --git a/swift/common/utils.py b/swift/common/utils.py index e4bb964bcd..68005f324b 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -186,6 +186,10 @@ O_TMPFILE = getattr(os, 'O_TMPFILE', 0o20000000 | os.O_DIRECTORY) IPV6_RE = re.compile("^\[(?P
.*)\](:(?P[0-9]+))?$") MD5_OF_EMPTY_STRING = 'd41d8cd98f00b204e9800998ecf8427e' +RESERVED_BYTE = b'\x00' +RESERVED_STR = u'\x00' +RESERVED = '\x00' + LOG_LINE_DEFAULT_FORMAT = '{remote_addr} - - [{time.d}/{time.b}/{time.Y}' \ ':{time.H}:{time.M}:{time.S} +0000] ' \ diff --git a/swift/container/backend.py b/swift/container/backend.py index 0915901a05..8dee9d243c 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -30,7 +30,8 @@ from swift.common.exceptions import LockTimeout from swift.common.utils import Timestamp, encode_timestamps, \ decode_timestamps, extract_swift_bytes, storage_directory, hash_path, \ ShardRange, renamer, find_shard_range, MD5_OF_EMPTY_STRING, mkdirs, \ - get_db_files, parse_db_filename, make_db_file_path, split_path + get_db_files, parse_db_filename, make_db_file_path, split_path, \ + RESERVED_BYTE from swift.common.db import DatabaseBroker, utf8encode, BROKER_TIMEOUT, \ zero_like, DatabaseAlreadyExists @@ -1028,7 +1029,8 @@ class ContainerBroker(DatabaseBroker): def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path=None, storage_policy_index=0, reverse=False, include_deleted=False, since_row=None, - transform_func=None, all_policies=False): + transform_func=None, all_policies=False, + allow_reserved=False): """ Get a list of objects sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not @@ -1054,6 +1056,8 @@ class ContainerBroker(DatabaseBroker): :meth:`~_transform_record`; defaults to :meth:`~_transform_record`. :param all_policies: if True, include objects for all storage policies ignoring any value given for ``storage_policy_index`` + :param allow_reserved: exclude names with reserved-byte by default + :returns: list of tuples of (name, created_at, size, content_type, etag, deleted) """ @@ -1110,6 +1114,9 @@ class ContainerBroker(DatabaseBroker): elif prefix: query_conditions.append('name >= ?') query_args.append(prefix) + if not allow_reserved: + query_conditions.append('name >= ?') + query_args.append(chr(ord(RESERVED_BYTE) + 1)) query_conditions.append(deleted_key + deleted_arg) if since_row: query_conditions.append('ROWID > ?') diff --git a/swift/container/server.py b/swift/container/server.py index 0e8a443442..5d0915aa07 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -32,7 +32,8 @@ from swift.container.replicator import ContainerReplicatorRpc from swift.common.db import DatabaseAlreadyExists from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.request_helpers import get_param, \ - split_and_validate_path, is_sys_or_user_meta + split_and_validate_path, is_sys_or_user_meta, \ + validate_internal_container, validate_internal_obj from swift.common.utils import get_logger, hash_path, public, \ Timestamp, storage_directory, validate_sync_to, \ config_true_value, timing_stats, replication, \ @@ -83,6 +84,33 @@ def gen_resp_headers(info, is_deleted=False): return headers +def get_container_name_and_placement(req): + """ + Split and validate path for a container. + + :param req: a swob request + + :returns: a tuple of path parts as strings + """ + drive, part, account, container = split_and_validate_path(req, 4) + validate_internal_container(account, container) + return drive, part, account, container + + +def get_obj_name_and_placement(req): + """ + Split and validate path for an object. + + :param req: a swob request + + :returns: a tuple of path parts as strings + """ + drive, part, account, container, obj = split_and_validate_path( + req, 4, 5, True) + validate_internal_obj(account, container, obj) + return drive, part, account, container, obj + + class ContainerController(BaseStorageServer): """WSGI Controller for the container server.""" @@ -311,8 +339,7 @@ class ContainerController(BaseStorageServer): @timing_stats() def DELETE(self, req): """Handle HTTP DELETE request.""" - drive, part, account, container, obj = split_and_validate_path( - req, 4, 5, True) + drive, part, account, container, obj = get_obj_name_and_placement(req) req_timestamp = valid_timestamp(req) try: check_drive(self.root, drive, self.mount_check) @@ -433,8 +460,7 @@ class ContainerController(BaseStorageServer): @timing_stats() def PUT(self, req): """Handle HTTP PUT request.""" - drive, part, account, container, obj = split_and_validate_path( - req, 4, 5, True) + drive, part, account, container, obj = get_obj_name_and_placement(req) req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( @@ -514,8 +540,7 @@ class ContainerController(BaseStorageServer): @timing_stats(sample_rate=0.1) def HEAD(self, req): """Handle HTTP HEAD request.""" - drive, part, account, container, obj = split_and_validate_path( - req, 4, 5, True) + drive, part, account, container, obj = get_obj_name_and_placement(req) out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) @@ -632,8 +657,7 @@ class ContainerController(BaseStorageServer): :param req: an instance of :class:`swift.common.swob.Request` :returns: an instance of :class:`swift.common.swob.Response` """ - drive, part, account, container, obj = split_and_validate_path( - req, 4, 5, True) + drive, part, account, container, obj = get_obj_name_and_placement(req) path = get_param(req, 'path') prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') @@ -696,7 +720,7 @@ class ContainerController(BaseStorageServer): container_list = src_broker.list_objects_iter( limit, marker, end_marker, prefix, delimiter, path, storage_policy_index=info['storage_policy_index'], - reverse=reverse) + reverse=reverse, allow_reserved=req.allow_reserved_names) return self.create_listing(req, out_content_type, info, resp_headers, broker.metadata, container_list, container) @@ -751,7 +775,7 @@ class ContainerController(BaseStorageServer): """ Handle HTTP UPDATE request (merge_items RPCs coming from the proxy.) """ - drive, part, account, container = split_and_validate_path(req, 4) + drive, part, account, container = get_container_name_and_placement(req) req_timestamp = valid_timestamp(req) try: check_drive(self.root, drive, self.mount_check) @@ -775,7 +799,7 @@ class ContainerController(BaseStorageServer): @timing_stats() def POST(self, req): """Handle HTTP POST request.""" - drive, part, account, container = split_and_validate_path(req, 4) + drive, part, account, container = get_container_name_and_placement(req) req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( @@ -800,7 +824,7 @@ class ContainerController(BaseStorageServer): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) - if not check_utf8(wsgi_to_str(req.path_info)): + if not check_utf8(wsgi_to_str(req.path_info), internal=True): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: diff --git a/swift/obj/server.py b/swift/obj/server.py index 98f6887c50..33bc7ff007 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -35,7 +35,8 @@ from swift.common.utils import public, get_logger, \ normalize_delete_at_timestamp, get_log_line, Timestamp, \ get_expirer_container, parse_mime_headers, \ iter_multipart_mime_documents, extract_swift_bytes, safe_json_loads, \ - config_auto_int_value, split_path, get_redirect_data, normalize_timestamp + config_auto_int_value, split_path, get_redirect_data, \ + normalize_timestamp from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, \ valid_timestamp, check_utf8 @@ -51,7 +52,7 @@ from swift.common.base_storage_server import BaseStorageServer from swift.common.header_key_dict import HeaderKeyDict from swift.common.request_helpers import get_name_and_placement, \ is_user_meta, is_sys_or_user_meta, is_object_transient_sysmeta, \ - resolve_etag_is_at_header, is_sys_meta + resolve_etag_is_at_header, is_sys_meta, validate_internal_obj from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \ HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \ @@ -89,6 +90,20 @@ def drain(file_like, read_size, timeout): break +def get_obj_name_and_placement(request): + """ + Split and validate path for an object. + + :param request: a swob request + + :returns: a tuple of path parts and storage policy + """ + device, partition, account, container, obj, policy = \ + get_name_and_placement(request, 5, 5, True) + validate_internal_obj(account, container, obj) + return device, partition, account, container, obj, policy + + def _make_backend_fragments_header(fragments): if fragments: result = {} @@ -603,7 +618,8 @@ class ObjectController(BaseStorageServer): def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj, policy = \ - get_name_and_placement(request, 5, 5, True) + get_obj_name_and_placement(request) + req_timestamp = valid_timestamp(request) new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < req_timestamp: @@ -995,7 +1011,7 @@ class ObjectController(BaseStorageServer): def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj, policy = \ - get_name_and_placement(request, 5, 5, True) + get_obj_name_and_placement(request) disk_file, fsize, orig_metadata = self._pre_create_checks( request, device, partition, account, container, obj, policy) writer = disk_file.writer(size=fsize) @@ -1037,7 +1053,7 @@ class ObjectController(BaseStorageServer): def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj, policy = \ - get_name_and_placement(request, 5, 5, True) + get_obj_name_and_placement(request) request.headers.setdefault('X-Timestamp', normalize_timestamp(time.time())) req_timestamp = valid_timestamp(request) @@ -1104,7 +1120,7 @@ class ObjectController(BaseStorageServer): def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" device, partition, account, container, obj, policy = \ - get_name_and_placement(request, 5, 5, True) + get_obj_name_and_placement(request) request.headers.setdefault('X-Timestamp', normalize_timestamp(time.time())) req_timestamp = valid_timestamp(request) @@ -1163,7 +1179,7 @@ class ObjectController(BaseStorageServer): def DELETE(self, request): """Handle HTTP DELETE requests for the Swift Object Server.""" device, partition, account, container, obj, policy = \ - get_name_and_placement(request, 5, 5, True) + get_obj_name_and_placement(request) req_timestamp = valid_timestamp(request) next_part_power = request.headers.get('X-Backend-Next-Part-Power') try: @@ -1275,7 +1291,7 @@ class ObjectController(BaseStorageServer): req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) - if not check_utf8(wsgi_to_str(req.path_info)): + if not check_utf8(wsgi_to_str(req.path_info), internal=True): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index b56bcd285f..1d25a95d9f 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -356,6 +356,9 @@ def get_container_info(env, app, swift_source=None): req = _prepare_pre_auth_info_request( env, ("/%s/%s/%s" % (version, wsgi_account, wsgi_container)), (swift_source or 'GET_CONTAINER_INFO')) + # *Always* allow reserved names for get-info requests -- it's on the + # caller to keep the result private-ish + req.headers['X-Backend-Allow-Reserved-Names'] = 'true' resp = req.get_response(app) close_if_possible(resp.app_iter) # Check in infocache to see if the proxy (or anyone else) already @@ -412,6 +415,9 @@ def get_account_info(env, app, swift_source=None): req = _prepare_pre_auth_info_request( env, "/%s/%s" % (version, wsgi_account), (swift_source or 'GET_ACCOUNT_INFO')) + # *Always* allow reserved names for get-info requests -- it's on the + # caller to keep the result private-ish + req.headers['X-Backend-Allow-Reserved-Names'] = 'true' resp = req.get_response(app) close_if_possible(resp.app_iter) # Check in infocache to see if the proxy (or anyone else) already @@ -739,6 +745,9 @@ def _get_object_info(app, env, account, container, obj, swift_source=None): # Not in cache, let's try the object servers path = '/v1/%s/%s/%s' % (account, container, obj) req = _prepare_pre_auth_info_request(env, path, swift_source) + # *Always* allow reserved names for get-info requests -- it's on the + # caller to keep the result private-ish + req.headers['X-Backend-Allow-Reserved-Names'] = 'true' resp = req.get_response(app) # Unlike get_account_info() and get_container_info(), we don't save # things in memcache, so we can store the info without network traffic, diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index e272287ab4..f237d98293 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -24,7 +24,7 @@ # These shenanigans are to ensure all related objects can be garbage # collected. We've seen objects hang around forever otherwise. -from six.moves.urllib.parse import unquote +from six.moves.urllib.parse import quote, unquote from six.moves import zip import collections @@ -72,7 +72,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ HTTPUnprocessableEntity, Response, HTTPException, \ HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError from swift.common.request_helpers import update_etag_is_at_header, \ - resolve_etag_is_at_header + resolve_etag_is_at_header, validate_internal_obj def check_content_type(req): @@ -168,6 +168,8 @@ class BaseObjectController(Controller): self.account_name = unquote(account_name) self.container_name = unquote(container_name) self.object_name = unquote(object_name) + validate_internal_obj( + self.account_name, self.container_name, self.object_name) def iter_nodes_local_first(self, ring, partition, policy=None, local_handoffs_first=False): @@ -637,7 +639,8 @@ class BaseObjectController(Controller): except (Exception, Timeout): self.app.exception_occurred( node, _('Object'), - _('Expect: 100-continue on %s') % req.swift_entity_path) + _('Expect: 100-continue on %s') % + quote(req.swift_entity_path)) def _get_put_connections(self, req, nodes, partition, outgoing_headers, policy): @@ -1399,10 +1402,10 @@ class ECAppIter(object): except ChunkReadTimeout: # unable to resume in GetOrHeadHandler self.logger.exception(_("Timeout fetching fragments for %r"), - self.path) + quote(self.path)) except: # noqa self.logger.exception(_("Exception fetching fragments for" - " %r"), self.path) + " %r"), quote(self.path)) finally: queue.resize(2) # ensure there's room queue.put(None) @@ -1431,7 +1434,7 @@ class ECAppIter(object): segment = self.policy.pyeclib_driver.decode(fragments) except ECDriverError: self.logger.exception(_("Error decoding fragments for" - " %r"), self.path) + " %r"), quote(self.path)) raise yield segment @@ -1670,7 +1673,7 @@ class Putter(object): self.failed = True self.send_exception_handler(self.node, _('Object'), _('Trying to write to %s') - % self.path) + % quote(self.path)) def close(self): # release reference to response to ensure connection really does close, diff --git a/swift/proxy/server.py b/swift/proxy/server.py index ae7ee8ee26..6b3e3c9bc5 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -480,7 +480,8 @@ class Application(object): body='Invalid Content-Length') try: - if not check_utf8(wsgi_to_str(req.path_info)): + if not check_utf8(wsgi_to_str(req.path_info), + internal=req.allow_reserved_names): self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') diff --git a/test/probe/common.py b/test/probe/common.py index 6392b00163..9652c163bc 100644 --- a/test/probe/common.py +++ b/test/probe/common.py @@ -487,6 +487,7 @@ class ProbeTest(unittest.TestCase): [app:proxy-server] use = egg:swift#proxy + allow_account_management = True [filter:copy] use = egg:swift#copy diff --git a/test/probe/test_account_reaper.py b/test/probe/test_account_reaper.py index 6530205faf..367e8a761c 100644 --- a/test/probe/test_account_reaper.py +++ b/test/probe/test_account_reaper.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from io import BytesIO from time import sleep import uuid import unittest @@ -23,6 +24,7 @@ from swift.common import utils from swift.common.manager import Manager from swift.common.direct_client import direct_delete_account, \ direct_get_object, direct_head_container, ClientException +from swift.common.request_helpers import get_reserved_name from test.probe.common import ReplProbeTest, ENABLED_POLICIES @@ -30,8 +32,9 @@ class TestAccountReaper(ReplProbeTest): def setUp(self): super(TestAccountReaper, self).setUp() self.all_objects = [] + int_client = self.make_internal_client() # upload some containers - body = 'test-body' + body = b'test-body' for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, @@ -39,6 +42,18 @@ class TestAccountReaper(ReplProbeTest): obj = 'object-%s' % uuid.uuid4() client.put_object(self.url, self.token, container, obj, body) self.all_objects.append((policy, container, obj)) + + # Also create some reserved names + container = get_reserved_name( + 'reserved', policy.name, str(uuid.uuid4())) + int_client.create_container( + self.account, container, + headers={'X-Storage-Policy': policy.name}) + obj = get_reserved_name('object', str(uuid.uuid4())) + int_client.upload_object( + BytesIO(body), self.account, container, obj) + self.all_objects.append((policy, container, obj)) + policy.load_ring('/etc/swift') Manager(['container-updater']).once() @@ -46,11 +61,11 @@ class TestAccountReaper(ReplProbeTest): headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), - len(ENABLED_POLICIES)) + len(self.all_objects)) self.assertEqual(int(headers['x-account-object-count']), - len(ENABLED_POLICIES)) + len(self.all_objects)) self.assertEqual(int(headers['x-account-bytes-used']), - len(ENABLED_POLICIES) * len(body)) + len(self.all_objects) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) diff --git a/test/probe/test_replication_servers_working.py b/test/probe/test_replication_servers_working.py index 1052d5ba7c..fae77b742f 100644 --- a/test/probe/test_replication_servers_working.py +++ b/test/probe/test_replication_servers_working.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from io import BytesIO from unittest import main from uuid import uuid4 import os @@ -25,6 +26,7 @@ from swiftclient import client from swift.obj.diskfile import get_data_dir from test.probe.common import ReplProbeTest +from swift.common.request_helpers import get_reserved_name from swift.common.utils import readconf EXCLUDE_FILES = re.compile('^(hashes\.(pkl|invalid)|lock(-\d+)?)$') @@ -80,6 +82,15 @@ class TestReplicatorFunctions(ReplProbeTest): different port values. """ + def put_data(self): + container = 'container-%s' % uuid4() + client.put_container(self.url, self.token, container, + headers={'X-Storage-Policy': + self.policy.name}) + + obj = 'object-%s' % uuid4() + client.put_object(self.url, self.token, container, obj, 'VERIFY') + def test_main(self): # Create one account, container and object file. # Find node with account, container and object replicas. @@ -102,13 +113,7 @@ class TestReplicatorFunctions(ReplProbeTest): path_list.append(os.path.join(device_path, device)) # Put data to storage nodes - container = 'container-%s' % uuid4() - client.put_container(self.url, self.token, container, - headers={'X-Storage-Policy': - self.policy.name}) - - obj = 'object-%s' % uuid4() - client.put_object(self.url, self.token, container, obj, 'VERIFY') + self.put_data() # Get all data file information (files_list, dir_list) = collect_info(path_list) @@ -200,5 +205,19 @@ class TestReplicatorFunctions(ReplProbeTest): self.replicators.stop() +class TestReplicatorFunctionsReservedNames(TestReplicatorFunctions): + def put_data(self): + int_client = self.make_internal_client() + int_client.create_account(self.account) + container = get_reserved_name('container', str(uuid4())) + int_client.create_container(self.account, container, + headers={'X-Storage-Policy': + self.policy.name}) + + obj = get_reserved_name('object', str(uuid4())) + int_client.upload_object( + BytesIO(b'VERIFY'), self.account, container, obj) + + if __name__ == '__main__': main() diff --git a/test/probe/test_reserved_name.py b/test/probe/test_reserved_name.py new file mode 100644 index 0000000000..bce0095058 --- /dev/null +++ b/test/probe/test_reserved_name.py @@ -0,0 +1,131 @@ +#!/usr/bin/python -u +# Copyright (c) 2019 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from io import BytesIO +from uuid import uuid4 + +from swift.common.request_helpers import get_reserved_name + +from test.probe.common import ReplProbeTest + +from swiftclient import client, ClientException + + +class TestReservedNames(ReplProbeTest): + + def test_simple_crud(self): + int_client = self.make_internal_client() + + # Create reserve named container + user_cont = 'container-%s' % uuid4() + reserved_cont = get_reserved_name('container-%s' % uuid4()) + client.put_container(self.url, self.token, user_cont) + int_client.create_container(self.account, reserved_cont) + + # Check that we can list both reserved and non-reserved containers + self.assertEqual([reserved_cont, user_cont], [ + c['name'] for c in int_client.iter_containers(self.account)]) + + # sanity, user can't get to reserved name + with self.assertRaises(ClientException) as cm: + client.head_container(self.url, self.token, reserved_cont) + self.assertEqual(412, cm.exception.http_status) + + user_obj = 'obj-%s' % uuid4() + reserved_obj = get_reserved_name('obj-%s' % uuid4()) + + # InternalClient can write & read reserved names fine + int_client.upload_object( + BytesIO(b'data'), self.account, reserved_cont, reserved_obj) + int_client.get_object_metadata( + self.account, reserved_cont, reserved_obj) + _, _, app_iter = int_client.get_object( + self.account, reserved_cont, reserved_obj) + self.assertEqual(b''.join(app_iter), b'data') + self.assertEqual([reserved_obj], [ + o['name'] + for o in int_client.iter_objects(self.account, reserved_cont)]) + + # But reserved objects must be in reserved containers, and + # user objects must be in user containers (at least for now) + int_client.upload_object( + BytesIO(b'data'), self.account, reserved_cont, user_obj, + acceptable_statuses=(400,)) + + int_client.upload_object( + BytesIO(b'data'), self.account, user_cont, reserved_obj, + acceptable_statuses=(400,)) + + # Make sure we can clean up, too + int_client.delete_object(self.account, reserved_cont, reserved_obj) + int_client.delete_container(self.account, reserved_cont) + + def test_symlink_target(self): + if 'symlink' not in self.cluster_info: + raise unittest.SkipTest( + "Symlink not enabled in proxy; can't test " + "symlink to reserved name") + int_client = self.make_internal_client() + + # create link container first, ensure account gets created too + client.put_container(self.url, self.token, 'c1') + + # Create reserve named container + tgt_cont = get_reserved_name('container-%s' % uuid4()) + int_client.create_container(self.account, tgt_cont) + + # sanity, user can't get to reserved name + with self.assertRaises(ClientException) as cm: + client.head_container(self.url, self.token, tgt_cont) + self.assertEqual(412, cm.exception.http_status) + + tgt_obj = get_reserved_name('obj-%s' % uuid4()) + int_client.upload_object( + BytesIO(b'target object'), self.account, tgt_cont, tgt_obj) + metadata = int_client.get_object_metadata( + self.account, tgt_cont, tgt_obj) + etag = metadata['etag'] + + # users can write a dynamic symlink that targets a reserved + # name object + client.put_object( + self.url, self.token, 'c1', 'symlink', + headers={ + 'X-Symlink-Target': '%s/%s' % (tgt_cont, tgt_obj), + 'Content-Type': 'application/symlink', + }) + + # but can't read the symlink + with self.assertRaises(ClientException) as cm: + client.get_object(self.url, self.token, 'c1', 'symlink') + self.assertEqual(412, cm.exception.http_status) + + # user's can't create static symlink to reserved name + with self.assertRaises(ClientException) as cm: + client.put_object( + self.url, self.token, 'c1', 'static-symlink', + headers={ + 'X-Symlink-Target': '%s/%s' % (tgt_cont, tgt_obj), + 'X-Symlink-Target-Etag': etag, + 'Content-Type': 'application/symlink', + }) + self.assertEqual(412, cm.exception.http_status) + + # clean-up + client.delete_object(self.url, self.token, 'c1', 'symlink') + int_client.delete_object(self.account, tgt_cont, tgt_obj) + int_client.delete_container(self.account, tgt_cont) diff --git a/test/unit/account/test_backend.py b/test/unit/account/test_backend.py index 82e1a77075..15422bd139 100644 --- a/test/unit/account/test_backend.py +++ b/test/unit/account/test_backend.py @@ -38,6 +38,7 @@ from swift.account.backend import AccountBroker from swift.common.utils import Timestamp from test.unit import patch_policies, with_tempdir, make_timestamp_iter from swift.common.db import DatabaseConnectionError +from swift.common.request_helpers import get_reserved_name from swift.common.storage_policy import StoragePolicy, POLICIES from test.unit.common import test_db @@ -545,6 +546,35 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual([row[0] for row in listing], ['c10', 'c1']) + def test_list_container_iter_with_reserved_name(self): + # Test ContainerBroker.list_objects_iter + broker = AccountBroker(':memory:', account='a') + broker.initialize(next(self.ts).internal, 0) + + broker.put_container( + 'foo', next(self.ts).internal, 0, 0, 0, POLICIES.default.idx) + broker.put_container( + get_reserved_name('foo'), next(self.ts).internal, 0, 0, 0, + POLICIES.default.idx) + + listing = broker.list_containers_iter(100, None, None, '', '') + self.assertEqual([row[0] for row in listing], ['foo']) + + listing = broker.list_containers_iter(100, None, None, '', '', + reverse=True) + self.assertEqual([row[0] for row in listing], ['foo']) + + listing = broker.list_containers_iter(100, None, None, '', '', + allow_reserved=True) + self.assertEqual([row[0] for row in listing], + [get_reserved_name('foo'), 'foo']) + + listing = broker.list_containers_iter(100, None, None, '', '', + reverse=True, + allow_reserved=True) + self.assertEqual([row[0] for row in listing], + ['foo', get_reserved_name('foo')]) + def test_reverse_prefix_delim(self): expectations = [ { diff --git a/test/unit/account/test_reaper.py b/test/unit/account/test_reaper.py index ebb5407f7f..d8b76eb9b2 100644 --- a/test/unit/account/test_reaper.py +++ b/test/unit/account/test_reaper.py @@ -50,7 +50,12 @@ class FakeAccountBroker(object): 'delete_timestamp': time.time() - 10} return info - def list_containers_iter(self, limit, marker, *args): + def list_containers_iter(self, limit, marker, *args, **kwargs): + if not kwargs.pop('allow_reserved'): + raise RuntimeError('Expected allow_reserved to be True!') + if kwargs: + raise RuntimeError('Got unexpected keyword arguments: %r' % ( + kwargs, )) for cont in self.containers: if cont > marker: yield cont, None, None, None, None @@ -710,11 +715,16 @@ class TestReaper(unittest.TestCase): devices = self.prepare_data_dir() self.called_amount = 0 conf = {'devices': devices} - r = self.init_reaper(conf, myips=['10.10.10.2']) + r = self.init_reaper(conf, myips=['10.10.10.2'], fakelogger=True) container_reaped = [0] - def fake_list_containers_iter(self, *args): + def fake_list_containers_iter(self, *args, **kwargs): + if not kwargs.pop('allow_reserved'): + raise RuntimeError('Expected allow_reserved to be True!') + if kwargs: + raise RuntimeError('Got unexpected keyword arguments: %r' % ( + kwargs, )) for container in self.containers: if container in self.containers_yielded: continue diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index 9bee9b5c6b..c1168a852f 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -27,17 +27,19 @@ from io import BytesIO import json from six import StringIO +from six.moves.urllib.parse import quote import xml.dom.minidom from swift import __version__ as swift_version from swift.common.swob import (Request, WsgiBytesIO, HTTPNoContent) -from swift.common import constraints +from swift.common.constraints import ACCOUNT_LISTING_LIMIT from swift.account.backend import AccountBroker from swift.account.server import AccountController from swift.common.utils import (normalize_timestamp, replication, public, mkdirs, storage_directory, Timestamp) -from swift.common.request_helpers import get_sys_meta_prefix -from test.unit import patch_policies, debug_logger, mock_check_drive +from swift.common.request_helpers import get_sys_meta_prefix, get_reserved_name +from test.unit import patch_policies, debug_logger, mock_check_drive, \ + make_timestamp_iter from swift.common.storage_policy import StoragePolicy, POLICIES @@ -52,6 +54,7 @@ class TestAccountController(unittest.TestCase): self.controller = AccountController( {'devices': self.testdir, 'mount_check': 'false'}, logger=debug_logger()) + self.ts = make_timestamp_iter() def tearDown(self): """Tear down for testing swift.account.server.AccountController""" @@ -501,6 +504,65 @@ class TestAccountController(unittest.TestCase): self.assertEqual(resp.body, b'Recently deleted') self.assertEqual(resp.headers['X-Account-Status'], 'Deleted') + def test_create_reserved_namespace_account(self): + path = '/sda1/p/%s' % get_reserved_name('a') + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status, '201 Created') + + path = '/sda1/p/%s' % get_reserved_name('foo', 'bar') + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status, '201 Created') + + def test_create_invalid_reserved_namespace_account(self): + account_name = get_reserved_name('foo', 'bar')[1:] + path = '/sda1/p/%s' % account_name + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status, '400 Bad Request') + + def test_create_reserved_container_in_account(self): + # create account + path = '/sda1/p/a' + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + # put null container in it + path += '/%s' % get_reserved_name('c', 'stuff') + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal, + 'X-Put-Timestamp': next(self.ts).internal, + 'X-Delete-Timestamp': 0, + 'X-Object-Count': 0, + 'X-Bytes-Used': 0, + }) + resp = req.get_response(self.controller) + self.assertEqual(resp.status, '201 Created') + + def test_create_invalid_reserved_container_in_account(self): + # create account + path = '/sda1/p/a' + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + # put invalid container in it + path += '/%s' % get_reserved_name('c', 'stuff')[1:] + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal, + 'X-Put-Timestamp': next(self.ts).internal, + 'X-Delete-Timestamp': 0, + 'X-Object-Count': 0, + 'X-Bytes-Used': 0, + }) + resp = req.get_response(self.controller) + self.assertEqual(resp.status, '400 Bad Request') + def test_PUT_non_utf8_metadata(self): # Set metadata header req = Request.blank( @@ -706,7 +768,7 @@ class TestAccountController(unittest.TestCase): headers={'X-Timestamp': normalize_timestamp(1), 'X-Account-Meta-Test': 'Value'}) resp = req.get_response(self.controller) - self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.status_int, 204, resp.body) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) @@ -935,7 +997,7 @@ class TestAccountController(unittest.TestCase): def test_GET_over_limit(self): req = Request.blank( - '/sda1/p/a?limit=%d' % (constraints.ACCOUNT_LISTING_LIMIT + 1), + '/sda1/p/a?limit=%d' % (ACCOUNT_LISTING_LIMIT + 1), environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 412) @@ -1752,6 +1814,382 @@ class TestAccountController(unittest.TestCase): for item in json.loads(resp.body)], [{"name": "US~~UT~~~B"}]) + def _expected_listing(self, containers): + return [dict( + last_modified=c['timestamp'].isoformat, **{ + k: v for k, v in c.items() + if k != 'timestamp' + }) for c in sorted(containers, key=lambda c: c['name'])] + + def _report_containers(self, containers, account='a'): + req = Request.blank('/sda1/p/%s' % account, method='PUT', headers={ + 'x-timestamp': next(self.ts).internal}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int // 100, 2, resp.body) + for container in containers: + path = '/sda1/p/%s/%s' % (account, container['name']) + req = Request.blank(path, method='PUT', headers={ + 'X-Put-Timestamp': container['timestamp'].internal, + 'X-Delete-Timestamp': container.get( + 'deleted', Timestamp(0)).internal, + 'X-Object-Count': container['count'], + 'X-Bytes-Used': container['bytes'], + }) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int // 100, 2, resp.body) + + def test_delimiter_with_reserved_and_no_public(self): + containers = [{ + 'name': get_reserved_name('null', 'test01'), + 'bytes': 200, + 'count': 2, + 'timestamp': next(self.ts), + }] + self._report_containers(containers) + + req = Request.blank('/sda1/p/a', headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), []) + + req = Request.blank('/sda1/p/a', headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)) + + req = Request.blank('/sda1/p/a?prefix=%s&delimiter=l' % + get_reserved_name('nul'), headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), []) + + req = Request.blank('/sda1/p/a?prefix=%s&delimiter=l' % + get_reserved_name('nul'), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), [{ + 'subdir': '%s' % get_reserved_name('null')}]) + + def test_delimiter_with_reserved_and_public(self): + containers = [{ + 'name': get_reserved_name('null', 'test01'), + 'bytes': 200, + 'count': 2, + 'timestamp': next(self.ts), + }, { + 'name': 'nullish', + 'bytes': 10, + 'count': 10, + 'timestamp': next(self.ts), + }] + self._report_containers(containers) + + req = Request.blank('/sda1/p/a?prefix=nul&delimiter=l', headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), [{'subdir': 'null'}]) + + # allow-reserved header doesn't really make a difference + req = Request.blank('/sda1/p/a?prefix=nul&delimiter=l', headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), [{'subdir': 'null'}]) + + req = Request.blank('/sda1/p/a?prefix=%s&delimiter=l' % + get_reserved_name('nul'), headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), []) + + req = Request.blank('/sda1/p/a?prefix=%s&delimiter=l' % + get_reserved_name('nul'), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), [{ + 'subdir': '%s' % get_reserved_name('null')}]) + + req = Request.blank('/sda1/p/a?delimiter=%00', headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)[1:]) + + req = Request.blank('/sda1/p/a?delimiter=%00', headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + [{'subdir': '\x00'}] + + self._expected_listing(containers)[1:]) + + def test_markers_with_reserved(self): + containers = [{ + 'name': get_reserved_name('null', 'test01'), + 'bytes': 200, + 'count': 2, + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'test02'), + 'bytes': 10, + 'count': 10, + 'timestamp': next(self.ts), + }] + self._report_containers(containers) + + req = Request.blank('/sda1/p/a?marker=%s' % + get_reserved_name('null', ''), headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), []) + + req = Request.blank('/sda1/p/a?marker=%s' % + get_reserved_name('null', ''), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)) + + req = Request.blank('/sda1/p/a?marker=%s' % quote( + self._expected_listing(containers)[0]['name']), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)[1:]) + + containers.append({ + 'name': get_reserved_name('null', 'test03'), + 'bytes': 300, + 'count': 30, + 'timestamp': next(self.ts), + }) + self._report_containers(containers) + + req = Request.blank('/sda1/p/a?marker=%s' % quote( + self._expected_listing(containers)[0]['name']), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)[1:]) + + req = Request.blank('/sda1/p/a?marker=%s' % quote( + self._expected_listing(containers)[1]['name']), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)[-1:]) + + def test_prefix_with_reserved(self): + containers = [{ + 'name': get_reserved_name('null', 'test01'), + 'bytes': 200, + 'count': 2, + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'test02'), + 'bytes': 10, + 'count': 10, + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'foo'), + 'bytes': 10, + 'count': 10, + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('nullish'), + 'bytes': 300, + 'count': 32, + 'timestamp': next(self.ts), + }] + self._report_containers(containers) + + req = Request.blank('/sda1/p/a?prefix=%s' % + get_reserved_name('null', 'test'), headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), []) + + req = Request.blank('/sda1/p/a?prefix=%s' % + get_reserved_name('null', 'test'), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers[:2])) + + def test_prefix_and_delim_with_reserved(self): + containers = [{ + 'name': get_reserved_name('null', 'test01'), + 'bytes': 200, + 'count': 2, + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'test02'), + 'bytes': 10, + 'count': 10, + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'foo'), + 'bytes': 10, + 'count': 10, + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('nullish'), + 'bytes': 300, + 'count': 32, + 'timestamp': next(self.ts), + }] + self._report_containers(containers) + + req = Request.blank('/sda1/p/a?prefix=%s&delimiter=%s' % ( + get_reserved_name('null'), get_reserved_name()), headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), []) + + req = Request.blank('/sda1/p/a?prefix=%s&delimiter=%s' % ( + get_reserved_name('null'), get_reserved_name()), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + expected = [{'subdir': get_reserved_name('null', '')}] + \ + self._expected_listing(containers[-1:]) + self.assertEqual(json.loads(resp.body), expected) + + def test_reserved_markers_with_non_reserved(self): + containers = [{ + 'name': get_reserved_name('null', 'test01'), + 'bytes': 200, + 'count': 2, + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'test02'), + 'bytes': 10, + 'count': 10, + 'timestamp': next(self.ts), + }, { + 'name': 'nullish', + 'bytes': 300, + 'count': 32, + 'timestamp': next(self.ts), + }] + self._report_containers(containers) + + req = Request.blank('/sda1/p/a?marker=%s' % + get_reserved_name('null', ''), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)) + + req = Request.blank('/sda1/p/a?marker=%s' % + get_reserved_name('null', ''), headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + [c for c in self._expected_listing(containers) + if get_reserved_name() not in c['name']]) + + req = Request.blank('/sda1/p/a?marker=%s' % + get_reserved_name('null', ''), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)) + + req = Request.blank('/sda1/p/a?marker=%s' % quote( + self._expected_listing(containers)[0]['name']), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)[1:]) + + def test_null_markers(self): + containers = [{ + 'name': get_reserved_name('null', ''), + 'bytes': 200, + 'count': 2, + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'test01'), + 'bytes': 200, + 'count': 2, + 'timestamp': next(self.ts), + }, { + 'name': 'null', + 'bytes': 300, + 'count': 32, + 'timestamp': next(self.ts), + }] + self._report_containers(containers) + + req = Request.blank('/sda1/p/a?marker=%s' % get_reserved_name('null'), + headers={'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)[-1:]) + + req = Request.blank('/sda1/p/a?marker=%s' % get_reserved_name('null'), + headers={'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)) + + req = Request.blank('/sda1/p/a?marker=%s' % + get_reserved_name('null', ''), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)[1:]) + + req = Request.blank('/sda1/p/a?marker=%s' % + get_reserved_name('null', 'test00'), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(containers)[1:]) + def test_through_call(self): inbuf = BytesIO() errbuf = StringIO() @@ -1814,7 +2252,7 @@ class TestAccountController(unittest.TestCase): self.controller.__call__({'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', - 'PATH_INFO': '\x00', + 'PATH_INFO': '/sda1/p/a/c\xd8\x3e%20', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', diff --git a/test/unit/account/test_utils.py b/test/unit/account/test_utils.py index f931ad58bd..eb332395a7 100644 --- a/test/unit/account/test_utils.py +++ b/test/unit/account/test_utils.py @@ -14,15 +14,18 @@ import itertools import time import unittest +import json import mock from swift.account import utils, backend -from swift.common.storage_policy import POLICIES +from swift.common.storage_policy import POLICIES, StoragePolicy +from swift.common.swob import Request from swift.common.utils import Timestamp from swift.common.header_key_dict import HeaderKeyDict +from swift.common.request_helpers import get_reserved_name -from test.unit import patch_policies +from test.unit import patch_policies, make_timestamp_iter class TestFakeAccountBroker(unittest.TestCase): @@ -57,6 +60,9 @@ class TestFakeAccountBroker(unittest.TestCase): class TestAccountUtils(unittest.TestCase): + def setUp(self): + self.ts = make_timestamp_iter() + def test_get_response_headers_fake_broker(self): broker = utils.FakeAccountBroker() now = time.time() @@ -187,3 +193,77 @@ class TestAccountUtils(unittest.TestCase): 'value for %r was %r not %r' % ( key, value, expected_value)) self.assertFalse(expected) + + def test_account_listing_response(self): + req = Request.blank('') + now = time.time() + with mock.patch('time.time', new=lambda: now): + resp = utils.account_listing_response('a', req, 'text/plain') + self.assertEqual(resp.status_int, 204) + expected = HeaderKeyDict({ + 'Content-Type': 'text/plain; charset=utf-8', + 'X-Account-Container-Count': 0, + 'X-Account-Object-Count': 0, + 'X-Account-Bytes-Used': 0, + 'X-Timestamp': Timestamp(now).normal, + 'X-PUT-Timestamp': Timestamp(now).normal, + }) + self.assertEqual(expected, resp.headers) + self.assertEqual(b'', resp.body) + + @patch_policies([StoragePolicy(0, 'zero', is_default=True)]) + def test_account_listing_reserved_names(self): + broker = backend.AccountBroker(':memory:', account='a') + put_timestamp = next(self.ts) + now = time.time() + with mock.patch('time.time', new=lambda: now): + broker.initialize(put_timestamp.internal) + container_timestamp = next(self.ts) + broker.put_container(get_reserved_name('foo'), + container_timestamp.internal, 0, 10, 100, 0) + + req = Request.blank('') + resp = utils.account_listing_response( + 'a', req, 'application/json', broker) + self.assertEqual(resp.status_int, 200) + expected = HeaderKeyDict({ + 'Content-Type': 'application/json; charset=utf-8', + 'Content-Length': 2, + 'X-Account-Container-Count': 1, + 'X-Account-Object-Count': 10, + 'X-Account-Bytes-Used': 100, + 'X-Timestamp': Timestamp(now).normal, + 'X-PUT-Timestamp': put_timestamp.normal, + 'X-Account-Storage-Policy-Zero-Container-Count': 1, + 'X-Account-Storage-Policy-Zero-Object-Count': 10, + 'X-Account-Storage-Policy-Zero-Bytes-Used': 100, + }) + self.assertEqual(expected, resp.headers) + self.assertEqual(b'[]', resp.body) + + req = Request.blank('', headers={ + 'X-Backend-Allow-Reserved-Names': 'true'}) + resp = utils.account_listing_response( + 'a', req, 'application/json', broker) + self.assertEqual(resp.status_int, 200) + expected = HeaderKeyDict({ + 'Content-Type': 'application/json; charset=utf-8', + 'Content-Length': 97, + 'X-Account-Container-Count': 1, + 'X-Account-Object-Count': 10, + 'X-Account-Bytes-Used': 100, + 'X-Timestamp': Timestamp(now).normal, + 'X-PUT-Timestamp': put_timestamp.normal, + 'X-Account-Storage-Policy-Zero-Container-Count': 1, + 'X-Account-Storage-Policy-Zero-Object-Count': 10, + 'X-Account-Storage-Policy-Zero-Bytes-Used': 100, + }) + self.assertEqual(expected, resp.headers) + expected = [{ + "last_modified": container_timestamp.isoformat, + "count": 10, + "bytes": 100, + "name": get_reserved_name('foo'), + }] + self.assertEqual(sorted(json.dumps(expected).encode('ascii')), + sorted(resp.body)) diff --git a/test/unit/common/middleware/test_gatekeeper.py b/test/unit/common/middleware/test_gatekeeper.py index 0fac175dbf..4f8cb480b7 100644 --- a/test/unit/common/middleware/test_gatekeeper.py +++ b/test/unit/common/middleware/test_gatekeeper.py @@ -245,6 +245,39 @@ class TestGatekeeper(unittest.TestCase): self._test_location_header('/v/a/c/o2?query=path#test') self._test_location_header('/v/a/c/o2;whatisparam?query=path#test') + def test_allow_reserved_names(self): + fake_app = FakeApp() + app = self.get_app(fake_app, {}) + headers = { + 'X-Allow-Reserved-Names': 'some-value' + } + + req = Request.blank('/v/a/c/o', method='GET', headers=headers) + resp = req.get_response(app) + self.assertEqual('200 OK', resp.status) + self.assertNotIn('X-Backend-Allow-Reserved-Names', + fake_app.req.headers) + self.assertIn('X-Allow-Reserved-Names', + fake_app.req.headers) + self.assertEqual( + 'some-value', + fake_app.req.headers['X-Allow-Reserved-Names']) + + app.allow_reserved_names_header = True + req = Request.blank('/v/a/c/o', method='GET', headers=headers) + resp = req.get_response(app) + self.assertEqual('200 OK', resp.status) + self.assertIn('X-Backend-Allow-Reserved-Names', + fake_app.req.headers) + self.assertEqual( + 'some-value', + fake_app.req.headers['X-Backend-Allow-Reserved-Names']) + self.assertEqual( + 'some-value', + req.headers['X-Backend-Allow-Reserved-Names']) + self.assertNotIn('X-Allow-Reserved-Names', fake_app.req.headers) + self.assertNotIn('X-Allow-Reserved-Names', req.headers) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/test_listing_formats.py b/test/unit/common/middleware/test_listing_formats.py index 6ac6d08735..22cf3e55f7 100644 --- a/test/unit/common/middleware/test_listing_formats.py +++ b/test/unit/common/middleware/test_listing_formats.py @@ -18,13 +18,17 @@ import unittest from swift.common.swob import Request, HTTPOk from swift.common.middleware import listing_formats +from swift.common.request_helpers import get_reserved_name +from test.unit import debug_logger from test.unit.common.middleware.helpers import FakeSwift class TestListingFormats(unittest.TestCase): def setUp(self): self.fake_swift = FakeSwift() - self.app = listing_formats.ListingFilter(self.fake_swift) + self.logger = debug_logger('test-listing') + self.app = listing_formats.ListingFilter(self.fake_swift, {}, + logger=self.logger) self.fake_account_listing = json.dumps([ {'name': 'bar', 'bytes': 0, 'count': 0, 'last_modified': '1970-01-01T00:00:00.000000'}, @@ -37,6 +41,25 @@ class TestListingFormats(unittest.TestCase): {'subdir': 'foo/'}, ]).encode('ascii') + self.fake_account_listing_with_reserved = json.dumps([ + {'name': 'bar', 'bytes': 0, 'count': 0, + 'last_modified': '1970-01-01T00:00:00.000000'}, + {'name': get_reserved_name('bar', 'versions'), 'bytes': 0, + 'count': 0, 'last_modified': '1970-01-01T00:00:00.000000'}, + {'subdir': 'foo_'}, + {'subdir': get_reserved_name('foo_')}, + ]).encode('ascii') + self.fake_container_listing_with_reserved = json.dumps([ + {'name': 'bar', 'hash': 'etag', 'bytes': 0, + 'content_type': 'text/plain', + 'last_modified': '1970-01-01T00:00:00.000000'}, + {'name': get_reserved_name('bar', 'extra data'), 'hash': 'etag', + 'bytes': 0, 'content_type': 'text/plain', + 'last_modified': '1970-01-01T00:00:00.000000'}, + {'subdir': 'foo/'}, + {'subdir': get_reserved_name('foo/')}, + ]).encode('ascii') + def test_valid_account(self): self.fake_swift.register('GET', '/v1/a', HTTPOk, { 'Content-Length': str(len(self.fake_account_listing)), @@ -60,7 +83,8 @@ class TestListingFormats(unittest.TestCase): req = Request.blank('/v1/a?format=json') resp = req.get_response(self.app) - self.assertEqual(resp.body, self.fake_account_listing) + self.assertEqual(json.loads(resp.body), + json.loads(self.fake_account_listing)) self.assertEqual(resp.headers['Content-Type'], 'application/json; charset=utf-8') self.assertEqual(self.fake_swift.calls[-1], ( @@ -82,6 +106,119 @@ class TestListingFormats(unittest.TestCase): self.assertEqual(self.fake_swift.calls[-1], ( 'GET', '/v1/a?format=json')) + def test_valid_account_with_reserved(self): + body_len = len(self.fake_account_listing_with_reserved) + self.fake_swift.register( + 'GET', '/v1/a\xe2\x98\x83', HTTPOk, { + 'Content-Length': str(body_len), + 'Content-Type': 'application/json', + }, self.fake_account_listing_with_reserved) + + req = Request.blank('/v1/a\xe2\x98\x83') + resp = req.get_response(self.app) + self.assertEqual(resp.body, b'bar\nfoo_\n') + self.assertEqual(resp.headers['Content-Type'], + 'text/plain; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', '/v1/a\xe2\x98\x83?format=json')) + self.assertEqual(self.logger.get_lines_for_level('warning'), [ + "Account listing for a%E2%98%83 had reserved byte in name: " + "'\\x00bar\\x00versions'", + "Account listing for a%E2%98%83 had reserved byte in subdir: " + "'\\x00foo_'", + ]) + + req = Request.blank('/v1/a\xe2\x98\x83', headers={ + 'X-Backend-Allow-Reserved-Names': 'true'}) + resp = req.get_response(self.app) + self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\n' % ( + get_reserved_name('bar', 'versions').encode('ascii'), + get_reserved_name('foo_').encode('ascii'), + )) + self.assertEqual(resp.headers['Content-Type'], + 'text/plain; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', '/v1/a\xe2\x98\x83?format=json')) + + req = Request.blank('/v1/a\xe2\x98\x83?format=txt') + resp = req.get_response(self.app) + self.assertEqual(resp.body, b'bar\nfoo_\n') + self.assertEqual(resp.headers['Content-Type'], + 'text/plain; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', '/v1/a\xe2\x98\x83?format=json')) + + req = Request.blank('/v1/a\xe2\x98\x83?format=txt', headers={ + 'X-Backend-Allow-Reserved-Names': 'true'}) + resp = req.get_response(self.app) + self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\n' % ( + get_reserved_name('bar', 'versions').encode('ascii'), + get_reserved_name('foo_').encode('ascii'), + )) + self.assertEqual(resp.headers['Content-Type'], + 'text/plain; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', '/v1/a\xe2\x98\x83?format=json')) + + req = Request.blank('/v1/a\xe2\x98\x83?format=json') + resp = req.get_response(self.app) + self.assertEqual(json.loads(resp.body), + json.loads(self.fake_account_listing)) + self.assertEqual(resp.headers['Content-Type'], + 'application/json; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', '/v1/a\xe2\x98\x83?format=json')) + + req = Request.blank('/v1/a\xe2\x98\x83?format=json', headers={ + 'X-Backend-Allow-Reserved-Names': 'true'}) + resp = req.get_response(self.app) + self.assertEqual(json.loads(resp.body), + json.loads(self.fake_account_listing_with_reserved)) + self.assertEqual(resp.headers['Content-Type'], + 'application/json; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', '/v1/a\xe2\x98\x83?format=json')) + + req = Request.blank('/v1/a\xe2\x98\x83?format=xml') + resp = req.get_response(self.app) + self.assertEqual(resp.body.split(b'\n'), [ + b'', + b'', + b'bar00' + b'1970-01-01T00:00:00.000000' + b'', + b'', + b'', + ]) + self.assertEqual(resp.headers['Content-Type'], + 'application/xml; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', '/v1/a\xe2\x98\x83?format=json')) + + req = Request.blank('/v1/a\xe2\x98\x83?format=xml', headers={ + 'X-Backend-Allow-Reserved-Names': 'true'}) + resp = req.get_response(self.app) + self.assertEqual(resp.body.split(b'\n'), [ + b'', + b'', + b'bar00' + b'1970-01-01T00:00:00.000000' + b'', + b'%s' + b'00' + b'1970-01-01T00:00:00.000000' + b'' % get_reserved_name( + 'bar', 'versions').encode('ascii'), + b'', + b'' % get_reserved_name( + 'foo_').encode('ascii'), + b'', + ]) + self.assertEqual(resp.headers['Content-Type'], + 'application/xml; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', '/v1/a\xe2\x98\x83?format=json')) + def test_valid_container(self): self.fake_swift.register('GET', '/v1/a/c', HTTPOk, { 'Content-Length': str(len(self.fake_container_listing)), @@ -105,7 +242,8 @@ class TestListingFormats(unittest.TestCase): req = Request.blank('/v1/a/c?format=json') resp = req.get_response(self.app) - self.assertEqual(resp.body, self.fake_container_listing) + self.assertEqual(json.loads(resp.body), + json.loads(self.fake_container_listing)) self.assertEqual(resp.headers['Content-Type'], 'application/json; charset=utf-8') self.assertEqual(self.fake_swift.calls[-1], ( @@ -129,6 +267,126 @@ class TestListingFormats(unittest.TestCase): self.assertEqual(self.fake_swift.calls[-1], ( 'GET', '/v1/a/c?format=json')) + def test_valid_container_with_reserved(self): + path = '/v1/a\xe2\x98\x83/c\xf0\x9f\x8c\xb4' + body_len = len(self.fake_container_listing_with_reserved) + self.fake_swift.register( + 'GET', path, HTTPOk, { + 'Content-Length': str(body_len), + 'Content-Type': 'application/json', + }, self.fake_container_listing_with_reserved) + + req = Request.blank(path) + resp = req.get_response(self.app) + self.assertEqual(resp.body, b'bar\nfoo/\n') + self.assertEqual(resp.headers['Content-Type'], + 'text/plain; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', path + '?format=json')) + self.assertEqual(self.logger.get_lines_for_level('warning'), [ + "Container listing for a%E2%98%83/c%F0%9F%8C%B4 had reserved byte " + "in name: '\\x00bar\\x00extra data'", + "Container listing for a%E2%98%83/c%F0%9F%8C%B4 had reserved byte " + "in subdir: '\\x00foo/'", + ]) + + req = Request.blank(path, headers={ + 'X-Backend-Allow-Reserved-Names': 'true'}) + resp = req.get_response(self.app) + self.assertEqual(resp.body, b'bar\n%s\nfoo/\n%s\n' % ( + get_reserved_name('bar', 'extra data').encode('ascii'), + get_reserved_name('foo/').encode('ascii'), + )) + self.assertEqual(resp.headers['Content-Type'], + 'text/plain; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', path + '?format=json')) + + req = Request.blank(path + '?format=txt') + resp = req.get_response(self.app) + self.assertEqual(resp.body, b'bar\nfoo/\n') + self.assertEqual(resp.headers['Content-Type'], + 'text/plain; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', path + '?format=json')) + + req = Request.blank(path + '?format=txt', headers={ + 'X-Backend-Allow-Reserved-Names': 'true'}) + resp = req.get_response(self.app) + self.assertEqual(resp.body, b'bar\n%s\nfoo/\n%s\n' % ( + get_reserved_name('bar', 'extra data').encode('ascii'), + get_reserved_name('foo/').encode('ascii'), + )) + self.assertEqual(resp.headers['Content-Type'], + 'text/plain; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', path + '?format=json')) + + req = Request.blank(path + '?format=json') + resp = req.get_response(self.app) + self.assertEqual(json.loads(resp.body), + json.loads(self.fake_container_listing)) + self.assertEqual(resp.headers['Content-Type'], + 'application/json; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', path + '?format=json')) + + req = Request.blank(path + '?format=json', headers={ + 'X-Backend-Allow-Reserved-Names': 'true'}) + resp = req.get_response(self.app) + self.assertEqual(json.loads(resp.body), + json.loads(self.fake_container_listing_with_reserved)) + self.assertEqual(resp.headers['Content-Type'], + 'application/json; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', path + '?format=json')) + + req = Request.blank(path + '?format=xml') + resp = req.get_response(self.app) + self.assertEqual( + resp.body, + b'\n' + b'' + b'baretag0' + b'text/plain' + b'1970-01-01T00:00:00.000000' + b'' + b'foo/' + b'' + ) + self.assertEqual(resp.headers['Content-Type'], + 'application/xml; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', path + '?format=json')) + + req = Request.blank(path + '?format=xml', headers={ + 'X-Backend-Allow-Reserved-Names': 'true'}) + resp = req.get_response(self.app) + self.assertEqual( + resp.body, + b'\n' + b'' + b'baretag0' + b'text/plain' + b'1970-01-01T00:00:00.000000' + b'' + b'%s' + b'etag0' + b'text/plain' + b'1970-01-01T00:00:00.000000' + b'' + b'foo/' + b'%s' + b'' % ( + get_reserved_name('bar', 'extra data').encode('ascii'), + get_reserved_name('foo/').encode('ascii'), + get_reserved_name('foo/').encode('ascii'), + )) + self.assertEqual(resp.headers['Content-Type'], + 'application/xml; charset=utf-8') + self.assertEqual(self.fake_swift.calls[-1], ( + 'GET', path + '?format=json')) + def test_blank_account(self): self.fake_swift.register('GET', '/v1/a', HTTPOk, { 'Content-Length': '2', 'Content-Type': 'application/json'}, b'[]') diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index aa42f243c8..5090fddeb6 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -506,7 +506,7 @@ class TestConstraints(unittest.TestCase): def test_check_utf8(self): unicode_sample = u'\uc77c\uc601' - unicode_with_null = u'abc\u0000def' + unicode_with_reserved = u'abc%sdef' % utils.RESERVED_STR # Some false-y values self.assertFalse(constraints.check_utf8(None)) @@ -518,15 +518,24 @@ class TestConstraints(unittest.TestCase): self.assertFalse(constraints.check_utf8( unicode_sample.encode('utf-8')[::-1])) # unicode with null - self.assertFalse(constraints.check_utf8(unicode_with_null)) + self.assertFalse(constraints.check_utf8(unicode_with_reserved)) # utf8 bytes with null self.assertFalse(constraints.check_utf8( - unicode_with_null.encode('utf8'))) + unicode_with_reserved.encode('utf8'))) self.assertTrue(constraints.check_utf8('this is ascii and utf-8, too')) self.assertTrue(constraints.check_utf8(unicode_sample)) self.assertTrue(constraints.check_utf8(unicode_sample.encode('utf8'))) + def test_check_utf8_internal(self): + unicode_with_reserved = u'abc%sdef' % utils.RESERVED_STR + # sanity + self.assertFalse(constraints.check_utf8(unicode_with_reserved)) + self.assertTrue(constraints.check_utf8('foobar', internal=True)) + # internal allows reserved names + self.assertTrue(constraints.check_utf8(unicode_with_reserved, + internal=True)) + def test_check_utf8_non_canonical(self): self.assertFalse(constraints.check_utf8(b'\xed\xa0\xbc\xed\xbc\xb8')) self.assertTrue(constraints.check_utf8(u'\U0001f338')) diff --git a/test/unit/common/test_direct_client.py b/test/unit/common/test_direct_client.py index ad711f378d..a714d16ba9 100644 --- a/test/unit/common/test_direct_client.py +++ b/test/unit/common/test_direct_client.py @@ -132,66 +132,76 @@ class TestDirectClient(unittest.TestCase): stub_user_agent = 'direct-client %s' % os.getpid() headers = direct_client.gen_headers(add_ts=False) - self.assertEqual(headers['user-agent'], stub_user_agent) - self.assertEqual(1, len(headers)) + self.assertEqual(dict(headers), { + 'User-Agent': stub_user_agent, + 'X-Backend-Allow-Reserved-Names': 'true', + }) - now = time.time() - headers = direct_client.gen_headers() - self.assertEqual(headers['user-agent'], stub_user_agent) - self.assertTrue(now - 1 < Timestamp(headers['x-timestamp']) < now + 1) - self.assertEqual(headers['x-timestamp'], - Timestamp(headers['x-timestamp']).internal) - self.assertEqual(2, len(headers)) + with mock.patch('swift.common.utils.Timestamp.now', + return_value=Timestamp('123.45')): + headers = direct_client.gen_headers() + self.assertEqual(dict(headers), { + 'User-Agent': stub_user_agent, + 'X-Backend-Allow-Reserved-Names': 'true', + 'X-Timestamp': '0000000123.45000', + }) headers = direct_client.gen_headers(hdrs_in={'x-timestamp': '15'}) - self.assertEqual(headers['x-timestamp'], '15') - self.assertEqual(headers['user-agent'], stub_user_agent) - self.assertEqual(2, len(headers)) + self.assertEqual(dict(headers), { + 'User-Agent': stub_user_agent, + 'X-Backend-Allow-Reserved-Names': 'true', + 'X-Timestamp': '15', + }) - headers = direct_client.gen_headers(hdrs_in={'foo-bar': '63'}) - self.assertEqual(headers['user-agent'], stub_user_agent) - self.assertEqual(headers['foo-bar'], '63') - self.assertTrue(now - 1 < Timestamp(headers['x-timestamp']) < now + 1) - self.assertEqual(headers['x-timestamp'], - Timestamp(headers['x-timestamp']).internal) - self.assertEqual(3, len(headers)) + with mock.patch('swift.common.utils.Timestamp.now', + return_value=Timestamp('12345.6789')): + headers = direct_client.gen_headers(hdrs_in={'foo-bar': '63'}) + self.assertEqual(dict(headers), { + 'User-Agent': stub_user_agent, + 'Foo-Bar': '63', + 'X-Backend-Allow-Reserved-Names': 'true', + 'X-Timestamp': '0000012345.67890', + }) hdrs_in = {'foo-bar': '55'} headers = direct_client.gen_headers(hdrs_in, add_ts=False) - self.assertEqual(headers['user-agent'], stub_user_agent) - self.assertEqual(headers['foo-bar'], '55') - self.assertEqual(2, len(headers)) + self.assertEqual(dict(headers), { + 'User-Agent': stub_user_agent, + 'Foo-Bar': '55', + 'X-Backend-Allow-Reserved-Names': 'true', + }) - headers = direct_client.gen_headers(hdrs_in={'user-agent': '32'}) - self.assertEqual(headers['user-agent'], '32') - self.assertTrue(now - 1 < Timestamp(headers['x-timestamp']) < now + 1) - self.assertEqual(headers['x-timestamp'], - Timestamp(headers['x-timestamp']).internal) - self.assertEqual(2, len(headers)) + with mock.patch('swift.common.utils.Timestamp.now', + return_value=Timestamp('12345')): + headers = direct_client.gen_headers(hdrs_in={'user-agent': '32'}) + self.assertEqual(dict(headers), { + 'User-Agent': '32', + 'X-Backend-Allow-Reserved-Names': 'true', + 'X-Timestamp': '0000012345.00000', + }) hdrs_in = {'user-agent': '47'} headers = direct_client.gen_headers(hdrs_in, add_ts=False) - self.assertEqual(headers['user-agent'], '47') - self.assertEqual(1, len(headers)) + self.assertEqual(dict(headers), { + 'User-Agent': '47', + 'X-Backend-Allow-Reserved-Names': 'true', + }) for policy in POLICIES: for add_ts in (True, False): - now = time.time() - headers = direct_client.gen_headers( - {'X-Backend-Storage-Policy-Index': policy.idx}, - add_ts=add_ts) - self.assertEqual(headers['user-agent'], stub_user_agent) - self.assertEqual(headers['X-Backend-Storage-Policy-Index'], - str(policy.idx)) - expected_header_count = 2 + with mock.patch('swift.common.utils.Timestamp.now', + return_value=Timestamp('123456789')): + headers = direct_client.gen_headers( + {'X-Backend-Storage-Policy-Index': policy.idx}, + add_ts=add_ts) + expected = { + 'User-Agent': stub_user_agent, + 'X-Backend-Storage-Policy-Index': str(policy.idx), + 'X-Backend-Allow-Reserved-Names': 'true', + } if add_ts: - expected_header_count += 1 - self.assertEqual( - headers['x-timestamp'], - Timestamp(headers['x-timestamp']).internal) - self.assertTrue( - now - 1 < Timestamp(headers['x-timestamp']) < now + 1) - self.assertEqual(expected_header_count, len(headers)) + expected['X-Timestamp'] = '0123456789.00000' + self.assertEqual(dict(headers), expected) def test_direct_get_account(self): def do_test(req_params): diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index 524be1054d..e957ce894c 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -1233,7 +1233,8 @@ class TestInternalClient(unittest.TestCase): self.assertEqual(app.call_count, 1) req_headers.update({ 'host': 'localhost:80', # from swob.Request.blank - 'user-agent': 'test', # from InternalClient.make_request + 'user-agent': 'test', # from InternalClient.make_request + 'x-backend-allow-reserved-names': 'true', # also from IC }) self.assertEqual(app.calls_with_headers, [( 'GET', path_info + '?symlink=get', HeaderKeyDict(req_headers))]) diff --git a/test/unit/common/test_request_helpers.py b/test/unit/common/test_request_helpers.py index 5e07ddb584..bbdc3be7ff 100644 --- a/test/unit/common/test_request_helpers.py +++ b/test/unit/common/test_request_helpers.py @@ -181,6 +181,174 @@ class TestRequestHelpers(unittest.TestCase): self.assertEqual(policy, POLICIES[1]) self.assertEqual(policy.policy_type, REPL_POLICY) + def test_validate_internal_name(self): + self.assertIsNone(rh._validate_internal_name('foo')) + self.assertIsNone(rh._validate_internal_name( + rh.get_reserved_name('foo'))) + self.assertIsNone(rh._validate_internal_name( + rh.get_reserved_name('foo', 'bar'))) + self.assertIsNone(rh._validate_internal_name('')) + self.assertIsNone(rh._validate_internal_name(rh.RESERVED)) + + def test_invalid_reserved_name(self): + with self.assertRaises(HTTPException) as raised: + rh._validate_internal_name('foo' + rh.RESERVED) + e = raised.exception + self.assertEqual(e.status_int, 400) + self.assertEqual(str(e), '400 Bad Request') + self.assertEqual(e.body, b"Invalid reserved-namespace name") + + def test_validate_internal_account(self): + self.assertIsNone(rh.validate_internal_account('AUTH_foo')) + self.assertIsNone(rh.validate_internal_account( + rh.get_reserved_name('AUTH_foo'))) + with self.assertRaises(HTTPException) as raised: + rh.validate_internal_account('AUTH_foo' + rh.RESERVED) + e = raised.exception + self.assertEqual(e.status_int, 400) + self.assertEqual(str(e), '400 Bad Request') + self.assertEqual(e.body, b"Invalid reserved-namespace account") + + def test_validate_internal_container(self): + self.assertIsNone(rh.validate_internal_container('AUTH_foo', 'bar')) + self.assertIsNone(rh.validate_internal_container( + rh.get_reserved_name('AUTH_foo'), 'bar')) + self.assertIsNone(rh.validate_internal_container( + 'foo', rh.get_reserved_name('bar'))) + self.assertIsNone(rh.validate_internal_container( + rh.get_reserved_name('AUTH_foo'), rh.get_reserved_name('bar'))) + with self.assertRaises(HTTPException) as raised: + rh.validate_internal_container('AUTH_foo' + rh.RESERVED, 'bar') + e = raised.exception + self.assertEqual(e.status_int, 400) + self.assertEqual(str(e), '400 Bad Request') + self.assertEqual(e.body, b"Invalid reserved-namespace account") + with self.assertRaises(HTTPException) as raised: + rh.validate_internal_container('AUTH_foo', 'bar' + rh.RESERVED) + e = raised.exception + self.assertEqual(e.status_int, 400) + self.assertEqual(str(e), '400 Bad Request') + self.assertEqual(e.body, b"Invalid reserved-namespace container") + + # These should always be operating on split_path outputs so this + # shouldn't really be an issue, but just in case... + for acct in ('', None): + with self.assertRaises(ValueError) as raised: + rh.validate_internal_container( + acct, 'bar') + self.assertEqual(raised.exception.args[0], 'Account is required') + + def test_validate_internal_object(self): + self.assertIsNone(rh.validate_internal_obj('AUTH_foo', 'bar', 'baz')) + self.assertIsNone(rh.validate_internal_obj( + rh.get_reserved_name('AUTH_foo'), 'bar', 'baz')) + for acct in ('AUTH_foo', rh.get_reserved_name('AUTH_foo')): + self.assertIsNone(rh.validate_internal_obj( + acct, + rh.get_reserved_name('bar'), + rh.get_reserved_name('baz'))) + for acct in ('AUTH_foo', rh.get_reserved_name('AUTH_foo')): + with self.assertRaises(HTTPException) as raised: + rh.validate_internal_obj( + acct, 'bar', rh.get_reserved_name('baz')) + e = raised.exception + self.assertEqual(e.status_int, 400) + self.assertEqual(str(e), '400 Bad Request') + self.assertEqual(e.body, b"Invalid reserved-namespace object " + b"in user-namespace container") + for acct in ('AUTH_foo', rh.get_reserved_name('AUTH_foo')): + with self.assertRaises(HTTPException) as raised: + rh.validate_internal_obj( + acct, rh.get_reserved_name('bar'), 'baz') + e = raised.exception + self.assertEqual(e.status_int, 400) + self.assertEqual(str(e), '400 Bad Request') + self.assertEqual(e.body, b"Invalid user-namespace object " + b"in reserved-namespace container") + + # These should always be operating on split_path outputs so this + # shouldn't really be an issue, but just in case... + for acct in ('', None): + with self.assertRaises(ValueError) as raised: + rh.validate_internal_obj( + acct, 'bar', 'baz') + self.assertEqual(raised.exception.args[0], 'Account is required') + + for cont in ('', None): + with self.assertRaises(ValueError) as raised: + rh.validate_internal_obj( + 'AUTH_foo', cont, 'baz') + self.assertEqual(raised.exception.args[0], 'Container is required') + + def test_invalid_reserved_names(self): + with self.assertRaises(HTTPException) as raised: + rh.validate_internal_obj('AUTH_foo' + rh.RESERVED, 'bar', 'baz') + e = raised.exception + self.assertEqual(e.status_int, 400) + self.assertEqual(str(e), '400 Bad Request') + self.assertEqual(e.body, b"Invalid reserved-namespace account") + with self.assertRaises(HTTPException) as raised: + rh.validate_internal_obj('AUTH_foo', 'bar' + rh.RESERVED, 'baz') + e = raised.exception + self.assertEqual(e.status_int, 400) + self.assertEqual(str(e), '400 Bad Request') + self.assertEqual(e.body, b"Invalid reserved-namespace container") + with self.assertRaises(HTTPException) as raised: + rh.validate_internal_obj('AUTH_foo', 'bar', 'baz' + rh.RESERVED) + e = raised.exception + self.assertEqual(e.status_int, 400) + self.assertEqual(str(e), '400 Bad Request') + self.assertEqual(e.body, b"Invalid reserved-namespace object") + + def test_get_reserved_name(self): + expectations = { + tuple(): rh.RESERVED, + ('',): rh.RESERVED, + ('foo',): rh.RESERVED + 'foo', + ('foo', 'bar'): rh.RESERVED + 'foo' + rh.RESERVED + 'bar', + ('foo', ''): rh.RESERVED + 'foo' + rh.RESERVED, + ('', ''): rh.RESERVED * 2, + } + failures = [] + for parts, expected in expectations.items(): + name = rh.get_reserved_name(*parts) + if name != expected: + failures.append('get given %r expected %r != %r' % ( + parts, expected, name)) + if failures: + self.fail('Unexpected reults:\n' + '\n'.join(failures)) + + def test_invalid_get_reserved_name(self): + self.assertRaises(ValueError) + with self.assertRaises(ValueError) as ctx: + rh.get_reserved_name('foo', rh.RESERVED + 'bar', 'baz') + self.assertEqual(str(ctx.exception), + 'Invalid reserved part in components') + + def test_split_reserved_name(self): + expectations = { + rh.RESERVED: ('',), + rh.RESERVED + 'foo': ('foo',), + rh.RESERVED + 'foo' + rh.RESERVED + 'bar': ('foo', 'bar'), + rh.RESERVED + 'foo' + rh.RESERVED: ('foo', ''), + rh.RESERVED * 2: ('', ''), + } + failures = [] + for name, expected in expectations.items(): + parts = rh.split_reserved_name(name) + if tuple(parts) != expected: + failures.append('split given %r expected %r != %r' % ( + name, expected, parts)) + if failures: + self.fail('Unexpected reults:\n' + '\n'.join(failures)) + + def test_invalid_split_reserved_name(self): + self.assertRaises(ValueError) + with self.assertRaises(ValueError) as ctx: + rh.split_reserved_name('foo') + self.assertEqual(str(ctx.exception), + 'Invalid reserved name') + class TestHTTPResponseToDocumentIters(unittest.TestCase): def test_200(self): diff --git a/test/unit/common/test_swob.py b/test/unit/common/test_swob.py index cfaaa1fd80..a6dbeed27a 100644 --- a/test/unit/common/test_swob.py +++ b/test/unit/common/test_swob.py @@ -1115,6 +1115,19 @@ class TestRequest(unittest.TestCase): else: self.fail("Expected an AttributeError raised for 'gzip,identity'") + def test_allow_reserved_names(self): + req = swift.common.swob.Request.blank('', headers={}) + self.assertFalse(req.allow_reserved_names) + req = swift.common.swob.Request.blank('', headers={ + 'X-Allow-Reserved-Names': 'true'}) + self.assertFalse(req.allow_reserved_names) + req = swift.common.swob.Request.blank('', headers={ + 'X-Backend-Allow-Reserved-Names': 'false'}) + self.assertFalse(req.allow_reserved_names) + req = swift.common.swob.Request.blank('', headers={ + 'X-Backend-Allow-Reserved-Names': 'true'}) + self.assertTrue(req.allow_reserved_names) + class TestStatusMap(unittest.TestCase): def test_status_map(self): diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py index b43f91d055..d53f1cda40 100644 --- a/test/unit/container/test_backend.py +++ b/test/unit/container/test_backend.py @@ -36,6 +36,7 @@ from swift.container.backend import ContainerBroker, \ update_new_item_from_existing, UNSHARDED, SHARDING, SHARDED, \ COLLAPSED, SHARD_LISTING_STATES, SHARD_UPDATE_STATES from swift.common.db import DatabaseAlreadyExists, GreenDBConnection +from swift.common.request_helpers import get_reserved_name from swift.common.utils import Timestamp, encode_timestamps, hash_path, \ ShardRange, make_db_file_path from swift.common.storage_policy import POLICIES @@ -2365,6 +2366,33 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(len(listing), 2) self.assertEqual([row[0] for row in listing], ['3/0000', '3/0001']) + def test_list_objects_iter_with_reserved_name(self): + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(next(self.ts).internal, 0) + + broker.put_object( + 'foo', next(self.ts).internal, 0, 0, 0, POLICIES.default.idx) + broker.put_object( + get_reserved_name('foo'), next(self.ts).internal, 0, 0, 0, + POLICIES.default.idx) + + listing = broker.list_objects_iter(100, None, None, '', '') + self.assertEqual([row[0] for row in listing], ['foo']) + + listing = broker.list_objects_iter(100, None, None, '', '', + reverse=True) + self.assertEqual([row[0] for row in listing], ['foo']) + + listing = broker.list_objects_iter(100, None, None, '', '', + allow_reserved=True) + self.assertEqual([row[0] for row in listing], + [get_reserved_name('foo'), 'foo']) + + listing = broker.list_objects_iter(100, None, None, '', '', + reverse=True, allow_reserved=True) + self.assertEqual([row[0] for row in listing], + ['foo', get_reserved_name('foo')]) + def test_reverse_prefix_delim(self): expectations = [ { diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 01d26ab8fc..c45ec01c9b 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -26,13 +26,13 @@ from contextlib import contextmanager from io import BytesIO from shutil import rmtree from tempfile import mkdtemp -from test.unit import make_timestamp_iter, mock_timestamp_now from xml.dom import minidom from eventlet import spawn, Timeout import json import six from six import StringIO +from six.moves.urllib.parse import quote from swift import __version__ as swift_version from swift.common.header_key_dict import HeaderKeyDict @@ -43,13 +43,13 @@ from swift.container import server as container_server from swift.common import constraints from swift.common.utils import (Timestamp, mkdirs, public, replication, storage_directory, lock_parent_directory, - ShardRange) + ShardRange, RESERVED_STR) from test.unit import fake_http_connect, debug_logger, mock_check_drive from swift.common.storage_policy import (POLICIES, StoragePolicy) -from swift.common.request_helpers import get_sys_meta_prefix +from swift.common.request_helpers import get_sys_meta_prefix, get_reserved_name from test import listen_zero, annotate_failure -from test.unit import patch_policies +from test.unit import patch_policies, make_timestamp_iter, mock_timestamp_now @contextmanager @@ -78,6 +78,7 @@ class TestContainerController(unittest.TestCase): logger=self.logger) # some of the policy tests want at least two policies self.assertTrue(len(POLICIES) > 1) + self.ts = make_timestamp_iter() def tearDown(self): rmtree(os.path.dirname(self.testdir), ignore_errors=1) @@ -282,11 +283,9 @@ class TestContainerController(unittest.TestCase): self.assertIsNone(resp.headers[header]) def test_deleted_headers(self): - ts = (Timestamp(t).internal for t in - itertools.count(int(time.time()))) request_method_times = { - 'PUT': next(ts), - 'DELETE': next(ts), + 'PUT': next(self.ts).internal, + 'DELETE': next(self.ts).internal, } # setup a deleted container for method in ('PUT', 'DELETE'): @@ -547,11 +546,10 @@ class TestContainerController(unittest.TestCase): self.assertFalse('X-Backend-Storage-Policy-Index' in resp.headers) def test_PUT_no_policy_change(self): - ts = (Timestamp(t).internal for t in itertools.count(time.time())) policy = random.choice(list(POLICIES)) # Set metadata header req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': next(ts), + 'X-Timestamp': next(self.ts).internal, 'X-Backend-Storage-Policy-Index': policy.idx}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) @@ -565,7 +563,7 @@ class TestContainerController(unittest.TestCase): # now try to update w/o changing the policy for method in ('POST', 'PUT'): req = Request.blank('/sda1/p/a/c', method=method, headers={ - 'X-Timestamp': next(ts), + 'X-Timestamp': next(self.ts).internal, 'X-Backend-Storage-Policy-Index': policy.idx }) resp = req.get_response(self.controller) @@ -578,11 +576,10 @@ class TestContainerController(unittest.TestCase): str(policy.idx)) def test_PUT_bad_policy_change(self): - ts = (Timestamp(t).internal for t in itertools.count(time.time())) policy = random.choice(list(POLICIES)) # Set metadata header req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': next(ts), + 'X-Timestamp': next(self.ts).internal, 'X-Backend-Storage-Policy-Index': policy.idx}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) @@ -597,7 +594,7 @@ class TestContainerController(unittest.TestCase): for other_policy in other_policies: # now try to change it and make sure we get a conflict req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': next(ts), + 'X-Timestamp': next(self.ts).internal, 'X-Backend-Storage-Policy-Index': other_policy.idx }) resp = req.get_response(self.controller) @@ -615,10 +612,9 @@ class TestContainerController(unittest.TestCase): str(policy.idx)) def test_POST_ignores_policy_change(self): - ts = (Timestamp(t).internal for t in itertools.count(time.time())) policy = random.choice(list(POLICIES)) req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': next(ts), + 'X-Timestamp': next(self.ts).internal, 'X-Backend-Storage-Policy-Index': policy.idx}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) @@ -633,7 +629,7 @@ class TestContainerController(unittest.TestCase): for other_policy in other_policies: # now try to change it and make sure we get a conflict req = Request.blank('/sda1/p/a/c', method='POST', headers={ - 'X-Timestamp': next(ts), + 'X-Timestamp': next(self.ts).internal, 'X-Backend-Storage-Policy-Index': other_policy.idx }) resp = req.get_response(self.controller) @@ -650,11 +646,9 @@ class TestContainerController(unittest.TestCase): str(policy.idx)) def test_PUT_no_policy_for_existing_default(self): - ts = (Timestamp(t).internal for t in - itertools.count(int(time.time()))) # create a container with the default storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': next(ts), + 'X-Timestamp': next(self.ts).internal, }) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) # sanity check @@ -668,7 +662,7 @@ class TestContainerController(unittest.TestCase): # put again without specifying the storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': next(ts), + 'X-Timestamp': next(self.ts).internal, }) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) # sanity check @@ -685,11 +679,9 @@ class TestContainerController(unittest.TestCase): # during a config change restart across a multi node cluster. proxy_default = random.choice([p for p in POLICIES if not p.is_default]) - ts = (Timestamp(t).internal for t in - itertools.count(int(time.time()))) # create a container with the default storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': next(ts), + 'X-Timestamp': next(self.ts).internal, 'X-Backend-Storage-Policy-Default': int(proxy_default), }) resp = req.get_response(self.controller) @@ -704,7 +696,7 @@ class TestContainerController(unittest.TestCase): # put again without proxy specifying the different default req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': next(ts), + 'X-Timestamp': next(self.ts).internal, 'X-Backend-Storage-Policy-Default': int(POLICIES.default), }) resp = req.get_response(self.controller) @@ -718,11 +710,10 @@ class TestContainerController(unittest.TestCase): int(proxy_default)) def test_PUT_no_policy_for_existing_non_default(self): - ts = (Timestamp(t).internal for t in itertools.count(time.time())) non_default_policy = [p for p in POLICIES if not p.is_default][0] # create a container with the non-default storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': next(ts), + 'X-Timestamp': next(self.ts).internal, 'X-Backend-Storage-Policy-Index': non_default_policy.idx, }) resp = req.get_response(self.controller) @@ -737,7 +728,7 @@ class TestContainerController(unittest.TestCase): # put again without specifying the storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': next(ts), + 'X-Timestamp': next(self.ts).internal, }) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) # sanity check @@ -749,6 +740,39 @@ class TestContainerController(unittest.TestCase): self.assertEqual(resp.headers['X-Backend-Storage-Policy-Index'], str(non_default_policy.idx)) + def test_create_reserved_namespace_container(self): + path = '/sda1/p/a/%sc' % RESERVED_STR + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status, '201 Created', resp.body) + + path = '/sda1/p/a/%sc%stest' % (RESERVED_STR, RESERVED_STR) + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status, '201 Created', resp.body) + + def test_create_reserved_object_in_container(self): + # create container + path = '/sda1/p/a/c/' + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + # put null object in it + path += '%so' % RESERVED_STR + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal, + 'X-Size': 0, + 'X-Content-Type': 'application/x-test', + 'X-Etag': 'x', + }) + resp = req.get_response(self.controller) + self.assertEqual(resp.status, '400 Bad Request') + self.assertEqual(resp.body, b'Invalid reserved-namespace object ' + b'in user-namespace container') + def test_PUT_non_utf8_metadata(self): # Set metadata header req = Request.blank( @@ -4106,6 +4130,238 @@ class TestContainerController(unittest.TestCase): for item in json.loads(resp.body)], [{"name": "US~~UT~~~B"}]) + def _report_objects(self, path, objects): + req = Request.blank(path, method='PUT', headers={ + 'x-timestamp': next(self.ts).internal}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int // 100, 2, resp.body) + for obj in objects: + obj_path = path + '/%s' % obj['name'] + req = Request.blank(obj_path, method='PUT', headers={ + 'X-Timestamp': obj['timestamp'].internal, + 'X-Size': obj['bytes'], + 'X-Content-Type': obj['content_type'], + 'X-Etag': obj['hash'], + }) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int // 100, 2, resp.body) + + def _expected_listing(self, objects): + return [dict( + last_modified=o['timestamp'].isoformat, **{ + k: v for k, v in o.items() + if k != 'timestamp' + }) for o in sorted(objects, key=lambda o: o['name'])] + + def test_listing_with_reserved(self): + objects = [{ + 'name': get_reserved_name('null', 'test01'), + 'bytes': 8, + 'content_type': 'application/octet-stream', + 'hash': '70c1db56f301c9e337b0099bd4174b28', + 'timestamp': next(self.ts), + }] + path = '/sda1/p/a/%s' % get_reserved_name('null') + self._report_objects(path, objects) + + req = Request.blank(path, headers={'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), []) + + req = Request.blank(path, headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(objects)) + + def test_delimiter_with_reserved(self): + objects = [{ + 'name': get_reserved_name('null', 'test01'), + 'bytes': 8, + 'content_type': 'application/octet-stream', + 'hash': '70c1db56f301c9e337b0099bd4174b28', + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'test02'), + 'bytes': 8, + 'content_type': 'application/octet-stream', + 'hash': '70c1db56f301c9e337b0099bd4174b28', + 'timestamp': next(self.ts), + }] + path = '/sda1/p/a/%s' % get_reserved_name('null') + self._report_objects(path, objects) + + req = Request.blank(path + '?prefix=%s&delimiter=l' % + get_reserved_name('nul'), headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), []) + + req = Request.blank(path + '?prefix=%s&delimiter=l' % + get_reserved_name('nul'), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), [{ + 'subdir': '%s' % get_reserved_name('null')}]) + + req = Request.blank(path + '?prefix=%s&delimiter=%s' % ( + get_reserved_name('nul'), get_reserved_name('')), + headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), [{ + 'subdir': '%s' % get_reserved_name('null', '')}]) + + def test_markers_with_reserved(self): + objects = [{ + 'name': get_reserved_name('null', 'test01'), + 'bytes': 8, + 'content_type': 'application/octet-stream', + 'hash': '70c1db56f301c9e337b0099bd4174b28', + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'test02'), + 'bytes': 10, + 'content_type': 'application/octet-stream', + 'hash': '912ec803b2ce49e4a541068d495ab570', + 'timestamp': next(self.ts), + }] + path = '/sda1/p/a/%s' % get_reserved_name('null') + self._report_objects(path, objects) + + req = Request.blank(path + '?marker=%s' % + get_reserved_name('null', ''), headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), []) + + req = Request.blank(path + '?marker=%s' % + get_reserved_name('null', ''), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(objects)) + + req = Request.blank(path + '?marker=%s' % + quote(json.loads(resp.body)[0]['name']), headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), []) + + req = Request.blank(path + '?marker=%s' % + quote(self._expected_listing(objects)[0]['name']), + headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(objects)[1:]) + + def test_prefix_with_reserved(self): + objects = [{ + 'name': get_reserved_name('null', 'test01'), + 'bytes': 8, + 'content_type': 'application/octet-stream', + 'hash': '70c1db56f301c9e337b0099bd4174b28', + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'test02'), + 'bytes': 10, + 'content_type': 'application/octet-stream', + 'hash': '912ec803b2ce49e4a541068d495ab570', + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'foo'), + 'bytes': 12, + 'content_type': 'application/octet-stream', + 'hash': 'acbd18db4cc2f85cedef654fccc4a4d8', + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('nullish'), + 'bytes': 13, + 'content_type': 'application/octet-stream', + 'hash': '37b51d194a7513e45b56f6524f2d51f2', + 'timestamp': next(self.ts), + }] + path = '/sda1/p/a/%s' % get_reserved_name('null') + self._report_objects(path, objects) + + req = Request.blank(path + '?prefix=%s' % + get_reserved_name('null', 'test'), headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), []) + + req = Request.blank(path + '?prefix=%s' % + get_reserved_name('null', 'test'), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), + self._expected_listing(objects[:2])) + + def test_prefix_and_delim_with_reserved(self): + objects = [{ + 'name': get_reserved_name('null', 'test01'), + 'bytes': 8, + 'content_type': 'application/octet-stream', + 'hash': '70c1db56f301c9e337b0099bd4174b28', + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'test02'), + 'bytes': 10, + 'content_type': 'application/octet-stream', + 'hash': '912ec803b2ce49e4a541068d495ab570', + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('null', 'foo'), + 'bytes': 12, + 'content_type': 'application/octet-stream', + 'hash': 'acbd18db4cc2f85cedef654fccc4a4d8', + 'timestamp': next(self.ts), + }, { + 'name': get_reserved_name('nullish'), + 'bytes': 13, + 'content_type': 'application/octet-stream', + 'hash': '37b51d194a7513e45b56f6524f2d51f2', + 'timestamp': next(self.ts), + }] + path = '/sda1/p/a/%s' % get_reserved_name('null') + self._report_objects(path, objects) + + req = Request.blank(path + '?prefix=%s&delimiter=%s' % ( + get_reserved_name('null'), get_reserved_name()), headers={ + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + self.assertEqual(json.loads(resp.body), []) + + req = Request.blank(path + '?prefix=%s&delimiter=%s' % ( + get_reserved_name('null'), get_reserved_name()), headers={ + 'X-Backend-Allow-Reserved-Names': 'true', + 'Accept': 'application/json'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200, resp.body) + expected = [{'subdir': get_reserved_name('null', '')}] + \ + self._expected_listing(objects)[-1:] + self.assertEqual(json.loads(resp.body), expected) + def test_GET_delimiter_non_ascii(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', @@ -4309,7 +4565,7 @@ class TestContainerController(unittest.TestCase): self.controller.__call__({'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', - 'PATH_INFO': '\x00', + 'PATH_INFO': '/sda1/p/a/c\xd8\x3e%20/%', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', diff --git a/test/unit/helpers.py b/test/unit/helpers.py index 67dac477c5..d14e8a38cd 100644 --- a/test/unit/helpers.py +++ b/test/unit/helpers.py @@ -212,7 +212,8 @@ def setup_servers(the_object_server=object_server, extra_conf=None): obj4srv, obj5srv, obj6srv) nl = NullLogger() logging_prosv = proxy_logging.ProxyLoggingMiddleware( - listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger) + listing_formats.ListingFilter(prosrv, {}, logger=prosrv.logger), + conf, logger=prosrv.logger) prospa = spawn(wsgi.server, prolis, logging_prosv, nl, protocol=SwiftHttpProtocol, capitalize_response_headers=False) diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 23ae40700e..3e5a5f16f7 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -55,6 +55,7 @@ from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ NullLogger, storage_directory, public, replication, encode_timestamps, \ Timestamp from swift.common import constraints +from swift.common.request_helpers import get_reserved_name from swift.common.swob import Request, WsgiBytesIO from swift.common.splice import splice from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy, @@ -7147,6 +7148,60 @@ class TestObjectController(unittest.TestCase): self.assertEqual(errbuf.getvalue(), '') self.assertEqual(outbuf.getvalue()[:4], '405 ') + def test_create_reserved_namespace_object(self): + path = '/sda1/p/a/%sc/%so' % (utils.RESERVED_STR, utils.RESERVED_STR) + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal, + 'Content-Type': 'application/x-test', + 'Content-Length': 0, + }) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status, '201 Created') + + def test_create_reserved_namespace_object_in_user_container(self): + path = '/sda1/p/a/c/%so' % utils.RESERVED_STR + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal, + 'Content-Type': 'application/x-test', + 'Content-Length': 0, + }) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status, '400 Bad Request', resp.body) + self.assertEqual(resp.body, b'Invalid reserved-namespace object in ' + b'user-namespace container') + + def test_other_methods_reserved_namespace_object(self): + container = get_reserved_name('c') + obj = get_reserved_name('o', 'v1') + path = '/sda1/p/a/%s/%s' % (container, obj) + req = Request.blank(path, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal, + 'Content-Type': 'application/x-test', + 'Content-Length': 0, + }) + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status, '201 Created') + + bad_req = Request.blank('/sda1/p/a/c/%s' % obj, method='PUT', headers={ + 'X-Timestamp': next(self.ts).internal}) + resp = bad_req.get_response(self.object_controller) + self.assertEqual(resp.status, '400 Bad Request') + self.assertEqual(resp.body, b'Invalid reserved-namespace object ' + b'in user-namespace container') + + for method in ('GET', 'POST', 'DELETE'): + req.method = method + req.headers['X-Timestamp'] = next(self.ts).internal + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int // 100, 2) + + bad_req.method = method + req.headers['X-Timestamp'] = next(self.ts).internal + resp = bad_req.get_response(self.object_controller) + self.assertEqual(resp.status, '400 Bad Request') + self.assertEqual(resp.body, b'Invalid reserved-namespace object ' + b'in user-namespace container') + def test_not_utf8_and_not_logging_requests(self): inbuf = WsgiBytesIO() errbuf = StringIO() @@ -7164,7 +7219,7 @@ class TestObjectController(unittest.TestCase): env = {'REQUEST_METHOD': method, 'SCRIPT_NAME': '', - 'PATH_INFO': '/sda1/p/a/c/\x00%20/%', + 'PATH_INFO': '/sda1/p/a/c/\xd8\x3e%20/%', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 4acc962069..19a7050538 100644 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -31,6 +31,7 @@ from eventlet import Timeout import six from six import StringIO from six.moves import range +from six.moves.urllib.parse import quote if six.PY2: from email.parser import FeedParser as EmailFeedParser else: @@ -1258,10 +1259,7 @@ class TestReplicatedObjController(CommonObjectControllerMixin, log_lines = self.app.logger.get_lines_for_level('error') self.assertFalse(log_lines[1:]) self.assertIn('ERROR with Object server', log_lines[0]) - if six.PY3: - self.assertIn(req.swift_entity_path, log_lines[0]) - else: - self.assertIn(req.swift_entity_path.decode('utf-8'), log_lines[0]) + self.assertIn(quote(req.swift_entity_path), log_lines[0]) self.assertIn('re: Expect: 100-continue', log_lines[0]) def test_PUT_get_expect_errors_with_unicode_path(self): @@ -1305,11 +1303,7 @@ class TestReplicatedObjController(CommonObjectControllerMixin, log_lines = self.app.logger.get_lines_for_level('error') self.assertFalse(log_lines[1:]) self.assertIn('ERROR with Object server', log_lines[0]) - if six.PY3: - self.assertIn(req.swift_entity_path, log_lines[0]) - else: - self.assertIn(req.swift_entity_path.decode('utf-8'), - log_lines[0]) + self.assertIn(quote(req.swift_entity_path), log_lines[0]) self.assertIn('Trying to write to', log_lines[0]) do_test(Exception('Exception while sending data on connection')) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 7a481509dc..3b509c380d 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -83,7 +83,7 @@ from swift.common.swob import Request, Response, HTTPUnauthorized, \ HTTPException, HTTPBadRequest, wsgi_to_str from swift.common.storage_policy import StoragePolicy, POLICIES import swift.common.request_helpers -from swift.common.request_helpers import get_sys_meta_prefix +from swift.common.request_helpers import get_sys_meta_prefix, get_reserved_name # mocks logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) @@ -698,6 +698,29 @@ class TestProxyServer(unittest.TestCase): self.assertEqual(sorted(resp.headers['Allow'].split(', ')), [ 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'POST', 'PUT', 'UPDATE']) + def test_internal_reserved_name_request(self): + # set account info + fake_cache = FakeMemcache() + fake_cache.store[get_cache_key('a')] = {'status': 200} + app = proxy_server.Application({}, fake_cache, + container_ring=FakeRing(), + account_ring=FakeRing()) + # build internal container request + container = get_reserved_name('c') + req = Request.blank('/v1/a/%s' % container) + app.update_request(req) + + # try client request to reserved name + resp = app.handle_request(req) + self.assertEqual(resp.status_int, 412) + self.assertEqual(resp.body, b'Invalid UTF8 or contains NULL') + + # set backend header + req.headers['X-Backend-Allow-Reserved-Names'] = 'true' + with mocked_http_conn(200): + resp = app.handle_request(req) + self.assertEqual(resp.status_int, 200) + def test_calls_authorize_allow(self): called = [False] @@ -10405,7 +10428,8 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase): self.app = listing_formats.ListingFilter( proxy_server.Application(conf, FakeMemcache(), account_ring=FakeRing(), - container_ring=FakeRing())) + container_ring=FakeRing()), + {}) self.app.app.memcache = FakeMemcacheReturnsNone() def test_GET_autocreate_accept_json(self): @@ -10823,7 +10847,8 @@ class TestSocketObjectVersions(unittest.TestCase): _test_servers[0], conf, logger=_test_servers[0].logger), {}), {} - ) + ), + {}, logger=_test_servers[0].logger ) self.coro = spawn(wsgi.server, prolis, prosrv, NullLogger(), protocol=SwiftHttpProtocol)