diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index de9517b12b..4f9260476c 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -69,7 +69,7 @@ # eventlet_debug = false [pipeline:main] -pipeline = catch_errors healthcheck proxy-logging cache bulk slo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk slo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server [app:proxy-server] use = egg:swift#proxy @@ -517,3 +517,12 @@ use = egg:swift#slo [filter:account-quotas] use = egg:swift#account_quotas + +[filter:gatekeeper] +use = egg:swift#gatekeeper +# You can override the default log routing for this filter here: +# set log_name = gatekeeper +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_headers = false +# set log_address = /dev/log diff --git a/setup.cfg b/setup.cfg index 02c115bbf8..1102b86502 100644 --- a/setup.cfg +++ b/setup.cfg @@ -86,6 +86,7 @@ paste.filter_factory = proxy_logging = swift.common.middleware.proxy_logging:filter_factory slo = swift.common.middleware.slo:filter_factory list_endpoints = swift.common.middleware.list_endpoints:filter_factory + gatekeeper = swift.common.middleware.gatekeeper:filter_factory [build_sphinx] all_files = 1 diff --git a/swift/account/server.py b/swift/account/server.py index 1c1ea0a4d2..8a0c12744d 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -37,6 +37,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, \ HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \ HTTPPreconditionFailed, HTTPConflict, Request, \ HTTPInsufficientStorage, HTTPException +from swift.common.request_helpers import is_sys_or_user_meta DATADIR = 'accounts' @@ -152,7 +153,7 @@ class AccountController(object): metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() - if key.lower().startswith('x-account-meta-')) + if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata) if created: @@ -258,7 +259,7 @@ class AccountController(object): metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() - if key.lower().startswith('x-account-meta-')) + if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata) return HTTPNoContent(request=req) diff --git a/swift/common/middleware/gatekeeper.py b/swift/common/middleware/gatekeeper.py new file mode 100644 index 0000000000..4dc67e81cb --- /dev/null +++ b/swift/common/middleware/gatekeeper.py @@ -0,0 +1,94 @@ +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +The ``gatekeeper`` middleware imposes restrictions on the headers that +may be included with requests and responses. Request headers are filtered +to remove headers that should never be generated by a client. Similarly, +response headers are filtered to remove private headers that should +never be passed to a client. + +The ``gatekeeper`` middleware must always be present in the proxy server +wsgi pipeline. It should be configured close to the start of the pipeline +specified in ``/etc/swift/proxy-server.conf``, immediately after catch_errors +and before any other middleware. It is essential that it is configured ahead +of all middlewares using system metadata in order that they function +correctly. + +If ``gatekeeper`` middleware is not configured in the pipeline then it will be +automatically inserted close to the start of the pipeline by the proxy server. +""" + + +from swift.common.swob import wsgify +from swift.common.utils import get_logger +from swift.common.request_helpers import remove_items, get_sys_meta_prefix +import re + +""" +A list of python regular expressions that will be used to +match against inbound request headers. Matching headers will +be removed from the request. +""" +# Exclude headers starting with a sysmeta prefix. +# If adding to this list, note that these are regex patterns, +# so use a trailing $ to constrain to an exact header match +# rather than prefix match. +inbound_exclusions = [get_sys_meta_prefix('account'), + get_sys_meta_prefix('container'), + get_sys_meta_prefix('object')] +# 'x-object-sysmeta' is reserved in anticipation of future support +# for system metadata being applied to objects + + +""" +A list of python regular expressions that will be used to +match against outbound response headers. Matching headers will +be removed from the response. +""" +outbound_exclusions = inbound_exclusions + + +def make_exclusion_test(exclusions): + expr = '|'.join(exclusions) + test = re.compile(expr, re.IGNORECASE) + return test.match + + +class GatekeeperMiddleware(object): + def __init__(self, app, conf): + self.app = app + self.logger = get_logger(conf, log_route='gatekeeper') + self.inbound_condition = make_exclusion_test(inbound_exclusions) + self.outbound_condition = make_exclusion_test(outbound_exclusions) + + @wsgify + def __call__(self, req): + removed = remove_items(req.headers, self.inbound_condition) + if removed: + self.logger.debug('removed request headers: %s' % removed) + resp = req.get_response(self.app) + removed = remove_items(resp.headers, self.outbound_condition) + if removed: + self.logger.debug('removed response headers: %s' % removed) + return resp + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + + def gatekeeper_filter(app): + return GatekeeperMiddleware(app, conf) + return gatekeeper_filter diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index c5584d5796..77672f1ce9 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -87,3 +87,109 @@ def split_and_validate_path(request, minsegs=1, maxsegs=None, except ValueError as err: raise HTTPBadRequest(body=str(err), request=request, content_type='text/plain') + + +def is_user_meta(server_type, key): + """ + Tests if a header key starts with and is longer than the user + metadata prefix for given server type. + + :param server_type: type of backend server i.e. [account|container|object] + :param key: header key + :returns: True if the key satisfies the test, False otherwise + """ + if len(key) <= 8 + len(server_type): + return False + return key.lower().startswith(get_user_meta_prefix(server_type)) + + +def is_sys_meta(server_type, key): + """ + Tests if a header key starts with and is longer than the system + metadata prefix for given server type. + + :param server_type: type of backend server i.e. [account|container|object] + :param key: header key + :returns: True if the key satisfies the test, False otherwise + """ + if len(key) <= 11 + len(server_type): + return False + return key.lower().startswith(get_sys_meta_prefix(server_type)) + + +def is_sys_or_user_meta(server_type, key): + """ + Tests if a header key starts with and is longer than the user or system + metadata prefix for given server type. + + :param server_type: type of backend server i.e. [account|container|object] + :param key: header key + :returns: True if the key satisfies the test, False otherwise + """ + return is_user_meta(server_type, key) or is_sys_meta(server_type, key) + + +def strip_user_meta_prefix(server_type, key): + """ + Removes the user metadata prefix for a given server type from the start + of a header key. + + :param server_type: type of backend server i.e. [account|container|object] + :param key: header key + :returns: stripped header key + """ + return key[len(get_user_meta_prefix(server_type)):] + + +def strip_sys_meta_prefix(server_type, key): + """ + Removes the system metadata prefix for a given server type from the start + of a header key. + + :param server_type: type of backend server i.e. [account|container|object] + :param key: header key + :returns: stripped header key + """ + return key[len(get_sys_meta_prefix(server_type)):] + + +def get_user_meta_prefix(server_type): + """ + Returns the prefix for user metadata headers for given server type. + + This prefix defines the namespace for headers that will be persisted + by backend servers. + + :param server_type: type of backend server i.e. [account|container|object] + :returns: prefix string for server type's user metadata headers + """ + return 'x-%s-%s-' % (server_type.lower(), 'meta') + + +def get_sys_meta_prefix(server_type): + """ + Returns the prefix for system metadata headers for given server type. + + This prefix defines the namespace for headers that will be persisted + by backend servers. + + :param server_type: type of backend server i.e. [account|container|object] + :returns: prefix string for server type's system metadata headers + """ + return 'x-%s-%s-' % (server_type.lower(), 'sysmeta') + + +def remove_items(headers, condition): + """ + Removes items from a dict whose keys satisfy + the given condition. + + :param headers: a dict of headers + :param condition: a function that will be passed the header key as a + single argument and should return True if the header is to be removed. + :returns: a dict, possibly empty, of headers that have been removed + """ + removed = {} + keys = filter(condition, headers) + removed.update((key, headers.pop(key)) for key in keys) + return removed diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index e61e1545c5..d2a75c6c0c 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -213,6 +213,21 @@ class PipelineWrapper(object): except ValueError: return False + def startswith(self, entry_point_name): + """ + Tests if the pipeline starts with the given entry point name. + + :param entry_point_name: entry point of middleware or app (Swift only) + + :returns: True if entry_point_name is first in pipeline, False + otherwise + """ + try: + first_ctx = self.context.filter_contexts[0] + except IndexError: + first_ctx = self.context.app_context + return first_ctx.entry_point_name == entry_point_name + def _format_for_display(self, ctx): if ctx.entry_point_name: return ctx.entry_point_name diff --git a/swift/container/server.py b/swift/container/server.py index 3aecff5efa..744e50e33b 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -26,7 +26,7 @@ import swift.common.db from swift.container.backend import ContainerBroker from swift.common.db import DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ - split_and_validate_path + split_and_validate_path, is_sys_or_user_meta from swift.common.utils import get_logger, hash_path, public, \ normalize_timestamp, storage_directory, validate_sync_to, \ config_true_value, json, timing_stats, replication, \ @@ -266,7 +266,7 @@ class ContainerController(object): (key, (value, timestamp)) for key, value in req.headers.iteritems() if key.lower() in self.save_headers or - key.lower().startswith('x-container-meta-')) + is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ @@ -307,7 +307,7 @@ class ContainerController(object): (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and (key.lower() in self.save_headers or - key.lower().startswith('x-container-meta-'))) + is_sys_or_user_meta('container', key))) headers['Content-Type'] = out_content_type return HTTPNoContent(request=req, headers=headers, charset='utf-8') @@ -374,7 +374,7 @@ class ContainerController(object): } for key, (value, timestamp) in broker.metadata.iteritems(): if value and (key.lower() in self.save_headers or - key.lower().startswith('x-container-meta-')): + is_sys_or_user_meta('container', key)): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') @@ -452,7 +452,7 @@ class ContainerController(object): metadata.update( (key, (value, timestamp)) for key, value in req.headers.iteritems() if key.lower() in self.save_headers or - key.lower().startswith('x-container-meta-')) + is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ diff --git a/swift/obj/server.py b/swift/obj/server.py index b22b8221f8..99a5db89cf 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -38,7 +38,7 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \ DiskFileDeviceUnavailable, DiskFileExpired from swift.obj import ssync_receiver from swift.common.http import is_success -from swift.common.request_helpers import split_and_validate_path +from swift.common.request_helpers import split_and_validate_path, is_user_meta from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \ HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \ @@ -338,7 +338,7 @@ class ObjectController(object): return HTTPConflict(request=request) metadata = {'X-Timestamp': request.headers['x-timestamp']} metadata.update(val for val in request.headers.iteritems() - if val[0].startswith('X-Object-Meta-')) + if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() @@ -422,8 +422,7 @@ class ObjectController(object): 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() - if val[0].lower().startswith('x-object-meta-') - and len(val[0]) > 14) + if is_user_meta('object', val[0])) for header_key in ( request.headers.get('X-Backend-Replication-Headers') or self.allowed_headers): @@ -504,7 +503,7 @@ class ObjectController(object): response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): - if key.lower().startswith('x-object-meta-') or \ + if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] @@ -545,7 +544,7 @@ class ObjectController(object): response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): - if key.lower().startswith('x-object-meta-') or \ + if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 0f76950836..5637841a29 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -48,6 +48,8 @@ from swift.common.http import is_informational, is_success, is_redirection, \ HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED from swift.common.swob import Request, Response, HeaderKeyDict, Range, \ HTTPException, HTTPRequestedRangeNotSatisfiable +from swift.common.request_helpers import strip_sys_meta_prefix, \ + strip_user_meta_prefix, is_user_meta, is_sys_meta, is_sys_or_user_meta def update_headers(response, headers): @@ -106,11 +108,32 @@ def get_container_memcache_key(account, container): return cache_key +def _prep_headers_to_info(headers, server_type): + """ + Helper method that iterates once over a dict of headers, + converting all keys to lower case and separating + into subsets containing user metadata, system metadata + and other headers. + """ + meta = {} + sysmeta = {} + other = {} + for key, val in dict(headers).iteritems(): + lkey = key.lower() + if is_user_meta(server_type, lkey): + meta[strip_user_meta_prefix(server_type, lkey)] = val + elif is_sys_meta(server_type, lkey): + sysmeta[strip_sys_meta_prefix(server_type, lkey)] = val + else: + other[lkey] = val + return other, meta, sysmeta + + def headers_to_account_info(headers, status_int=HTTP_OK): """ Construct a cacheable dict of account info based on response headers. """ - headers = dict((k.lower(), v) for k, v in dict(headers).iteritems()) + headers, meta, sysmeta = _prep_headers_to_info(headers, 'account') return { 'status': status_int, # 'container_count' anomaly: @@ -120,9 +143,8 @@ def headers_to_account_info(headers, status_int=HTTP_OK): 'container_count': headers.get('x-account-container-count'), 'total_object_count': headers.get('x-account-object-count'), 'bytes': headers.get('x-account-bytes-used'), - 'meta': dict((key[15:], value) - for key, value in headers.iteritems() - if key.startswith('x-account-meta-')) + 'meta': meta, + 'sysmeta': sysmeta } @@ -130,7 +152,7 @@ def headers_to_container_info(headers, status_int=HTTP_OK): """ Construct a cacheable dict of container info based on response headers. """ - headers = dict((k.lower(), v) for k, v in dict(headers).iteritems()) + headers, meta, sysmeta = _prep_headers_to_info(headers, 'container') return { 'status': status_int, 'read_acl': headers.get('x-container-read'), @@ -140,16 +162,12 @@ def headers_to_container_info(headers, status_int=HTTP_OK): 'bytes': headers.get('x-container-bytes-used'), 'versions': headers.get('x-versions-location'), 'cors': { - 'allow_origin': headers.get( - 'x-container-meta-access-control-allow-origin'), - 'expose_headers': headers.get( - 'x-container-meta-access-control-expose-headers'), - 'max_age': headers.get( - 'x-container-meta-access-control-max-age') + 'allow_origin': meta.get('access-control-allow-origin'), + 'expose_headers': meta.get('access-control-expose-headers'), + 'max_age': meta.get('access-control-max-age') }, - 'meta': dict((key[17:], value) - for key, value in headers.iteritems() - if key.startswith('x-container-meta-')) + 'meta': meta, + 'sysmeta': sysmeta } @@ -157,14 +175,12 @@ def headers_to_object_info(headers, status_int=HTTP_OK): """ Construct a cacheable dict of object info based on response headers. """ - headers = dict((k.lower(), v) for k, v in dict(headers).iteritems()) + headers, meta, sysmeta = _prep_headers_to_info(headers, 'object') info = {'status': status_int, 'length': headers.get('content-length'), 'type': headers.get('content-type'), 'etag': headers.get('etag'), - 'meta': dict((key[14:], value) - for key, value in headers.iteritems() - if key.startswith('x-object-meta-')) + 'meta': meta } return info @@ -854,11 +870,10 @@ class Controller(object): if k.lower().startswith(x_remove) or k.lower() in self._x_remove_headers()) - x_meta = 'x-%s-meta-' % st dst_headers.update((k.lower(), v) for k, v in src_headers.iteritems() if k.lower() in self.pass_through_headers or - k.lower().startswith(x_meta)) + is_sys_or_user_meta(st, k)) def generate_request_headers(self, orig_req=None, additional=None, transfer=False): diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 189e909756..603c4f6165 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -59,6 +59,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \ HTTPServerError, HTTPServiceUnavailable, Request, Response, \ HTTPClientDisconnect, HTTPNotImplemented, HTTPException +from swift.common.request_helpers import is_user_meta def segment_listing_iter(listing): @@ -78,7 +79,7 @@ def copy_headers_into(from_r, to_r): """ pass_headers = ['x-delete-at'] for k, v in from_r.headers.items(): - if k.lower().startswith('x-object-meta-') or k.lower() in pass_headers: + if is_user_meta('object', k) or k.lower() in pass_headers: to_r.headers[k] = v diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 1d5f968d3c..2a1f0e5850 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -51,7 +51,17 @@ from swift.common.swob import HTTPBadRequest, HTTPForbidden, \ # example, 'after: ["catch_errors", "bulk"]' would install this middleware # after catch_errors and bulk if both were present, but if bulk were absent, # would just install it after catch_errors. -required_filters = [{'name': 'catch_errors'}] +# +# "after_fn" (optional) a function that takes a PipelineWrapper object as its +# single argument and returns a list of middlewares that this middleware should +# come after. This list overrides any defined by the "after" field. +required_filters = [ + {'name': 'catch_errors'}, + {'name': 'gatekeeper', + 'after_fn': lambda pipe: (['catch_errors'] + if pipe.startswith("catch_errors") + else [])} +] class Application(object): @@ -505,7 +515,10 @@ class Application(object): for filter_spec in reversed(required_filters): filter_name = filter_spec['name'] if filter_name not in pipe: - afters = filter_spec.get('after', []) + if 'after_fn' in filter_spec: + afters = filter_spec['after_fn'](pipe) + else: + afters = filter_spec.get('after', []) insert_at = 0 for after in afters: try: diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index 748c3173c6..c91ff0d416 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -26,6 +26,7 @@ import xml.dom.minidom from swift.common.swob import Request from swift.account.server import AccountController, ACCOUNT_LISTING_LIMIT from swift.common.utils import normalize_timestamp, replication, public +from swift.common.request_helpers import get_sys_meta_prefix class TestAccountController(unittest.TestCase): @@ -371,6 +372,67 @@ class TestAccountController(unittest.TestCase): self.assertEqual(resp.status_int, 204) self.assert_('x-account-meta-test' not in resp.headers) + def test_PUT_GET_sys_metadata(self): + prefix = get_sys_meta_prefix('account') + hdr = '%stest' % prefix + hdr2 = '%stest2' % prefix + # Set metadata header + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(1), + hdr.title(): 'Value'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'Value') + # Set another metadata header, ensuring old one doesn't disappear + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(1), + hdr2.title(): 'Value2'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'Value') + self.assertEqual(resp.headers.get(hdr2), 'Value2') + # Update metadata header + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(3), + hdr.title(): 'New Value'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 202) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'New Value') + # Send old update to metadata header + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(2), + hdr.title(): 'Old Value'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 202) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'New Value') + # Remove metadata header (by setting it to empty) + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(4), + hdr.title(): ''}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 202) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assert_(hdr not in resp.headers) + def test_PUT_invalid_partition(self): req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) @@ -435,6 +497,59 @@ class TestAccountController(unittest.TestCase): self.assertEqual(resp.status_int, 204) self.assert_('x-account-meta-test' not in resp.headers) + def test_POST_HEAD_sys_metadata(self): + prefix = get_sys_meta_prefix('account') + hdr = '%stest' % prefix + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(1)}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + # Set metadata header + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(1), + hdr.title(): 'Value'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'Value') + # Update metadata header + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(3), + hdr.title(): 'New Value'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'New Value') + # Send old update to metadata header + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(2), + hdr.title(): 'Old Value'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'New Value') + # Remove metadata header (by setting it to empty) + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(4), + hdr.title(): ''}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assert_(hdr not in resp.headers) + def test_POST_invalid_partition(self): req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_TIMESTAMP': '1'}) diff --git a/test/unit/common/middleware/test_gatekeeper.py b/test/unit/common/middleware/test_gatekeeper.py new file mode 100644 index 0000000000..294273171a --- /dev/null +++ b/test/unit/common/middleware/test_gatekeeper.py @@ -0,0 +1,115 @@ +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from swift.common.swob import Request, Response +from swift.common.middleware import gatekeeper + + +class FakeApp(object): + def __init__(self, headers={}): + self.headers = headers + self.req = None + + def __call__(self, env, start_response): + self.req = Request(env) + return Response(request=self.req, body='FAKE APP', + headers=self.headers)(env, start_response) + + +class TestGatekeeper(unittest.TestCase): + methods = ['PUT', 'POST', 'GET', 'DELETE', 'HEAD', 'COPY', 'OPTIONS'] + + allowed_headers = {'xx-account-sysmeta-foo': 'value', + 'xx-container-sysmeta-foo': 'value', + 'xx-object-sysmeta-foo': 'value', + 'x-account-meta-foo': 'value', + 'x-container-meta-foo': 'value', + 'x-object-meta-foo': 'value', + 'x-timestamp-foo': 'value'} + + sysmeta_headers = {'x-account-sysmeta-': 'value', + 'x-container-sysmeta-': 'value', + 'x-object-sysmeta-': 'value', + 'x-account-sysmeta-foo': 'value', + 'x-container-sysmeta-foo': 'value', + 'x-object-sysmeta-foo': 'value', + 'X-Account-Sysmeta-BAR': 'value', + 'X-Container-Sysmeta-BAR': 'value', + 'X-Object-Sysmeta-BAR': 'value'} + + forbidden_headers_out = dict(sysmeta_headers) + forbidden_headers_in = dict(sysmeta_headers) + + def _assertHeadersEqual(self, expected, actual): + for key in expected: + self.assertTrue(key.lower() in actual, + '%s missing from %s' % (key, actual)) + + def _assertHeadersAbsent(self, unexpected, actual): + for key in unexpected: + self.assertTrue(key.lower() not in actual, + '%s is in %s' % (key, actual)) + + def get_app(self, app, global_conf, **local_conf): + factory = gatekeeper.filter_factory(global_conf, **local_conf) + return factory(app) + + def test_ok_header(self): + req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers=self.allowed_headers) + fake_app = FakeApp() + app = self.get_app(fake_app, {}) + resp = req.get_response(app) + self.assertEquals('200 OK', resp.status) + self.assertEquals(resp.body, 'FAKE APP') + self._assertHeadersEqual(self.allowed_headers, fake_app.req.headers) + + def _test_reserved_header_removed_inbound(self, method): + headers = dict(self.forbidden_headers_in) + headers.update(self.allowed_headers) + req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method}, + headers=headers) + fake_app = FakeApp() + app = self.get_app(fake_app, {}) + resp = req.get_response(app) + self.assertEquals('200 OK', resp.status) + self._assertHeadersEqual(self.allowed_headers, fake_app.req.headers) + self._assertHeadersAbsent(self.forbidden_headers_in, + fake_app.req.headers) + + def test_reserved_header_removed_inbound(self): + for method in self.methods: + self._test_reserved_header_removed_inbound(method) + + def _test_reserved_header_removed_outbound(self, method): + headers = dict(self.forbidden_headers_out) + headers.update(self.allowed_headers) + req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method}) + fake_app = FakeApp(headers=headers) + app = self.get_app(fake_app, {}) + resp = req.get_response(app) + self.assertEquals('200 OK', resp.status) + self._assertHeadersEqual(self.allowed_headers, resp.headers) + self._assertHeadersAbsent(self.forbidden_headers_out, resp.headers) + + def test_reserved_header_removed_outbound(self): + for method in self.methods: + self._test_reserved_header_removed_outbound(method) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_request_helpers.py b/test/unit/common/test_request_helpers.py new file mode 100644 index 0000000000..8bb382db1d --- /dev/null +++ b/test/unit/common/test_request_helpers.py @@ -0,0 +1,70 @@ +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for swift.common.request_helpers""" + +import unittest +from swift.common.request_helpers import is_sys_meta, is_user_meta, \ + is_sys_or_user_meta, strip_sys_meta_prefix, strip_user_meta_prefix, \ + remove_items + +server_types = ['account', 'container', 'object'] + + +class TestRequestHelpers(unittest.TestCase): + def test_is_user_meta(self): + m_type = 'meta' + for st in server_types: + self.assertTrue(is_user_meta(st, 'x-%s-%s-foo' % (st, m_type))) + self.assertFalse(is_user_meta(st, 'x-%s-%s-' % (st, m_type))) + self.assertFalse(is_user_meta(st, 'x-%s-%sfoo' % (st, m_type))) + + def test_is_sys_meta(self): + m_type = 'sysmeta' + for st in server_types: + self.assertTrue(is_sys_meta(st, 'x-%s-%s-foo' % (st, m_type))) + self.assertFalse(is_sys_meta(st, 'x-%s-%s-' % (st, m_type))) + self.assertFalse(is_sys_meta(st, 'x-%s-%sfoo' % (st, m_type))) + + def test_is_sys_or_user_meta(self): + m_types = ['sysmeta', 'meta'] + for mt in m_types: + for st in server_types: + self.assertTrue(is_sys_or_user_meta(st, 'x-%s-%s-foo' + % (st, mt))) + self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%s-' + % (st, mt))) + self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%sfoo' + % (st, mt))) + + def test_strip_sys_meta_prefix(self): + mt = 'sysmeta' + for st in server_types: + self.assertEquals(strip_sys_meta_prefix(st, 'x-%s-%s-a' + % (st, mt)), 'a') + + def test_strip_user_meta_prefix(self): + mt = 'meta' + for st in server_types: + self.assertEquals(strip_user_meta_prefix(st, 'x-%s-%s-a' + % (st, mt)), 'a') + + def test_remove_items(self): + src = {'a': 'b', + 'c': 'd'} + test = lambda x: x == 'a' + rem = remove_items(src, test) + self.assertEquals(src, {'c': 'd'}) + self.assertEquals(rem, {'a': 'b'}) diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index d1e8f5130c..6172c3fda9 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -35,6 +35,7 @@ from eventlet import listen import mock import swift.common.middleware.catch_errors +import swift.common.middleware.gatekeeper import swift.proxy.server from swift.common.swob import Request @@ -143,6 +144,9 @@ class TestWSGI(unittest.TestCase): # verify pipeline is catch_errors -> proxy-server expected = swift.common.middleware.catch_errors.CatchErrorMiddleware self.assert_(isinstance(app, expected)) + app = app.app + expected = swift.common.middleware.gatekeeper.GatekeeperMiddleware + self.assert_(isinstance(app, expected)) self.assert_(isinstance(app.app, swift.proxy.server.Application)) # config settings applied to app instance self.assertEquals(0.2, app.app.conn_timeout) @@ -706,6 +710,31 @@ class TestPipelineWrapper(unittest.TestCase): # filters in the pipeline. return [c.entry_point_name for c in self.pipe.context.filter_contexts] + def test_startswith(self): + self.assertTrue(self.pipe.startswith("healthcheck")) + self.assertFalse(self.pipe.startswith("tempurl")) + + def test_startswith_no_filters(self): + config = """ + [DEFAULT] + swift_dir = TEMPDIR + + [pipeline:main] + pipeline = proxy-server + + [app:proxy-server] + use = egg:swift#proxy + conn_timeout = 0.2 + """ + contents = dedent(config) + with temptree(['proxy-server.conf']) as t: + conf_file = os.path.join(t, 'proxy-server.conf') + with open(conf_file, 'w') as f: + f.write(contents.replace('TEMPDIR', t)) + ctx = wsgi.loadcontext(loadwsgi.APP, conf_file, global_conf={}) + pipe = wsgi.PipelineWrapper(ctx) + self.assertTrue(pipe.startswith('proxy')) + def test_insert_filter(self): original_modules = ['healthcheck', 'catch_errors', None] self.assertEqual(self._entry_point_names(), original_modules) @@ -789,7 +818,7 @@ class TestPipelineModification(unittest.TestCase): swift_dir = TEMPDIR [pipeline:main] - pipeline = catch_errors proxy-server + pipeline = catch_errors gatekeeper proxy-server [app:proxy-server] use = egg:swift#proxy @@ -797,6 +826,9 @@ class TestPipelineModification(unittest.TestCase): [filter:catch_errors] use = egg:swift#catch_errors + + [filter:gatekeeper] + use = egg:swift#gatekeeper """ contents = dedent(config) @@ -809,6 +841,7 @@ class TestPipelineModification(unittest.TestCase): self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', + 'swift.common.middleware.gatekeeper', 'swift.proxy.server']) def test_proxy_modify_wsgi_pipeline(self): @@ -835,8 +868,11 @@ class TestPipelineModification(unittest.TestCase): _fake_rings(t) app = wsgi.loadapp(conf_file, global_conf={}) - self.assertEqual(self.pipeline_modules(app)[0], - 'swift.common.middleware.catch_errors') + self.assertEqual(self.pipeline_modules(app), + ['swift.common.middleware.catch_errors', + 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.healthcheck', + 'swift.proxy.server']) def test_proxy_modify_wsgi_pipeline_ordering(self): config = """ @@ -892,6 +928,69 @@ class TestPipelineModification(unittest.TestCase): 'swift.common.middleware.tempurl', 'swift.proxy.server']) + def _proxy_modify_wsgi_pipeline(self, pipe): + config = """ + [DEFAULT] + swift_dir = TEMPDIR + + [pipeline:main] + pipeline = %s + + [app:proxy-server] + use = egg:swift#proxy + conn_timeout = 0.2 + + [filter:healthcheck] + use = egg:swift#healthcheck + + [filter:catch_errors] + use = egg:swift#catch_errors + + [filter:gatekeeper] + use = egg:swift#gatekeeper + """ + config = config % (pipe,) + contents = dedent(config) + with temptree(['proxy-server.conf']) as t: + conf_file = os.path.join(t, 'proxy-server.conf') + with open(conf_file, 'w') as f: + f.write(contents.replace('TEMPDIR', t)) + _fake_rings(t) + app = wsgi.loadapp(conf_file, global_conf={}) + return app + + def test_gatekeeper_insertion_catch_errors_configured_at_start(self): + # catch_errors is configured at start, gatekeeper is not configured, + # so gatekeeper should be inserted just after catch_errors + pipe = 'catch_errors healthcheck proxy-server' + app = self._proxy_modify_wsgi_pipeline(pipe) + self.assertEqual(self.pipeline_modules(app), [ + 'swift.common.middleware.catch_errors', + 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.healthcheck', + 'swift.proxy.server']) + + def test_gatekeeper_insertion_catch_errors_configured_not_at_start(self): + # catch_errors is configured, gatekeeper is not configured, so + # gatekeeper should be inserted at start of pipeline + pipe = 'healthcheck catch_errors proxy-server' + app = self._proxy_modify_wsgi_pipeline(pipe) + self.assertEqual(self.pipeline_modules(app), [ + 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.healthcheck', + 'swift.common.middleware.catch_errors', + 'swift.proxy.server']) + + def test_catch_errors_gatekeeper_configured_not_at_start(self): + # catch_errors is configured, gatekeeper is configured, so + # no change should be made to pipeline + pipe = 'healthcheck catch_errors gatekeeper proxy-server' + app = self._proxy_modify_wsgi_pipeline(pipe) + self.assertEqual(self.pipeline_modules(app), [ + 'swift.common.middleware.healthcheck', + 'swift.common.middleware.catch_errors', + 'swift.common.middleware.gatekeeper', + 'swift.proxy.server']) if __name__ == '__main__': unittest.main() diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 7662006fec..a65e5d83ad 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -31,6 +31,7 @@ import swift.container from swift.container import server as container_server from swift.common.utils import normalize_timestamp, mkdirs, public, replication from test.unit import fake_http_connect +from swift.common.request_helpers import get_sys_meta_prefix @contextmanager @@ -292,6 +293,64 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 204) self.assert_('x-container-meta-test' not in resp.headers) + def test_PUT_GET_sys_metadata(self): + prefix = get_sys_meta_prefix('container') + key = '%sTest' % prefix + key2 = '%sTest2' % prefix + # Set metadata header + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(1), + key: 'Value'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c') + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), 'Value') + # Set another metadata header, ensuring old one doesn't disappear + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(1), + key2: 'Value2'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c') + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), 'Value') + self.assertEquals(resp.headers.get(key2.lower()), 'Value2') + # Update metadata header + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(3), + key: 'New Value'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 202) + req = Request.blank('/sda1/p/a/c') + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), + 'New Value') + # Send old update to metadata header + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(2), + key: 'Old Value'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 202) + req = Request.blank('/sda1/p/a/c') + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), + 'New Value') + # Remove metadata header (by setting it to empty) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(4), + key: ''}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 202) + req = Request.blank('/sda1/p/a/c') + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + self.assert_(key.lower() not in resp.headers) + def test_PUT_invalid_partition(self): req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) @@ -369,6 +428,56 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 204) self.assert_('x-container-meta-test' not in resp.headers) + def test_POST_HEAD_sys_metadata(self): + prefix = get_sys_meta_prefix('container') + key = '%sTest' % prefix + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(1)}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + # Set metadata header + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(1), + key: 'Value'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), 'Value') + # Update metadata header + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(3), + key: 'New Value'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), + 'New Value') + # Send old update to metadata header + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(2), + key: 'Old Value'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), + 'New Value') + # Remove metadata header (by setting it to empty) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(4), + key: ''}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 204) + self.assert_(key.lower() not in resp.headers) + def test_POST_invalid_partition(self): req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_TIMESTAMP': '1'}) diff --git a/test/unit/proxy/controllers/test_account.py b/test/unit/proxy/controllers/test_account.py index 29ed09bd62..eefd57dd28 100644 --- a/test/unit/proxy/controllers/test_account.py +++ b/test/unit/proxy/controllers/test_account.py @@ -21,6 +21,7 @@ from swift.proxy import server as proxy_server from swift.proxy.controllers.base import headers_to_account_info from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN from test.unit import fake_http_connect, FakeRing, FakeMemcache +from swift.common.request_helpers import get_sys_meta_prefix class TestAccountController(unittest.TestCase): @@ -95,6 +96,62 @@ class TestAccountController(unittest.TestCase): resp = controller.POST(req) self.assertEquals(400, resp.status_int) + def _make_callback_func(self, context): + def callback(ipaddr, port, device, partition, method, path, + headers=None, query_string=None, ssl=False): + context['method'] = method + context['path'] = path + context['headers'] = headers or {} + return callback + + def test_sys_meta_headers_PUT(self): + # check that headers in sys meta namespace make it through + # the proxy controller + sys_meta_key = '%stest' % get_sys_meta_prefix('account') + sys_meta_key = sys_meta_key.title() + user_meta_key = 'X-Account-Meta-Test' + # allow PUTs to account... + self.app.allow_account_management = True + controller = proxy_server.AccountController(self.app, 'a') + context = {} + callback = self._make_callback_func(context) + hdrs_in = {sys_meta_key: 'foo', + user_meta_key: 'bar', + 'x-timestamp': '1.0'} + req = Request.blank('/v1/a', headers=hdrs_in) + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, give_connect=callback)): + controller.PUT(req) + self.assertEqual(context['method'], 'PUT') + self.assertTrue(sys_meta_key in context['headers']) + self.assertEqual(context['headers'][sys_meta_key], 'foo') + self.assertTrue(user_meta_key in context['headers']) + self.assertEqual(context['headers'][user_meta_key], 'bar') + self.assertNotEqual(context['headers']['x-timestamp'], '1.0') + + def test_sys_meta_headers_POST(self): + # check that headers in sys meta namespace make it through + # the proxy controller + sys_meta_key = '%stest' % get_sys_meta_prefix('account') + sys_meta_key = sys_meta_key.title() + user_meta_key = 'X-Account-Meta-Test' + controller = proxy_server.AccountController(self.app, 'a') + context = {} + callback = self._make_callback_func(context) + hdrs_in = {sys_meta_key: 'foo', + user_meta_key: 'bar', + 'x-timestamp': '1.0'} + req = Request.blank('/v1/a', headers=hdrs_in) + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, give_connect=callback)): + controller.POST(req) + self.assertEqual(context['method'], 'POST') + self.assertTrue(sys_meta_key in context['headers']) + self.assertEqual(context['headers'][sys_meta_key], 'foo') + self.assertTrue(user_meta_key in context['headers']) + self.assertEqual(context['headers'][user_meta_key], 'bar') + self.assertNotEqual(context['headers']['x-timestamp'], '1.0') + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index e72eedcf32..0c94f90171 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -20,10 +20,11 @@ from swift.proxy.controllers.base import headers_to_container_info, \ get_container_memcache_key, get_account_info, get_account_memcache_key, \ get_object_env_key, _get_cache_key, get_info, get_object_info, \ Controller, GetOrHeadHandler -from swift.common.swob import Request, HTTPException +from swift.common.swob import Request, HTTPException, HeaderKeyDict from swift.common.utils import split_path from test.unit import fake_http_connect, FakeRing, FakeMemcache from swift.proxy import server as proxy_server +from swift.common.request_helpers import get_sys_meta_prefix FakeResponse_status_int = 201 @@ -365,6 +366,15 @@ class TestFuncs(unittest.TestCase): self.assertEquals(resp['meta']['whatevs'], 14) self.assertEquals(resp['meta']['somethingelse'], 0) + def test_headers_to_container_info_sys_meta(self): + prefix = get_sys_meta_prefix('container') + headers = {'%sWhatevs' % prefix: 14, + '%ssomethingelse' % prefix: 0} + resp = headers_to_container_info(headers.items(), 200) + self.assertEquals(len(resp['sysmeta']), 2) + self.assertEquals(resp['sysmeta']['whatevs'], 14) + self.assertEquals(resp['sysmeta']['somethingelse'], 0) + def test_headers_to_container_info_values(self): headers = { 'x-container-read': 'readvalue', @@ -396,6 +406,15 @@ class TestFuncs(unittest.TestCase): self.assertEquals(resp['meta']['whatevs'], 14) self.assertEquals(resp['meta']['somethingelse'], 0) + def test_headers_to_account_info_sys_meta(self): + prefix = get_sys_meta_prefix('account') + headers = {'%sWhatevs' % prefix: 14, + '%ssomethingelse' % prefix: 0} + resp = headers_to_account_info(headers.items(), 200) + self.assertEquals(len(resp['sysmeta']), 2) + self.assertEquals(resp['sysmeta']['whatevs'], 14) + self.assertEquals(resp['sysmeta']['somethingelse'], 0) + def test_headers_to_account_info_values(self): headers = { 'x-account-object-count': '10', @@ -473,3 +492,43 @@ class TestFuncs(unittest.TestCase): {'Range': 'bytes=-100'}) handler.fast_forward(20) self.assertEquals(handler.backend_headers['Range'], 'bytes=-80') + + def test_transfer_headers_with_sysmeta(self): + base = Controller(self.app) + good_hdrs = {'x-base-sysmeta-foo': 'ok', + 'X-Base-sysmeta-Bar': 'also ok'} + bad_hdrs = {'x-base-sysmeta-': 'too short'} + hdrs = dict(good_hdrs) + hdrs.update(bad_hdrs) + dst_hdrs = HeaderKeyDict() + base.transfer_headers(hdrs, dst_hdrs) + self.assertEqual(HeaderKeyDict(good_hdrs), dst_hdrs) + + def test_generate_request_headers(self): + base = Controller(self.app) + src_headers = {'x-remove-base-meta-owner': 'x', + 'x-base-meta-size': '151M', + 'new-owner': 'Kun'} + req = Request.blank('/v1/a/c/o', headers=src_headers) + dst_headers = base.generate_request_headers(req, transfer=True) + expected_headers = {'x-base-meta-owner': '', + 'x-base-meta-size': '151M'} + for k, v in expected_headers.iteritems(): + self.assertTrue(k in dst_headers) + self.assertEqual(v, dst_headers[k]) + self.assertFalse('new-owner' in dst_headers) + + def test_generate_request_headers_with_sysmeta(self): + base = Controller(self.app) + good_hdrs = {'x-base-sysmeta-foo': 'ok', + 'X-Base-sysmeta-Bar': 'also ok'} + bad_hdrs = {'x-base-sysmeta-': 'too short'} + hdrs = dict(good_hdrs) + hdrs.update(bad_hdrs) + req = Request.blank('/v1/a/c/o', headers=hdrs) + dst_headers = base.generate_request_headers(req, transfer=True) + for k, v in good_hdrs.iteritems(): + self.assertTrue(k.lower() in dst_headers) + self.assertEqual(v, dst_headers[k.lower()]) + for k, v in bad_hdrs.iteritems(): + self.assertFalse(k.lower() in dst_headers) diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py index e98e04fe8d..7c8ecf7075 100644 --- a/test/unit/proxy/controllers/test_container.py +++ b/test/unit/proxy/controllers/test_container.py @@ -20,6 +20,7 @@ from swift.common.swob import Request from swift.proxy import server as proxy_server from swift.proxy.controllers.base import headers_to_container_info from test.unit import fake_http_connect, FakeRing, FakeMemcache +from swift.common.request_helpers import get_sys_meta_prefix class TestContainerController(unittest.TestCase): @@ -62,6 +63,61 @@ class TestContainerController(unittest.TestCase): for key in owner_headers: self.assertTrue(key in resp.headers) + def _make_callback_func(self, context): + def callback(ipaddr, port, device, partition, method, path, + headers=None, query_string=None, ssl=False): + context['method'] = method + context['path'] = path + context['headers'] = headers or {} + return callback + + def test_sys_meta_headers_PUT(self): + # check that headers in sys meta namespace make it through + # the container controller + sys_meta_key = '%stest' % get_sys_meta_prefix('container') + sys_meta_key = sys_meta_key.title() + user_meta_key = 'X-Container-Meta-Test' + controller = proxy_server.ContainerController(self.app, 'a', 'c') + + context = {} + callback = self._make_callback_func(context) + hdrs_in = {sys_meta_key: 'foo', + user_meta_key: 'bar', + 'x-timestamp': '1.0'} + req = Request.blank('/v1/a/c', headers=hdrs_in) + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, give_connect=callback)): + controller.PUT(req) + self.assertEqual(context['method'], 'PUT') + self.assertTrue(sys_meta_key in context['headers']) + self.assertEqual(context['headers'][sys_meta_key], 'foo') + self.assertTrue(user_meta_key in context['headers']) + self.assertEqual(context['headers'][user_meta_key], 'bar') + self.assertNotEqual(context['headers']['x-timestamp'], '1.0') + + def test_sys_meta_headers_POST(self): + # check that headers in sys meta namespace make it through + # the container controller + sys_meta_key = '%stest' % get_sys_meta_prefix('container') + sys_meta_key = sys_meta_key.title() + user_meta_key = 'X-Container-Meta-Test' + controller = proxy_server.ContainerController(self.app, 'a', 'c') + context = {} + callback = self._make_callback_func(context) + hdrs_in = {sys_meta_key: 'foo', + user_meta_key: 'bar', + 'x-timestamp': '1.0'} + req = Request.blank('/v1/a/c', headers=hdrs_in) + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, give_connect=callback)): + controller.POST(req) + self.assertEqual(context['method'], 'POST') + self.assertTrue(sys_meta_key in context['headers']) + self.assertEqual(context['headers'][sys_meta_key], 'foo') + self.assertTrue(user_meta_key in context['headers']) + self.assertEqual(context['headers'][user_meta_key], 'bar') + self.assertNotEqual(context['headers']['x-timestamp'], '1.0') + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 8803e4a797..80a13155bd 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -340,7 +340,8 @@ class TestController(unittest.TestCase): 'container_count': '12345', 'total_object_count': None, 'bytes': None, - 'meta': {}} + 'meta': {}, + 'sysmeta': {}} self.assertEquals(container_info, self.memcache.get(cache_key)) @@ -366,7 +367,8 @@ class TestController(unittest.TestCase): 'container_count': None, # internally keep None 'total_object_count': None, 'bytes': None, - 'meta': {}} + 'meta': {}, + 'sysmeta': {}} self.assertEquals(account_info, self.memcache.get(cache_key))