Merge "Generic means for persisting system metadata."

etc/proxy-server.conf-sample

@@ -69,7 +69,7 @@
 # eventlet_debug = false
 
 [pipeline:main]
-pipeline = catch_errors healthcheck proxy-logging cache bulk slo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server
+pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk slo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server
 
 [app:proxy-server]
 use = egg:swift#proxy
@@ -517,3 +517,12 @@ use = egg:swift#slo
 
 [filter:account-quotas]
 use = egg:swift#account_quotas
+
+[filter:gatekeeper]
+use = egg:swift#gatekeeper
+# You can override the default log routing for this filter here:
+# set log_name = gatekeeper
+# set log_facility = LOG_LOCAL0
+# set log_level = INFO
+# set log_headers = false
+# set log_address = /dev/log

setup.cfg

@@ -86,6 +86,7 @@ paste.filter_factory =
     proxy_logging = swift.common.middleware.proxy_logging:filter_factory
     slo = swift.common.middleware.slo:filter_factory
     list_endpoints = swift.common.middleware.list_endpoints:filter_factory
+    gatekeeper = swift.common.middleware.gatekeeper:filter_factory
 
 [build_sphinx]
 all_files = 1

swift/account/server.py

@@ -37,6 +37,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, \
     HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \
     HTTPPreconditionFailed, HTTPConflict, Request, \
     HTTPInsufficientStorage, HTTPException
+from swift.common.request_helpers import is_sys_or_user_meta
 
 
 DATADIR = 'accounts'
@@ -152,7 +153,7 @@ class AccountController(object):
             metadata = {}
             metadata.update((key, (value, timestamp))
                             for key, value in req.headers.iteritems()
-                            if key.lower().startswith('x-account-meta-'))
+                            if is_sys_or_user_meta('account', key))
             if metadata:
                 broker.update_metadata(metadata)
             if created:
@@ -258,7 +259,7 @@ class AccountController(object):
         metadata = {}
         metadata.update((key, (value, timestamp))
                         for key, value in req.headers.iteritems()
-                        if key.lower().startswith('x-account-meta-'))
+                        if is_sys_or_user_meta('account', key))
         if metadata:
             broker.update_metadata(metadata)
         return HTTPNoContent(request=req)

swift/common/middleware/gatekeeper.py

@@ -0,0 +1,94 @@
+# Copyright (c) 2010-2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+The ``gatekeeper`` middleware imposes restrictions on the headers that
+may be included with requests and responses. Request headers are filtered
+to remove headers that should never be generated by a client. Similarly,
+response headers are filtered to remove private headers that should
+never be passed to a client.
+
+The ``gatekeeper`` middleware must always be present in the proxy server
+wsgi pipeline. It should be configured close to the start of the pipeline
+specified in ``/etc/swift/proxy-server.conf``, immediately after catch_errors
+and before any other middleware. It is essential that it is configured ahead
+of all middlewares using system metadata in order that they function
+correctly.
+
+If ``gatekeeper`` middleware is not configured in the pipeline then it will be
+automatically inserted close to the start of the pipeline by the proxy server.
+"""
+
+
+from swift.common.swob import wsgify
+from swift.common.utils import get_logger
+from swift.common.request_helpers import remove_items, get_sys_meta_prefix
+import re
+
+"""
+A list of python regular expressions that will be used to
+match against inbound request headers. Matching headers will
+be removed from the request.
+"""
+# Exclude headers starting with a sysmeta prefix.
+# If adding to this list, note that these are regex patterns,
+# so use a trailing $ to constrain to an exact header match
+# rather than prefix match.
+inbound_exclusions = [get_sys_meta_prefix('account'),
+                      get_sys_meta_prefix('container'),
+                      get_sys_meta_prefix('object')]
+# 'x-object-sysmeta' is reserved in anticipation of future support
+# for system metadata being applied to objects
+
+
+"""
+A list of python regular expressions that will be used to
+match against outbound response headers. Matching headers will
+be removed from the response.
+"""
+outbound_exclusions = inbound_exclusions
+
+
+def make_exclusion_test(exclusions):
+    expr = '|'.join(exclusions)
+    test = re.compile(expr, re.IGNORECASE)
+    return test.match
+
+
+class GatekeeperMiddleware(object):
+    def __init__(self, app, conf):
+        self.app = app
+        self.logger = get_logger(conf, log_route='gatekeeper')
+        self.inbound_condition = make_exclusion_test(inbound_exclusions)
+        self.outbound_condition = make_exclusion_test(outbound_exclusions)
+
+    @wsgify
+    def __call__(self, req):
+        removed = remove_items(req.headers, self.inbound_condition)
+        if removed:
+            self.logger.debug('removed request headers: %s' % removed)
+        resp = req.get_response(self.app)
+        removed = remove_items(resp.headers, self.outbound_condition)
+        if removed:
+            self.logger.debug('removed response headers: %s' % removed)
+        return resp
+
+
+def filter_factory(global_conf, **local_conf):
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def gatekeeper_filter(app):
+        return GatekeeperMiddleware(app, conf)
+    return gatekeeper_filter

swift/common/request_helpers.py

@@ -87,3 +87,109 @@ def split_and_validate_path(request, minsegs=1, maxsegs=None,
     except ValueError as err:
         raise HTTPBadRequest(body=str(err), request=request,
                              content_type='text/plain')
+
+
+def is_user_meta(server_type, key):
+    """
+    Tests if a header key starts with and is longer than the user
+    metadata prefix for given server type.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :param key: header key
+    :returns: True if the key satisfies the test, False otherwise
+    """
+    if len(key) <= 8 + len(server_type):
+        return False
+    return key.lower().startswith(get_user_meta_prefix(server_type))
+
+
+def is_sys_meta(server_type, key):
+    """
+    Tests if a header key starts with and is longer than the system
+    metadata prefix for given server type.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :param key: header key
+    :returns: True if the key satisfies the test, False otherwise
+    """
+    if len(key) <= 11 + len(server_type):
+        return False
+    return key.lower().startswith(get_sys_meta_prefix(server_type))
+
+
+def is_sys_or_user_meta(server_type, key):
+    """
+    Tests if a header key starts with and is longer than the user or system
+    metadata prefix for given server type.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :param key: header key
+    :returns: True if the key satisfies the test, False otherwise
+    """
+    return is_user_meta(server_type, key) or is_sys_meta(server_type, key)
+
+
+def strip_user_meta_prefix(server_type, key):
+    """
+    Removes the user metadata prefix for a given server type from the start
+    of a header key.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :param key: header key
+    :returns: stripped header key
+    """
+    return key[len(get_user_meta_prefix(server_type)):]
+
+
+def strip_sys_meta_prefix(server_type, key):
+    """
+    Removes the system metadata prefix for a given server type from the start
+    of a header key.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :param key: header key
+    :returns: stripped header key
+    """
+    return key[len(get_sys_meta_prefix(server_type)):]
+
+
+def get_user_meta_prefix(server_type):
+    """
+    Returns the prefix for user metadata headers for given server type.
+
+    This prefix defines the namespace for headers that will be persisted
+    by backend servers.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :returns: prefix string for server type's user metadata headers
+    """
+    return 'x-%s-%s-' % (server_type.lower(), 'meta')
+
+
+def get_sys_meta_prefix(server_type):
+    """
+    Returns the prefix for system metadata headers for given server type.
+
+    This prefix defines the namespace for headers that will be persisted
+    by backend servers.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :returns: prefix string for server type's system metadata headers
+    """
+    return 'x-%s-%s-' % (server_type.lower(), 'sysmeta')
+
+
+def remove_items(headers, condition):
+    """
+    Removes items from a dict whose keys satisfy
+    the given condition.
+
+    :param headers: a dict of headers
+    :param condition: a function that will be passed the header key as a
+    single argument and should return True if the header is to be removed.
+    :returns: a dict, possibly empty, of headers that have been removed
+    """
+    removed = {}
+    keys = filter(condition, headers)
+    removed.update((key, headers.pop(key)) for key in keys)
+    return removed

swift/common/wsgi.py

@@ -213,6 +213,21 @@ class PipelineWrapper(object):
         except ValueError:
             return False
 
+    def startswith(self, entry_point_name):
+        """
+        Tests if the pipeline starts with the given entry point name.
+
+        :param entry_point_name: entry point of middleware or app (Swift only)
+
+        :returns: True if entry_point_name is first in pipeline, False
+        otherwise
+        """
+        try:
+            first_ctx = self.context.filter_contexts[0]
+        except IndexError:
+            first_ctx = self.context.app_context
+        return first_ctx.entry_point_name == entry_point_name
+
     def _format_for_display(self, ctx):
         if ctx.entry_point_name:
             return ctx.entry_point_name

swift/container/server.py

@@ -26,7 +26,7 @@ import swift.common.db
 from swift.container.backend import ContainerBroker
 from swift.common.db import DatabaseAlreadyExists
 from swift.common.request_helpers import get_param, get_listing_content_type, \
-    split_and_validate_path
+    split_and_validate_path, is_sys_or_user_meta
 from swift.common.utils import get_logger, hash_path, public, \
     normalize_timestamp, storage_directory, validate_sync_to, \
     config_true_value, json, timing_stats, replication, \
@@ -266,7 +266,7 @@ class ContainerController(object):
                 (key, (value, timestamp))
                 for key, value in req.headers.iteritems()
                 if key.lower() in self.save_headers or
-                key.lower().startswith('x-container-meta-'))
+                is_sys_or_user_meta('container', key))
             if metadata:
                 if 'X-Container-Sync-To' in metadata:
                     if 'X-Container-Sync-To' not in broker.metadata or \
@@ -307,7 +307,7 @@ class ContainerController(object):
             (key, value)
             for key, (value, timestamp) in broker.metadata.iteritems()
             if value != '' and (key.lower() in self.save_headers or
-                                key.lower().startswith('x-container-meta-')))
+                                is_sys_or_user_meta('container', key)))
         headers['Content-Type'] = out_content_type
         return HTTPNoContent(request=req, headers=headers, charset='utf-8')
 
@@ -374,7 +374,7 @@ class ContainerController(object):
         }
         for key, (value, timestamp) in broker.metadata.iteritems():
             if value and (key.lower() in self.save_headers or
-                          key.lower().startswith('x-container-meta-')):
+                          is_sys_or_user_meta('container', key)):
                 resp_headers[key] = value
         ret = Response(request=req, headers=resp_headers,
                        content_type=out_content_type, charset='utf-8')
@@ -452,7 +452,7 @@ class ContainerController(object):
         metadata.update(
             (key, (value, timestamp)) for key, value in req.headers.iteritems()
             if key.lower() in self.save_headers or
-            key.lower().startswith('x-container-meta-'))
+            is_sys_or_user_meta('container', key))
         if metadata:
             if 'X-Container-Sync-To' in metadata:
                 if 'X-Container-Sync-To' not in broker.metadata or \

swift/obj/server.py

@@ -38,7 +38,7 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \
     DiskFileDeviceUnavailable, DiskFileExpired
 from swift.obj import ssync_receiver
 from swift.common.http import is_success
-from swift.common.request_helpers import split_and_validate_path
+from swift.common.request_helpers import split_and_validate_path, is_user_meta
 from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
     HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \
     HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
@@ -338,7 +338,7 @@ class ObjectController(object):
             return HTTPConflict(request=request)
         metadata = {'X-Timestamp': request.headers['x-timestamp']}
         metadata.update(val for val in request.headers.iteritems()
-                        if val[0].startswith('X-Object-Meta-'))
+                        if is_user_meta('object', val[0]))
         for header_key in self.allowed_headers:
             if header_key in request.headers:
                 header_caps = header_key.title()
@@ -422,8 +422,7 @@ class ObjectController(object):
                     'Content-Length': str(upload_size),
                 }
                 metadata.update(val for val in request.headers.iteritems()
-                                if val[0].lower().startswith('x-object-meta-')
+                                if is_user_meta('object', val[0]))
-                                and len(val[0]) > 14)
                 for header_key in (
                         request.headers.get('X-Backend-Replication-Headers') or
                         self.allowed_headers):
@@ -504,7 +503,7 @@ class ObjectController(object):
                 response.headers['Content-Type'] = metadata.get(
                     'Content-Type', 'application/octet-stream')
                 for key, value in metadata.iteritems():
-                    if key.lower().startswith('x-object-meta-') or \
+                    if is_user_meta('object', key) or \
                             key.lower() in self.allowed_headers:
                         response.headers[key] = value
                 response.etag = metadata['ETag']
@@ -545,7 +544,7 @@ class ObjectController(object):
         response.headers['Content-Type'] = metadata.get(
             'Content-Type', 'application/octet-stream')
         for key, value in metadata.iteritems():
-            if key.lower().startswith('x-object-meta-') or \
+            if is_user_meta('object', key) or \
                     key.lower() in self.allowed_headers:
                 response.headers[key] = value
         response.etag = metadata['ETag']

swift/proxy/controllers/base.py

@@ -48,6 +48,8 @@ from swift.common.http import is_informational, is_success, is_redirection, \
     HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED
 from swift.common.swob import Request, Response, HeaderKeyDict, Range, \
     HTTPException, HTTPRequestedRangeNotSatisfiable
+from swift.common.request_helpers import strip_sys_meta_prefix, \
+    strip_user_meta_prefix, is_user_meta, is_sys_meta, is_sys_or_user_meta
 
 
 def update_headers(response, headers):
@@ -106,11 +108,32 @@ def get_container_memcache_key(account, container):
     return cache_key
 
 
+def _prep_headers_to_info(headers, server_type):
+    """
+    Helper method that iterates once over a dict of headers,
+    converting all keys to lower case and separating
+    into subsets containing user metadata, system metadata
+    and other headers.
+    """
+    meta = {}
+    sysmeta = {}
+    other = {}
+    for key, val in dict(headers).iteritems():
+        lkey = key.lower()
+        if is_user_meta(server_type, lkey):
+            meta[strip_user_meta_prefix(server_type, lkey)] = val
+        elif is_sys_meta(server_type, lkey):
+            sysmeta[strip_sys_meta_prefix(server_type, lkey)] = val
+        else:
+            other[lkey] = val
+    return other, meta, sysmeta
+
+
 def headers_to_account_info(headers, status_int=HTTP_OK):
     """
     Construct a cacheable dict of account info based on response headers.
     """
-    headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
+    headers, meta, sysmeta = _prep_headers_to_info(headers, 'account')
     return {
         'status': status_int,
         # 'container_count' anomaly:
@@ -120,9 +143,8 @@ def headers_to_account_info(headers, status_int=HTTP_OK):
         'container_count': headers.get('x-account-container-count'),
         'total_object_count': headers.get('x-account-object-count'),
         'bytes': headers.get('x-account-bytes-used'),
-        'meta': dict((key[15:], value)
+        'meta': meta,
-                     for key, value in headers.iteritems()
+        'sysmeta': sysmeta
-                     if key.startswith('x-account-meta-'))
     }
 
 
@@ -130,7 +152,7 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
     """
     Construct a cacheable dict of container info based on response headers.
     """
-    headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
+    headers, meta, sysmeta = _prep_headers_to_info(headers, 'container')
     return {
         'status': status_int,
         'read_acl': headers.get('x-container-read'),
@@ -140,16 +162,12 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
         'bytes': headers.get('x-container-bytes-used'),
         'versions': headers.get('x-versions-location'),
         'cors': {
-            'allow_origin': headers.get(
+            'allow_origin': meta.get('access-control-allow-origin'),
-                'x-container-meta-access-control-allow-origin'),
+            'expose_headers': meta.get('access-control-expose-headers'),
-            'expose_headers': headers.get(
+            'max_age': meta.get('access-control-max-age')
-                'x-container-meta-access-control-expose-headers'),
-            'max_age': headers.get(
-                'x-container-meta-access-control-max-age')
         },
-        'meta': dict((key[17:], value)
+        'meta': meta,
-                     for key, value in headers.iteritems()
+        'sysmeta': sysmeta
-                     if key.startswith('x-container-meta-'))
     }
 
 
@@ -157,14 +175,12 @@ def headers_to_object_info(headers, status_int=HTTP_OK):
     """
     Construct a cacheable dict of object info based on response headers.
     """
-    headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
+    headers, meta, sysmeta = _prep_headers_to_info(headers, 'object')
     info = {'status': status_int,
             'length': headers.get('content-length'),
             'type': headers.get('content-type'),
             'etag': headers.get('etag'),
-            'meta': dict((key[14:], value)
+            'meta': meta
-                         for key, value in headers.iteritems()
-                         if key.startswith('x-object-meta-'))
             }
     return info
 
@@ -854,11 +870,10 @@ class Controller(object):
                            if k.lower().startswith(x_remove) or
                            k.lower() in self._x_remove_headers())
 
-        x_meta = 'x-%s-meta-' % st
         dst_headers.update((k.lower(), v)
                            for k, v in src_headers.iteritems()
                            if k.lower() in self.pass_through_headers or
-                           k.lower().startswith(x_meta))
+                           is_sys_or_user_meta(st, k))
 
     def generate_request_headers(self, orig_req=None, additional=None,
                                  transfer=False):

swift/proxy/controllers/obj.py

@@ -59,6 +59,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
     HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \
     HTTPServerError, HTTPServiceUnavailable, Request, Response, \
     HTTPClientDisconnect, HTTPNotImplemented, HTTPException
+from swift.common.request_helpers import is_user_meta
 
 
 def segment_listing_iter(listing):
@@ -78,7 +79,7 @@ def copy_headers_into(from_r, to_r):
     """
     pass_headers = ['x-delete-at']
     for k, v in from_r.headers.items():
-        if k.lower().startswith('x-object-meta-') or k.lower() in pass_headers:
+        if is_user_meta('object', k) or k.lower() in pass_headers:
             to_r.headers[k] = v
 
 

swift/proxy/server.py

@@ -51,7 +51,17 @@ from swift.common.swob import HTTPBadRequest, HTTPForbidden, \
 # example, 'after: ["catch_errors", "bulk"]' would install this middleware
 # after catch_errors and bulk if both were present, but if bulk were absent,
 # would just install it after catch_errors.
-required_filters = [{'name': 'catch_errors'}]
+#
+# "after_fn" (optional) a function that takes a PipelineWrapper object as its
+# single argument and returns a list of middlewares that this middleware should
+# come after. This list overrides any defined by the "after" field.
+required_filters = [
+    {'name': 'catch_errors'},
+    {'name': 'gatekeeper',
+     'after_fn': lambda pipe: (['catch_errors']
+                               if pipe.startswith("catch_errors")
+                               else [])}
+]
 
 
 class Application(object):
@@ -505,6 +515,9 @@ class Application(object):
         for filter_spec in reversed(required_filters):
             filter_name = filter_spec['name']
             if filter_name not in pipe:
+                if 'after_fn' in filter_spec:
+                    afters = filter_spec['after_fn'](pipe)
+                else:
                     afters = filter_spec.get('after', [])
                 insert_at = 0
                 for after in afters:

test/unit/account/test_server.py

@@ -26,6 +26,7 @@ import xml.dom.minidom
 from swift.common.swob import Request
 from swift.account.server import AccountController, ACCOUNT_LISTING_LIMIT
 from swift.common.utils import normalize_timestamp, replication, public
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 class TestAccountController(unittest.TestCase):
@@ -371,6 +372,67 @@ class TestAccountController(unittest.TestCase):
         self.assertEqual(resp.status_int, 204)
         self.assert_('x-account-meta-test' not in resp.headers)
 
+    def test_PUT_GET_sys_metadata(self):
+        prefix = get_sys_meta_prefix('account')
+        hdr = '%stest' % prefix
+        hdr2 = '%stest2' % prefix
+        # Set metadata header
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
+            headers={'X-Timestamp': normalize_timestamp(1),
+                     hdr.title(): 'Value'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 201)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'Value')
+        # Set another metadata header, ensuring old one doesn't disappear
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
+            headers={'X-Timestamp': normalize_timestamp(1),
+                     hdr2.title(): 'Value2'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'Value')
+        self.assertEqual(resp.headers.get(hdr2), 'Value2')
+        # Update metadata header
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
+            headers={'X-Timestamp': normalize_timestamp(3),
+                     hdr.title(): 'New Value'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 202)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'New Value')
+        # Send old update to metadata header
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
+            headers={'X-Timestamp': normalize_timestamp(2),
+                     hdr.title(): 'Old Value'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 202)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'New Value')
+        # Remove metadata header (by setting it to empty)
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
+            headers={'X-Timestamp': normalize_timestamp(4),
+                     hdr.title(): ''})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 202)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assert_(hdr not in resp.headers)
+
     def test_PUT_invalid_partition(self):
         req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'PUT',
                                                   'HTTP_X_TIMESTAMP': '1'})
@@ -435,6 +497,59 @@ class TestAccountController(unittest.TestCase):
         self.assertEqual(resp.status_int, 204)
         self.assert_('x-account-meta-test' not in resp.headers)
 
+    def test_POST_HEAD_sys_metadata(self):
+        prefix = get_sys_meta_prefix('account')
+        hdr = '%stest' % prefix
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
+            headers={'X-Timestamp': normalize_timestamp(1)})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 201)
+        # Set metadata header
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
+            headers={'X-Timestamp': normalize_timestamp(1),
+                     hdr.title(): 'Value'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'Value')
+        # Update metadata header
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
+            headers={'X-Timestamp': normalize_timestamp(3),
+                     hdr.title(): 'New Value'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'New Value')
+        # Send old update to metadata header
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
+            headers={'X-Timestamp': normalize_timestamp(2),
+                     hdr.title(): 'Old Value'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'New Value')
+        # Remove metadata header (by setting it to empty)
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
+            headers={'X-Timestamp': normalize_timestamp(4),
+                     hdr.title(): ''})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assert_(hdr not in resp.headers)
+
     def test_POST_invalid_partition(self):
         req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'POST',
                                                   'HTTP_X_TIMESTAMP': '1'})

test/unit/common/middleware/test_gatekeeper.py

@@ -0,0 +1,115 @@
+# Copyright (c) 2010-2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from swift.common.swob import Request, Response
+from swift.common.middleware import gatekeeper
+
+
+class FakeApp(object):
+    def __init__(self, headers={}):
+        self.headers = headers
+        self.req = None
+
+    def __call__(self, env, start_response):
+        self.req = Request(env)
+        return Response(request=self.req, body='FAKE APP',
+                        headers=self.headers)(env, start_response)
+
+
+class TestGatekeeper(unittest.TestCase):
+    methods = ['PUT', 'POST', 'GET', 'DELETE', 'HEAD', 'COPY', 'OPTIONS']
+
+    allowed_headers = {'xx-account-sysmeta-foo': 'value',
+                       'xx-container-sysmeta-foo': 'value',
+                       'xx-object-sysmeta-foo': 'value',
+                       'x-account-meta-foo': 'value',
+                       'x-container-meta-foo': 'value',
+                       'x-object-meta-foo': 'value',
+                       'x-timestamp-foo': 'value'}
+
+    sysmeta_headers = {'x-account-sysmeta-': 'value',
+                       'x-container-sysmeta-': 'value',
+                       'x-object-sysmeta-': 'value',
+                       'x-account-sysmeta-foo': 'value',
+                       'x-container-sysmeta-foo': 'value',
+                       'x-object-sysmeta-foo': 'value',
+                       'X-Account-Sysmeta-BAR': 'value',
+                       'X-Container-Sysmeta-BAR': 'value',
+                       'X-Object-Sysmeta-BAR': 'value'}
+
+    forbidden_headers_out = dict(sysmeta_headers)
+    forbidden_headers_in = dict(sysmeta_headers)
+
+    def _assertHeadersEqual(self, expected, actual):
+        for key in expected:
+            self.assertTrue(key.lower() in actual,
+                            '%s missing from %s' % (key, actual))
+
+    def _assertHeadersAbsent(self, unexpected, actual):
+        for key in unexpected:
+            self.assertTrue(key.lower() not in actual,
+                            '%s is in %s' % (key, actual))
+
+    def get_app(self, app, global_conf, **local_conf):
+        factory = gatekeeper.filter_factory(global_conf, **local_conf)
+        return factory(app)
+
+    def test_ok_header(self):
+        req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                            headers=self.allowed_headers)
+        fake_app = FakeApp()
+        app = self.get_app(fake_app, {})
+        resp = req.get_response(app)
+        self.assertEquals('200 OK', resp.status)
+        self.assertEquals(resp.body, 'FAKE APP')
+        self._assertHeadersEqual(self.allowed_headers, fake_app.req.headers)
+
+    def _test_reserved_header_removed_inbound(self, method):
+        headers = dict(self.forbidden_headers_in)
+        headers.update(self.allowed_headers)
+        req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method},
+                            headers=headers)
+        fake_app = FakeApp()
+        app = self.get_app(fake_app, {})
+        resp = req.get_response(app)
+        self.assertEquals('200 OK', resp.status)
+        self._assertHeadersEqual(self.allowed_headers, fake_app.req.headers)
+        self._assertHeadersAbsent(self.forbidden_headers_in,
+                                  fake_app.req.headers)
+
+    def test_reserved_header_removed_inbound(self):
+        for method in self.methods:
+            self._test_reserved_header_removed_inbound(method)
+
+    def _test_reserved_header_removed_outbound(self, method):
+        headers = dict(self.forbidden_headers_out)
+        headers.update(self.allowed_headers)
+        req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method})
+        fake_app = FakeApp(headers=headers)
+        app = self.get_app(fake_app, {})
+        resp = req.get_response(app)
+        self.assertEquals('200 OK', resp.status)
+        self._assertHeadersEqual(self.allowed_headers, resp.headers)
+        self._assertHeadersAbsent(self.forbidden_headers_out, resp.headers)
+
+    def test_reserved_header_removed_outbound(self):
+        for method in self.methods:
+            self._test_reserved_header_removed_outbound(method)
+
+
+if __name__ == '__main__':
+    unittest.main()

test/unit/common/test_request_helpers.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2010-2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for swift.common.request_helpers"""
+
+import unittest
+from swift.common.request_helpers import is_sys_meta, is_user_meta, \
+    is_sys_or_user_meta, strip_sys_meta_prefix, strip_user_meta_prefix, \
+    remove_items
+
+server_types = ['account', 'container', 'object']
+
+
+class TestRequestHelpers(unittest.TestCase):
+    def test_is_user_meta(self):
+        m_type = 'meta'
+        for st in server_types:
+            self.assertTrue(is_user_meta(st, 'x-%s-%s-foo' % (st, m_type)))
+            self.assertFalse(is_user_meta(st, 'x-%s-%s-' % (st, m_type)))
+            self.assertFalse(is_user_meta(st, 'x-%s-%sfoo' % (st, m_type)))
+
+    def test_is_sys_meta(self):
+        m_type = 'sysmeta'
+        for st in server_types:
+            self.assertTrue(is_sys_meta(st, 'x-%s-%s-foo' % (st, m_type)))
+            self.assertFalse(is_sys_meta(st, 'x-%s-%s-' % (st, m_type)))
+            self.assertFalse(is_sys_meta(st, 'x-%s-%sfoo' % (st, m_type)))
+
+    def test_is_sys_or_user_meta(self):
+        m_types = ['sysmeta', 'meta']
+        for mt in m_types:
+            for st in server_types:
+                self.assertTrue(is_sys_or_user_meta(st, 'x-%s-%s-foo'
+                                                    % (st, mt)))
+                self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%s-'
+                                                     % (st, mt)))
+                self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%sfoo'
+                                                     % (st, mt)))
+
+    def test_strip_sys_meta_prefix(self):
+        mt = 'sysmeta'
+        for st in server_types:
+            self.assertEquals(strip_sys_meta_prefix(st, 'x-%s-%s-a'
+                                                    % (st, mt)), 'a')
+
+    def test_strip_user_meta_prefix(self):
+        mt = 'meta'
+        for st in server_types:
+            self.assertEquals(strip_user_meta_prefix(st, 'x-%s-%s-a'
+                                                     % (st, mt)), 'a')
+
+    def test_remove_items(self):
+        src = {'a': 'b',
+               'c': 'd'}
+        test = lambda x: x == 'a'
+        rem = remove_items(src, test)
+        self.assertEquals(src, {'c': 'd'})
+        self.assertEquals(rem, {'a': 'b'})

test/unit/common/test_wsgi.py

@@ -35,6 +35,7 @@ from eventlet import listen
 import mock
 
 import swift.common.middleware.catch_errors
+import swift.common.middleware.gatekeeper
 import swift.proxy.server
 
 from swift.common.swob import Request
@@ -143,6 +144,9 @@ class TestWSGI(unittest.TestCase):
         # verify pipeline is catch_errors -> proxy-server
         expected = swift.common.middleware.catch_errors.CatchErrorMiddleware
         self.assert_(isinstance(app, expected))
+        app = app.app
+        expected = swift.common.middleware.gatekeeper.GatekeeperMiddleware
+        self.assert_(isinstance(app, expected))
         self.assert_(isinstance(app.app, swift.proxy.server.Application))
         # config settings applied to app instance
         self.assertEquals(0.2, app.app.conn_timeout)
@@ -706,6 +710,31 @@ class TestPipelineWrapper(unittest.TestCase):
         # filters in the pipeline.
         return [c.entry_point_name for c in self.pipe.context.filter_contexts]
 
+    def test_startswith(self):
+        self.assertTrue(self.pipe.startswith("healthcheck"))
+        self.assertFalse(self.pipe.startswith("tempurl"))
+
+    def test_startswith_no_filters(self):
+        config = """
+        [DEFAULT]
+        swift_dir = TEMPDIR
+
+        [pipeline:main]
+        pipeline = proxy-server
+
+        [app:proxy-server]
+        use = egg:swift#proxy
+        conn_timeout = 0.2
+        """
+        contents = dedent(config)
+        with temptree(['proxy-server.conf']) as t:
+            conf_file = os.path.join(t, 'proxy-server.conf')
+            with open(conf_file, 'w') as f:
+                f.write(contents.replace('TEMPDIR', t))
+            ctx = wsgi.loadcontext(loadwsgi.APP, conf_file, global_conf={})
+            pipe = wsgi.PipelineWrapper(ctx)
+        self.assertTrue(pipe.startswith('proxy'))
+
     def test_insert_filter(self):
         original_modules = ['healthcheck', 'catch_errors', None]
         self.assertEqual(self._entry_point_names(), original_modules)
@@ -789,7 +818,7 @@ class TestPipelineModification(unittest.TestCase):
         swift_dir = TEMPDIR
 
         [pipeline:main]
-        pipeline = catch_errors proxy-server
+        pipeline = catch_errors gatekeeper proxy-server
 
         [app:proxy-server]
         use = egg:swift#proxy
@@ -797,6 +826,9 @@ class TestPipelineModification(unittest.TestCase):
 
         [filter:catch_errors]
         use = egg:swift#catch_errors
+
+        [filter:gatekeeper]
+        use = egg:swift#gatekeeper
         """
 
         contents = dedent(config)
@@ -809,6 +841,7 @@ class TestPipelineModification(unittest.TestCase):
 
         self.assertEqual(self.pipeline_modules(app),
                          ['swift.common.middleware.catch_errors',
+                         'swift.common.middleware.gatekeeper',
                           'swift.proxy.server'])
 
     def test_proxy_modify_wsgi_pipeline(self):
@@ -835,8 +868,11 @@ class TestPipelineModification(unittest.TestCase):
             _fake_rings(t)
             app = wsgi.loadapp(conf_file, global_conf={})
 
-        self.assertEqual(self.pipeline_modules(app)[0],
+        self.assertEqual(self.pipeline_modules(app),
-                         'swift.common.middleware.catch_errors')
+                         ['swift.common.middleware.catch_errors',
+                         'swift.common.middleware.gatekeeper',
+                         'swift.common.middleware.healthcheck',
+                         'swift.proxy.server'])
 
     def test_proxy_modify_wsgi_pipeline_ordering(self):
         config = """
@@ -892,6 +928,69 @@ class TestPipelineModification(unittest.TestCase):
             'swift.common.middleware.tempurl',
             'swift.proxy.server'])
 
+    def _proxy_modify_wsgi_pipeline(self, pipe):
+        config = """
+        [DEFAULT]
+        swift_dir = TEMPDIR
+
+        [pipeline:main]
+        pipeline = %s
+
+        [app:proxy-server]
+        use = egg:swift#proxy
+        conn_timeout = 0.2
+
+        [filter:healthcheck]
+        use = egg:swift#healthcheck
+
+        [filter:catch_errors]
+        use = egg:swift#catch_errors
+
+        [filter:gatekeeper]
+        use = egg:swift#gatekeeper
+        """
+        config = config % (pipe,)
+        contents = dedent(config)
+        with temptree(['proxy-server.conf']) as t:
+            conf_file = os.path.join(t, 'proxy-server.conf')
+            with open(conf_file, 'w') as f:
+                f.write(contents.replace('TEMPDIR', t))
+            _fake_rings(t)
+            app = wsgi.loadapp(conf_file, global_conf={})
+        return app
+
+    def test_gatekeeper_insertion_catch_errors_configured_at_start(self):
+        # catch_errors is configured at start, gatekeeper is not configured,
+        # so gatekeeper should be inserted just after catch_errors
+        pipe = 'catch_errors healthcheck proxy-server'
+        app = self._proxy_modify_wsgi_pipeline(pipe)
+        self.assertEqual(self.pipeline_modules(app), [
+            'swift.common.middleware.catch_errors',
+            'swift.common.middleware.gatekeeper',
+            'swift.common.middleware.healthcheck',
+            'swift.proxy.server'])
+
+    def test_gatekeeper_insertion_catch_errors_configured_not_at_start(self):
+        # catch_errors is configured, gatekeeper is not configured, so
+        # gatekeeper should be inserted at start of pipeline
+        pipe = 'healthcheck catch_errors proxy-server'
+        app = self._proxy_modify_wsgi_pipeline(pipe)
+        self.assertEqual(self.pipeline_modules(app), [
+            'swift.common.middleware.gatekeeper',
+            'swift.common.middleware.healthcheck',
+            'swift.common.middleware.catch_errors',
+            'swift.proxy.server'])
+
+    def test_catch_errors_gatekeeper_configured_not_at_start(self):
+        # catch_errors is configured, gatekeeper is configured, so
+        # no change should be made to pipeline
+        pipe = 'healthcheck catch_errors gatekeeper proxy-server'
+        app = self._proxy_modify_wsgi_pipeline(pipe)
+        self.assertEqual(self.pipeline_modules(app), [
+            'swift.common.middleware.healthcheck',
+            'swift.common.middleware.catch_errors',
+            'swift.common.middleware.gatekeeper',
+            'swift.proxy.server'])
 
 if __name__ == '__main__':
     unittest.main()

test/unit/container/test_server.py

@@ -31,6 +31,7 @@ import swift.container
 from swift.container import server as container_server
 from swift.common.utils import normalize_timestamp, mkdirs, public, replication
 from test.unit import fake_http_connect
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 @contextmanager
@@ -292,6 +293,64 @@ class TestContainerController(unittest.TestCase):
         self.assertEquals(resp.status_int, 204)
         self.assert_('x-container-meta-test' not in resp.headers)
 
+    def test_PUT_GET_sys_metadata(self):
+        prefix = get_sys_meta_prefix('container')
+        key = '%sTest' % prefix
+        key2 = '%sTest2' % prefix
+        # Set metadata header
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': normalize_timestamp(1),
+                                     key: 'Value'})
+        resp = self.controller.PUT(req)
+        self.assertEquals(resp.status_int, 201)
+        req = Request.blank('/sda1/p/a/c')
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()), 'Value')
+        # Set another metadata header, ensuring old one doesn't disappear
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
+                            headers={'X-Timestamp': normalize_timestamp(1),
+                                     key2: 'Value2'})
+        resp = self.controller.POST(req)
+        self.assertEquals(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a/c')
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()), 'Value')
+        self.assertEquals(resp.headers.get(key2.lower()), 'Value2')
+        # Update metadata header
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': normalize_timestamp(3),
+                                     key: 'New Value'})
+        resp = self.controller.PUT(req)
+        self.assertEquals(resp.status_int, 202)
+        req = Request.blank('/sda1/p/a/c')
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()),
+                          'New Value')
+        # Send old update to metadata header
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': normalize_timestamp(2),
+                                     key: 'Old Value'})
+        resp = self.controller.PUT(req)
+        self.assertEquals(resp.status_int, 202)
+        req = Request.blank('/sda1/p/a/c')
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()),
+                          'New Value')
+        # Remove metadata header (by setting it to empty)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': normalize_timestamp(4),
+                                     key: ''})
+        resp = self.controller.PUT(req)
+        self.assertEquals(resp.status_int, 202)
+        req = Request.blank('/sda1/p/a/c')
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assert_(key.lower() not in resp.headers)
+
     def test_PUT_invalid_partition(self):
         req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'PUT',
                                                     'HTTP_X_TIMESTAMP': '1'})
@@ -369,6 +428,56 @@ class TestContainerController(unittest.TestCase):
         self.assertEquals(resp.status_int, 204)
         self.assert_('x-container-meta-test' not in resp.headers)
 
+    def test_POST_HEAD_sys_metadata(self):
+        prefix = get_sys_meta_prefix('container')
+        key = '%sTest' % prefix
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': normalize_timestamp(1)})
+        resp = self.controller.PUT(req)
+        self.assertEquals(resp.status_int, 201)
+        # Set metadata header
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
+                            headers={'X-Timestamp': normalize_timestamp(1),
+                                     key: 'Value'})
+        resp = self.controller.POST(req)
+        self.assertEquals(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = self.controller.HEAD(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()), 'Value')
+        # Update metadata header
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
+                            headers={'X-Timestamp': normalize_timestamp(3),
+                                     key: 'New Value'})
+        resp = self.controller.POST(req)
+        self.assertEquals(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = self.controller.HEAD(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()),
+                          'New Value')
+        # Send old update to metadata header
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
+                            headers={'X-Timestamp': normalize_timestamp(2),
+                                     key: 'Old Value'})
+        resp = self.controller.POST(req)
+        self.assertEquals(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = self.controller.HEAD(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()),
+                          'New Value')
+        # Remove metadata header (by setting it to empty)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
+                            headers={'X-Timestamp': normalize_timestamp(4),
+                                     key: ''})
+        resp = self.controller.POST(req)
+        self.assertEquals(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = self.controller.HEAD(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assert_(key.lower() not in resp.headers)
+
     def test_POST_invalid_partition(self):
         req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'POST',
                                                     'HTTP_X_TIMESTAMP': '1'})

test/unit/proxy/controllers/test_account.py

@@ -21,6 +21,7 @@ from swift.proxy import server as proxy_server
 from swift.proxy.controllers.base import headers_to_account_info
 from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN
 from test.unit import fake_http_connect, FakeRing, FakeMemcache
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 class TestAccountController(unittest.TestCase):
@@ -95,6 +96,62 @@ class TestAccountController(unittest.TestCase):
             resp = controller.POST(req)
         self.assertEquals(400, resp.status_int)
 
+    def _make_callback_func(self, context):
+        def callback(ipaddr, port, device, partition, method, path,
+                     headers=None, query_string=None, ssl=False):
+            context['method'] = method
+            context['path'] = path
+            context['headers'] = headers or {}
+        return callback
+
+    def test_sys_meta_headers_PUT(self):
+        # check that headers in sys meta namespace make it through
+        # the proxy controller
+        sys_meta_key = '%stest' % get_sys_meta_prefix('account')
+        sys_meta_key = sys_meta_key.title()
+        user_meta_key = 'X-Account-Meta-Test'
+        # allow PUTs to account...
+        self.app.allow_account_management = True
+        controller = proxy_server.AccountController(self.app, 'a')
+        context = {}
+        callback = self._make_callback_func(context)
+        hdrs_in = {sys_meta_key: 'foo',
+                   user_meta_key: 'bar',
+                   'x-timestamp': '1.0'}
+        req = Request.blank('/v1/a', headers=hdrs_in)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200, 200, give_connect=callback)):
+            controller.PUT(req)
+        self.assertEqual(context['method'], 'PUT')
+        self.assertTrue(sys_meta_key in context['headers'])
+        self.assertEqual(context['headers'][sys_meta_key], 'foo')
+        self.assertTrue(user_meta_key in context['headers'])
+        self.assertEqual(context['headers'][user_meta_key], 'bar')
+        self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+
+    def test_sys_meta_headers_POST(self):
+        # check that headers in sys meta namespace make it through
+        # the proxy controller
+        sys_meta_key = '%stest' % get_sys_meta_prefix('account')
+        sys_meta_key = sys_meta_key.title()
+        user_meta_key = 'X-Account-Meta-Test'
+        controller = proxy_server.AccountController(self.app, 'a')
+        context = {}
+        callback = self._make_callback_func(context)
+        hdrs_in = {sys_meta_key: 'foo',
+                   user_meta_key: 'bar',
+                   'x-timestamp': '1.0'}
+        req = Request.blank('/v1/a', headers=hdrs_in)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200, 200, give_connect=callback)):
+            controller.POST(req)
+        self.assertEqual(context['method'], 'POST')
+        self.assertTrue(sys_meta_key in context['headers'])
+        self.assertEqual(context['headers'][sys_meta_key], 'foo')
+        self.assertTrue(user_meta_key in context['headers'])
+        self.assertEqual(context['headers'][user_meta_key], 'bar')
+        self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+
 
 if __name__ == '__main__':
     unittest.main()

test/unit/proxy/controllers/test_base.py

@@ -20,10 +20,11 @@ from swift.proxy.controllers.base import headers_to_container_info, \
     get_container_memcache_key, get_account_info, get_account_memcache_key, \
     get_object_env_key, _get_cache_key, get_info, get_object_info, \
     Controller, GetOrHeadHandler
-from swift.common.swob import Request, HTTPException
+from swift.common.swob import Request, HTTPException, HeaderKeyDict
 from swift.common.utils import split_path
 from test.unit import fake_http_connect, FakeRing, FakeMemcache
 from swift.proxy import server as proxy_server
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 FakeResponse_status_int = 201
@@ -365,6 +366,15 @@ class TestFuncs(unittest.TestCase):
         self.assertEquals(resp['meta']['whatevs'], 14)
         self.assertEquals(resp['meta']['somethingelse'], 0)
 
+    def test_headers_to_container_info_sys_meta(self):
+        prefix = get_sys_meta_prefix('container')
+        headers = {'%sWhatevs' % prefix: 14,
+                   '%ssomethingelse' % prefix: 0}
+        resp = headers_to_container_info(headers.items(), 200)
+        self.assertEquals(len(resp['sysmeta']), 2)
+        self.assertEquals(resp['sysmeta']['whatevs'], 14)
+        self.assertEquals(resp['sysmeta']['somethingelse'], 0)
+
     def test_headers_to_container_info_values(self):
         headers = {
             'x-container-read': 'readvalue',
@@ -396,6 +406,15 @@ class TestFuncs(unittest.TestCase):
         self.assertEquals(resp['meta']['whatevs'], 14)
         self.assertEquals(resp['meta']['somethingelse'], 0)
 
+    def test_headers_to_account_info_sys_meta(self):
+        prefix = get_sys_meta_prefix('account')
+        headers = {'%sWhatevs' % prefix: 14,
+                   '%ssomethingelse' % prefix: 0}
+        resp = headers_to_account_info(headers.items(), 200)
+        self.assertEquals(len(resp['sysmeta']), 2)
+        self.assertEquals(resp['sysmeta']['whatevs'], 14)
+        self.assertEquals(resp['sysmeta']['somethingelse'], 0)
+
     def test_headers_to_account_info_values(self):
         headers = {
             'x-account-object-count': '10',
@@ -473,3 +492,43 @@ class TestFuncs(unittest.TestCase):
                                    {'Range': 'bytes=-100'})
         handler.fast_forward(20)
         self.assertEquals(handler.backend_headers['Range'], 'bytes=-80')
+
+    def test_transfer_headers_with_sysmeta(self):
+        base = Controller(self.app)
+        good_hdrs = {'x-base-sysmeta-foo': 'ok',
+                     'X-Base-sysmeta-Bar': 'also ok'}
+        bad_hdrs = {'x-base-sysmeta-': 'too short'}
+        hdrs = dict(good_hdrs)
+        hdrs.update(bad_hdrs)
+        dst_hdrs = HeaderKeyDict()
+        base.transfer_headers(hdrs, dst_hdrs)
+        self.assertEqual(HeaderKeyDict(good_hdrs), dst_hdrs)
+
+    def test_generate_request_headers(self):
+        base = Controller(self.app)
+        src_headers = {'x-remove-base-meta-owner': 'x',
+                       'x-base-meta-size': '151M',
+                       'new-owner': 'Kun'}
+        req = Request.blank('/v1/a/c/o', headers=src_headers)
+        dst_headers = base.generate_request_headers(req, transfer=True)
+        expected_headers = {'x-base-meta-owner': '',
+                            'x-base-meta-size': '151M'}
+        for k, v in expected_headers.iteritems():
+            self.assertTrue(k in dst_headers)
+            self.assertEqual(v, dst_headers[k])
+        self.assertFalse('new-owner' in dst_headers)
+
+    def test_generate_request_headers_with_sysmeta(self):
+        base = Controller(self.app)
+        good_hdrs = {'x-base-sysmeta-foo': 'ok',
+                     'X-Base-sysmeta-Bar': 'also ok'}
+        bad_hdrs = {'x-base-sysmeta-': 'too short'}
+        hdrs = dict(good_hdrs)
+        hdrs.update(bad_hdrs)
+        req = Request.blank('/v1/a/c/o', headers=hdrs)
+        dst_headers = base.generate_request_headers(req, transfer=True)
+        for k, v in good_hdrs.iteritems():
+            self.assertTrue(k.lower() in dst_headers)
+            self.assertEqual(v, dst_headers[k.lower()])
+        for k, v in bad_hdrs.iteritems():
+            self.assertFalse(k.lower() in dst_headers)

test/unit/proxy/controllers/test_container.py

@@ -20,6 +20,7 @@ from swift.common.swob import Request
 from swift.proxy import server as proxy_server
 from swift.proxy.controllers.base import headers_to_container_info
 from test.unit import fake_http_connect, FakeRing, FakeMemcache
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 class TestContainerController(unittest.TestCase):
@@ -62,6 +63,61 @@ class TestContainerController(unittest.TestCase):
         for key in owner_headers:
             self.assertTrue(key in resp.headers)
 
+    def _make_callback_func(self, context):
+        def callback(ipaddr, port, device, partition, method, path,
+                     headers=None, query_string=None, ssl=False):
+            context['method'] = method
+            context['path'] = path
+            context['headers'] = headers or {}
+        return callback
+
+    def test_sys_meta_headers_PUT(self):
+        # check that headers in sys meta namespace make it through
+        # the container controller
+        sys_meta_key = '%stest' % get_sys_meta_prefix('container')
+        sys_meta_key = sys_meta_key.title()
+        user_meta_key = 'X-Container-Meta-Test'
+        controller = proxy_server.ContainerController(self.app, 'a', 'c')
+
+        context = {}
+        callback = self._make_callback_func(context)
+        hdrs_in = {sys_meta_key: 'foo',
+                   user_meta_key: 'bar',
+                   'x-timestamp': '1.0'}
+        req = Request.blank('/v1/a/c', headers=hdrs_in)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200, 200, give_connect=callback)):
+            controller.PUT(req)
+        self.assertEqual(context['method'], 'PUT')
+        self.assertTrue(sys_meta_key in context['headers'])
+        self.assertEqual(context['headers'][sys_meta_key], 'foo')
+        self.assertTrue(user_meta_key in context['headers'])
+        self.assertEqual(context['headers'][user_meta_key], 'bar')
+        self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+
+    def test_sys_meta_headers_POST(self):
+        # check that headers in sys meta namespace make it through
+        # the container controller
+        sys_meta_key = '%stest' % get_sys_meta_prefix('container')
+        sys_meta_key = sys_meta_key.title()
+        user_meta_key = 'X-Container-Meta-Test'
+        controller = proxy_server.ContainerController(self.app, 'a', 'c')
+        context = {}
+        callback = self._make_callback_func(context)
+        hdrs_in = {sys_meta_key: 'foo',
+                   user_meta_key: 'bar',
+                   'x-timestamp': '1.0'}
+        req = Request.blank('/v1/a/c', headers=hdrs_in)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200, 200, give_connect=callback)):
+            controller.POST(req)
+        self.assertEqual(context['method'], 'POST')
+        self.assertTrue(sys_meta_key in context['headers'])
+        self.assertEqual(context['headers'][sys_meta_key], 'foo')
+        self.assertTrue(user_meta_key in context['headers'])
+        self.assertEqual(context['headers'][user_meta_key], 'bar')
+        self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+
 
 if __name__ == '__main__':
     unittest.main()

test/unit/proxy/test_server.py

@@ -340,7 +340,8 @@ class TestController(unittest.TestCase):
                               'container_count': '12345',
                               'total_object_count': None,
                               'bytes': None,
-                              'meta': {}}
+                              'meta': {},
+                              'sysmeta': {}}
             self.assertEquals(container_info,
                               self.memcache.get(cache_key))
 
@@ -366,7 +367,8 @@ class TestController(unittest.TestCase):
                             'container_count': None,  # internally keep None
                             'total_object_count': None,
                             'bytes': None,
-                            'meta': {}}
+                            'meta': {},
+                            'sysmeta': {}}
             self.assertEquals(account_info,
                               self.memcache.get(cache_key))