From 8e5b38b1ddb71d631c12368e6d917beb25d3a160 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 7 Oct 2015 16:47:11 -0700 Subject: [PATCH] Expose tempurl's header restrictions via /info Also, clean up the module documentation a bit. Change-Id: Iaeb5eb264b118b78738187db9242540275e77444 --- swift/common/middleware/tempurl.py | 176 ++++++++++---------- swift/common/swob.py | 29 ++-- test/unit/common/middleware/test_tempurl.py | 71 +++++--- 3 files changed, 153 insertions(+), 123 deletions(-) diff --git a/swift/common/middleware/tempurl.py b/swift/common/middleware/tempurl.py index 59d9d44277..fe9c9f0bac 100644 --- a/swift/common/middleware/tempurl.py +++ b/swift/common/middleware/tempurl.py @@ -44,14 +44,18 @@ If the user were to share the link with all his friends, or accidentally post it on a forum, etc. the direct access would be limited to the expiration time set when the website created the link. -To create such temporary URLs, first an X-Account-Meta-Temp-URL-Key +------------ +Client Usage +------------ + +To create such temporary URLs, first an ``X-Account-Meta-Temp-URL-Key`` header must be set on the Swift account. Then, an HMAC-SHA1 (RFC 2104) -signature is generated using the HTTP method to allow (GET, PUT, -DELETE, etc.), the Unix timestamp the access should be allowed until, +signature is generated using the HTTP method to allow (``GET``, ``PUT``, +``DELETE``, etc.), the Unix timestamp the access should be allowed until, the full path to the object, and the key set on the account. -For example, here is code generating the signature for a GET for 60 -seconds on /v1/AUTH_account/container/object:: +For example, here is code generating the signature for a ``GET`` for 60 +seconds on ``/v1/AUTH_account/container/object``:: import hmac from hashlib import sha1 @@ -63,19 +67,20 @@ seconds on /v1/AUTH_account/container/object:: hmac_body = '%s\\n%s\\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() -Be certain to use the full path, from the /v1/ onward. +Be certain to use the full path, from the ``/v1/`` onward. -Let's say the sig ends up equaling -da39a3ee5e6b4b0d3255bfef95601890afd80709 and expires ends up -1323479485. Then, for example, the website could provide a link to:: +Let's say ``sig`` ends up equaling +``da39a3ee5e6b4b0d3255bfef95601890afd80709`` and ``expires`` ends up +``1323479485``. Then, for example, the website could provide a link to:: https://swift-cluster.example.com/v1/AUTH_account/container/object? temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709& temp_url_expires=1323479485 -Any alteration of the resource path or query arguments would result -in 401 Unauthorized. Similarly, a PUT where GET was the allowed method -would 401. HEAD is allowed if GET, PUT, or POST is allowed. +Any alteration of the resource path or query arguments would result in +``401 Unauthorized``. Similarly, a ``PUT`` where ``GET`` was the allowed method +would be rejected with ``401 Unauthorized``. However, ``HEAD`` is allowed if +``GET``, ``PUT``, or ``POST`` is allowed. Using this in combination with browser form post translation middleware could also allow direct-from-browser uploads to specific @@ -83,13 +88,13 @@ locations in Swift. TempURL supports both account and container level keys. Each allows up to two keys to be set, allowing key rotation without invalidating all existing -temporary URLs. Account keys are specified by X-Account-Meta-Temp-URL-Key and -X-Account-Meta-Temp-URL-Key-2, while container keys are specified by -X-Container-Meta-Temp-URL-Key and X-Container-Meta-Temp-URL-Key-2. +temporary URLs. Account keys are specified by ``X-Account-Meta-Temp-URL-Key`` +and ``X-Account-Meta-Temp-URL-Key-2``, while container keys are specified by +``X-Container-Meta-Temp-URL-Key`` and ``X-Container-Meta-Temp-URL-Key-2``. Signatures are checked against account and container keys, if present. -With GET TempURLs, a Content-Disposition header will be set on the +With ``GET`` TempURLs, a ``Content-Disposition`` header will be set on the response so that browsers will interpret this as a file attachment to be saved. The filename chosen is based on the object name, but you can override this with a filename query parameter. Modifying the @@ -100,13 +105,54 @@ above example:: temp_url_expires=1323479485&filename=My+Test+File.pdf If you do not want the object to be downloaded, you can cause -"Content-Disposition: inline" to be set on the response by adding the "inline" -parameter to the query string, like so:: +``Content-Disposition: inline`` to be set on the response by adding the +``inline`` parameter to the query string, like so:: https://swift-cluster.example.com/v1/AUTH_account/container/object? temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709& temp_url_expires=1323479485&inline +--------------------- +Cluster Configuration +--------------------- + +This middleware understands the following configuration settings: + +``incoming_remove_headers`` + A whitespace-delimited list of the headers to remove from + incoming requests. Names may optionally end with ``*`` to + indicate a prefix match. ``incoming_allow_headers`` is a + list of exceptions to these removals. + Default: ``x-timestamp`` + +``incoming_allow_headers`` + A whitespace-delimited list of the headers allowed as + exceptions to ``incoming_remove_headers``. Names may + optionally end with ``*`` to indicate a prefix match. + + Default: None + +``outgoing_remove_headers`` + A whitespace-delimited list of the headers to remove from + outgoing responses. Names may optionally end with ``*`` to + indicate a prefix match. ``outgoing_allow_headers`` is a + list of exceptions to these removals. + + Default: ``x-object-meta-*`` + +``outgoing_allow_headers`` + A whitespace-delimited list of the headers allowed as + exceptions to ``outgoing_remove_headers``. Names may + optionally end with ``*`` to indicate a prefix match. + + Default: ``x-object-meta-public-*`` + +``methods`` + A whitespace delimited list of request methods that are + allowed to be used with a temporary URL. + + Default: ``GET HEAD PUT POST DELETE`` + """ __all__ = ['TempURL', 'filter_factory', @@ -123,7 +169,8 @@ from six.moves.urllib.parse import parse_qs from six.moves.urllib.parse import urlencode from swift.proxy.controllers.base import get_account_info, get_container_info -from swift.common.swob import HeaderKeyDict, HTTPUnauthorized, HTTPBadRequest +from swift.common.swob import HeaderKeyDict, header_to_environ_key, \ + HTTPUnauthorized, HTTPBadRequest from swift.common.utils import split_path, get_valid_utf8_str, \ register_swift_info, get_hmac, streq_const_time, quote @@ -214,43 +261,6 @@ class TempURL(object): WSGI Middleware to grant temporary URLs specific access to Swift resources. See the overview for more information. - This middleware understands the following configuration settings:: - - incoming_remove_headers - The headers to remove from incoming requests. Simply a - whitespace delimited list of header names and names can - optionally end with '*' to indicate a prefix match. - incoming_allow_headers is a list of exceptions to these - removals. - Default: x-timestamp - - incoming_allow_headers - The headers allowed as exceptions to - incoming_remove_headers. Simply a whitespace delimited - list of header names and names can optionally end with - '*' to indicate a prefix match. - Default: None - - outgoing_remove_headers - The headers to remove from outgoing responses. Simply a - whitespace delimited list of header names and names can - optionally end with '*' to indicate a prefix match. - outgoing_allow_headers is a list of exceptions to these - removals. - Default: x-object-meta-* - - outgoing_allow_headers - The headers allowed as exceptions to - outgoing_remove_headers. Simply a whitespace delimited - list of header names and names can optionally end with - '*' to indicate a prefix match. - Default: x-object-meta-public-* - - methods - A whitespace delimited list of request methods that are - allowed to be used with a temporary URL. - Default: 'GET HEAD PUT POST DELETE' - The proxy logs created for any subrequests made will have swift.source set to "TU". @@ -259,25 +269,19 @@ class TempURL(object): :param conf: The configuration dict for the middleware. """ - def __init__(self, app, conf, - methods=('GET', 'HEAD', 'PUT', 'POST', 'DELETE')): + def __init__(self, app, conf): #: The next WSGI application/filter in the paste.deploy pipeline. self.app = app #: The filter configuration dict. self.conf = conf - #: The methods allowed with Temp URLs. - self.methods = methods - self.disallowed_headers = set( - 'HTTP_' + h.upper().replace('-', '_') + header_to_environ_key(h) for h in DISALLOWED_INCOMING_HEADERS.split()) - headers = DEFAULT_INCOMING_REMOVE_HEADERS - if 'incoming_remove_headers' in conf: - headers = conf['incoming_remove_headers'] - headers = \ - ['HTTP_' + h.upper().replace('-', '_') for h in headers.split()] + headers = [header_to_environ_key(h) + for h in conf.get('incoming_remove_headers', + DEFAULT_INCOMING_REMOVE_HEADERS.split())] #: Headers to remove from incoming requests. Uppercase WSGI env style, #: like `HTTP_X_PRIVATE`. self.incoming_remove_headers = [h for h in headers if h[-1] != '*'] @@ -286,11 +290,9 @@ class TempURL(object): self.incoming_remove_headers_startswith = \ [h[:-1] for h in headers if h[-1] == '*'] - headers = DEFAULT_INCOMING_ALLOW_HEADERS - if 'incoming_allow_headers' in conf: - headers = conf['incoming_allow_headers'] - headers = \ - ['HTTP_' + h.upper().replace('-', '_') for h in headers.split()] + headers = [header_to_environ_key(h) + for h in conf.get('incoming_allow_headers', + DEFAULT_INCOMING_ALLOW_HEADERS.split())] #: Headers to allow in incoming requests. Uppercase WSGI env style, #: like `HTTP_X_MATCHES_REMOVE_PREFIX_BUT_OKAY`. self.incoming_allow_headers = [h for h in headers if h[-1] != '*'] @@ -299,10 +301,9 @@ class TempURL(object): self.incoming_allow_headers_startswith = \ [h[:-1] for h in headers if h[-1] == '*'] - headers = DEFAULT_OUTGOING_REMOVE_HEADERS - if 'outgoing_remove_headers' in conf: - headers = conf['outgoing_remove_headers'] - headers = [h.title() for h in headers.split()] + headers = [h.title() + for h in conf.get('outgoing_remove_headers', + DEFAULT_OUTGOING_REMOVE_HEADERS.split())] #: Headers to remove from outgoing responses. Lowercase, like #: `x-account-meta-temp-url-key`. self.outgoing_remove_headers = [h for h in headers if h[-1] != '*'] @@ -311,10 +312,9 @@ class TempURL(object): self.outgoing_remove_headers_startswith = \ [h[:-1] for h in headers if h[-1] == '*'] - headers = DEFAULT_OUTGOING_ALLOW_HEADERS - if 'outgoing_allow_headers' in conf: - headers = conf['outgoing_allow_headers'] - headers = [h.title() for h in headers.split()] + headers = [h.title() + for h in conf.get('outgoing_allow_headers', + DEFAULT_OUTGOING_ALLOW_HEADERS.split())] #: Headers to allow in outgoing responses. Lowercase, like #: `x-matches-remove-prefix-but-okay`. self.outgoing_allow_headers = [h for h in headers if h[-1] != '*'] @@ -434,7 +434,7 @@ class TempURL(object): :param env: The WSGI environment for the request. :returns: (Account str, container str) or (None, None). """ - if env['REQUEST_METHOD'] in self.methods: + if env['REQUEST_METHOD'] in self.conf['methods']: try: ver, acc, cont, obj = split_path(env['PATH_INFO'], 4, 4, True) except ValueError: @@ -607,7 +607,15 @@ def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) - methods = conf.get('methods', 'GET HEAD PUT POST DELETE').split() - register_swift_info('tempurl', methods=methods) + defaults = { + 'methods': 'GET HEAD PUT POST DELETE', + 'incoming_remove_headers': DEFAULT_INCOMING_REMOVE_HEADERS, + 'incoming_allow_headers': DEFAULT_INCOMING_ALLOW_HEADERS, + 'outgoing_remove_headers': DEFAULT_OUTGOING_REMOVE_HEADERS, + 'outgoing_allow_headers': DEFAULT_OUTGOING_ALLOW_HEADERS, + } + info_conf = {k: conf.get(k, v).split() for k, v in defaults.items()} + register_swift_info('tempurl', **info_conf) + conf.update(info_conf) - return lambda app: TempURL(app, conf, methods=methods) + return lambda app: TempURL(app, conf) diff --git a/swift/common/swob.py b/swift/common/swob.py index a64c4cf6d1..a0a26c7c03 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -217,6 +217,15 @@ def _header_int_property(header): doc="Retrieve and set the %s header as an int" % header) +def header_to_environ_key(header_name): + header_name = 'HTTP_' + header_name.replace('-', '_').upper() + if header_name == 'HTTP_CONTENT_LENGTH': + return 'CONTENT_LENGTH' + if header_name == 'HTTP_CONTENT_TYPE': + return 'CONTENT_TYPE' + return header_name + + class HeaderEnvironProxy(MutableMapping): """ A dict-like object that proxies requests to a wsgi environ, @@ -235,30 +244,22 @@ class HeaderEnvironProxy(MutableMapping): def __len__(self): return len(self.keys()) - def _normalize(self, key): - key = 'HTTP_' + key.replace('-', '_').upper() - if key == 'HTTP_CONTENT_LENGTH': - return 'CONTENT_LENGTH' - if key == 'HTTP_CONTENT_TYPE': - return 'CONTENT_TYPE' - return key - def __getitem__(self, key): - return self.environ[self._normalize(key)] + return self.environ[header_to_environ_key(key)] def __setitem__(self, key, value): if value is None: - self.environ.pop(self._normalize(key), None) + self.environ.pop(header_to_environ_key(key), None) elif isinstance(value, six.text_type): - self.environ[self._normalize(key)] = value.encode('utf-8') + self.environ[header_to_environ_key(key)] = value.encode('utf-8') else: - self.environ[self._normalize(key)] = str(value) + self.environ[header_to_environ_key(key)] = str(value) def __contains__(self, key): - return self._normalize(key) in self.environ + return header_to_environ_key(key) in self.environ def __delitem__(self, key): - del self.environ[self._normalize(key)] + del self.environ[header_to_environ_key(key)] def keys(self): keys = [key[5:].replace('_', '-').title() diff --git a/test/unit/common/middleware/test_tempurl.py b/test/unit/common/middleware/test_tempurl.py index 170736751a..ff06eb8510 100644 --- a/test/unit/common/middleware/test_tempurl.py +++ b/test/unit/common/middleware/test_tempurl.py @@ -525,7 +525,7 @@ class TestTempURL(unittest.TestCase): self.assertTrue('Www-Authenticate' in resp.headers) def test_post_when_forbidden_by_config(self): - self.tempurl.methods.remove('POST') + self.tempurl.conf['methods'].remove('POST') method = 'POST' expires = int(time() + 86400) path = '/v1/a/c/o' @@ -543,7 +543,7 @@ class TestTempURL(unittest.TestCase): self.assertTrue('Www-Authenticate' in resp.headers) def test_delete_when_forbidden_by_config(self): - self.tempurl.methods.remove('DELETE') + self.tempurl.conf['methods'].remove('DELETE') method = 'DELETE' expires = int(time() + 86400) path = '/v1/a/c/o' @@ -1039,8 +1039,8 @@ class TestTempURL(unittest.TestCase): self.assertTrue('Www-Authenticate' in resp.headers) def test_clean_incoming_headers(self): - irh = '' - iah = '' + irh = [] + iah = [] env = {'HTTP_TEST_HEADER': 'value'} tempurl.TempURL( None, {'incoming_remove_headers': irh, @@ -1048,8 +1048,8 @@ class TestTempURL(unittest.TestCase): )._clean_incoming_headers(env) self.assertTrue('HTTP_TEST_HEADER' in env) - irh = 'test-header' - iah = '' + irh = ['test-header'] + iah = [] env = {'HTTP_TEST_HEADER': 'value'} tempurl.TempURL( None, {'incoming_remove_headers': irh, @@ -1057,8 +1057,8 @@ class TestTempURL(unittest.TestCase): )._clean_incoming_headers(env) self.assertTrue('HTTP_TEST_HEADER' not in env) - irh = 'test-header-*' - iah = '' + irh = ['test-header-*'] + iah = [] env = {'HTTP_TEST_HEADER_ONE': 'value', 'HTTP_TEST_HEADER_TWO': 'value'} tempurl.TempURL( @@ -1068,8 +1068,8 @@ class TestTempURL(unittest.TestCase): self.assertTrue('HTTP_TEST_HEADER_ONE' not in env) self.assertTrue('HTTP_TEST_HEADER_TWO' not in env) - irh = 'test-header-*' - iah = 'test-header-two' + irh = ['test-header-*'] + iah = ['test-header-two'] env = {'HTTP_TEST_HEADER_ONE': 'value', 'HTTP_TEST_HEADER_TWO': 'value'} tempurl.TempURL( @@ -1079,8 +1079,8 @@ class TestTempURL(unittest.TestCase): self.assertTrue('HTTP_TEST_HEADER_ONE' not in env) self.assertTrue('HTTP_TEST_HEADER_TWO' in env) - irh = 'test-header-* test-other-header' - iah = 'test-header-two test-header-yes-*' + irh = ['test-header-*', 'test-other-header'] + iah = ['test-header-two', 'test-header-yes-*'] env = {'HTTP_TEST_HEADER_ONE': 'value', 'HTTP_TEST_HEADER_TWO': 'value', 'HTTP_TEST_OTHER_HEADER': 'value', @@ -1097,8 +1097,8 @@ class TestTempURL(unittest.TestCase): self.assertTrue('HTTP_TEST_HEADER_YES_THIS' in env) def test_clean_outgoing_headers(self): - orh = '' - oah = '' + orh = [] + oah = [] hdrs = {'test-header': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( None, @@ -1106,8 +1106,8 @@ class TestTempURL(unittest.TestCase): )._clean_outgoing_headers(hdrs.items())) self.assertTrue('test-header' in hdrs) - orh = 'test-header' - oah = '' + orh = ['test-header'] + oah = [] hdrs = {'test-header': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( None, @@ -1115,8 +1115,8 @@ class TestTempURL(unittest.TestCase): )._clean_outgoing_headers(hdrs.items())) self.assertTrue('test-header' not in hdrs) - orh = 'test-header-*' - oah = '' + orh = ['test-header-*'] + oah = [] hdrs = {'test-header-one': 'value', 'test-header-two': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( @@ -1126,8 +1126,8 @@ class TestTempURL(unittest.TestCase): self.assertTrue('test-header-one' not in hdrs) self.assertTrue('test-header-two' not in hdrs) - orh = 'test-header-*' - oah = 'test-header-two' + orh = ['test-header-*'] + oah = ['test-header-two'] hdrs = {'test-header-one': 'value', 'test-header-two': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( @@ -1137,8 +1137,8 @@ class TestTempURL(unittest.TestCase): self.assertTrue('test-header-one' not in hdrs) self.assertTrue('test-header-two' in hdrs) - orh = 'test-header-* test-other-header' - oah = 'test-header-two test-header-yes-*' + orh = ['test-header-*', 'test-other-header'] + oah = ['test-header-two', 'test-header-yes-*'] hdrs = {'test-header-one': 'value', 'test-header-two': 'value', 'test-other-header': 'value', @@ -1170,15 +1170,36 @@ class TestSwiftInfo(unittest.TestCase): tempurl.filter_factory({}) swift_info = utils.get_swift_info() self.assertTrue('tempurl' in swift_info) - self.assertEqual(set(swift_info['tempurl']['methods']), + info = swift_info['tempurl'] + self.assertEqual(set(info['methods']), set(('GET', 'HEAD', 'PUT', 'POST', 'DELETE'))) + self.assertEqual(set(info['incoming_remove_headers']), + set(('x-timestamp',))) + self.assertEqual(set(info['incoming_allow_headers']), set()) + self.assertEqual(set(info['outgoing_remove_headers']), + set(('x-object-meta-*',))) + self.assertEqual(set(info['outgoing_allow_headers']), + set(('x-object-meta-public-*',))) def test_non_default_methods(self): - tempurl.filter_factory({'methods': 'GET HEAD PUT DELETE BREW'}) + tempurl.filter_factory({ + 'methods': 'GET HEAD PUT DELETE BREW', + 'incoming_remove_headers': '', + 'incoming_allow_headers': 'x-timestamp x-versions-location', + 'outgoing_remove_headers': 'x-*', + 'outgoing_allow_headers': 'x-object-meta-* content-type', + }) swift_info = utils.get_swift_info() self.assertTrue('tempurl' in swift_info) - self.assertEqual(set(swift_info['tempurl']['methods']), + info = swift_info['tempurl'] + self.assertEqual(set(info['methods']), set(('GET', 'HEAD', 'PUT', 'DELETE', 'BREW'))) + self.assertEqual(set(info['incoming_remove_headers']), set()) + self.assertEqual(set(info['incoming_allow_headers']), + set(('x-timestamp', 'x-versions-location'))) + self.assertEqual(set(info['outgoing_remove_headers']), set(('x-*', ))) + self.assertEqual(set(info['outgoing_allow_headers']), + set(('x-object-meta-*', 'content-type'))) if __name__ == '__main__':