openstack/swift
Code Issues Proposed changes

Merge "Allow 2 TempURL keys per account."

Browse Source
This commit is contained in:
Jenkins 2013-05-12 01:35:54 +00:00 committed by Gerrit Code Review
commit 7d625f6ea4
2 changed files with 117 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
swift/common/middleware/tempurl.py
View File

@ -66,9 +66,15 @@ Using this in combination with browser form post translation
middleware could also allow direct-from-browser uploads to specific
locations in Swift.
Note that changing the X-Account-Meta-Temp-URL-Key will invalidate
any previously generated temporary URLs within 60 seconds (the
memcache time for the key).
TempURL supports up to two keys, specified by X-Account-Meta-Temp-URL-Key and
X-Account-Meta-Temp-URL-Key-2. Signatures are checked against both keys, if
present. This is to allow for key rotation without invalidating all existing
temporary URLs.
Note that changing either X-Account-Meta-Temp-URL-Key or
X-Account-Meta-Temp-URL-Key-2 will invalidate any previously generated
temporary URLs signed with that key within 60 seconds (the memcache lifetime
for the key). It is not instantaneous.
With GET TempURLs, a Content-Disposition header will be set on the
response so that browsers will interpret this as a file attachment to
@ -246,20 +252,20 @@ class TempURL(object):
account = self._get_account(env)
if not account:
return self._invalid(env, start_response)
key = self._get_key(env, account)
if not key:
keys = self._get_keys(env, account)
if not keys:
return self._invalid(env, start_response)
if env['REQUEST_METHOD'] == 'HEAD':
hmac_val = self._get_hmac(env, temp_url_expires, key,
request_method='GET')
if temp_url_sig != hmac_val:
hmac_val = self._get_hmac(env, temp_url_expires, key,
request_method='PUT')
if temp_url_sig != hmac_val:
hmac_vals = self._get_hmacs(env, temp_url_expires, keys,
request_method='GET')
if temp_url_sig not in hmac_vals:
hmac_vals = self._get_hmacs(env, temp_url_expires, keys,
request_method='PUT')
if temp_url_sig not in hmac_vals:
return self._invalid(env, start_response)
else:
hmac_val = self._get_hmac(env, temp_url_expires, key)
if temp_url_sig != hmac_val:
hmac_vals = self._get_hmacs(env, temp_url_expires, keys)
if temp_url_sig not in hmac_vals:
return self._invalid(env, start_response)
self._clean_incoming_headers(env)
env['swift.authorize'] = lambda req: None
@ -339,40 +345,57 @@ class TempURL(object):
filename = qs['filename'][0]
return temp_url_sig, temp_url_expires, filename
def _get_key(self, env, account):
def _get_keys(self, env, account):
"""
Returns the X-Account-Meta-Temp-URL-Key header value for the
account, or None if none is set.
Returns the X-Account-Meta-Temp-URL-Key[-2] header values for the
account, or an empty list if none is set.
Returns 0, 1, or 2 elements depending on how many keys are set
in the account's metadata.
:param env: The WSGI environment for the request.
:param account: Account str.
:returns: X-Account-Meta-Temp-URL-Key str value, or None.
:returns: [X-Account-Meta-Temp-URL-Key str value if set,
X-Account-Meta-Temp-URL-Key-2 str value if set]
"""
key = None
keys = None
memcache = env.get('swift.cache')
memcache_hash_key = 'temp-url-keys/%s' % account
if memcache:
key = memcache.get('temp-url-key/%s' % account)
if not key:
keys = memcache.get(memcache_hash_key)
if keys is None:
newenv = make_pre_authed_env(env, 'HEAD', '/v1/' + account,
self.agent, swift_source='TU')
newenv['CONTENT_LENGTH'] = '0'
newenv['wsgi.input'] = StringIO('')
key = [None]
keys = []
def _start_response(status, response_headers, exc_info=None):
for h, v in response_headers:
if h.lower() == 'x-account-meta-temp-url-key':
key[0] = v
keys.append(v)
elif h.lower() == 'x-account-meta-temp-url-key-2':
keys.append(v)
i = iter(self.app(newenv, _start_response))
try:
i.next()
except StopIteration:
pass
key = key[0]
if key and memcache:
memcache.set('temp-url-key/%s' % account, key, time=60)
return key
if memcache:
memcache.set(memcache_hash_key, keys, time=60)
return keys
def _get_hmacs(self, env, expires, keys, request_method=None):
"""
:param env: The WSGI environment for the request.
:param expires: Unix timestamp as an int for when the URL
expires.
:param keys: Key strings, from the X-Account-Meta-Temp-URL-Key[-2] of
the account.
"""
return [self._get_hmac(env, expires, key, request_method)
for key in keys]
def _get_hmac(self, env, expires, key, request_method=None):
"""

95
test/unit/common/middleware/test_tempurl.py
View File

@ -110,7 +110,7 @@ class TestTempURL(unittest.TestCase):
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')]))
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 200)
@ -119,6 +119,28 @@ class TestTempURL(unittest.TestCase):
self.assertEquals(req.environ['swift.authorize_override'], True)
self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl')
def test_get_valid_key2(self):
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key1 = 'abc123'
key2 = 'def456'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig1 = hmac.new(key1, hmac_body, sha1).hexdigest()
sig2 = hmac.new(key2, hmac_body, sha1).hexdigest()
for sig in (sig1, sig2):
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-keys/a', [key1, key2])
self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')]))
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.headers['content-disposition'],
'attachment; filename="o"')
self.assertEquals(req.environ['swift.authorize_override'], True)
self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl')
def test_get_valid_with_filename(self):
method = 'GET'
expires = int(time() + 86400)
@ -129,7 +151,7 @@ class TestTempURL(unittest.TestCase):
req = self._make_request(path, environ={
'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s&'
'filename=bob%%20%%22killer%%22.txt' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')]))
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 200)
@ -148,7 +170,7 @@ class TestTempURL(unittest.TestCase):
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertFalse('content-disposition' in resp.headers)
@ -166,7 +188,7 @@ class TestTempURL(unittest.TestCase):
environ={'REQUEST_METHOD': 'PUT',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
@ -182,7 +204,7 @@ class TestTempURL(unittest.TestCase):
environ={'REQUEST_METHOD': 'PUT',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertEquals(req.environ['swift.authorize_override'], True)
@ -198,7 +220,7 @@ class TestTempURL(unittest.TestCase):
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
@ -212,7 +234,7 @@ class TestTempURL(unittest.TestCase):
hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'QUERY_STRING': 'temp_url_expires=%s' % expires})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
@ -271,7 +293,7 @@ class TestTempURL(unittest.TestCase):
environ={'REQUEST_METHOD': 'HEAD',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertEquals(req.environ['swift.authorize_override'], True)
@ -288,7 +310,7 @@ class TestTempURL(unittest.TestCase):
environ={'REQUEST_METHOD': 'HEAD',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertEquals(req.environ['swift.authorize_override'], True)
@ -356,7 +378,7 @@ class TestTempURL(unittest.TestCase):
environ={'REQUEST_METHOD': 'DELETE',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
@ -371,7 +393,7 @@ class TestTempURL(unittest.TestCase):
environ={'REQUEST_METHOD': 'UNKNOWN',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
@ -386,7 +408,7 @@ class TestTempURL(unittest.TestCase):
req = self._make_request(path + '2',
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
@ -405,7 +427,7 @@ class TestTempURL(unittest.TestCase):
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
@ -421,7 +443,7 @@ class TestTempURL(unittest.TestCase):
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' %
(sig, expires + 1)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
@ -436,7 +458,7 @@ class TestTempURL(unittest.TestCase):
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key + '2')
req.environ['swift.cache'].set('temp-url-keys/a', [key + '2'])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
@ -453,7 +475,7 @@ class TestTempURL(unittest.TestCase):
req = self._make_request(path, headers={'x-remove-this': 'value'},
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertTrue('x-remove-this' not in self.app.request.headers)
@ -473,7 +495,7 @@ class TestTempURL(unittest.TestCase):
'x-remove-this-except-this': 'value2'},
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertTrue('x-remove-this-one' not in self.app.request.headers)
@ -492,7 +514,7 @@ class TestTempURL(unittest.TestCase):
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertTrue('x-test-header-one-a' not in resp.headers)
@ -511,7 +533,7 @@ class TestTempURL(unittest.TestCase):
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
req.environ['swift.cache'].set('temp-url-keys/a', [key])
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertEquals(resp.headers['x-test-header-one-a'], 'value1')
@ -571,25 +593,44 @@ class TestTempURL(unittest.TestCase):
def test_get_key_memcache(self):
self.app.status_headers_body_iter = iter([('404 Not Found', {}, '')])
self.assertEquals(
self.tempurl._get_key({}, 'a'), None)
self.tempurl._get_keys({}, 'a'), [])
self.app.status_headers_body_iter = iter([('404 Not Found', {}, '')])
self.assertEquals(
self.tempurl._get_key({'swift.cache': None}, 'a'), None)
self.tempurl._get_keys({'swift.cache': None}, 'a'), [])
mc = FakeMemcache()
self.app.status_headers_body_iter = iter([('404 Not Found', {}, '')])
self.assertEquals(
self.tempurl._get_key({'swift.cache': mc}, 'a'), None)
mc.set('temp-url-key/a', 'abc')
self.tempurl._get_keys({'swift.cache': mc}, 'a'), [])
mc.set('temp-url-keys/a', ['abc', 'def'])
self.assertEquals(
self.tempurl._get_key({'swift.cache': mc}, 'a'), 'abc')
self.tempurl._get_keys({'swift.cache': mc}, 'a'), ['abc', 'def'])
def test_get_key_from_source(self):
def test_get_keys_from_source(self):
self.app.status_headers_body_iter = \
iter([('200 Ok', {'x-account-meta-temp-url-key': 'abc'}, '')])
mc = FakeMemcache()
self.assertEquals(
self.tempurl._get_key({'swift.cache': mc}, 'a'), 'abc')
self.assertEquals(mc.get('temp-url-key/a'), 'abc')
self.tempurl._get_keys({'swift.cache': mc}, 'a'), ['abc'])
self.assertEquals(mc.get('temp-url-keys/a'), ['abc'])
self.app.status_headers_body_iter = \
iter([('200 Ok',
{'x-account-meta-temp-url-key': 'abc',
'x-account-meta-temp-url-key-2': 'def'},
'')])
mc = FakeMemcache()
self.assertEquals(
sorted(self.tempurl._get_keys({'swift.cache': mc}, 'a')),
['abc', 'def'])
self.assertEquals(sorted(mc.get('temp-url-keys/a')), ['abc', 'def'])
# no keys at all: still gets cached
self.app.status_headers_body_iter = iter([('200 Ok', {}, '')])
mc = FakeMemcache()
self.assertEquals(
sorted(self.tempurl._get_keys({'swift.cache': mc}, 'a')),
[])
self.assertEquals(sorted(mc.get('temp-url-keys/a')), [])
def test_get_hmac(self):
self.assertEquals(self.tempurl._get_hmac(