Allow 2 TempURL keys per account.
This allows users to rotate their TempURL keys without invalidating all existing signed URLs. This is handy if you have multiple systems generating signed URLs, but you want to change your keys for some reason (e.g. keys compromised, company policy, general paranoia). Both the first and second keys are optional, so existing accounts' signed URLs will continue to work as before. This commit does change the memcache key used to store the fetched TempURL keys. This is because we were storing the old key as a string in memcached, but the new one is a list of keys. Since the key cache lifetime here is only 60 seconds, it doesn't seem like too big a deal to completely flush the TempURL cache. Also, this commit adds caching of a negative TempURL result. If the account HEAD reveals no TempURL keys at all, that result is now stored for 60 seconds the same way that a positive result would be. Change-Id: I40a02bd607283fbce11aa52a9bb8a5846ab17f5e
This commit is contained in:
Samuel Merritt
Gerrit Code Review
parent
c3e6f3a1d6
commit
21343ab038
@ -66,9 +66,15 @@ Using this in combination with browser form post translation
|
||||
middleware could also allow direct-from-browser uploads to specific
|
||||
locations in Swift.
|
||||
|
||||
Note that changing the X-Account-Meta-Temp-URL-Key will invalidate
|
||||
any previously generated temporary URLs within 60 seconds (the
|
||||
memcache time for the key).
|
||||
TempURL supports up to two keys, specified by X-Account-Meta-Temp-URL-Key and
|
||||
X-Account-Meta-Temp-URL-Key-2. Signatures are checked against both keys, if
|
||||
present. This is to allow for key rotation without invalidating all existing
|
||||
temporary URLs.
|
||||
|
||||
Note that changing either X-Account-Meta-Temp-URL-Key or
|
||||
X-Account-Meta-Temp-URL-Key-2 will invalidate any previously generated
|
||||
temporary URLs signed with that key within 60 seconds (the memcache lifetime
|
||||
for the key). It is not instantaneous.
|
||||
|
||||
With GET TempURLs, a Content-Disposition header will be set on the
|
||||
response so that browsers will interpret this as a file attachment to
|
||||
@ -246,20 +252,20 @@ class TempURL(object):
|
||||
account = self._get_account(env)
|
||||
if not account:
|
||||
return self._invalid(env, start_response)
|
||||
key = self._get_key(env, account)
|
||||
if not key:
|
||||
keys = self._get_keys(env, account)
|
||||
if not keys:
|
||||
return self._invalid(env, start_response)
|
||||
if env['REQUEST_METHOD'] == 'HEAD':
|
||||
hmac_val = self._get_hmac(env, temp_url_expires, key,
|
||||
request_method='GET')
|
||||
if temp_url_sig != hmac_val:
|
||||
hmac_val = self._get_hmac(env, temp_url_expires, key,
|
||||
request_method='PUT')
|
||||
if temp_url_sig != hmac_val:
|
||||
hmac_vals = self._get_hmacs(env, temp_url_expires, keys,
|
||||
request_method='GET')
|
||||
if temp_url_sig not in hmac_vals:
|
||||
hmac_vals = self._get_hmacs(env, temp_url_expires, keys,
|
||||
request_method='PUT')
|
||||
if temp_url_sig not in hmac_vals:
|
||||
return self._invalid(env, start_response)
|
||||
else:
|
||||
hmac_val = self._get_hmac(env, temp_url_expires, key)
|
||||
if temp_url_sig != hmac_val:
|
||||
hmac_vals = self._get_hmacs(env, temp_url_expires, keys)
|
||||
if temp_url_sig not in hmac_vals:
|
||||
return self._invalid(env, start_response)
|
||||
self._clean_incoming_headers(env)
|
||||
env['swift.authorize'] = lambda req: None
|
||||
@ -339,40 +345,57 @@ class TempURL(object):
|
||||
filename = qs['filename'][0]
|
||||
return temp_url_sig, temp_url_expires, filename
|
||||
|
||||
def _get_key(self, env, account):
|
||||
def _get_keys(self, env, account):
|
||||
"""
|
||||
Returns the X-Account-Meta-Temp-URL-Key header value for the
|
||||
account, or None if none is set.
|
||||
Returns the X-Account-Meta-Temp-URL-Key[-2] header values for the
|
||||
account, or an empty list if none is set.
|
||||
|
||||
Returns 0, 1, or 2 elements depending on how many keys are set
|
||||
in the account's metadata.
|
||||
|
||||
:param env: The WSGI environment for the request.
|
||||
:param account: Account str.
|
||||
:returns: X-Account-Meta-Temp-URL-Key str value, or None.
|
||||
:returns: [X-Account-Meta-Temp-URL-Key str value if set,
|
||||
X-Account-Meta-Temp-URL-Key-2 str value if set]
|
||||
"""
|
||||
key = None
|
||||
keys = None
|
||||
memcache = env.get('swift.cache')
|
||||
memcache_hash_key = 'temp-url-keys/%s' % account
|
||||
if memcache:
|
||||
key = memcache.get('temp-url-key/%s' % account)
|
||||
if not key:
|
||||
keys = memcache.get(memcache_hash_key)
|
||||
if keys is None:
|
||||
newenv = make_pre_authed_env(env, 'HEAD', '/v1/' + account,
|
||||
self.agent, swift_source='TU')
|
||||
newenv['CONTENT_LENGTH'] = '0'
|
||||
newenv['wsgi.input'] = StringIO('')
|
||||
key = [None]
|
||||
keys = []
|
||||
|
||||
def _start_response(status, response_headers, exc_info=None):
|
||||
for h, v in response_headers:
|
||||
if h.lower() == 'x-account-meta-temp-url-key':
|
||||
key[0] = v
|
||||
keys.append(v)
|
||||
elif h.lower() == 'x-account-meta-temp-url-key-2':
|
||||
keys.append(v)
|
||||
|
||||
i = iter(self.app(newenv, _start_response))
|
||||
try:
|
||||
i.next()
|
||||
except StopIteration:
|
||||
pass
|
||||
key = key[0]
|
||||
if key and memcache:
|
||||
memcache.set('temp-url-key/%s' % account, key, time=60)
|
||||
return key
|
||||
if memcache:
|
||||
memcache.set(memcache_hash_key, keys, time=60)
|
||||
return keys
|
||||
|
||||
def _get_hmacs(self, env, expires, keys, request_method=None):
|
||||
"""
|
||||
:param env: The WSGI environment for the request.
|
||||
:param expires: Unix timestamp as an int for when the URL
|
||||
expires.
|
||||
:param keys: Key strings, from the X-Account-Meta-Temp-URL-Key[-2] of
|
||||
the account.
|
||||
"""
|
||||
return [self._get_hmac(env, expires, key, request_method)
|
||||
for key in keys]
|
||||
|
||||
def _get_hmac(self, env, expires, key, request_method=None):
|
||||
"""
|
||||
|
||||
@ -110,7 +110,7 @@ class TestTempURL(unittest.TestCase):
|
||||
req = self._make_request(path,
|
||||
environ={'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')]))
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 200)
|
||||
@ -119,6 +119,28 @@ class TestTempURL(unittest.TestCase):
|
||||
self.assertEquals(req.environ['swift.authorize_override'], True)
|
||||
self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl')
|
||||
|
||||
def test_get_valid_key2(self):
|
||||
method = 'GET'
|
||||
expires = int(time() + 86400)
|
||||
path = '/v1/a/c/o'
|
||||
key1 = 'abc123'
|
||||
key2 = 'def456'
|
||||
hmac_body = '%s\n%s\n%s' % (method, expires, path)
|
||||
sig1 = hmac.new(key1, hmac_body, sha1).hexdigest()
|
||||
sig2 = hmac.new(key2, hmac_body, sha1).hexdigest()
|
||||
for sig in (sig1, sig2):
|
||||
req = self._make_request(path,
|
||||
environ={'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key1, key2])
|
||||
self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')]))
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 200)
|
||||
self.assertEquals(resp.headers['content-disposition'],
|
||||
'attachment; filename="o"')
|
||||
self.assertEquals(req.environ['swift.authorize_override'], True)
|
||||
self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl')
|
||||
|
||||
def test_get_valid_with_filename(self):
|
||||
method = 'GET'
|
||||
expires = int(time() + 86400)
|
||||
@ -129,7 +151,7 @@ class TestTempURL(unittest.TestCase):
|
||||
req = self._make_request(path, environ={
|
||||
'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s&'
|
||||
'filename=bob%%20%%22killer%%22.txt' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')]))
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 200)
|
||||
@ -148,7 +170,7 @@ class TestTempURL(unittest.TestCase):
|
||||
req = self._make_request(path,
|
||||
environ={'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 404)
|
||||
self.assertFalse('content-disposition' in resp.headers)
|
||||
@ -166,7 +188,7 @@ class TestTempURL(unittest.TestCase):
|
||||
environ={'REQUEST_METHOD': 'PUT',
|
||||
'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 401)
|
||||
self.assertTrue('Temp URL invalid' in resp.body)
|
||||
@ -182,7 +204,7 @@ class TestTempURL(unittest.TestCase):
|
||||
environ={'REQUEST_METHOD': 'PUT',
|
||||
'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 404)
|
||||
self.assertEquals(req.environ['swift.authorize_override'], True)
|
||||
@ -198,7 +220,7 @@ class TestTempURL(unittest.TestCase):
|
||||
req = self._make_request(path,
|
||||
environ={'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 401)
|
||||
self.assertTrue('Temp URL invalid' in resp.body)
|
||||
@ -212,7 +234,7 @@ class TestTempURL(unittest.TestCase):
|
||||
hmac.new(key, hmac_body, sha1).hexdigest()
|
||||
req = self._make_request(path,
|
||||
environ={'QUERY_STRING': 'temp_url_expires=%s' % expires})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 401)
|
||||
self.assertTrue('Temp URL invalid' in resp.body)
|
||||
@ -271,7 +293,7 @@ class TestTempURL(unittest.TestCase):
|
||||
environ={'REQUEST_METHOD': 'HEAD',
|
||||
'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 404)
|
||||
self.assertEquals(req.environ['swift.authorize_override'], True)
|
||||
@ -288,7 +310,7 @@ class TestTempURL(unittest.TestCase):
|
||||
environ={'REQUEST_METHOD': 'HEAD',
|
||||
'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 404)
|
||||
self.assertEquals(req.environ['swift.authorize_override'], True)
|
||||
@ -356,7 +378,7 @@ class TestTempURL(unittest.TestCase):
|
||||
environ={'REQUEST_METHOD': 'DELETE',
|
||||
'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 404)
|
||||
|
||||
@ -371,7 +393,7 @@ class TestTempURL(unittest.TestCase):
|
||||
environ={'REQUEST_METHOD': 'UNKNOWN',
|
||||
'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 401)
|
||||
self.assertTrue('Temp URL invalid' in resp.body)
|
||||
@ -386,7 +408,7 @@ class TestTempURL(unittest.TestCase):
|
||||
req = self._make_request(path + '2',
|
||||
environ={'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 401)
|
||||
self.assertTrue('Temp URL invalid' in resp.body)
|
||||
@ -405,7 +427,7 @@ class TestTempURL(unittest.TestCase):
|
||||
req = self._make_request(path,
|
||||
environ={'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 401)
|
||||
self.assertTrue('Temp URL invalid' in resp.body)
|
||||
@ -421,7 +443,7 @@ class TestTempURL(unittest.TestCase):
|
||||
environ={'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' %
|
||||
(sig, expires + 1)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 401)
|
||||
self.assertTrue('Temp URL invalid' in resp.body)
|
||||
@ -436,7 +458,7 @@ class TestTempURL(unittest.TestCase):
|
||||
req = self._make_request(path,
|
||||
environ={'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key + '2')
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key + '2'])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 401)
|
||||
self.assertTrue('Temp URL invalid' in resp.body)
|
||||
@ -453,7 +475,7 @@ class TestTempURL(unittest.TestCase):
|
||||
req = self._make_request(path, headers={'x-remove-this': 'value'},
|
||||
environ={'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 404)
|
||||
self.assertTrue('x-remove-this' not in self.app.request.headers)
|
||||
@ -473,7 +495,7 @@ class TestTempURL(unittest.TestCase):
|
||||
'x-remove-this-except-this': 'value2'},
|
||||
environ={'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 404)
|
||||
self.assertTrue('x-remove-this-one' not in self.app.request.headers)
|
||||
@ -492,7 +514,7 @@ class TestTempURL(unittest.TestCase):
|
||||
req = self._make_request(path,
|
||||
environ={'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 404)
|
||||
self.assertTrue('x-test-header-one-a' not in resp.headers)
|
||||
@ -511,7 +533,7 @@ class TestTempURL(unittest.TestCase):
|
||||
req = self._make_request(path,
|
||||
environ={'QUERY_STRING':
|
||||
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
|
||||
req.environ['swift.cache'].set('temp-url-key/a', key)
|
||||
req.environ['swift.cache'].set('temp-url-keys/a', [key])
|
||||
resp = req.get_response(self.tempurl)
|
||||
self.assertEquals(resp.status_int, 404)
|
||||
self.assertEquals(resp.headers['x-test-header-one-a'], 'value1')
|
||||
@ -571,25 +593,44 @@ class TestTempURL(unittest.TestCase):
|
||||
def test_get_key_memcache(self):
|
||||
self.app.status_headers_body_iter = iter([('404 Not Found', {}, '')])
|
||||
self.assertEquals(
|
||||
self.tempurl._get_key({}, 'a'), None)
|
||||
self.tempurl._get_keys({}, 'a'), [])
|
||||
self.app.status_headers_body_iter = iter([('404 Not Found', {}, '')])
|
||||
self.assertEquals(
|
||||
self.tempurl._get_key({'swift.cache': None}, 'a'), None)
|
||||
self.tempurl._get_keys({'swift.cache': None}, 'a'), [])
|
||||
mc = FakeMemcache()
|
||||
self.app.status_headers_body_iter = iter([('404 Not Found', {}, '')])
|
||||
self.assertEquals(
|
||||
self.tempurl._get_key({'swift.cache': mc}, 'a'), None)
|
||||
mc.set('temp-url-key/a', 'abc')
|
||||
self.tempurl._get_keys({'swift.cache': mc}, 'a'), [])
|
||||
mc.set('temp-url-keys/a', ['abc', 'def'])
|
||||
self.assertEquals(
|
||||
self.tempurl._get_key({'swift.cache': mc}, 'a'), 'abc')
|
||||
self.tempurl._get_keys({'swift.cache': mc}, 'a'), ['abc', 'def'])
|
||||
|
||||
def test_get_key_from_source(self):
|
||||
def test_get_keys_from_source(self):
|
||||
self.app.status_headers_body_iter = \
|
||||
iter([('200 Ok', {'x-account-meta-temp-url-key': 'abc'}, '')])
|
||||
mc = FakeMemcache()
|
||||
self.assertEquals(
|
||||
self.tempurl._get_key({'swift.cache': mc}, 'a'), 'abc')
|
||||
self.assertEquals(mc.get('temp-url-key/a'), 'abc')
|
||||
self.tempurl._get_keys({'swift.cache': mc}, 'a'), ['abc'])
|
||||
self.assertEquals(mc.get('temp-url-keys/a'), ['abc'])
|
||||
|
||||
self.app.status_headers_body_iter = \
|
||||
iter([('200 Ok',
|
||||
{'x-account-meta-temp-url-key': 'abc',
|
||||
'x-account-meta-temp-url-key-2': 'def'},
|
||||
'')])
|
||||
mc = FakeMemcache()
|
||||
self.assertEquals(
|
||||
sorted(self.tempurl._get_keys({'swift.cache': mc}, 'a')),
|
||||
['abc', 'def'])
|
||||
self.assertEquals(sorted(mc.get('temp-url-keys/a')), ['abc', 'def'])
|
||||
|
||||
# no keys at all: still gets cached
|
||||
self.app.status_headers_body_iter = iter([('200 Ok', {}, '')])
|
||||
mc = FakeMemcache()
|
||||
self.assertEquals(
|
||||
sorted(self.tempurl._get_keys({'swift.cache': mc}, 'a')),
|
||||
[])
|
||||
self.assertEquals(sorted(mc.get('temp-url-keys/a')), [])
|
||||
|
||||
def test_get_hmac(self):
|
||||
self.assertEquals(self.tempurl._get_hmac(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user