Merge "quotas: Add account-level per-policy quotas"
This commit is contained in:
Zuul
Gerrit Code Review
commit
6d3d419715
@ -1,3 +1,5 @@
|
||||
.. _container_quotas:
|
||||
|
||||
================
|
||||
Container quotas
|
||||
================
|
||||
|
||||
@ -1110,11 +1110,11 @@ use = egg:swift#dlo
|
||||
# Time limit on GET requests (seconds)
|
||||
# max_get_time = 86400
|
||||
|
||||
# Note: Put after auth in the pipeline.
|
||||
# Note: Put after auth and server-side copy in the pipeline.
|
||||
[filter:container-quotas]
|
||||
use = egg:swift#container_quotas
|
||||
|
||||
# Note: Put after auth in the pipeline.
|
||||
# Note: Put after auth and server-side copy in the pipeline.
|
||||
[filter:account-quotas]
|
||||
use = egg:swift#account_quotas
|
||||
|
||||
|
||||
@ -19,9 +19,19 @@ given account quota (in bytes) is exceeded while DELETE requests are still
|
||||
allowed.
|
||||
|
||||
``account_quotas`` uses the ``x-account-meta-quota-bytes`` metadata entry to
|
||||
store the quota. Write requests to this metadata entry are only permitted for
|
||||
resellers. There is no quota limit if ``x-account-meta-quota-bytes`` is not
|
||||
set.
|
||||
store the overall account quota. Write requests to this metadata entry are
|
||||
only permitted for resellers. There is no overall account quota limit if
|
||||
``x-account-meta-quota-bytes`` is not set.
|
||||
|
||||
Additionally, account quotas may be set for each storage policy, using metadata
|
||||
of the form ``x-account-quota-bytes-policy-<policy name>``. Again, only
|
||||
resellers may update these metadata, and there will be no limit for a
|
||||
particular policy if the corresponding metadata is not set.
|
||||
|
||||
.. note::
|
||||
Per-policy quotas need not sum to the overall account quota, and the sum of
|
||||
all :ref:`container_quotas` for a given policy need not sum to the account's
|
||||
policy quota.
|
||||
|
||||
The ``account_quotas`` middleware should be added to the pipeline in your
|
||||
``/etc/swift/proxy-server.conf`` file just after any auth middleware.
|
||||
@ -55,7 +65,8 @@ account size has been updated.
|
||||
from swift.common.swob import HTTPForbidden, HTTPBadRequest, \
|
||||
HTTPRequestEntityTooLarge, wsgify
|
||||
from swift.common.registry import register_swift_info
|
||||
from swift.proxy.controllers.base import get_account_info
|
||||
from swift.common.storage_policy import POLICIES
|
||||
from swift.proxy.controllers.base import get_account_info, get_container_info
|
||||
|
||||
|
||||
class AccountQuotaMiddleware(object):
|
||||
@ -68,29 +79,49 @@ class AccountQuotaMiddleware(object):
|
||||
self.app = app
|
||||
|
||||
def handle_account(self, request):
|
||||
# account request, so we pay attention to the quotas
|
||||
new_quota = request.headers.get(
|
||||
'X-Account-Meta-Quota-Bytes')
|
||||
if request.headers.get(
|
||||
'X-Remove-Account-Meta-Quota-Bytes'):
|
||||
new_quota = 0 # X-Remove dominates if both are present
|
||||
if request.method in ("POST", "PUT"):
|
||||
# account request, so we pay attention to the quotas
|
||||
new_quotas = {}
|
||||
new_quotas[None] = request.headers.get(
|
||||
'X-Account-Meta-Quota-Bytes')
|
||||
if request.headers.get(
|
||||
'X-Remove-Account-Meta-Quota-Bytes'):
|
||||
new_quotas[None] = 0 # X-Remove dominates if both are present
|
||||
|
||||
if request.environ.get('reseller_request') is True:
|
||||
if new_quota and not new_quota.isdigit():
|
||||
return HTTPBadRequest()
|
||||
return self.app
|
||||
for policy in POLICIES:
|
||||
tail = 'Account-Quota-Bytes-Policy-%s' % policy.name
|
||||
if request.headers.get('X-Remove-' + tail):
|
||||
new_quotas[policy.idx] = 0
|
||||
else:
|
||||
quota = request.headers.pop('X-' + tail, None)
|
||||
new_quotas[policy.idx] = quota
|
||||
|
||||
# deny quota set for non-reseller
|
||||
if new_quota is not None:
|
||||
return HTTPForbidden()
|
||||
return self.app
|
||||
if request.environ.get('reseller_request') is True:
|
||||
if any(quota and not quota.isdigit()
|
||||
for quota in new_quotas.values()):
|
||||
return HTTPBadRequest()
|
||||
for idx, quota in new_quotas.items():
|
||||
if idx is None:
|
||||
continue # For legacy reasons, it's in user meta
|
||||
hdr = 'X-Account-Sysmeta-Quota-Bytes-Policy-%d' % idx
|
||||
request.headers[hdr] = quota
|
||||
elif any(quota is not None for quota in new_quotas.values()):
|
||||
# deny quota set for non-reseller
|
||||
return HTTPForbidden()
|
||||
|
||||
resp = request.get_response(self.app)
|
||||
# Non-resellers can't update quotas, but they *can* see them
|
||||
for policy in POLICIES:
|
||||
infix = 'Quota-Bytes-Policy'
|
||||
value = resp.headers.get('X-Account-Sysmeta-%s-%d' % (
|
||||
infix, policy.idx))
|
||||
if value:
|
||||
resp.headers['X-Account-%s-%s' % (infix, policy.name)] = value
|
||||
return resp
|
||||
|
||||
@wsgify
|
||||
def __call__(self, request):
|
||||
|
||||
if request.method not in ("POST", "PUT"):
|
||||
return self.app
|
||||
|
||||
try:
|
||||
ver, account, container, obj = request.split_path(
|
||||
2, 4, rest_with_last=True)
|
||||
@ -102,7 +133,7 @@ class AccountQuotaMiddleware(object):
|
||||
# container or object request; even if the quota headers are set
|
||||
# in the request, they're meaningless
|
||||
|
||||
if request.method == "POST" or not obj:
|
||||
if not (request.method == "PUT" and obj):
|
||||
return self.app
|
||||
# OK, object PUT
|
||||
|
||||
@ -110,6 +141,7 @@ class AccountQuotaMiddleware(object):
|
||||
# but resellers aren't constrained by quotas :-)
|
||||
return self.app
|
||||
|
||||
# Object PUT request
|
||||
content_length = (request.content_length or 0)
|
||||
|
||||
account_info = get_account_info(request.environ, self.app,
|
||||
@ -119,24 +151,50 @@ class AccountQuotaMiddleware(object):
|
||||
try:
|
||||
quota = int(account_info['meta'].get('quota-bytes', -1))
|
||||
except ValueError:
|
||||
return self.app
|
||||
if quota < 0:
|
||||
return self.app
|
||||
quota = -1
|
||||
if quota >= 0:
|
||||
new_size = int(account_info['bytes']) + content_length
|
||||
if quota < new_size:
|
||||
resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.')
|
||||
if 'swift.authorize' in request.environ:
|
||||
orig_authorize = request.environ['swift.authorize']
|
||||
|
||||
new_size = int(account_info['bytes']) + content_length
|
||||
if quota < new_size:
|
||||
resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.')
|
||||
if 'swift.authorize' in request.environ:
|
||||
orig_authorize = request.environ['swift.authorize']
|
||||
|
||||
def reject_authorize(*args, **kwargs):
|
||||
aresp = orig_authorize(*args, **kwargs)
|
||||
if aresp:
|
||||
return aresp
|
||||
def reject_authorize(*args, **kwargs):
|
||||
aresp = orig_authorize(*args, **kwargs)
|
||||
if aresp:
|
||||
return aresp
|
||||
return resp
|
||||
request.environ['swift.authorize'] = reject_authorize
|
||||
else:
|
||||
return resp
|
||||
|
||||
container_info = get_container_info(request.environ, self.app,
|
||||
swift_source='AQ')
|
||||
if not container_info:
|
||||
return self.app
|
||||
policy_idx = container_info['storage_policy']
|
||||
sysmeta_key = 'quota-bytes-policy-%s' % policy_idx
|
||||
try:
|
||||
policy_quota = int(account_info['sysmeta'].get(sysmeta_key, -1))
|
||||
except ValueError:
|
||||
policy_quota = -1
|
||||
if policy_quota >= 0:
|
||||
policy_stats = account_info['storage_policies'].get(policy_idx, {})
|
||||
new_size = int(policy_stats.get('bytes', 0)) + content_length
|
||||
if policy_quota < new_size:
|
||||
resp = HTTPRequestEntityTooLarge(
|
||||
body='Upload exceeds policy quota.')
|
||||
if 'swift.authorize' in request.environ:
|
||||
orig_authorize = request.environ['swift.authorize']
|
||||
|
||||
def reject_authorize(*args, **kwargs):
|
||||
aresp = orig_authorize(*args, **kwargs)
|
||||
if aresp:
|
||||
return aresp
|
||||
return resp
|
||||
request.environ['swift.authorize'] = reject_authorize
|
||||
else:
|
||||
return resp
|
||||
request.environ['swift.authorize'] = reject_authorize
|
||||
else:
|
||||
return resp
|
||||
|
||||
return self.app
|
||||
|
||||
|
||||
@ -18,6 +18,7 @@ from swift.common.swob import Request, wsgify, HTTPForbidden, HTTPOk, \
|
||||
|
||||
from swift.common.middleware import account_quotas, copy
|
||||
|
||||
from test.unit import patch_policies
|
||||
from test.unit.common.middleware.helpers import FakeSwift
|
||||
|
||||
|
||||
@ -53,6 +54,8 @@ class TestAccountQuota(unittest.TestCase):
|
||||
self.app = FakeSwift()
|
||||
self.app.register('HEAD', '/v1/a', HTTPOk, {
|
||||
'x-account-bytes-used': '1000'})
|
||||
self.app.register('HEAD', '/v1/a/c', HTTPOk, {
|
||||
'x-backend-storage-policy-index': '1'})
|
||||
self.app.register('POST', '/v1/a', HTTPOk, {})
|
||||
self.app.register('PUT', '/v1/a/c/o', HTTPOk, {})
|
||||
|
||||
@ -128,6 +131,48 @@ class TestAccountQuota(unittest.TestCase):
|
||||
self.assertEqual(res.status_int, 413)
|
||||
self.assertEqual(res.body, b'Upload exceeds quota.')
|
||||
|
||||
@patch_policies
|
||||
def test_exceed_per_policy_quota(self):
|
||||
self.app.register('HEAD', '/v1/a', HTTPOk, {
|
||||
'x-account-bytes-used': '100',
|
||||
'x-account-storage-policy-unu-bytes-used': '100',
|
||||
'x-account-sysmeta-quota-bytes-policy-1': '10',
|
||||
'x-account-meta-quota-bytes': '1000'})
|
||||
app = account_quotas.AccountQuotaMiddleware(self.app)
|
||||
cache = FakeCache(None)
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT',
|
||||
'swift.cache': cache})
|
||||
res = req.get_response(app)
|
||||
self.assertEqual(res.status_int, 413)
|
||||
self.assertEqual(res.body, b'Upload exceeds policy quota.')
|
||||
|
||||
@patch_policies
|
||||
def test_policy_quota_translation(self):
|
||||
def do_test(method):
|
||||
self.app.register(method, '/v1/a', HTTPOk, {
|
||||
'x-account-bytes-used': '100',
|
||||
'x-account-storage-policy-unu-bytes-used': '100',
|
||||
'x-account-sysmeta-quota-bytes-policy-1': '10',
|
||||
'x-account-meta-quota-bytes': '1000'})
|
||||
app = account_quotas.AccountQuotaMiddleware(self.app)
|
||||
cache = FakeCache(None)
|
||||
req = Request.blank('/v1/a', method=method, environ={
|
||||
'swift.cache': cache})
|
||||
res = req.get_response(app)
|
||||
self.assertEqual(res.status_int, 200)
|
||||
self.assertEqual(res.headers.get(
|
||||
'X-Account-Meta-Quota-Bytes'), '1000')
|
||||
self.assertEqual(res.headers.get(
|
||||
'X-Account-Sysmeta-Quota-Bytes-Policy-1'), '10')
|
||||
self.assertEqual(res.headers.get(
|
||||
'X-Account-Quota-Bytes-Policy-Unu'), '10')
|
||||
self.assertEqual(res.headers.get(
|
||||
'X-Account-Storage-Policy-Unu-Bytes-Used'), '100')
|
||||
|
||||
do_test('GET')
|
||||
do_test('HEAD')
|
||||
|
||||
def test_exceed_quota_not_authorized(self):
|
||||
self.app.register('HEAD', '/v1/a', HTTPOk, {
|
||||
'x-account-bytes-used': '1000',
|
||||
@ -335,6 +380,19 @@ class TestAccountQuota(unittest.TestCase):
|
||||
'reseller_request': True})
|
||||
res = req.get_response(app)
|
||||
self.assertEqual(res.status_int, 400)
|
||||
self.assertEqual(self.app.calls, [])
|
||||
|
||||
def test_invalid_policy_quota(self):
|
||||
app = account_quotas.AccountQuotaMiddleware(self.app)
|
||||
cache = FakeCache(None)
|
||||
req = Request.blank('/v1/a', environ={
|
||||
'REQUEST_METHOD': 'POST',
|
||||
'swift.cache': cache,
|
||||
'HTTP_X_ACCOUNT_QUOTA_BYTES_POLICY_POLICY_0': 'abc',
|
||||
'reseller_request': True})
|
||||
res = req.get_response(app)
|
||||
self.assertEqual(res.status_int, 400)
|
||||
self.assertEqual(self.app.calls, [])
|
||||
|
||||
def test_valid_quotas_admin(self):
|
||||
app = account_quotas.AccountQuotaMiddleware(self.app)
|
||||
@ -345,6 +403,18 @@ class TestAccountQuota(unittest.TestCase):
|
||||
'HTTP_X_ACCOUNT_META_QUOTA_BYTES': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEqual(res.status_int, 403)
|
||||
self.assertEqual(self.app.calls, [])
|
||||
|
||||
def test_valid_policy_quota_admin(self):
|
||||
app = account_quotas.AccountQuotaMiddleware(self.app)
|
||||
cache = FakeCache(None)
|
||||
req = Request.blank('/v1/a', environ={
|
||||
'REQUEST_METHOD': 'POST',
|
||||
'swift.cache': cache,
|
||||
'HTTP_X_ACCOUNT_QUOTA_BYTES_POLICY_POLICY_0': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEqual(res.status_int, 403)
|
||||
self.assertEqual(self.app.calls, [])
|
||||
|
||||
def test_valid_quotas_reseller(self):
|
||||
app = account_quotas.AccountQuotaMiddleware(self.app)
|
||||
@ -356,6 +426,24 @@ class TestAccountQuota(unittest.TestCase):
|
||||
'reseller_request': True})
|
||||
res = req.get_response(app)
|
||||
self.assertEqual(res.status_int, 200)
|
||||
self.assertEqual(self.app.calls_with_headers, [
|
||||
('POST', '/v1/a', {'Host': 'localhost:80',
|
||||
'X-Account-Meta-Quota-Bytes': '100'})])
|
||||
|
||||
def test_valid_policy_quota_reseller(self):
|
||||
app = account_quotas.AccountQuotaMiddleware(self.app)
|
||||
cache = FakeCache(None)
|
||||
req = Request.blank('/v1/a', environ={
|
||||
'REQUEST_METHOD': 'POST',
|
||||
'swift.cache': cache,
|
||||
'HTTP_X_ACCOUNT_QUOTA_BYTES_POLICY_POLICY_0': '100',
|
||||
'reseller_request': True})
|
||||
res = req.get_response(app)
|
||||
self.assertEqual(res.status_int, 200)
|
||||
self.assertEqual(self.app.calls_with_headers, [
|
||||
('POST', '/v1/a', {
|
||||
'Host': 'localhost:80',
|
||||
'X-Account-Sysmeta-Quota-Bytes-Policy-0': '100'})])
|
||||
|
||||
def test_delete_quotas(self):
|
||||
app = account_quotas.AccountQuotaMiddleware(self.app)
|
||||
@ -414,6 +502,8 @@ class AccountQuotaCopyingTestCases(unittest.TestCase):
|
||||
self.headers = []
|
||||
self.app = FakeSwift()
|
||||
self.app.register('HEAD', '/v1/a', HTTPOk, self.headers)
|
||||
self.app.register('HEAD', '/v1/a/c', HTTPOk, {
|
||||
'x-backend-storage-policy-index': '1'})
|
||||
self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {
|
||||
'content-length': '1000'})
|
||||
self.aq_filter = account_quotas.filter_factory({})(self.app)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user