openstack/zaqar
Code Issues Proposed changes

feat(transport): place request size limit for JSON

Browse Source 
To not deserialize over-sized requests with content body,
we also need transport-specific size restrictions.  They are
come before input validation, and are also configurable.

Change-Id: I06434431b3df9250c05472b4d91a53dfedb682d2
This commit is contained in:
Zhihao Yuan 2013-08-02 11:26:16 -04:00
parent 5cbfcefe29
commit a5bdd9c516
7 changed files with 75 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
marconi/tests/transport/wsgi/test_claims.py
View File

@ -29,6 +29,9 @@ class ClaimsBaseTest(base.TestBase):
def setUp(self):
super(ClaimsBaseTest, self).setUp()
self.wsgi_cfg = config.namespace(
'drivers:transport:wsgi').from_options()
self.project_id = '480924'
self.queue_path = '/v1/queues/fizbit'
self.claims_path = self.queue_path + '/claims'
@ -64,6 +67,20 @@ class ClaimsBaseTest(base.TestBase):
self.simulate_patch(href, self.project_id, body=doc)
self.assertEquals(self.srmock.status, falcon.HTTP_400)
def test_too_much_metadata(self):
doc = '{"ttl": 100, "grace": 60}'
long_doc = doc + (' ' *
(self.wsgi_cfg.metadata_max_length - len(doc) + 1))
self.simulate_post(self.claims_path, self.project_id, body=long_doc)
self.assertEquals(self.srmock.status, falcon.HTTP_400)
self.simulate_post(self.claims_path, self.project_id, body=doc)
href = self.srmock.headers_dict['Location']
self.simulate_patch(href, self.project_id, body=long_doc)
self.assertEquals(self.srmock.status, falcon.HTTP_400)
def test_lifecycle(self):
doc = '{"ttl": 10, "grace": 30}'

16
marconi/tests/transport/wsgi/test_messages.py
View File

@ -18,6 +18,7 @@ import os
import falcon
from marconi.common import config
from marconi.tests.transport.wsgi import base
@ -26,6 +27,9 @@ class MessagesBaseTest(base.TestBase):
def setUp(self):
super(MessagesBaseTest, self).setUp()
self.wsgi_cfg = config.namespace(
'drivers:transport:wsgi').from_options()
self.project_id = '7e55e1a7e'
self.queue_path = '/v1/queues/fizbit'
@ -119,6 +123,18 @@ class MessagesBaseTest(base.TestBase):
self.assertEquals(self.srmock.status, falcon.HTTP_400)
def test_exceeded_message_posting(self):
#TODO(zyuan): read `20` from the input validation module
doc = json.dumps([{'body': "some body", 'ttl': 100}] * 20, indent=4)
long_doc = doc + (' ' *
(self.wsgi_cfg.content_max_length - len(doc) + 1))
self.simulate_post(self.queue_path + '/messages',
body=long_doc,
headers=self.headers)
self.assertEquals(self.srmock.status, falcon.HTTP_400)
def test_unsupported_json(self):
for document in ('{"overflow": 9223372036854775808}',
'{"underflow": -9223372036854775809}'):

13
marconi/tests/transport/wsgi/test_queue_lifecycle.py
View File

@ -22,13 +22,18 @@ import pymongo
from marconi.common import config
from marconi.tests.transport.wsgi import base
from marconi import transport
class QueueLifecycleBaseTest(base.TestBase):
config_filename = None
def setUp(self):
super(QueueLifecycleBaseTest, self).setUp()
self.wsgi_cfg = config.namespace(
'drivers:transport:wsgi').from_options()
def test_basics_thoroughly(self):
path = '/v1/queues/gumshoe'
@ -95,7 +100,7 @@ class QueueLifecycleBaseTest(base.TestBase):
self.simulate_put('/v1/queues/fizbat', '7e55e1a7e')
self.assertEquals(self.srmock.status, falcon.HTTP_201)
doc = '{"messages": {"ttl": 600}, "padding": "%s"}'
padding_len = transport.MAX_QUEUE_METADATA_SIZE - (len(doc) - 2) + 1
padding_len = self.wsgi_cfg.metadata_max_length - (len(doc) - 2) + 1
doc = doc % ('x' * padding_len)
self.simulate_put('/v1/queues/fizbat/metadata', '7e55e1a7e', body=doc)
@ -105,7 +110,7 @@ class QueueLifecycleBaseTest(base.TestBase):
self.simulate_put('/v1/queues/fizbat', '7e55e1a7e')
self.assertEquals(self.srmock.status, falcon.HTTP_201)
doc = '{"messages": {"ttl": 600}, "padding": "%s"}'
padding_len = transport.MAX_QUEUE_METADATA_SIZE * 100
padding_len = self.wsgi_cfg.metadata_max_length * 100
doc = doc % ('x' * padding_len)
self.simulate_put('/v1/queues/fizbat/metadata', '7e55e1a7e', body=doc)
@ -117,7 +122,7 @@ class QueueLifecycleBaseTest(base.TestBase):
# Set
doc = '{"messages": {"ttl": 600}, "padding": "%s"}'
padding_len = transport.MAX_QUEUE_METADATA_SIZE - (len(doc) - 2)
padding_len = self.wsgi_cfg.metadata_max_length - (len(doc) - 2)
doc = doc % ('x' * padding_len)
self.simulate_put('/v1/queues/fizbat/metadata', '480924', body=doc)
self.assertEquals(self.srmock.status, falcon.HTTP_204)

4
marconi/transport/__init__.py
View File

@ -9,9 +9,5 @@ OPTIONS = {
CFG = config.project('marconi').from_options(**OPTIONS)
MAX_QUEUE_METADATA_SIZE = 64 * 1024
"""Maximum metadata size per queue when serialized as JSON"""
# Hoist into package namespace
DriverBase = base.DriverBase

17
marconi/transport/wsgi/claims.py
View File

@ -15,6 +15,7 @@
import falcon
from marconi.common import config
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import helpers
@ -23,6 +24,10 @@ from marconi.transport.wsgi import helpers as wsgi_helpers
LOG = logging.getLogger(__name__)
CFG = config.namespace('drivers:transport:wsgi').from_options(
metadata_max_length=64 * 1024
)
CLAIM_POST_SPEC = (('ttl', int), ('grace', int))
CLAIM_PATCH_SPEC = (('ttl', int),)
@ -38,10 +43,16 @@ class CollectionResource(object):
LOG.debug(_("Claims collection POST - queue: %(queue)s, "
"project: %(project)s") %
{"queue": queue_name, "project": project_id})
# Check for an explicit limit on the # of messages to claim
limit = req.get_param_as_int('limit')
claim_options = {} if limit is None else {'limit': limit}
# Place JSON size restriction before parsing
if req.content_length > CFG.metadata_max_length:
description = _('Claim metadata size is too large.')
raise wsgi_exceptions.HTTPBadRequestBody(description)
# Read claim metadata (e.g., TTL) and raise appropriate
# HTTP errors as needed.
metadata, = wsgi_helpers.filter_stream(req.stream, req.content_length,
@ -132,6 +143,12 @@ class ItemResource(object):
{"queue_name": queue_name,
"project_id": project_id,
"claim_id": claim_id})
# Place JSON size restriction before parsing
if req.content_length > CFG.metadata_max_length:
description = _('Claim metadata size is too large.')
raise wsgi_exceptions.HTTPBadRequestBody(description)
# Read claim metadata (e.g., TTL) and raise appropriate
# HTTP errors as needed.
metadata, = wsgi_helpers.filter_stream(req.stream, req.content_length,

10
marconi/transport/wsgi/messages.py
View File

@ -17,6 +17,7 @@ import itertools
import falcon
from marconi.common import config
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import helpers
@ -25,6 +26,10 @@ from marconi.transport.wsgi import helpers as wsgi_helpers
LOG = logging.getLogger(__name__)
CFG = config.namespace('drivers:transport:wsgi').from_options(
content_max_length=256 * 1024
)
MESSAGE_POST_SPEC = (('ttl', int), ('body', '*'))
@ -135,6 +140,11 @@ class CollectionResource(object):
uuid = req.get_header('Client-ID', required=True)
# Place JSON size restriction before parsing
if req.content_length > CFG.content_max_length:
description = _('Message collection size is too large.')
raise wsgi_exceptions.HTTPBadRequestBody(description)
# Pull out just the fields we care about
messages = wsgi_helpers.filter_stream(
req.stream,

9
marconi/transport/wsgi/metadata.py
View File

@ -15,14 +15,17 @@
import falcon
from marconi.common import config
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi import transport
from marconi.transport import helpers
from marconi.transport.wsgi import exceptions as wsgi_exceptions
LOG = logging.getLogger(__name__)
CFG = config.namespace('drivers:transport:wsgi').from_options(
metadata_max_length=64 * 1024
)
class Resource(object):
@ -57,8 +60,8 @@ class Resource(object):
"project: %(project)s") %
{"queue": queue_name, "project": project_id})
# TODO(kgriffs): Migrate this check to input validator middleware
if req.content_length > transport.MAX_QUEUE_METADATA_SIZE:
# Place JSON size restriction before parsing
if req.content_length > CFG.metadata_max_length:
description = _('Queue metadata size is too large.')
raise wsgi_exceptions.HTTPBadRequestBody(description)