openstack/zaqar
Code Issues Proposed changes

Merge "feat(validation): check metadata JSON size"

Browse Source
This commit is contained in:
Jenkins 2013-08-13 20:14:22 +00:00 committed by Gerrit Code Review
commit 5e9f60da85
8 changed files with 139 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
marconi/tests/etc/wsgi_sqlite_validation.conf
View File

@ -3,4 +3,5 @@ transport = wsgi
storage = sqlite
[limits:transport]
metadata_size_uplimit = 64
message_size_uplimit = 256

12
marconi/tests/transport/wsgi/test_messages.py
View File

@ -176,16 +176,6 @@ class MessagesBaseTest(base.TestBase):
self.assertEquals(self.srmock.status, falcon.HTTP_400)
# Each message's size
for long_body in ('a' * 255, # +2 string quotes
{'a': 0, 'b': 'x' * 243}): # w/o whitespaces
doc = json.dumps([{'body': long_body, 'ttl': 100}])
self.simulate_post(self.queue_path + '/messages',
body=doc,
headers=self.headers)
self.assertEquals(self.srmock.status, falcon.HTTP_400)
def test_unsupported_json(self):
for document in ('{"overflow": 9223372036854775808}',
'{"underflow": -9223372036854775809}'):
@ -346,7 +336,7 @@ class MessagesBaseTest(base.TestBase):
class MessagesSQLiteTests(MessagesBaseTest):
config_filename = 'wsgi_sqlite_validation.conf'
config_filename = 'wsgi_sqlite.conf'
class MessagesMongoDBTests(MessagesBaseTest):

89
marconi/tests/transport/wsgi/test_validation.py Normal file
View File

@ -0,0 +1,89 @@
# Copyright (c) 2013 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import falcon
from marconi.tests.transport.wsgi import base
class ValidationTest(base.TestBase):
config_filename = 'wsgi_sqlite_validation.conf'
def setUp(self):
super(ValidationTest, self).setUp()
self.project_id = '7e55e1a7e'
self.queue_path = '/v1/queues/noein'
self.simulate_put(self.queue_path, self.project_id)
self.headers = {
'Client-ID': '30387f00',
}
def tearDown(self):
self.simulate_delete(self.queue_path, self.project_id)
super(ValidationTest, self).tearDown()
def test_metadata_deserialization(self):
# Normal case
self.simulate_put(self.queue_path + '/metadata',
self.project_id,
body='{"timespace": "Shangri-la"}')
self.assertEquals(self.srmock.status, falcon.HTTP_204)
# Too long
metadata_size_uplimit = 64
doc_tmpl = '{"Dragon Torc":"%s"}'
doc_tmpl_ws = '{ "Dragon Torc" : "%s" }' # with whitespace
envelop_length = len(doc_tmpl % '')
for tmpl in doc_tmpl, doc_tmpl_ws:
doc = tmpl % ('0' * (metadata_size_uplimit - envelop_length + 1))
self.simulate_put(self.queue_path + '/metadata',
self.project_id,
body=doc)
self.assertEquals(self.srmock.status, falcon.HTTP_400)
def test_message_deserialization(self):
# Normal case
self.simulate_post(self.queue_path + '/messages',
self.project_id,
body='[{"body": "Dragon Knights", "ttl": 100}]',
headers=self.headers)
self.assertEquals(self.srmock.status, falcon.HTTP_201)
# Both messages' size are too long
message_size_uplimit = 256
obj = {'a': 0, 'b': ''}
envelop_length = len(json.dumps(obj, separators=(',', ':')))
obj['b'] = 'x' * (message_size_uplimit - envelop_length + 1)
for long_body in ('a' * (message_size_uplimit - 2 + 1), obj):
doc = json.dumps([{'body': long_body, 'ttl': 100}])
self.simulate_post(self.queue_path + '/messages',
self.project_id,
body=doc,
headers=self.headers)
self.assertEquals(self.srmock.status, falcon.HTTP_400)

33
marconi/transport/validation.py
View File

@ -22,6 +22,7 @@ from marconi.common import exceptions
OPTIONS = {
'queue_payload_uplimit': 20,
'metadata_size_uplimit': 64 * 1024,
'message_payload_uplimit': 20,
'message_size_uplimit': 256 * 1024,
'message_ttl_max': 1209600,
@ -66,6 +67,22 @@ def queue_listing(limit=None, **kwargs):
CFG.queue_payload_uplimit)
def queue_content(metadata, check_size):
"""Restrictions on queue metadata.
:param metadata: Metadata as a Python dict
:param check_size: Whether this size checking is required
:raises: ValidationFailed if the metadata is oversize.
"""
if check_size:
length = _compact_json_length(metadata)
if length > CFG.metadata_size_uplimit:
raise exceptions.ValidationFailed(
'queue metadata larger than %d bytes' %
CFG.metadata_size_uplimit)
def message_posting(messages, check_size=True):
"""Restrictions on a list of messages.
@ -91,12 +108,7 @@ def message_content(message, check_size):
CFG.message_ttl_max)
if check_size:
# UTF-8 encoded, without whitespace
# TODO(zyuan): Replace this redundent re-serialization
# with a sizing-only parser.
body_length = len(json.dumps(message['body'],
ensure_ascii=False,
separators=(',', ':')))
body_length = _compact_json_length(message['body'])
if body_length > CFG.message_size_uplimit:
raise exceptions.ValidationFailed(
'message body larger than %d bytes' %
@ -147,3 +159,12 @@ def claim_updating(metadata):
raise exceptions.ValidationFailed(
'claim TTL not in [60, %d]' %
CFG.claim_ttl_max)
def _compact_json_length(obj):
# UTF-8 encoded, without whitespace
# TODO(zyuan): Replace this redundent re-serialization
# with a sizing-only parser.
return len(json.dumps(obj,
ensure_ascii=False,
separators=(',', ':')))

6
marconi/transport/wsgi/claims.py
View File

@ -20,7 +20,7 @@ from marconi.common import exceptions as input_exceptions
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import utils
from marconi.transport import validation
from marconi.transport import validation as validate
from marconi.transport.wsgi import exceptions as wsgi_exceptions
from marconi.transport.wsgi import utils as wsgi_utils
@ -62,7 +62,7 @@ class CollectionResource(object):
# Claim some messages
try:
validation.claim_creation(metadata, **claim_options)
validate.claim_creation(metadata, **claim_options)
cid, msgs = self.claim_controller.create(
queue_name,
metadata=metadata,
@ -162,7 +162,7 @@ class ItemResource(object):
CLAIM_PATCH_SPEC)
try:
validation.claim_updating(metadata)
validate.claim_updating(metadata)
self.claim_controller.update(queue_name,
claim_id=claim_id,
metadata=metadata,

12
marconi/transport/wsgi/messages.py
View File

@ -20,7 +20,7 @@ from marconi.common import exceptions as input_exceptions
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import utils
from marconi.transport import validation
from marconi.transport import validation as validate
from marconi.transport.wsgi import exceptions as wsgi_exceptions
from marconi.transport.wsgi import utils as wsgi_utils
@ -47,7 +47,7 @@ class CollectionResource(object):
def _get_by_id(self, base_path, project_id, queue_name, ids):
"""Returns one or more messages from the queue by ID."""
try:
validation.message_listing(limit=len(ids))
validate.message_listing(limit=len(ids))
messages = self.message_controller.bulk_get(
queue_name,
message_ids=ids,
@ -85,7 +85,7 @@ class CollectionResource(object):
})
try:
validation.message_listing(**kwargs)
validate.message_listing(**kwargs)
results = self.message_controller.list(
queue_name,
project=project_id,
@ -173,9 +173,9 @@ class CollectionResource(object):
try:
# No need to check each message's size if it
# can not exceed the request size limit
validation.message_posting(
validate.message_posting(
messages, check_size=(
validation.CFG.message_size_uplimit <
validate.CFG.message_size_uplimit <
CFG.content_max_length))
message_ids = self.message_controller.post(
queue_name,
@ -241,7 +241,7 @@ class CollectionResource(object):
ids = req.get_param_as_list('ids', required=True)
try:
validation.message_listing(limit=len(ids))
validate.message_listing(limit=len(ids))
self.message_controller.bulk_delete(
queue_name,
message_ids=ids,

9
marconi/transport/wsgi/metadata.py
View File

@ -16,9 +16,11 @@
import falcon
from marconi.common import config
from marconi.common import exceptions as input_exceptions
import marconi.openstack.common.log as logging
from marconi.storage import exceptions as storage_exceptions
from marconi.transport import utils
from marconi.transport import validation as validate
from marconi.transport.wsgi import exceptions as wsgi_exceptions
from marconi.transport.wsgi import utils as wsgi_utils
@ -72,10 +74,17 @@ class Resource(object):
spec=None)
try:
validate.queue_content(
metadata, check_size=(
validate.CFG.metadata_size_uplimit <
CFG.metadata_max_length))
self.queue_ctrl.set_metadata(queue_name,
metadata=metadata,
project=project_id)
except input_exceptions.ValidationFailed as ex:
raise wsgi_exceptions.HTTPBadRequestBody(str(ex))
except storage_exceptions.QueueDoesNotExist:
raise falcon.HTTPNotFound()

6
marconi/transport/wsgi/queues.py
View File

@ -18,7 +18,7 @@ import falcon
from marconi.common import exceptions as input_exceptions
import marconi.openstack.common.log as logging
from marconi.transport import utils
from marconi.transport import validation
from marconi.transport import validation as validate
from marconi.transport.wsgi import exceptions as wsgi_exceptions
@ -39,7 +39,7 @@ class ItemResource(object):
{'queue': queue_name, 'project': project_id})
try:
validation.queue_creation(name=queue_name)
validate.queue_creation(name=queue_name)
created = self.queue_controller.create(
queue_name,
project=project_id)
@ -101,7 +101,7 @@ class CollectionResource(object):
})
try:
validation.queue_listing(**kwargs)
validate.queue_listing(**kwargs)
results = self.queue_controller.list(project=project_id, **kwargs)
except input_exceptions.ValidationFailed as ex: