openstack/zaqar
Code Issues Proposed changes

Fix handling of request/response body for Python 3

Browse Source 
In Python 3 request/response body is represented as `bytes`. This commit
adds implicit decoding from/to bytes before parsing json from it. Also
the same problem was fixed for the tests: for example, parsing response
from simulate_request requires decoding it to text string.

Change-Id: I0d210a9bc4e3f24b4440588c96e7c06ca2411b64
Partially-implements: blueprint py3k-support
This commit is contained in:
Nataliia Uvarova 2014-05-29 13:40:03 +02:00
parent 7adaf3aa4d
commit a83d378a00
7 changed files with 45 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
marconi/queues/transport/utils.py
View File

@ -15,6 +15,8 @@
import json
from marconi.openstack.common import strutils
class MalformedJSON(ValueError):
"""JSON string is not valid."""
@ -42,10 +44,12 @@ def read_json(stream, len):
:param len: the number of bytes to read from stream
"""
try:
return json.loads(stream.read(len), parse_int=_json_int)
content = strutils.safe_decode(stream.read(len), 'utf-8')
return json.loads(content, parse_int=_json_int)
except ValueError as ex:
raise MalformedJSON(ex)
except UnicodeDecodeError as ex:
raise MalformedJSON(ex)
def to_json(obj):

4
marconi/tests/functional/http.py
View File

@ -20,6 +20,8 @@ from falcon import testing as ftest
import requests
import six
from marconi.openstack.common import jsonutils
def _build_url(method):
@ -100,7 +102,7 @@ class ResponseMock(object):
self.headers = srmock.headers_dict
def json(self):
return json.loads(self._body, encoding='utf-8')
return jsonutils.loads(self._body)
class WSGIClient(object):

7
marconi/tests/queues/transport/wsgi/test_default_limits.py
View File

@ -21,6 +21,7 @@ import falcon
from . import base # noqa
from marconi.openstack.common import jsonutils
from marconi.queues import storage
@ -51,7 +52,7 @@ class TestDefaultLimits(base.TestBase):
result = self.simulate_get(self.queue_path)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
queues = json.loads(result[0])['queues']
queues = jsonutils.loads(result[0])['queues']
self.assertEqual(len(queues), storage.DEFAULT_QUEUES_PER_PAGE)
def test_message_listing(self):
@ -62,7 +63,7 @@ class TestDefaultLimits(base.TestBase):
self.assertEqual(self.srmock.status, falcon.HTTP_200)
messages = json.loads(result[0])['messages']
messages = jsonutils.loads(result[0])['messages']
self.assertEqual(len(messages), storage.DEFAULT_MESSAGES_PER_PAGE)
def test_claim_creation(self):
@ -73,7 +74,7 @@ class TestDefaultLimits(base.TestBase):
self.assertEqual(self.srmock.status, falcon.HTTP_201)
messages = json.loads(result[0])
messages = jsonutils.loads(result[0])
self.assertEqual(len(messages), storage.DEFAULT_MESSAGES_PER_CLAIM)
@contextlib.contextmanager

7
marconi/tests/queues/transport/wsgi/test_home.py
View File

@ -12,12 +12,11 @@
# License for the specific language governing permissions and limitations under
# the License.
import json
import falcon
import six.moves.urllib.parse as urlparse
from . import base # noqa
from marconi.openstack.common import jsonutils
class TestHomeDocument(base.TestBase):
@ -32,14 +31,14 @@ class TestHomeDocument(base.TestBase):
self.assertEqual(content_type, 'application/json-home')
try:
json.loads(body[0])
jsonutils.loads(body[0])
except ValueError:
self.fail('Home document is not valid JSON')
def test_href_template(self):
body = self.simulate_get(self.url_prefix)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
resp = json.loads(body[0])
resp = jsonutils.loads(body[0])
queue_href_template = resp['resources']['rel/queue']['href-template']
path_1 = 'https://marconi.example.com' + self.url_prefix
path_2 = 'https://marconi.example.com' + self.url_prefix + '/'

15
marconi/tests/queues/transport/wsgi/test_messages.py
View File

@ -24,6 +24,7 @@ import six
from testtools import matchers
from . import base # noqa
from marconi.openstack.common import jsonutils
from marconi.openstack.common import timeutils
from marconi.queues.transport import validation
from marconi import tests as testing
@ -73,7 +74,7 @@ class MessagesBaseTest(base.TestBase):
body=sample_doc, headers=self.headers)
self.assertEqual(self.srmock.status, falcon.HTTP_201)
result_doc = json.loads(result[0])
result_doc = jsonutils.loads(result[0])
msg_ids = self._get_msg_ids(self.srmock.headers_dict)
self.assertEqual(len(msg_ids), len(sample_messages))
@ -107,7 +108,7 @@ class MessagesBaseTest(base.TestBase):
message_uri)
# Check message properties
message = json.loads(result[0])
message = jsonutils.loads(result[0])
self.assertEqual(message['href'], message_uri)
self.assertEqual(message['body'], lookup[message['ttl']])
@ -122,7 +123,7 @@ class MessagesBaseTest(base.TestBase):
query_string=query_string)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
result_doc = json.loads(result[0])
result_doc = jsonutils.loads(result[0])
expected_ttls = set(m['ttl'] for m in sample_messages)
actual_ttls = set(m['ttl'] for m in result_doc)
self.assertFalse(expected_ttls - actual_ttls)
@ -320,7 +321,7 @@ class MessagesBaseTest(base.TestBase):
cnt = 0
while self.srmock.status == falcon.HTTP_200:
contents = json.loads(body[0])
contents = jsonutils.loads(body[0])
[target, params] = contents['links'][0]['href'].split('?')
for msg in contents['messages']:
@ -339,7 +340,7 @@ class MessagesBaseTest(base.TestBase):
body = self.simulate_get(self.queue_path + '/stats', self.project_id)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
message_stats = json.loads(body[0])['messages']
message_stats = jsonutils.loads(body[0])['messages']
self.assertEqual(self.srmock.headers_dict['Content-Location'],
self.queue_path + '/stats')
@ -417,7 +418,7 @@ class MessagesBaseTest(base.TestBase):
def test_delete_message_with_invalid_claim_doesnt_delete_message(self):
path = self.queue_path
resp = self._post_messages(path + '/messages', 1)
location = json.loads(resp[0])['resources'][0]
location = jsonutils.loads(resp[0])['resources'][0]
self.simulate_delete(location, query_string='claim_id=invalid')
self.assertEqual(self.srmock.status, falcon.HTTP_204)
@ -437,7 +438,7 @@ class MessagesBaseTest(base.TestBase):
body = self.simulate_get(path, self.project_id,
query_string=query_string,
headers=self.headers)
messages = json.loads(body[0])
messages = jsonutils.loads(body[0])
self.assertNotIn(self.queue_path + '/messages/messages',
messages[0]['href'])

29
marconi/tests/queues/transport/wsgi/test_queue_lifecycle.py
View File

@ -12,13 +12,12 @@
# License for the specific language governing permissions and limitations under
# the License.
import json
import ddt
import falcon
import six
from . import base # noqa
from marconi.openstack.common import jsonutils
from marconi import tests as testing
@ -81,9 +80,9 @@ class QueueLifecycleBaseTest(base.TestBase):
# Fetch metadata
result = self.simulate_get(gumshoe_queue_path_metadata,
project_id)
result_doc = json.loads(result[0])
result_doc = jsonutils.loads(result[0])
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(result_doc, json.loads(doc))
self.assertEqual(result_doc, jsonutils.loads(doc))
# Stats empty queue
self.simulate_get(gumshoe_queue_path_stats, project_id)
@ -209,8 +208,8 @@ class QueueLifecycleBaseTest(base.TestBase):
# Get
result = self.simulate_get(self.fizbat_queue_path_metadata, '480924')
result_doc = json.loads(result[0])
self.assertEqual(result_doc, json.loads(doc))
result_doc = jsonutils.loads(result[0])
self.assertEqual(result_doc, jsonutils.loads(doc))
self.assertEqual(self.srmock.status, falcon.HTTP_200)
def test_update_metadata(self):
@ -234,9 +233,9 @@ class QueueLifecycleBaseTest(base.TestBase):
# Get
result = self.simulate_get(xyz_queue_path_metadata, project_id)
result_doc = json.loads(result[0])
result_doc = jsonutils.loads(result[0])
self.assertEqual(result_doc, json.loads(doc2))
self.assertEqual(result_doc, jsonutils.loads(doc2))
self.assertEqual(self.srmock.headers_dict['Content-Location'],
xyz_queue_path_metadata)
@ -276,7 +275,7 @@ class QueueLifecycleBaseTest(base.TestBase):
result = self.simulate_get(self.queue_path, None,
query_string='limit=2&detailed=true')
result_doc = json.loads(result[0])
result_doc = jsonutils.loads(result[0])
queues = result_doc['queues']
self.assertEqual(len(queues), 2)
@ -287,14 +286,14 @@ class QueueLifecycleBaseTest(base.TestBase):
result = self.simulate_get(self.queue_path, project_id,
query_string='limit=2')
result_doc = json.loads(result[0])
result_doc = jsonutils.loads(result[0])
self.assertEqual(len(result_doc['queues']), 2)
# List (no metadata, get all)
result = self.simulate_get(self.queue_path,
project_id, query_string='limit=5')
result_doc = json.loads(result[0])
result_doc = jsonutils.loads(result[0])
[target, params] = result_doc['links'][0]['href'].split('?')
self.assertEqual(self.srmock.status, falcon.HTTP_200)
@ -319,12 +318,12 @@ class QueueLifecycleBaseTest(base.TestBase):
query_string='detailed=true')
self.assertEqual(self.srmock.status, falcon.HTTP_200)
result_doc = json.loads(result[0])
result_doc = jsonutils.loads(result[0])
[target, params] = result_doc['links'][0]['href'].split('?')
queue = result_doc['queues'][0]
result = self.simulate_get(queue['href'] + '/metadata', project_id)
result_doc = json.loads(result[0])
result_doc = jsonutils.loads(result[0])
self.assertEqual(result_doc, queue['metadata'])
self.assertEqual(result_doc, {'node': 31})
@ -376,9 +375,9 @@ class TestQueueLifecycleFaultyDriver(base.TestBaseFaulty):
self.assertNotIn(location, self.srmock.headers)
result = self.simulate_get(gumshoe_queue_path + '/metadata', '480924')
result_doc = json.loads(result[0])
result_doc = jsonutils.loads(result[0])
self.assertEqual(self.srmock.status, falcon.HTTP_503)
self.assertNotEqual(result_doc, json.loads(doc))
self.assertNotEqual(result_doc, jsonutils.loads(doc))
self.simulate_get(gumshoe_queue_path + '/stats', '480924')
self.assertEqual(self.srmock.status, falcon.HTTP_503)

13
marconi/tests/queues/transport/wsgi/test_shards.py
View File

@ -20,6 +20,7 @@ import ddt
import falcon
from . import base # noqa
from marconi.openstack.common import jsonutils
from marconi import tests as testing
@ -137,7 +138,7 @@ class ShardsBaseTest(base.TestBase):
result = self.simulate_get(self.shard)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
doc = json.loads(result[0])
doc = jsonutils.loads(result[0])
self.assertEqual(doc['weight'], expect['weight'])
self.assertEqual(doc['uri'], expect['uri'])
@ -163,7 +164,7 @@ class ShardsBaseTest(base.TestBase):
def test_get_works(self):
result = self.simulate_get(self.shard)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
shard = json.loads(result[0])
shard = jsonutils.loads(result[0])
self._shard_expect(shard, self.shard, self.doc['weight'],
self.doc['uri'])
@ -171,7 +172,7 @@ class ShardsBaseTest(base.TestBase):
result = self.simulate_get(self.shard,
query_string='?detailed=True')
self.assertEqual(self.srmock.status, falcon.HTTP_200)
shard = json.loads(result[0])
shard = jsonutils.loads(result[0])
self._shard_expect(shard, self.shard, self.doc['weight'],
self.doc['uri'])
self.assertIn('options', shard)
@ -190,7 +191,7 @@ class ShardsBaseTest(base.TestBase):
result = self.simulate_get(self.shard,
query_string='?detailed=True')
self.assertEqual(self.srmock.status, falcon.HTTP_200)
shard = json.loads(result[0])
shard = jsonutils.loads(result[0])
self._shard_expect(shard, self.shard, doc['weight'],
doc['uri'])
self.assertEqual(shard['options'], doc['options'])
@ -245,7 +246,7 @@ class ShardsBaseTest(base.TestBase):
result = self.simulate_get(self.url_prefix + '/shards',
query_string=query)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
results = json.loads(result[0])
results = jsonutils.loads(result[0])
self.assertIsInstance(results, dict)
self.assertIn('shards', results)
shard_list = results['shards']
@ -283,7 +284,7 @@ class ShardsBaseTest(base.TestBase):
result = self.simulate_get(self.url_prefix + '/shards',
query_string='?marker=3')
self.assertEqual(self.srmock.status, falcon.HTTP_200)
shard_list = json.loads(result[0])['shards']
shard_list = jsonutils.loads(result[0])['shards']
self.assertEqual(len(shard_list), 6)
path, weight = expected[4][:2]
self._shard_expect(shard_list[0], path, weight, self.doc['uri'])