From a83d378a00c5dd438d3c3df5c86c360364361aec Mon Sep 17 00:00:00 2001 From: Nataliia Uvarova Date: Thu, 29 May 2014 13:40:03 +0200 Subject: [PATCH] Fix handling of request/response body for Python 3 In Python 3 request/response body is represented as `bytes`. This commit adds implicit decoding from/to bytes before parsing json from it. Also the same problem was fixed for the tests: for example, parsing response from simulate_request requires decoding it to text string. Change-Id: I0d210a9bc4e3f24b4440588c96e7c06ca2411b64 Partially-implements: blueprint py3k-support --- marconi/queues/transport/utils.py | 8 +++-- marconi/tests/functional/http.py | 4 ++- .../transport/wsgi/test_default_limits.py | 7 +++-- .../tests/queues/transport/wsgi/test_home.py | 7 ++--- .../queues/transport/wsgi/test_messages.py | 15 +++++----- .../transport/wsgi/test_queue_lifecycle.py | 29 +++++++++---------- .../queues/transport/wsgi/test_shards.py | 13 +++++---- 7 files changed, 45 insertions(+), 38 deletions(-) diff --git a/marconi/queues/transport/utils.py b/marconi/queues/transport/utils.py index 3b388d6ba..c3cfc3428 100644 --- a/marconi/queues/transport/utils.py +++ b/marconi/queues/transport/utils.py @@ -15,6 +15,8 @@ import json +from marconi.openstack.common import strutils + class MalformedJSON(ValueError): """JSON string is not valid.""" @@ -42,10 +44,12 @@ def read_json(stream, len): :param len: the number of bytes to read from stream """ try: - return json.loads(stream.read(len), parse_int=_json_int) - + content = strutils.safe_decode(stream.read(len), 'utf-8') + return json.loads(content, parse_int=_json_int) except ValueError as ex: raise MalformedJSON(ex) + except UnicodeDecodeError as ex: + raise MalformedJSON(ex) def to_json(obj): diff --git a/marconi/tests/functional/http.py b/marconi/tests/functional/http.py index dff6626f9..e2b9397f0 100755 --- a/marconi/tests/functional/http.py +++ b/marconi/tests/functional/http.py @@ -20,6 +20,8 @@ from falcon import testing as ftest import requests import six +from marconi.openstack.common import jsonutils + def _build_url(method): @@ -100,7 +102,7 @@ class ResponseMock(object): self.headers = srmock.headers_dict def json(self): - return json.loads(self._body, encoding='utf-8') + return jsonutils.loads(self._body) class WSGIClient(object): diff --git a/marconi/tests/queues/transport/wsgi/test_default_limits.py b/marconi/tests/queues/transport/wsgi/test_default_limits.py index 0d8922dc4..f035c3d93 100644 --- a/marconi/tests/queues/transport/wsgi/test_default_limits.py +++ b/marconi/tests/queues/transport/wsgi/test_default_limits.py @@ -21,6 +21,7 @@ import falcon from . import base # noqa +from marconi.openstack.common import jsonutils from marconi.queues import storage @@ -51,7 +52,7 @@ class TestDefaultLimits(base.TestBase): result = self.simulate_get(self.queue_path) self.assertEqual(self.srmock.status, falcon.HTTP_200) - queues = json.loads(result[0])['queues'] + queues = jsonutils.loads(result[0])['queues'] self.assertEqual(len(queues), storage.DEFAULT_QUEUES_PER_PAGE) def test_message_listing(self): @@ -62,7 +63,7 @@ class TestDefaultLimits(base.TestBase): self.assertEqual(self.srmock.status, falcon.HTTP_200) - messages = json.loads(result[0])['messages'] + messages = jsonutils.loads(result[0])['messages'] self.assertEqual(len(messages), storage.DEFAULT_MESSAGES_PER_PAGE) def test_claim_creation(self): @@ -73,7 +74,7 @@ class TestDefaultLimits(base.TestBase): self.assertEqual(self.srmock.status, falcon.HTTP_201) - messages = json.loads(result[0]) + messages = jsonutils.loads(result[0]) self.assertEqual(len(messages), storage.DEFAULT_MESSAGES_PER_CLAIM) @contextlib.contextmanager diff --git a/marconi/tests/queues/transport/wsgi/test_home.py b/marconi/tests/queues/transport/wsgi/test_home.py index 6e4cfbe9b..c817d7d03 100644 --- a/marconi/tests/queues/transport/wsgi/test_home.py +++ b/marconi/tests/queues/transport/wsgi/test_home.py @@ -12,12 +12,11 @@ # License for the specific language governing permissions and limitations under # the License. -import json - import falcon import six.moves.urllib.parse as urlparse from . import base # noqa +from marconi.openstack.common import jsonutils class TestHomeDocument(base.TestBase): @@ -32,14 +31,14 @@ class TestHomeDocument(base.TestBase): self.assertEqual(content_type, 'application/json-home') try: - json.loads(body[0]) + jsonutils.loads(body[0]) except ValueError: self.fail('Home document is not valid JSON') def test_href_template(self): body = self.simulate_get(self.url_prefix) self.assertEqual(self.srmock.status, falcon.HTTP_200) - resp = json.loads(body[0]) + resp = jsonutils.loads(body[0]) queue_href_template = resp['resources']['rel/queue']['href-template'] path_1 = 'https://marconi.example.com' + self.url_prefix path_2 = 'https://marconi.example.com' + self.url_prefix + '/' diff --git a/marconi/tests/queues/transport/wsgi/test_messages.py b/marconi/tests/queues/transport/wsgi/test_messages.py index 8b61cd5e7..adbbcfbc2 100644 --- a/marconi/tests/queues/transport/wsgi/test_messages.py +++ b/marconi/tests/queues/transport/wsgi/test_messages.py @@ -24,6 +24,7 @@ import six from testtools import matchers from . import base # noqa +from marconi.openstack.common import jsonutils from marconi.openstack.common import timeutils from marconi.queues.transport import validation from marconi import tests as testing @@ -73,7 +74,7 @@ class MessagesBaseTest(base.TestBase): body=sample_doc, headers=self.headers) self.assertEqual(self.srmock.status, falcon.HTTP_201) - result_doc = json.loads(result[0]) + result_doc = jsonutils.loads(result[0]) msg_ids = self._get_msg_ids(self.srmock.headers_dict) self.assertEqual(len(msg_ids), len(sample_messages)) @@ -107,7 +108,7 @@ class MessagesBaseTest(base.TestBase): message_uri) # Check message properties - message = json.loads(result[0]) + message = jsonutils.loads(result[0]) self.assertEqual(message['href'], message_uri) self.assertEqual(message['body'], lookup[message['ttl']]) @@ -122,7 +123,7 @@ class MessagesBaseTest(base.TestBase): query_string=query_string) self.assertEqual(self.srmock.status, falcon.HTTP_200) - result_doc = json.loads(result[0]) + result_doc = jsonutils.loads(result[0]) expected_ttls = set(m['ttl'] for m in sample_messages) actual_ttls = set(m['ttl'] for m in result_doc) self.assertFalse(expected_ttls - actual_ttls) @@ -320,7 +321,7 @@ class MessagesBaseTest(base.TestBase): cnt = 0 while self.srmock.status == falcon.HTTP_200: - contents = json.loads(body[0]) + contents = jsonutils.loads(body[0]) [target, params] = contents['links'][0]['href'].split('?') for msg in contents['messages']: @@ -339,7 +340,7 @@ class MessagesBaseTest(base.TestBase): body = self.simulate_get(self.queue_path + '/stats', self.project_id) self.assertEqual(self.srmock.status, falcon.HTTP_200) - message_stats = json.loads(body[0])['messages'] + message_stats = jsonutils.loads(body[0])['messages'] self.assertEqual(self.srmock.headers_dict['Content-Location'], self.queue_path + '/stats') @@ -417,7 +418,7 @@ class MessagesBaseTest(base.TestBase): def test_delete_message_with_invalid_claim_doesnt_delete_message(self): path = self.queue_path resp = self._post_messages(path + '/messages', 1) - location = json.loads(resp[0])['resources'][0] + location = jsonutils.loads(resp[0])['resources'][0] self.simulate_delete(location, query_string='claim_id=invalid') self.assertEqual(self.srmock.status, falcon.HTTP_204) @@ -437,7 +438,7 @@ class MessagesBaseTest(base.TestBase): body = self.simulate_get(path, self.project_id, query_string=query_string, headers=self.headers) - messages = json.loads(body[0]) + messages = jsonutils.loads(body[0]) self.assertNotIn(self.queue_path + '/messages/messages', messages[0]['href']) diff --git a/marconi/tests/queues/transport/wsgi/test_queue_lifecycle.py b/marconi/tests/queues/transport/wsgi/test_queue_lifecycle.py index 0c947d5b1..12c47307f 100644 --- a/marconi/tests/queues/transport/wsgi/test_queue_lifecycle.py +++ b/marconi/tests/queues/transport/wsgi/test_queue_lifecycle.py @@ -12,13 +12,12 @@ # License for the specific language governing permissions and limitations under # the License. -import json - import ddt import falcon import six from . import base # noqa +from marconi.openstack.common import jsonutils from marconi import tests as testing @@ -81,9 +80,9 @@ class QueueLifecycleBaseTest(base.TestBase): # Fetch metadata result = self.simulate_get(gumshoe_queue_path_metadata, project_id) - result_doc = json.loads(result[0]) + result_doc = jsonutils.loads(result[0]) self.assertEqual(self.srmock.status, falcon.HTTP_200) - self.assertEqual(result_doc, json.loads(doc)) + self.assertEqual(result_doc, jsonutils.loads(doc)) # Stats empty queue self.simulate_get(gumshoe_queue_path_stats, project_id) @@ -209,8 +208,8 @@ class QueueLifecycleBaseTest(base.TestBase): # Get result = self.simulate_get(self.fizbat_queue_path_metadata, '480924') - result_doc = json.loads(result[0]) - self.assertEqual(result_doc, json.loads(doc)) + result_doc = jsonutils.loads(result[0]) + self.assertEqual(result_doc, jsonutils.loads(doc)) self.assertEqual(self.srmock.status, falcon.HTTP_200) def test_update_metadata(self): @@ -234,9 +233,9 @@ class QueueLifecycleBaseTest(base.TestBase): # Get result = self.simulate_get(xyz_queue_path_metadata, project_id) - result_doc = json.loads(result[0]) + result_doc = jsonutils.loads(result[0]) - self.assertEqual(result_doc, json.loads(doc2)) + self.assertEqual(result_doc, jsonutils.loads(doc2)) self.assertEqual(self.srmock.headers_dict['Content-Location'], xyz_queue_path_metadata) @@ -276,7 +275,7 @@ class QueueLifecycleBaseTest(base.TestBase): result = self.simulate_get(self.queue_path, None, query_string='limit=2&detailed=true') - result_doc = json.loads(result[0]) + result_doc = jsonutils.loads(result[0]) queues = result_doc['queues'] self.assertEqual(len(queues), 2) @@ -287,14 +286,14 @@ class QueueLifecycleBaseTest(base.TestBase): result = self.simulate_get(self.queue_path, project_id, query_string='limit=2') - result_doc = json.loads(result[0]) + result_doc = jsonutils.loads(result[0]) self.assertEqual(len(result_doc['queues']), 2) # List (no metadata, get all) result = self.simulate_get(self.queue_path, project_id, query_string='limit=5') - result_doc = json.loads(result[0]) + result_doc = jsonutils.loads(result[0]) [target, params] = result_doc['links'][0]['href'].split('?') self.assertEqual(self.srmock.status, falcon.HTTP_200) @@ -319,12 +318,12 @@ class QueueLifecycleBaseTest(base.TestBase): query_string='detailed=true') self.assertEqual(self.srmock.status, falcon.HTTP_200) - result_doc = json.loads(result[0]) + result_doc = jsonutils.loads(result[0]) [target, params] = result_doc['links'][0]['href'].split('?') queue = result_doc['queues'][0] result = self.simulate_get(queue['href'] + '/metadata', project_id) - result_doc = json.loads(result[0]) + result_doc = jsonutils.loads(result[0]) self.assertEqual(result_doc, queue['metadata']) self.assertEqual(result_doc, {'node': 31}) @@ -376,9 +375,9 @@ class TestQueueLifecycleFaultyDriver(base.TestBaseFaulty): self.assertNotIn(location, self.srmock.headers) result = self.simulate_get(gumshoe_queue_path + '/metadata', '480924') - result_doc = json.loads(result[0]) + result_doc = jsonutils.loads(result[0]) self.assertEqual(self.srmock.status, falcon.HTTP_503) - self.assertNotEqual(result_doc, json.loads(doc)) + self.assertNotEqual(result_doc, jsonutils.loads(doc)) self.simulate_get(gumshoe_queue_path + '/stats', '480924') self.assertEqual(self.srmock.status, falcon.HTTP_503) diff --git a/marconi/tests/queues/transport/wsgi/test_shards.py b/marconi/tests/queues/transport/wsgi/test_shards.py index d8ceb2567..561e6f8ed 100644 --- a/marconi/tests/queues/transport/wsgi/test_shards.py +++ b/marconi/tests/queues/transport/wsgi/test_shards.py @@ -20,6 +20,7 @@ import ddt import falcon from . import base # noqa +from marconi.openstack.common import jsonutils from marconi import tests as testing @@ -137,7 +138,7 @@ class ShardsBaseTest(base.TestBase): result = self.simulate_get(self.shard) self.assertEqual(self.srmock.status, falcon.HTTP_200) - doc = json.loads(result[0]) + doc = jsonutils.loads(result[0]) self.assertEqual(doc['weight'], expect['weight']) self.assertEqual(doc['uri'], expect['uri']) @@ -163,7 +164,7 @@ class ShardsBaseTest(base.TestBase): def test_get_works(self): result = self.simulate_get(self.shard) self.assertEqual(self.srmock.status, falcon.HTTP_200) - shard = json.loads(result[0]) + shard = jsonutils.loads(result[0]) self._shard_expect(shard, self.shard, self.doc['weight'], self.doc['uri']) @@ -171,7 +172,7 @@ class ShardsBaseTest(base.TestBase): result = self.simulate_get(self.shard, query_string='?detailed=True') self.assertEqual(self.srmock.status, falcon.HTTP_200) - shard = json.loads(result[0]) + shard = jsonutils.loads(result[0]) self._shard_expect(shard, self.shard, self.doc['weight'], self.doc['uri']) self.assertIn('options', shard) @@ -190,7 +191,7 @@ class ShardsBaseTest(base.TestBase): result = self.simulate_get(self.shard, query_string='?detailed=True') self.assertEqual(self.srmock.status, falcon.HTTP_200) - shard = json.loads(result[0]) + shard = jsonutils.loads(result[0]) self._shard_expect(shard, self.shard, doc['weight'], doc['uri']) self.assertEqual(shard['options'], doc['options']) @@ -245,7 +246,7 @@ class ShardsBaseTest(base.TestBase): result = self.simulate_get(self.url_prefix + '/shards', query_string=query) self.assertEqual(self.srmock.status, falcon.HTTP_200) - results = json.loads(result[0]) + results = jsonutils.loads(result[0]) self.assertIsInstance(results, dict) self.assertIn('shards', results) shard_list = results['shards'] @@ -283,7 +284,7 @@ class ShardsBaseTest(base.TestBase): result = self.simulate_get(self.url_prefix + '/shards', query_string='?marker=3') self.assertEqual(self.srmock.status, falcon.HTTP_200) - shard_list = json.loads(result[0])['shards'] + shard_list = jsonutils.loads(result[0])['shards'] self.assertEqual(len(shard_list), 6) path, weight = expected[4][:2] self._shard_expect(shard_list[0], path, weight, self.doc['uri'])