From 386382b18caab6a3a8a111fa8e24875dca27b0aa Mon Sep 17 00:00:00 2001 From: Jesse Shapiro Date: Wed, 4 May 2016 20:09:07 -0400 Subject: [PATCH 1/4] Encoding JSON requests to bytes for urllib3 to handle; ensuring same with testing. --- AUTHORS.rst | 1 + requests/models.py | 7 +++++-- tests/test_requests.py | 12 ++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index e684d850..37b66698 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -165,3 +165,4 @@ Patches and Suggestions - Brian Samek (`@bsamek `_) - Dmitry Dygalo (`@Stranger6667 `_) - piotrjurkiewicz +- Jesse Shapiro (`@haikuginger `_) diff --git a/requests/models.py b/requests/models.py index fe4bec1b..05ec3e47 100644 --- a/requests/models.py +++ b/requests/models.py @@ -420,8 +420,11 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): length = None if not data and json is not None: - content_type = 'application/json' - body = complexjson.dumps(json) + # When urllib3 uses pyOpenSSL, it can only resume large uploads + # properly if receiving a bytes-like object. In Python 2, json.dumps() + # returns just that, but Python 3 returns a Unicode string. + content_type = 'application/json; charset=utf-8' + body = complexjson.dumps(json).encode('utf-8') is_stream = all([ hasattr(data, '__iter__'), diff --git a/tests/test_requests.py b/tests/test_requests.py index 0a87b52c..427675d5 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1516,6 +1516,18 @@ class RedirectSession(SessionRedirectMixin): return string +def test_json_encodes_as_bytes(): + # urllib3 expects bodies as bytes-like objects + body = {"key": "value"} + p = PreparedRequest() + p.prepare( + method='GET', + url='https://www.example.com/', + json='body' + ) + assert isinstance(p.body, bytes) + + def test_requests_are_updated_each_time(httpbin): session = RedirectSession([303, 307]) prep = requests.Request('POST', httpbin('post')).prepare() From 9ff2e43cd6de340c152145adc0e922f94e0aaf1c Mon Sep 17 00:00:00 2001 From: Jesse Shapiro Date: Thu, 5 May 2016 06:27:12 -0400 Subject: [PATCH 2/4] Removing charset from JSON content type; tightening requirements on .encode() --- requests/models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 05ec3e47..2341c1b8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -423,8 +423,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): # When urllib3 uses pyOpenSSL, it can only resume large uploads # properly if receiving a bytes-like object. In Python 2, json.dumps() # returns just that, but Python 3 returns a Unicode string. - content_type = 'application/json; charset=utf-8' - body = complexjson.dumps(json).encode('utf-8') + content_type = 'application/json' + body = complexjson.dumps(json) + if not isinstance(body, bytes): + body = body.encode('utf-8') is_stream = all([ hasattr(data, '__iter__'), From 52c0daff5f87adb1e89678139fdd054c8005e580 Mon Sep 17 00:00:00 2001 From: Jesse Shapiro Date: Thu, 5 May 2016 12:12:49 -0400 Subject: [PATCH 3/4] Cleaning up comment on JSON encoding to be more strictly relevant. --- requests/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 2341c1b8..e22a67ad 100644 --- a/requests/models.py +++ b/requests/models.py @@ -420,9 +420,8 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): length = None if not data and json is not None: - # When urllib3 uses pyOpenSSL, it can only resume large uploads - # properly if receiving a bytes-like object. In Python 2, json.dumps() - # returns just that, but Python 3 returns a Unicode string. + # urllib3 requires a bytes-like body. Python 2's json.dumps + # provides this natively, but Python 3 gives a Unicode string. content_type = 'application/json' body = complexjson.dumps(json) if not isinstance(body, bytes): From 04bb965c87218c74437cd3e1c2fa09866b07b3c0 Mon Sep 17 00:00:00 2001 From: Jesse Shapiro Date: Thu, 5 May 2016 12:23:59 -0400 Subject: [PATCH 4/4] Fixing test; it was accomplishing the right thing, but doing it in the wrong way. --- tests/test_requests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_requests.py b/tests/test_requests.py index 427675d5..f5f233f7 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1523,7 +1523,7 @@ def test_json_encodes_as_bytes(): p.prepare( method='GET', url='https://www.example.com/', - json='body' + json=body ) assert isinstance(p.body, bytes)