Fix inconsistent exception type in response.json() method (#5856)

HISTORY.md

@@ -4,7 +4,12 @@ Release History
 dev
 ---
 
--   \[Short description of non-trivial change.\]
+- \[Short description of non-trivial change.\]
+
+- Added a `requests.exceptions.JSONDecodeError` to decrease inconsistencies
+  in the library. This gets raised in the `response.json()` method, and is
+  backwards compatible as it inherits from previously thrown exceptions.
+  Can be caught from `requests.exceptions.RequestException` as well.
 
 2.26.0 (2021-07-13)
 -------------------

docs/user/quickstart.rst

@@ -153,9 +153,9 @@ There's also a builtin JSON decoder, in case you're dealing with JSON data::
 
 In case the JSON decoding fails, ``r.json()`` raises an exception. For example, if
 the response gets a 204 (No Content), or if the response contains invalid JSON,
-attempting ``r.json()`` raises ``simplejson.JSONDecodeError`` if simplejson is
-installed or raises ``ValueError: No JSON object could be decoded`` on Python 2 or
-``json.JSONDecodeError`` on Python 3.
+attempting ``r.json()`` raises ``requests.exceptions.JSONDecodeError``. This wrapper exception
+provides interoperability for multiple exceptions that may be thrown by different
+python versions and json serialization libraries.
 
 It should be noted that the success of the call to ``r.json()`` does **not**
 indicate the success of the response. Some servers may return a JSON object in a

requests/__init__.py

@@ -139,7 +139,7 @@ from .status_codes import codes
 from .exceptions import (
     RequestException, Timeout, URLRequired,
     TooManyRedirects, HTTPError, ConnectionError,
-    FileModeWarning, ConnectTimeout, ReadTimeout
+    FileModeWarning, ConnectTimeout, ReadTimeout, JSONDecodeError
 )
 
 # Set default logging handler to avoid "No handler found" warnings.

requests/compat.py

@@ -28,8 +28,10 @@ is_py2 = (_ver[0] == 2)
 #: Python 3.x?
 is_py3 = (_ver[0] == 3)
 
+has_simplejson = False
 try:
     import simplejson as json
+    has_simplejson = True
 except ImportError:
     import json
 
@@ -49,13 +51,13 @@ if is_py2:
     # Keep OrderedDict for backwards compatibility.
     from collections import Callable, Mapping, MutableMapping, OrderedDict
 
-
     builtin_str = str
     bytes = str
     str = unicode
     basestring = basestring
     numeric_types = (int, long, float)
     integer_types = (int, long)
+    JSONDecodeError = ValueError
 
 elif is_py3:
     from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag
@@ -66,6 +68,10 @@ elif is_py3:
     # Keep OrderedDict for backwards compatibility.
     from collections import OrderedDict
     from collections.abc import Callable, Mapping, MutableMapping
+    if has_simplejson:
+        from simplejson import JSONDecodeError
+    else:
+        from json import JSONDecodeError
 
     builtin_str = str
     str = str

requests/exceptions.py

@@ -8,6 +8,8 @@ This module contains the set of Requests' exceptions.
 """
 from urllib3.exceptions import HTTPError as BaseHTTPError
 
+from .compat import JSONDecodeError as CompatJSONDecodeError
+
 
 class RequestException(IOError):
     """There was an ambiguous exception that occurred while handling your
@@ -29,6 +31,10 @@ class InvalidJSONError(RequestException):
     """A JSON error occurred."""
 
 
+class JSONDecodeError(InvalidJSONError, CompatJSONDecodeError):
+    """Couldn't decode the text into json"""
+
+
 class HTTPError(RequestException):
     """An HTTP error occurred."""
 

requests/models.py

@@ -29,7 +29,9 @@ from .auth import HTTPBasicAuth
 from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar
 from .exceptions import (
     HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError,
-    ContentDecodingError, ConnectionError, StreamConsumedError, InvalidJSONError)
+    ContentDecodingError, ConnectionError, StreamConsumedError,
+    InvalidJSONError)
+from .exceptions import JSONDecodeError as RequestsJSONDecodeError
 from ._internal_utils import to_native_string, unicode_is_ascii
 from .utils import (
     guess_filename, get_auth_from_url, requote_uri,
@@ -38,7 +40,7 @@ from .utils import (
 from .compat import (
     Callable, Mapping,
     cookielib, urlunparse, urlsplit, urlencode, str, bytes,
-    is_py2, chardet, builtin_str, basestring)
+    is_py2, chardet, builtin_str, basestring, JSONDecodeError)
 from .compat import json as complexjson
 from .status_codes import codes
 
@@ -468,9 +470,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
             content_type = 'application/json'
 
             try:
-              body = complexjson.dumps(json, allow_nan=False)
+                body = complexjson.dumps(json, allow_nan=False)
             except ValueError as ve:
-              raise InvalidJSONError(ve, request=self)
+                raise InvalidJSONError(ve, request=self)
 
             if not isinstance(body, bytes):
                 body = body.encode('utf-8')
@@ -882,12 +884,8 @@ class Response(object):
         r"""Returns the json-encoded content of a response, if any.
 
         :param \*\*kwargs: Optional arguments that ``json.loads`` takes.
-        :raises simplejson.JSONDecodeError: If the response body does not
-            contain valid json and simplejson is installed.
-        :raises json.JSONDecodeError: If the response body does not contain
-            valid json and simplejson is not installed on Python 3.
-        :raises ValueError: If the response body does not contain valid
-            json and simplejson is not installed on Python 2.        
+        :raises requests.exceptions.JSONDecodeError: If the response body does not
+            contain valid json.
         """
 
         if not self.encoding and self.content and len(self.content) > 3:
@@ -907,7 +905,16 @@ class Response(object):
                     # and the server didn't bother to tell us what codec *was*
                     # used.
                     pass
-        return complexjson.loads(self.text, **kwargs)
+
+        try:
+            return complexjson.loads(self.text, **kwargs)
+        except JSONDecodeError as e:
+            # Catch JSON-related errors and raise as requests.JSONDecodeError
+            # This aliases json.JSONDecodeError and simplejson.JSONDecodeError
+            if is_py2: # e is a ValueError
+                raise RequestsJSONDecodeError(e.message)
+            else:
+                raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
 
     @property
     def links(self):

tests/test_requests.py

@@ -2570,4 +2570,9 @@ class TestPreparingURLs(object):
     def test_post_json_nan(self, httpbin):
         data = {"foo": float("nan")}
         with pytest.raises(requests.exceptions.InvalidJSONError):
-          r = requests.post(httpbin('post'), json=data)
+          r = requests.post(httpbin('post'), json=data)
+
+    def test_json_decode_compatibility(self, httpbin):
+        r = requests.get(httpbin('bytes/20'))
+        with pytest.raises(requests.exceptions.JSONDecodeError):
+            r.json()