Typecheck Json inner type (#4332)

* Add .venv/ to .gitignore

* Allow typecheckers to infer Json inner type

* Fix and improve mypy tests

* Add type tests

* Add Json[Any] case to schema test

* Update example in docs

* Add changes file

* Use <3.9 compatible annotations for tests

.gitignore

@@ -1,6 +1,7 @@
 .idea/
 env/
 venv/
+.venv/
 env3*/
 Pipfile
 *.lock

changes/4332-Bobronium.md

@@ -0,0 +1,3 @@
+Allow type checkers to infer inner type of `Json` type. `Json[list[str]]` will be now inferred as `list[str]`. 
+`Json[Any]` should be used instead of plain `Json`.
+Runtime behaviour is not changed.

docs/examples/types_json_type.py

@@ -1,29 +1,29 @@
-from typing import List
+from typing import Any, List
 
 from pydantic import BaseModel, Json, ValidationError
 
 
-class SimpleJsonModel(BaseModel):
-    json_obj: Json
+class AnyJsonModel(BaseModel):
+    json_obj: Json[Any]
 
 
-class ComplexJsonModel(BaseModel):
+class ConstrainedJsonModel(BaseModel):
     json_obj: Json[List[int]]
 
 
-print(SimpleJsonModel(json_obj='{"b": 1}'))
-print(ComplexJsonModel(json_obj='[1, 2, 3]'))
+print(AnyJsonModel(json_obj='{"b": 1}'))
+print(ConstrainedJsonModel(json_obj='[1, 2, 3]'))
 try:
-    ComplexJsonModel(json_obj=12)
+    ConstrainedJsonModel(json_obj=12)
 except ValidationError as e:
     print(e)
 
 try:
-    ComplexJsonModel(json_obj='[a, b]')
+    ConstrainedJsonModel(json_obj='[a, b]')
 except ValidationError as e:
     print(e)
 
 try:
-    ComplexJsonModel(json_obj='["a", "b"]')
+    ConstrainedJsonModel(json_obj='["a", "b"]')
 except ValidationError as e:
     print(e)

pydantic/types.py

@@ -114,6 +114,8 @@ OptionalIntFloatDecimal = Union[OptionalIntFloat, Decimal]
 StrIntFloat = Union[str, int, float]
 
 if TYPE_CHECKING:
+    from typing_extensions import Annotated
+
     from .dataclasses import Dataclass
     from .main import BaseModel
     from .typing import CallableGenerator
@@ -791,11 +793,14 @@ class JsonWrapper:
 
 class JsonMeta(type):
     def __getitem__(self, t: Type[Any]) -> Type[JsonWrapper]:
+        if t is Any:
+            return Json  # allow Json[Any] to replecate plain Json
         return _registered(type('JsonWrapperValue', (JsonWrapper,), {'inner_type': t}))
 
 
 if TYPE_CHECKING:
-    Json = str
+    Json = Annotated[T, ...]  # Json[list[str]] will be recognized by type checkers as list[str]
+
 else:
 
     class Json(metaclass=JsonMeta):

tests/mypy/modules/fail1.py

@@ -5,6 +5,7 @@ from datetime import datetime
 from typing import List, Optional
 
 from pydantic import BaseModel, NoneStr
+from pydantic.types import Json
 
 
 class Model(BaseModel):
@@ -13,8 +14,10 @@ class Model(BaseModel):
     last_name: NoneStr = None
     signup_ts: Optional[datetime] = None
     list_of_ints: List[int]
+    json_list_of_ints: Json[List[int]]
 
 
 m = Model(age=42, list_of_ints=[1, '2', b'3'])
 
 print(m.age + 'not integer')
+m.json_list_of_ints[0] + 'not integer'

tests/mypy/modules/success.py

@@ -230,7 +230,8 @@ class PydanticTypes(BaseModel):
     my_dir_path: DirectoryPath = Path('.')
     my_dir_path_str: DirectoryPath = '.'  # type: ignore
     # Json
-    my_json: Json = '{"hello": "world"}'
+    my_json: Json[Dict[str, str]] = '{"hello": "world"}'  # type: ignore
+    my_json_list: Json[List[str]] = '["hello", "world"]'  # type: ignore
     # Date
     my_past_date: PastDate = date.today() - timedelta(1)
     my_future_date: FutureDate = date.today() + timedelta(1)
@@ -248,6 +249,8 @@ validated.my_file_path.absolute()
 validated.my_file_path_str.absolute()
 validated.my_dir_path.absolute()
 validated.my_dir_path_str.absolute()
+validated.my_json['hello'].capitalize()
+validated.my_json_list[0].capitalize()
 
 stricturl(allowed_schemes={'http'})
 stricturl(allowed_schemes=frozenset({'http'}))

tests/mypy/outputs/fail1.txt

@@ -1 +1,2 @@
-20: error: Unsupported operand types for + ("int" and "str")  [operator]
+22: error: Unsupported operand types for + ("int" and "str")  [operator]
+23: error: Unsupported operand types for + ("int" and "str")  [operator]

tests/test_schema.py

@@ -951,6 +951,7 @@ def test_json_type():
     class Model(BaseModel):
         a: Json
         b: Json[int]
+        c: Json[Any]
 
     assert Model.schema() == {
         'title': 'Model',
@@ -958,6 +959,7 @@ def test_json_type():
         'properties': {
             'a': {'title': 'A', 'type': 'string', 'format': 'json-string'},
             'b': {'title': 'B', 'type': 'integer'},
+            'c': {'title': 'C', 'type': 'string', 'format': 'json-string'},
         },
         'required': ['b'],
     }

tests/test_types.py

@@ -9,6 +9,7 @@ from decimal import Decimal
 from enum import Enum, IntEnum
 from pathlib import Path
 from typing import (
+    Any,
     Deque,
     Dict,
     FrozenSet,
@@ -2342,6 +2343,11 @@ def test_new_type_fails():
     ]
 
 
+def test_json_any_is_json():
+    """Mypy doesn't allow plain Json, so Json[Any] must behave just as Json did."""
+    assert Json[Any] is Json
+
+
 def test_valid_simple_json():
     class JsonModel(BaseModel):
         json_obj: Json
@@ -2350,6 +2356,14 @@ def test_valid_simple_json():
     assert JsonModel(json_obj=obj).dict() == {'json_obj': {'a': 1, 'b': [2, 3]}}
 
 
+def test_valid_simple_json_any():
+    class JsonModel(BaseModel):
+        json_obj: Json[Any]
+
+    obj = '{"a": 1, "b": [2, 3]}'
+    assert JsonModel(json_obj=obj).dict() == {'json_obj': {'a': 1, 'b': [2, 3]}}
+
+
 def test_invalid_simple_json():
     class JsonModel(BaseModel):
         json_obj: Json
@@ -2360,6 +2374,16 @@ def test_invalid_simple_json():
     assert exc_info.value.errors()[0] == {'loc': ('json_obj',), 'msg': 'Invalid JSON', 'type': 'value_error.json'}
 
 
+def test_invalid_simple_json_any():
+    class JsonModel(BaseModel):
+        json_obj: Json[Any]
+
+    obj = '{a: 1, b: [2, 3]}'
+    with pytest.raises(ValidationError) as exc_info:
+        JsonModel(json_obj=obj)
+    assert exc_info.value.errors()[0] == {'loc': ('json_obj',), 'msg': 'Invalid JSON', 'type': 'value_error.json'}
+
+
 def test_valid_simple_json_bytes():
     class JsonModel(BaseModel):
         json_obj: Json
@@ -2394,6 +2418,38 @@ def test_valid_detailed_json_bytes():
     assert JsonDetailedModel(json_obj=obj).dict() == {'json_obj': [1, 2, 3]}
 
 
+def test_valid_model_json():
+    class Model(BaseModel):
+        a: int
+        b: List[int]
+
+    class JsonDetailedModel(BaseModel):
+        json_obj: Json[Model]
+
+    obj = '{"a": 1, "b": [2, 3]}'
+    m = JsonDetailedModel(json_obj=obj)
+    assert isinstance(m.json_obj, Model)
+    assert m.json_obj.a == 1
+    assert m.dict() == {'json_obj': {'a': 1, 'b': [2, 3]}}
+
+
+def test_invalid_model_json():
+    class Model(BaseModel):
+        a: int
+        b: List[int]
+
+    class JsonDetailedModel(BaseModel):
+        json_obj: Json[Model]
+
+    obj = '{"a": 1, "c": [2, 3]}'
+    with pytest.raises(ValidationError) as exc_info:
+        JsonDetailedModel(json_obj=obj)
+
+    assert exc_info.value.errors() == [
+        {'loc': ('json_obj', 'b'), 'msg': 'field required', 'type': 'value_error.missing'}
+    ]
+
+
 def test_invalid_detailed_json_type_error():
     class JsonDetailedModel(BaseModel):
         json_obj: Json[List[int]]