fix: error checking inheritance when using PEP585 and PEP604 type hints (#3681)

* Add tests

* Fix the issue

* Add changes file

* Improved convert_generics

* Add default fallback to convert_generics
Improved Annotated and Literal handling

* Fix Cython doesn't support generic types (PEP560)
Watch cython issue cython/cython#2753
Previous implementation can be used after cython 3.0 release

* Add custom type test

* Cosmetic fixes

Co-authored-by: Samuel Colvin <samcolvin@gmail.com>

* Fix typos

* Add SelfReferencing test validation
Add parametrization to

* Fix: parametrization caused test discovery problem

* Better explanation for a test case

* Better assertions for model creation tests

* Rerun CI

Co-authored-by: Samuel Colvin <samcolvin@gmail.com>

changes/3681-aleksul.md

@@ -0,0 +1 @@
+Fixed "error checking inheritance of" when using PEP585 and PEP604 type hints

pydantic/fields.py

@@ -35,6 +35,7 @@ from .typing import (
     Callable,
     ForwardRef,
     NoArgAnyCallable,
+    convert_generics,
     display_as_type,
     get_args,
     get_origin,
@@ -396,7 +397,7 @@ class ModelField(Representation):
         self.name: str = name
         self.has_alias: bool = bool(alias)
         self.alias: str = alias or name
-        self.type_: Any = type_
+        self.type_: Any = convert_generics(type_)
         self.outer_type_: Any = type_
         self.class_validators = class_validators or {}
         self.default: Any = default

pydantic/typing.py

@@ -38,6 +38,12 @@ except ImportError:
     # python < 3.9 does not have GenericAlias (list[int], tuple[str, ...] and so on)
     TypingGenericAlias = ()
 
+try:
+    from types import UnionType as TypesUnionType  # type: ignore
+except ImportError:
+    # python < 3.10 does not have UnionType (str | int, byte | bool and so on)
+    TypesUnionType = ()
+
 
 if sys.version_info < (3, 9):
 
@@ -145,6 +151,63 @@ else:
         return _typing_get_args(tp) or getattr(tp, '__args__', ()) or _generic_get_args(tp)
 
 
+if sys.version_info < (3, 9):
+
+    def convert_generics(tp: Type[Any]) -> Type[Any]:
+        """Python 3.9 and older only supports generics from `typing` module.
+        They convert strings to ForwardRef automatically.
+
+        Examples::
+            typing.List['Hero'] == typing.List[ForwardRef('Hero')]
+        """
+        return tp
+
+else:
+    from typing import _UnionGenericAlias  # type: ignore
+
+    from typing_extensions import _AnnotatedAlias
+
+    def convert_generics(tp: Type[Any]) -> Type[Any]:
+        """
+        Recursively searches for `str` type hints and replaces them with ForwardRef.
+
+        Examples::
+            convert_generics(list['Hero']) == list[ForwardRef('Hero')]
+            convert_generics(dict['Hero', 'Team']) == dict[ForwardRef('Hero'), ForwardRef('Team')]
+            convert_generics(typing.Dict['Hero', 'Team']) == typing.Dict[ForwardRef('Hero'), ForwardRef('Team')]
+            convert_generics(list[str | 'Hero'] | int) == list[str | ForwardRef('Hero')] | int
+        """
+        origin = get_origin(tp)
+        if not origin or not hasattr(tp, '__args__'):
+            return tp
+
+        args = get_args(tp)
+
+        # typing.Annotated needs special treatment
+        if origin is Annotated:
+            return _AnnotatedAlias(convert_generics(args[0]), args[1:])
+
+        # recursively replace `str` instances inside of `GenericAlias` with `ForwardRef(arg)`
+        converted = tuple(
+            ForwardRef(arg) if isinstance(arg, str) and isinstance(tp, TypingGenericAlias) else convert_generics(arg)
+            for arg in args
+        )
+
+        if converted == args:
+            return tp
+        elif isinstance(tp, TypingGenericAlias):
+            return TypingGenericAlias(origin, converted)
+        elif isinstance(tp, TypesUnionType):
+            # recreate types.UnionType (PEP604, Python >= 3.10)
+            return _UnionGenericAlias(origin, converted)
+        else:
+            try:
+                setattr(tp, '__args__', converted)
+            except AttributeError:
+                pass
+            return tp
+
+
 if sys.version_info < (3, 10):
 
     def is_union(tp: Optional[Type[Any]]) -> bool:

tests/test_forward_ref.py

@@ -672,6 +672,48 @@ class User(BaseModel):
     assert m.json(models_as_dict=False) == '{"name": "anne", "friends": ["User(ben)", "User(charlie)"]}'
 
 
+skip_pep585 = pytest.mark.skipif(
+    sys.version_info < (3, 9), reason='PEP585 generics only supported for python 3.9 and above'
+)
+
+
+@skip_pep585
+def test_pep585_self_referencing_generics():
+    class SelfReferencing(BaseModel):
+        names: list['SelfReferencing']  # noqa: F821
+
+    SelfReferencing.update_forward_refs()  # will raise an exception if the forward ref isn't resolvable
+    # test the class
+    assert SelfReferencing.__fields__['names'].type_ is SelfReferencing
+    # NOTE: outer_type_ is not converted
+    assert SelfReferencing.__fields__['names'].outer_type_ == list['SelfReferencing']
+    # test that object creation works
+    obj = SelfReferencing(names=[SelfReferencing(names=[])])
+    assert obj.names == [SelfReferencing(names=[])]
+
+
+@skip_pep585
+def test_pep585_recursive_generics(create_module):
+    @create_module
+    def module():
+        from pydantic import BaseModel
+
+        class Team(BaseModel):
+            name: str
+            heroes: list['Hero']  # noqa: F821
+
+        class Hero(BaseModel):
+            name: str
+            teams: list[Team]
+
+        Team.update_forward_refs()
+
+    assert module.Team.__fields__['heroes'].type_ is module.Hero
+    assert module.Hero.__fields__['teams'].type_ is module.Team
+
+    module.Hero(name='Ivan', teams=[module.Team(name='TheBest', heroes=[])])
+
+
 @pytest.mark.skipif(sys.version_info < (3, 9), reason='needs 3.9 or newer')
 def test_class_var_forward_ref(create_module):
     # see #3679

tests/test_typing.py

@@ -1,9 +1,12 @@
+import sys
 from collections import namedtuple
-from typing import Callable as TypingCallable, NamedTuple
+from typing import Any, Callable as TypingCallable, Dict, ForwardRef, List, NamedTuple, NewType, Union  # noqa: F401
 
 import pytest
+from typing_extensions import Annotated  # noqa: F401
 
-from pydantic.typing import Literal, is_namedtuple, is_none_type, is_typeddict
+from pydantic import Field  # noqa: F401
+from pydantic.typing import Literal, convert_generics, is_namedtuple, is_none_type, is_typeddict
 
 try:
     from typing import TypedDict as typing_TypedDict
@@ -66,3 +69,58 @@ def test_is_none_type():
     # `collections.abc.Callable` (even with python >= 3.9) as they behave
     # differently
     assert is_none_type(TypingCallable) is False
+
+
+class Hero:
+    pass
+
+
+class Team:
+    pass
+
+
+@pytest.mark.skipif(sys.version_info < (3, 9), reason='PEP585 generics only supported for python 3.9 and above.')
+@pytest.mark.parametrize(
+    ['type_', 'expectations'],
+    [
+        ('int', 'int'),
+        ('Union[list["Hero"], int]', 'Union[list[ForwardRef("Hero")], int]'),
+        ('list["Hero"]', 'list[ForwardRef("Hero")]'),
+        ('dict["Hero", "Team"]', 'dict[ForwardRef("Hero"), ForwardRef("Team")]'),
+        ('dict["Hero", list["Team"]]', 'dict[ForwardRef("Hero"), list[ForwardRef("Team")]]'),
+        ('dict["Hero", List["Team"]]', 'dict[ForwardRef("Hero"), List[ForwardRef("Team")]]'),
+        ('Dict["Hero", list["Team"]]', 'Dict[ForwardRef("Hero"), list[ForwardRef("Team")]]'),
+        (
+            'Annotated[list["Hero"], Field(min_length=2)]',
+            'Annotated[list[ForwardRef("Hero")], Field(min_length=2)]',
+        ),
+    ],
+)
+def test_convert_generics(type_, expectations):
+    assert str(convert_generics(eval(type_))) == str(eval(expectations))
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason='NewType class was added in python 3.10.')
+def test_convert_generics_unsettable_args():
+    class User(NewType):
+
+        __origin__ = type(list[str])
+        __args__ = (list['Hero'],)
+
+        def __init__(self, name: str, tp: type) -> None:
+            super().__init__(name, tp)
+
+        def __setattr__(self, __name: str, __value: Any) -> None:
+            if __name == '__args__':
+                raise AttributeError  # will be thrown during the generics conversion
+            return super().__setattr__(__name, __value)
+
+    # tests that convert_generics will not throw an exception even if __args__ isn't settable
+    assert convert_generics(User('MyUser', str)).__args__ == (list['Hero'],)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason='PEP604 unions only supported for python 3.10 and above.')
+def test_convert_generics_pep604():
+    assert (
+        convert_generics(dict['Hero', list['Team']] | int) == dict[ForwardRef('Hero'), list[ForwardRef('Team')]] | int
+    )