From f69012a5aae2d22162dfbd17c3d9e77f5165204b Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 14 May 2022 18:26:12 +0200 Subject: [PATCH] fix: `error checking inheritance` when using PEP585 and PEP604 type hints (#3681) * Add tests * Fix the issue * Add changes file * Improved convert_generics * Add default fallback to convert_generics Improved Annotated and Literal handling * Fix Cython doesn't support generic types (PEP560) Watch cython issue cython/cython#2753 Previous implementation can be used after cython 3.0 release * Add custom type test * Cosmetic fixes Co-authored-by: Samuel Colvin * Fix typos * Add SelfReferencing test validation Add parametrization to * Fix: parametrization caused test discovery problem * Better explanation for a test case * Better assertions for model creation tests * Rerun CI Co-authored-by: Samuel Colvin --- changes/3681-aleksul.md | 1 + pydantic/fields.py | 3 +- pydantic/typing.py | 63 +++++++++++++++++++++++++++++++++++++++ tests/test_forward_ref.py | 42 ++++++++++++++++++++++++++ tests/test_typing.py | 62 ++++++++++++++++++++++++++++++++++++-- 5 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 changes/3681-aleksul.md diff --git a/changes/3681-aleksul.md b/changes/3681-aleksul.md new file mode 100644 index 0000000..13f745a --- /dev/null +++ b/changes/3681-aleksul.md @@ -0,0 +1 @@ +Fixed "error checking inheritance of" when using PEP585 and PEP604 type hints diff --git a/pydantic/fields.py b/pydantic/fields.py index 081f922..3d8a0e1 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -35,6 +35,7 @@ from .typing import ( Callable, ForwardRef, NoArgAnyCallable, + convert_generics, display_as_type, get_args, get_origin, @@ -396,7 +397,7 @@ class ModelField(Representation): self.name: str = name self.has_alias: bool = bool(alias) self.alias: str = alias or name - self.type_: Any = type_ + self.type_: Any = convert_generics(type_) self.outer_type_: Any = type_ self.class_validators = class_validators or {} self.default: Any = default diff --git a/pydantic/typing.py b/pydantic/typing.py index c6e9da1..11c5932 100644 --- a/pydantic/typing.py +++ b/pydantic/typing.py @@ -38,6 +38,12 @@ except ImportError: # python < 3.9 does not have GenericAlias (list[int], tuple[str, ...] and so on) TypingGenericAlias = () +try: + from types import UnionType as TypesUnionType # type: ignore +except ImportError: + # python < 3.10 does not have UnionType (str | int, byte | bool and so on) + TypesUnionType = () + if sys.version_info < (3, 9): @@ -145,6 +151,63 @@ else: return _typing_get_args(tp) or getattr(tp, '__args__', ()) or _generic_get_args(tp) +if sys.version_info < (3, 9): + + def convert_generics(tp: Type[Any]) -> Type[Any]: + """Python 3.9 and older only supports generics from `typing` module. + They convert strings to ForwardRef automatically. + + Examples:: + typing.List['Hero'] == typing.List[ForwardRef('Hero')] + """ + return tp + +else: + from typing import _UnionGenericAlias # type: ignore + + from typing_extensions import _AnnotatedAlias + + def convert_generics(tp: Type[Any]) -> Type[Any]: + """ + Recursively searches for `str` type hints and replaces them with ForwardRef. + + Examples:: + convert_generics(list['Hero']) == list[ForwardRef('Hero')] + convert_generics(dict['Hero', 'Team']) == dict[ForwardRef('Hero'), ForwardRef('Team')] + convert_generics(typing.Dict['Hero', 'Team']) == typing.Dict[ForwardRef('Hero'), ForwardRef('Team')] + convert_generics(list[str | 'Hero'] | int) == list[str | ForwardRef('Hero')] | int + """ + origin = get_origin(tp) + if not origin or not hasattr(tp, '__args__'): + return tp + + args = get_args(tp) + + # typing.Annotated needs special treatment + if origin is Annotated: + return _AnnotatedAlias(convert_generics(args[0]), args[1:]) + + # recursively replace `str` instances inside of `GenericAlias` with `ForwardRef(arg)` + converted = tuple( + ForwardRef(arg) if isinstance(arg, str) and isinstance(tp, TypingGenericAlias) else convert_generics(arg) + for arg in args + ) + + if converted == args: + return tp + elif isinstance(tp, TypingGenericAlias): + return TypingGenericAlias(origin, converted) + elif isinstance(tp, TypesUnionType): + # recreate types.UnionType (PEP604, Python >= 3.10) + return _UnionGenericAlias(origin, converted) + else: + try: + setattr(tp, '__args__', converted) + except AttributeError: + pass + return tp + + if sys.version_info < (3, 10): def is_union(tp: Optional[Type[Any]]) -> bool: diff --git a/tests/test_forward_ref.py b/tests/test_forward_ref.py index 969c3f5..5406215 100644 --- a/tests/test_forward_ref.py +++ b/tests/test_forward_ref.py @@ -672,6 +672,48 @@ class User(BaseModel): assert m.json(models_as_dict=False) == '{"name": "anne", "friends": ["User(ben)", "User(charlie)"]}' +skip_pep585 = pytest.mark.skipif( + sys.version_info < (3, 9), reason='PEP585 generics only supported for python 3.9 and above' +) + + +@skip_pep585 +def test_pep585_self_referencing_generics(): + class SelfReferencing(BaseModel): + names: list['SelfReferencing'] # noqa: F821 + + SelfReferencing.update_forward_refs() # will raise an exception if the forward ref isn't resolvable + # test the class + assert SelfReferencing.__fields__['names'].type_ is SelfReferencing + # NOTE: outer_type_ is not converted + assert SelfReferencing.__fields__['names'].outer_type_ == list['SelfReferencing'] + # test that object creation works + obj = SelfReferencing(names=[SelfReferencing(names=[])]) + assert obj.names == [SelfReferencing(names=[])] + + +@skip_pep585 +def test_pep585_recursive_generics(create_module): + @create_module + def module(): + from pydantic import BaseModel + + class Team(BaseModel): + name: str + heroes: list['Hero'] # noqa: F821 + + class Hero(BaseModel): + name: str + teams: list[Team] + + Team.update_forward_refs() + + assert module.Team.__fields__['heroes'].type_ is module.Hero + assert module.Hero.__fields__['teams'].type_ is module.Team + + module.Hero(name='Ivan', teams=[module.Team(name='TheBest', heroes=[])]) + + @pytest.mark.skipif(sys.version_info < (3, 9), reason='needs 3.9 or newer') def test_class_var_forward_ref(create_module): # see #3679 diff --git a/tests/test_typing.py b/tests/test_typing.py index af0b8d9..9e59365 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -1,9 +1,12 @@ +import sys from collections import namedtuple -from typing import Callable as TypingCallable, NamedTuple +from typing import Any, Callable as TypingCallable, Dict, ForwardRef, List, NamedTuple, NewType, Union # noqa: F401 import pytest +from typing_extensions import Annotated # noqa: F401 -from pydantic.typing import Literal, is_namedtuple, is_none_type, is_typeddict +from pydantic import Field # noqa: F401 +from pydantic.typing import Literal, convert_generics, is_namedtuple, is_none_type, is_typeddict try: from typing import TypedDict as typing_TypedDict @@ -66,3 +69,58 @@ def test_is_none_type(): # `collections.abc.Callable` (even with python >= 3.9) as they behave # differently assert is_none_type(TypingCallable) is False + + +class Hero: + pass + + +class Team: + pass + + +@pytest.mark.skipif(sys.version_info < (3, 9), reason='PEP585 generics only supported for python 3.9 and above.') +@pytest.mark.parametrize( + ['type_', 'expectations'], + [ + ('int', 'int'), + ('Union[list["Hero"], int]', 'Union[list[ForwardRef("Hero")], int]'), + ('list["Hero"]', 'list[ForwardRef("Hero")]'), + ('dict["Hero", "Team"]', 'dict[ForwardRef("Hero"), ForwardRef("Team")]'), + ('dict["Hero", list["Team"]]', 'dict[ForwardRef("Hero"), list[ForwardRef("Team")]]'), + ('dict["Hero", List["Team"]]', 'dict[ForwardRef("Hero"), List[ForwardRef("Team")]]'), + ('Dict["Hero", list["Team"]]', 'Dict[ForwardRef("Hero"), list[ForwardRef("Team")]]'), + ( + 'Annotated[list["Hero"], Field(min_length=2)]', + 'Annotated[list[ForwardRef("Hero")], Field(min_length=2)]', + ), + ], +) +def test_convert_generics(type_, expectations): + assert str(convert_generics(eval(type_))) == str(eval(expectations)) + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason='NewType class was added in python 3.10.') +def test_convert_generics_unsettable_args(): + class User(NewType): + + __origin__ = type(list[str]) + __args__ = (list['Hero'],) + + def __init__(self, name: str, tp: type) -> None: + super().__init__(name, tp) + + def __setattr__(self, __name: str, __value: Any) -> None: + if __name == '__args__': + raise AttributeError # will be thrown during the generics conversion + return super().__setattr__(__name, __value) + + # tests that convert_generics will not throw an exception even if __args__ isn't settable + assert convert_generics(User('MyUser', str)).__args__ == (list['Hero'],) + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason='PEP604 unions only supported for python 3.10 and above.') +def test_convert_generics_pep604(): + assert ( + convert_generics(dict['Hero', list['Team']] | int) == dict[ForwardRef('Hero'), list[ForwardRef('Team')]] | int + )