From b84df079a7b8bd49762460b02fe87c2103c91da8 Mon Sep 17 00:00:00 2001 From: dmontagu <35119617+dmontagu@users.noreply.github.com> Date: Wed, 19 Jun 2019 03:22:06 -0700 Subject: [PATCH] Add generic functionality (#595), fix #556 * Added generic functionality * Skip tests in python 3.6 * double quote -> single quote * Simplified, with more comprehensive tests * double quote -> single quote * Remove unintentional file * Add caching * don't cythonize generics.py * Make work with mypy * Remove __parameters__ * double quote -> single quote i'll remember one day * More cleanup and validation * Removed unwanted file * A little more cleanup, and finish the PR * Add proper inheritance * Added note about inheritance to docs * Added error for double-parameterizing * Should build for python3.7 * Works with both 3.6 and 3.7 * Fixed bug with caching for single argument * handle __name__ for generic models * double quote -> single quote * Updated error messages --- HISTORY.rst | 1 + Makefile | 3 + docs/examples/generics.py | 61 +++++++++ docs/index.rst | 39 ++++++ pydantic/generics.py | 85 ++++++++++++ pydantic/schema.py | 2 + setup.py | 1 + tests/mypy_test_fails3.py | 22 ++++ tests/mypy_test_success.py | 23 +++- tests/test_generics.py | 259 +++++++++++++++++++++++++++++++++++++ 10 files changed, 495 insertions(+), 1 deletion(-) create mode 100644 docs/examples/generics.py create mode 100644 pydantic/generics.py create mode 100644 tests/mypy_test_fails3.py create mode 100644 tests/test_generics.py diff --git a/HISTORY.rst b/HISTORY.rst index 3666767..683ba2a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ v0.29 (unreleased) * Updated documentation to elucidate the usage of ``Union`` when defining multiple types under an attribute's annotation and showcase how the type-order can affect marshalling of provided values, #594 by @somada141 * add ``conlist`` type, #583 by @hmvp +* add support for generics, #595 by @dmontagu v0.28 (2019-06-06) .................. diff --git a/Makefile b/Makefile index edc0f9a..c34c7b6 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,9 @@ external-mypy: @mypy tests/mypy_test_fails2.py 1>/dev/null; \ test $$? -eq 1 || \ (echo "mypy_test_fails2: mypy passed when it should have failed!"; exit 1) + @mypy tests/mypy_test_fails3.py 1>/dev/null; \ + test $$? -eq 1 || \ + (echo "mypy_test_fails3: mypy passed when it should have failed!"; exit 1) .PHONY: testcov testcov: test diff --git a/docs/examples/generics.py b/docs/examples/generics.py new file mode 100644 index 0000000..987e342 --- /dev/null +++ b/docs/examples/generics.py @@ -0,0 +1,61 @@ +from typing import Generic, TypeVar, Optional, List + +from pydantic import BaseModel, validator, ValidationError +from pydantic.generics import GenericModel + + +DataT = TypeVar('DataT') + + +class Error(BaseModel): + code: int + message: str + + +class DataModel(BaseModel): + numbers: List[int] + people: List[str] + + +class Response(GenericModel, Generic[DataT]): + data: Optional[DataT] + error: Optional[Error] + + @validator('error', always=True) + def check_consistency(cls, v, values): + if v is not None and values['data'] is not None: + raise ValueError('must not provide both data and error') + if v is None and values.get('data') is None: + raise ValueError('must provide data or error') + return v + + +data = DataModel(numbers=[1, 2, 3], people=[]) +error = Error(code=404, message='Not found') + +print(Response[int](data=1)) +# > Response[int] data=1 error=None +print(Response[str](data='value')) +# > Response[str] data='value' error=None +print(Response[str](data='value').dict()) +# > {'data': 'value', 'error': None} +print(Response[DataModel](data=data).dict()) +# > {'data': {'numbers': [1, 2, 3], 'people': []}, 'error': None} +print(Response[DataModel](error=error).dict()) +# > {'data': None, 'error': {'code': 404, 'message': 'Not found'}} + +try: + Response[int](data='value') +except ValidationError as e: + print(e) +""" +4 validation errors +data + value is not a valid integer (type=type_error.integer) +data + value is not none (type=type_error.none.allowed) +error + value is not a valid dict (type=type_error.dict) +error + must provide data or error (type=value_error) +""" diff --git a/docs/index.rst b/docs/index.rst index a970fa1..7702bcd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -295,6 +295,45 @@ and pydantic versions). (This script is complete, it should run "as is") +.. _generic_models: + +Generic Models +.............. + +.. note:: + + New in version v0.29. + + This feature requires Python 3.7+. + +Pydantic supports the creation of generic models to make it easier to reuse a common model structure. + +In order to declare a generic model, you perform the following steps: + +* Declare one or more ``typing.TypeVar`` instances to use to parameterize your model. +* Declare a pydantic model that inherits from ``pydantic.generics.GenericModel`` and ``typing.Generic``, + where you pass the ``TypeVar`` instances as parameters to ``typing.Generic``. +* Use the ``TypeVar`` instances as annotations where you will want to replace them with other types or + pydantic models. + +Here is an example using ``GenericModel`` to create an easily-reused HTTP response payload wrapper: + +.. literalinclude:: examples/generics.py + +(This script is complete, it should run "as is") + +If you set ``Config`` or make use of ``validator`` in your generic model definition, it is applied +to concrete subclasses in the same way as when inheriting from ``BaseModel``. Any methods defined on +your generic class will also be inherited. + +Pydantic's generics also integrate properly with mypy, so you get all the type checking +you would expect mypy to provide if you were to declare the type without using ``GenericModel``. + +.. note:: + + Internally, pydantic uses ``create_model`` to generate a (cached) concrete ``BaseModel`` at runtime, + so there is essentially zero overhead introduced by making use of ``GenericModel``. + .. _orm_mode: ORM Mode (aka Arbitrary Class Instances) diff --git a/pydantic/generics.py b/pydantic/generics.py new file mode 100644 index 0000000..244a6ce --- /dev/null +++ b/pydantic/generics.py @@ -0,0 +1,85 @@ +from typing import Any, ClassVar, Dict, Generic, Tuple, Type, TypeVar, Union, get_type_hints + +from pydantic import BaseModel, create_model +from pydantic.class_validators import gather_validators + +_generic_types_cache: Dict[Tuple[Type[Any], Union[Any, Tuple[Any, ...]]], Type[BaseModel]] = {} +GenericModelT = TypeVar('GenericModelT', bound='GenericModel') + + +class GenericModel(BaseModel): + __slots__ = () + __concrete__: ClassVar[bool] = False + + def __new__(cls, *args: Any, **kwargs: Any) -> Any: + if cls.__concrete__: + return super().__new__(cls) + else: + raise TypeError(f'Type {cls.__name__} cannot be used without generic parameters, e.g. {cls.__name__}[T]') + + def __class_getitem__( # type: ignore + cls: Type[GenericModelT], params: Union[Type[Any], Tuple[Type[Any], ...]] + ) -> Type[BaseModel]: + cached = _generic_types_cache.get((cls, params)) + if cached is not None: + return cached + if cls.__concrete__: + raise TypeError('Cannot parameterize a concrete instantiation of a generic model') + if not isinstance(params, tuple): + params = (params,) + if any(isinstance(param, TypeVar) for param in params): # type: ignore + raise TypeError(f'Type parameters should be placed on typing.Generic, not GenericModel') + if Generic not in cls.__bases__: + raise TypeError(f'Type {cls.__name__} must inherit from typing.Generic before being parameterized') + + check_parameters_count(cls, params) + typevars_map: Dict[Any, Any] = dict(zip(cls.__parameters__, params)) # type: ignore + type_hints = get_type_hints(cls).items() + instance_type_hints = {k: v for k, v in type_hints if getattr(v, '__origin__', None) is not ClassVar} + concrete_type_hints: Dict[str, Type[Any]] = { + k: resolve_type_hint(v, typevars_map) for k, v in instance_type_hints.items() + } + + model_name = concrete_name(cls, params) + validators = gather_validators(cls) + fields: Dict[str, Tuple[Type[Any], Any]] = { + k: (v, getattr(cls, k, ...)) for k, v in concrete_type_hints.items() + } + created_model = create_model( + model_name=model_name, + __module__=cls.__module__, + __base__=cls, + __config__=None, + __validators__=validators, + **fields, + ) + created_model.Config = cls.Config + created_model.__concrete__ = True # type: ignore + _generic_types_cache[(cls, params)] = created_model + if len(params) == 1: + _generic_types_cache[(cls, params[0])] = created_model + return created_model + + +def concrete_name(cls: Type[Any], params: Tuple[Type[Any], ...]) -> str: + param_names = [param.__name__ if hasattr(param, '__name__') else str(param) for param in params] + params_component = ', '.join(param_names) + return f'{cls.__name__}[{params_component}]' + + +def resolve_type_hint(type_: Any, typevars_map: Dict[Any, Any]) -> Type[Any]: + if hasattr(type_, '__origin__'): + new_args = tuple(resolve_type_hint(x, typevars_map) for x in type_.__args__) + if type_.__origin__ is Union: + return type_.__origin__[new_args] + new_args = tuple([typevars_map[x] for x in type_.__parameters__]) + return type_[new_args] + return typevars_map.get(type_, type_) + + +def check_parameters_count(cls: Type[GenericModel], parameters: Tuple[Any, ...]) -> None: + actual = len(parameters) + expected = len(cls.__parameters__) # type: ignore + if actual != expected: + description = 'many' if actual > expected else 'few' + raise TypeError(f'Too {description} parameters for {cls.__name__}; actual {actual}, expected {expected}') diff --git a/pydantic/schema.py b/pydantic/schema.py index b1168fa..4e178bd 100644 --- a/pydantic/schema.py +++ b/pydantic/schema.py @@ -1,3 +1,4 @@ +import re import warnings from datetime import date, datetime, time, timedelta from decimal import Decimal @@ -354,6 +355,7 @@ def get_model_name_map(unique_models: Set[Type['BaseModel']]) -> Dict[Type['Base conflicting_names: Set[str] = set() for model in unique_models: model_name = model.__name__ + model_name = re.sub(r'[^a-zA-Z0-9.\-_]', '_', model_name) if model_name in conflicting_names: model_name = get_long_model_name(model) name_model_map[model_name] = model diff --git a/setup.py b/setup.py index 44c2cac..198daec 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ if not any(arg in sys.argv for arg in ['clean', 'check']) and 'SKIP_CYTHON' not os.environ['CFLAGS'] = '-O3' ext_modules = cythonize( 'pydantic/*.py', + exclude=['pydantic/generics.py'], nthreads=4, language_level=3, compiler_directives=compiler_directives, diff --git a/tests/mypy_test_fails3.py b/tests/mypy_test_fails3.py new file mode 100644 index 0000000..0af21ca --- /dev/null +++ b/tests/mypy_test_fails3.py @@ -0,0 +1,22 @@ +""" +Test mypy failure with invalid types. +""" +from typing import Generic, List, TypeVar + +from pydantic import BaseModel +from pydantic.generics import GenericModel + +T = TypeVar('T') + + +class Model(BaseModel): + list_of_ints: List[int] + + +class WrapperModel(GenericModel, Generic[T]): + payload: T + + +model_instance = Model(list_of_ints=[1]) +wrapper_instance = WrapperModel[Model](payload=model_instance) +wrapper_instance.payload.list_of_ints.append('1') diff --git a/tests/mypy_test_success.py b/tests/mypy_test_success.py index 714f1f8..b11007f 100644 --- a/tests/mypy_test_success.py +++ b/tests/mypy_test_success.py @@ -4,11 +4,13 @@ Test pydantic's compliance with mypy. Do a little skipping about with types to demonstrate its usage. """ import json +import sys from datetime import datetime -from typing import List, Optional +from typing import Generic, List, Optional, TypeVar from pydantic import BaseModel, NoneStr from pydantic.dataclasses import dataclass +from pydantic.generics import GenericModel class Model(BaseModel): @@ -82,3 +84,22 @@ class AddProject: p = AddProject(name='x', slug='y', description='z') + + +if sys.version_info >= (3, 7): + T = TypeVar('T') + + class WrapperModel(GenericModel, Generic[T]): + payload: T + + int_instance = WrapperModel[int](payload=1) + int_instance.payload += 1 + assert int_instance.payload == 2 + + str_instance = WrapperModel[str](payload='a') + str_instance.payload += 'a' + assert str_instance.payload == 'aa' + + model_instance = WrapperModel[Model](payload=m) + model_instance.payload.list_of_ints.append(4) + assert model_instance.payload.list_of_ints == [1, 2, 3, 4] diff --git a/tests/test_generics.py b/tests/test_generics.py new file mode 100644 index 0000000..eede582 --- /dev/null +++ b/tests/test_generics.py @@ -0,0 +1,259 @@ +import sys +from enum import Enum +from typing import Any, Dict, Generic, List, Optional, TypeVar, Union + +import pytest + +from pydantic import BaseModel, ValidationError, validator +from pydantic.generics import GenericModel, _generic_types_cache + +skip_36 = pytest.mark.skipif(sys.version_info < (3, 7), reason='generics only supported for python 3.7 and above') + + +@skip_36 +def test_generic_name(): + data_type = TypeVar('data_type') + + class Result(GenericModel, Generic[data_type]): + data: data_type + + assert Result[List[int]].__name__ == 'Result[typing.List[int]]' + + +@skip_36 +def test_double_parameterize_error(): + data_type = TypeVar('data_type') + + class Result(GenericModel, Generic[data_type]): + data: data_type + + with pytest.raises(TypeError) as exc_info: + Result[int][int] + + assert str(exc_info.value) == 'Cannot parameterize a concrete instantiation of a generic model' + + +@skip_36 +def test_methods_are_inherited(): + class CustomGenericModel(GenericModel): + def method(self): + return self.data + + T = TypeVar('T') + + class Model(CustomGenericModel, Generic[T]): + data: T + + instance = Model[int](data=1) + + assert instance.method() == 1 + + +@skip_36 +def test_config_is_inherited(): + class CustomGenericModel(GenericModel): + class Config: + allow_mutation = False + + T = TypeVar('T') + + class Model(CustomGenericModel, Generic[T]): + data: T + + instance = Model[int](data=1) + + with pytest.raises(TypeError) as exc_info: + instance.data = 2 + + assert str(exc_info.value) == '"Model[int]" is immutable and does not support item assignment' + + +@skip_36 +def test_must_inherit_from_generic(): + with pytest.raises(TypeError) as exc_info: + + class Result(GenericModel): + pass + + Result[int] + + assert str(exc_info.value) == f'Type Result must inherit from typing.Generic before being parameterized' + + +@skip_36 +def test_parameters_must_be_typevar(): + T = TypeVar('T') + with pytest.raises(TypeError) as exc_info: + + class Result(GenericModel[T]): + pass + + assert str(exc_info.value) == f'Type parameters should be placed on typing.Generic, not GenericModel' + + +@skip_36 +def test_parameter_count(): + T = TypeVar('T') + S = TypeVar('S') + + class Model(GenericModel, Generic[T, S]): + x: T + y: S + + with pytest.raises(TypeError) as exc_info: + Model[int, int, int] + assert str(exc_info.value) == 'Too many parameters for Model; actual 3, expected 2' + + with pytest.raises(TypeError) as exc_info: + Model[int] + assert str(exc_info.value) == 'Too few parameters for Model; actual 1, expected 2' + + +@skip_36 +def test_cover_cache(): + cache_size = len(_generic_types_cache) + T = TypeVar('T') + + class Model(GenericModel, Generic[T]): + x: T + + Model[int] # adds both with-tuple and without-tuple version to cache + assert len(_generic_types_cache) == cache_size + 2 + Model[int] # uses the cache + assert len(_generic_types_cache) == cache_size + 2 + + +@skip_36 +def test_generic_config(): + data_type = TypeVar('data_type') + + class Result(GenericModel, Generic[data_type]): + data: data_type + + class Config: + allow_mutation = False + + result = Result[int](data=1) + assert result.data == 1 + with pytest.raises(TypeError): + result.data = 2 + + +@skip_36 +def test_generic_instantiation_error(): + with pytest.raises(TypeError) as exc_info: + GenericModel() + assert str(exc_info.value) == 'Type GenericModel cannot be used without generic parameters, e.g. GenericModel[T]' + + +@skip_36 +def test_parameterized_generic_instantiation_error(): + data_type = TypeVar('data_type') + + class Result(GenericModel, Generic[data_type]): + data: data_type + + with pytest.raises(TypeError) as exc_info: + Result(data=1) + assert str(exc_info.value) == 'Type Result cannot be used without generic parameters, e.g. Result[T]' + + +@skip_36 +def test_deep_generic(): + T = TypeVar('T') + S = TypeVar('S') + R = TypeVar('R') + + class OuterModel(GenericModel, Generic[T, S, R]): + a: Dict[R, Optional[List[T]]] + b: Optional[Union[S, R]] + c: R + d: float + + class InnerModel(GenericModel, Generic[T, R]): + c: T + d: R + + class NormalModel(BaseModel): + e: int + f: str + + inner_model = InnerModel[int, str] + generic_model = OuterModel[inner_model, NormalModel, int] + + inner_models = [inner_model(c=1, d='a')] + generic_model(a={1: inner_models, 2: None}, b=None, c=1, d=1.5) + generic_model(a={}, b=NormalModel(e=1, f='a'), c=1, d=1.5) + generic_model(a={}, b=1, c=1, d=1.5) + + +@skip_36 +def test_enum_generic(): + T = TypeVar('T') + + class MyEnum(Enum): + x = 1 + y = 2 + + class Model(GenericModel, Generic[T]): + enum: T + + Model[MyEnum](enum=MyEnum.x) + Model[MyEnum](enum=2) + + +@skip_36 +def test_generic(): + data_type = TypeVar('data_type') + error_type = TypeVar('error_type') + + class Result(GenericModel, Generic[data_type, error_type]): + data: Optional[List[data_type]] + error: Optional[error_type] + positive_number: int + + @validator('error', always=True) + def validate_error(cls, v: Optional[error_type], values: Dict[str, Any]) -> Optional[error_type]: + if values.get('data', None) is None and v is None: + raise ValueError('Must provide data or error') + if values.get('data', None) is not None and v is not None: + raise ValueError('Must not provide both data and error') + return v + + @validator('positive_number') + def validate_positive_number(cls, v: int) -> int: + if v < 0: + raise ValueError + return v + + class Error(BaseModel): + message: str + + class Data(BaseModel): + number: int + text: str + + success1 = Result[Data, Error](data=[Data(number=1, text='a')], positive_number=1) + assert success1.dict() == {'data': [{'number': 1, 'text': 'a'}], 'error': None, 'positive_number': 1} + assert str(success1) == 'Result[Data, Error] data=[] error=None positive_number=1' + + success2 = Result[Data, Error](error=Error(message='error'), positive_number=1) + assert success2.dict() == {'data': None, 'error': {'message': 'error'}, 'positive_number': 1} + assert str(success2) == 'Result[Data, Error] data=None error= positive_number=1' + with pytest.raises(ValidationError) as exc_info: + Result[Data, Error](error=Error(message='error'), positive_number=-1) + assert exc_info.value.errors() == [{'loc': ('positive_number',), 'msg': '', 'type': 'value_error'}] + + with pytest.raises(ValidationError) as exc_info: + Result[Data, Error](data=[Data(number=1, text='a')], error=Error(message='error'), positive_number=1) + assert exc_info.value.errors() == [ + {'loc': ('error',), 'msg': 'Must not provide both data and error', 'type': 'value_error'}, + {'loc': ('error',), 'msg': 'value is not none', 'type': 'type_error.none.allowed'}, + ] + + with pytest.raises(ValidationError) as exc_info: + Result[Data, Error](data=[Data(number=1, text='a')], error=Error(message='error'), positive_number=1) + assert exc_info.value.errors() == [ + {'loc': ('error',), 'msg': 'Must not provide both data and error', 'type': 'value_error'}, + {'loc': ('error',), 'msg': 'value is not none', 'type': 'type_error.none.allowed'}, + ]