Add generic functionality (#595), fix #556

* Added generic functionality

* Skip tests in python 3.6

* double quote -> single quote

* Simplified, with more comprehensive tests

* double quote -> single quote

* Remove unintentional file

* Add caching

* don't cythonize generics.py

* Make work with mypy

* Remove __parameters__

* double quote -> single quote i'll remember one day

* More cleanup and validation

* Removed unwanted file

* A little more cleanup, and finish the PR

* Add proper inheritance

* Added note about inheritance to docs

* Added error for double-parameterizing

* Should build for python3.7

* Works with both 3.6 and 3.7

* Fixed bug with caching for single argument

* handle __name__ for generic models

* double quote -> single quote

* Updated error messages

HISTORY.rst

@@ -9,6 +9,7 @@ v0.29 (unreleased)
 * Updated documentation to elucidate the usage of ``Union`` when defining multiple types under an attribute's
   annotation and showcase how the type-order can affect marshalling of provided values, #594 by @somada141
 * add ``conlist`` type, #583 by @hmvp
+* add support for generics, #595 by @dmontagu
 
 v0.28 (2019-06-06)
 ..................

Makefile

@@ -48,6 +48,9 @@ external-mypy:
 	@mypy tests/mypy_test_fails2.py 1>/dev/null; \
 	  test $$? -eq 1 || \
 	  (echo "mypy_test_fails2: mypy passed when it should have failed!"; exit 1)
+	@mypy tests/mypy_test_fails3.py 1>/dev/null; \
+	  test $$? -eq 1 || \
+	  (echo "mypy_test_fails3: mypy passed when it should have failed!"; exit 1)
 
 .PHONY: testcov
 testcov: test

docs/examples/generics.py

@@ -0,0 +1,61 @@
+from typing import Generic, TypeVar, Optional, List
+
+from pydantic import BaseModel, validator, ValidationError
+from pydantic.generics import GenericModel
+
+
+DataT = TypeVar('DataT')
+
+
+class Error(BaseModel):
+    code: int
+    message: str
+
+
+class DataModel(BaseModel):
+    numbers: List[int]
+    people: List[str]
+
+
+class Response(GenericModel, Generic[DataT]):
+    data: Optional[DataT]
+    error: Optional[Error]
+
+    @validator('error', always=True)
+    def check_consistency(cls, v, values):
+        if v is not None and values['data'] is not None:
+            raise ValueError('must not provide both data and error')
+        if v is None and values.get('data') is None:
+            raise ValueError('must provide data or error')
+        return v
+
+
+data = DataModel(numbers=[1, 2, 3], people=[])
+error = Error(code=404, message='Not found')
+
+print(Response[int](data=1))
+# > Response[int] data=1 error=None
+print(Response[str](data='value'))
+# > Response[str] data='value' error=None
+print(Response[str](data='value').dict())
+# > {'data': 'value', 'error': None}
+print(Response[DataModel](data=data).dict())
+# > {'data': {'numbers': [1, 2, 3], 'people': []}, 'error': None}
+print(Response[DataModel](error=error).dict())
+# > {'data': None, 'error': {'code': 404, 'message': 'Not found'}}
+
+try:
+    Response[int](data='value')
+except ValidationError as e:
+    print(e)
+"""
+4 validation errors
+data
+  value is not a valid integer (type=type_error.integer)
+data
+  value is not none (type=type_error.none.allowed)
+error
+  value is not a valid dict (type=type_error.dict)
+error
+  must provide data or error (type=value_error)
+"""

docs/index.rst

@@ -295,6 +295,45 @@ and pydantic versions).
 
 (This script is complete, it should run "as is")
 
+.. _generic_models:
+
+Generic Models
+..............
+
+.. note::
+
+   New in version v0.29.
+
+   This feature requires Python 3.7+.
+
+Pydantic supports the creation of generic models to make it easier to reuse a common model structure.
+
+In order to declare a generic model, you perform the following steps:
+
+* Declare one or more ``typing.TypeVar`` instances to use to parameterize your model.
+* Declare a pydantic model that inherits from ``pydantic.generics.GenericModel`` and ``typing.Generic``,
+  where you pass the ``TypeVar`` instances as parameters to ``typing.Generic``.
+* Use the ``TypeVar`` instances as annotations where you will want to replace them with other types or
+  pydantic models.
+
+Here is an example using ``GenericModel`` to create an easily-reused HTTP response payload wrapper:
+
+.. literalinclude:: examples/generics.py
+
+(This script is complete, it should run "as is")
+
+If you set ``Config`` or make use of ``validator`` in your generic model definition, it is applied
+to concrete subclasses in the same way as when inheriting from ``BaseModel``. Any methods defined on
+your generic class will also be inherited.
+
+Pydantic's generics also integrate properly with mypy, so you get all the type checking
+you would expect mypy to provide if you were to declare the type without using ``GenericModel``.
+
+.. note::
+
+   Internally, pydantic uses ``create_model`` to generate a (cached) concrete ``BaseModel`` at runtime,
+   so there is essentially zero overhead introduced by making use of ``GenericModel``.
+
 .. _orm_mode:
 
 ORM Mode (aka Arbitrary Class Instances)

pydantic/generics.py

@@ -0,0 +1,85 @@
+from typing import Any, ClassVar, Dict, Generic, Tuple, Type, TypeVar, Union, get_type_hints
+
+from pydantic import BaseModel, create_model
+from pydantic.class_validators import gather_validators
+
+_generic_types_cache: Dict[Tuple[Type[Any], Union[Any, Tuple[Any, ...]]], Type[BaseModel]] = {}
+GenericModelT = TypeVar('GenericModelT', bound='GenericModel')
+
+
+class GenericModel(BaseModel):
+    __slots__ = ()
+    __concrete__: ClassVar[bool] = False
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> Any:
+        if cls.__concrete__:
+            return super().__new__(cls)
+        else:
+            raise TypeError(f'Type {cls.__name__} cannot be used without generic parameters, e.g. {cls.__name__}[T]')
+
+    def __class_getitem__(  # type: ignore
+        cls: Type[GenericModelT], params: Union[Type[Any], Tuple[Type[Any], ...]]
+    ) -> Type[BaseModel]:
+        cached = _generic_types_cache.get((cls, params))
+        if cached is not None:
+            return cached
+        if cls.__concrete__:
+            raise TypeError('Cannot parameterize a concrete instantiation of a generic model')
+        if not isinstance(params, tuple):
+            params = (params,)
+        if any(isinstance(param, TypeVar) for param in params):  # type: ignore
+            raise TypeError(f'Type parameters should be placed on typing.Generic, not GenericModel')
+        if Generic not in cls.__bases__:
+            raise TypeError(f'Type {cls.__name__} must inherit from typing.Generic before being parameterized')
+
+        check_parameters_count(cls, params)
+        typevars_map: Dict[Any, Any] = dict(zip(cls.__parameters__, params))  # type: ignore
+        type_hints = get_type_hints(cls).items()
+        instance_type_hints = {k: v for k, v in type_hints if getattr(v, '__origin__', None) is not ClassVar}
+        concrete_type_hints: Dict[str, Type[Any]] = {
+            k: resolve_type_hint(v, typevars_map) for k, v in instance_type_hints.items()
+        }
+
+        model_name = concrete_name(cls, params)
+        validators = gather_validators(cls)
+        fields: Dict[str, Tuple[Type[Any], Any]] = {
+            k: (v, getattr(cls, k, ...)) for k, v in concrete_type_hints.items()
+        }
+        created_model = create_model(
+            model_name=model_name,
+            __module__=cls.__module__,
+            __base__=cls,
+            __config__=None,
+            __validators__=validators,
+            **fields,
+        )
+        created_model.Config = cls.Config
+        created_model.__concrete__ = True  # type: ignore
+        _generic_types_cache[(cls, params)] = created_model
+        if len(params) == 1:
+            _generic_types_cache[(cls, params[0])] = created_model
+        return created_model
+
+
+def concrete_name(cls: Type[Any], params: Tuple[Type[Any], ...]) -> str:
+    param_names = [param.__name__ if hasattr(param, '__name__') else str(param) for param in params]
+    params_component = ', '.join(param_names)
+    return f'{cls.__name__}[{params_component}]'
+
+
+def resolve_type_hint(type_: Any, typevars_map: Dict[Any, Any]) -> Type[Any]:
+    if hasattr(type_, '__origin__'):
+        new_args = tuple(resolve_type_hint(x, typevars_map) for x in type_.__args__)
+        if type_.__origin__ is Union:
+            return type_.__origin__[new_args]
+        new_args = tuple([typevars_map[x] for x in type_.__parameters__])
+        return type_[new_args]
+    return typevars_map.get(type_, type_)
+
+
+def check_parameters_count(cls: Type[GenericModel], parameters: Tuple[Any, ...]) -> None:
+    actual = len(parameters)
+    expected = len(cls.__parameters__)  # type: ignore
+    if actual != expected:
+        description = 'many' if actual > expected else 'few'
+        raise TypeError(f'Too {description} parameters for {cls.__name__}; actual {actual}, expected {expected}')

pydantic/schema.py

@@ -1,3 +1,4 @@
+import re
 import warnings
 from datetime import date, datetime, time, timedelta
 from decimal import Decimal
@@ -354,6 +355,7 @@ def get_model_name_map(unique_models: Set[Type['BaseModel']]) -> Dict[Type['Base
     conflicting_names: Set[str] = set()
     for model in unique_models:
         model_name = model.__name__
+        model_name = re.sub(r'[^a-zA-Z0-9.\-_]', '_', model_name)
         if model_name in conflicting_names:
             model_name = get_long_model_name(model)
             name_model_map[model_name] = model

setup.py

@@ -57,6 +57,7 @@ if not any(arg in sys.argv for arg in ['clean', 'check']) and 'SKIP_CYTHON' not
         os.environ['CFLAGS'] = '-O3'
         ext_modules = cythonize(
             'pydantic/*.py',
+            exclude=['pydantic/generics.py'],
             nthreads=4,
             language_level=3,
             compiler_directives=compiler_directives,

tests/mypy_test_fails3.py

@@ -0,0 +1,22 @@
+"""
+Test mypy failure with invalid types.
+"""
+from typing import Generic, List, TypeVar
+
+from pydantic import BaseModel
+from pydantic.generics import GenericModel
+
+T = TypeVar('T')
+
+
+class Model(BaseModel):
+    list_of_ints: List[int]
+
+
+class WrapperModel(GenericModel, Generic[T]):
+    payload: T
+
+
+model_instance = Model(list_of_ints=[1])
+wrapper_instance = WrapperModel[Model](payload=model_instance)
+wrapper_instance.payload.list_of_ints.append('1')

tests/mypy_test_success.py

@@ -4,11 +4,13 @@ Test pydantic's compliance with mypy.
 Do a little skipping about with types to demonstrate its usage.
 """
 import json
+import sys
 from datetime import datetime
-from typing import List, Optional
+from typing import Generic, List, Optional, TypeVar
 
 from pydantic import BaseModel, NoneStr
 from pydantic.dataclasses import dataclass
+from pydantic.generics import GenericModel
 
 
 class Model(BaseModel):
@@ -82,3 +84,22 @@ class AddProject:
 
 
 p = AddProject(name='x', slug='y', description='z')
+
+
+if sys.version_info >= (3, 7):
+    T = TypeVar('T')
+
+    class WrapperModel(GenericModel, Generic[T]):
+        payload: T
+
+    int_instance = WrapperModel[int](payload=1)
+    int_instance.payload += 1
+    assert int_instance.payload == 2
+
+    str_instance = WrapperModel[str](payload='a')
+    str_instance.payload += 'a'
+    assert str_instance.payload == 'aa'
+
+    model_instance = WrapperModel[Model](payload=m)
+    model_instance.payload.list_of_ints.append(4)
+    assert model_instance.payload.list_of_ints == [1, 2, 3, 4]

tests/test_generics.py

@@ -0,0 +1,259 @@
+import sys
+from enum import Enum
+from typing import Any, Dict, Generic, List, Optional, TypeVar, Union
+
+import pytest
+
+from pydantic import BaseModel, ValidationError, validator
+from pydantic.generics import GenericModel, _generic_types_cache
+
+skip_36 = pytest.mark.skipif(sys.version_info < (3, 7), reason='generics only supported for python 3.7 and above')
+
+
+@skip_36
+def test_generic_name():
+    data_type = TypeVar('data_type')
+
+    class Result(GenericModel, Generic[data_type]):
+        data: data_type
+
+    assert Result[List[int]].__name__ == 'Result[typing.List[int]]'
+
+
+@skip_36
+def test_double_parameterize_error():
+    data_type = TypeVar('data_type')
+
+    class Result(GenericModel, Generic[data_type]):
+        data: data_type
+
+    with pytest.raises(TypeError) as exc_info:
+        Result[int][int]
+
+    assert str(exc_info.value) == 'Cannot parameterize a concrete instantiation of a generic model'
+
+
+@skip_36
+def test_methods_are_inherited():
+    class CustomGenericModel(GenericModel):
+        def method(self):
+            return self.data
+
+    T = TypeVar('T')
+
+    class Model(CustomGenericModel, Generic[T]):
+        data: T
+
+    instance = Model[int](data=1)
+
+    assert instance.method() == 1
+
+
+@skip_36
+def test_config_is_inherited():
+    class CustomGenericModel(GenericModel):
+        class Config:
+            allow_mutation = False
+
+    T = TypeVar('T')
+
+    class Model(CustomGenericModel, Generic[T]):
+        data: T
+
+    instance = Model[int](data=1)
+
+    with pytest.raises(TypeError) as exc_info:
+        instance.data = 2
+
+    assert str(exc_info.value) == '"Model[int]" is immutable and does not support item assignment'
+
+
+@skip_36
+def test_must_inherit_from_generic():
+    with pytest.raises(TypeError) as exc_info:
+
+        class Result(GenericModel):
+            pass
+
+        Result[int]
+
+    assert str(exc_info.value) == f'Type Result must inherit from typing.Generic before being parameterized'
+
+
+@skip_36
+def test_parameters_must_be_typevar():
+    T = TypeVar('T')
+    with pytest.raises(TypeError) as exc_info:
+
+        class Result(GenericModel[T]):
+            pass
+
+    assert str(exc_info.value) == f'Type parameters should be placed on typing.Generic, not GenericModel'
+
+
+@skip_36
+def test_parameter_count():
+    T = TypeVar('T')
+    S = TypeVar('S')
+
+    class Model(GenericModel, Generic[T, S]):
+        x: T
+        y: S
+
+    with pytest.raises(TypeError) as exc_info:
+        Model[int, int, int]
+    assert str(exc_info.value) == 'Too many parameters for Model; actual 3, expected 2'
+
+    with pytest.raises(TypeError) as exc_info:
+        Model[int]
+    assert str(exc_info.value) == 'Too few parameters for Model; actual 1, expected 2'
+
+
+@skip_36
+def test_cover_cache():
+    cache_size = len(_generic_types_cache)
+    T = TypeVar('T')
+
+    class Model(GenericModel, Generic[T]):
+        x: T
+
+    Model[int]  # adds both with-tuple and without-tuple version to cache
+    assert len(_generic_types_cache) == cache_size + 2
+    Model[int]  # uses the cache
+    assert len(_generic_types_cache) == cache_size + 2
+
+
+@skip_36
+def test_generic_config():
+    data_type = TypeVar('data_type')
+
+    class Result(GenericModel, Generic[data_type]):
+        data: data_type
+
+        class Config:
+            allow_mutation = False
+
+    result = Result[int](data=1)
+    assert result.data == 1
+    with pytest.raises(TypeError):
+        result.data = 2
+
+
+@skip_36
+def test_generic_instantiation_error():
+    with pytest.raises(TypeError) as exc_info:
+        GenericModel()
+    assert str(exc_info.value) == 'Type GenericModel cannot be used without generic parameters, e.g. GenericModel[T]'
+
+
+@skip_36
+def test_parameterized_generic_instantiation_error():
+    data_type = TypeVar('data_type')
+
+    class Result(GenericModel, Generic[data_type]):
+        data: data_type
+
+    with pytest.raises(TypeError) as exc_info:
+        Result(data=1)
+    assert str(exc_info.value) == 'Type Result cannot be used without generic parameters, e.g. Result[T]'
+
+
+@skip_36
+def test_deep_generic():
+    T = TypeVar('T')
+    S = TypeVar('S')
+    R = TypeVar('R')
+
+    class OuterModel(GenericModel, Generic[T, S, R]):
+        a: Dict[R, Optional[List[T]]]
+        b: Optional[Union[S, R]]
+        c: R
+        d: float
+
+    class InnerModel(GenericModel, Generic[T, R]):
+        c: T
+        d: R
+
+    class NormalModel(BaseModel):
+        e: int
+        f: str
+
+    inner_model = InnerModel[int, str]
+    generic_model = OuterModel[inner_model, NormalModel, int]
+
+    inner_models = [inner_model(c=1, d='a')]
+    generic_model(a={1: inner_models, 2: None}, b=None, c=1, d=1.5)
+    generic_model(a={}, b=NormalModel(e=1, f='a'), c=1, d=1.5)
+    generic_model(a={}, b=1, c=1, d=1.5)
+
+
+@skip_36
+def test_enum_generic():
+    T = TypeVar('T')
+
+    class MyEnum(Enum):
+        x = 1
+        y = 2
+
+    class Model(GenericModel, Generic[T]):
+        enum: T
+
+    Model[MyEnum](enum=MyEnum.x)
+    Model[MyEnum](enum=2)
+
+
+@skip_36
+def test_generic():
+    data_type = TypeVar('data_type')
+    error_type = TypeVar('error_type')
+
+    class Result(GenericModel, Generic[data_type, error_type]):
+        data: Optional[List[data_type]]
+        error: Optional[error_type]
+        positive_number: int
+
+        @validator('error', always=True)
+        def validate_error(cls, v: Optional[error_type], values: Dict[str, Any]) -> Optional[error_type]:
+            if values.get('data', None) is None and v is None:
+                raise ValueError('Must provide data or error')
+            if values.get('data', None) is not None and v is not None:
+                raise ValueError('Must not provide both data and error')
+            return v
+
+        @validator('positive_number')
+        def validate_positive_number(cls, v: int) -> int:
+            if v < 0:
+                raise ValueError
+            return v
+
+    class Error(BaseModel):
+        message: str
+
+    class Data(BaseModel):
+        number: int
+        text: str
+
+    success1 = Result[Data, Error](data=[Data(number=1, text='a')], positive_number=1)
+    assert success1.dict() == {'data': [{'number': 1, 'text': 'a'}], 'error': None, 'positive_number': 1}
+    assert str(success1) == 'Result[Data, Error] data=[<Data number=1 text=\'a\'>] error=None positive_number=1'
+
+    success2 = Result[Data, Error](error=Error(message='error'), positive_number=1)
+    assert success2.dict() == {'data': None, 'error': {'message': 'error'}, 'positive_number': 1}
+    assert str(success2) == 'Result[Data, Error] data=None error=<Error message=\'error\'> positive_number=1'
+    with pytest.raises(ValidationError) as exc_info:
+        Result[Data, Error](error=Error(message='error'), positive_number=-1)
+    assert exc_info.value.errors() == [{'loc': ('positive_number',), 'msg': '', 'type': 'value_error'}]
+
+    with pytest.raises(ValidationError) as exc_info:
+        Result[Data, Error](data=[Data(number=1, text='a')], error=Error(message='error'), positive_number=1)
+    assert exc_info.value.errors() == [
+        {'loc': ('error',), 'msg': 'Must not provide both data and error', 'type': 'value_error'},
+        {'loc': ('error',), 'msg': 'value is not none', 'type': 'type_error.none.allowed'},
+    ]
+
+    with pytest.raises(ValidationError) as exc_info:
+        Result[Data, Error](data=[Data(number=1, text='a')], error=Error(message='error'), positive_number=1)
+    assert exc_info.value.errors() == [
+        {'loc': ('error',), 'msg': 'Must not provide both data and error', 'type': 'value_error'},
+        {'loc': ('error',), 'msg': 'value is not none', 'type': 'type_error.none.allowed'},
+    ]