feat: add support for NamedTuple and TypedDict types (#2216)

* feat: add support for `NamedTuple` and `TypedDict` types

* support `total=False`

* tests: fix ci with python < 3.8 without typing-extensions

* chore: improve mypy

* chore: @samuelcolvin remarks

* refactor: move tests in dedicated file

* docs: add annotated types section with examples

* feat: support properly required and optional fields

* chore(deps-dev): bump typing_extensions

* docs: add a note for `typing_extensions`

* chore: update message to be more accurate

* feat: pass down config to created models

* feat: add util methods to create model from TypedDict or NamedTuple

* refactor: rename into typeddict and namedtuple

* test: add utils tests

* chore: fix lint

* chore: improve test

* refactor: rename utils to match the rest

* chore: update change

* docs: add section for create_model_from_{namedtuple,typeddict}

* refactor: rename typed_dict/named_tuple

* feat: support schema with TypedDict

* feat: support schema for NamedTuple

* feat: add json support for NamedTuple

* chore: rewording

* refactor: use parse_obj

* fix: add check for max items in tuple

* docs: separate typing.NamedTuple and collections.namedtuple
This commit is contained in:
Eric Jolibois
2021-02-13 11:05:57 +01:00
committed by GitHub
parent 502570706a
commit c314f5a909
16 changed files with 629 additions and 4 deletions
+3
View File
@@ -0,0 +1,3 @@
Add support for `NamedTuple` and `TypedDict` types.
Those two types are now handled and validated when used inside `BaseModel` or _pydantic_ `dataclass`.
Two utils are also added `create_model_from_namedtuple` and `create_model_from_typeddict`.
@@ -0,0 +1,20 @@
from typing import NamedTuple
from pydantic import BaseModel, ValidationError
class Point(NamedTuple):
x: int
y: int
class Model(BaseModel):
p: Point
print(Model(p=('1', '2')))
try:
Model(p=('1.3', '2'))
except ValidationError as e:
print(e)
@@ -0,0 +1,45 @@
from typing import TypedDict
from pydantic import BaseModel, Extra, ValidationError
# `total=False` means keys are non-required
class UserIdentity(TypedDict, total=False):
name: str
surname: str
class User(TypedDict):
identity: UserIdentity
age: int
class Model(BaseModel):
u: User
class Config:
extra = Extra.forbid
print(Model(u={'identity': {'name': 'Smith', 'surname': 'John'}, 'age': '37'}))
print(Model(u={'identity': {'name': None, 'surname': 'John'}, 'age': '37'}))
print(Model(u={'identity': {}, 'age': '37'}))
try:
Model(u={'identity': {'name': ['Smith'], 'surname': 'John'}, 'age': '24'})
except ValidationError as e:
print(e)
try:
Model(
u={
'identity': {'name': 'Smith', 'surname': 'John'},
'age': '37',
'email': 'john.smith@me.com',
}
)
except ValidationError as e:
print(e)
+21
View File
@@ -0,0 +1,21 @@
from typing import TypedDict
from pydantic import ValidationError, create_model_from_typeddict
class User(TypedDict):
name: str
id: int
class Config:
extra = 'forbid'
UserM = create_model_from_typeddict(User, __config__=Config)
print(repr(UserM(name=123, id='3')))
try:
UserM(name=123, id='3', other='no')
except ValidationError as e:
print(e)
+12
View File
@@ -384,6 +384,18 @@ You can also add validators by passing a dict to the `__validators__` argument.
{!.tmp_examples/models_dynamic_validators.py!}
```
## Model creation from `NamedTuple` or `TypedDict`
Sometimes you already use in your application classes that inherit from `NamedTuple` or `TypedDict`
and you don't want to duplicate all your information to have a `BaseModel`.
For this _pydantic_ provides `create_model_from_namedtuple` and `create_model_from_typeddict` methods.
Those methods have the exact same keyword arguments as `create_model`.
```py
{!.tmp_examples/models_from_typeddict.py!}
```
## Custom Root Types
Pydantic models can be defined with a custom root type by declaring the `__root__` field.
+34
View File
@@ -88,9 +88,20 @@ with custom properties and validation.
`typing.Tuple`
: see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation
`subclass of typing.NamedTuple`
: Same as `tuple` but instantiates with the given namedtuple and validates fields since they are annotated.
See [Annotated Types](#annotated-types) below for more detail on parsing and validation
`subclass of collections.namedtuple`
: Same as `subclass of typing.NamedTuple` but all fields will have type `Any` since they are not annotated
`typing.Dict`
: see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation
`subclass of typing.TypedDict`
: Same as `dict` but _pydantic_ will validate the dictionary since keys are annotated.
See [Annotated Types](#annotated-types) below for more detail on parsing and validation
`typing.Set`
: see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation
@@ -395,6 +406,29 @@ With proper ordering in an annotated `Union`, you can use this to parse types of
```
_(This script is complete, it should run "as is")_
## Annotated Types
### NamedTuple
```py
{!.tmp_examples/annotated_types_named_tuple.py!}
```
_(This script is complete, it should run "as is")_
### TypedDict
!!! note
This is a new feature of the python standard library as of python 3.8.
Prior to python 3.8, it requires the [typing-extensions](https://pypi.org/project/typing-extensions/) package.
But required and optional fields are properly differentiated only since python 3.9.
We therefore recommend using [typing-extensions](https://pypi.org/project/typing-extensions/) with python 3.8 as well.
```py
{!.tmp_examples/annotated_types_typed_dict.py!}
```
_(This script is complete, it should run "as is")_
## Pydantic Types
*pydantic* also provides a variety of other useful types:
+4
View File
@@ -1,5 +1,6 @@
# flake8: noqa
from . import dataclasses
from .annotated_types import create_model_from_namedtuple, create_model_from_typeddict
from .class_validators import root_validator, validator
from .decorator import validate_arguments
from .env_settings import BaseSettings
@@ -16,6 +17,9 @@ from .version import VERSION
# WARNING __all__ from .errors is not included here, it will be removed as an export here in v2
# please use "from pydantic.errors import ..." instead
__all__ = [
# annotated types utils
'create_model_from_namedtuple',
'create_model_from_typeddict',
# dataclasses
'dataclasses',
# class_validators
+59
View File
@@ -0,0 +1,59 @@
from typing import TYPE_CHECKING, Any, Dict, FrozenSet, NamedTuple, Type
from .fields import Required
from .main import BaseModel, create_model
if TYPE_CHECKING:
class TypedDict(Dict[str, Any]):
__annotations__: Dict[str, Type[Any]]
__total__: bool
__required_keys__: FrozenSet[str]
__optional_keys__: FrozenSet[str]
def create_model_from_typeddict(typeddict_cls: Type['TypedDict'], **kwargs: Any) -> Type['BaseModel']:
"""
Create a `BaseModel` based on the fields of a `TypedDict`.
Since `typing.TypedDict` in Python 3.8 does not store runtime information about optional keys,
we warn the user if that's the case (see https://bugs.python.org/issue38834).
"""
field_definitions: Dict[str, Any]
# Best case scenario: with python 3.9+ or when `TypedDict` is imported from `typing_extensions`
if hasattr(typeddict_cls, '__required_keys__'):
field_definitions = {
field_name: (field_type, Required if field_name in typeddict_cls.__required_keys__ else None)
for field_name, field_type in typeddict_cls.__annotations__.items()
}
else:
import warnings
warnings.warn(
'You should use `typing_extensions.TypedDict` instead of `typing.TypedDict` for better support! '
'Without it, there is no way to differentiate required and optional fields when subclassed. '
'Fields will therefore be considered all required or all optional depending on class totality.',
UserWarning,
)
default_value = Required if typeddict_cls.__total__ else None
field_definitions = {
field_name: (field_type, default_value) for field_name, field_type in typeddict_cls.__annotations__.items()
}
return create_model(typeddict_cls.__name__, **kwargs, **field_definitions)
def create_model_from_namedtuple(namedtuple_cls: Type['NamedTuple'], **kwargs: Any) -> Type['BaseModel']:
"""
Create a `BaseModel` based on the fields of a named tuple.
A named tuple can be created with `typing.NamedTuple` and declared annotations
but also with `collections.namedtuple`, in this case we consider all fields
to have type `Any`.
"""
namedtuple_annotations: Dict[str, Type[Any]] = getattr(
namedtuple_cls, '__annotations__', {k: Any for k in namedtuple_cls._fields}
)
field_definitions: Dict[str, Any] = {
field_name: (field_type, Required) for field_name, field_type in namedtuple_annotations.items()
}
return create_model(namedtuple_cls.__name__, **kwargs, **field_definitions)
+3
View File
@@ -38,6 +38,7 @@ from .typing import (
get_origin,
is_literal_type,
is_new_type,
is_typeddict,
new_type_supertype,
)
from .utils import PyObjectStr, Representation, lenient_issubclass, sequence_like, smart_deepcopy
@@ -416,6 +417,8 @@ class ModelField(Representation):
return
elif is_literal_type(self.type_):
return
elif is_typeddict(self.type_):
return
origin = get_origin(self.type_)
if origin is None:
+12 -2
View File
@@ -34,7 +34,15 @@ from .json import custom_pydantic_encoder, pydantic_encoder
from .parse import Protocol, load_file, load_str_bytes
from .schema import default_ref_template, model_schema
from .types import PyObject, StrBytes
from .typing import AnyCallable, get_args, get_origin, is_classvar, resolve_annotations, update_field_forward_refs
from .typing import (
AnyCallable,
get_args,
get_origin,
is_classvar,
is_namedtuple,
resolve_annotations,
update_field_forward_refs,
)
from .utils import (
ROOT_KEY,
ClassAttribute,
@@ -745,7 +753,7 @@ class BaseModel(Representation, metaclass=ModelMetaclass):
}
elif sequence_like(v):
return v.__class__(
seq_args = (
cls._get_value(
v_,
to_dict=to_dict,
@@ -761,6 +769,8 @@ class BaseModel(Representation, metaclass=ModelMetaclass):
and (not value_include or value_include.is_included(i))
)
return v.__class__(*seq_args) if is_namedtuple(v.__class__) else v.__class__(seq_args)
elif isinstance(v, Enum) and getattr(cls.Config, 'use_enum_values', False):
return v.value
+12 -1
View File
@@ -65,6 +65,7 @@ from .typing import (
get_origin,
is_callable_type,
is_literal_type,
is_namedtuple,
literal_values,
)
from .utils import ROOT_KEY, get_model, lenient_issubclass, sequence_like
@@ -800,7 +801,17 @@ def field_singleton_schema( # noqa: C901 (ignore complexity)
f_schema, schema_overrides = get_field_info_schema(field)
f_schema.update(get_schema_ref(enum_name, ref_prefix, ref_template, schema_overrides))
definitions[enum_name] = enum_process_schema(field_type)
else:
elif is_namedtuple(field_type):
sub_schema, *_ = model_process_schema(
field_type.__pydantic_model__,
by_alias=by_alias,
model_name_map=model_name_map,
ref_prefix=ref_prefix,
ref_template=ref_template,
known_models=known_models,
)
f_schema.update({'type': 'array', 'items': list(sub_schema['properties'].values())})
elif not hasattr(field_type, '__pydantic_model__'):
add_field_type_to_schema(field_type, f_schema)
modify_schema = getattr(field_type, '__modify_schema__', None)
+22
View File
@@ -188,6 +188,8 @@ __all__ = (
'is_literal_type',
'literal_values',
'Literal',
'is_namedtuple',
'is_typeddict',
'is_new_type',
'new_type_supertype',
'is_classvar',
@@ -299,6 +301,26 @@ def all_literal_values(type_: Type[Any]) -> Tuple[Any, ...]:
return tuple(x for value in values for x in all_literal_values(value))
def is_namedtuple(type_: Type[Any]) -> bool:
"""
Check if a given class is a named tuple.
It can be either a `typing.NamedTuple` or `collections.namedtuple`
"""
from .utils import lenient_issubclass
return lenient_issubclass(type_, tuple) and hasattr(type_, '_fields')
def is_typeddict(type_: Type[Any]) -> bool:
"""
Check if a given class is a typed dict (from `typing` or `typing_extensions`)
In 3.10, there will be a public method (https://docs.python.org/3.10/library/typing.html#typing.is_typeddict)
"""
from .utils import lenient_issubclass
return lenient_issubclass(type_, dict) and hasattr(type_, '__total__')
test_type = NewType('test_type', str)
+47
View File
@@ -15,6 +15,7 @@ from typing import (
FrozenSet,
Generator,
List,
NamedTuple,
Pattern,
Set,
Tuple,
@@ -36,10 +37,13 @@ from .typing import (
get_class,
is_callable_type,
is_literal_type,
is_namedtuple,
is_typeddict,
)
from .utils import almost_equal_floats, lenient_issubclass, sequence_like
if TYPE_CHECKING:
from .annotated_types import TypedDict
from .fields import ModelField
from .main import BaseConfig
from .types import ConstrainedDecimal, ConstrainedFloat, ConstrainedInt
@@ -547,6 +551,42 @@ def pattern_validator(v: Any) -> Pattern[str]:
raise errors.PatternError()
NamedTupleT = TypeVar('NamedTupleT', bound=NamedTuple)
def make_namedtuple_validator(namedtuple_cls: Type[NamedTupleT]) -> Callable[[Tuple[Any, ...]], NamedTupleT]:
from .annotated_types import create_model_from_namedtuple
NamedTupleModel = create_model_from_namedtuple(namedtuple_cls)
namedtuple_cls.__pydantic_model__ = NamedTupleModel # type: ignore[attr-defined]
def namedtuple_validator(values: Tuple[Any, ...]) -> NamedTupleT:
annotations = NamedTupleModel.__annotations__
if len(values) > len(annotations):
raise errors.ListMaxLengthError(limit_value=len(annotations))
dict_values: Dict[str, Any] = dict(zip(annotations, values))
validated_dict_values: Dict[str, Any] = dict(NamedTupleModel(**dict_values))
return namedtuple_cls(**validated_dict_values)
return namedtuple_validator
def make_typeddict_validator(
typeddict_cls: Type['TypedDict'], config: Type['BaseConfig']
) -> Callable[[Any], Dict[str, Any]]:
from .annotated_types import create_model_from_typeddict
TypedDictModel = create_model_from_typeddict(typeddict_cls, __config__=config)
typeddict_cls.__pydantic_model__ = TypedDictModel # type: ignore[attr-defined]
def typeddict_validator(values: 'TypedDict') -> Dict[str, Any]:
return TypedDictModel.parse_obj(values).dict(exclude_unset=True)
return typeddict_validator
class IfConfig:
def __init__(self, validator: AnyCallable, *config_attr_names: str) -> None:
self.validator = validator
@@ -639,6 +679,13 @@ def find_validators( # noqa: C901 (ignore complexity)
if type_ is IntEnum:
yield int_enum_validator
return
if is_namedtuple(type_):
yield tuple_validator
yield make_namedtuple_validator(type_)
return
if is_typeddict(type_):
yield make_typeddict_validator(type_, config)
return
class_ = get_class(type_)
if class_ is not None:
+1 -1
View File
@@ -4,5 +4,5 @@ Cython==0.29.21;sys_platform!='win32'
devtools==0.6.1
email-validator==1.1.2
dataclasses==0.6; python_version < '3.7'
typing-extensions==3.7.4.1; python_version < '3.8'
typing-extensions==3.7.4.3; python_version < '3.9'
python-dotenv==0.15.0
+278
View File
@@ -0,0 +1,278 @@
"""
Tests for annotated types that _pydantic_ can validate like
- NamedTuple
- TypedDict
"""
import json
import sys
from collections import namedtuple
from typing import List, NamedTuple, Tuple
if sys.version_info < (3, 9):
try:
from typing import TypedDict as LegacyTypedDict
except ImportError:
LegacyTypedDict = None
try:
from typing_extensions import TypedDict
except ImportError:
TypedDict = None
else:
from typing import TypedDict
LegacyTypedDict = None
import pytest
from pydantic import BaseModel, ValidationError
def test_namedtuple():
Position = namedtuple('Pos', 'x y')
class Event(NamedTuple):
a: int
b: int
c: int
d: str
class Model(BaseModel):
pos: Position
events: List[Event]
model = Model(pos=('1', 2), events=[[b'1', '2', 3, 'qwe']])
assert isinstance(model.pos, Position)
assert isinstance(model.events[0], Event)
assert model.pos.x == '1'
assert model.pos == Position('1', 2)
assert model.events[0] == Event(1, 2, 3, 'qwe')
assert repr(model) == "Model(pos=Pos(x='1', y=2), events=[Event(a=1, b=2, c=3, d='qwe')])"
assert model.json() == json.dumps(model.dict()) == '{"pos": ["1", 2], "events": [[1, 2, 3, "qwe"]]}'
with pytest.raises(ValidationError) as exc_info:
Model(pos=('1', 2), events=[['qwe', '2', 3, 'qwe']])
assert exc_info.value.errors() == [
{
'loc': ('events', 0, 'a'),
'msg': 'value is not a valid integer',
'type': 'type_error.integer',
}
]
def test_namedtuple_schema():
class Position1(NamedTuple):
x: int
y: int
Position2 = namedtuple('Position2', 'x y')
class Model(BaseModel):
pos1: Position1
pos2: Position2
pos3: Tuple[int, int]
assert Model.schema() == {
'title': 'Model',
'type': 'object',
'properties': {
'pos1': {
'title': 'Pos1',
'type': 'array',
'items': [
{'title': 'X', 'type': 'integer'},
{'title': 'Y', 'type': 'integer'},
],
},
'pos2': {
'title': 'Pos2',
'type': 'array',
'items': [
{'title': 'X'},
{'title': 'Y'},
],
},
'pos3': {
'title': 'Pos3',
'type': 'array',
'items': [
{'type': 'integer'},
{'type': 'integer'},
],
},
},
'required': ['pos1', 'pos2', 'pos3'],
}
def test_namedtuple_right_length():
class Point(NamedTuple):
x: int
y: int
class Model(BaseModel):
p: Point
assert isinstance(Model(p=(1, 2)), Model)
with pytest.raises(ValidationError) as exc_info:
Model(p=(1, 2, 3))
assert exc_info.value.errors() == [
{
'loc': ('p',),
'msg': 'ensure this value has at most 2 items',
'type': 'value_error.list.max_items',
'ctx': {'limit_value': 2},
}
]
@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed')
def test_typeddict():
class TD(TypedDict):
a: int
b: int
c: int
d: str
class Model(BaseModel):
td: TD
m = Model(td={'a': '3', 'b': b'1', 'c': 4, 'd': 'qwe'})
assert m.td == {'a': 3, 'b': 1, 'c': 4, 'd': 'qwe'}
with pytest.raises(ValidationError) as exc_info:
Model(td={'a': [1], 'b': 2, 'c': 3, 'd': 'qwe'})
assert exc_info.value.errors() == [
{
'loc': ('td', 'a'),
'msg': 'value is not a valid integer',
'type': 'type_error.integer',
}
]
@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed')
def test_typeddict_non_total():
class FullMovie(TypedDict, total=True):
name: str
year: int
class Model(BaseModel):
movie: FullMovie
with pytest.raises(ValidationError) as exc_info:
Model(movie={'year': '2002'})
assert exc_info.value.errors() == [
{
'loc': ('movie', 'name'),
'msg': 'field required',
'type': 'value_error.missing',
}
]
class PartialMovie(TypedDict, total=False):
name: str
year: int
class Model(BaseModel):
movie: PartialMovie
m = Model(movie={'year': '2002'})
assert m.movie == {'year': 2002}
@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed')
def test_partial_new_typeddict():
class OptionalUser(TypedDict, total=False):
name: str
class User(OptionalUser):
id: int
class Model(BaseModel):
user: User
m = Model(user={'id': 1})
assert m.user == {'id': 1}
@pytest.mark.skipif(not LegacyTypedDict, reason='python 3.9+ is used or typing_extensions is installed')
def test_partial_legacy_typeddict():
class OptionalUser(LegacyTypedDict, total=False):
name: str
class User(OptionalUser):
id: int
with pytest.warns(
UserWarning,
match='You should use `typing_extensions.TypedDict` instead of `typing.TypedDict` for better support!',
):
class Model(BaseModel):
user: User
with pytest.raises(ValidationError) as exc_info:
Model(user={'id': 1})
assert exc_info.value.errors() == [
{
'loc': ('user', 'name'),
'msg': 'field required',
'type': 'value_error.missing',
}
]
@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed')
def test_typeddict_extra():
class User(TypedDict):
name: str
age: int
class Model(BaseModel):
u: User
class Config:
extra = 'forbid'
with pytest.raises(ValidationError) as exc_info:
Model(u={'name': 'pika', 'age': 7, 'rank': 1})
assert exc_info.value.errors() == [
{'loc': ('u', 'rank'), 'msg': 'extra fields not permitted', 'type': 'value_error.extra'},
]
@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed')
def test_typeddict_schema():
class Data(BaseModel):
a: int
class DataTD(TypedDict):
a: int
class Model(BaseModel):
data: Data
data_td: DataTD
assert Model.schema() == {
'title': 'Model',
'type': 'object',
'properties': {'data': {'$ref': '#/definitions/Data'}, 'data_td': {'$ref': '#/definitions/DataTD'}},
'required': ['data', 'data_td'],
'definitions': {
'Data': {
'type': 'object',
'title': 'Data',
'properties': {'a': {'title': 'A', 'type': 'integer'}},
'required': ['a'],
},
'DataTD': {
'type': 'object',
'title': 'DataTD',
'properties': {'a': {'title': 'A', 'type': 'integer'}},
'required': ['a'],
},
},
}
+56
View File
@@ -0,0 +1,56 @@
from collections import namedtuple
from typing import NamedTuple
import pytest
from pydantic.typing import is_namedtuple, is_typeddict
try:
from typing import TypedDict as typing_TypedDict
except ImportError:
typing_TypedDict = None
try:
from typing_extensions import TypedDict as typing_extensions_TypedDict
except ImportError:
typing_extensions_TypedDict = None
try:
from mypy_extensions import TypedDict as mypy_extensions_TypedDict
except ImportError:
mypy_extensions_TypedDict = None
ALL_TYPEDDICT_KINDS = (typing_TypedDict, typing_extensions_TypedDict, mypy_extensions_TypedDict)
def test_is_namedtuple():
class Employee(NamedTuple):
name: str
id: int = 3
assert is_namedtuple(namedtuple('Point', 'x y')) is True
assert is_namedtuple(Employee) is True
assert is_namedtuple(NamedTuple('Employee', [('name', str), ('id', int)])) is True
class Other(tuple):
name: str
id: int
assert is_namedtuple(Other) is False
@pytest.mark.parametrize('TypedDict', (t for t in ALL_TYPEDDICT_KINDS if t is not None))
def test_is_typeddict_typing(TypedDict):
class Employee(TypedDict):
name: str
id: int
assert is_typeddict(Employee) is True
assert is_typeddict(TypedDict('Employee', {'name': str, 'id': int})) is True
class Other(dict):
name: str
id: int
assert is_typeddict(Other) is False