From 1d3f7824ecdc72d63acd2ece653230ebdff5ae3e Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Thu, 7 Nov 2019 12:31:26 +0000 Subject: [PATCH] Fix broken Any and TypeVar behaviour (#962) * tests for broken Dict behaviour * fix Any, support TypeVar * change type of field.type_ * add docs and example for TypeVar --- changes/962-samuelcolvin.md | 2 + docs/examples/types_typevar.py | 16 +++++++ docs/usage/types.md | 17 +++++++ pydantic/fields.py | 43 ++++++++++------- pydantic/schema.py | 4 +- pydantic/validators.py | 12 ++--- pydantic/version.py | 2 +- tests/test_edge_cases.py | 85 +++++++++++++++++++++++++++++++++- tests/test_schema.py | 8 +--- 9 files changed, 156 insertions(+), 33 deletions(-) create mode 100644 changes/962-samuelcolvin.md create mode 100644 docs/examples/types_typevar.py diff --git a/changes/962-samuelcolvin.md b/changes/962-samuelcolvin.md new file mode 100644 index 0000000..af848d1 --- /dev/null +++ b/changes/962-samuelcolvin.md @@ -0,0 +1,2 @@ +Fix usage of `Any` to allow `None`, also support `TypeVar` thus allowing use of un-parameterised collection types +e.g. `Dict` and `List` diff --git a/docs/examples/types_typevar.py b/docs/examples/types_typevar.py new file mode 100644 index 0000000..c8c34f6 --- /dev/null +++ b/docs/examples/types_typevar.py @@ -0,0 +1,16 @@ +from typing import TypeVar +from pydantic import BaseModel + +Foobar = TypeVar('Foobar') +BoundFloat = TypeVar('BoundFloat', bound=float) +IntStr = TypeVar('IntStr', int, str) + +class Model(BaseModel): + a: Foobar # equivalent of ": Any" + b: BoundFloat # equivalent of ": float" + c: IntStr # equivalent of ": Union[int, str]" + +print(Model(a=[1], b=4.2, c='x')) + +# a may be None and is therefore optional +print(Model(b=1, c=1)) diff --git a/docs/usage/types.md b/docs/usage/types.md index 7ddcbf8..adcca18 100644 --- a/docs/usage/types.md +++ b/docs/usage/types.md @@ -62,6 +62,12 @@ with custom properties and validation. `datetime.timedelta` : see [Datetime Types](#datetime-types) below for more detail on parsing and validation +`typing.Any` +: allows any value include `None`, thus an `Any` field is optional + +`typing.TypeVar` +: constrains the values allowed based on `constraints` or `bound`, see [TypeVar](#typevar) + `typing.Union` : see [Unions](#unions) below for more detail on parsing and validation @@ -275,12 +281,23 @@ that are subclasses of `T`. ```py {!.tmp_examples/type_type.py!} ``` +_(This script is complete, it should run "as is")_ You may also use `Type` to specify that any class is allowed. ```py {!.tmp_examples/bare_type_type.py!} ``` +_(This script is complete, it should run "as is")_ + +### TypeVar + +`TypeVar` is supported either unconstrained, constrained or with a bound. + +```py +{!.tmp_examples/types_typevar.py!} +``` +_(This script is complete, it should run "as is")_ ## Literal Type diff --git a/pydantic/fields.py b/pydantic/fields.py index cf042e3..057863d 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -14,6 +14,7 @@ from typing import ( Set, Tuple, Type, + TypeVar, Union, cast, ) @@ -216,7 +217,7 @@ class ModelField(Representation): self.name: str = name self.has_alias: bool = bool(alias) self.alias: str = alias or name - self.type_: type = type_ + self.type_: Any = type_ self.class_validators = class_validators or {} self.default: Any = default self.required: bool = required @@ -303,17 +304,29 @@ class ModelField(Representation): def _type_analysis(self) -> None: # noqa: C901 (ignore complexity) # typing interface is horrible, we have to do some ugly checks if lenient_issubclass(self.type_, JsonWrapper): - self.type_ = self.type_.inner_type # type: ignore + self.type_ = self.type_.inner_type self.parse_json = True elif lenient_issubclass(self.type_, Json): - self.type_ = Any # type: ignore + self.type_ = Any self.parse_json = True + elif isinstance(self.type_, TypeVar): # type: ignore + if self.type_.__bound__: + self.type_ = self.type_.__bound__ + elif self.type_.__constraints__: + self.type_ = Union[self.type_.__constraints__] + else: + self.type_ = Any - if self.type_ is Pattern: + if self.type_ is Any: + self.required = False + self.allow_none = True + return + elif self.type_ is Pattern: # python 3.7 only, Pattern is a typing object but without sub fields return - if is_literal_type(self.type_): + elif is_literal_type(self.type_): return + origin = getattr(self.type_, '__origin__', None) if origin is None: # field is not "typing" object eg. Union, Dict, List etc. @@ -322,7 +335,7 @@ class ModelField(Representation): return if origin is Union: types_ = [] - for type_ in self.type_.__args__: # type: ignore + for type_ in self.type_.__args__: if type_ is NoneType: # type: ignore self.required = False self.allow_none = True @@ -340,9 +353,9 @@ class ModelField(Representation): if issubclass(origin, Tuple): # type: ignore self.shape = SHAPE_TUPLE self.sub_fields = [] - for i, t in enumerate(self.type_.__args__): # type: ignore + for i, t in enumerate(self.type_.__args__): if t is Ellipsis: - self.type_ = self.type_.__args__[0] # type: ignore + self.type_ = self.type_.__args__[0] self.shape = SHAPE_TUPLE_ELLIPSIS return self.sub_fields.append(self._create_sub_type(t, f'{self.name}_{i}')) @@ -359,22 +372,20 @@ class ModelField(Representation): } ) - self.type_ = self.type_.__args__[0] # type: ignore + self.type_ = self.type_.__args__[0] self.shape = SHAPE_LIST elif issubclass(origin, Set): - self.type_ = self.type_.__args__[0] # type: ignore + self.type_ = self.type_.__args__[0] self.shape = SHAPE_SET elif issubclass(origin, FrozenSet): - self.type_ = self.type_.__args__[0] # type: ignore + self.type_ = self.type_.__args__[0] self.shape = SHAPE_FROZENSET elif issubclass(origin, Sequence): - self.type_ = self.type_.__args__[0] # type: ignore + self.type_ = self.type_.__args__[0] self.shape = SHAPE_SEQUENCE elif issubclass(origin, Mapping): - self.key_field = self._create_sub_type( - self.type_.__args__[0], 'key_' + self.name, for_keys=True # type: ignore - ) - self.type_ = self.type_.__args__[1] # type: ignore + self.key_field = self._create_sub_type(self.type_.__args__[0], 'key_' + self.name, for_keys=True) + self.type_ = self.type_.__args__[1] self.shape = SHAPE_MAPPING elif issubclass(origin, Type): # type: ignore return diff --git a/pydantic/schema.py b/pydantic/schema.py index 5a02ad4..a6e9849 100644 --- a/pydantic/schema.py +++ b/pydantic/schema.py @@ -319,7 +319,7 @@ def get_flat_models_from_field(field: ModelField, known_models: Set[Type['BaseMo # Handle dataclass-based models field_type = field.type_ if lenient_issubclass(getattr(field_type, '__pydantic_model__', None), BaseModel): - field_type = field_type.__pydantic_model__ # type: ignore + field_type = field_type.__pydantic_model__ if field.sub_fields: flat_models |= get_flat_models_from_fields(field.sub_fields, known_models=known_models) elif lenient_issubclass(field_type, BaseModel) and field_type not in known_models: @@ -700,7 +700,7 @@ def field_singleton_schema( # noqa: C901 (ignore complexity) return t_schema, definitions, nested_models # Handle dataclass-based models if lenient_issubclass(getattr(field_type, '__pydantic_model__', None), BaseModel): - field_type = field_type.__pydantic_model__ # type: ignore + field_type = field_type.__pydantic_model__ if issubclass(field_type, BaseModel): model_name = model_name_map[field_type] if field_type not in known_models: diff --git a/pydantic/validators.py b/pydantic/validators.py index 2ccefaf..14030e2 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -128,7 +128,7 @@ def strict_float_validator(v: Any) -> float: def number_multiple_validator(v: 'Number', field: 'ModelField') -> 'Number': - field_type: ConstrainedNumber = field.type_ # type: ignore + field_type: ConstrainedNumber = field.type_ if field_type.multiple_of is not None: mod = float(v) / float(field_type.multiple_of) % 1 if not almost_equal_floats(mod, 0.0) and not almost_equal_floats(mod, 1.0): @@ -137,7 +137,7 @@ def number_multiple_validator(v: 'Number', field: 'ModelField') -> 'Number': def number_size_validator(v: 'Number', field: 'ModelField') -> 'Number': - field_type: ConstrainedNumber = field.type_ # type: ignore + field_type: ConstrainedNumber = field.type_ if field_type.gt is not None and not v > field_type.gt: raise errors.NumberNotGtError(limit_value=field_type.gt) elif field_type.ge is not None and not v >= field_type.ge: @@ -243,7 +243,7 @@ def enum_validator(v: Any, field: 'ModelField', config: 'BaseConfig') -> Enum: enum_v = field.type_(v) except ValueError: # field.type_ should be an enum, so will be iterable - raise errors.EnumError(enum_values=list(field.type_)) # type: ignore + raise errors.EnumError(enum_values=list(field.type_)) return enum_v.value if config.use_enum_values else enum_v @@ -404,11 +404,11 @@ def make_literal_validator(type_: Any) -> Callable[[Any], Any]: def constr_length_validator(v: 'StrBytes', field: 'ModelField', config: 'BaseConfig') -> 'StrBytes': v_len = len(v) - min_length = field.type_.min_length or config.min_anystr_length # type: ignore + min_length = field.type_.min_length or config.min_anystr_length if min_length is not None and v_len < min_length: raise errors.AnyStrMinLengthError(limit_value=min_length) - max_length = field.type_.max_length or config.max_anystr_length # type: ignore + max_length = field.type_.max_length or config.max_anystr_length if max_length is not None and v_len > max_length: raise errors.AnyStrMaxLengthError(limit_value=max_length) @@ -416,7 +416,7 @@ def constr_length_validator(v: 'StrBytes', field: 'ModelField', config: 'BaseCon def constr_strip_whitespace(v: 'StrBytes', field: 'ModelField', config: 'BaseConfig') -> 'StrBytes': - strip_whitespace = field.type_.strip_whitespace or config.anystr_strip_whitespace # type: ignore + strip_whitespace = field.type_.strip_whitespace or config.anystr_strip_whitespace if strip_whitespace: v = v.strip() diff --git a/pydantic/version.py b/pydantic/version.py index 174338c..aed6c16 100644 --- a/pydantic/version.py +++ b/pydantic/version.py @@ -2,4 +2,4 @@ from distutils.version import StrictVersion __all__ = ['VERSION'] -VERSION = StrictVersion('1.0') +VERSION = StrictVersion('1.1a1') diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index 2a78276..2db55c3 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -2,7 +2,7 @@ import re import sys from decimal import Decimal from enum import Enum -from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union +from typing import Any, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union import pytest @@ -1220,3 +1220,86 @@ def test_field_type_display(type_, expected): a: type_ assert Model.__fields__['a']._type_display() == expected + + +def test_any_none(): + class MyModel(BaseModel): + foo: Any + + m = MyModel(foo=None) + assert dict(m) == {'foo': None} + + +def test_type_var_any(): + Foobar = TypeVar('Foobar') + + class MyModel(BaseModel): + foo: Foobar + + assert MyModel.schema() == {'title': 'MyModel', 'type': 'object', 'properties': {'foo': {'title': 'Foo'}}} + assert MyModel(foo=None).foo is None + assert MyModel(foo='x').foo == 'x' + assert MyModel(foo=123).foo == 123 + + +def test_type_var_constraint(): + Foobar = TypeVar('Foobar', int, str) + + class MyModel(BaseModel): + foo: Foobar + + assert MyModel.schema() == { + 'title': 'MyModel', + 'type': 'object', + 'properties': {'foo': {'title': 'Foo', 'anyOf': [{'type': 'integer'}, {'type': 'string'}]}}, + 'required': ['foo'], + } + with pytest.raises(ValidationError, match='none is not an allowed value'): + MyModel(foo=None) + with pytest.raises(ValidationError, match='value is not a valid integer'): + MyModel(foo=[1, 2, 3]) + assert MyModel(foo='x').foo == 'x' + assert MyModel(foo=123).foo == 123 + + +def test_type_var_bound(): + Foobar = TypeVar('Foobar', bound=int) + + class MyModel(BaseModel): + foo: Foobar + + assert MyModel.schema() == { + 'title': 'MyModel', + 'type': 'object', + 'properties': {'foo': {'title': 'Foo', 'type': 'integer'}}, + 'required': ['foo'], + } + with pytest.raises(ValidationError, match='none is not an allowed value'): + MyModel(foo=None) + with pytest.raises(ValidationError, match='value is not a valid integer'): + MyModel(foo='x') + assert MyModel(foo=123).foo == 123 + + +def test_dict_bare(): + class MyModel(BaseModel): + foo: Dict + + m = MyModel(foo={'x': 'a', 'y': None}) + assert m.foo == {'x': 'a', 'y': None} + + +def test_list_bare(): + class MyModel(BaseModel): + foo: List + + m = MyModel(foo=[1, 2, None]) + assert m.foo == [1, 2, None] + + +def test_dict_any(): + class MyModel(BaseModel): + foo: Dict[str, Any] + + m = MyModel(foo={'x': 'a', 'y': None}) + assert m.foo == {'x': 'a', 'y': None} diff --git a/tests/test_schema.py b/tests/test_schema.py index 5743709..4fd7489 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -261,12 +261,7 @@ def test_any(): class Model(BaseModel): a: Any - assert Model.schema() == { - 'title': 'Model', - 'type': 'object', - 'properties': {'a': {'title': 'A'}}, - 'required': ['a'], - } + assert Model.schema() == {'title': 'Model', 'type': 'object', 'properties': {'a': {'title': 'A'}}} def test_set(): @@ -677,7 +672,6 @@ def test_json_type(): 'title': 'Model', 'type': 'object', 'properties': {'a': {'title': 'A', 'type': 'string', 'format': 'json-string'}}, - 'required': ['a'], }