Fix broken Any and TypeVar behaviour (#962)

* tests for broken Dict behaviour

* fix Any, support TypeVar

* change type of field.type_

* add docs and example for TypeVar

changes/962-samuelcolvin.md

@@ -0,0 +1,2 @@
+Fix usage of `Any` to allow `None`, also support `TypeVar` thus allowing use of un-parameterised collection types
+e.g. `Dict` and `List`

docs/examples/types_typevar.py

@@ -0,0 +1,16 @@
+from typing import TypeVar
+from pydantic import BaseModel
+
+Foobar = TypeVar('Foobar')
+BoundFloat = TypeVar('BoundFloat', bound=float)
+IntStr = TypeVar('IntStr', int, str)
+
+class Model(BaseModel):
+    a: Foobar  # equivalent of ": Any"
+    b: BoundFloat  # equivalent of ": float"
+    c: IntStr  # equivalent of ": Union[int, str]"
+
+print(Model(a=[1], b=4.2, c='x'))
+
+# a may be None and is therefore optional
+print(Model(b=1, c=1))

docs/usage/types.md

@@ -62,6 +62,12 @@ with custom properties and validation.
 `datetime.timedelta`
 : see [Datetime Types](#datetime-types) below for more detail on parsing and validation
 
+`typing.Any`
+: allows any value include `None`, thus an `Any` field is optional
+
+`typing.TypeVar`
+: constrains the values allowed based on `constraints` or `bound`, see [TypeVar](#typevar)
+
 `typing.Union`
 : see [Unions](#unions) below for more detail on parsing and validation
 
@@ -275,12 +281,23 @@ that are subclasses of `T`.
 ```py
 {!.tmp_examples/type_type.py!}
 ```
+_(This script is complete, it should run "as is")_
 
 You may also use `Type` to specify that any class is allowed.
 
 ```py
 {!.tmp_examples/bare_type_type.py!}
 ```
+_(This script is complete, it should run "as is")_
+
+### TypeVar
+
+`TypeVar` is supported either unconstrained, constrained or with a bound.
+
+```py
+{!.tmp_examples/types_typevar.py!}
+```
+_(This script is complete, it should run "as is")_
 
 ## Literal Type
 

pydantic/fields.py

@@ -14,6 +14,7 @@ from typing import (
     Set,
     Tuple,
     Type,
+    TypeVar,
     Union,
     cast,
 )
@@ -216,7 +217,7 @@ class ModelField(Representation):
         self.name: str = name
         self.has_alias: bool = bool(alias)
         self.alias: str = alias or name
-        self.type_: type = type_
+        self.type_: Any = type_
         self.class_validators = class_validators or {}
         self.default: Any = default
         self.required: bool = required
@@ -303,17 +304,29 @@ class ModelField(Representation):
     def _type_analysis(self) -> None:  # noqa: C901 (ignore complexity)
         # typing interface is horrible, we have to do some ugly checks
         if lenient_issubclass(self.type_, JsonWrapper):
-            self.type_ = self.type_.inner_type  # type: ignore
+            self.type_ = self.type_.inner_type
             self.parse_json = True
         elif lenient_issubclass(self.type_, Json):
-            self.type_ = Any  # type: ignore
+            self.type_ = Any
             self.parse_json = True
+        elif isinstance(self.type_, TypeVar):  # type: ignore
+            if self.type_.__bound__:
+                self.type_ = self.type_.__bound__
+            elif self.type_.__constraints__:
+                self.type_ = Union[self.type_.__constraints__]
+            else:
+                self.type_ = Any
 
-        if self.type_ is Pattern:
+        if self.type_ is Any:
+            self.required = False
+            self.allow_none = True
+            return
+        elif self.type_ is Pattern:
             # python 3.7 only, Pattern is a typing object but without sub fields
             return
-        if is_literal_type(self.type_):
+        elif is_literal_type(self.type_):
             return
+
         origin = getattr(self.type_, '__origin__', None)
         if origin is None:
             # field is not "typing" object eg. Union, Dict, List etc.
@@ -322,7 +335,7 @@ class ModelField(Representation):
             return
         if origin is Union:
             types_ = []
-            for type_ in self.type_.__args__:  # type: ignore
+            for type_ in self.type_.__args__:
                 if type_ is NoneType:  # type: ignore
                     self.required = False
                     self.allow_none = True
@@ -340,9 +353,9 @@ class ModelField(Representation):
         if issubclass(origin, Tuple):  # type: ignore
             self.shape = SHAPE_TUPLE
             self.sub_fields = []
-            for i, t in enumerate(self.type_.__args__):  # type: ignore
+            for i, t in enumerate(self.type_.__args__):
                 if t is Ellipsis:
-                    self.type_ = self.type_.__args__[0]  # type: ignore
+                    self.type_ = self.type_.__args__[0]
                     self.shape = SHAPE_TUPLE_ELLIPSIS
                     return
                 self.sub_fields.append(self._create_sub_type(t, f'{self.name}_{i}'))
@@ -359,22 +372,20 @@ class ModelField(Representation):
                     }
                 )
 
-            self.type_ = self.type_.__args__[0]  # type: ignore
+            self.type_ = self.type_.__args__[0]
             self.shape = SHAPE_LIST
         elif issubclass(origin, Set):
-            self.type_ = self.type_.__args__[0]  # type: ignore
+            self.type_ = self.type_.__args__[0]
             self.shape = SHAPE_SET
         elif issubclass(origin, FrozenSet):
-            self.type_ = self.type_.__args__[0]  # type: ignore
+            self.type_ = self.type_.__args__[0]
             self.shape = SHAPE_FROZENSET
         elif issubclass(origin, Sequence):
-            self.type_ = self.type_.__args__[0]  # type: ignore
+            self.type_ = self.type_.__args__[0]
             self.shape = SHAPE_SEQUENCE
         elif issubclass(origin, Mapping):
-            self.key_field = self._create_sub_type(
-                self.type_.__args__[0], 'key_' + self.name, for_keys=True  # type: ignore
-            )
-            self.type_ = self.type_.__args__[1]  # type: ignore
+            self.key_field = self._create_sub_type(self.type_.__args__[0], 'key_' + self.name, for_keys=True)
+            self.type_ = self.type_.__args__[1]
             self.shape = SHAPE_MAPPING
         elif issubclass(origin, Type):  # type: ignore
             return

pydantic/schema.py

@@ -319,7 +319,7 @@ def get_flat_models_from_field(field: ModelField, known_models: Set[Type['BaseMo
     # Handle dataclass-based models
     field_type = field.type_
     if lenient_issubclass(getattr(field_type, '__pydantic_model__', None), BaseModel):
-        field_type = field_type.__pydantic_model__  # type: ignore
+        field_type = field_type.__pydantic_model__
     if field.sub_fields:
         flat_models |= get_flat_models_from_fields(field.sub_fields, known_models=known_models)
     elif lenient_issubclass(field_type, BaseModel) and field_type not in known_models:
@@ -700,7 +700,7 @@ def field_singleton_schema(  # noqa: C901 (ignore complexity)
             return t_schema, definitions, nested_models
     # Handle dataclass-based models
     if lenient_issubclass(getattr(field_type, '__pydantic_model__', None), BaseModel):
-        field_type = field_type.__pydantic_model__  # type: ignore
+        field_type = field_type.__pydantic_model__
     if issubclass(field_type, BaseModel):
         model_name = model_name_map[field_type]
         if field_type not in known_models:

pydantic/validators.py

@@ -128,7 +128,7 @@ def strict_float_validator(v: Any) -> float:
 
 
 def number_multiple_validator(v: 'Number', field: 'ModelField') -> 'Number':
-    field_type: ConstrainedNumber = field.type_  # type: ignore
+    field_type: ConstrainedNumber = field.type_
     if field_type.multiple_of is not None:
         mod = float(v) / float(field_type.multiple_of) % 1
         if not almost_equal_floats(mod, 0.0) and not almost_equal_floats(mod, 1.0):
@@ -137,7 +137,7 @@ def number_multiple_validator(v: 'Number', field: 'ModelField') -> 'Number':
 
 
 def number_size_validator(v: 'Number', field: 'ModelField') -> 'Number':
-    field_type: ConstrainedNumber = field.type_  # type: ignore
+    field_type: ConstrainedNumber = field.type_
     if field_type.gt is not None and not v > field_type.gt:
         raise errors.NumberNotGtError(limit_value=field_type.gt)
     elif field_type.ge is not None and not v >= field_type.ge:
@@ -243,7 +243,7 @@ def enum_validator(v: Any, field: 'ModelField', config: 'BaseConfig') -> Enum:
         enum_v = field.type_(v)
     except ValueError:
         # field.type_ should be an enum, so will be iterable
-        raise errors.EnumError(enum_values=list(field.type_))  # type: ignore
+        raise errors.EnumError(enum_values=list(field.type_))
     return enum_v.value if config.use_enum_values else enum_v
 
 
@@ -404,11 +404,11 @@ def make_literal_validator(type_: Any) -> Callable[[Any], Any]:
 def constr_length_validator(v: 'StrBytes', field: 'ModelField', config: 'BaseConfig') -> 'StrBytes':
     v_len = len(v)
 
-    min_length = field.type_.min_length or config.min_anystr_length  # type: ignore
+    min_length = field.type_.min_length or config.min_anystr_length
     if min_length is not None and v_len < min_length:
         raise errors.AnyStrMinLengthError(limit_value=min_length)
 
-    max_length = field.type_.max_length or config.max_anystr_length  # type: ignore
+    max_length = field.type_.max_length or config.max_anystr_length
     if max_length is not None and v_len > max_length:
         raise errors.AnyStrMaxLengthError(limit_value=max_length)
 
@@ -416,7 +416,7 @@ def constr_length_validator(v: 'StrBytes', field: 'ModelField', config: 'BaseCon
 
 
 def constr_strip_whitespace(v: 'StrBytes', field: 'ModelField', config: 'BaseConfig') -> 'StrBytes':
-    strip_whitespace = field.type_.strip_whitespace or config.anystr_strip_whitespace  # type: ignore
+    strip_whitespace = field.type_.strip_whitespace or config.anystr_strip_whitespace
     if strip_whitespace:
         v = v.strip()
 

pydantic/version.py

@@ -2,4 +2,4 @@ from distutils.version import StrictVersion
 
 __all__ = ['VERSION']
 
-VERSION = StrictVersion('1.0')
+VERSION = StrictVersion('1.1a1')

tests/test_edge_cases.py

@@ -2,7 +2,7 @@ import re
 import sys
 from decimal import Decimal
 from enum import Enum
-from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
+from typing import Any, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union
 
 import pytest
 
@@ -1220,3 +1220,86 @@ def test_field_type_display(type_, expected):
         a: type_
 
     assert Model.__fields__['a']._type_display() == expected
+
+
+def test_any_none():
+    class MyModel(BaseModel):
+        foo: Any
+
+    m = MyModel(foo=None)
+    assert dict(m) == {'foo': None}
+
+
+def test_type_var_any():
+    Foobar = TypeVar('Foobar')
+
+    class MyModel(BaseModel):
+        foo: Foobar
+
+    assert MyModel.schema() == {'title': 'MyModel', 'type': 'object', 'properties': {'foo': {'title': 'Foo'}}}
+    assert MyModel(foo=None).foo is None
+    assert MyModel(foo='x').foo == 'x'
+    assert MyModel(foo=123).foo == 123
+
+
+def test_type_var_constraint():
+    Foobar = TypeVar('Foobar', int, str)
+
+    class MyModel(BaseModel):
+        foo: Foobar
+
+    assert MyModel.schema() == {
+        'title': 'MyModel',
+        'type': 'object',
+        'properties': {'foo': {'title': 'Foo', 'anyOf': [{'type': 'integer'}, {'type': 'string'}]}},
+        'required': ['foo'],
+    }
+    with pytest.raises(ValidationError, match='none is not an allowed value'):
+        MyModel(foo=None)
+    with pytest.raises(ValidationError, match='value is not a valid integer'):
+        MyModel(foo=[1, 2, 3])
+    assert MyModel(foo='x').foo == 'x'
+    assert MyModel(foo=123).foo == 123
+
+
+def test_type_var_bound():
+    Foobar = TypeVar('Foobar', bound=int)
+
+    class MyModel(BaseModel):
+        foo: Foobar
+
+    assert MyModel.schema() == {
+        'title': 'MyModel',
+        'type': 'object',
+        'properties': {'foo': {'title': 'Foo', 'type': 'integer'}},
+        'required': ['foo'],
+    }
+    with pytest.raises(ValidationError, match='none is not an allowed value'):
+        MyModel(foo=None)
+    with pytest.raises(ValidationError, match='value is not a valid integer'):
+        MyModel(foo='x')
+    assert MyModel(foo=123).foo == 123
+
+
+def test_dict_bare():
+    class MyModel(BaseModel):
+        foo: Dict
+
+    m = MyModel(foo={'x': 'a', 'y': None})
+    assert m.foo == {'x': 'a', 'y': None}
+
+
+def test_list_bare():
+    class MyModel(BaseModel):
+        foo: List
+
+    m = MyModel(foo=[1, 2, None])
+    assert m.foo == [1, 2, None]
+
+
+def test_dict_any():
+    class MyModel(BaseModel):
+        foo: Dict[str, Any]
+
+    m = MyModel(foo={'x': 'a', 'y': None})
+    assert m.foo == {'x': 'a', 'y': None}

tests/test_schema.py

@@ -261,12 +261,7 @@ def test_any():
     class Model(BaseModel):
         a: Any
 
-    assert Model.schema() == {
-        'title': 'Model',
-        'type': 'object',
-        'properties': {'a': {'title': 'A'}},
-        'required': ['a'],
-    }
+    assert Model.schema() == {'title': 'Model', 'type': 'object', 'properties': {'a': {'title': 'A'}}}
 
 
 def test_set():
@@ -677,7 +672,6 @@ def test_json_type():
         'title': 'Model',
         'type': 'object',
         'properties': {'a': {'title': 'A', 'type': 'string', 'format': 'json-string'}},
-        'required': ['a'],
     }