add support for annotation only fields (#41)

* add support for annotation only fields, fix #34

* adding tests with mypy

* adding docs for mypy usage

* adding mypy failure test

* adding alias tests

* tweak mypy tests

.travis.yml

@@ -23,6 +23,7 @@ install:
 script:
 - make lint
 - make test
+- make mypy
 - make docs-lint
 - make docs
 - make benchmark

HISTORY.rst

@@ -3,6 +3,10 @@
 History
 -------
 
+v0.2.0 (TBC)
+............
+* allow annotation only fields to support mypy
+
 v0.1.0 (2017-06-03)
 ...................
 * add docs

Makefile

@@ -1,3 +1,5 @@
+.DEFAULT_GOAL := all
+
 .PHONY: install
 install:
 	pip install -U setuptools pip
@@ -19,12 +21,25 @@ lint:
 test:
 	pytest --cov=pydantic
 
+.PHONY: mypy
+mypy:
+	@echo "testing simple example with mypy (and python to check it's sane)..."
+	mypy --ignore-missing-imports --follow-imports=skip --strict-optional tests/mypy_test_success.py
+	python tests/mypy_test_success.py
+	@echo "checking code with bad type annotations fails..."
+	@mypy --ignore-missing-imports --follow-imports=skip tests/mypy_test_fails.py 1>/dev/null; \
+	  test $$? -eq 1 || \
+	  (echo "mypy passed when it shouldn't"; exit 1)
+	python tests/mypy_test_fails.py
+
 .PHONY: testcov
 testcov:
-	pytest --cov=pydantic && (echo "building coverage html"; coverage html)
+	pytest --cov=pydantic
+	@echo "building coverage html"
+	@coverage html
 
 .PHONY: all
-all: testcov lint
+all: testcov mypy lint
 
 .PHONY: benchmark
 benchmark:

docs/examples/mypy.py

@@ -0,0 +1,17 @@
+from datetime import datetime
+from typing import List, Optional
+from pydantic import BaseModel, NoneStr
+
+class Model(BaseModel):
+    age: int
+    first_name = 'John'
+    last_name: NoneStr = None
+    signup_ts: Optional[datetime] = None
+    list_of_ints: List[int]
+
+m = Model(age=42, list_of_ints=[1, '2', b'3'])
+print(m.age)
+# > 42
+
+Model()
+# will raise a validation error for age and list_of_ints

docs/index.rst

@@ -19,7 +19,7 @@ Define how data should be in pure, canonical python; validate it with *pydantic*
 
 A simple example:
 
-.. literalinclude:: example1.py
+.. literalinclude:: examples/example1.py
 
 (This script is complete, it should run "as is")
 
@@ -35,7 +35,7 @@ What's going on here:
 
 If validation fails pydantic with raise an error with a breakdown of what was wrong:
 
-.. literalinclude:: example2.py
+.. literalinclude:: examples/example2.py
 
 Rationale
 ---------
@@ -48,7 +48,7 @@ So *pydantic* uses some cool new language feature, but why should I actually go
 
 **plays nicely with your IDE/linter/brain**
     because pydantic data structures are just instances of classes you define; auto-completion, linting,
-    `mypy <http://mypy-lang.org/>`_ and your intuition should all work properly with your validated data.
+    :ref:`mypy <usage_mypy>` and your intuition should all work properly with your validated data.
 
 **dual use**
     pydantic's :ref:`BaseSettings <settings>` class allows it to be used in both a "validate this request data" context
@@ -85,7 +85,7 @@ PEP 484 Types
 
 pydantic uses ``typing`` types to define more complex objects.
 
-.. literalinclude:: usage_typing.py
+.. literalinclude:: examples/ex_typing.py
 
 (This script is complete, it should run "as is")
 
@@ -94,7 +94,7 @@ Choices
 
 pydantic uses python's standard ``enum`` classes to define choices.
 
-.. literalinclude:: usage_choices.py
+.. literalinclude:: examples/choices.py
 
 (This script is complete, it should run "as is")
 
@@ -103,14 +103,14 @@ Recursive Models
 
 More complex hierarchical data structures can be defined using models as types in annotations themselves.
 
-.. literalinclude:: usage_recursive.py
+.. literalinclude:: examples/recursive.py
 
 (This script is complete, it should run "as is")
 
 Error Handling
 ..............
 
-.. literalinclude:: usage_errors.py
+.. literalinclude:: examples/errors.py
 
 (This script is complete, it should run "as is")
 
@@ -119,11 +119,10 @@ Exotic Types
 
 pydantic comes with a number of utilities for parsing or validating common objects.
 
-.. literalinclude:: usage_exotic.py
+.. literalinclude:: examples/exotic.py
 
 (This script is complete, it should run "as is")
 
-
 Model Config
 ............
 
@@ -131,7 +130,7 @@ Behaviour of pydantic can be controlled via the ``Config`` class on a model.
 
 Here default for config parameter are shown together with their meaning.
 
-.. literalinclude:: usage_config.py
+.. literalinclude:: examples/config.py
 
 .. _settings:
 
@@ -143,11 +142,53 @@ environment variables or keyword arguments (e.g. in unit tests).
 
 This usage example comes last as it uses numerous concepts described above.
 
-.. literalinclude:: usage_settings.py
+.. literalinclude:: examples/settings.py
 
 Here ``redis_port`` could be modified via ``export MY_PREFIX_REDIS_PORT=6380`` or ``auth_key`` by
 ``export my_api_key=6380``.
 
+.. _usage_mypy:
+
+Usage with mypy
+...............
+
+Pydantic works with `mypy <http://mypy-lang.org/>`_ provided you use the "annotation only" version of
+required variables:
+
+.. literalinclude:: examples/mypy.py
+
+This script is complete, it should run "as is". You can also run it through mypy with::
+
+    mypy --ignore-missing-imports --follow-imports=skip --strict-optional pydantic_mypy_test.py
+
+Strict Optional
+~~~~~~~~~~~~~~~
+
+For your code to pass with ``--strict-optional`` you need to to use ``Optional[]`` or an alias of ``Optional[]``
+for all fields with ``None`` default, this is standard with mypy.
+
+Pydantic provides a few useful optional or union types:
+
+* ``NoneStr`` aka. ``Optional[str]``
+* ``NoneBytes`` aka. ``Optional[bytes]``
+* ``StrBytes`` aka. ``Union[str, bytes]``
+* ``NoneStrBytes`` aka. ``Optional[StrBytes]``
+
+If these aren't sufficient you can of course define your own.
+
+Required Fields and mypy
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ellipsis notation ``...`` will not work with mypy, you need to use annotation only fields as in the example above.
+
+.. warning::
+
+   Be aware that using annotation only fields will alter the order of your fields in metadata and errors:
+   annotation only fields will always come last, but still in the order they were defined.
+
+To get round this you can use the ``Required`` (via ``from pydantic import Required``) field as an alias for
+ellipses or annotation only.
+
 .. include:: ../HISTORY.rst
 
 .. |pypi| image:: https://img.shields.io/pypi/v/pydantic.svg

docs/spelling_wordlist.txt

@@ -1,9 +1,11 @@
+aren
 cerberus
 Config
 config
 ints
 jsonmodels
 pydantic
+metadata
 schemas
 timestamp
 unix

pydantic/__init__.py

@@ -1,6 +1,7 @@
 # flake8: noqa
 from .env_settings import BaseSettings
 from .exceptions import *
+from .fields import Required
 from .main import BaseModel
 from .types import *
 from .version import VERSION

pydantic/fields.py

@@ -6,6 +6,8 @@ from typing import Any, List, Mapping, Set, Type, Union
 from .exceptions import ConfigError, Error, type_display
 from .validators import NoneType, find_validators, not_none_validator
 
+Required: Any = Ellipsis
+
 
 class ValidatorSignature(IntEnum):
     JUST_VALUE = 1
@@ -54,7 +56,7 @@ class Field:
 
     @classmethod
     def infer(cls, *, name, value, annotation, class_validators, field_config):
-        required = value == Ellipsis
+        required = value == Required
         return cls(
             name=name,
             type_=annotation,
@@ -255,7 +257,10 @@ class Field:
         return f'<Field {self}>'
 
     def __str__(self):
-        return f'{self.name}: ' + ', '.join(f'{k}={v!r}' for k, v in self.info.items())
+        if self.alt_alias:
+            return f"{self.name} (alias '{self.alias}'): " + ', '.join(f'{k}={v!r}' for k, v in self.info.items())
+        else:
+            return f'{self.name}: ' + ', '.join(f'{k}={v!r}' for k, v in self.info.items())
 
 
 def _get_validator_signature(validator):

pydantic/main.py

@@ -55,14 +55,29 @@ class MetaModel(type):
             field_config = config_fields.get(var_name)
             if isinstance(field_config, str):
                 field_config = {'alias': field_config}
-            field = Field.infer(
+            fields[var_name] = Field.infer(
                 name=var_name,
                 value=value,
-                annotation=annotations and annotations.get(var_name),
+                annotation=annotations and annotations.pop(var_name, None),
                 class_validators=class_validators,
                 field_config=field_config,
             )
-            fields[var_name] = field
+
+        if annotations:
+            for ann_name, ann_type in annotations.items():
+                if ann_name.startswith('_'):
+                    continue
+                field_config = config_fields.get(ann_name)
+                if isinstance(field_config, str):
+                    field_config = {'alias': field_config}
+                fields[ann_name] = Field.infer(
+                    name=ann_name,
+                    value=...,
+                    annotation=ann_type,
+                    class_validators=class_validators,
+                    field_config=field_config,
+                )
+
         namespace.update(
             config=config,
             __fields__=fields,
@@ -78,6 +93,7 @@ EXTRA_ERROR = Error(Extra('extra fields not permitted'), None, None)
 class BaseModel(metaclass=MetaModel):
     # populated by the metaclass, defined here to help IDEs only
     __fields__ = {}
+    Config = BaseConfig
 
     def __init__(self, **values):
         self.__values__ = {}

pydantic/version.py

@@ -2,4 +2,4 @@ from distutils.version import StrictVersion
 
 __all__ = ['VERSION']
 
-VERSION = StrictVersion('0.1.0')
+VERSION = StrictVersion('0.2.0')

tests/mypy_test_fails.py

@@ -0,0 +1,21 @@
+"""
+Test mypy failure with invalid types.
+"""
+from datetime import datetime
+from typing import List, Optional
+
+from pydantic import BaseModel, NoneStr
+
+
+class Model(BaseModel):
+    age: int
+    first_name = 'John'
+    last_name: NoneStr = None
+    signup_ts: Optional[datetime] = None
+    list_of_ints: List[int]
+
+
+m = Model(age=42, list_of_ints=[1, '2', b'3'])
+
+assert m.age == 42, m.age
+m.age = 'not integer'

tests/mypy_test_success.py

@@ -0,0 +1,46 @@
+"""
+Test pydantic's compliance with mypy.
+
+Do a little skipping about with types to demonstrate its usage.
+"""
+from datetime import datetime
+from typing import List, Optional
+
+from pydantic import BaseModel, NoneStr
+
+
+class Model(BaseModel):
+    age: int
+    first_name = 'John'
+    last_name: NoneStr = None
+    signup_ts: Optional[datetime] = None
+    list_of_ints: List[int]
+
+
+def dog_years(age: int) -> int:
+    return age * 7
+
+
+def day_of_week(dt: datetime) -> int:
+    return dt.date().isoweekday()
+
+
+m = Model(age=21, list_of_ints=[1, '2', b'3'])
+
+assert m.age == 21, m.age
+m.age = 42
+assert m.age == 42, m.age
+assert m.first_name == 'John', m.first_name
+assert m.last_name is None, m.last_name
+assert m.list_of_ints == [1, 2, 3], m.list_of_ints
+
+dog_age = dog_years(m.age)
+assert dog_age == 294, dog_age
+
+
+m = Model(age=2, first_name=b'Woof', last_name=b'Woof', signup_ts='2017-06-07 00:00', list_of_ints=[1, '2', b'3'])
+
+assert m.first_name == 'Woof', m.first_name
+assert m.last_name == 'Woof', m.last_name
+assert m.signup_ts == datetime(2017, 6, 7), m.signup_ts
+assert day_of_week(m.signup_ts) == 3

tests/requirements.txt

@@ -1,5 +1,6 @@
 coverage==4.4.1
 flake8==3.3.0
+mypy==0.511
 pycodestyle==2.3.1
 pyflakes==1.5.0
 pytest==3.1.1

tests/test_complex.py

@@ -369,6 +369,9 @@ def test_infer_alias():
             fields = {'a': '_a'}
 
     assert Model(_a='different').a == 'different'
+    assert repr(Model.__fields__['a']) == ("<Field a (alias '_a'): type='str', default='foobar',"
+                                           " required=False, validators=['not_none_validator', 'str_validator',"
+                                           " 'anystr_length_validator']>")
 
 
 def test_alias_error():
@@ -385,3 +388,17 @@ def test_alias_error():
 1 error validating input
 _a:
   invalid literal for int() with base 10: 'foo' (error_type=ValueError track=int)""" == str(exc_info.value)
+
+
+def test_annotation_config():
+    class Model(BaseModel):
+        a: float
+        b: int = 10
+        _c: str
+
+        class Config:
+            fields = {'a': 'foobar'}
+
+    assert list(Model.__fields__.keys()) == ['b', 'a']
+    assert [f.alias for f in Model.__fields__.values()] == ['b', 'foobar']
+    assert Model(foobar='123').a == 123.0

tests/test_main.py

@@ -3,7 +3,18 @@ from typing import Any
 
 import pytest
 
-from pydantic import BaseModel, ConfigError, NoneBytes, NoneStr, ValidationError, pretty_errors
+from pydantic import BaseModel, ConfigError, NoneBytes, NoneStr, Required, ValidationError, pretty_errors
+
+
+def test_success():
+    # same as below but defined here so class definition occurs inside the test
+    class Model(BaseModel):
+        a: float
+        b: int = 10
+
+    m = Model(a=10.2)
+    assert m.a == 10.2
+    assert m.b == 10
 
 
 class UltraSimpleModel(BaseModel):
@@ -11,12 +22,6 @@ class UltraSimpleModel(BaseModel):
     b: int = 10
 
 
-def test_ultra_simple_success():
-    m = UltraSimpleModel(a=10.2)
-    assert m.a == 10.2
-    assert m.b == 10
-
-
 def test_ultra_simple_missing():
     with pytest.raises(ValidationError) as exc_info:
         UltraSimpleModel()
@@ -258,3 +263,32 @@ def test_alias():
     assert Model().values == {'a': 'foobar'}
     assert Model(_a='different').a == 'different'
     assert Model(_a='different').values == {'a': 'different'}
+
+
+def test_field_order():
+    class Model(BaseModel):
+        c: float
+        b: int = 10
+        a: str
+        d: dict = {}
+
+    # fields are ordered as defined except annotation-only fields come last
+    assert list(Model.__fields__.keys()) == ['b', 'd', 'c', 'a']
+
+
+def test_required():
+    # same as below but defined here so class definition occurs inside the test
+    class Model(BaseModel):
+        a: float = Required
+        b: int = 10
+
+    m = Model(a=10.2)
+    assert m.values == dict(a=10.2, b=10)
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model()
+    assert """\
+1 error validating input
+a:
+  field required (error_type=Missing)\
+""" == str(exc_info.value)