diff --git a/.travis.yml b/.travis.yml index 6d6c14b..05c1e9f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ install: script: - make lint - make test +- make mypy - make docs-lint - make docs - make benchmark diff --git a/HISTORY.rst b/HISTORY.rst index c27b3e5..e74331d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,10 @@ History ------- +v0.2.0 (TBC) +............ +* allow annotation only fields to support mypy + v0.1.0 (2017-06-03) ................... * add docs diff --git a/Makefile b/Makefile index a059216..c41bc9d 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +.DEFAULT_GOAL := all + .PHONY: install install: pip install -U setuptools pip @@ -19,12 +21,25 @@ lint: test: pytest --cov=pydantic +.PHONY: mypy +mypy: + @echo "testing simple example with mypy (and python to check it's sane)..." + mypy --ignore-missing-imports --follow-imports=skip --strict-optional tests/mypy_test_success.py + python tests/mypy_test_success.py + @echo "checking code with bad type annotations fails..." + @mypy --ignore-missing-imports --follow-imports=skip tests/mypy_test_fails.py 1>/dev/null; \ + test $$? -eq 1 || \ + (echo "mypy passed when it shouldn't"; exit 1) + python tests/mypy_test_fails.py + .PHONY: testcov testcov: - pytest --cov=pydantic && (echo "building coverage html"; coverage html) + pytest --cov=pydantic + @echo "building coverage html" + @coverage html .PHONY: all -all: testcov lint +all: testcov mypy lint .PHONY: benchmark benchmark: diff --git a/docs/usage_choices.py b/docs/examples/choices.py similarity index 100% rename from docs/usage_choices.py rename to docs/examples/choices.py diff --git a/docs/usage_config.py b/docs/examples/config.py similarity index 100% rename from docs/usage_config.py rename to docs/examples/config.py diff --git a/docs/usage_errors.py b/docs/examples/errors.py similarity index 100% rename from docs/usage_errors.py rename to docs/examples/errors.py diff --git a/docs/usage_typing.py b/docs/examples/ex_typing.py similarity index 100% rename from docs/usage_typing.py rename to docs/examples/ex_typing.py diff --git a/docs/example1.py b/docs/examples/example1.py similarity index 100% rename from docs/example1.py rename to docs/examples/example1.py diff --git a/docs/example2.py b/docs/examples/example2.py similarity index 100% rename from docs/example2.py rename to docs/examples/example2.py diff --git a/docs/usage_exotic.py b/docs/examples/exotic.py similarity index 100% rename from docs/usage_exotic.py rename to docs/examples/exotic.py diff --git a/docs/examples/mypy.py b/docs/examples/mypy.py new file mode 100644 index 0000000..d57f41e --- /dev/null +++ b/docs/examples/mypy.py @@ -0,0 +1,17 @@ +from datetime import datetime +from typing import List, Optional +from pydantic import BaseModel, NoneStr + +class Model(BaseModel): + age: int + first_name = 'John' + last_name: NoneStr = None + signup_ts: Optional[datetime] = None + list_of_ints: List[int] + +m = Model(age=42, list_of_ints=[1, '2', b'3']) +print(m.age) +# > 42 + +Model() +# will raise a validation error for age and list_of_ints diff --git a/docs/usage_recursive.py b/docs/examples/recursive.py similarity index 100% rename from docs/usage_recursive.py rename to docs/examples/recursive.py diff --git a/docs/usage_settings.py b/docs/examples/settings.py similarity index 100% rename from docs/usage_settings.py rename to docs/examples/settings.py diff --git a/docs/index.rst b/docs/index.rst index db9e711..4e5bed3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,7 +19,7 @@ Define how data should be in pure, canonical python; validate it with *pydantic* A simple example: -.. literalinclude:: example1.py +.. literalinclude:: examples/example1.py (This script is complete, it should run "as is") @@ -35,7 +35,7 @@ What's going on here: If validation fails pydantic with raise an error with a breakdown of what was wrong: -.. literalinclude:: example2.py +.. literalinclude:: examples/example2.py Rationale --------- @@ -48,7 +48,7 @@ So *pydantic* uses some cool new language feature, but why should I actually go **plays nicely with your IDE/linter/brain** because pydantic data structures are just instances of classes you define; auto-completion, linting, - `mypy `_ and your intuition should all work properly with your validated data. + :ref:`mypy ` and your intuition should all work properly with your validated data. **dual use** pydantic's :ref:`BaseSettings ` class allows it to be used in both a "validate this request data" context @@ -85,7 +85,7 @@ PEP 484 Types pydantic uses ``typing`` types to define more complex objects. -.. literalinclude:: usage_typing.py +.. literalinclude:: examples/ex_typing.py (This script is complete, it should run "as is") @@ -94,7 +94,7 @@ Choices pydantic uses python's standard ``enum`` classes to define choices. -.. literalinclude:: usage_choices.py +.. literalinclude:: examples/choices.py (This script is complete, it should run "as is") @@ -103,14 +103,14 @@ Recursive Models More complex hierarchical data structures can be defined using models as types in annotations themselves. -.. literalinclude:: usage_recursive.py +.. literalinclude:: examples/recursive.py (This script is complete, it should run "as is") Error Handling .............. -.. literalinclude:: usage_errors.py +.. literalinclude:: examples/errors.py (This script is complete, it should run "as is") @@ -119,11 +119,10 @@ Exotic Types pydantic comes with a number of utilities for parsing or validating common objects. -.. literalinclude:: usage_exotic.py +.. literalinclude:: examples/exotic.py (This script is complete, it should run "as is") - Model Config ............ @@ -131,7 +130,7 @@ Behaviour of pydantic can be controlled via the ``Config`` class on a model. Here default for config parameter are shown together with their meaning. -.. literalinclude:: usage_config.py +.. literalinclude:: examples/config.py .. _settings: @@ -143,11 +142,53 @@ environment variables or keyword arguments (e.g. in unit tests). This usage example comes last as it uses numerous concepts described above. -.. literalinclude:: usage_settings.py +.. literalinclude:: examples/settings.py Here ``redis_port`` could be modified via ``export MY_PREFIX_REDIS_PORT=6380`` or ``auth_key`` by ``export my_api_key=6380``. +.. _usage_mypy: + +Usage with mypy +............... + +Pydantic works with `mypy `_ provided you use the "annotation only" version of +required variables: + +.. literalinclude:: examples/mypy.py + +This script is complete, it should run "as is". You can also run it through mypy with:: + + mypy --ignore-missing-imports --follow-imports=skip --strict-optional pydantic_mypy_test.py + +Strict Optional +~~~~~~~~~~~~~~~ + +For your code to pass with ``--strict-optional`` you need to to use ``Optional[]`` or an alias of ``Optional[]`` +for all fields with ``None`` default, this is standard with mypy. + +Pydantic provides a few useful optional or union types: + +* ``NoneStr`` aka. ``Optional[str]`` +* ``NoneBytes`` aka. ``Optional[bytes]`` +* ``StrBytes`` aka. ``Union[str, bytes]`` +* ``NoneStrBytes`` aka. ``Optional[StrBytes]`` + +If these aren't sufficient you can of course define your own. + +Required Fields and mypy +~~~~~~~~~~~~~~~~~~~~~~~~ + +The ellipsis notation ``...`` will not work with mypy, you need to use annotation only fields as in the example above. + +.. warning:: + + Be aware that using annotation only fields will alter the order of your fields in metadata and errors: + annotation only fields will always come last, but still in the order they were defined. + +To get round this you can use the ``Required`` (via ``from pydantic import Required``) field as an alias for +ellipses or annotation only. + .. include:: ../HISTORY.rst .. |pypi| image:: https://img.shields.io/pypi/v/pydantic.svg diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 855bd85..70796f6 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -1,9 +1,11 @@ +aren cerberus Config config ints jsonmodels pydantic +metadata schemas timestamp unix diff --git a/pydantic/__init__.py b/pydantic/__init__.py index 1e623d7..cfe551d 100644 --- a/pydantic/__init__.py +++ b/pydantic/__init__.py @@ -1,6 +1,7 @@ # flake8: noqa from .env_settings import BaseSettings from .exceptions import * +from .fields import Required from .main import BaseModel from .types import * from .version import VERSION diff --git a/pydantic/fields.py b/pydantic/fields.py index b9de7ea..a3d891b 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -6,6 +6,8 @@ from typing import Any, List, Mapping, Set, Type, Union from .exceptions import ConfigError, Error, type_display from .validators import NoneType, find_validators, not_none_validator +Required: Any = Ellipsis + class ValidatorSignature(IntEnum): JUST_VALUE = 1 @@ -54,7 +56,7 @@ class Field: @classmethod def infer(cls, *, name, value, annotation, class_validators, field_config): - required = value == Ellipsis + required = value == Required return cls( name=name, type_=annotation, @@ -255,7 +257,10 @@ class Field: return f'' def __str__(self): - return f'{self.name}: ' + ', '.join(f'{k}={v!r}' for k, v in self.info.items()) + if self.alt_alias: + return f"{self.name} (alias '{self.alias}'): " + ', '.join(f'{k}={v!r}' for k, v in self.info.items()) + else: + return f'{self.name}: ' + ', '.join(f'{k}={v!r}' for k, v in self.info.items()) def _get_validator_signature(validator): diff --git a/pydantic/main.py b/pydantic/main.py index d0b080d..7638919 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -55,14 +55,29 @@ class MetaModel(type): field_config = config_fields.get(var_name) if isinstance(field_config, str): field_config = {'alias': field_config} - field = Field.infer( + fields[var_name] = Field.infer( name=var_name, value=value, - annotation=annotations and annotations.get(var_name), + annotation=annotations and annotations.pop(var_name, None), class_validators=class_validators, field_config=field_config, ) - fields[var_name] = field + + if annotations: + for ann_name, ann_type in annotations.items(): + if ann_name.startswith('_'): + continue + field_config = config_fields.get(ann_name) + if isinstance(field_config, str): + field_config = {'alias': field_config} + fields[ann_name] = Field.infer( + name=ann_name, + value=..., + annotation=ann_type, + class_validators=class_validators, + field_config=field_config, + ) + namespace.update( config=config, __fields__=fields, @@ -78,6 +93,7 @@ EXTRA_ERROR = Error(Extra('extra fields not permitted'), None, None) class BaseModel(metaclass=MetaModel): # populated by the metaclass, defined here to help IDEs only __fields__ = {} + Config = BaseConfig def __init__(self, **values): self.__values__ = {} diff --git a/pydantic/version.py b/pydantic/version.py index 6859261..626d41c 100644 --- a/pydantic/version.py +++ b/pydantic/version.py @@ -2,4 +2,4 @@ from distutils.version import StrictVersion __all__ = ['VERSION'] -VERSION = StrictVersion('0.1.0') +VERSION = StrictVersion('0.2.0') diff --git a/tests/mypy_test_fails.py b/tests/mypy_test_fails.py new file mode 100644 index 0000000..e6c6fc8 --- /dev/null +++ b/tests/mypy_test_fails.py @@ -0,0 +1,21 @@ +""" +Test mypy failure with invalid types. +""" +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel, NoneStr + + +class Model(BaseModel): + age: int + first_name = 'John' + last_name: NoneStr = None + signup_ts: Optional[datetime] = None + list_of_ints: List[int] + + +m = Model(age=42, list_of_ints=[1, '2', b'3']) + +assert m.age == 42, m.age +m.age = 'not integer' diff --git a/tests/mypy_test_success.py b/tests/mypy_test_success.py new file mode 100644 index 0000000..2bbb6cd --- /dev/null +++ b/tests/mypy_test_success.py @@ -0,0 +1,46 @@ +""" +Test pydantic's compliance with mypy. + +Do a little skipping about with types to demonstrate its usage. +""" +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel, NoneStr + + +class Model(BaseModel): + age: int + first_name = 'John' + last_name: NoneStr = None + signup_ts: Optional[datetime] = None + list_of_ints: List[int] + + +def dog_years(age: int) -> int: + return age * 7 + + +def day_of_week(dt: datetime) -> int: + return dt.date().isoweekday() + + +m = Model(age=21, list_of_ints=[1, '2', b'3']) + +assert m.age == 21, m.age +m.age = 42 +assert m.age == 42, m.age +assert m.first_name == 'John', m.first_name +assert m.last_name is None, m.last_name +assert m.list_of_ints == [1, 2, 3], m.list_of_ints + +dog_age = dog_years(m.age) +assert dog_age == 294, dog_age + + +m = Model(age=2, first_name=b'Woof', last_name=b'Woof', signup_ts='2017-06-07 00:00', list_of_ints=[1, '2', b'3']) + +assert m.first_name == 'Woof', m.first_name +assert m.last_name == 'Woof', m.last_name +assert m.signup_ts == datetime(2017, 6, 7), m.signup_ts +assert day_of_week(m.signup_ts) == 3 diff --git a/tests/requirements.txt b/tests/requirements.txt index 70c05e7..6335873 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,6 @@ coverage==4.4.1 flake8==3.3.0 +mypy==0.511 pycodestyle==2.3.1 pyflakes==1.5.0 pytest==3.1.1 diff --git a/tests/test_complex.py b/tests/test_complex.py index 1c612a1..9fbd6f6 100644 --- a/tests/test_complex.py +++ b/tests/test_complex.py @@ -369,6 +369,9 @@ def test_infer_alias(): fields = {'a': '_a'} assert Model(_a='different').a == 'different' + assert repr(Model.__fields__['a']) == ("") def test_alias_error(): @@ -385,3 +388,17 @@ def test_alias_error(): 1 error validating input _a: invalid literal for int() with base 10: 'foo' (error_type=ValueError track=int)""" == str(exc_info.value) + + +def test_annotation_config(): + class Model(BaseModel): + a: float + b: int = 10 + _c: str + + class Config: + fields = {'a': 'foobar'} + + assert list(Model.__fields__.keys()) == ['b', 'a'] + assert [f.alias for f in Model.__fields__.values()] == ['b', 'foobar'] + assert Model(foobar='123').a == 123.0 diff --git a/tests/test_main.py b/tests/test_main.py index 29c2b7a..2434aa6 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,7 +3,18 @@ from typing import Any import pytest -from pydantic import BaseModel, ConfigError, NoneBytes, NoneStr, ValidationError, pretty_errors +from pydantic import BaseModel, ConfigError, NoneBytes, NoneStr, Required, ValidationError, pretty_errors + + +def test_success(): + # same as below but defined here so class definition occurs inside the test + class Model(BaseModel): + a: float + b: int = 10 + + m = Model(a=10.2) + assert m.a == 10.2 + assert m.b == 10 class UltraSimpleModel(BaseModel): @@ -11,12 +22,6 @@ class UltraSimpleModel(BaseModel): b: int = 10 -def test_ultra_simple_success(): - m = UltraSimpleModel(a=10.2) - assert m.a == 10.2 - assert m.b == 10 - - def test_ultra_simple_missing(): with pytest.raises(ValidationError) as exc_info: UltraSimpleModel() @@ -258,3 +263,32 @@ def test_alias(): assert Model().values == {'a': 'foobar'} assert Model(_a='different').a == 'different' assert Model(_a='different').values == {'a': 'different'} + + +def test_field_order(): + class Model(BaseModel): + c: float + b: int = 10 + a: str + d: dict = {} + + # fields are ordered as defined except annotation-only fields come last + assert list(Model.__fields__.keys()) == ['b', 'd', 'c', 'a'] + + +def test_required(): + # same as below but defined here so class definition occurs inside the test + class Model(BaseModel): + a: float = Required + b: int = 10 + + m = Model(a=10.2) + assert m.values == dict(a=10.2, b=10) + + with pytest.raises(ValidationError) as exc_info: + Model() + assert """\ +1 error validating input +a: + field required (error_type=Missing)\ +""" == str(exc_info.value)