diff --git a/.gitignore b/.gitignore index c085eec..0614b89 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ docs/.tmp_schema_mappings.rst .pytest_cache/ .vscode/ _build/ +pydantic/*.c +pydantic/*.so +.auto-format diff --git a/.travis.yml b/.travis.yml index 35883d8..e5f40fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,58 +1,112 @@ +os: linux +dist: xenial +sudo: required language: python - cache: pip -matrix: - include: - - python: '3.6' - - python: '3.7' - dist: xenial - sudo: required - - python: 3.8-dev - dist: xenial - sudo: required - - allow_failures: - - python: 3.8-dev +python: +- '3.6' +- '3.7' +- '3.8-dev' install: - make install - pip freeze script: +# test without cython but with ujson and email-validator +- python -c "import sys, pydantic; print('compiled:', pydantic.compiled); sys.exit(1 if pydantic.compiled else 0)" +- make test + - make lint - -# test with and without ujson and email-validator then combine coverage -- make test && mv .coverage .coverage.extra -- pip uninstall -y ujson email-validator -- make test && mv .coverage .coverage.no-extra -- coverage combine - - make mypy - make external-mypy -- make docs -- BENCHMARK_REPEATS=1 make benchmark-all -- ./tests/check_tag.py after_success: - ls -lha - bash <(curl -s https://codecov.io/bash) -env: -- secure: "vpTd8bkwPBP0CV3EJBAwSMNMnNK3m/71dvTvBd1T4YGuefyJvYhtA7wauA5xRL9jpK2mu5QR5eo0owTUJhKi4DjpafMMd1bc4PnXlrdZFzkn3VsGmlKt74D/aJgiuiNyhd/Qvq4OxMHrMhf4f6lKWoMM1vh6yT0yp3+51SexSh2Me0Q+npxbjXwoxX5XUHRcoSLtFk4GbYI88a2I+08XWI6v+Awo/giQ5QurUJhjAklbosrrQVr1FCOkU0em5jeyZvEbZSLmaMtbX1JlRdKoJm6WMU+y9I7zj35w6ue/vgfcLz7b/HDZrBx7/L9g1LxRo80briueX/IbHvN7DOVFKvaXVmnEa6lIDdCeOLOyESpIbmjqmDKi8JeexdPNxKq4Tvo2VEA9dL2w2aw+aALNtU2OF5iEMfPTUQyosu/CNu2PKtiuZkSOdvpYbSy1WUNHJRvomdR4Olzg8ZIScNsxU3IIPdrlG/LUA8auXcE9juFeZfD6D2hQZATqWeEe/C2J7amNSD+mLLaTf6nMQw8oNtKYOvYK17M7xyvi7HXDy711Bi18U3x6Ye0xGx8CDbFwl0ICNzIk9rrSAh9hEHTvfdUUkk35pxifvO0Hrh4SArCA20ozcH/hHWBhyqGdxoIQ6KoDgNbIFIGQ6/vugxL/pt8z1sJwPfJnq8tRDAyWZvE=" +jobs: + allow_failures: + - python: '3.8-dev' -deploy: -- provider: pypi - user: samuelcolvin - password: - secure: QbXFF2puEWjhFUpD0yu2R+wP4QI1IKIomBkMizsiCyMutlexERElranyYB8bsakvjPaJ+zU14ufffh2u7UA7Zhep/iE4skRHq4XWxnnRLHGu5nyGf3+zSM3F9MOzV32eZ4CDLJtFb6I0ensjTpodJH2EsIYHYxTgndIZn56Qbh6CStj7Xg1zm0Ujxdzm4ZLgcS28SOF/tpjsDW9+GXwc6L1mAZWYiS98gVgzL1vBd9tL9uFbbuFwGz9uhFMzFJko7vXSl8urWB4qeCspKXa9iKH7/AOYSwXTCwcg8U2hhC9UsOapnga2BubZKlU5HRfSs9fQcpnzcP2lwhSmkrEFa8VOw83hX6+bL564xK1Q4kanfGZ1fLU4FYge3iOnqjH7ajO7xEcUrcOEYUPfxM4EfdiDw0xnAzE1ITGH1/pZikF+wjlu+ez7RmmnejgK7quT1WU7keo7pSlRSfQtNgNl6xu818x0xZ1TScfN6e9npNy4TYyIooMOOeI4tMdfcR4JClkjGKhAtBk81DH7isZgPv3uwocGnKZ2S7La97CE3ADzU3MTA9xVIOSOjzwuvAe72uS2nwzqXkS9KATdATkC9QCvheJ9jIBB4UcqnHbD8L1gkqdmZwXZqHZldq8wcqNYZb+81lumy5EZ6xSoEzlLDpXHe80EjMUOBkb5fz3D44s= - distributions: sdist bdist_wheel - skip_upload_docs: true - on: - tags: true + include: + - stage: test python: 3.6 -- provider: script - script: make publish - on: - tags: true + name: 'Cython 3.6' + script: + # test with cython, ujson and email-validator + - make build-cython-trace + - python -c "import sys, pydantic; print('compiled:', pydantic.compiled); sys.exit(0 if pydantic.compiled else 1)" + - make test + - stage: test + python: 3.7 + name: 'Cython 3.7' + script: + # test with cython, ujson and email-validator + - make build-cython-trace + - python -c "import sys, pydantic; print('compiled:', pydantic.compiled); sys.exit(0 if pydantic.compiled else 1)" + - make test + + - stage: test python: 3.6 + name: 'Without Deps 3.6' + script: + # test without cython, ujson and email-validator + - pip uninstall -y ujson email-validator + - make test + - stage: test + python: 3.7 + name: 'Without Deps 3.7' + script: + # test without cython, ujson and email-validator + - pip uninstall -y ujson email-validator cython + - make test + + - stage: test + python: 3.7 + name: 'Benchmarks' + script: + # default install skips cython compilation, need to compile for benchmarks + - make build-cython + - BENCHMARK_REPEATS=1 make benchmark-all + after_success: skip + + - stage: build + name: 'PyPI Build and Upload' + if: type = push AND (branch = master OR tag IS present) + python: 3.7 + services: + - docker + install: skip + script: + - ./tests/check_tag.py + - pip install -U cibuildwheel + - cibuildwheel --output-dir dist + - ls -lha dist + env: + - 'PIP=pip' + - 'CIBW_BUILD="cp36-manylinux1_x86_64 cp36-manylinux1_i686 cp37-manylinux1_x86_64 cp37-manylinux1_i686"' + - 'CIBW_BEFORE_BUILD="pip install -U cython"' + deploy: + provider: pypi + skip_cleanup: true + user: samuelcolvin + password: + secure: 'QbXFF2puEWjhFUpD0yu2R+wP4QI1IKIomBkMizsiCyMutlexERElranyYB8bsakvjPaJ+zU14ufffh2u7UA7Zhep/iE4skRHq4XWxnnRLHGu5nyGf3+zSM3F9MOzV32eZ4CDLJtFb6I0ensjTpodJH2EsIYHYxTgndIZn56Qbh6CStj7Xg1zm0Ujxdzm4ZLgcS28SOF/tpjsDW9+GXwc6L1mAZWYiS98gVgzL1vBd9tL9uFbbuFwGz9uhFMzFJko7vXSl8urWB4qeCspKXa9iKH7/AOYSwXTCwcg8U2hhC9UsOapnga2BubZKlU5HRfSs9fQcpnzcP2lwhSmkrEFa8VOw83hX6+bL564xK1Q4kanfGZ1fLU4FYge3iOnqjH7ajO7xEcUrcOEYUPfxM4EfdiDw0xnAzE1ITGH1/pZikF+wjlu+ez7RmmnejgK7quT1WU7keo7pSlRSfQtNgNl6xu818x0xZ1TScfN6e9npNy4TYyIooMOOeI4tMdfcR4JClkjGKhAtBk81DH7isZgPv3uwocGnKZ2S7La97CE3ADzU3MTA9xVIOSOjzwuvAe72uS2nwzqXkS9KATdATkC9QCvheJ9jIBB4UcqnHbD8L1gkqdmZwXZqHZldq8wcqNYZb+81lumy5EZ6xSoEzlLDpXHe80EjMUOBkb5fz3D44s=' + on: + tags: true + all_branches: true + + - stage: build + name: 'Docs Build and Upload' + if: type = push AND (branch = master OR tag IS present) + python: 3.7 + script: make docs + env: + - secure: "vpTd8bkwPBP0CV3EJBAwSMNMnNK3m/71dvTvBd1T4YGuefyJvYhtA7wauA5xRL9jpK2mu5QR5eo0owTUJhKi4DjpafMMd1bc4PnXlrdZFzkn3VsGmlKt74D/aJgiuiNyhd/Qvq4OxMHrMhf4f6lKWoMM1vh6yT0yp3+51SexSh2Me0Q+npxbjXwoxX5XUHRcoSLtFk4GbYI88a2I+08XWI6v+Awo/giQ5QurUJhjAklbosrrQVr1FCOkU0em5jeyZvEbZSLmaMtbX1JlRdKoJm6WMU+y9I7zj35w6ue/vgfcLz7b/HDZrBx7/L9g1LxRo80briueX/IbHvN7DOVFKvaXVmnEa6lIDdCeOLOyESpIbmjqmDKi8JeexdPNxKq4Tvo2VEA9dL2w2aw+aALNtU2OF5iEMfPTUQyosu/CNu2PKtiuZkSOdvpYbSy1WUNHJRvomdR4Olzg8ZIScNsxU3IIPdrlG/LUA8auXcE9juFeZfD6D2hQZATqWeEe/C2J7amNSD+mLLaTf6nMQw8oNtKYOvYK17M7xyvi7HXDy711Bi18U3x6Ye0xGx8CDbFwl0ICNzIk9rrSAh9hEHTvfdUUkk35pxifvO0Hrh4SArCA20ozcH/hHWBhyqGdxoIQ6KoDgNbIFIGQ6/vugxL/pt8z1sJwPfJnq8tRDAyWZvE=" + deploy: + provider: script + script: make publish + on: + tags: true diff --git a/HISTORY.rst b/HISTORY.rst index f1e88a6..0354e9b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,12 +3,18 @@ History ------- -v0.xx (xxxx-xx-xx) +v0.27 (unreleased) .................. -* fix JSON Schema for ``list``, ``tuple``, and ``set``, #540 by @tiangolo -* Change `_pydantic_post_init` to execute dataclass' original `__post_init__` before +* Change ``_pydantic_post_init`` to execute dataclass' original ``__post_init__`` before validation, #560 by @HeavenVolkoff * fix handling of generic types without specified parameters, #550 by @dmontagu +* **breaking change** (maybe): this is the first release compiled with **cython**, see the docs and please + submit an issue if you run into problems + +v0.27.0a1 (2019-05-26) +...................... +* fix JSON Schema for ``list``, ``tuple``, and ``set``, #540 by @tiangolo +* compiling with cython, ``manylinux`` binaries, some other performance improvements, #548 by @samuelcolvin v0.26 (2019-05-22) .................. diff --git a/Makefile b/Makefile index c08cc0b..7fbe9f2 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,15 @@ black = black -S -l 120 --target-version py36 pydantic tests install: pip install -U setuptools pip pip install -U -r requirements.txt - pip install -e . + SKIP_CYTHON=1 pip install -e . + +.PHONY: build-cython-trace +build-cython-trace: clean + python setup.py build_ext --force --inplace --define CYTHON_TRACE + +.PHONY: build-cython +build-cython: clean + python setup.py build_ext --inplace .PHONY: format format: @@ -42,8 +50,12 @@ external-mypy: (echo "mypy_test_fails2: mypy passed when it should have failed!"; exit 1) .PHONY: testcov -testcov: - pytest --cov=pydantic +testcov: test + @echo "building coverage html" + @coverage html + +.PHONY: testcov-compile +testcov-compile: build-cython-trace test @echo "building coverage html" @coverage html @@ -72,6 +84,7 @@ clean: rm -f .coverage rm -f .coverage.* rm -rf build + rm -f pydantic/*.c pydantic/*.so python setup.py clean make -C docs clean diff --git a/docs/index.rst b/docs/index.rst index 668d765..65d7d63 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -80,8 +80,19 @@ Just:: *pydantic* has no required dependencies except python 3.6 or 3.7 (and the dataclasses package in python 3.6). If you've got python 3.6 and ``pip`` installed - you're good to go. +*pydantic* can optionally be compiled with `cython `_ which should give a 30-50% performance +improvement. ``manylinux`` binaries exist for python 3.6 and 3.7, so if you're installing from PyPI on linux, you +should get *pydantic* compiled with no extra work. If you're installing manually, install ``cython`` before installing +*pydantic* and you should get *pydandic* compiled. Compilation with cython is not tested on windows or mac. +`[issue] `_ + +To test if *pydantic* is compiled run:: + + import pydantic + print('compiled:', pydantic.compiled) + If you want *pydantic* to parse json faster you can add `ujson `_ -as an optional dependency. Similarly if *pydantic's* email validation relies on +as an optional dependency. Similarly *pydantic's* email validation relies on `email-validator `_ :: pip install pydantic[ujson] diff --git a/pydantic/__init__.py b/pydantic/__init__.py index 67cd874..717f1f7 100644 --- a/pydantic/__init__.py +++ b/pydantic/__init__.py @@ -5,7 +5,7 @@ from .env_settings import BaseSettings from .error_wrappers import ValidationError from .errors import * from .fields import Required -from .main import BaseConfig, BaseModel, Extra, create_model, validate_model +from .main import BaseConfig, BaseModel, Extra, compiled, create_model, validate_model from .parse import Protocol from .schema import Schema from .types import * diff --git a/pydantic/class_validators.py b/pydantic/class_validators.py index ca0023b..8c389fd 100644 --- a/pydantic/class_validators.py +++ b/pydantic/class_validators.py @@ -1,5 +1,4 @@ from collections import ChainMap -from dataclasses import dataclass from functools import wraps from inspect import Signature, signature from itertools import chain @@ -17,13 +16,13 @@ if TYPE_CHECKING: # pragma: no cover ValidatorCallable = Callable[[Optional[ModelOrDc], Any, Dict[str, Any], Field, Type[BaseConfig]], Any] -@dataclass class Validator: - func: AnyCallable - pre: bool - whole: bool - always: bool - check_fields: bool + def __init__(self, func: AnyCallable, pre: bool, whole: bool, always: bool, check_fields: bool): + self.func = func + self.pre = pre + self.whole = whole + self.always = always + self.check_fields = check_fields _FUNCS: Set[str] = set() @@ -57,7 +56,10 @@ def validator( raise ConfigError(f'duplicate validator function "{ref}"') _FUNCS.add(ref) f_cls = classmethod(f) - f_cls.__validator_config = fields, Validator(f, pre, whole, always, check_fields) # type: ignore + f_cls.__validator_config = ( # type: ignore + fields, + Validator(func=f, pre=pre, whole=whole, always=always, check_fields=check_fields), + ) return f_cls return dec diff --git a/pydantic/color.py b/pydantic/color.py index c21e03f..aaa13ae 100644 --- a/pydantic/color.py +++ b/pydantic/color.py @@ -10,7 +10,7 @@ eg. Color((0, 255, 255)).as_named() == 'cyan' because "cyan" comes after "aqua". import math import re from colorsys import hls_to_rgb, rgb_to_hls -from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Tuple, Union, cast +from typing import TYPE_CHECKING, Any, Optional, Tuple, Union, cast from pydantic.validators import not_none_validator @@ -20,21 +20,28 @@ from .utils import almost_equal_floats if TYPE_CHECKING: # pragma: no cover from .types import CallableGenerator - ColorTuple = Union[Tuple[int, int, int], Tuple[int, int, int, float]] ColorType = Union[ColorTuple, str] HslColorTuple = Union[Tuple[float, float, float], Tuple[float, float, float, float]] -class RGBA(NamedTuple): +class RGBA: """ Internal use only as a representation of a color. """ - r: float - g: float - b: float - alpha: Optional[float] + __slots__ = 'r', 'g', 'b', 'alpha', '_tuple' + + def __init__(self, r: float, g: float, b: float, alpha: Optional[float]): + self.r = r + self.g = g + self.b = b + self.alpha = alpha + + self._tuple: Tuple[float, float, float, Optional[float]] = (r, g, b, alpha) + + def __getitem__(self, item: Any) -> Any: + return self._tuple[item] r_hex_short = re.compile(r'\s*(?:#|0x)?([0-9a-f])([0-9a-f])([0-9a-f])([0-9a-f])?\s*') diff --git a/pydantic/error_wrappers.py b/pydantic/error_wrappers.py index c30db5e..daf0e53 100644 --- a/pydantic/error_wrappers.py +++ b/pydantic/error_wrappers.py @@ -1,6 +1,6 @@ import json from functools import lru_cache -from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Sequence, Tuple, Type, Union, cast +from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Sequence, Tuple, Type, Union if TYPE_CHECKING: # pragma: no cover from pydantic import BaseConfig # noqa: F401 @@ -9,13 +9,14 @@ __all__ = ('ErrorWrapper', 'ValidationError') class ErrorWrapper: - __slots__ = 'exc', 'loc', 'msg_template' + __slots__ = 'exc', 'type_', 'loc', 'msg_template' def __init__( self, exc: Exception, *, loc: Union[Tuple[str, ...], str], config: Optional[Type['BaseConfig']] = None ) -> None: self.exc = exc - self.loc: Tuple[str, ...] = cast(Tuple[str, ...], loc if isinstance(loc, tuple) else (loc,)) + self.type_ = get_exc_type(type(exc)) + self.loc: Tuple[str, ...] = loc if isinstance(loc, tuple) else (loc,) # type: ignore self.msg_template = config.error_msg_templates.get(self.type_) if config else None @property @@ -31,10 +32,6 @@ class ErrorWrapper: return str(self.exc) - @property - def type_(self) -> str: - return get_exc_type(self.exc) - def dict(self, *, loc_prefix: Optional[Tuple[str, ...]] = None) -> Dict[str, Any]: loc = self.loc if loc_prefix is None else loc_prefix + self.loc @@ -107,8 +104,7 @@ def flatten_errors( @lru_cache() -def get_exc_type(exc: Exception) -> str: - cls = type(exc) +def get_exc_type(cls: Type[Exception]) -> str: base_name = 'type_error' if issubclass(cls, TypeError) else 'value_error' if cls in (TypeError, ValueError): diff --git a/pydantic/fields.py b/pydantic/fields.py index 551cb84..f6f3c99 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -249,11 +249,7 @@ class Field: ) v_funcs = ( *[v.func for v in class_validators_ if not v.whole and v.pre], - *( - get_validators() - if get_validators - else find_validators(self.type_, self.model_config.arbitrary_types_allowed) - ), + *(get_validators() if get_validators else list(find_validators(self.type_, self.model_config))), self.schema is not None and self.schema.const and constant_validator, *[v.func for v in class_validators_ if not v.whole and not v.pre], ) @@ -300,13 +296,13 @@ class Field: v, errors = self._apply_validators(v, values, loc, cls, self.whole_post_validators) return v, errors - def _validate_json(self, v: str, loc: Tuple[str, ...]) -> Tuple[Optional[Any], Optional[ErrorWrapper]]: + def _validate_json(self, v: Any, loc: Tuple[str, ...]) -> Tuple[Optional[Any], Optional[ErrorWrapper]]: try: return Json.validate(v), None except (ValueError, TypeError) as exc: return v, ErrorWrapper(exc, loc=loc, config=self.model_config) - def _validate_sequence_like( # noqa: C901 (ignore complexity) + def _validate_sequence_like( self, v: Any, values: Dict[str, Any], loc: 'LocType', cls: Optional['ModelOrDc'] ) -> 'ValidateReturn': """ diff --git a/pydantic/main.py b/pydantic/main.py index 5100fe6..b16c4de 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -60,6 +60,14 @@ if TYPE_CHECKING: # pragma: no cover Model = TypeVar('Model', bound='BaseModel') +try: + import cython # type: ignore +except ImportError: + compiled: bool = False +else: # pragma: no cover + compiled = cython.compiled + + class Extra(str, Enum): allow = 'allow' ignore = 'ignore' @@ -168,34 +176,35 @@ class MetaModel(ABCMeta): annotations = resolve_annotations(annotations, namespace.get('__module__', None)) class_vars = set() - # annotation only fields need to come first in fields - for ann_name, ann_type in annotations.items(): - if is_classvar(ann_type): - class_vars.add(ann_name) - elif not ann_name.startswith('_') and ann_name not in namespace: - validate_field_name(bases, ann_name) - fields[ann_name] = Field.infer( - name=ann_name, - value=..., - annotation=ann_type, - class_validators=vg.get_validators(ann_name), - config=config, - ) + if (namespace.get('__module__'), namespace.get('__qualname__')) != ('pydantic.main', 'BaseModel'): + # annotation only fields need to come first in fields + for ann_name, ann_type in annotations.items(): + if is_classvar(ann_type): + class_vars.add(ann_name) + elif not ann_name.startswith('_') and ann_name not in namespace: + validate_field_name(bases, ann_name) + fields[ann_name] = Field.infer( + name=ann_name, + value=..., + annotation=ann_type, + class_validators=vg.get_validators(ann_name), + config=config, + ) - for var_name, value in namespace.items(): - if ( - not var_name.startswith('_') - and (annotations.get(var_name) == PyObject or not isinstance(value, TYPE_BLACKLIST)) - and var_name not in class_vars - ): - validate_field_name(bases, var_name) - fields[var_name] = Field.infer( - name=var_name, - value=value, - annotation=annotations.get(var_name), - class_validators=vg.get_validators(var_name), - config=config, - ) + for var_name, value in namespace.items(): + if ( + not var_name.startswith('_') + and (annotations.get(var_name) == PyObject or not isinstance(value, TYPE_BLACKLIST)) + and var_name not in class_vars + ): + validate_field_name(bases, var_name) + fields[var_name] = Field.infer( + name=var_name, + value=value, + annotation=annotations.get(var_name), + class_validators=vg.get_validators(var_name), + config=config, + ) vg.check_for_unused() if config.json_encoders: @@ -213,9 +222,6 @@ class MetaModel(ABCMeta): return super().__new__(mcs, name, bases, new_namespace) -_missing = object() - - class BaseModel(metaclass=MetaModel): if TYPE_CHECKING: # pragma: no cover # populated by the metaclass, defined here to help IDEs only @@ -511,7 +517,7 @@ class BaseModel(metaclass=MetaModel): return ret -def create_model( # noqa: C901 (ignore complexity) +def create_model( model_name: str, *, __config__: Type[BaseConfig] = None, @@ -567,6 +573,9 @@ def create_model( # noqa: C901 (ignore complexity) return type(model_name, (__base__,), namespace) +_missing = object() + + def validate_model( # noqa: C901 (ignore complexity) model: Union[BaseModel, Type[BaseModel]], input_data: 'DictStrAny', raise_exc: bool = True, cls: 'ModelOrDc' = None ) -> Tuple['DictStrAny', 'SetStr', Optional[ValidationError]]: diff --git a/pydantic/schema.py b/pydantic/schema.py index 2438142..fb1603c 100644 --- a/pydantic/schema.py +++ b/pydantic/schema.py @@ -6,7 +6,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Set, Tuple, Type, Union, cast from uuid import UUID -from . import main +import pydantic + from .fields import Field, Shape from .json import pydantic_encoder from .types import ( @@ -46,6 +47,9 @@ from .utils import clean_docstring, is_callable_type, lenient_issubclass if TYPE_CHECKING: # pragma: no cover from . import dataclasses # noqa: F401 + BaseModel = pydantic.main.BaseModel + + __all__ = [ 'Schema', 'schema', @@ -155,7 +159,7 @@ class Schema: def schema( - models: Sequence[Type['main.BaseModel']], + models: Sequence[Type['BaseModel']], *, by_alias: bool = True, title: Optional[str] = None, @@ -199,9 +203,7 @@ def schema( return output_schema -def model_schema( - model: Type['main.BaseModel'], by_alias: bool = True, ref_prefix: Optional[str] = None -) -> Dict[str, Any]: +def model_schema(model: Type['BaseModel'], by_alias: bool = True, ref_prefix: Optional[str] = None) -> Dict[str, Any]: """ Generate a JSON Schema for one model. With all the sub-models defined in the ``definitions`` top-level JSON key. @@ -230,7 +232,7 @@ def field_schema( field: Field, *, by_alias: bool = True, - model_name_map: Dict[Type['main.BaseModel'], str], + model_name_map: Dict[Type['BaseModel'], str], ref_prefix: Optional[str] = None, ) -> Tuple[Dict[str, Any], Dict[str, Any]]: """ @@ -321,7 +323,7 @@ def get_field_schema_validations(field: Field) -> Dict[str, Any]: return f_schema -def get_model_name_map(unique_models: Set[Type['main.BaseModel']]) -> Dict[Type['main.BaseModel'], str]: +def get_model_name_map(unique_models: Set[Type['BaseModel']]) -> Dict[Type['BaseModel'], str]: """ Process a set of models and generate unique names for them to be used as keys in the JSON Schema definitions. By default the names are the same as the class name. But if two models in different Python @@ -348,7 +350,7 @@ def get_model_name_map(unique_models: Set[Type['main.BaseModel']]) -> Dict[Type[ return {v: k for k, v in name_model_map.items()} -def get_flat_models_from_model(model: Type['main.BaseModel']) -> Set[Type['main.BaseModel']]: +def get_flat_models_from_model(model: Type['BaseModel']) -> Set[Type['BaseModel']]: """ Take a single ``model`` and generate a set with itself and all the sub-models in the tree. I.e. if you pass model ``Foo`` (subclass of Pydantic ``BaseModel``) as ``model``, and it has a field of type ``Bar`` (also @@ -358,14 +360,14 @@ def get_flat_models_from_model(model: Type['main.BaseModel']) -> Set[Type['main. :param model: a Pydantic ``BaseModel`` subclass :return: a set with the initial model and all its sub-models """ - flat_models: Set[Type['main.BaseModel']] = set() + flat_models: Set[Type['BaseModel']] = set() flat_models.add(model) fields = cast(Sequence[Field], model.__fields__.values()) flat_models |= get_flat_models_from_fields(fields) return flat_models -def get_flat_models_from_field(field: Field) -> Set[Type['main.BaseModel']]: +def get_flat_models_from_field(field: Field) -> Set[Type['BaseModel']]: """ Take a single Pydantic ``Field`` (from a model) that could have been declared as a sublcass of BaseModel (so, it could be a submodel), and generate a set with its model and all the sub-models in the tree. @@ -376,18 +378,18 @@ def get_flat_models_from_field(field: Field) -> Set[Type['main.BaseModel']]: :param field: a Pydantic ``Field`` :return: a set with the model used in the declaration for this field, if any, and all its sub-models """ - flat_models: Set[Type['main.BaseModel']] = set() + flat_models: Set[Type['BaseModel']] = set() if field.sub_fields: flat_models |= get_flat_models_from_fields(field.sub_fields) - elif lenient_issubclass(field.type_, main.BaseModel): + elif lenient_issubclass(field.type_, pydantic.BaseModel): flat_models |= get_flat_models_from_model(field.type_) - elif lenient_issubclass(getattr(field.type_, '__pydantic_model__', None), main.BaseModel): + elif lenient_issubclass(getattr(field.type_, '__pydantic_model__', None), pydantic.BaseModel): field.type_ = cast(Type['dataclasses.DataclassType'], field.type_) flat_models |= get_flat_models_from_model(field.type_.__pydantic_model__) return flat_models -def get_flat_models_from_fields(fields: Sequence[Field]) -> Set[Type['main.BaseModel']]: +def get_flat_models_from_fields(fields: Sequence[Field]) -> Set[Type['BaseModel']]: """ Take a list of Pydantic ``Field``s (from a model) that could have been declared as sublcasses of ``BaseModel`` (so, any of them could be a submodel), and generate a set with their models and all the sub-models in the tree. @@ -398,25 +400,25 @@ def get_flat_models_from_fields(fields: Sequence[Field]) -> Set[Type['main.BaseM :param fields: a list of Pydantic ``Field``s :return: a set with any model declared in the fields, and all their sub-models """ - flat_models: Set[Type['main.BaseModel']] = set() + flat_models: Set[Type['BaseModel']] = set() for field in fields: flat_models |= get_flat_models_from_field(field) return flat_models -def get_flat_models_from_models(models: Sequence[Type['main.BaseModel']]) -> Set[Type['main.BaseModel']]: +def get_flat_models_from_models(models: Sequence[Type['BaseModel']]) -> Set[Type['BaseModel']]: """ Take a list of ``models`` and generate a set with them and all their sub-models in their trees. I.e. if you pass a list of two models, ``Foo`` and ``Bar``, both subclasses of Pydantic ``BaseModel`` as models, and ``Bar`` has a field of type ``Baz`` (also subclass of ``BaseModel``), the return value will be ``set([Foo, Bar, Baz])``. """ - flat_models: Set[Type['main.BaseModel']] = set() + flat_models: Set[Type['BaseModel']] = set() for model in models: flat_models |= get_flat_models_from_model(model) return flat_models -def get_long_model_name(model: Type['main.BaseModel']) -> str: +def get_long_model_name(model: Type['BaseModel']) -> str: return f'{model.__module__}__{model.__name__}'.replace('.', '__') @@ -424,7 +426,7 @@ def field_type_schema( field: Field, *, by_alias: bool, - model_name_map: Dict[Type['main.BaseModel'], str], + model_name_map: Dict[Type['BaseModel'], str], schema_overrides: bool = False, ref_prefix: Optional[str] = None, ) -> Tuple[Dict[str, Any], Dict[str, Any]]: @@ -490,10 +492,10 @@ def field_type_schema( def model_process_schema( - model: Type['main.BaseModel'], + model: Type['BaseModel'], *, by_alias: bool = True, - model_name_map: Dict[Type['main.BaseModel'], str], + model_name_map: Dict[Type['BaseModel'], str], ref_prefix: Optional[str] = None, ) -> Tuple[Dict[str, Any], Dict[str, Any]]: """ @@ -515,10 +517,10 @@ def model_process_schema( def model_type_schema( - model: Type['main.BaseModel'], + model: Type['BaseModel'], *, by_alias: bool, - model_name_map: Dict[Type['main.BaseModel'], str], + model_name_map: Dict[Type['BaseModel'], str], ref_prefix: Optional[str] = None, ) -> Tuple[Dict[str, Any], Dict[str, Any]]: """ @@ -558,7 +560,7 @@ def field_singleton_sub_fields_schema( sub_fields: Sequence[Field], *, by_alias: bool, - model_name_map: Dict[Type['main.BaseModel'], str], + model_name_map: Dict[Type['BaseModel'], str], schema_overrides: bool = False, ref_prefix: Optional[str] = None, ) -> Tuple[Dict[str, Any], Dict[str, Any]]: @@ -657,7 +659,7 @@ def field_singleton_schema( # noqa: C901 (ignore complexity) field: Field, *, by_alias: bool, - model_name_map: Dict[Type['main.BaseModel'], str], + model_name_map: Dict[Type['BaseModel'], str], schema_overrides: bool = False, ref_prefix: Optional[str] = None, ) -> Tuple[Dict[str, Any], Dict[str, Any]]: @@ -705,10 +707,10 @@ def field_singleton_schema( # noqa: C901 (ignore complexity) return t_schema, definitions # Handle dataclass-based models field_type = field.type_ - if lenient_issubclass(getattr(field_type, '__pydantic_model__', None), main.BaseModel): + if lenient_issubclass(getattr(field_type, '__pydantic_model__', None), pydantic.BaseModel): field_type = cast(Type['dataclasses.DataclassType'], field_type) field_type = field_type.__pydantic_model__ - if issubclass(field_type, main.BaseModel): + if issubclass(field_type, pydantic.BaseModel): sub_schema, sub_definitions = model_process_schema( field_type, by_alias=by_alias, model_name_map=model_name_map, ref_prefix=ref_prefix ) diff --git a/pydantic/types.py b/pydantic/types.py index 05eec58..8c379c0 100644 --- a/pydantic/types.py +++ b/pydantic/types.py @@ -18,8 +18,6 @@ from uuid import UUID from . import errors from .utils import AnyType, change_exception, import_string, make_dsn, url_regex_generator, validate_email from .validators import ( - anystr_length_validator, - anystr_strip_whitespace, bytes_validator, decimal_validator, float_validator, @@ -87,10 +85,10 @@ OptionalIntFloat = Union[OptionalInt, float] OptionalIntFloatDecimal = Union[OptionalIntFloat, Decimal] NetworkType = Union[str, bytes, int, Tuple[Union[str, bytes, int], Union[str, int]]] - if TYPE_CHECKING: # pragma: no cover + from .fields import Field from .dataclasses import DataclassType # noqa: F401 - from .main import BaseModel # noqa: F401 + from .main import BaseModel, BaseConfig # noqa: F401 from .utils import AnyCallable CallableGenerator = Generator[AnyCallable, None, None] @@ -118,8 +116,8 @@ class ConstrainedBytes(bytes): def __get_validators__(cls) -> 'CallableGenerator': yield not_none_validator yield bytes_validator - yield anystr_strip_whitespace - yield anystr_length_validator + yield constr_strip_whitespace + yield constr_length_validator def conbytes(*, strip_whitespace: bool = False, min_length: int = None, max_length: int = None) -> Type[bytes]: @@ -139,8 +137,8 @@ class ConstrainedStr(str): def __get_validators__(cls) -> 'CallableGenerator': yield not_none_validator yield str_validator - yield anystr_strip_whitespace - yield anystr_length_validator + yield constr_strip_whitespace + yield constr_length_validator yield cls.validate @classmethod @@ -201,8 +199,8 @@ class UrlStr(str): def __get_validators__(cls) -> 'CallableGenerator': yield not_none_validator yield str_validator - yield anystr_strip_whitespace - yield anystr_length_validator + yield constr_strip_whitespace + yield constr_length_validator yield cls.validate @classmethod @@ -517,7 +515,7 @@ class Json(metaclass=JsonMeta): yield cls.validate @classmethod - def validate(cls, v: str) -> Any: + def validate(cls, v: Any) -> Any: try: return json.loads(v) except ValueError: @@ -593,7 +591,7 @@ class SecretStr: return "SecretStr('**********')" if self._secret_value else "SecretStr('')" def __str__(self) -> str: - return repr(self) + return self.__repr__() def display(self) -> str: return '**********' if self._secret_value else '' @@ -619,10 +617,32 @@ class SecretBytes: return "SecretBytes(b'**********')" if self._secret_value else "SecretBytes(b'')" def __str__(self) -> str: - return repr(self) + return self.__repr__() def display(self) -> str: return '**********' if self._secret_value else '' def get_secret_value(self) -> bytes: return self._secret_value + + +def constr_length_validator(v: 'StrBytes', field: 'Field', config: 'BaseConfig') -> 'StrBytes': + v_len = len(v) + + min_length = field.type_.min_length or config.min_anystr_length # type: ignore + if min_length is not None and v_len < min_length: + raise errors.AnyStrMinLengthError(limit_value=min_length) + + max_length = field.type_.max_length or config.max_anystr_length # type: ignore + if max_length is not None and v_len > max_length: + raise errors.AnyStrMaxLengthError(limit_value=max_length) + + return v + + +def constr_strip_whitespace(v: 'StrBytes', field: 'Field', config: 'BaseConfig') -> 'StrBytes': + strip_whitespace = field.type_.strip_whitespace or config.anystr_strip_whitespace # type: ignore + if strip_whitespace: + v = v.strip() + + return v diff --git a/pydantic/utils.py b/pydantic/utils.py index 21c6644..4cf5380 100644 --- a/pydantic/utils.py +++ b/pydantic/utils.py @@ -9,7 +9,7 @@ from textwrap import dedent from typing import _eval_type # type: ignore from typing import TYPE_CHECKING, Any, ClassVar, Dict, Generator, List, Optional, Pattern, Tuple, Type, Union -from . import errors +import pydantic try: import email_validator @@ -30,6 +30,7 @@ except ImportError: if TYPE_CHECKING: # pragma: no cover from .main import BaseModel # noqa: F401 from .main import Field # noqa: F401 + from . import errors # noqa: F401 if sys.version_info < (3, 7): from typing import Callable @@ -70,7 +71,7 @@ def validate_email(value: str) -> Tuple[str, str]: try: email_validator.validate_email(email, check_deliverability=False) except email_validator.EmailNotValidError as e: - raise errors.EmailError() from e + raise pydantic.errors.EmailError() from e return name or email[: email.index('@')], email.lower() @@ -134,14 +135,14 @@ def import_string(dotted_path: str) -> Any: raise ImportError(f'Module "{module_path}" does not define a "{class_name}" attribute') from e -def truncate(v: str, *, max_len: int = 80) -> str: +def truncate(v: Union[str], *, max_len: int = 80) -> str: """ Truncate a value and add a unicode ellipsis (three dots) to the end if it was too long """ if isinstance(v, str) and len(v) > (max_len - 2): # -3 so quote + string + … + quote has correct length - return repr(v[: (max_len - 3)] + '…') - v = repr(v) + return (v[: (max_len - 3)] + '…').__repr__() + v = v.__repr__() if len(v) > max_len: v = v[: max_len - 1] + '…' return v @@ -237,7 +238,7 @@ def in_ipython() -> bool: Check whether we're in an ipython environment, including jupyter notebooks. """ try: - __IPYTHON__ # type: ignore + eval('__IPYTHON__') except NameError: return False else: # pragma: no cover diff --git a/pydantic/validators.py b/pydantic/validators.py index d3ef5b9..62c5444 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -5,7 +5,21 @@ from decimal import Decimal, DecimalException from enum import Enum, IntEnum from ipaddress import IPv4Address, IPv4Interface, IPv4Network, IPv6Address, IPv6Interface, IPv6Network from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Pattern, Set, Tuple, Type, TypeVar, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + List, + Optional, + Pattern, + Set, + Tuple, + Type, + TypeVar, + Union, +) from uuid import UUID from . import errors @@ -36,14 +50,19 @@ def is_none_validator(v: Any) -> None: raise errors.NoneIsAllowedError() -def str_validator(v: Any) -> str: - if isinstance(v, (str, NoneType)): # type: ignore - return v - elif isinstance(v, (bytes, bytearray)): - return v.decode() +def str_validator(v: Any) -> Optional[str]: + if isinstance(v, str): + if isinstance(v, Enum): + return v.value + else: + return v + elif v is None: + return None elif isinstance(v, (float, int, Decimal)): # is there anything else we want to add here? If you think so, create an issue. return str(v) + elif isinstance(v, (bytes, bytearray)): + return v.decode() else: raise errors.StrError() @@ -75,7 +94,7 @@ def bool_validator(v: Any) -> bool: def int_validator(v: Any) -> int: - if not isinstance(v, bool) and isinstance(v, int): + if isinstance(v, int) and not isinstance(v, bool): return v with change_exception(errors.IntegerError, TypeError, ValueError): @@ -91,7 +110,7 @@ def float_validator(v: Any) -> float: def number_multiple_validator(v: 'Number', field: 'Field') -> 'Number': - field_type = cast('ConstrainedNumber', field.type_) + field_type: ConstrainedNumber = field.type_ # type: ignore if field_type.multiple_of is not None and v % field_type.multiple_of != 0: # type: ignore raise errors.NumberNotMultipleError(multiple_of=field_type.multiple_of) @@ -99,7 +118,7 @@ def number_multiple_validator(v: 'Number', field: 'Field') -> 'Number': def number_size_validator(v: 'Number', field: 'Field') -> 'Number': - field_type = cast('ConstrainedNumber', field.type_) + field_type: ConstrainedNumber = field.type_ # type: ignore if field_type.gt is not None and not v > field_type.gt: raise errors.NumberNotGtError(limit_value=field_type.gt) elif field_type.ge is not None and not v >= field_type.ge: @@ -129,11 +148,11 @@ def constant_validator(v: 'Any', field: 'Field') -> 'Any': def anystr_length_validator(v: 'StrBytes', field: 'Field', config: 'BaseConfig') -> 'StrBytes': v_len = len(v) - min_length = getattr(field.type_, 'min_length', config.min_anystr_length) + min_length = config.min_anystr_length if min_length is not None and v_len < min_length: raise errors.AnyStrMinLengthError(limit_value=min_length) - max_length = getattr(field.type_, 'max_length', config.max_anystr_length) + max_length = config.max_anystr_length if max_length is not None and v_len > max_length: raise errors.AnyStrMaxLengthError(limit_value=max_length) @@ -141,11 +160,7 @@ def anystr_length_validator(v: 'StrBytes', field: 'Field', config: 'BaseConfig') def anystr_strip_whitespace(v: 'StrBytes', field: 'Field', config: 'BaseConfig') -> 'StrBytes': - strip_whitespace = getattr(field.type_, 'strip_whitespace', config.anystr_strip_whitespace) - if strip_whitespace: - v = v.strip() - - return v + return v.strip() def ordered_dict_validator(v: Any) -> 'AnyOrderedDict': @@ -336,14 +351,39 @@ def pattern_validator(v: Any) -> Pattern[str]: return re.compile(v) +class IfConfig: + def __init__(self, validator: AnyCallable, *config_attr_names: str) -> None: + self.validator = validator + self.config_attr_names = config_attr_names + + def check(self, config: Type['BaseConfig']) -> bool: + return any(getattr(config, name) not in {None, False} for name in self.config_attr_names) + + pattern_validators = [not_none_validator, str_validator, pattern_validator] # order is important here, for example: bool is a subclass of int so has to come first, datetime before date same, # IPv4Interface before IPv4Address, etc -_VALIDATORS: List[Tuple[AnyType, List[AnyCallable]]] = [ +_VALIDATORS: List[Tuple[AnyType, List[Any]]] = [ (IntEnum, [int_validator, enum_validator]), (Enum, [enum_validator]), - (str, [not_none_validator, str_validator, anystr_strip_whitespace, anystr_length_validator]), - (bytes, [not_none_validator, bytes_validator, anystr_strip_whitespace, anystr_length_validator]), + ( + str, + [ + not_none_validator, + str_validator, + IfConfig(anystr_strip_whitespace, 'anystr_strip_whitespace'), + IfConfig(anystr_length_validator, 'min_anystr_length', 'max_anystr_length'), + ], + ), + ( + bytes, + [ + not_none_validator, + bytes_validator, + IfConfig(anystr_strip_whitespace, 'anystr_strip_whitespace'), + IfConfig(anystr_length_validator, 'min_anystr_length', 'max_anystr_length'), + ], + ), (bool, [bool_validator]), (int, [int_validator]), (float, [float_validator]), @@ -369,13 +409,18 @@ _VALIDATORS: List[Tuple[AnyType, List[AnyCallable]]] = [ ] -def find_validators(type_: AnyType, arbitrary_types_allowed: bool = False) -> List[AnyCallable]: - if type_ is Any or type(type_) in (ForwardRef, TypeVar): - return [] +def find_validators(type_: AnyType, config: Type['BaseConfig']) -> Generator[AnyCallable, None, None]: + if type_ is Any: + return + type_type = type(type_) + if type_type == ForwardRef or type_type == TypeVar: + return if type_ is Pattern: - return pattern_validators + yield from pattern_validators + return if is_callable_type(type_): - return [callable_validator] + yield callable_validator + return supertype = _find_supertype(type_) if supertype is not None: @@ -384,12 +429,18 @@ def find_validators(type_: AnyType, arbitrary_types_allowed: bool = False) -> Li for val_type, validators in _VALIDATORS: try: if issubclass(type_, val_type): - return validators + for v in validators: + if isinstance(v, IfConfig): + if v.check(config): + yield v.validator + else: + yield v + return except TypeError as e: raise RuntimeError(f'error checking inheritance of {type_!r} (type: {display_as_type(type_)})') from e - if arbitrary_types_allowed: - return [make_arbitrary_type_validator(type_)] + if config.arbitrary_types_allowed: + yield make_arbitrary_type_validator(type_) else: raise RuntimeError(f'no validator found for {type_}') diff --git a/pydantic/version.py b/pydantic/version.py index ec60f47..3136436 100644 --- a/pydantic/version.py +++ b/pydantic/version.py @@ -2,4 +2,4 @@ from distutils.version import StrictVersion __all__ = ['VERSION'] -VERSION = StrictVersion('0.26') +VERSION = StrictVersion('0.27a1') diff --git a/setup.cfg b/setup.cfg index 78e988b..eb5a62d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ filterwarnings = error [flake8] max-line-length = 120 -max-complexity = 12 +max-complexity = 14 [bdist_wheel] python-tag = py36.py37.py38 diff --git a/setup.py b/setup.py index e90eab1..85dc463 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,9 @@ +import os import re +import sys from importlib.machinery import SourceFileLoader from pathlib import Path + from setuptools import setup @@ -40,6 +43,24 @@ except FileNotFoundError: # avoid loading the package before requirements are installed: version = SourceFileLoader('version', 'pydantic/version.py').load_module() +ext_modules = None +if 'clean' not in sys.argv and 'SKIP_CYTHON' not in os.environ: + try: + from Cython.Build import cythonize + except ImportError: + pass + else: + # For cython test coverage install with `make install-trace` + compiler_directives = {} + if 'CYTHON_TRACE' in sys.argv: + compiler_directives['linetrace'] = True + ext_modules = cythonize( + 'pydantic/*.py', + nthreads=4, + language_level=3, + compiler_directives=compiler_directives, + ) + setup( name='pydantic', version=str(version.VERSION), @@ -78,5 +99,6 @@ setup( extras_require={ 'ujson': ['ujson>=1.35'], 'email': ['email-validator>=1.0.3'], - } + }, + ext_modules=ext_modules, ) diff --git a/tests/check_tag.py b/tests/check_tag.py index 8b07de1..d44204a 100755 --- a/tests/check_tag.py +++ b/tests/check_tag.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 import os import sys +from importlib.machinery import SourceFileLoader -from pydantic.version import VERSION +VERSION = SourceFileLoader('version', 'pydantic/version.py').load_module().VERSION git_tag = os.getenv('TRAVIS_TAG') if git_tag: diff --git a/tests/requirements.txt b/tests/requirements.txt index 2c31cbd..abc3724 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,7 @@ attrs==19.1.0 black==19.3b0 coverage==4.5.3 +Cython==0.29.9;sys_platform!='win32' flake8==3.7.7 isort==4.3.17 mypy==0.701 diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index d11ae25..8e88d9d 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -473,6 +473,11 @@ def test_invalid_type(): assert "error checking inheritance of 43 (type: int)" in str(exc_info) +class CustomStr(str): + def foobar(self): + return 7 + + @pytest.mark.parametrize( 'value,expected', [ @@ -485,6 +490,7 @@ def test_invalid_type(): (True, 'True'), (False, 'False'), (StrEnum.a, 'a10'), + (CustomStr('whatever'), 'whatever'), ], ) def test_valid_string_types(value, expected): diff --git a/tests/test_error_wrappers.py b/tests/test_error_wrappers.py index f019cf4..9092d03 100644 --- a/tests/test_error_wrappers.py +++ b/tests/test_error_wrappers.py @@ -212,10 +212,10 @@ def test_errors_unknown_error_object(): ) def test_get_exc_type(exc, type_): if isinstance(type_, str): - assert get_exc_type(exc) == type_ + assert get_exc_type(type(exc)) == type_ else: with pytest.raises(type_) as exc_info: - get_exc_type(exc) + get_exc_type(type(exc)) assert isinstance(exc_info.value, type_)