From 044fd42f4fda6b06bedbe5f42fc8324020706e27 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Sat, 8 Jul 2017 18:39:24 +0100 Subject: [PATCH] parser methods (#58), fix #39 * working on parsers * starting parse tests * proper tests for parse * adding parse docs * tweaks and history * add test for datetime direct, fix tests * tweak docs --- .travis.yml | 8 ++- HISTORY.rst | 1 + docs/examples/config.py | 2 +- docs/examples/errors.py | 2 +- docs/examples/parse.py | 43 ++++++++++++ docs/index.rst | 40 ++++++++++- docs/spelling_wordlist.txt | 1 + pydantic/__init__.py | 1 + pydantic/datetime_parse.py | 3 + pydantic/exceptions.py | 12 +++- pydantic/main.py | 36 +++++++++- pydantic/parse.py | 73 ++++++++++++++++++++ requirements.txt | 3 + setup.py | 4 ++ tests/requirements.txt | 2 + tests/test_complex.py | 6 ++ tests/test_datetime_parse.py | 1 + tests/test_main.py | 3 +- tests/test_parse.py | 126 +++++++++++++++++++++++++++++++++++ 19 files changed, 359 insertions(+), 8 deletions(-) create mode 100644 docs/examples/parse.py create mode 100644 pydantic/parse.py create mode 100644 tests/test_parse.py diff --git a/.travis.yml b/.travis.yml index ab9021d..ce91618 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,13 @@ install: script: - make lint -- make test + +# test with and without msgpack and ujson then combine coverage +- make test && mv .coverage .coverage.extra +- pip uninstall -y msgpack-python ujson +- make test && mv .coverage .coverage.no-extra +- coverage combine + - make mypy - make docs-lint - make docs diff --git a/HISTORY.rst b/HISTORY.rst index 89559f6..242983f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ v0.4.0 (2017-XX-XX) * fix aliases in config during inheritance #55 * simplify error display * use unicode ellipsis in ``truncate`` +* add ``parse_obj``, ``parse_raw`` and ``parse_file`` helper functions #58 v0.3.0 (2017-06-21) ................... diff --git a/docs/examples/config.py b/docs/examples/config.py index d9d7e91..8123656 100644 --- a/docs/examples/config.py +++ b/docs/examples/config.py @@ -13,7 +13,7 @@ try: except ValidationError as e: print(e) """ -1 error validating input +error validating input v: length not in range 0 to 10 (error_type=ValueError track=str) """ diff --git a/docs/examples/errors.py b/docs/examples/errors.py index 83943d8..d1cff02 100644 --- a/docs/examples/errors.py +++ b/docs/examples/errors.py @@ -27,7 +27,7 @@ a_float: is_required: field required (error_type=Missing) recursive_model: - 1 error validating input (error_type=ValidationError track=Location) + error validating input (error_type=ValidationError track=Location) lng: could not convert string to float: 'New York' (error_type=ValueError track=float """ diff --git a/docs/examples/parse.py b/docs/examples/parse.py new file mode 100644 index 0000000..43a07f0 --- /dev/null +++ b/docs/examples/parse.py @@ -0,0 +1,43 @@ +import pickle +from pathlib import Path +from datetime import datetime +import msgpack +from pydantic import BaseModel, ValidationError + +class User(BaseModel): + id: int + name = 'John Doe' + signup_ts: datetime = None + +m = User.parse_obj({'id': 123, 'name': 'James'}) +print(m) +# > User id=123 name='James' signup_ts=None + +try: + User.parse_obj(['not', 'a', 'dict']) +except ValidationError as e: + print(e) +# > error validating input +# > User expected dict not list (error_type=TypeError) + +m = User.parse_raw('{"id": 123, "name": "James"}') # assumes json as no content type passed +print(m) +# > User id=123 name='James' signup_ts=None + +msgpack_data = msgpack.packb({'id': 123, 'name': 'James', 'signup_ts': 1500000000}) +m = User.parse_raw(msgpack_data, content_type='application/msgpack') +print(m) +# > User id=123 name='James' signup_ts=datetime.datetime(2017, 7, 14, 2, 40, tzinfo=datetime.timezone.utc) + + +pickle_data = pickle.dumps({'id': 123, 'name': 'James', 'signup_ts': datetime(2017, 7, 14)}) +m = User.parse_raw(pickle_data, content_type='application/pickle', allow_pickle=True) +print(m) +# > User id=123 name='James' signup_ts=datetime.datetime(2017, 7, 14, 0, 0) + + +Path('/tmp/data.mp').write_bytes(msgpack_data) +# data.json: {"id": 123, "name": "James"} +m = User.parse_file('/tmp/data.mp') +print(m) +# > User id=123 name='James' signup_ts=datetime.datetime(2017, 7, 14, 2, 40, tzinfo=datetime.timezone.utc) diff --git a/docs/index.rst b/docs/index.rst index 153a59d..a0ba1dc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -75,7 +75,17 @@ Just:: pip install pydantic -pydantic has no dependencies except python 3.6+. If you've got python 3.6 and ``pip`` installed - you're good to go. +pydantic has no required dependencies except python 3.6+. If you've got python 3.6 and ``pip`` installed - +you're good to go. + +If you want *pydantic* to parse msgpack you can add `msgpack-python `_ +as an optional dependency, same goes for reading json faster with `ujson `_:: + + pip install pydantic[msgpack] + # or + pip install pydantic[ujson] + # or just + pip install pydantic[msgpack,ujson] Usage ----- @@ -125,6 +135,28 @@ pydantic comes with a number of utilities for parsing or validating common objec (This script is complete, it should run "as is") +Helper Functions +................ + +*Pydantic* provides three ``classmethod`` helper functions on models for parsing data: + +:parse_obj: this is almost identical to the ``__init__`` method of the model except if the object passed is not + a dict ``ValidationError`` will be raised (rather than python raising a ``TypeError``). +:parse_raw: takes a *str* or *bytes* parses it as *json*, *msgpack* or *pickle* data and then passes + the result to ``parse_obj``. The data type is inferred from the ``content_type`` argument, + otherwise *json* is assumed. +:parse_file: reads a file and passes the contents to ``parse_raw``, if ``content_type`` is omitted it is inferred + from the file's extension. + +.. literalinclude:: examples/parse.py + +(This script is complete, it should run "as is" provided ``msgpack-python`` is installed) + +.. note:: + + Since ``pickle`` allows complex objects to be encoded, to use it you need to explicitly pass ``allow_pickle`` to + the parsing function. + Model Config ............ @@ -144,6 +176,8 @@ Options: .. literalinclude:: examples/config.py +(This script is complete, it should run "as is") + .. _settings: Settings @@ -156,6 +190,8 @@ This usage example comes last as it uses numerous concepts described above. .. literalinclude:: examples/settings.py +(This script is complete, it should run "as is") + Here ``redis_port`` could be modified via ``export MY_PREFIX_REDIS_PORT=6380`` or ``auth_key`` by ``export my_api_key=6380``. @@ -169,6 +205,8 @@ required variables: .. literalinclude:: examples/mypy.py +(This script is complete, it should run "as is") + This script is complete, it should run "as is". You can also run it through mypy with:: mypy --ignore-missing-imports --follow-imports=skip --strict-optional pydantic_mypy_test.py diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 0fb0cf8..3106451 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -12,6 +12,7 @@ jsonmodels pydantic pypi metadata +msgpack schemas timestamp travis diff --git a/pydantic/__init__.py b/pydantic/__init__.py index cfe551d..a6bc7ea 100644 --- a/pydantic/__init__.py +++ b/pydantic/__init__.py @@ -3,5 +3,6 @@ from .env_settings import BaseSettings from .exceptions import * from .fields import Required from .main import BaseModel +from .parse import Protocol from .types import * from .version import VERSION diff --git a/pydantic/datetime_parse.py b/pydantic/datetime_parse.py index a6df292..f5b7f76 100644 --- a/pydantic/datetime_parse.py +++ b/pydantic/datetime_parse.py @@ -129,6 +129,9 @@ def parse_datetime(value: StrIntFloat) -> datetime: Raise ValueError if the input is well formatted but not a valid datetime. Raise ValueError if the input isn't well formatted. """ + if isinstance(value, datetime): + return value + number = get_numeric(value) if number: return from_unix_seconds(number) diff --git a/pydantic/exceptions.py b/pydantic/exceptions.py index 659d0ee..b1f06ec 100644 --- a/pydantic/exceptions.py +++ b/pydantic/exceptions.py @@ -2,6 +2,14 @@ import json from collections import OrderedDict, namedtuple from itertools import chain +__all__ = ( + 'Error', + 'ValidationError', + 'ConfigError', + 'Missing', + 'Extra', +) + def type_display(type_: type): if type_: @@ -32,8 +40,10 @@ def pretty_errors(e): return d elif isinstance(e, dict): return OrderedDict([(k, pretty_errors(v)) for k, v in e.items()]) - else: + elif isinstance(e, (list, tuple)): return [pretty_errors(e_) for e_ in e] + else: + raise TypeError(f'Unknown error object: {e}') E_KEYS = 'error_type', 'track', 'index' diff --git a/pydantic/main.py b/pydantic/main.py index 0027313..c3700c1 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -1,9 +1,12 @@ from collections import OrderedDict +from pathlib import Path from types import FunctionType -from typing import Any, Dict, Set +from typing import Any, Dict, Set, Union from .exceptions import Error, Extra, Missing, ValidationError from .fields import Field +from .parse import Protocol, load_file, load_str_bytes +from .types import StrBytes from .utils import truncate from .validators import dict_validator @@ -128,6 +131,35 @@ class BaseModel(metaclass=MetaModel): if k not in exclude and (not include or k in include) } + @classmethod + def parse_obj(cls, obj): + if not isinstance(obj, dict): + exc = TypeError(f'{cls.__name__} expected dict not {type(obj).__name__}') + raise ValidationError([Error(exc, None, None)]) + return cls(**obj) + + @classmethod + def parse_raw(cls, b: StrBytes, *, + content_type: str=None, + encoding: str='utf8', + proto: Protocol=None, + allow_pickle: bool=False): + try: + obj = load_str_bytes(b, proto=proto, content_type=content_type, encoding=encoding, + allow_pickle=allow_pickle) + except (ValueError, TypeError, UnicodeDecodeError) as e: + raise ValidationError([Error(e, None, None)]) + return cls.parse_obj(obj) + + @classmethod + def parse_file(cls, path: Union[str, Path], *, + content_type: str=None, + encoding: str='utf8', + proto: Protocol=None, + allow_pickle: bool=False): + obj = load_file(path, proto=proto, content_type=content_type, encoding=encoding, allow_pickle=allow_pickle) + return cls.parse_obj(obj) + @classmethod def construct(cls, **values): """ @@ -172,7 +204,7 @@ class BaseModel(metaclass=MetaModel): def validate(cls, value): return cls(**value) - def _process_values(self, input_data: dict) -> OrderedDict: # noqa: C901 + def _process_values(self, input_data: dict) -> OrderedDict: # noqa: C901 (ignore complexity) values = OrderedDict() errors = OrderedDict() diff --git a/pydantic/parse.py b/pydantic/parse.py new file mode 100644 index 0000000..9f304dd --- /dev/null +++ b/pydantic/parse.py @@ -0,0 +1,73 @@ +import pickle +from enum import Enum +from pathlib import Path +from typing import Any, Union + +from .types import StrBytes + +try: + import ujson as json +except ImportError: + import json + +try: + import msgpack +except ImportError: + msgpack = None + + +class Protocol(str, Enum): + json = 'json' + msgpack = 'msgpack' + pickle = 'pickle' + + +def load_str_bytes(b: StrBytes, *, # noqa: C901 (ignore complexity) + content_type: str=None, + encoding: str='utf8', + proto: Protocol=None, + allow_pickle: bool=False) -> Any: + if proto is None and content_type: + if content_type.endswith(('json', 'javascript')): + pass + elif msgpack and content_type.endswith('msgpack'): + proto = Protocol.msgpack + elif allow_pickle and content_type.endswith('pickle'): + proto = Protocol.pickle + else: + raise TypeError(f'Unknown content-type: {content_type}') + + proto = proto or Protocol.json + + if proto == Protocol.json: + if isinstance(b, bytes): + b = b.decode(encoding) + return json.loads(b) + elif proto == Protocol.msgpack: + if msgpack is None: + raise ImportError("msgpack not installed, can't parse data") + return msgpack.unpackb(b, encoding=encoding) + elif proto == Protocol.pickle: + if not allow_pickle: + raise RuntimeError('Trying to decode with pickle with allow_pickle=False') + return pickle.loads(b) + else: + raise TypeError(f'Unknown protocol: {proto}') + + +def load_file(path: Union[str, Path], *, + content_type: str=None, + encoding: str='utf8', + proto: Protocol=None, + allow_pickle: bool=False) -> Any: + path = Path(path) + b = path.read_bytes() + if content_type is None: + if path.suffix in ('.js', '.json'): + proto = Protocol.json + elif path.suffix in ('.mp', '.msgpack'): + proto = Protocol.msgpack + elif path.suffix == '.pkl': + proto = Protocol.pickle + + return load_str_bytes(b, proto=proto, content_type=content_type, encoding=encoding, allow_pickle=allow_pickle) diff --git a/requirements.txt b/requirements.txt index e11b90d..1b3ad07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ -r benchmarks/requirements.txt -r docs/requirements.txt -r tests/requirements.txt + +msgpack-python==0.4.8 +ujson==1.35 diff --git a/setup.py b/setup.py index 42901d3..d5ddc65 100644 --- a/setup.py +++ b/setup.py @@ -37,4 +37,8 @@ setup( packages=['pydantic'], python_requires='>=3.6', zip_safe=True, + extras_require={ + 'msgpack': ['msgpack-python>=0.4.8'], + 'ujson': ['ujson>=1.35'], + } ) diff --git a/tests/requirements.txt b/tests/requirements.txt index 44a0f1b..7f93105 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,9 +1,11 @@ coverage==4.4.1 flake8==3.3.0 +isort==4.2.15 mypy==0.511 pycodestyle==2.3.1 pyflakes==1.5.0 pytest==3.1.3 pytest-cov==2.5.1 pytest-isort==0.1.0 +pytest-mock==1.6.0 pytest-sugar==0.8.0 diff --git a/tests/test_complex.py b/tests/test_complex.py index 9201905..70524bf 100644 --- a/tests/test_complex.py +++ b/tests/test_complex.py @@ -5,6 +5,7 @@ from typing import Any, Dict, List, Set, Union import pytest from pydantic import BaseModel, NoneStrBytes, StrBytes, ValidationError, constr +from pydantic.exceptions import pretty_errors def test_str_bytes(): @@ -530,3 +531,8 @@ def test_string_none(): with pytest.raises(ValidationError) as exc_info: Model(a=None) assert 'None is not an allow value' in str(exc_info.value) + + +def test_pretty_error_no_recursion(): + with pytest.raises(TypeError): + pretty_errors('foobar') diff --git a/tests/test_datetime_parse.py b/tests/test_datetime_parse.py index 54e1636..3bf69aa 100644 --- a/tests/test_datetime_parse.py +++ b/tests/test_datetime_parse.py @@ -73,6 +73,7 @@ def test_time_parsing(value, result): ('2012-04-23T10:20:30.400+02:30', datetime(2012, 4, 23, 10, 20, 30, 400000, create_tz(150))), ('2012-04-23T10:20:30.400+02', datetime(2012, 4, 23, 10, 20, 30, 400000, create_tz(120))), ('2012-04-23T10:20:30.400-02', datetime(2012, 4, 23, 10, 20, 30, 400000, create_tz(-120))), + (datetime(2017, 5, 5), datetime(2017, 5, 5)), # Invalid inputs ('x20120423091500', ValueError), ('2012-04-56T09:15:90', ValueError), diff --git a/tests/test_main.py b/tests/test_main.py index 128bb3f..baa96be 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,7 +3,8 @@ from typing import Any import pytest -from pydantic import BaseModel, ConfigError, NoneBytes, NoneStr, Required, ValidationError, pretty_errors +from pydantic import BaseModel, ConfigError, NoneBytes, NoneStr, Required, ValidationError +from pydantic.exceptions import pretty_errors def test_success(): diff --git a/tests/test_parse.py b/tests/test_parse.py new file mode 100644 index 0000000..3fc9153 --- /dev/null +++ b/tests/test_parse.py @@ -0,0 +1,126 @@ +import pickle + +import pytest + +from pydantic import BaseModel, Protocol, ValidationError + +try: + import msgpack +except ImportError: + msgpack = None + + +class Model(BaseModel): + a: float = ... + b: int = 10 + + +def test_obj(): + m = Model.parse_obj(dict(a=10.2)) + assert str(m) == 'Model a=10.2 b=10' + + +def test_fails(): + with pytest.raises(ValidationError) as exc_info: + Model.parse_obj([1, 2, 3]) + assert """\ +error validating input +Model expected dict not list (error_type=TypeError)""" == str(exc_info.value) + + +def test_json(): + assert Model.parse_raw('{"a": 12, "b": 8}') == Model.construct(a=12, b=8) + + +def test_json_ct(): + assert Model.parse_raw('{"a": 12, "b": 8}', content_type='application/json') == Model.construct(a=12, b=8) + + +@pytest.mark.skipif(not msgpack, reason='msgpack not installed') +def test_msgpack_proto(mocker): + # b'\x82\xa1a\x0c\xa1b\x08' == msgpack.packb(dict(a=12, b=8)) + assert Model.parse_raw(b'\x82\xa1a\x0c\xa1b\x08', proto=Protocol.msgpack) == Model.construct(a=12, b=8) + + +@pytest.mark.skipif(not msgpack, reason='msgpack not installed') +def test_msgpack_ct(): + assert Model.parse_raw(b'\x82\xa1a\x0c\xa1b\x08', content_type='application/msgpack') == Model.construct(a=12, b=8) + + +@pytest.mark.skipif(msgpack, reason='msgpack installed') +def test_msgpack_not_installed_proto(mocker): + with pytest.raises(ImportError) as exc_info: + Model.parse_raw(b'\x82\xa1a\x0c\xa1b\x08', proto=Protocol.msgpack) + assert "ImportError: msgpack not installed, can't parse data" in str(exc_info) + + +@pytest.mark.skipif(msgpack, reason='msgpack installed') +def test_msgpack_not_installed_ct(): + with pytest.raises(ValidationError) as exc_info: + Model.parse_raw(b'\x82\xa1a\x0c\xa1b\x08', content_type='application/msgpack') + assert """\ +error validating input +Unknown content-type: application/msgpack (error_type=TypeError)""" == str(exc_info.value) + + +def test_pickle_ct(): + data = pickle.dumps(dict(a=12, b=8)) + assert Model.parse_raw(data, content_type='application/pickle', allow_pickle=True) == Model.construct(a=12, b=8) + + +def test_pickle_proto(): + data = pickle.dumps(dict(a=12, b=8)) + assert Model.parse_raw(data, proto=Protocol.pickle, allow_pickle=True) == Model.construct(a=12, b=8) + + +def test_pickle_not_allowed(): + data = pickle.dumps(dict(a=12, b=8)) + with pytest.raises(RuntimeError): + Model.parse_raw(data, proto=Protocol.pickle) + + +def test_bad_ct(): + with pytest.raises(ValidationError) as exc_info: + Model.parse_raw('{"a": 12, "b": 8}', content_type='application/missing') + assert """\ +error validating input +Unknown content-type: application/missing (error_type=TypeError)""" == str(exc_info.value) + + +def test_bad_proto(): + with pytest.raises(ValidationError) as exc_info: + Model.parse_raw('{"a": 12, "b": 8}', proto='foobar') + assert """\ +error validating input +Unknown protocol: foobar (error_type=TypeError)""" == str(exc_info.value) + + +def test_file_json(tmpdir): + p = tmpdir.join('test.json') + p.write('{"a": 12, "b": 8}') + assert Model.parse_file(str(p)) == Model.construct(a=12, b=8) + + +def test_file_json_no_ext(tmpdir): + p = tmpdir.join('test') + p.write('{"a": 12, "b": 8}') + assert Model.parse_file(str(p)) == Model.construct(a=12, b=8) + + +@pytest.mark.skipif(not msgpack, reason='msgpack not installed') +def test_file_msgpack(tmpdir): + p = tmpdir.join('test.mp') + p.write_binary(b'\x82\xa1a\x0c\xa1b\x08') + assert Model.parse_file(str(p)) == Model.construct(a=12, b=8) + + +def test_file_pickle(tmpdir): + p = tmpdir.join('test.pkl') + p.write_binary(pickle.dumps(dict(a=12, b=8))) + assert Model.parse_file(str(p), allow_pickle=True) == Model.construct(a=12, b=8) + + +def test_file_pickle_no_ext(tmpdir): + p = tmpdir.join('test') + p.write_binary(pickle.dumps(dict(a=12, b=8))) + assert Model.parse_file(str(p), content_type='application/pickle', allow_pickle=True) == Model.construct(a=12, b=8)