parser methods (#58), fix #39

* working on parsers

* starting parse tests

* proper tests for parse

* adding parse docs

* tweaks and history

* add test for datetime direct, fix tests

* tweak docs

.travis.yml

@@ -22,7 +22,13 @@ install:
 
 script:
 - make lint
-- make test
+
+# test with and without msgpack and ujson then combine coverage
+- make test && mv .coverage .coverage.extra
+- pip uninstall -y msgpack-python ujson
+- make test && mv .coverage .coverage.no-extra
+- coverage combine
+
 - make mypy
 - make docs-lint
 - make docs

HISTORY.rst

@@ -9,6 +9,7 @@ v0.4.0 (2017-XX-XX)
 * fix aliases in config during inheritance #55
 * simplify error display
 * use unicode ellipsis in ``truncate``
+* add ``parse_obj``, ``parse_raw`` and ``parse_file`` helper functions #58
 
 v0.3.0 (2017-06-21)
 ...................

docs/examples/config.py

@@ -13,7 +13,7 @@ try:
 except ValidationError as e:
     print(e)
 """
-1 error validating input
+error validating input
 v:
   length not in range 0 to 10 (error_type=ValueError track=str)
 """

docs/examples/errors.py

@@ -27,7 +27,7 @@ a_float:
 is_required:
   field required (error_type=Missing)
 recursive_model:
-  1 error validating input (error_type=ValidationError track=Location)
+  error validating input (error_type=ValidationError track=Location)
     lng:
       could not convert string to float: 'New York' (error_type=ValueError track=float
 """

docs/examples/parse.py

@@ -0,0 +1,43 @@
+import pickle
+from pathlib import Path
+from datetime import datetime
+import msgpack
+from pydantic import BaseModel, ValidationError
+
+class User(BaseModel):
+    id: int
+    name = 'John Doe'
+    signup_ts: datetime = None
+
+m = User.parse_obj({'id': 123, 'name': 'James'})
+print(m)
+# > User id=123 name='James' signup_ts=None
+
+try:
+    User.parse_obj(['not', 'a', 'dict'])
+except ValidationError as e:
+    print(e)
+# > error validating input
+# > User expected dict not list (error_type=TypeError)
+
+m = User.parse_raw('{"id": 123, "name": "James"}')  # assumes json as no content type passed
+print(m)
+# > User id=123 name='James' signup_ts=None
+
+msgpack_data = msgpack.packb({'id': 123, 'name': 'James', 'signup_ts': 1500000000})
+m = User.parse_raw(msgpack_data, content_type='application/msgpack')
+print(m)
+# > User id=123 name='James' signup_ts=datetime.datetime(2017, 7, 14, 2, 40, tzinfo=datetime.timezone.utc)
+
+
+pickle_data = pickle.dumps({'id': 123, 'name': 'James', 'signup_ts': datetime(2017, 7, 14)})
+m = User.parse_raw(pickle_data, content_type='application/pickle', allow_pickle=True)
+print(m)
+# > User id=123 name='James' signup_ts=datetime.datetime(2017, 7, 14, 0, 0)
+
+
+Path('/tmp/data.mp').write_bytes(msgpack_data)
+# data.json: {"id": 123, "name": "James"}
+m = User.parse_file('/tmp/data.mp')
+print(m)
+# > User id=123 name='James' signup_ts=datetime.datetime(2017, 7, 14, 2, 40, tzinfo=datetime.timezone.utc)

docs/index.rst

@@ -75,7 +75,17 @@ Just::
 
     pip install pydantic
 
-pydantic has no dependencies except python 3.6+. If you've got python 3.6 and ``pip`` installed - you're good to go.
+pydantic has no required dependencies except python 3.6+. If you've got python 3.6 and ``pip`` installed -
+you're good to go.
+
+If you want *pydantic* to parse msgpack you can add `msgpack-python <https://pypi.python.org/pypi/msgpack-python>`_
+as an optional dependency, same goes for reading json faster with `ujson <https://pypi.python.org/pypi/ujson>`_::
+
+    pip install pydantic[msgpack]
+    # or
+    pip install pydantic[ujson]
+    # or just
+    pip install pydantic[msgpack,ujson]
 
 Usage
 -----
@@ -125,6 +135,28 @@ pydantic comes with a number of utilities for parsing or validating common objec
 
 (This script is complete, it should run "as is")
 
+Helper Functions
+................
+
+*Pydantic* provides three ``classmethod`` helper functions on models for parsing data:
+
+:parse_obj: this is almost identical to the ``__init__`` method of the model except if the object passed is not
+  a dict ``ValidationError`` will be raised (rather than python raising a ``TypeError``).
+:parse_raw: takes a *str* or *bytes* parses it as *json*, *msgpack* or *pickle* data and then passes
+  the result to ``parse_obj``. The data type is inferred from the ``content_type`` argument,
+  otherwise *json* is assumed.
+:parse_file: reads a file and passes the contents to ``parse_raw``, if ``content_type`` is omitted it is inferred
+  from the file's extension.
+
+.. literalinclude:: examples/parse.py
+
+(This script is complete, it should run "as is" provided ``msgpack-python`` is installed)
+
+.. note::
+
+   Since ``pickle`` allows complex objects to be encoded, to use it you need to explicitly pass ``allow_pickle`` to
+   the parsing function.
+
 Model Config
 ............
 
@@ -144,6 +176,8 @@ Options:
 
 .. literalinclude:: examples/config.py
 
+(This script is complete, it should run "as is")
+
 .. _settings:
 
 Settings
@@ -156,6 +190,8 @@ This usage example comes last as it uses numerous concepts described above.
 
 .. literalinclude:: examples/settings.py
 
+(This script is complete, it should run "as is")
+
 Here ``redis_port`` could be modified via ``export MY_PREFIX_REDIS_PORT=6380`` or ``auth_key`` by
 ``export my_api_key=6380``.
 
@@ -169,6 +205,8 @@ required variables:
 
 .. literalinclude:: examples/mypy.py
 
+(This script is complete, it should run "as is")
+
 This script is complete, it should run "as is". You can also run it through mypy with::
 
     mypy --ignore-missing-imports --follow-imports=skip --strict-optional pydantic_mypy_test.py

docs/spelling_wordlist.txt

@@ -12,6 +12,7 @@ jsonmodels
 pydantic
 pypi
 metadata
+msgpack
 schemas
 timestamp
 travis

pydantic/__init__.py

@@ -3,5 +3,6 @@ from .env_settings import BaseSettings
 from .exceptions import *
 from .fields import Required
 from .main import BaseModel
+from .parse import Protocol
 from .types import *
 from .version import VERSION

pydantic/datetime_parse.py

@@ -129,6 +129,9 @@ def parse_datetime(value: StrIntFloat) -> datetime:
     Raise ValueError if the input is well formatted but not a valid datetime.
     Raise ValueError if the input isn't well formatted.
     """
+    if isinstance(value, datetime):
+        return value
+
     number = get_numeric(value)
     if number:
         return from_unix_seconds(number)

pydantic/exceptions.py

@@ -2,6 +2,14 @@ import json
 from collections import OrderedDict, namedtuple
 from itertools import chain
 
+__all__ = (
+    'Error',
+    'ValidationError',
+    'ConfigError',
+    'Missing',
+    'Extra',
+)
+
 
 def type_display(type_: type):
     if type_:
@@ -32,8 +40,10 @@ def pretty_errors(e):
         return d
     elif isinstance(e, dict):
         return OrderedDict([(k, pretty_errors(v)) for k, v in e.items()])
-    else:
+    elif isinstance(e, (list, tuple)):
         return [pretty_errors(e_) for e_ in e]
+    else:
+        raise TypeError(f'Unknown error object: {e}')
 
 
 E_KEYS = 'error_type', 'track', 'index'

pydantic/main.py

@@ -1,9 +1,12 @@
 from collections import OrderedDict
+from pathlib import Path
 from types import FunctionType
-from typing import Any, Dict, Set
+from typing import Any, Dict, Set, Union
 
 from .exceptions import Error, Extra, Missing, ValidationError
 from .fields import Field
+from .parse import Protocol, load_file, load_str_bytes
+from .types import StrBytes
 from .utils import truncate
 from .validators import dict_validator
 
@@ -128,6 +131,35 @@ class BaseModel(metaclass=MetaModel):
             if k not in exclude and (not include or k in include)
         }
 
+    @classmethod
+    def parse_obj(cls, obj):
+        if not isinstance(obj, dict):
+            exc = TypeError(f'{cls.__name__} expected dict not {type(obj).__name__}')
+            raise ValidationError([Error(exc, None, None)])
+        return cls(**obj)
+
+    @classmethod
+    def parse_raw(cls, b: StrBytes, *,
+                  content_type: str=None,
+                  encoding: str='utf8',
+                  proto: Protocol=None,
+                  allow_pickle: bool=False):
+        try:
+            obj = load_str_bytes(b, proto=proto, content_type=content_type, encoding=encoding,
+                                 allow_pickle=allow_pickle)
+        except (ValueError, TypeError, UnicodeDecodeError) as e:
+            raise ValidationError([Error(e, None, None)])
+        return cls.parse_obj(obj)
+
+    @classmethod
+    def parse_file(cls, path: Union[str, Path], *,
+                   content_type: str=None,
+                   encoding: str='utf8',
+                   proto: Protocol=None,
+                   allow_pickle: bool=False):
+        obj = load_file(path, proto=proto, content_type=content_type, encoding=encoding, allow_pickle=allow_pickle)
+        return cls.parse_obj(obj)
+
     @classmethod
     def construct(cls, **values):
         """
@@ -172,7 +204,7 @@ class BaseModel(metaclass=MetaModel):
     def validate(cls, value):
         return cls(**value)
 
-    def _process_values(self, input_data: dict) -> OrderedDict:  # noqa: C901
+    def _process_values(self, input_data: dict) -> OrderedDict:  # noqa: C901 (ignore complexity)
         values = OrderedDict()
         errors = OrderedDict()
 

pydantic/parse.py

@@ -0,0 +1,73 @@
+import pickle
+from enum import Enum
+from pathlib import Path
+from typing import Any, Union
+
+from .types import StrBytes
+
+try:
+    import ujson as json
+except ImportError:
+    import json
+
+try:
+    import msgpack
+except ImportError:
+    msgpack = None
+
+
+class Protocol(str, Enum):
+    json = 'json'
+    msgpack = 'msgpack'
+    pickle = 'pickle'
+
+
+def load_str_bytes(b: StrBytes, *,  # noqa: C901 (ignore complexity)
+                   content_type: str=None,
+                   encoding: str='utf8',
+                   proto: Protocol=None,
+                   allow_pickle: bool=False) -> Any:
+    if proto is None and content_type:
+        if content_type.endswith(('json', 'javascript')):
+            pass
+        elif msgpack and content_type.endswith('msgpack'):
+            proto = Protocol.msgpack
+        elif allow_pickle and content_type.endswith('pickle'):
+            proto = Protocol.pickle
+        else:
+            raise TypeError(f'Unknown content-type: {content_type}')
+
+    proto = proto or Protocol.json
+
+    if proto == Protocol.json:
+        if isinstance(b, bytes):
+            b = b.decode(encoding)
+        return json.loads(b)
+    elif proto == Protocol.msgpack:
+        if msgpack is None:
+            raise ImportError("msgpack not installed, can't parse data")
+        return msgpack.unpackb(b, encoding=encoding)
+    elif proto == Protocol.pickle:
+        if not allow_pickle:
+            raise RuntimeError('Trying to decode with pickle with allow_pickle=False')
+        return pickle.loads(b)
+    else:
+        raise TypeError(f'Unknown protocol: {proto}')
+
+
+def load_file(path: Union[str, Path], *,
+              content_type: str=None,
+              encoding: str='utf8',
+              proto: Protocol=None,
+              allow_pickle: bool=False) -> Any:
+    path = Path(path)
+    b = path.read_bytes()
+    if content_type is None:
+        if path.suffix in ('.js', '.json'):
+            proto = Protocol.json
+        elif path.suffix in ('.mp', '.msgpack'):
+            proto = Protocol.msgpack
+        elif path.suffix == '.pkl':
+            proto = Protocol.pickle
+
+    return load_str_bytes(b, proto=proto, content_type=content_type, encoding=encoding, allow_pickle=allow_pickle)

requirements.txt

@@ -1,3 +1,6 @@
 -r benchmarks/requirements.txt
 -r docs/requirements.txt
 -r tests/requirements.txt
+
+msgpack-python==0.4.8
+ujson==1.35

setup.py

@@ -37,4 +37,8 @@ setup(
     packages=['pydantic'],
     python_requires='>=3.6',
     zip_safe=True,
+    extras_require={
+        'msgpack': ['msgpack-python>=0.4.8'],
+        'ujson': ['ujson>=1.35'],
+    }
 )

tests/requirements.txt

@@ -1,9 +1,11 @@
 coverage==4.4.1
 flake8==3.3.0
+isort==4.2.15
 mypy==0.511
 pycodestyle==2.3.1
 pyflakes==1.5.0
 pytest==3.1.3
 pytest-cov==2.5.1
 pytest-isort==0.1.0
+pytest-mock==1.6.0
 pytest-sugar==0.8.0

tests/test_complex.py

@@ -5,6 +5,7 @@ from typing import Any, Dict, List, Set, Union
 import pytest
 
 from pydantic import BaseModel, NoneStrBytes, StrBytes, ValidationError, constr
+from pydantic.exceptions import pretty_errors
 
 
 def test_str_bytes():
@@ -530,3 +531,8 @@ def test_string_none():
     with pytest.raises(ValidationError) as exc_info:
         Model(a=None)
     assert 'None is not an allow value' in str(exc_info.value)
+
+
+def test_pretty_error_no_recursion():
+    with pytest.raises(TypeError):
+        pretty_errors('foobar')

tests/test_datetime_parse.py

@@ -73,6 +73,7 @@ def test_time_parsing(value, result):
     ('2012-04-23T10:20:30.400+02:30', datetime(2012, 4, 23, 10, 20, 30, 400000, create_tz(150))),
     ('2012-04-23T10:20:30.400+02', datetime(2012, 4, 23, 10, 20, 30, 400000, create_tz(120))),
     ('2012-04-23T10:20:30.400-02', datetime(2012, 4, 23, 10, 20, 30, 400000, create_tz(-120))),
+    (datetime(2017, 5, 5), datetime(2017, 5, 5)),
     # Invalid inputs
     ('x20120423091500', ValueError),
     ('2012-04-56T09:15:90', ValueError),

tests/test_main.py

@@ -3,7 +3,8 @@ from typing import Any
 
 import pytest
 
-from pydantic import BaseModel, ConfigError, NoneBytes, NoneStr, Required, ValidationError, pretty_errors
+from pydantic import BaseModel, ConfigError, NoneBytes, NoneStr, Required, ValidationError
+from pydantic.exceptions import pretty_errors
 
 
 def test_success():

tests/test_parse.py

@@ -0,0 +1,126 @@
+import pickle
+
+import pytest
+
+from pydantic import BaseModel, Protocol, ValidationError
+
+try:
+    import msgpack
+except ImportError:
+    msgpack = None
+
+
+class Model(BaseModel):
+    a: float = ...
+    b: int = 10
+
+
+def test_obj():
+    m = Model.parse_obj(dict(a=10.2))
+    assert str(m) == 'Model a=10.2 b=10'
+
+
+def test_fails():
+    with pytest.raises(ValidationError) as exc_info:
+        Model.parse_obj([1, 2, 3])
+    assert """\
+error validating input
+Model expected dict not list (error_type=TypeError)""" == str(exc_info.value)
+
+
+def test_json():
+    assert Model.parse_raw('{"a": 12, "b": 8}') == Model.construct(a=12, b=8)
+
+
+def test_json_ct():
+    assert Model.parse_raw('{"a": 12, "b": 8}', content_type='application/json') == Model.construct(a=12, b=8)
+
+
+@pytest.mark.skipif(not msgpack, reason='msgpack not installed')
+def test_msgpack_proto(mocker):
+    # b'\x82\xa1a\x0c\xa1b\x08' == msgpack.packb(dict(a=12, b=8))
+    assert Model.parse_raw(b'\x82\xa1a\x0c\xa1b\x08', proto=Protocol.msgpack) == Model.construct(a=12, b=8)
+
+
+@pytest.mark.skipif(not msgpack, reason='msgpack not installed')
+def test_msgpack_ct():
+    assert Model.parse_raw(b'\x82\xa1a\x0c\xa1b\x08', content_type='application/msgpack') == Model.construct(a=12, b=8)
+
+
+@pytest.mark.skipif(msgpack, reason='msgpack installed')
+def test_msgpack_not_installed_proto(mocker):
+    with pytest.raises(ImportError) as exc_info:
+        Model.parse_raw(b'\x82\xa1a\x0c\xa1b\x08', proto=Protocol.msgpack)
+    assert "ImportError: msgpack not installed, can't parse data" in str(exc_info)
+
+
+@pytest.mark.skipif(msgpack, reason='msgpack installed')
+def test_msgpack_not_installed_ct():
+    with pytest.raises(ValidationError) as exc_info:
+        Model.parse_raw(b'\x82\xa1a\x0c\xa1b\x08', content_type='application/msgpack')
+    assert """\
+error validating input
+Unknown content-type: application/msgpack (error_type=TypeError)""" == str(exc_info.value)
+
+
+def test_pickle_ct():
+    data = pickle.dumps(dict(a=12, b=8))
+    assert Model.parse_raw(data, content_type='application/pickle', allow_pickle=True) == Model.construct(a=12, b=8)
+
+
+def test_pickle_proto():
+    data = pickle.dumps(dict(a=12, b=8))
+    assert Model.parse_raw(data, proto=Protocol.pickle, allow_pickle=True) == Model.construct(a=12, b=8)
+
+
+def test_pickle_not_allowed():
+    data = pickle.dumps(dict(a=12, b=8))
+    with pytest.raises(RuntimeError):
+        Model.parse_raw(data, proto=Protocol.pickle)
+
+
+def test_bad_ct():
+    with pytest.raises(ValidationError) as exc_info:
+        Model.parse_raw('{"a": 12, "b": 8}', content_type='application/missing')
+    assert """\
+error validating input
+Unknown content-type: application/missing (error_type=TypeError)""" == str(exc_info.value)
+
+
+def test_bad_proto():
+    with pytest.raises(ValidationError) as exc_info:
+        Model.parse_raw('{"a": 12, "b": 8}', proto='foobar')
+    assert """\
+error validating input
+Unknown protocol: foobar (error_type=TypeError)""" == str(exc_info.value)
+
+
+def test_file_json(tmpdir):
+    p = tmpdir.join('test.json')
+    p.write('{"a": 12, "b": 8}')
+    assert Model.parse_file(str(p)) == Model.construct(a=12, b=8)
+
+
+def test_file_json_no_ext(tmpdir):
+    p = tmpdir.join('test')
+    p.write('{"a": 12, "b": 8}')
+    assert Model.parse_file(str(p)) == Model.construct(a=12, b=8)
+
+
+@pytest.mark.skipif(not msgpack, reason='msgpack not installed')
+def test_file_msgpack(tmpdir):
+    p = tmpdir.join('test.mp')
+    p.write_binary(b'\x82\xa1a\x0c\xa1b\x08')
+    assert Model.parse_file(str(p)) == Model.construct(a=12, b=8)
+
+
+def test_file_pickle(tmpdir):
+    p = tmpdir.join('test.pkl')
+    p.write_binary(pickle.dumps(dict(a=12, b=8)))
+    assert Model.parse_file(str(p), allow_pickle=True) == Model.construct(a=12, b=8)
+
+
+def test_file_pickle_no_ext(tmpdir):
+    p = tmpdir.join('test')
+    p.write_binary(pickle.dumps(dict(a=12, b=8)))
+    assert Model.parse_file(str(p), content_type='application/pickle', allow_pickle=True) == Model.construct(a=12, b=8)