cleaning up error display, enabling list parsing

pydantic/exceptions.py

@@ -1,14 +1,45 @@
 import json
+from collections import OrderedDict, namedtuple
+
+
+def type_json(type_: type):
+    if type_:
+        try:
+            return type_.__name__
+        except AttributeError:
+            # happens with unions
+            return str(type_)
+
+
+Error = namedtuple('Error', ['exc', 'validator', 'track_type', 'index'])
+
+
+def jsonify_errors(e):
+    if isinstance(e, Error):
+        return {
+            'error_type': e.exc.__class__.__name__,
+            'error_msg': str(e.exc),
+            'validator': e.validator and e.validator.__qualname__,
+            'track': type_json(e.track_type),
+            'index': e.index,
+        }
+    elif isinstance(e, OrderedDict):
+        return OrderedDict([(k, jsonify_errors(v)) for k, v in e.items()])
+    else:
+        return [jsonify_errors(e_) for e_ in e]
 
 
 class ValidationError(ValueError):
     def __init__(self, errors):
         self.errors = errors
-        e_count = len(errors)
+        e_count = len(self.errors)
         s = '' if e_count == 1 else 's'
         self.message = f'{e_count} error{s} validating input'
-        self.pretty_errors = json.dumps(errors, sort_keys=True)
-        super().__init__(f'{self.message}: {self.pretty_errors}')
+        self.errors_jsonable = jsonify_errors(errors)
+        super().__init__(f'{self.message}: {self.json()}')
+
+    def json(self, indent=None):
+        return json.dumps(self.errors_jsonable, indent=indent, sort_keys=True)
 
 
 class ConfigError(RuntimeError):

pydantic/fields.py

@@ -1,9 +1,9 @@
 import inspect
 from collections import OrderedDict
 from enum import IntEnum
-from typing import Any, List, Type, Union  # noqa
+from typing import Any, List, Sequence, Type, Union  # noqa
 
-from .exceptions import ConfigError
+from .exceptions import ConfigError, Error, type_json
 from .validators import NoneType, find_validator, not_none_validator
 
 
@@ -12,17 +12,15 @@ class ValidatorSignature(IntEnum):
     VALUE_KWARGS = 2
 
 
-def type_str(type_: type):
-    try:
-        return type_.__name__
-    except AttributeError:
-        # happens with unions
-        return str(type_)
+class Shape(IntEnum):
+    SINGLETON = 1
+    SEQUENCE = 2
+    MAPPING = 3
 
 
 class Field:
-    __slots__ = ('type_', 'validator_tracks', 'default', 'required', 'name', 'description',
-                 'info', 'validate_always', 'allow_none')
+    __slots__ = ('type_', 'validator_tracks', 'track_count', 'default', 'required', 'name', 'description',
+                 'info', 'validate_always', 'allow_none', 'shape')
 
     def __init__(
             self, *,
@@ -40,6 +38,7 @@ class Field:
         self.name: str = name
         self.description: str = description
         self.allow_none: bool = False
+        self.shape: Shape = Shape.SINGLETON
 
     def prepare(self, name, class_validators):
         self.name = self.name or name
@@ -49,14 +48,22 @@ class Field:
         if self.type_ is None:
             raise ConfigError(f'unable to infer type for attribute "{self.name}"')
 
+        # typing interface is horrible, we have to do some ugly checks
+        origin = getattr(self.type_, '__origin__', None)
+        if origin not in (None, Union):
+            if issubclass(origin, Sequence):
+                self.type_ = self.type_.__args__[0]
+                self.shape = Shape.SEQUENCE
+            # TODO mapping
+
         self._populate_validator_tracks(class_validators)
         validators = {
-            type_str(r.type_): [v[1].__qualname__ for v in r.validators] for r in self.validator_tracks
+            type_json(r.type_): [v[1].__qualname__ for v in r.validators] for r in self.validator_tracks
         }
         if len(validators) == 1:
             validators = list(validators.values())[0]
         self.info = OrderedDict([
-            ('type', type_str(self.type_)),
+            ('type', type_json(self.type_)),
             ('default', self.default),
             ('required', self.required),
             ('validators', validators)
@@ -92,12 +99,35 @@ class Field:
         if self.allow_none and v is None:
             return None, None
 
+        if self.shape == Shape.SINGLETON:
+            return self._validate_singleton(v, model)
+        elif self.shape == Shape.SEQUENCE:
+            result, errors = [], []
+            try:
+                v_iter = enumerate(v)
+            except TypeError as exc:
+                return v, Error(exc, iter, None, None)
+            for i, v_ in v_iter:
+                single_result, single_errors = self._validate_singleton(v_, model, i)
+                if errors or single_errors:
+                    errors.append(single_errors)
+                else:
+                    result.append(single_result)
+            if errors:
+                return v, errors
+            else:
+                return result, None
+        else:
+            # mapping
+            raise NotImplemented('TODO')
+
+    def _validate_singleton(self, v, model, index=None):
         errors = []
         result = ...
         for track in self.validator_tracks:
-            value, error, validator = track.validate(v, model, self)
-            if error:
-                errors.append((error, validator, track.type_))
+            value, exc, validator = track.validate(v, model, self)
+            if exc:
+                errors.append(Error(exc, validator, track.type_, index))
             elif isinstance(v, track.type_):
                 # exact match: return immediately
                 return value, None
@@ -105,7 +135,10 @@ class Field:
                 result = value
         if result is not ...:
             return result, None
-        return v, errors
+        elif len(self.validator_tracks) == 1:
+            return v, errors[0]
+        else:
+            return v, errors
 
     @classmethod
     def infer(cls, *, name, value, annotation, class_validators):

pydantic/main.py

@@ -1,8 +1,8 @@
 from collections import OrderedDict
 from types import FunctionType
 
-from .exceptions import ValidationError
-from .fields import Field, type_str
+from .exceptions import Error, ValidationError
+from .fields import Field
 from .validators import dict_validator
 
 
@@ -13,7 +13,7 @@ class BaseConfig:
     max_number_size = 2 ** 64
     raise_exception = True
     validate_all = False
-    allow_extra = True
+    ignore_extra = True
 
 
 def inherit_config(self_config, parent_config) -> BaseConfig:
@@ -63,9 +63,17 @@ class MetaModel(type):
         return super().__new__(mcs, name, bases, namespace)
 
 
-MISSING = object()
-MISSING_ERROR = {'type': 'Missing', 'msg': 'field required'}
-EXTRA_ERROR = {'type': 'Extra', 'msg': 'extra field not permitted'}
+class Missing(ValueError):
+    pass
+
+
+class Extra(ValueError):
+    pass
+
+
+MISSING = Missing('field required')
+MISSING_ERROR = Error(MISSING, None, None, None)
+EXTRA_ERROR = Error(Extra('extra fields not permitted'), None, None, None)
 
 
 class BaseModel(metaclass=MetaModel):
@@ -103,7 +111,7 @@ class BaseModel(metaclass=MetaModel):
             value = values.get(name, MISSING)
             self._process_value(name, field, value)
 
-        if not self.config.allow_extra:
+        if not self.config.ignore_extra:
             extra = values.keys() - self.__fields__.keys()
             if extra:
                 for field in sorted(extra):
@@ -127,20 +135,7 @@ class BaseModel(metaclass=MetaModel):
 
         value, errors = field.validate(value, self)
         if errors:
-            if len(errors) == 1:
-                error, validator, _ = errors[0]
-                self.__errors__[name] = {
-                    'type': error.__class__.__name__,
-                    'msg': str(error),
-                    'validator': validator.__qualname__,
-                }
-            else:
-                self.__errors__[name] = [{
-                    'type': error.__class__.__name__,
-                    'route': type_str(type_),
-                    'msg': str(error),
-                    'validator': validator.__qualname__,
-                } for error, validator, type_ in errors]
+            self.__errors__[name] = errors
         self.__values__[name] = value
         setattr(self, name, value)
 

pydantic/validators.py

@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from datetime import date, datetime, time, timedelta
 from enum import Enum
 from pathlib import Path
@@ -58,12 +59,24 @@ def anystr_length_validator(v, model, **kwargs):
     raise ValueError(f'length not in range {model.config.max_anystr_length} to {model.config.max_anystr_length}')
 
 
+def ordered_dict_validator(v) -> OrderedDict:
+    if isinstance(v, OrderedDict):
+        return v
+    return OrderedDict(v)
+
+
 def dict_validator(v) -> dict:
     if isinstance(v, dict):
         return v
     return dict(v)
 
 
+def list_validator(v) -> list:
+    if isinstance(v, list):
+        return v
+    return list(v)
+
+
 def enum_validator(v, field, **kwargs) -> Enum:
     return field.type_(v)
 
@@ -86,9 +99,10 @@ _VALIDATORS = [
     (time, [parse_time]),
     (timedelta, [parse_duration]),
 
-    (dict, [not_none_validator, dict_validator]),
+    (OrderedDict, [ordered_dict_validator]),
+    (dict, [dict_validator]),
+    (list, [list_validator]),
 ]
-# TODO list, List, Dict
 
 
 def find_validator(type_):

tests/test_complex.py

@@ -1,5 +1,4 @@
-from collections import OrderedDict
-from typing import Union
+from typing import List, Union
 
 import pytest
 
@@ -24,13 +23,25 @@ def test_str_bytes():
     with pytest.raises(ValidationError) as exc_info:
         StrBytesModel(v=None)
     assert exc_info.value.message == '1 error validating input'
-    assert exc_info.value.errors == OrderedDict(
-        [
-            ('v', [{'type': 'TypeError', 'route': 'str', 'msg': 'None is not an allow value',
-                    'validator': 'not_none_validator'},
-                   {'type': 'TypeError', 'route': 'bytes', 'msg': 'None is not an allow value',
-                    'validator': 'not_none_validator'}])
-        ])
+    assert """\
+{
+  "v": [
+    {
+      "error_msg": "None is not an allow value",
+      "error_type": "TypeError",
+      "index": null,
+      "track": "str",
+      "validator": "not_none_validator"
+    },
+    {
+      "error_msg": "None is not an allow value",
+      "error_type": "TypeError",
+      "index": null,
+      "track": "bytes",
+      "validator": "not_none_validator"
+    }
+  ]
+}""" == exc_info.value.json(2)
 
 
 def test_str_bytes_none():
@@ -76,10 +87,67 @@ def test_union_int_str():
     with pytest.raises(ValidationError) as exc_info:
         Model(v=None)
     assert exc_info.value.message == '1 error validating input'
-    assert exc_info.value.errors == OrderedDict(
-        [
-            ('v', [{'type': 'TypeError', 'route': 'int', 'validator': 'int',
-                    'msg': "int() argument must be a string, a bytes-like object or a number, not 'NoneType'"},
-                   {'type': 'TypeError', 'route': 'str', 'msg': 'None is not an allow value',
-                    'validator': 'not_none_validator'}])
-        ])
+    assert """\
+{
+  "v": [
+    {
+      "error_msg": "int() argument must be a string, a bytes-like object or a number, not 'NoneType'",
+      "error_type": "TypeError",
+      "index": null,
+      "track": "int",
+      "validator": "int"
+    },
+    {
+      "error_msg": "None is not an allow value",
+      "error_type": "TypeError",
+      "index": null,
+      "track": "str",
+      "validator": "not_none_validator"
+    }
+  ]
+}""" == exc_info.value.json(2)
+
+
+def test_typed_list():
+    class Model(BaseModel):
+        v: List[int] = ...
+
+    m = Model(v=[1, 2, '3'])
+    assert m.v == [1, 2, 3]
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model(v=[1, 'x', 'y'])
+    assert exc_info.value.message == '1 error validating input'
+    assert """\
+{
+  "v": [
+    {
+      "error_msg": "invalid literal for int() with base 10: 'x'",
+      "error_type": "ValueError",
+      "index": 1,
+      "track": "int",
+      "validator": "int"
+    },
+    {
+      "error_msg": "invalid literal for int() with base 10: 'y'",
+      "error_type": "ValueError",
+      "index": 2,
+      "track": "int",
+      "validator": "int"
+    }
+  ]
+}""" == exc_info.value.json(2)
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model(v=1)
+    assert exc_info.value.message == '1 error validating input'
+    assert """\
+{
+  "v": {
+    "error_msg": "'int' object is not iterable",
+    "error_type": "TypeError",
+    "index": null,
+    "track": null,
+    "validator": "iter"
+  }
+}""" == exc_info.value.json(2)

tests/test_main.py

@@ -1,8 +1,8 @@
-from collections import OrderedDict
+import json
 
 import pytest
 
-from pydantic import BaseModel, ConfigError, NoneBytes, NoneStr, ValidationError
+from pydantic import BaseModel, ConfigError, NoneBytes, NoneStr, ValidationError, jsonify_errors
 
 
 class UltraSimpleModel(BaseModel):
@@ -19,18 +19,40 @@ def test_ultra_simple_success():
 def test_ultra_simple_missing():
     with pytest.raises(ValidationError) as exc_info:
         UltraSimpleModel()
-    assert exc_info.value.args[0] == '1 error validating input: {"a": {"msg": "field required", "type": "Missing"}}'
+    assert exc_info.value.message == '1 error validating input'
+    assert """\
+{
+  "a": {
+    "error_msg": "field required",
+    "error_type": "Missing",
+    "index": null,
+    "track": null,
+    "validator": null
+  }
+}""" == exc_info.value.json(2)
 
 
 def test_ultra_simple_failed():
     with pytest.raises(ValidationError) as exc_info:
         UltraSimpleModel(a='x', b='x')
-    assert exc_info.value.errors == OrderedDict(
-        [
-            ('a', {'type': 'ValueError', 'msg': "could not convert string to float: 'x'", 'validator': 'float'}),
-            ('b', {'type': 'ValueError', 'msg': "invalid literal for int() with base 10: 'x'", 'validator': 'int'})
-        ]
-    )
+    assert exc_info.value.message == '2 errors validating input'
+    assert """\
+{
+  "a": {
+    "error_msg": "could not convert string to float: 'x'",
+    "error_type": "ValueError",
+    "index": null,
+    "track": "float",
+    "validator": "float"
+  },
+  "b": {
+    "error_msg": "invalid literal for int() with base 10: 'x'",
+    "error_type": "ValueError",
+    "index": null,
+    "track": "int",
+    "validator": "int"
+  }
+}""" == exc_info.value.json(2)
 
 
 def test_ultra_simple_repr():
@@ -47,7 +69,8 @@ class ConfigModel(UltraSimpleModel):
 
 def test_config_doesnt_raise():
     m = ConfigModel()
-    assert m.errors == OrderedDict([('a', {'type': 'Missing', 'msg': 'field required'})])
+    assert len(m.errors) == 1
+    assert m.errors['a'].exc.args[0] == 'field required'
     assert m.config.raise_exception is False
     assert m.config.max_anystr_length == 65536
 
@@ -90,13 +113,23 @@ def test_nullable_strings_fails():
         required_bytes_value=None,
         required_bytes_none_value=None,
     )
-    assert m.errors == OrderedDict(
-        [
-            ('required_str_value', {'type': 'TypeError', 'msg': 'None is not an allow value',
-                                    'validator': 'not_none_validator'}),
-            ('required_bytes_value', {'type': 'TypeError', 'msg': 'None is not an allow value',
-                                      'validator': 'not_none_validator'})
-        ])
+    assert """\
+{
+  "required_bytes_value": {
+    "error_msg": "None is not an allow value",
+    "error_type": "TypeError",
+    "index": null,
+    "track": "bytes",
+    "validator": "not_none_validator"
+  },
+  "required_str_value": {
+    "error_msg": "None is not an allow value",
+    "error_type": "TypeError",
+    "index": null,
+    "track": "str",
+    "validator": "not_none_validator"
+  }
+}""" == json.dumps(jsonify_errors(m.errors), indent=2, sort_keys=True)
 
 
 class RecursiveModel(BaseModel):
@@ -121,7 +154,7 @@ class PreventExtraModel(BaseModel):
     foo = 'whatever'
 
     class Config:
-        allow_extra = False
+        ignore_extra = False
 
 
 def test_prevent_extra_success():
@@ -135,8 +168,23 @@ def test_prevent_extra_fails():
     with pytest.raises(ValidationError) as exc_info:
         PreventExtraModel(foo='ok', bar='wrong', spam='xx')
     assert exc_info.value.message == '2 errors validating input'
-    assert exc_info.value.pretty_errors == ('{"bar": {"msg": "extra field not permitted", "type": "Extra"}, '
-                                            '"spam": {"msg": "extra field not permitted", "type": "Extra"}}')
+    assert """\
+{
+  "bar": {
+    "error_msg": "extra fields not permitted",
+    "error_type": "Extra",
+    "index": null,
+    "track": null,
+    "validator": null
+  },
+  "spam": {
+    "error_msg": "extra fields not permitted",
+    "error_type": "Extra",
+    "index": null,
+    "track": null,
+    "validator": null
+  }
+}""" == exc_info.value.json(2)
 
 
 class InvalidValidator:

tests/test_types.py

@@ -1,5 +1,4 @@
 import os
-from collections import OrderedDict
 from datetime import date, datetime, time, timedelta
 from enum import Enum, IntEnum
 
@@ -25,10 +24,16 @@ def test_constrained_str_default():
 def test_constrained_str_too_long():
     with pytest.raises(ValidationError) as exc_info:
         ConStringModel(v='this is too long')
-    assert exc_info.value.args[0] == ('1 error validating input: {"v": {'
-                                      '"msg": "length greater than maximum allowed: 10", '
-                                      '"type": "ValueError", '
-                                      '"validator": "ConstrainedStr.validate"}}')
+    assert """\
+{
+  "v": {
+    "error_msg": "length greater than maximum allowed: 10",
+    "error_type": "ValueError",
+    "index": null,
+    "track": "ConstrainedStrValue",
+    "validator": "ConstrainedStr.validate"
+  }
+}""" == exc_info.value.json(2)
 
 
 class DsnModel(BaseModel):
@@ -165,13 +170,37 @@ def test_datetime_errors():
             duration='15:30.0001 broken',
         )
     assert exc_info.value.message == '4 errors validating input'
-    assert exc_info.value.errors == OrderedDict(
-        [
-            ('dt', {'type': 'ValueError', 'msg': 'month must be in 1..12', 'validator': 'parse_datetime'}),
-            ('date_', {'type': 'ValueError', 'msg': 'Invalid date format', 'validator': 'parse_date'}),
-            ('time_', {'type': 'ValueError', 'msg': 'hour must be in 0..23', 'validator': 'parse_time'}),
-            ('duration', {'type': 'ValueError', 'msg': 'Invalid duration format', 'validator': 'parse_duration'})
-        ])
+    assert """\
+{
+  "date_": {
+    "error_msg": "Invalid date format",
+    "error_type": "ValueError",
+    "index": null,
+    "track": "date",
+    "validator": "parse_date"
+  },
+  "dt": {
+    "error_msg": "month must be in 1..12",
+    "error_type": "ValueError",
+    "index": null,
+    "track": "datetime",
+    "validator": "parse_datetime"
+  },
+  "duration": {
+    "error_msg": "Invalid duration format",
+    "error_type": "ValueError",
+    "index": null,
+    "track": "timedelta",
+    "validator": "parse_duration"
+  },
+  "time_": {
+    "error_msg": "hour must be in 0..23",
+    "error_type": "ValueError",
+    "index": null,
+    "track": "time",
+    "validator": "parse_time"
+  }
+}""" == exc_info.value.json(2)
 
 
 class FruitEnum(str, Enum):
@@ -200,8 +229,16 @@ def test_enum_fails():
     with pytest.raises(ValueError) as exc_info:
         CookingModel(tool=3)
     assert exc_info.value.message == '1 error validating input'
-    assert exc_info.value.pretty_errors == ('{"tool": {"msg": "3 is not a valid ToolEnum", "type": "ValueError", '
-                                            '"validator": "enum_validator"}}')
+    assert """\
+{
+  "tool": {
+    "error_msg": "3 is not a valid ToolEnum",
+    "error_type": "ValueError",
+    "index": null,
+    "track": "ToolEnum",
+    "validator": "enum_validator"
+  }
+}""" == exc_info.value.json(2)
 
 
 class MoreStringsModel(BaseModel):
@@ -237,14 +274,34 @@ def test_string_fails():
             name_email='foobar @example.com',
         )
     assert exc_info.value.message == '4 errors validating input'
-    assert exc_info.value.errors == OrderedDict(
-        [
-            ('str_regex', {'type': 'ValueError', 'msg': 'string does not match regex "^xxx\\d{3}$"',
-                           'validator': 'ConstrainedStr.validate'}),
-            ('str_min_length', {'msg': 'length less than minimum allowed: 5', 'type': 'ValueError',
-                                'validator': 'ConstrainedStr.validate'}),
-            ('str_email', {'type': 'ValueError', 'msg': 'Email address is not valid',
-                           'validator': 'EmailStr.validate'}),
-            ('name_email', {'type': 'ValueError', 'msg': 'Email address is not valid',
-                            'validator': 'NameEmail.validate'})
-        ])
+    assert """\
+{
+  "name_email": {
+    "error_msg": "Email address is not valid",
+    "error_type": "ValueError",
+    "index": null,
+    "track": "NameEmail",
+    "validator": "NameEmail.validate"
+  },
+  "str_email": {
+    "error_msg": "Email address is not valid",
+    "error_type": "ValueError",
+    "index": null,
+    "track": "EmailStr",
+    "validator": "EmailStr.validate"
+  },
+  "str_min_length": {
+    "error_msg": "length less than minimum allowed: 5",
+    "error_type": "ValueError",
+    "index": null,
+    "track": "ConstrainedStrValue",
+    "validator": "ConstrainedStr.validate"
+  },
+  "str_regex": {
+    "error_msg": "string does not match regex \\"^xxx\\\\d{3}$\\"",
+    "error_type": "ValueError",
+    "index": null,
+    "track": "ConstrainedStrValue",
+    "validator": "ConstrainedStr.validate"
+  }
+}""" == exc_info.value.json(2)