Add additional parameters to Schema for validation and annotation (#319)

* Add additional parameters to Schema for validation and annotation (#311)

* Add tests for validation declared via Schema class in defaults

* Add validations to field from declarations in Schema

* Add annotations in generated JSON Schema from validations via Schema

* Augment tests for schema generation

* Simplify validations from Schema in fields, from coverage hints

* Update schema test to use the spec plural "examples"

* Add docs and simple example of the additional parameters for Schema

* Update history

* Fix number of PR in HISTORY

* Refactor check for numeric types, remove break to make coverage happy

* Fix typo in docs, I confused gt with maximum

* Finish docstring for Schema (I had forgotten about it)

* Implement code review requests and lenient_issubclass with tests

* Move Schema to its now file to extract from fields.py but avoid circular imports

* Control coverage

* Schema fixes (#318)

* rearrange code

* cleanup get_annotation_from_schema

* fix typo

* rename _schema to schema
This commit is contained in:
Samuel Colvin
2018-12-03 12:37:39 +00:00
committed by GitHub
parent a34cfd23da
commit 7a06736ead
11 changed files with 391 additions and 39 deletions
+1
View File
@@ -6,6 +6,7 @@ History
v0.16.0 (2018-XX-XX)
....................
* add additional fields to ``Schema`` class to declare validation for ``str`` and numeric values, #311 by @tiangolo
* refactor schema generation to be compatible with JSON Schema and OpenAPI specs, #308 by @tiangolo
* add ``schema`` to ``schema`` module to generate top-level schemas from base models, #308 by @tiangolo
* add ``case_insensitive`` option to ``BaseSettings`` ``Config``, #277 by @jasonkuhrt
+3 -1
View File
@@ -18,8 +18,10 @@
},
"snap": {
"title": "The Snap",
"default": 42,
"description": "this is the value of snap",
"default": 42,
"exclusiveMinimum": 30,
"exclusiveMaximum": 50,
"type": "integer"
}
},
+2
View File
@@ -24,6 +24,8 @@ class MainModel(BaseModel):
42,
title='The Snap',
description='this is the value of snap',
gt=30,
lt=50,
)
class Config:
+20 -4
View File
@@ -245,15 +245,31 @@ and reference.
"sub-models" with modifications (via the ``Schema`` class) like a custom title, description or default value,
are recursively included instead of referenced.
The ``description`` for models is taken from the docstring of the class.
The ``description`` for models is taken from the docstring of the class or the argument ``description`` to the ``Schema``
class.
Optionally the ``Schema`` class can be used to provide extra information about the field, arguments:
Optionally the ``Schema`` class can be used to provide extra information about the field and validations, arguments:
* ``default`` (positional argument), since the ``Schema`` is replacing the field's default, its first
argument is used to set the default, use ellipsis (``...``) to indicate the field is required
* ``alias`` - the public name of the field
* ``title`` if omitted ``field_name.title()`` is used
* ``alias`` - the public name of the field.
* ``**`` any other keyword arguments (eg. ``description``) will be added verbatim to the field's schema
* ``description`` if omitted and the annotation is a sub-model, the docstring of the sub-model will be used
* ``gt`` for numeric values (``int``, ``float``, ``Decimal``), adds a validation of "greater than" and an annotation
of ``exclusiveMinimum`` to the JSON Schema
* ``ge`` for numeric values, adds a validation of "greater than or equal" and an annotation of ``minimum`` to the
JSON Schema
* ``lt`` for numeric values, adds a validation of "less than" and an annotation of ``exclusiveMaximum`` to the
JSON Schema
* ``le`` for numeric values, adds a validation of "less than or equal" and an annotation of ``maximum`` to the
JSON Schema
* ``min_length`` for string values, adds a corresponding validation and an annotation of ``minLength`` to the
JSON Schema
* ``max_length`` for string values, adds a corresponding validation and an annotation of ``maxLength`` to the
JSON Schema
* ``regex`` for string values, adds a Regular Expression validation generated from the passed string and an
annotation of ``pattern`` to the JSON Schema
* ``**`` any other keyword arguments (eg. ``examples``) will be added verbatim to the field's schema
Instead of using ``Schema``, the ``fields`` property of :ref:`the Config class <config>` can be used
to set all the arguments above except ``default``.
+2 -1
View File
@@ -2,9 +2,10 @@
from .env_settings import BaseSettings
from .error_wrappers import ValidationError
from .errors import *
from .fields import Required, Schema
from .fields import Required
from .main import BaseConfig, BaseModel, create_model, validate_model, validator
from .parse import Protocol
from .types import *
from .version import VERSION
from . import dataclasses
from .schema import Schema
+10 -21
View File
@@ -5,7 +5,7 @@ from typing import Any, Callable, List, Mapping, NamedTuple, Pattern, Set, Tuple
from . import errors as errors_
from .error_wrappers import ErrorWrapper
from .types import Json, JsonWrapper
from .utils import display_as_type, list_like
from .utils import display_as_type, lenient_issubclass, list_like
from .validators import NoneType, dict_validator, find_validators, not_none_validator
Required: Any = Ellipsis
@@ -34,20 +34,6 @@ class Validator(NamedTuple):
check_fields: bool
class Schema:
"""
Used to provide extra information about a field in a model schema.
"""
__slots__ = 'default', 'alias', 'title', 'extra'
def __init__(self, default, *, alias=None, title=None, **extra):
self.default = default
self.alias = alias
self.title = title
self.extra = extra
class Field:
__slots__ = (
'type_',
@@ -62,7 +48,7 @@ class Field:
'name',
'alias',
'has_alias',
'_schema',
'schema',
'validate_always',
'allow_none',
'shape',
@@ -81,7 +67,7 @@ class Field:
model_config: Any,
alias: str = None,
allow_none: bool = False,
schema: Schema = None,
schema=None,
):
self.name: str = name
@@ -101,12 +87,14 @@ class Field:
self.allow_none: bool = allow_none
self.parse_json: bool = False
self.shape: Shape = Shape.SINGLETON
self._schema: Schema = schema
self.schema: 'schema.Schema' = schema
self.prepare()
@classmethod
def infer(cls, *, name, value, annotation, class_validators, config):
schema_from_config = config.get_field_schema(name)
from .schema import Schema, get_annotation_from_schema
if isinstance(value, Schema):
schema = value
value = schema.default
@@ -114,6 +102,7 @@ class Field:
schema = Schema(value, **schema_from_config)
schema.alias = schema.alias or schema_from_config.get('alias')
required = value == Required
annotation = get_annotation_from_schema(annotation, schema)
return cls(
name=name,
type_=annotation,
@@ -129,8 +118,8 @@ class Field:
self.model_config = config
schema_from_config = config.get_field_schema(self.name)
if schema_from_config:
self._schema.alias = self._schema.alias or schema_from_config.get('alias')
self.alias = self._schema.alias
self.schema.alias = self.schema.alias or schema_from_config.get('alias')
self.alias = self.schema.alias
@property
def alt_alias(self):
@@ -155,7 +144,7 @@ class Field:
def _populate_sub_fields(self): # noqa: C901 (ignore complexity)
# typing interface is horrible, we have to do some ugly checks
if isinstance(self.type_, type) and issubclass(self.type_, JsonWrapper):
if lenient_issubclass(self.type_, JsonWrapper):
self.type_ = self.type_.inner_type
self.parse_json = True
+185 -9
View File
@@ -8,10 +8,31 @@ from uuid import UUID
from . import main
from .fields import Field, Shape
from .json import pydantic_encoder
from .types import DSN, UUID1, UUID3, UUID4, UUID5, DirectoryPath, EmailStr, FilePath, Json, NameEmail, UrlStr
from .utils import clean_docstring
from .types import (
DSN,
UUID1,
UUID3,
UUID4,
UUID5,
ConstrainedDecimal,
ConstrainedFloat,
ConstrainedInt,
ConstrainedStr,
DirectoryPath,
EmailStr,
FilePath,
Json,
NameEmail,
UrlStr,
condecimal,
confloat,
conint,
constr,
)
from .utils import clean_docstring, lenient_issubclass
__all__ = [
'Schema',
'schema',
'model_schema',
'field_schema',
@@ -26,11 +47,89 @@ __all__ = [
'model_type_schema',
'field_singleton_sub_fields_schema',
'field_singleton_schema',
'get_annotation_from_schema',
]
default_prefix = '#/definitions/'
class Schema:
"""
Used to provide extra information about a field in a model schema. The parameters will be
converted to validations and will add annotations to the generated JSON Schema. Some arguments
apply only to number fields (``int``, ``float``, ``Decimal``) and some apply only to ``str``
:param default: since the Schema is replacing the fields default, its first argument is used
to set the default, use ellipsis (``...``) to indicate the field is required
:param alias: the public name of the field
:param title: can be any string, used in the schema
:param description: can be any string, used in the schema
:param gt: only applies to numbers, requires the field to be "greater than". The schema
will have an ``exclusiveMinimum`` validation keyword
:param ge: only applies to numbers, requires the field to be "greater than or equal to". The
schema will have a ``minimum`` validation keyword
:param lt: only applies to numbers, requires the field to be "less than". The schema
will have an ``exclusiveMaximum`` validation keyword
:param le: only applies to numbers, requires the field to be "less than or equal to". The
schema will have a ``maximum`` validation keyword
:param min_length: only applies to strings, requires the field to have a minimum length. The
schema will have a ``maximum`` validation keyword
:param max_length: only applies to strings, requires the field to have a maximum length. The
schema will have a ``maxLength`` validation keyword
:param regex: only applies to strings, requires the field match agains a regular expression
pattern string. The schema will have a ``pattern`` validation keyword
:param **extra: any additional keyword arguments will be added as is to the schema
"""
__slots__ = (
'default',
'alias',
'title',
'description',
'gt',
'ge',
'lt',
'le',
'min_length',
'max_length',
'regex',
'extra',
)
def __init__(
self,
default,
*,
alias: str = None,
title: str = None,
description: str = None,
gt: float = None,
ge: float = None,
lt: float = None,
le: float = None,
min_length: int = None,
max_length: int = None,
regex: str = None,
**extra,
):
self.default = default
self.alias = alias
self.title = title
self.description = description
self.extra = extra
self.gt = gt
self.ge = ge
self.lt = lt
self.le = le
self.min_length = min_length
self.max_length = max_length
self.regex = regex
def __repr__(self):
attrs = ((s, getattr(self, s)) for s in self.__slots__)
return 'Schema({})'.format(', '.join(f'{a}: {v!r}' for a, v in attrs if v is not None))
def schema(
models: Sequence[Type['main.BaseModel']], *, by_alias=True, title=None, description=None, ref_prefix=None
) -> Dict:
@@ -114,15 +213,21 @@ def field_schema(
"""
ref_prefix = ref_prefix or default_prefix
schema_overrides = False
s = dict(title=field._schema.title or field.alias.title())
if field._schema.title:
s = dict(title=field.schema.title or field.alias.title())
if field.schema.title:
schema_overrides = True
if field.schema.description:
s['description'] = field.schema.description
schema_overrides = True
if not field.required and field.default is not None:
schema_overrides = True
s['default'] = encode_default(field.default)
if field._schema.extra:
s.update(field._schema.extra)
schema_overrides = True
validation_schema = get_field_schema_validations(field)
if validation_schema:
s.update(validation_schema)
schema_overrides = True
f_schema, f_definitions = field_type_schema(
@@ -140,6 +245,42 @@ def field_schema(
return s, f_definitions
numeric_types = (int, float, Decimal)
_str_types_attrs = (
('max_length', numeric_types, 'maxLength'),
('min_length', numeric_types, 'minLength'),
('regex', str, 'pattern'),
)
_numeric_types_attrs = (
('gt', numeric_types, 'exclusiveMinimum'),
('lt', numeric_types, 'exclusiveMaximum'),
('ge', numeric_types, 'minimum'),
('le', numeric_types, 'maximum'),
)
def get_field_schema_validations(field):
"""
Get the JSON Schema validation keywords for a ``field`` with an annotation of
a Pydantic ``Schema`` with validation arguments.
"""
f_schema = {}
if lenient_issubclass(field.type_, (str, bytes)):
for attr_name, t, keyword in _str_types_attrs:
attr = getattr(field.schema, attr_name, None)
if isinstance(attr, t):
f_schema[keyword] = attr
if lenient_issubclass(field.type_, numeric_types) and not issubclass(field.type_, bool):
for attr_name, t, keyword in _numeric_types_attrs:
attr = getattr(field.schema, attr_name, None)
if isinstance(attr, t):
f_schema[keyword] = attr
if field.schema.extra:
f_schema.update(field.schema.extra)
return f_schema
def get_model_name_map(unique_models: Set[Type['main.BaseModel']]) -> Dict[Type['main.BaseModel'], str]:
"""
Process a set of models and generate unique names for them to be used as keys in the JSON Schema
@@ -197,7 +338,7 @@ def get_flat_models_from_field(field: Field) -> Set[Type['main.BaseModel']]:
flat_models = set()
if field.sub_fields:
flat_models |= get_flat_models_from_fields(field.sub_fields)
elif isinstance(field.type_, type) and issubclass(field.type_, main.BaseModel):
elif lenient_issubclass(field.type_, main.BaseModel):
flat_models |= get_flat_models_from_model(field.type_)
return flat_models
@@ -320,7 +461,7 @@ def model_process_schema(
def model_type_schema(
model: 'main.BaseModel', *, by_alias: bool, model_name_map: Dict[Type['main.BaseModel'], str], ref_prefix=None
model: Type['main.BaseModel'], *, by_alias: bool, model_name_map: Dict[Type['main.BaseModel'], str], ref_prefix=None
):
"""
You probably should be using ``model_schema()``, this function is indirectly used by that function.
@@ -503,3 +644,38 @@ def encode_default(dft):
return {encode_default(k): encode_default(v) for k, v in dft.items()}
else:
return pydantic_encoder(dft)
_map_types_constraint = {int: conint, float: confloat, Decimal: condecimal}
def get_annotation_from_schema(annotation, schema):
"""
Get an annotation with validation implemented for numbers and strings based on the schema.
:param annotation: an annotation from a field specification, as ``str``, ``ConstrainedStr``
:param schema: an instance of Schema, possibly with declarations for validations and JSON Schema
:return: the same ``annotation`` if unmodified or a new annotation with validation in place
"""
if isinstance(annotation, type):
attrs = constraint_func = None
if issubclass(annotation, str) and not issubclass(annotation, (EmailStr, DSN, UrlStr, ConstrainedStr)):
attrs = ('max_length', 'min_length', 'regex')
constraint_func = constr
elif lenient_issubclass(annotation, numeric_types) and not issubclass(
annotation, (ConstrainedInt, ConstrainedFloat, ConstrainedDecimal, bool)
):
# Is numeric type
attrs = ('gt', 'lt', 'ge', 'le')
numeric_type = next(t for t in numeric_types if issubclass(annotation, t)) # pragma: no branch
constraint_func = _map_types_constraint[numeric_type]
if attrs:
kwargs = {
attr_name: attr
for attr_name, attr in ((attr_name, getattr(schema, attr_name)) for attr_name in attrs)
if attr is not None
}
if kwargs:
return constraint_func(**kwargs)
return annotation
+2 -1
View File
@@ -56,6 +56,7 @@ __all__ = [
'FilePath',
'DirectoryPath',
'Json',
'JsonWrapper',
]
NoneStr = Optional[str]
@@ -103,7 +104,7 @@ class ConstrainedStr(str):
return value
def constr(*, strip_whitespace=False, min_length=0, max_length=2 ** 16, curtail_length=None, regex=None) -> Type[str]:
def constr(*, strip_whitespace=False, min_length=None, max_length=None, curtail_length=None, regex=None) -> Type[str]:
# use kwargs then define conf in a dict to aid with IDE type hinting
namespace = dict(
strip_whitespace=strip_whitespace,
+5 -1
View File
@@ -129,7 +129,7 @@ def display_as_type(v):
if not isinstance(v, typing_base) and not isinstance(v, type):
v = type(v)
if isinstance(v, type) and issubclass(v, Enum):
if lenient_issubclass(v, Enum):
if issubclass(v, int):
return 'int'
elif issubclass(v, str):
@@ -201,3 +201,7 @@ def url_regex_generator(*, relative: bool, require_tld: bool) -> Pattern:
),
re.IGNORECASE,
)
def lenient_issubclass(cls, class_or_tuple):
return isinstance(cls, type) and issubclass(cls, class_or_tuple)
+149
View File
@@ -147,6 +147,12 @@ def test_schema_class():
}
def test_schema_repr():
s = Schema(4, title='Foo is Great')
assert repr(s) == "Schema(default: 4, title: 'Foo is Great', extra: {})"
assert str(s) == "Schema(default: 4, title: 'Foo is Great', extra: {})"
def test_schema_class_by_alias():
class Model(BaseModel):
foo: int = Schema(4, alias='foofoo')
@@ -855,3 +861,146 @@ def test_dict_default():
}
},
}
@pytest.mark.parametrize(
'kwargs,type_,expected_extra',
[
({'max_length': 5}, str, {'type': 'string', 'maxLength': 5}),
({'max_length': 5}, constr(max_length=6), {'type': 'string', 'maxLength': 6}),
({'min_length': 2}, str, {'type': 'string', 'minLength': 2}),
({'max_length': 5}, bytes, {'type': 'string', 'maxLength': 5, 'format': 'binary'}),
({'regex': '^foo$'}, str, {'type': 'string', 'pattern': '^foo$'}),
({'gt': 2}, int, {'type': 'integer', 'exclusiveMinimum': 2}),
({'lt': 5}, int, {'type': 'integer', 'exclusiveMaximum': 5}),
({'ge': 2}, int, {'type': 'integer', 'minimum': 2}),
({'le': 5}, int, {'type': 'integer', 'maximum': 5}),
({'gt': 2}, float, {'type': 'number', 'exclusiveMinimum': 2}),
({'lt': 5}, float, {'type': 'number', 'exclusiveMaximum': 5}),
({'ge': 2}, float, {'type': 'number', 'minimum': 2}),
({'le': 5}, float, {'type': 'number', 'maximum': 5}),
({'gt': 2}, Decimal, {'type': 'number', 'exclusiveMinimum': 2}),
({'lt': 5}, Decimal, {'type': 'number', 'exclusiveMaximum': 5}),
({'ge': 2}, Decimal, {'type': 'number', 'minimum': 2}),
({'le': 5}, Decimal, {'type': 'number', 'maximum': 5}),
],
)
def test_constraints_schema(kwargs, type_, expected_extra):
class Foo(BaseModel):
a: type_ = Schema('foo', title='A title', description='A description', **kwargs)
expected_schema = {
'title': 'Foo',
'type': 'object',
'properties': {'a': {'title': 'A title', 'description': 'A description', 'default': 'foo'}},
}
expected_schema['properties']['a'].update(expected_extra)
assert Foo.schema() == expected_schema
@pytest.mark.parametrize(
'kwargs,type_,expected',
[
({'max_length': 5}, int, {'type': 'integer'}),
({'min_length': 2}, float, {'type': 'number'}),
({'max_length': 5}, Decimal, {'type': 'number'}),
({'regex': '^foo$'}, int, {'type': 'integer'}),
({'gt': 2}, str, {'type': 'string'}),
({'lt': 5}, bytes, {'type': 'string', 'format': 'binary'}),
({'ge': 2}, str, {'type': 'string'}),
({'le': 5}, bool, {'type': 'boolean'}),
],
)
def test_not_constraints_schema(kwargs, type_, expected):
class Foo(BaseModel):
a: type_ = Schema('foo', title='A title', description='A description', **kwargs)
base_schema = {
'title': 'Foo',
'type': 'object',
'properties': {'a': {'title': 'A title', 'description': 'A description', 'default': 'foo'}},
}
base_schema['properties']['a'].update(expected)
assert Foo.schema() == base_schema
@pytest.mark.parametrize(
'kwargs,type_,value',
[
({'max_length': 5}, str, 'foo'),
({'max_length': 5}, constr(max_length=6), 'foo'),
({'min_length': 2}, str, 'foo'),
({'max_length': 5}, bytes, b'foo'),
({'regex': '^foo$'}, str, 'foo'),
({'max_length': 5}, bool, True),
({'gt': 2}, int, 3),
({'lt': 5}, int, 3),
({'ge': 2}, int, 3),
({'ge': 2}, int, 2),
({'gt': 2}, int, '3'),
({'le': 5}, int, 3),
({'le': 5}, int, 5),
({'gt': 2}, float, 3.0),
({'gt': 2}, float, 2.1),
({'lt': 5}, float, 3.0),
({'lt': 5}, float, 4.9),
({'ge': 2}, float, 3.0),
({'ge': 2}, float, 2.0),
({'le': 5}, float, 3.0),
({'le': 5}, float, 5.0),
({'gt': 2}, float, 3),
({'gt': 2}, float, '3'),
({'gt': 2}, Decimal, Decimal(3)),
({'lt': 5}, Decimal, Decimal(3)),
({'ge': 2}, Decimal, Decimal(3)),
({'ge': 2}, Decimal, Decimal(2)),
({'le': 5}, Decimal, Decimal(3)),
({'le': 5}, Decimal, Decimal(5)),
],
)
def test_constraints_schema_validation(kwargs, type_, value):
class Foo(BaseModel):
a: type_ = Schema('foo', title='A title', description='A description', **kwargs)
assert Foo(a=value)
@pytest.mark.parametrize(
'kwargs,type_,value',
[
({'max_length': 5}, str, 'foobar'),
({'min_length': 2}, str, 'f'),
({'regex': '^foo$'}, str, 'bar'),
({'gt': 2}, int, 2),
({'lt': 5}, int, 5),
({'ge': 2}, int, 1),
({'le': 5}, int, 6),
({'gt': 2}, float, 2.0),
({'lt': 5}, float, 5.0),
({'ge': 2}, float, 1.9),
({'le': 5}, float, 5.1),
({'gt': 2}, Decimal, Decimal(2)),
({'lt': 5}, Decimal, Decimal(5)),
({'ge': 2}, Decimal, Decimal(1)),
({'le': 5}, Decimal, Decimal(6)),
],
)
def test_constraints_schema_validation_raises(kwargs, type_, value):
class Foo(BaseModel):
a: type_ = Schema('foo', title='A title', description='A description', **kwargs)
with pytest.raises(ValidationError):
Foo(a=value)
def test_schema_kwargs():
class Foo(BaseModel):
a: str = Schema('foo', examples=['bar'])
assert Foo.schema() == {
'title': 'Foo',
'type': 'object',
'properties': {'a': {'type': 'string', 'title': 'A', 'default': 'foo', 'examples': ['bar']}},
}
+12 -1
View File
@@ -4,7 +4,7 @@ from typing import Union
import pytest
from pydantic.utils import display_as_type, import_string, make_dsn, validate_email
from pydantic.utils import display_as_type, import_string, lenient_issubclass, make_dsn, validate_email
try:
import email_validator
@@ -135,3 +135,14 @@ def test_display_as_type_enum_str():
displayed = display_as_type(SubField)
assert displayed == 'str'
def test_lenient_issubclass():
class A(str):
pass
assert lenient_issubclass(A, str) is True
def test_lenient_issubclass_is_lenient():
assert lenient_issubclass('a', 'a') is False