From 3dfae212083a35fabe9a402bed4472b29fe1cabf Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Thu, 6 Jun 2019 11:29:09 +0100 Subject: [PATCH] ORM mode: Add support for arbitrary class instances (#562) * Support ORM objects to 'parse_obj', replace #520 * switch to GetterDict and orm_mode * tweaks * update docs * split tests and add @tiangolo's suggestion * split tests and add @tiangolo's suggestion * fix coverage --- .gitignore | 1 + HISTORY.rst | 1 + docs/examples/orm_mode.py | 30 +++++ docs/examples/orm_mode_recursive.py | 35 ++++++ docs/index.rst | 26 +++++ pydantic/main.py | 23 +++- pydantic/utils.py | 37 +++++- setup.py | 2 +- tests/test_orm_mode.py | 168 ++++++++++++++++++++++++++++ 9 files changed, 318 insertions(+), 5 deletions(-) create mode 100644 docs/examples/orm_mode.py create mode 100644 docs/examples/orm_mode_recursive.py create mode 100644 tests/test_orm_mode.py diff --git a/.gitignore b/.gitignore index 0614b89..2388e6c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ _build/ pydantic/*.c pydantic/*.so .auto-format +/sandbox/ diff --git a/HISTORY.rst b/HISTORY.rst index fb3942a..447d136 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,7 @@ v0.28 (unreleased) * fix support for JSON Schema generation when using models with circular references in Python 3.7, #572 by @tiangolo * support ``__post_init_post_parse__`` on dataclasses, #567 by @sevaho * allow dumping dataclasses to JSON, #575 by @samuelcolvin and @DanielOberg +* ORM mode, #562 by @samuelcolvin * fix ``pydantic.compiled`` on ipython, #573 by @dmontagu and @samuelcolvin v0.27 (2019-05-30) diff --git a/docs/examples/orm_mode.py b/docs/examples/orm_mode.py new file mode 100644 index 0000000..36a03ef --- /dev/null +++ b/docs/examples/orm_mode.py @@ -0,0 +1,30 @@ +from typing import List +from sqlalchemy import Column, Integer, String +from sqlalchemy.dialects.postgresql import ARRAY +from sqlalchemy.ext.declarative import declarative_base +from pydantic import BaseModel, constr + +Base = declarative_base() + +class CompanyOrm(Base): + __tablename__ = 'companies' + id = Column(Integer, primary_key=True, nullable=False) + public_key = Column(String(20), index=True, nullable=False, unique=True) + name = Column(String(63), unique=True) + domains = Column(ARRAY(String(255))) + +class CompanyModel(BaseModel): + id: int + public_key: constr(max_length=20) + name: constr(max_length=63) + domains: List[constr(max_length=255)] + + class Config: + orm_mode = True + +co_orm = CompanyOrm(id=123, public_key='foobar', name='Testing', domains=['example.com', 'foobar.com']) +print(co_orm) +#> <__main__.CompanyOrm object at 0x7ff4bf918278> +co_model = CompanyModel.from_orm(co_orm) +print(co_model) +#> CompanyModel id=123 public_key='foobar' name='Testing' domains=['example.com', 'foobar.com'] diff --git a/docs/examples/orm_mode_recursive.py b/docs/examples/orm_mode_recursive.py new file mode 100644 index 0000000..87d04a2 --- /dev/null +++ b/docs/examples/orm_mode_recursive.py @@ -0,0 +1,35 @@ +from typing import List +from pydantic import BaseModel + +class PetCls: + def __init__(self, *, name: str, species: str): + self.name = name + self.species = species + +class PersonCls: + def __init__(self, *, name: str, age: float = None, pets: List[PetCls]): + self.name = name + self.age = age + self.pets = pets + +class Pet(BaseModel): + name: str + species: str + + class Config: + orm_mode = True + +class Person(BaseModel): + name: str + age: float = None + pets: List[Pet] + + class Config: + orm_mode = True + +bones = PetCls(name='Bones', species='dog') +orion = PetCls(name='Orion', species='cat') +anna = PersonCls(name='Anna', age=20, pets=[bones, orion]) +anna_model = Person.from_orm(anna) +print(anna_model) +#> Person name='Anna' pets=[, ] age=20.0 diff --git a/docs/index.rst b/docs/index.rst index 9f6ea0e..6d43c3c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -295,6 +295,31 @@ and pydantic versions). (This script is complete, it should run "as is") +.. _orm_mode: + +ORM Mode (aka Arbitrary Class Instances) +........................................ + +Pydantic models can be created from arbitrary class instances to support models that map to ORM objects. + +To do this: +1. The :ref:`Config ` property ``orm_mode`` must be set to ``True``. +2. The special constructor ``from_orm`` must be used to create the model instance. + +The example here uses SQLAlchemy but the same approach should work for any ORM. + +.. literalinclude:: examples/orm_mode.py + +(This script is complete, it should run "as is") + +ORM instances will be parsed with ``from_orm`` recursively as well as at the top level. + +Here a vanilla class is used to demonstrate the principle, but any ORM could be used instead. + +.. literalinclude:: examples/orm_mode_recursive.py + +(This script is complete, it should run "as is") + .. _schema: Schema Creation @@ -642,6 +667,7 @@ Options: value is instance of that type). If False - RuntimeError will be raised on model declaration (default: ``False``) :json_encoders: customise the way types are encoded to json, see :ref:`JSON Serialisation ` for more details. +:orm_mode: allows usage of :ref:`ORM mode ` .. warning:: diff --git a/pydantic/main.py b/pydantic/main.py index 34d4de1..d81433e 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -37,6 +37,7 @@ from .utils import ( AnyCallable, AnyType, ForwardRef, + GetterDict, change_exception, is_classvar, resolve_annotations, @@ -92,6 +93,7 @@ class BaseConfig: error_msg_templates: Dict[str, str] = {} arbitrary_types_allowed = False json_encoders: Dict[AnyType, AnyCallable] = {} + orm_mode: bool = False @classmethod def get_field_schema(cls, name: str) -> Dict[str, str]: @@ -362,6 +364,17 @@ class BaseModel(metaclass=MetaModel): obj = load_file(path, proto=proto, content_type=content_type, encoding=encoding, allow_pickle=allow_pickle) return cls.parse_obj(obj) + @classmethod + def from_orm(cls: Type['Model'], obj: Any) -> 'Model': + if not cls.__config__.orm_mode: + raise ConfigError('You must have the config attribute orm_mode=True to use from_orm') + obj = cls._decompose_class(obj) + m = cls.__new__(cls) + values, fields_set, _ = validate_model(m, obj) + object.__setattr__(m, '__values__', values) + object.__setattr__(m, '__fields_set__', fields_set) + return m + @classmethod def construct(cls: Type['Model'], values: 'DictAny', fields_set: 'SetStr') -> 'Model': """ @@ -430,14 +443,20 @@ class BaseModel(metaclass=MetaModel): yield cls.validate @classmethod - def validate(cls: Type['Model'], value: Union['DictStrAny', 'Model']) -> 'Model': + def validate(cls: Type['Model'], value: Any) -> 'Model': if isinstance(value, dict): return cls(**value) elif isinstance(value, cls): return value.copy() + elif cls.__config__.orm_mode: + return cls.from_orm(value) else: with change_exception(DictError, TypeError, ValueError): - return cls(**dict(value)) # type: ignore + return cls(**dict(value)) + + @classmethod + def _decompose_class(cls: Type['Model'], obj: Any) -> GetterDict: + return GetterDict(obj) @classmethod def _get_value(cls, v: Any, by_alias: bool, skip_defaults: bool) -> Any: diff --git a/pydantic/utils.py b/pydantic/utils.py index 4cf5380..01f630a 100644 --- a/pydantic/utils.py +++ b/pydantic/utils.py @@ -6,8 +6,21 @@ from enum import Enum from functools import lru_cache from importlib import import_module from textwrap import dedent -from typing import _eval_type # type: ignore -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Generator, List, Optional, Pattern, Tuple, Type, Union +from typing import ( # type: ignore + TYPE_CHECKING, + Any, + ClassVar, + Dict, + Generator, + List, + Optional, + Pattern, + Set, + Tuple, + Type, + Union, + _eval_type, +) import pydantic @@ -297,3 +310,23 @@ def almost_equal_floats(value_1: float, value_2: float, *, delta: float = 1e-8) Return True if two floats are almost equal """ return abs(value_1 - value_2) <= delta + + +class GetterDict: + """ + Hack to make object's smell just enough like dicts for validate_model. + """ + + __slots__ = ('_obj',) + + def __init__(self, obj: Any): + self._obj = obj + + def get(self, item: Any, default: Any) -> Any: + return getattr(self._obj, item, default) + + def keys(self) -> Set[Any]: + """ + We don't want to get any other attributes of obj if the model didn't explicitly ask for them + """ + return set() diff --git a/setup.py b/setup.py index a3e1a22..44c2cac 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ except FileNotFoundError: version = SourceFileLoader('version', 'pydantic/version.py').load_module() ext_modules = None -if 'clean' not in sys.argv and 'SKIP_CYTHON' not in os.environ: +if not any(arg in sys.argv for arg in ['clean', 'check']) and 'SKIP_CYTHON' not in os.environ: try: from Cython.Build import cythonize except ImportError: diff --git a/tests/test_orm_mode.py b/tests/test_orm_mode.py new file mode 100644 index 0000000..c18a791 --- /dev/null +++ b/tests/test_orm_mode.py @@ -0,0 +1,168 @@ +from typing import List + +import pytest + +from pydantic import BaseModel, ConfigError, ValidationError +from pydantic.utils import GetterDict + + +def test_getdict(): + class TestCls: + a = 1 + b: int + + def __init__(self): + self.c = 3 + + @property + def d(self): + return 4 + + def __getattr__(self, key): + if key == 'e': + return 5 + else: + raise AttributeError() + + t = TestCls() + gd = GetterDict(t) + assert gd.keys() == set() + assert gd.get('a', None) == 1 + assert gd.get('b', None) is None + assert gd.get('b', 1234) == 1234 + assert gd.get('c', None) == 3 + assert gd.get('d', None) == 4 + assert gd.get('e', None) == 5 + assert gd.get('f', 'missing') == 'missing' + + +def test_orm_mode(): + class PetCls: + def __init__(self, *, name: str, species: str): + self.name = name + self.species = species + + class PersonCls: + def __init__(self, *, name: str, age: float = None, pets: List[PetCls]): + self.name = name + self.age = age + self.pets = pets + + class Pet(BaseModel): + name: str + species: str + + class Config: + orm_mode = True + + class Person(BaseModel): + name: str + age: float = None + pets: List[Pet] + + class Config: + orm_mode = True + + bones = PetCls(name='Bones', species='dog') + orion = PetCls(name='Orion', species='cat') + anna = PersonCls(name='Anna', age=20, pets=[bones, orion]) + + anna_model = Person.from_orm(anna) + + assert anna_model.dict() == { + 'name': 'Anna', + 'pets': [{'name': 'Bones', 'species': 'dog'}, {'name': 'Orion', 'species': 'cat'}], + 'age': 20.0, + } + + +def test_not_orm_mode(): + class Pet(BaseModel): + name: str + species: str + + with pytest.raises(ConfigError): + Pet.from_orm(None) + + +def test_object_with_getattr(): + class FooGetAttr: + def __getattr__(self, key: str): + if key == 'foo': + return 'Foo' + else: + raise AttributeError + + class Model(BaseModel): + foo: str + bar: int = 1 + + class Config: + orm_mode = True + + class ModelInvalid(BaseModel): + foo: str + bar: int + + class Config: + orm_mode = True + + foo = FooGetAttr() + model = Model.from_orm(foo) + assert model.foo == 'Foo' + assert model.bar == 1 + assert model.dict(skip_defaults=True) == {'foo': 'Foo'} + with pytest.raises(ValidationError): + ModelInvalid.from_orm(foo) + + +def test_properties(): + class XyProperty: + x = 4 + + @property + def y(self): + return '5' + + class Model(BaseModel): + x: int + y: int + + class Config: + orm_mode = True + + model = Model.from_orm(XyProperty()) + assert model.x == 4 + assert model.y == 5 + + +def test_extra_allow(): + class TestCls: + x = 1 + y = 2 + + class Model(BaseModel): + x: int + + class Config: + orm_mode = True + extra = 'allow' + + model = Model.from_orm(TestCls()) + assert model.dict() == {'x': 1} + + +def test_extra_forbid(): + class TestCls: + x = 1 + y = 2 + + class Model(BaseModel): + x: int + + class Config: + orm_mode = True + extra = 'forbid' + + model = Model.from_orm(TestCls()) + assert model.dict() == {'x': 1}