ORM mode: Add support for arbitrary class instances (#562)

* Support ORM objects to 'parse_obj', replace #520

* switch to GetterDict and orm_mode

* tweaks

* update docs

* split tests and add @tiangolo's suggestion

* split tests and add @tiangolo's suggestion

* fix coverage

.gitignore

@@ -21,3 +21,4 @@ _build/
 pydantic/*.c
 pydantic/*.so
 .auto-format
+/sandbox/

HISTORY.rst

@@ -8,6 +8,7 @@ v0.28 (unreleased)
 * fix support for JSON Schema generation when using models with circular references in Python 3.7, #572 by @tiangolo
 * support ``__post_init_post_parse__`` on dataclasses, #567 by @sevaho
 * allow dumping dataclasses to JSON, #575 by @samuelcolvin and @DanielOberg
+* ORM mode, #562 by @samuelcolvin
 * fix ``pydantic.compiled`` on ipython, #573 by @dmontagu and @samuelcolvin
 
 v0.27 (2019-05-30)

docs/examples/orm_mode.py

@@ -0,0 +1,30 @@
+from typing import List
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.dialects.postgresql import ARRAY
+from sqlalchemy.ext.declarative import declarative_base
+from pydantic import BaseModel, constr
+
+Base = declarative_base()
+
+class CompanyOrm(Base):
+    __tablename__ = 'companies'
+    id = Column(Integer, primary_key=True, nullable=False)
+    public_key = Column(String(20), index=True, nullable=False, unique=True)
+    name = Column(String(63), unique=True)
+    domains = Column(ARRAY(String(255)))
+
+class CompanyModel(BaseModel):
+    id: int
+    public_key: constr(max_length=20)
+    name: constr(max_length=63)
+    domains: List[constr(max_length=255)]
+
+    class Config:
+        orm_mode = True
+
+co_orm = CompanyOrm(id=123, public_key='foobar', name='Testing', domains=['example.com', 'foobar.com'])
+print(co_orm)
+#> <__main__.CompanyOrm object at 0x7ff4bf918278>
+co_model = CompanyModel.from_orm(co_orm)
+print(co_model)
+#> CompanyModel id=123 public_key='foobar' name='Testing' domains=['example.com', 'foobar.com']

docs/examples/orm_mode_recursive.py

@@ -0,0 +1,35 @@
+from typing import List
+from pydantic import BaseModel
+
+class PetCls:
+    def __init__(self, *, name: str, species: str):
+        self.name = name
+        self.species = species
+
+class PersonCls:
+    def __init__(self, *, name: str, age: float = None, pets: List[PetCls]):
+        self.name = name
+        self.age = age
+        self.pets = pets
+
+class Pet(BaseModel):
+    name: str
+    species: str
+
+    class Config:
+        orm_mode = True
+
+class Person(BaseModel):
+    name: str
+    age: float = None
+    pets: List[Pet]
+
+    class Config:
+        orm_mode = True
+
+bones = PetCls(name='Bones', species='dog')
+orion = PetCls(name='Orion', species='cat')
+anna = PersonCls(name='Anna', age=20, pets=[bones, orion])
+anna_model = Person.from_orm(anna)
+print(anna_model)
+#> Person name='Anna' pets=[<Pet name='Bones' species='dog'>, <Pet name='Orion' species='cat'>] age=20.0

docs/index.rst

@@ -295,6 +295,31 @@ and pydantic versions).
 
 (This script is complete, it should run "as is")
 
+.. _orm_mode:
+
+ORM Mode (aka Arbitrary Class Instances)
+........................................
+
+Pydantic models can be created from arbitrary class instances to support models that map to ORM objects.
+
+To do this:
+1. The :ref:`Config <config>` property ``orm_mode`` must be set to ``True``.
+2. The special constructor ``from_orm`` must be used to create the model instance.
+
+The example here uses SQLAlchemy but the same approach should work for any ORM.
+
+.. literalinclude:: examples/orm_mode.py
+
+(This script is complete, it should run "as is")
+
+ORM instances will be parsed with ``from_orm`` recursively as well as at the top level.
+
+Here a vanilla class is used to demonstrate the principle, but any ORM could be used instead.
+
+.. literalinclude:: examples/orm_mode_recursive.py
+
+(This script is complete, it should run "as is")
+
 .. _schema:
 
 Schema Creation
@@ -642,6 +667,7 @@ Options:
     value is instance of that type). If False - RuntimeError will be raised on model declaration (default: ``False``)
 :json_encoders: customise the way types are encoded to json, see :ref:`JSON Serialisation <json_dump>` for more
     details.
+:orm_mode: allows usage of :ref:`ORM mode <orm_mode>`
 
 .. warning::
 

pydantic/main.py

@@ -37,6 +37,7 @@ from .utils import (
     AnyCallable,
     AnyType,
     ForwardRef,
+    GetterDict,
     change_exception,
     is_classvar,
     resolve_annotations,
@@ -92,6 +93,7 @@ class BaseConfig:
     error_msg_templates: Dict[str, str] = {}
     arbitrary_types_allowed = False
     json_encoders: Dict[AnyType, AnyCallable] = {}
+    orm_mode: bool = False
 
     @classmethod
     def get_field_schema(cls, name: str) -> Dict[str, str]:
@@ -362,6 +364,17 @@ class BaseModel(metaclass=MetaModel):
         obj = load_file(path, proto=proto, content_type=content_type, encoding=encoding, allow_pickle=allow_pickle)
         return cls.parse_obj(obj)
 
+    @classmethod
+    def from_orm(cls: Type['Model'], obj: Any) -> 'Model':
+        if not cls.__config__.orm_mode:
+            raise ConfigError('You must have the config attribute orm_mode=True to use from_orm')
+        obj = cls._decompose_class(obj)
+        m = cls.__new__(cls)
+        values, fields_set, _ = validate_model(m, obj)
+        object.__setattr__(m, '__values__', values)
+        object.__setattr__(m, '__fields_set__', fields_set)
+        return m
+
     @classmethod
     def construct(cls: Type['Model'], values: 'DictAny', fields_set: 'SetStr') -> 'Model':
         """
@@ -430,14 +443,20 @@ class BaseModel(metaclass=MetaModel):
         yield cls.validate
 
     @classmethod
-    def validate(cls: Type['Model'], value: Union['DictStrAny', 'Model']) -> 'Model':
+    def validate(cls: Type['Model'], value: Any) -> 'Model':
         if isinstance(value, dict):
             return cls(**value)
         elif isinstance(value, cls):
             return value.copy()
+        elif cls.__config__.orm_mode:
+            return cls.from_orm(value)
         else:
             with change_exception(DictError, TypeError, ValueError):
-                return cls(**dict(value))  # type: ignore
+                return cls(**dict(value))
+
+    @classmethod
+    def _decompose_class(cls: Type['Model'], obj: Any) -> GetterDict:
+        return GetterDict(obj)
 
     @classmethod
     def _get_value(cls, v: Any, by_alias: bool, skip_defaults: bool) -> Any:

pydantic/utils.py

@@ -6,8 +6,21 @@ from enum import Enum
 from functools import lru_cache
 from importlib import import_module
 from textwrap import dedent
-from typing import _eval_type  # type: ignore
-from typing import TYPE_CHECKING, Any, ClassVar, Dict, Generator, List, Optional, Pattern, Tuple, Type, Union
+from typing import (  # type: ignore
+    TYPE_CHECKING,
+    Any,
+    ClassVar,
+    Dict,
+    Generator,
+    List,
+    Optional,
+    Pattern,
+    Set,
+    Tuple,
+    Type,
+    Union,
+    _eval_type,
+)
 
 import pydantic
 
@@ -297,3 +310,23 @@ def almost_equal_floats(value_1: float, value_2: float, *, delta: float = 1e-8)
     Return True if two floats are almost equal
     """
     return abs(value_1 - value_2) <= delta
+
+
+class GetterDict:
+    """
+    Hack to make object's smell just enough like dicts for validate_model.
+    """
+
+    __slots__ = ('_obj',)
+
+    def __init__(self, obj: Any):
+        self._obj = obj
+
+    def get(self, item: Any, default: Any) -> Any:
+        return getattr(self._obj, item, default)
+
+    def keys(self) -> Set[Any]:
+        """
+        We don't want to get any other attributes of obj if the model didn't explicitly ask for them
+        """
+        return set()

setup.py

@@ -44,7 +44,7 @@ except FileNotFoundError:
 version = SourceFileLoader('version', 'pydantic/version.py').load_module()
 
 ext_modules = None
-if 'clean' not in sys.argv and 'SKIP_CYTHON' not in os.environ:
+if not any(arg in sys.argv for arg in ['clean', 'check']) and 'SKIP_CYTHON' not in os.environ:
     try:
         from Cython.Build import cythonize
     except ImportError:

tests/test_orm_mode.py

@@ -0,0 +1,168 @@
+from typing import List
+
+import pytest
+
+from pydantic import BaseModel, ConfigError, ValidationError
+from pydantic.utils import GetterDict
+
+
+def test_getdict():
+    class TestCls:
+        a = 1
+        b: int
+
+        def __init__(self):
+            self.c = 3
+
+        @property
+        def d(self):
+            return 4
+
+        def __getattr__(self, key):
+            if key == 'e':
+                return 5
+            else:
+                raise AttributeError()
+
+    t = TestCls()
+    gd = GetterDict(t)
+    assert gd.keys() == set()
+    assert gd.get('a', None) == 1
+    assert gd.get('b', None) is None
+    assert gd.get('b', 1234) == 1234
+    assert gd.get('c', None) == 3
+    assert gd.get('d', None) == 4
+    assert gd.get('e', None) == 5
+    assert gd.get('f', 'missing') == 'missing'
+
+
+def test_orm_mode():
+    class PetCls:
+        def __init__(self, *, name: str, species: str):
+            self.name = name
+            self.species = species
+
+    class PersonCls:
+        def __init__(self, *, name: str, age: float = None, pets: List[PetCls]):
+            self.name = name
+            self.age = age
+            self.pets = pets
+
+    class Pet(BaseModel):
+        name: str
+        species: str
+
+        class Config:
+            orm_mode = True
+
+    class Person(BaseModel):
+        name: str
+        age: float = None
+        pets: List[Pet]
+
+        class Config:
+            orm_mode = True
+
+    bones = PetCls(name='Bones', species='dog')
+    orion = PetCls(name='Orion', species='cat')
+    anna = PersonCls(name='Anna', age=20, pets=[bones, orion])
+
+    anna_model = Person.from_orm(anna)
+
+    assert anna_model.dict() == {
+        'name': 'Anna',
+        'pets': [{'name': 'Bones', 'species': 'dog'}, {'name': 'Orion', 'species': 'cat'}],
+        'age': 20.0,
+    }
+
+
+def test_not_orm_mode():
+    class Pet(BaseModel):
+        name: str
+        species: str
+
+    with pytest.raises(ConfigError):
+        Pet.from_orm(None)
+
+
+def test_object_with_getattr():
+    class FooGetAttr:
+        def __getattr__(self, key: str):
+            if key == 'foo':
+                return 'Foo'
+            else:
+                raise AttributeError
+
+    class Model(BaseModel):
+        foo: str
+        bar: int = 1
+
+        class Config:
+            orm_mode = True
+
+    class ModelInvalid(BaseModel):
+        foo: str
+        bar: int
+
+        class Config:
+            orm_mode = True
+
+    foo = FooGetAttr()
+    model = Model.from_orm(foo)
+    assert model.foo == 'Foo'
+    assert model.bar == 1
+    assert model.dict(skip_defaults=True) == {'foo': 'Foo'}
+    with pytest.raises(ValidationError):
+        ModelInvalid.from_orm(foo)
+
+
+def test_properties():
+    class XyProperty:
+        x = 4
+
+        @property
+        def y(self):
+            return '5'
+
+    class Model(BaseModel):
+        x: int
+        y: int
+
+        class Config:
+            orm_mode = True
+
+    model = Model.from_orm(XyProperty())
+    assert model.x == 4
+    assert model.y == 5
+
+
+def test_extra_allow():
+    class TestCls:
+        x = 1
+        y = 2
+
+    class Model(BaseModel):
+        x: int
+
+        class Config:
+            orm_mode = True
+            extra = 'allow'
+
+    model = Model.from_orm(TestCls())
+    assert model.dict() == {'x': 1}
+
+
+def test_extra_forbid():
+    class TestCls:
+        x = 1
+        y = 2
+
+    class Model(BaseModel):
+        x: int
+
+        class Config:
+            orm_mode = True
+            extra = 'forbid'
+
+    model = Model.from_orm(TestCls())
+    assert model.dict() == {'x': 1}