linting and adding .travis.yml

.gitignore

@@ -4,3 +4,5 @@ env/
 *.egg-info/
 build/
 dist/
+.cache/
+test.py

.travis.yml

@@ -0,0 +1,38 @@
+language: python
+
+cache: pip
+
+services:
+- postgresql
+
+python:
+- '3.5'
+- '3.6'
+- 'nightly'  # currently 3.7
+
+matrix:
+  allow_failures:
+  - python: 'nightly'
+
+install:
+- make install
+- pip freeze
+
+script:
+- make lint
+#- make test
+- ./tests/check_tag.py
+
+after_success:
+- ls -lha
+#- bash <(curl -s https://codecov.io/bash)
+
+deploy:
+  provider: pypi
+  user: samuelcolvin
+  password:
+    secure: QbXFF2puEWjhFUpD0yu2R+wP4QI1IKIomBkMizsiCyMutlexERElranyYB8bsakvjPaJ+zU14ufffh2u7UA7Zhep/iE4skRHq4XWxnnRLHGu5nyGf3+zSM3F9MOzV32eZ4CDLJtFb6I0ensjTpodJH2EsIYHYxTgndIZn56Qbh6CStj7Xg1zm0Ujxdzm4ZLgcS28SOF/tpjsDW9+GXwc6L1mAZWYiS98gVgzL1vBd9tL9uFbbuFwGz9uhFMzFJko7vXSl8urWB4qeCspKXa9iKH7/AOYSwXTCwcg8U2hhC9UsOapnga2BubZKlU5HRfSs9fQcpnzcP2lwhSmkrEFa8VOw83hX6+bL564xK1Q4kanfGZ1fLU4FYge3iOnqjH7ajO7xEcUrcOEYUPfxM4EfdiDw0xnAzE1ITGH1/pZikF+wjlu+ez7RmmnejgK7quT1WU7keo7pSlRSfQtNgNl6xu818x0xZ1TScfN6e9npNy4TYyIooMOOeI4tMdfcR4JClkjGKhAtBk81DH7isZgPv3uwocGnKZ2S7La97CE3ADzU3MTA9xVIOSOjzwuvAe72uS2nwzqXkS9KATdATkC9QCvheJ9jIBB4UcqnHbD8L1gkqdmZwXZqHZldq8wcqNYZb+81lumy5EZ6xSoEzlLDpXHe80EjMUOBkb5fz3D44s=
+  distributions: sdist bdist_wheel
+  on:
+    tags: true
+    python: 3.6

README.rst

@@ -2,3 +2,11 @@ pydantic
 ========
 
 Data validation and settings management using python 3.6 type hinting
+
+
+TODO:
+* testing
+* fields as values
+* sub instances
+* datetime types
+* exotic typing: Union, List, Dict

pydantic/fields.py

@@ -1,11 +1,11 @@
+import inspect
 from collections import OrderedDict
-from functools import partial, wraps
-from inspect import signature
+from enum import IntEnum
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Type
+from typing import Any, Callable, List, Type
 
 
-def str_validator(v) -> str:  # TODO config
+def str_validator(v) -> str:
     if isinstance(v, str):
         return v
     elif isinstance(v, bytes):
@@ -36,21 +36,21 @@ def bool_validator(v) -> bool:
     return bool(v)
 
 
-def number_size_validator(v, *, config):
-    if config.min_number_size <= v <= config.max_number_size:
-        raise ValueError(f'size not in range {config.min_number_size} to {config.max_number_size}')
+def number_size_validator(v, m):
+    if m.config.min_number_size <= v <= m.config.max_number_size:
+        raise ValueError(f'size not in range {m.config.min_number_size} to {m.config.max_number_size}')
     return v
 
 
-def anystr_length_validator(v, *, config):
-    if config.max_anystr_length <= len(v) <= config.max_anystr_length:
-        raise ValueError(f'length not in range {config.max_anystr_length} to {config.max_anystr_length}')
+def anystr_length_validator(v, *, m):
+    if m.config.max_anystr_length <= len(v) <= m.config.max_anystr_length:
+        raise ValueError(f'length not in range {m.config.max_anystr_length} to {m.config.max_anystr_length}')
     return v
 
 
 class ValidatorsLookup:
     def __init__(self):
-        self._validators_lookup: Dict[Type, List[Callable]] = {
+        self._validators_lookup = {
             int: [int, number_size_validator],
             float: [float, number_size_validator],
             Path: [Path],
@@ -74,16 +74,9 @@ class ValidatorsLookup:
 validators_lookup = ValidatorsLookup()
 
 
-def wrap_validator(func, config):
-    multi = False
-    try:
-        multi = len(signature(func).parameters) > 1
-    except ValueError:
-        # happens on builtins like float
-        pass
-    if multi:
-        return wraps(func)(partial(func, config=config))
-    return func
+class ValidatorSignature(IntEnum):
+    JUST_VALUE = 1
+    VALUE_MODEL = 2
 
 
 class Field:
@@ -108,7 +101,7 @@ class Field:
         self.name = name
         self.description = description
 
-    def prepare(self, name, config, class_validators):
+    def prepare(self, name, class_validators):
         self.name = self.name or name
         if self.default and self.type_ is None:
             self.type_ = type(self.default)
@@ -126,12 +119,12 @@ class Field:
         self.validators.append(class_validators.get(f'validate_{self.name}'))
         self.validators.append(class_validators.get(f'validate_{self.name}_post'))
 
-        self.validators = tuple(wrap_validator(v, config) for v in self.validators if v)
+        self.validators = tuple(self._process_validator(v) for v in self.validators if v)
         self.info = OrderedDict([
             ('type', self.type_.__name__),
             ('default', self.default),
             ('required', self.required),
-            ('validators', [f.__qualname__ for f in self.validators])
+            ('validators', [f[1].__qualname__ for f in self.validators])
         ])
         if self.required:
             self.info.pop('default')
@@ -144,20 +137,37 @@ class Field:
             return list(get_validators())
         return validators_lookup.find(self.type_)
 
-    def validate(self, v):
-        for validator in self.validators:
-            v = validator(v)
+    @classmethod
+    def _process_validator(cls, validator):
+        try:
+            signature = inspect.signature(validator)
+        except ValueError:
+            # happens on builtins like float
+            return ValidatorSignature.JUST_VALUE, validator
+
+        try:
+            return ValidatorSignature(len(signature.parameters)), validator
+        except ValueError as e:
+            raise RuntimeError(f'Invalid signature for validator {validator}: {signature}, should be: '
+                               f'(value), (value, model)') from e
+
+    def validate(self, v, model):
+        for signature, validator in self.validators:
+            if signature == ValidatorSignature.JUST_VALUE:
+                v = validator(v)
+            else:
+                v = validator(v, model)
         return v
 
     @classmethod
-    def infer(cls, *, name, value, annotation, config, class_validators):
+    def infer(cls, *, name, value, annotation, class_validators):
         required = value == Ellipsis
         instance = cls(
             type_=annotation,
             default=None if required else value,
             required=required
         )
-        instance.prepare(name, config, class_validators)
+        instance.prepare(name, class_validators)
         return instance
 
     def __repr__(self):
@@ -165,9 +175,3 @@ class Field:
 
     def __str__(self):
         return ', '.join(f'{k}={v!r}' for k, v in self.info.items())
-
-
-class EnvField(Field):
-    def __init__(self, *, env=None, **kwargs):
-        super().__init__(**kwargs)
-        self.env_var_name = env

pydantic/main.py

@@ -1,9 +1,9 @@
 import json
-from types import FunctionType
 from collections import OrderedDict, namedtuple
+from types import FunctionType
 from typing import Any, Dict
 
-from pydantic.fields import Field
+from .fields import Field
 
 
 DEFAULT_CONFIG: Dict[str, Any] = dict(
@@ -50,7 +50,6 @@ class MetaModel(type):
                 name=var_name,
                 value=value,
                 annotation=annotations.get(var_name),
-                config=config,
                 class_validators=class_validators,
             )
             fields[field.name] = field
@@ -80,8 +79,8 @@ class BaseModel(metaclass=MetaModel):
                     errors[name] = {'type': 'Missing', 'msg': 'field required'}
                 continue
             try:
-                value = field.validate(value)
-            except (ValueError, TypeError) as e:
+                value = field.validate(value, self)
+            except (ValueError, TypeError, ImportError) as e:
                 errors[name] = {'type': e.__class__.__name__, 'msg': str(e)}
             else:
                 self.__values__[name] = value

pydantic/settings.py

@@ -1,68 +1,30 @@
+import os
 from .main import BaseModel
 
 
 class BaseSettings(BaseModel):
     """
-    Base class for settings, any setting defined on inheriting classes here can be overridden by:
+    Base class for settings, allowing values to be overridden by environment variables.
 
-    Setting the appropriate environment variable, eg. to override FOOBAR, `export APP_FOOBAR="whatever"`.
-    This is useful in production for secrets you do not wish to save in code and
-    also plays nicely with docker(-compose). Settings will attempt to convert environment variables to match the
-    type of the value here.
+    Environment variables must be upper case. Eg. to override foobar, `export APP_FOOBAR="whatever"`.
 
-    Or, passing the custom setting as a keyword argument when initialising settings (useful when testing)
+    This is useful in production for secrets you do not wish to save in code, it places nicely with docker(-compose),
+    Heroku and any 12 factor app design.
     """
-    _ENV_PREFIX = 'APP_'
 
-    DB_DATABASE = None
-    DB_USER = None
-    DB_PASSWORD = None
-    DB_HOST = 'localhost'
-    DB_PORT = '5432'
-    DB_DRIVER = 'postgres'
+    def __init__(self, **values):
+        values.update(self._substitute_environ())
+        super().__init__(**values)
 
-    def _substitute_environ(self, custom_settings):
+    def _substitute_environ(self):
         """
-        Substitute environment variables into settings.
+        Substitute environment variables into values.
         """
+        env_prefix = getattr(self.config, 'env_prefix', 'APP_')
         d = {}
-        for attr_name in dir(self):
-            if attr_name.startswith('_') or attr_name.upper() != attr_name:
-                continue
-
-            orig_value = getattr(self, attr_name)
-
-            if isinstance(orig_value, Setting):
-                is_required = orig_value.required
-                default = orig_value.default
-                orig_type = orig_value.v_type
-                env_var_name = orig_value.env_var_name
-            else:
-                default = orig_value
-                is_required = False
-                orig_type = type(orig_value)
-                env_var_name = self._ENV_PREFIX + attr_name
-
-            env_var = os.getenv(env_var_name, None)
-            d[attr_name] = default
-
-            if env_var is not None:
-                if issubclass(orig_type, bool):
-                    env_var = env_var.upper() in ('1', 'TRUE')
-                elif issubclass(orig_type, int):
-                    env_var = int(env_var)
-                elif issubclass(orig_type, Path):
-                    env_var = Path(env_var)
-                elif issubclass(orig_type, bytes):
-                    env_var = env_var.encode()
-                elif issubclass(orig_type, str) and env_var.startswith('py::'):
-                    env_var = self._import_string(env_var[4:])
-                elif issubclass(orig_type, (list, tuple, dict)):
-                    # TODO more checks and validation
-                    env_var = json.loads(env_var)
-                d[attr_name] = env_var
-            elif is_required and attr_name not in custom_settings:
-                raise RuntimeError('The required environment variable "{0}" is currently not set, '
-                                   'you\'ll need to set the environment variable with '
-                                   '`export {0}="<value>"`'.format(env_var_name))
+        for name, field in self.__fields__.items():
+            env_name = env_prefix + field.name.upper()
+            env_var = os.getenv(env_name, None)
+            if env_var:
+                d[name] = env_var
         return d

pydantic/types.py

@@ -1,11 +1,7 @@
-"""
-json
-JsonList
-JsonDict
-"""
 from typing import Type
 
-from pydantic.fields import str_validator
+from .fields import str_validator
+from .utils import import_string, make_dsn
 
 
 class ConstrainedStr(str):
@@ -42,3 +38,32 @@ def constr(*, min_length=0, max_length=2**16, curtail_length=None) -> Type[str]:
     )
     return type('ConstrainedStrValue', (ConstrainedStr,), namespace)
 
+
+class Module:
+    @classmethod
+    def get_validators(cls):
+        yield str_validator
+        yield cls.validate
+
+    @classmethod
+    def validate(cls, value):
+        return import_string(value)
+
+
+class DSN(str):
+    prefix = 'db_'
+
+    @classmethod
+    def get_validators(cls):
+        yield str_validator
+        yield cls.validate
+
+    @classmethod
+    def validate(cls, value, model):
+        if value:
+            return value
+        d = model.__values__
+        return make_dsn(**{f: d[cls.prefix + f] for f in ('name', 'password', 'host', 'port', 'user', 'driver')})
+
+
+# TODO, JsonEither, JsonList, JsonDict

pydantic/utils.py

@@ -1,20 +1,11 @@
 import re
-
-from .settings import BaseSettings
+from importlib import import_module
 
 
 def _rfc_1738_quote(text):
     return re.sub(r'[:@/]', lambda m: '%{:X}'.format(ord(m.group(0))), text)
 
 
-def make_settings_dsn(settings: BaseSettings, prefix='DB'):
-    kwargs = {
-        f: settings.dict['{}_{}'.format(prefix, f.upper())]
-        for f in ('name', 'password', 'host', 'port', 'user', 'driver')
-    }
-    return make_dsn(**kwargs)
-
-
 def make_dsn(
         driver: str = None,
         user: str = None,
@@ -49,3 +40,20 @@ def make_dsn(
         keys.sort()
         s += '?' + '&'.join('{}={}'.format(k, query[k]) for k in keys)
     return s
+
+
+def import_string(dotted_path):
+    """
+    Stolen approximately from django. Import a dotted module path and return the attribute/class designated by the
+    last name in the path. Raise ImportError if the import failed.
+    """
+    try:
+        module_path, class_name = dotted_path.strip(' ').rsplit('.', 1)
+    except ValueError as e:
+        raise ImportError("{} doesn't look like a module path".format(dotted_path)) from e
+
+    module = import_module(module_path)
+    try:
+        return getattr(module, class_name)
+    except AttributeError as e:
+        raise ImportError('Module "{}" does not define a "{}" attribute'.format(module_path, class_name)) from e

pydantic/utils/python.py

@@ -1,15 +0,0 @@
-def _import_string(cls, dotted_path):
-    """
-    Stolen from django. Import a dotted module path and return the attribute/class designated by the
-    last name in the path. Raise ImportError if the import failed.
-    """
-    try:
-        module_path, class_name = dotted_path.strip(' ').rsplit('.', 1)
-    except ValueError as e:
-        raise ImportError("{} doesn't look like a module path".format(dotted_path)) from e
-
-    module = import_module(module_path)
-    try:
-        return getattr(module, class_name)
-    except AttributeError as e:
-        raise ImportError('Module "{}" does not define a "{}" attribute'.format(module_path, class_name)) from e

setup.py

@@ -33,9 +33,6 @@ setup(
     author_email='s@muelcolvin.com',
     url='https://github.com/samuelcolvin/pydantic',
     license='MIT',
-    packages=[
-        'pydantic',
-        'pydantic.utils',
-    ],
+    packages=['pydantic'],
     zip_safe=True,
 )

tests/check_tag.py

@@ -2,7 +2,7 @@
 import os
 import sys
 
-from aiohttp_prodtools.version import VERSION
+from pydantic.version import VERSION
 
 git_tag = os.getenv('TRAVIS_TAG')
 if git_tag: