From 6d2e6b6d6027caaec6d041f92ab1e76366013c24 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Fri, 5 May 2017 12:10:23 +0100 Subject: [PATCH] linting and adding .travis.yml --- .gitignore | 2 + .travis.yml | 38 +++++++++++++++ README.rst | 8 ++++ pydantic/fields.py | 74 +++++++++++++++-------------- pydantic/main.py | 9 ++-- pydantic/settings.py | 70 +++++++-------------------- pydantic/types.py | 37 ++++++++++++--- pydantic/{utils/dsn.py => utils.py} | 28 +++++++---- pydantic/utils/__init__.py | 0 pydantic/utils/python.py | 15 ------ setup.py | 5 +- tests/check_tag.py | 2 +- 12 files changed, 158 insertions(+), 130 deletions(-) create mode 100644 .travis.yml rename pydantic/{utils/dsn.py => utils.py} (59%) delete mode 100644 pydantic/utils/__init__.py delete mode 100644 pydantic/utils/python.py diff --git a/.gitignore b/.gitignore index f70490b..ed93c99 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ env/ *.egg-info/ build/ dist/ +.cache/ +test.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..97ed286 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,38 @@ +language: python + +cache: pip + +services: +- postgresql + +python: +- '3.5' +- '3.6' +- 'nightly' # currently 3.7 + +matrix: + allow_failures: + - python: 'nightly' + +install: +- make install +- pip freeze + +script: +- make lint +#- make test +- ./tests/check_tag.py + +after_success: +- ls -lha +#- bash <(curl -s https://codecov.io/bash) + +deploy: + provider: pypi + user: samuelcolvin + password: + secure: QbXFF2puEWjhFUpD0yu2R+wP4QI1IKIomBkMizsiCyMutlexERElranyYB8bsakvjPaJ+zU14ufffh2u7UA7Zhep/iE4skRHq4XWxnnRLHGu5nyGf3+zSM3F9MOzV32eZ4CDLJtFb6I0ensjTpodJH2EsIYHYxTgndIZn56Qbh6CStj7Xg1zm0Ujxdzm4ZLgcS28SOF/tpjsDW9+GXwc6L1mAZWYiS98gVgzL1vBd9tL9uFbbuFwGz9uhFMzFJko7vXSl8urWB4qeCspKXa9iKH7/AOYSwXTCwcg8U2hhC9UsOapnga2BubZKlU5HRfSs9fQcpnzcP2lwhSmkrEFa8VOw83hX6+bL564xK1Q4kanfGZ1fLU4FYge3iOnqjH7ajO7xEcUrcOEYUPfxM4EfdiDw0xnAzE1ITGH1/pZikF+wjlu+ez7RmmnejgK7quT1WU7keo7pSlRSfQtNgNl6xu818x0xZ1TScfN6e9npNy4TYyIooMOOeI4tMdfcR4JClkjGKhAtBk81DH7isZgPv3uwocGnKZ2S7La97CE3ADzU3MTA9xVIOSOjzwuvAe72uS2nwzqXkS9KATdATkC9QCvheJ9jIBB4UcqnHbD8L1gkqdmZwXZqHZldq8wcqNYZb+81lumy5EZ6xSoEzlLDpXHe80EjMUOBkb5fz3D44s= + distributions: sdist bdist_wheel + on: + tags: true + python: 3.6 diff --git a/README.rst b/README.rst index c67aa9f..1eda9a0 100644 --- a/README.rst +++ b/README.rst @@ -2,3 +2,11 @@ pydantic ======== Data validation and settings management using python 3.6 type hinting + + +TODO: +* testing +* fields as values +* sub instances +* datetime types +* exotic typing: Union, List, Dict diff --git a/pydantic/fields.py b/pydantic/fields.py index c7ea74e..a3a0b58 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -1,11 +1,11 @@ +import inspect from collections import OrderedDict -from functools import partial, wraps -from inspect import signature +from enum import IntEnum from pathlib import Path -from typing import Any, Callable, Dict, List, Type +from typing import Any, Callable, List, Type -def str_validator(v) -> str: # TODO config +def str_validator(v) -> str: if isinstance(v, str): return v elif isinstance(v, bytes): @@ -36,21 +36,21 @@ def bool_validator(v) -> bool: return bool(v) -def number_size_validator(v, *, config): - if config.min_number_size <= v <= config.max_number_size: - raise ValueError(f'size not in range {config.min_number_size} to {config.max_number_size}') +def number_size_validator(v, m): + if m.config.min_number_size <= v <= m.config.max_number_size: + raise ValueError(f'size not in range {m.config.min_number_size} to {m.config.max_number_size}') return v -def anystr_length_validator(v, *, config): - if config.max_anystr_length <= len(v) <= config.max_anystr_length: - raise ValueError(f'length not in range {config.max_anystr_length} to {config.max_anystr_length}') +def anystr_length_validator(v, *, m): + if m.config.max_anystr_length <= len(v) <= m.config.max_anystr_length: + raise ValueError(f'length not in range {m.config.max_anystr_length} to {m.config.max_anystr_length}') return v class ValidatorsLookup: def __init__(self): - self._validators_lookup: Dict[Type, List[Callable]] = { + self._validators_lookup = { int: [int, number_size_validator], float: [float, number_size_validator], Path: [Path], @@ -74,16 +74,9 @@ class ValidatorsLookup: validators_lookup = ValidatorsLookup() -def wrap_validator(func, config): - multi = False - try: - multi = len(signature(func).parameters) > 1 - except ValueError: - # happens on builtins like float - pass - if multi: - return wraps(func)(partial(func, config=config)) - return func +class ValidatorSignature(IntEnum): + JUST_VALUE = 1 + VALUE_MODEL = 2 class Field: @@ -108,7 +101,7 @@ class Field: self.name = name self.description = description - def prepare(self, name, config, class_validators): + def prepare(self, name, class_validators): self.name = self.name or name if self.default and self.type_ is None: self.type_ = type(self.default) @@ -126,12 +119,12 @@ class Field: self.validators.append(class_validators.get(f'validate_{self.name}')) self.validators.append(class_validators.get(f'validate_{self.name}_post')) - self.validators = tuple(wrap_validator(v, config) for v in self.validators if v) + self.validators = tuple(self._process_validator(v) for v in self.validators if v) self.info = OrderedDict([ ('type', self.type_.__name__), ('default', self.default), ('required', self.required), - ('validators', [f.__qualname__ for f in self.validators]) + ('validators', [f[1].__qualname__ for f in self.validators]) ]) if self.required: self.info.pop('default') @@ -144,20 +137,37 @@ class Field: return list(get_validators()) return validators_lookup.find(self.type_) - def validate(self, v): - for validator in self.validators: - v = validator(v) + @classmethod + def _process_validator(cls, validator): + try: + signature = inspect.signature(validator) + except ValueError: + # happens on builtins like float + return ValidatorSignature.JUST_VALUE, validator + + try: + return ValidatorSignature(len(signature.parameters)), validator + except ValueError as e: + raise RuntimeError(f'Invalid signature for validator {validator}: {signature}, should be: ' + f'(value), (value, model)') from e + + def validate(self, v, model): + for signature, validator in self.validators: + if signature == ValidatorSignature.JUST_VALUE: + v = validator(v) + else: + v = validator(v, model) return v @classmethod - def infer(cls, *, name, value, annotation, config, class_validators): + def infer(cls, *, name, value, annotation, class_validators): required = value == Ellipsis instance = cls( type_=annotation, default=None if required else value, required=required ) - instance.prepare(name, config, class_validators) + instance.prepare(name, class_validators) return instance def __repr__(self): @@ -165,9 +175,3 @@ class Field: def __str__(self): return ', '.join(f'{k}={v!r}' for k, v in self.info.items()) - - -class EnvField(Field): - def __init__(self, *, env=None, **kwargs): - super().__init__(**kwargs) - self.env_var_name = env diff --git a/pydantic/main.py b/pydantic/main.py index 03e82fd..0c7da96 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -1,9 +1,9 @@ import json -from types import FunctionType from collections import OrderedDict, namedtuple +from types import FunctionType from typing import Any, Dict -from pydantic.fields import Field +from .fields import Field DEFAULT_CONFIG: Dict[str, Any] = dict( @@ -50,7 +50,6 @@ class MetaModel(type): name=var_name, value=value, annotation=annotations.get(var_name), - config=config, class_validators=class_validators, ) fields[field.name] = field @@ -80,8 +79,8 @@ class BaseModel(metaclass=MetaModel): errors[name] = {'type': 'Missing', 'msg': 'field required'} continue try: - value = field.validate(value) - except (ValueError, TypeError) as e: + value = field.validate(value, self) + except (ValueError, TypeError, ImportError) as e: errors[name] = {'type': e.__class__.__name__, 'msg': str(e)} else: self.__values__[name] = value diff --git a/pydantic/settings.py b/pydantic/settings.py index 2590883..83f98f2 100644 --- a/pydantic/settings.py +++ b/pydantic/settings.py @@ -1,68 +1,30 @@ +import os from .main import BaseModel class BaseSettings(BaseModel): """ - Base class for settings, any setting defined on inheriting classes here can be overridden by: + Base class for settings, allowing values to be overridden by environment variables. - Setting the appropriate environment variable, eg. to override FOOBAR, `export APP_FOOBAR="whatever"`. - This is useful in production for secrets you do not wish to save in code and - also plays nicely with docker(-compose). Settings will attempt to convert environment variables to match the - type of the value here. + Environment variables must be upper case. Eg. to override foobar, `export APP_FOOBAR="whatever"`. - Or, passing the custom setting as a keyword argument when initialising settings (useful when testing) + This is useful in production for secrets you do not wish to save in code, it places nicely with docker(-compose), + Heroku and any 12 factor app design. """ - _ENV_PREFIX = 'APP_' - DB_DATABASE = None - DB_USER = None - DB_PASSWORD = None - DB_HOST = 'localhost' - DB_PORT = '5432' - DB_DRIVER = 'postgres' + def __init__(self, **values): + values.update(self._substitute_environ()) + super().__init__(**values) - def _substitute_environ(self, custom_settings): + def _substitute_environ(self): """ - Substitute environment variables into settings. + Substitute environment variables into values. """ + env_prefix = getattr(self.config, 'env_prefix', 'APP_') d = {} - for attr_name in dir(self): - if attr_name.startswith('_') or attr_name.upper() != attr_name: - continue - - orig_value = getattr(self, attr_name) - - if isinstance(orig_value, Setting): - is_required = orig_value.required - default = orig_value.default - orig_type = orig_value.v_type - env_var_name = orig_value.env_var_name - else: - default = orig_value - is_required = False - orig_type = type(orig_value) - env_var_name = self._ENV_PREFIX + attr_name - - env_var = os.getenv(env_var_name, None) - d[attr_name] = default - - if env_var is not None: - if issubclass(orig_type, bool): - env_var = env_var.upper() in ('1', 'TRUE') - elif issubclass(orig_type, int): - env_var = int(env_var) - elif issubclass(orig_type, Path): - env_var = Path(env_var) - elif issubclass(orig_type, bytes): - env_var = env_var.encode() - elif issubclass(orig_type, str) and env_var.startswith('py::'): - env_var = self._import_string(env_var[4:]) - elif issubclass(orig_type, (list, tuple, dict)): - # TODO more checks and validation - env_var = json.loads(env_var) - d[attr_name] = env_var - elif is_required and attr_name not in custom_settings: - raise RuntimeError('The required environment variable "{0}" is currently not set, ' - 'you\'ll need to set the environment variable with ' - '`export {0}=""`'.format(env_var_name)) + for name, field in self.__fields__.items(): + env_name = env_prefix + field.name.upper() + env_var = os.getenv(env_name, None) + if env_var: + d[name] = env_var return d diff --git a/pydantic/types.py b/pydantic/types.py index d4d58d7..1876024 100644 --- a/pydantic/types.py +++ b/pydantic/types.py @@ -1,11 +1,7 @@ -""" -json -JsonList -JsonDict -""" from typing import Type -from pydantic.fields import str_validator +from .fields import str_validator +from .utils import import_string, make_dsn class ConstrainedStr(str): @@ -42,3 +38,32 @@ def constr(*, min_length=0, max_length=2**16, curtail_length=None) -> Type[str]: ) return type('ConstrainedStrValue', (ConstrainedStr,), namespace) + +class Module: + @classmethod + def get_validators(cls): + yield str_validator + yield cls.validate + + @classmethod + def validate(cls, value): + return import_string(value) + + +class DSN(str): + prefix = 'db_' + + @classmethod + def get_validators(cls): + yield str_validator + yield cls.validate + + @classmethod + def validate(cls, value, model): + if value: + return value + d = model.__values__ + return make_dsn(**{f: d[cls.prefix + f] for f in ('name', 'password', 'host', 'port', 'user', 'driver')}) + + +# TODO, JsonEither, JsonList, JsonDict diff --git a/pydantic/utils/dsn.py b/pydantic/utils.py similarity index 59% rename from pydantic/utils/dsn.py rename to pydantic/utils.py index e0298a0..171c396 100644 --- a/pydantic/utils/dsn.py +++ b/pydantic/utils.py @@ -1,20 +1,11 @@ import re - -from .settings import BaseSettings +from importlib import import_module def _rfc_1738_quote(text): return re.sub(r'[:@/]', lambda m: '%{:X}'.format(ord(m.group(0))), text) -def make_settings_dsn(settings: BaseSettings, prefix='DB'): - kwargs = { - f: settings.dict['{}_{}'.format(prefix, f.upper())] - for f in ('name', 'password', 'host', 'port', 'user', 'driver') - } - return make_dsn(**kwargs) - - def make_dsn( driver: str = None, user: str = None, @@ -49,3 +40,20 @@ def make_dsn( keys.sort() s += '?' + '&'.join('{}={}'.format(k, query[k]) for k in keys) return s + + +def import_string(dotted_path): + """ + Stolen approximately from django. Import a dotted module path and return the attribute/class designated by the + last name in the path. Raise ImportError if the import failed. + """ + try: + module_path, class_name = dotted_path.strip(' ').rsplit('.', 1) + except ValueError as e: + raise ImportError("{} doesn't look like a module path".format(dotted_path)) from e + + module = import_module(module_path) + try: + return getattr(module, class_name) + except AttributeError as e: + raise ImportError('Module "{}" does not define a "{}" attribute'.format(module_path, class_name)) from e diff --git a/pydantic/utils/__init__.py b/pydantic/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pydantic/utils/python.py b/pydantic/utils/python.py deleted file mode 100644 index 76517b2..0000000 --- a/pydantic/utils/python.py +++ /dev/null @@ -1,15 +0,0 @@ -def _import_string(cls, dotted_path): - """ - Stolen from django. Import a dotted module path and return the attribute/class designated by the - last name in the path. Raise ImportError if the import failed. - """ - try: - module_path, class_name = dotted_path.strip(' ').rsplit('.', 1) - except ValueError as e: - raise ImportError("{} doesn't look like a module path".format(dotted_path)) from e - - module = import_module(module_path) - try: - return getattr(module, class_name) - except AttributeError as e: - raise ImportError('Module "{}" does not define a "{}" attribute'.format(module_path, class_name)) from e diff --git a/setup.py b/setup.py index 007af64..808199e 100644 --- a/setup.py +++ b/setup.py @@ -33,9 +33,6 @@ setup( author_email='s@muelcolvin.com', url='https://github.com/samuelcolvin/pydantic', license='MIT', - packages=[ - 'pydantic', - 'pydantic.utils', - ], + packages=['pydantic'], zip_safe=True, ) diff --git a/tests/check_tag.py b/tests/check_tag.py index f7983b7..da16dd4 100755 --- a/tests/check_tag.py +++ b/tests/check_tag.py @@ -2,7 +2,7 @@ import os import sys -from aiohttp_prodtools.version import VERSION +from pydantic.version import VERSION git_tag = os.getenv('TRAVIS_TAG') if git_tag: