kennethreitz/pydantic
1
0
Fork 0
mirror of https://github.com/kennethreitz/pydantic.git synced 2026-06-05 23:00:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move settings to pydantic-settings (#4492)

Browse Source 
* Move settings to pydantic-settings

* fix docs, remove dotenv

* fix coverage

* removing unused test fixture
This commit is contained in:
Samuel Colvin
2022-09-07 13:05:51 +01:00
committed by GitHub
parent f341049b9e
commit 85e4596958
31 changed files with 55 additions and 2305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/examples/settings_add_custom_source.py
-41
View File
@@ -1,41 +0,0 @@
import json
from pathlib import Path
from typing import Dict, Any
from pydantic import BaseSettings
def json_config_settings_source(settings: BaseSettings) -> Dict[str, Any]:
"""
A simple settings source that loads variables from a JSON file
at the project's root.
Here we happen to choose to use the `env_file_encoding` from Config
when reading `config.json`
"""
encoding = settings.__config__.env_file_encoding
return json.loads(Path('config.json').read_text(encoding))
class Settings(BaseSettings):
foobar: str
class Config:
env_file_encoding = 'utf-8'
@classmethod
def customise_sources(
cls,
init_settings,
env_settings,
file_secret_settings,
):
return (
init_settings,
json_config_settings_source,
env_settings,
file_secret_settings,
)
print(Settings())
docs/examples/settings_case_sensitive.py
-8
View File
@@ -1,8 +0,0 @@
from pydantic import BaseSettings
class Settings(BaseSettings):
redis_host = 'localhost'
class Config:
case_sensitive = True
docs/examples/settings_disable_source.py
-23
View File
@@ -1,23 +0,0 @@
from typing import Tuple
from pydantic import BaseSettings
from pydantic.env_settings import SettingsSourceCallable
class Settings(BaseSettings):
my_api_key: str
class Config:
@classmethod
def customise_sources(
cls,
init_settings: SettingsSourceCallable,
env_settings: SettingsSourceCallable,
file_secret_settings: SettingsSourceCallable,
) -> Tuple[SettingsSourceCallable, ...]:
# here we choose to ignore arguments from init_settings
return env_settings, file_secret_settings
print(Settings(my_api_key='this is ignored'))
# requires: `MY_API_KEY` env variable to be set, e.g. `export MY_API_KEY=xxx`
docs/examples/settings_env_priority.py
-20
View File
@@ -1,20 +0,0 @@
from typing import Tuple
from pydantic import BaseSettings, PostgresDsn
from pydantic.env_settings import SettingsSourceCallable
class Settings(BaseSettings):
database_dsn: PostgresDsn
class Config:
@classmethod
def customise_sources(
cls,
init_settings: SettingsSourceCallable,
env_settings: SettingsSourceCallable,
file_secret_settings: SettingsSourceCallable,
) -> Tuple[SettingsSourceCallable, ...]:
return env_settings, init_settings, file_secret_settings
print(Settings(database_dsn='postgres://postgres@localhost:5432/kwargs_db'))
docs/examples/settings_main.py
-49
View File
@@ -1,49 +0,0 @@
from typing import Set
from pydantic import (
BaseModel,
BaseSettings,
PyObject,
RedisDsn,
PostgresDsn,
AmqpDsn,
Field,
)
class SubModel(BaseModel):
foo = 'bar'
apple = 1
class Settings(BaseSettings):
auth_key: str
api_key: str = Field(..., env='my_api_key')
redis_dsn: RedisDsn = 'redis://user:pass@localhost:6379/1'
pg_dsn: PostgresDsn = 'postgres://user:pass@localhost:5432/foobar'
amqp_dsn: AmqpDsn = 'amqp://user:pass@localhost:5672/'
special_function: PyObject = 'math.cos'
# to override domains:
# export my_prefix_domains='["foo.com", "bar.com"]'
domains: Set[str] = set()
# to override more_settings:
# export my_prefix_more_settings='{"foo": "x", "apple": 1}'
more_settings: SubModel = SubModel()
class Config:
env_prefix = 'my_prefix_' # defaults to no prefix, i.e. ""
fields = {
'auth_key': {
'env': 'my_auth_key',
},
'redis_dsn': {
'env': ['service_redis_dsn', 'redis_url']
}
}
print(Settings().dict())
docs/examples/settings_nested_env.py
-23
View File
@@ -1,23 +0,0 @@
from pydantic import BaseModel, BaseSettings
class DeepSubModel(BaseModel):
v4: str
class SubModel(BaseModel):
v1: str
v2: bytes
v3: int
deep: DeepSubModel
class Settings(BaseSettings):
v0: str
sub_model: SubModel
class Config:
env_nested_delimiter = '__'
print(Settings().dict())
docs/examples/settings_with_custom_parsing.py
-19
View File
@@ -1,19 +0,0 @@
import os
from typing import Any, List
from pydantic import BaseSettings
class Settings(BaseSettings):
numbers: List[int]
class Config:
@classmethod
def parse_env_var(cls, field_name: str, raw_val: str) -> Any:
if field_name == 'numbers':
return [int(x) for x in raw_val.split(',')]
return cls.json_loads(raw_val)
os.environ['numbers'] = '1,2,3'
print(Settings().dict())
docs/index.md
+1 -6
View File
@@ -7,7 +7,7 @@
{!.version.md!}
Data validation and settings management using Python type annotations.
Data validation using Python type annotations.
*pydantic* enforces type hints at runtime, and provides user friendly errors when data is invalid.
@@ -107,11 +107,6 @@ So *pydantic* uses some cool new language features, but why should I actually go
so auto-completion, linting, [mypy](usage/mypy.md), IDEs (especially [PyCharm](pycharm_plugin.md)),
and your intuition should all work properly with your validated data.
**dual use**
: *pydantic's* [BaseSettings](usage/settings.md) class allows *pydantic* to be used in both a "validate this request
data" context and in a "load my system settings" context. The main differences are that system settings can
be read from environment variables, and more complex objects like DSNs and Python objects are often required.
**fast**
: *pydantic* has always taken performance seriously, most of the library is compiled with cython giving a ~50% speedup,
it's generally as fast or faster than most similar libraries.
docs/install.md
+3 -10
View File
@@ -52,23 +52,16 @@ CFLAGS="-Os -g0 -s" pip install \
## Optional dependencies
*pydantic* has two optional dependencies:
*pydantic* has one optional dependencies:
* If you require email validation you can add [email-validator](https://github.com/JoshData/python-email-validator)
* [dotenv file support](usage/settings.md#dotenv-env-support) with `Settings` requires
[python-dotenv](https://pypi.org/project/python-dotenv)
To install these along with *pydantic*:
```bash
pip install pydantic[email]
# or
pip install pydantic[dotenv]
# or just
pip install pydantic[email,dotenv]
```
Of course, you can also install these requirements manually with `pip install email-validator` and/or `pip install python-dotenv`.
Of course, you can also install these requirements manually with `pip install email-validator`.
## Install from repository
@@ -76,5 +69,5 @@ And if you prefer to install *pydantic* directly from the repository:
```bash
pip install git+git://github.com/pydantic/pydantic@main#egg=pydantic
# or with extras
pip install git+git://github.com/pydantic/pydantic@main#egg=pydantic[email,dotenv]
pip install git+git://github.com/pydantic/pydantic@main#egg=pydantic[email]
```
docs/mypy_plugin.md
-27
View File
@@ -31,8 +31,6 @@ There are other benefits too! See below for more details.
keyword arguments.
* If `Config.allow_population_by_field_name=True`, the generated signature will use the field names,
rather than aliases.
* For subclasses of [`BaseSettings`](usage/settings.md), all fields are treated as optional since they may be
read from the environment.
* If `Config.extra="forbid"` and you don't make use of dynamically-determined aliases, the generated signature
will not allow unexpected inputs.
* **Optional:** If the [`init_forbid_extra` **plugin setting**](#plugin-settings) is set to `True`, unexpected inputs to
@@ -94,31 +92,6 @@ The plugin is compatible with mypy versions `>=0.930`.
See the [mypy usage](usage/mypy.md) and [plugin configuration](#configuring-the-plugin) docs for more details.
### Plugin Settings
The plugin offers a few optional strictness flags if you want even stronger checks:
* `init_forbid_extra`
If enabled, disallow extra arguments to the `__init__` call even when `Config.extra` is not `"forbid"`.
* `init_typed`
If enabled, include the field types as type hints in the generated signature for the `__init__` method.
This means that you'll get mypy errors if you pass an argument that is not already the right type to
`__init__`, even if parsing could safely convert the type.
* `warn_required_dynamic_aliases`
If enabled, raise a mypy error whenever a model is created for which
calls to its `__init__` or `construct` methods require the use of aliases that cannot be statically determined.
This is the case, for example, if `allow_population_by_field_name=False` and the model uses an alias generator.
* `warn_untyped_fields`
If enabled, raise a mypy error whenever a field is declared on a model without explicitly specifying its type.
#### Configuring the Plugin
To change the values of the plugin settings, create a section in your mypy config file called `[pydantic-mypy]`,
and add any key-value pairs for settings you want to override.
docs/usage/settings.md
-292
View File
@@ -1,292 +0,0 @@
One of pydantic's most useful applications is settings management.
If you create a model that inherits from `BaseSettings`, the model initialiser will attempt to determine
the values of any fields not passed as keyword arguments by reading from the environment. (Default values
will still be used if the matching environment variable is not set.)
This makes it easy to:
* Create a clearly-defined, type-hinted application configuration class
* Automatically read modifications to the configuration from environment variables
* Manually override specific settings in the initialiser where desired (e.g. in unit tests)
For example:
{!.tmp_examples/settings_main.md!}
## Environment variable names
The following rules are used to determine which environment variable(s) are read for a given field:
* By default, the environment variable name is built by concatenating the prefix and field name.
* For example, to override `special_function` above, you could use:
export my_prefix_special_function='foo.bar'
* Note 1: The default prefix is an empty string.
* Note 2: Field aliases are ignored when building the environment variable name.
* Custom environment variable names can be set in two ways:
* `Config.fields['field_name']['env']` (see `auth_key` and `redis_dsn` above)
* `Field(..., env=...)` (see `api_key` above)
* When specifying custom environment variable names, either a string or a list of strings may be provided.
* When specifying a list of strings, order matters: the first detected value is used.
* For example, for `redis_dsn` above, `service_redis_dsn` would take precedence over `redis_url`.
!!! warning
Since **v1.0** *pydantic* does not consider field aliases when finding environment variables to populate settings
models, use `env` instead as described above.
To aid the transition from aliases to `env`, a warning will be raised when aliases are used on settings models
without a custom env var name. If you really mean to use aliases, either ignore the warning or set `env` to
suppress it.
Case-sensitivity can be turned on through the `Config`:
{!.tmp_examples/settings_case_sensitive.md!}
When `case_sensitive` is `True`, the environment variable names must match field names (optionally with a prefix),
so in this example
`redis_host` could only be modified via `export redis_host`. If you want to name environment variables
all upper-case, you should name attribute all upper-case too. You can still name environment variables anything
you like through `Field(..., env=...)`.
In Pydantic **v1** `case_sensitive` is `False` by default and all variable names are converted to lower-case internally.
If you want to define upper-case variable names on nested models like `SubModel` you have to
set `case_sensitive=True` to disable this behaviour.
!!! note
On Windows, Python's `os` module always treats environment variables as case-insensitive, so the
`case_sensitive` config setting will have no effect - settings will always be updated ignoring case.
## Parsing environment variable values
For most simple field types (such as `int`, `float`, `str`, etc.),
the environment variable value is parsed the same way it would
be if passed directly to the initialiser (as a string).
Complex types like `list`, `set`, `dict`, and sub-models are populated from the environment
by treating the environment variable's value as a JSON-encoded string.
Another way to populate nested complex variables is to configure your model with the `env_nested_delimiter`
config setting, then use an env variable with a name pointing to the nested module fields.
What it does is simply explodes your variable into nested models or dicts.
So if you define a variable `FOO__BAR__BAZ=123` it will convert it into `FOO={'BAR': {'BAZ': 123}}`
If you have multiple variables with the same structure they will be merged.
With the following environment variables:
```bash
# your environment
export V0=0
export SUB_MODEL='{"v1": "json-1", "v2": "json-2"}'
export SUB_MODEL__V2=nested-2
export SUB_MODEL__V3=3
export SUB_MODEL__DEEP__V4=v4
```
You could load a settings module thus:
{!.tmp_examples/settings_nested_env.md!}
`env_nested_delimiter` can be configured via the `Config` class as shown above, or via the
`_env_nested_delimiter` keyword argument on instantiation.
JSON is only parsed in top-level fields, if you need to parse JSON in sub-models, you will need to implement
validators on those models.
Nested environment variables take precedence over the top-level environment variable JSON
(e.g. in the example above, `SUB_MODEL__V2` trumps `SUB_MODEL`).
You may also populate a complex type by providing your own parsing function to
the `parse_env_var` classmethod in the Config object.
{!.tmp_examples/settings_with_custom_parsing.md!}
## Dotenv (.env) support
!!! note
dotenv file parsing requires [python-dotenv](https://pypi.org/project/python-dotenv/) to be installed.
This can be done with either `pip install python-dotenv` or `pip install pydantic[dotenv]`.
Dotenv files (generally named `.env`) are a common pattern that make it easy to use environment variables in a
platform-independent manner.
A dotenv file follows the same general principles of all environment variables,
and looks something like:
```bash
# ignore comment
ENVIRONMENT="production"
REDIS_ADDRESS=localhost:6379
MEANING_OF_LIFE=42
MY_VAR='Hello world'
```
Once you have your `.env` file filled with variables, *pydantic* supports loading it in two ways:
**1.** setting `env_file` (and `env_file_encoding` if you don't want the default encoding of your OS) on `Config`
in a `BaseSettings` class:
```py
class Settings(BaseSettings):
...
class Config:
env_file = '.env'
env_file_encoding = 'utf-8'
```
**2.** instantiating a `BaseSettings` derived class with the `_env_file` keyword argument
(and the `_env_file_encoding` if needed):
```py
settings = Settings(_env_file='prod.env', _env_file_encoding='utf-8')
```
In either case, the value of the passed argument can be any valid path or filename, either absolute or relative to the
current working directory. From there, *pydantic* will handle everything for you by loading in your variables and
validating them.
!!! note
If a filename is specified for `env_file`, Pydantic will only check the current working directory and
won't check any parent directories for the `.env` file.
Even when using a dotenv file, *pydantic* will still read environment variables as well as the dotenv file,
**environment variables will always take priority over values loaded from a dotenv file**.
Passing a file path via the `_env_file` keyword argument on instantiation (method 2) will override
the value (if any) set on the `Config` class. If the above snippets were used in conjunction, `prod.env` would be loaded
while `.env` would be ignored.
If you need to load multiple dotenv files, you can pass the file paths as a `list` or `tuple`.
Later files in the list/tuple will take priority over earlier files.
```py
from pydantic import BaseSettings
class Settings(BaseSettings):
...
class Config:
# `.env.prod` takes priority over `.env`
env_file = '.env', '.env.prod'
```
You can also use the keyword argument override to tell Pydantic not to load any file at all (even if one is set in
the `Config` class) by passing `None` as the instantiation keyword argument, e.g. `settings = Settings(_env_file=None)`.
Because python-dotenv is used to parse the file, bash-like semantics such as `export` can be used which
(depending on your OS and environment) may allow your dotenv file to also be used with `source`,
see [python-dotenv's documentation](https://saurabh-kumar.com/python-dotenv/#usages) for more details.
## Secret Support
Placing secret values in files is a common pattern to provide sensitive configuration to an application.
A secret file follows the same principal as a dotenv file except it only contains a single value and the file name
is used as the key. A secret file will look like the following:
`/var/run/database_password`:
```
super_secret_database_password
```
Once you have your secret files, *pydantic* supports loading it in two ways:
**1.** setting `secrets_dir` on `Config` in a `BaseSettings` class to the directory where your secret files are stored:
```py
class Settings(BaseSettings):
...
database_password: str
class Config:
secrets_dir = '/var/run'
```
**2.** instantiating a `BaseSettings` derived class with the `_secrets_dir` keyword argument:
```py
settings = Settings(_secrets_dir='/var/run')
```
In either case, the value of the passed argument can be any valid directory, either absolute or relative to the
current working directory. **Note that a non existent directory will only generate a warning**.
From there, *pydantic* will handle everything for you by loading in your variables and validating them.
Even when using a secrets directory, *pydantic* will still read environment variables from a dotenv file or the environment,
**a dotenv file and environment variables will always take priority over values loaded from the secrets directory**.
Passing a file path via the `_secrets_dir` keyword argument on instantiation (method 2) will override
the value (if any) set on the `Config` class.
### Use Case: Docker Secrets
Docker Secrets can be used to provide sensitive configuration to an application running in a Docker container.
To use these secrets in a *pydantic* application the process is simple. More information regarding creating, managing
and using secrets in Docker see the official
[Docker documentation](https://docs.docker.com/engine/reference/commandline/secret/).
First, define your Settings
```py
class Settings(BaseSettings):
my_secret_data: str
class Config:
secrets_dir = '/run/secrets'
```
!!! note
By default Docker uses `/run/secrets` as the target mount point. If you want to use a different location, change
`Config.secrets_dir` accordingly.
Then, create your secret via the Docker CLI
```bash
printf "This is a secret" | docker secret create my_secret_data -
```
Last, run your application inside a Docker container and supply your newly created secret
```bash
docker service create --name pydantic-with-secrets --secret my_secret_data pydantic-app:latest
```
## Field value priority
In the case where a value is specified for the same `Settings` field in multiple ways,
the selected value is determined as follows (in descending order of priority):
1. Arguments passed to the `Settings` class initialiser.
2. Environment variables, e.g. `my_prefix_special_function` as described above.
3. Variables loaded from a dotenv (`.env`) file.
4. Variables loaded from the secrets directory.
5. The default field values for the `Settings` model.
## Customise settings sources
If the default order of priority doesn't match your needs, it's possible to change it by overriding
the `customise_sources` method on the `Config` class of your `Settings` .
`customise_sources` takes three callables as arguments and returns any number of callables as a tuple. In turn these
callables are called to build the inputs to the fields of the settings class.
Each callable should take an instance of the settings class as its sole argument and return a `dict`.
### Changing Priority
The order of the returned callables decides the priority of inputs; first item is the highest priority.
{!.tmp_examples/settings_env_priority.md!}
By flipping `env_settings` and `init_settings`, environment variables now have precedence over `__init__` kwargs.
### Adding sources
As explained earlier, *pydantic* ships with multiples built-in settings sources. However, you may occasionally
need to add your own custom sources, `customise_sources` makes this very easy:
{!.tmp_examples/settings_add_custom_source.md!}
### Removing sources
You might also want to disable a source:
{!.tmp_examples/settings_disable_source.md!}
docs/visual_studio_code.md
+1 -16
View File
@@ -257,21 +257,6 @@ in a model that is "frozen".
![VS Code strict type errors with model](./img/vs_code_08.png)
## BaseSettings and ignoring Pylance/pyright errors
Pylance/pyright does not work well with [`BaseSettings`](./usage/settings.md) - fields in settings classes can be
configured via environment variables and therefore "required" fields do not have to be explicitly set when
initialising a settings instance. However, pyright considers these fields as "required" and will therefore
show an error when they're not set.
See [#3753](https://github.com/pydantic/pydantic/issues/3753#issuecomment-1087417884) for an explanation of the
reasons behind this, and why we can't avoid the problem.
There are two potential workarounds:
* use an ignore comment (`# pyright: ignore`) when initialising `settings`
* or, use `settings.parse_obj({})` to avoid the warning
## Adding a default with `Field`
Pylance/pyright requires `default` to be a keyword argument to `Field` in order to infer that the field is optional.
@@ -287,7 +272,7 @@ class Knight(BaseModel):
lance = Knight() # error: Argument missing for parameter "age"
```
Like the issue with `BaseSettings`, this is a limitation of dataclass transforms and cannot be fixed in pydantic.
This is a limitation of dataclass transforms and cannot be fixed in pydantic.
## Technical Details
mkdocs.yml
-1
View File
@@ -52,7 +52,6 @@ nav:
- usage/exporting_models.md
- usage/dataclasses.md
- usage/validation_decorator.md
- 'Settings management': usage/settings.md
- usage/postponed_annotations.md
- 'Usage with mypy': usage/mypy.md
- 'Usage with devtools': usage/devtools.md
pydantic/__init__.py
-3
View File
@@ -4,7 +4,6 @@ from .annotated_types import create_model_from_namedtuple, create_model_from_typ
from .class_validators import root_validator, validator
from .config import BaseConfig, ConfigDict, Extra
from .decorator import validate_arguments
from .env_settings import BaseSettings
from .error_wrappers import ValidationError
from .errors import *
from .fields import Field, PrivateAttr, Required
@@ -34,8 +33,6 @@ __all__ = [
'Extra',
# decorator
'validate_arguments',
# env_settings
'BaseSettings',
# error_wrappers
'ValidationError',
# fields
pydantic/env_settings.py
-346
View File
@@ -1,346 +0,0 @@
import os
import warnings
from pathlib import Path
from typing import AbstractSet, Any, Callable, ClassVar, Dict, List, Mapping, Optional, Tuple, Type, Union
from .config import BaseConfig, Extra
from .fields import ModelField
from .main import BaseModel
from .typing import StrPath, display_as_type, get_origin, is_union
from .utils import deep_update, path_type, sequence_like
env_file_sentinel = str(object())
SettingsSourceCallable = Callable[['BaseSettings'], Dict[str, Any]]
DotenvType = Union[StrPath, List[StrPath], Tuple[StrPath, ...]]
class SettingsError(ValueError):
pass
class BaseSettings(BaseModel):
"""
Base class for settings, allowing values to be overridden by environment variables.
This is useful in production for secrets you do not wish to save in code, it plays nicely with docker(-compose),
Heroku and any 12 factor app design.
"""
def __init__(
__pydantic_self__,
_env_file: Optional[DotenvType] = env_file_sentinel,
_env_file_encoding: Optional[str] = None,
_env_nested_delimiter: Optional[str] = None,
_secrets_dir: Optional[StrPath] = None,
**values: Any,
) -> None:
# Uses something other than `self` the first arg to allow "self" as a settable attribute
super().__init__(
**__pydantic_self__._build_values(
values,
_env_file=_env_file,
_env_file_encoding=_env_file_encoding,
_env_nested_delimiter=_env_nested_delimiter,
_secrets_dir=_secrets_dir,
)
)
def _build_values(
self,
init_kwargs: Dict[str, Any],
_env_file: Optional[DotenvType] = None,
_env_file_encoding: Optional[str] = None,
_env_nested_delimiter: Optional[str] = None,
_secrets_dir: Optional[StrPath] = None,
) -> Dict[str, Any]:
# Configure built-in sources
init_settings = InitSettingsSource(init_kwargs=init_kwargs)
env_settings = EnvSettingsSource(
env_file=(_env_file if _env_file != env_file_sentinel else self.__config__.env_file),
env_file_encoding=(
_env_file_encoding if _env_file_encoding is not None else self.__config__.env_file_encoding
),
env_nested_delimiter=(
_env_nested_delimiter if _env_nested_delimiter is not None else self.__config__.env_nested_delimiter
),
env_prefix_len=len(self.__config__.env_prefix),
)
file_secret_settings = SecretsSettingsSource(secrets_dir=_secrets_dir or self.__config__.secrets_dir)
# Provide a hook to set built-in sources priority and add / remove sources
sources = self.__config__.customise_sources(
init_settings=init_settings, env_settings=env_settings, file_secret_settings=file_secret_settings
)
if sources:
return deep_update(*reversed([source(self) for source in sources]))
else:
# no one should mean to do this, but I think returning an empty dict is marginally preferable
# to an informative error and much better than a confusing error
return {}
class Config(BaseConfig):
env_prefix: str = ''
env_file: Optional[DotenvType] = None
env_file_encoding: Optional[str] = None
env_nested_delimiter: Optional[str] = None
secrets_dir: Optional[StrPath] = None
validate_all: bool = True
extra: Extra = Extra.forbid
arbitrary_types_allowed: bool = True
case_sensitive: bool = False
@classmethod
def prepare_field(cls, field: ModelField) -> None:
env_names: Union[List[str], AbstractSet[str]]
field_info_from_config = cls.get_field_info(field.name)
env = field_info_from_config.get('env') or field.field_info.extra.get('env')
if env is None:
if field.has_alias:
warnings.warn(
'aliases are no longer used by BaseSettings to define which environment variables to read. '
'Instead use the "env" field setting. '
'See https://pydantic-docs.helpmanual.io/usage/settings/#environment-variable-names',
FutureWarning,
)
env_names = {cls.env_prefix + field.name}
elif isinstance(env, str):
env_names = {env}
elif isinstance(env, (set, frozenset)):
env_names = env
elif sequence_like(env):
env_names = list(env)
else:
raise TypeError(f'invalid field env: {env!r} ({display_as_type(env)}); should be string, list or set')
if not cls.case_sensitive:
env_names = env_names.__class__(n.lower() for n in env_names)
field.field_info.extra['env_names'] = env_names
@classmethod
def customise_sources(
cls,
init_settings: SettingsSourceCallable,
env_settings: SettingsSourceCallable,
file_secret_settings: SettingsSourceCallable,
) -> Tuple[SettingsSourceCallable, ...]:
return init_settings, env_settings, file_secret_settings
@classmethod
def parse_env_var(cls, field_name: str, raw_val: str) -> Any:
return cls.json_loads(raw_val)
# populated by the metaclass using the Config class defined above, annotated here to help IDEs only
__config__: ClassVar[Type[Config]]
class InitSettingsSource:
__slots__ = ('init_kwargs',)
def __init__(self, init_kwargs: Dict[str, Any]):
self.init_kwargs = init_kwargs
def __call__(self, settings: BaseSettings) -> Dict[str, Any]:
return self.init_kwargs
def __repr__(self) -> str:
return f'InitSettingsSource(init_kwargs={self.init_kwargs!r})'
class EnvSettingsSource:
__slots__ = ('env_file', 'env_file_encoding', 'env_nested_delimiter', 'env_prefix_len')
def __init__(
self,
env_file: Optional[DotenvType],
env_file_encoding: Optional[str],
env_nested_delimiter: Optional[str] = None,
env_prefix_len: int = 0,
):
self.env_file: Optional[DotenvType] = env_file
self.env_file_encoding: Optional[str] = env_file_encoding
self.env_nested_delimiter: Optional[str] = env_nested_delimiter
self.env_prefix_len: int = env_prefix_len
def __call__(self, settings: BaseSettings) -> Dict[str, Any]: # noqa C901
"""
Build environment variables suitable for passing to the Model.
"""
d: Dict[str, Any] = {}
if settings.__config__.case_sensitive:
env_vars: Mapping[str, Optional[str]] = os.environ
else:
env_vars = {k.lower(): v for k, v in os.environ.items()}
dotenv_vars = self._read_env_files(settings.__config__.case_sensitive)
if dotenv_vars:
env_vars = {**dotenv_vars, **env_vars}
for field in settings.__fields__.values():
env_val: Optional[str] = None
for env_name in field.field_info.extra['env_names']:
env_val = env_vars.get(env_name)
if env_val is not None:
break
is_complex, allow_parse_failure = self.field_is_complex(field)
if is_complex:
if env_val is None:
# field is complex but no value found so far, try explode_env_vars
env_val_built = self.explode_env_vars(field, env_vars)
if env_val_built:
d[field.alias] = env_val_built
else:
# field is complex and there's a value, decode that as JSON, then add explode_env_vars
try:
env_val = settings.__config__.parse_env_var(field.name, env_val)
except ValueError as e:
if not allow_parse_failure:
raise SettingsError(f'error parsing env var "{env_name}"') from e
if isinstance(env_val, dict):
d[field.alias] = deep_update(env_val, self.explode_env_vars(field, env_vars))
else:
d[field.alias] = env_val
elif env_val is not None:
# simplest case, field is not complex, we only need to add the value if it was found
d[field.alias] = env_val
return d
def _read_env_files(self, case_sensitive: bool) -> Dict[str, Optional[str]]:
env_files = self.env_file
if env_files is None:
return {}
if isinstance(env_files, (str, os.PathLike)):
env_files = [env_files]
dotenv_vars = {}
for env_file in env_files:
env_path = Path(env_file).expanduser()
if env_path.is_file():
dotenv_vars.update(
read_env_file(env_path, encoding=self.env_file_encoding, case_sensitive=case_sensitive)
)
return dotenv_vars
def field_is_complex(self, field: ModelField) -> Tuple[bool, bool]:
"""
Find out if a field is complex, and if so whether JSON errors should be ignored
"""
if field.is_complex():
allow_parse_failure = False
elif is_union(get_origin(field.type_)) and field.sub_fields and any(f.is_complex() for f in field.sub_fields):
allow_parse_failure = True
else:
return False, False
return True, allow_parse_failure
def explode_env_vars(self, field: ModelField, env_vars: Mapping[str, Optional[str]]) -> Dict[str, Any]:
"""
Process env_vars and extract the values of keys containing env_nested_delimiter into nested dictionaries.
This is applied to a single field, hence filtering by env_var prefix.
"""
prefixes = [f'{env_name}{self.env_nested_delimiter}' for env_name in field.field_info.extra['env_names']]
result: Dict[str, Any] = {}
for env_name, env_val in env_vars.items():
if not any(env_name.startswith(prefix) for prefix in prefixes):
continue
# we remove the prefix before splitting in case the prefix has characters in common with the delimiter
env_name_without_prefix = env_name[self.env_prefix_len :]
_, *keys, last_key = env_name_without_prefix.split(self.env_nested_delimiter)
env_var = result
for key in keys:
env_var = env_var.setdefault(key, {})
env_var[last_key] = env_val
return result
def __repr__(self) -> str:
return (
f'EnvSettingsSource(env_file={self.env_file!r}, env_file_encoding={self.env_file_encoding!r}, '
f'env_nested_delimiter={self.env_nested_delimiter!r})'
)
class SecretsSettingsSource:
__slots__ = ('secrets_dir',)
def __init__(self, secrets_dir: Optional[StrPath]):
self.secrets_dir: Optional[StrPath] = secrets_dir
def __call__(self, settings: BaseSettings) -> Dict[str, Any]:
"""
Build fields from "secrets" files.
"""
secrets: Dict[str, Optional[str]] = {}
if self.secrets_dir is None:
return secrets
secrets_path = Path(self.secrets_dir).expanduser()
if not secrets_path.exists():
warnings.warn(f'directory "{secrets_path}" does not exist')
return secrets
if not secrets_path.is_dir():
raise SettingsError(f'secrets_dir must reference a directory, not a {path_type(secrets_path)}')
for field in settings.__fields__.values():
for env_name in field.field_info.extra['env_names']:
path = find_case_path(secrets_path, env_name, settings.__config__.case_sensitive)
if not path:
# path does not exist, we curently don't return a warning for this
continue
if path.is_file():
secret_value = path.read_text().strip()
if field.is_complex():
try:
secret_value = settings.__config__.parse_env_var(field.name, secret_value)
except ValueError as e:
raise SettingsError(f'error parsing env var "{env_name}"') from e
secrets[field.alias] = secret_value
else:
warnings.warn(
f'attempted to load secret file "{path}" but found a {path_type(path)} instead.',
stacklevel=4,
)
return secrets
def __repr__(self) -> str:
return f'SecretsSettingsSource(secrets_dir={self.secrets_dir!r})'
def read_env_file(
file_path: StrPath, *, encoding: str = None, case_sensitive: bool = False
) -> Dict[str, Optional[str]]:
try:
from dotenv import dotenv_values
except ImportError as e:
raise ImportError('python-dotenv is not installed, run `pip install pydantic[dotenv]`') from e
file_vars: Dict[str, Optional[str]] = dotenv_values(file_path, encoding=encoding or 'utf8')
if not case_sensitive:
return {k.lower(): v for k, v in file_vars.items()}
else:
return file_vars
def find_case_path(dir_path: Path, file_name: str, case_sensitive: bool) -> Optional[Path]:
"""
Find a file within path's directory matching filename, optionally ignoring case.
"""
for f in dir_path.iterdir():
if f.name == file_name:
return f
elif not case_sensitive and f.name.lower() == file_name.lower():
return f
return None
pydantic/fields.py
-12
View File
@@ -1150,18 +1150,6 @@ class ModelField(Representation):
return v, ErrorWrapper(exc, loc)
return v, None
def is_complex(self) -> bool:
"""
Whether the field is "complex" eg. env variables should be parsed as JSON.
"""
from .main import BaseModel
return (
self.shape != SHAPE_SINGLETON
or hasattr(self.type_, '__pydantic_model__')
or lenient_issubclass(self.type_, (BaseModel, list, set, frozenset, dict))
)
def _type_display(self) -> PyObjectStr:
t = display_as_type(self.type_)
pydantic/mypy.py
+3 -7
View File
@@ -74,7 +74,6 @@ except ImportError: # pragma: no cover
CONFIGFILE_KEY = 'pydantic-mypy'
METADATA_KEY = 'pydantic-mypy-metadata'
BASEMODEL_FULLNAME = 'pydantic.main.BaseModel'
BASESETTINGS_FULLNAME = 'pydantic.env_settings.BaseSettings'
FIELD_FULLNAME = 'pydantic.fields.Field'
DATACLASS_FULLNAME = 'pydantic.dataclasses.dataclass'
@@ -263,8 +262,7 @@ class PydanticModelTransformer:
if info[field.name].type is None:
if not ctx.api.final_iteration:
ctx.api.defer()
is_settings = any(get_fullname(base) == BASESETTINGS_FULLNAME for base in info.mro[:-1])
self.add_initializer(fields, config, is_settings)
self.add_initializer(fields, config)
self.add_construct_method(fields)
self.set_frozen(fields, frozen=config.allow_mutation is False or config.frozen is True)
info.metadata[METADATA_KEY] = {
@@ -405,7 +403,7 @@ class PydanticModelTransformer:
all_fields = superclass_fields + all_fields
return all_fields
def add_initializer(self, fields: List['PydanticModelField'], config: 'ModelConfigData', is_settings: bool) -> None:
def add_initializer(self, fields: List['PydanticModelField'], config: 'ModelConfigData') -> None:
"""
Adds a fields-aware `__init__` method to the class.
@@ -414,9 +412,7 @@ class PydanticModelTransformer:
ctx = self._ctx
typed = self.plugin_config.init_typed
use_alias = config.allow_population_by_field_name is not True
force_all_optional = is_settings or bool(
config.has_alias_generator and not config.allow_population_by_field_name
)
force_all_optional = bool(config.has_alias_generator and not config.allow_population_by_field_name)
init_arguments = self.get_field_arguments(
fields, typed=typed, force_all_optional=force_all_optional, use_alias=use_alias
)
pydantic/version.py
+1 -1
View File
@@ -10,7 +10,7 @@ def version_info() -> str:
from pathlib import Path
optional_deps = []
for p in ('devtools', 'dotenv', 'email-validator', 'typing-extensions'):
for p in 'devtools', 'email-validator', 'typing-extensions':
try:
import_module(p.replace('-', '_'))
except ImportError:
pyproject.toml
+1 -2
View File
@@ -39,7 +39,7 @@ classifiers = [
]
requires-python = '>=3.7'
dependencies = [ 'typing-extensions>=4.1.0' ]
optional-dependencies = { email = ['email-validator>=1.0.3'], dotenv = ['python-dotenv>=0.10.4'] }
optional-dependencies = { email = ['email-validator>=1.0.3'] }
dynamic = ['version']
entry-points.hypothesis = {_ = 'pydantic._hypothesis_plugin'}
@@ -135,7 +135,6 @@ disallow_untyped_defs = true
[[tool.mypy.overrides]]
module = [
'email_validator.*',
'dotenv.*',
'toml.*',
'ansi2html.*',
'devtools.*',
requirements/pyproject-all.txt
+1 -3
View File
@@ -2,7 +2,7 @@
# This file is autogenerated by pip-compile with python 3.10
# To update, run:
#
# pip-compile --extra=email,dotenv --output-file=requirements/pyproject-all.txt pyproject.toml
# pip-compile --extra=email --output-file=requirements/pyproject-all.txt pyproject.toml
#
dnspython==2.2.1
# via email-validator
@@ -10,7 +10,5 @@ email-validator==1.2.1
# via pydantic (pyproject.toml)
idna==3.3
# via email-validator
python-dotenv==0.20.0
# via pydantic (pyproject.toml)
typing-extensions==4.3.0
# via pydantic (pyproject.toml)
tests/conftest.py
-23
View File
@@ -1,6 +1,5 @@
import importlib
import inspect
import os
import secrets
import sys
import textwrap
@@ -33,28 +32,6 @@ def _create_module_file(code, tmp_path, name):
return name, str(path)
class SetEnv:
def __init__(self):
self.envars = set()
def set(self, name, value):
self.envars.add(name)
os.environ[name] = value
def clear(self):
for n in self.envars:
os.environ.pop(n)
@pytest.fixture
def env():
setenv = SetEnv()
yield setenv
setenv.clear()
@pytest.fixture
def create_module(tmp_path, request):
def run(source_code_or_function, rewrite_assertions=True):
tests/mypy/modules/plugin_fail.py
+1 -8
View File
@@ -1,6 +1,6 @@
from typing import Any, Generic, List, Optional, Set, TypeVar, Union
from pydantic import BaseModel, BaseSettings, Extra, Field, validator
from pydantic import BaseModel, Extra, Field, validator
from pydantic.dataclasses import dataclass
from pydantic.generics import GenericModel
@@ -94,19 +94,12 @@ class UndefinedAnnotationModel(BaseModel):
UndefinedAnnotationModel()
class Settings(BaseSettings):
x: int
Model.construct(x=1)
Model.construct(_fields_set={'x'}, x=1, y='2')
Model.construct(x='1', y='2')
Settings() # should pass here due to possibly reading from environment
# Strict mode fails
inheriting = InheritingModel(x='1', y='1')
Settings(x='1')
Model(x='1', y='2')
tests/mypy/modules/plugin_success.py
+1 -8
View File
@@ -1,6 +1,6 @@
from typing import ClassVar, Generic, List, Optional, TypeVar, Union
from pydantic import BaseModel, BaseSettings, Field, create_model, validator
from pydantic import BaseModel, Field, create_model, validator
from pydantic.dataclasses import dataclass
from pydantic.generics import GenericModel
@@ -165,13 +165,6 @@ class ModelWithSelfField(BaseModel):
self: str
class SettingsModel(BaseSettings):
pass
settings = SettingsModel.construct()
def f(name: str) -> str:
return name
tests/mypy/modules/settings_config.py
-7
View File
@@ -1,7 +0,0 @@
from pydantic import BaseSettings
class Settings(BaseSettings):
class Config(BaseSettings.Config):
env_file = '.env'
env_file_encoding = 'utf-8'
tests/mypy/modules/success.py
-10
View File
@@ -16,7 +16,6 @@ from pydantic import (
UUID1,
BaseConfig,
BaseModel,
BaseSettings,
DirectoryPath,
Extra,
FilePath,
@@ -273,10 +272,6 @@ class Config(BaseConfig):
max_anystr_length = 1234
class Settings(BaseSettings):
...
class CustomPath(PurePath):
def __init__(self, *args: str):
self.path = os.path.join(*args)
@@ -285,10 +280,5 @@ class CustomPath(PurePath):
return f'a/custom/{self.path}'
def dont_check_path_existence() -> None:
Settings(_env_file='a/path', _secrets_dir='a/path')
Settings(_env_file=CustomPath('a/path'), _secrets_dir=CustomPath('a/path'))
create_model_from_typeddict(SomeDict)(**obj)
DynamicModel = create_model('DynamicModel')
tests/mypy/outputs/plugin-fail-strict.txt
+22 -23
View File
@@ -18,26 +18,25 @@
87: error: Missing named argument "e" for "DefaultTestingModel" [call-arg]
91: error: Name "Undefined" is not defined [name-defined]
94: error: Missing named argument "undefined" for "UndefinedAnnotationModel" [call-arg]
101: error: Missing named argument "y" for "construct" of "Model" [call-arg]
103: error: Argument "x" to "construct" of "Model" has incompatible type "str"; expected "int" [arg-type]
108: error: Argument "x" to "InheritingModel" has incompatible type "str"; expected "int" [arg-type]
109: error: Argument "x" to "Settings" has incompatible type "str"; expected "int" [arg-type]
110: error: Argument "x" to "Model" has incompatible type "str"; expected "int" [arg-type]
127: error: Argument "data" to "Response" has incompatible type "int"; expected "Model" [arg-type]
135: error: Argument "y" to "AliasModel" has incompatible type "int"; expected "str" [arg-type]
141: error: Required dynamic aliases disallowed [pydantic-alias]
145: error: Argument "z" to "DynamicAliasModel" has incompatible type "str"; expected "int" [arg-type]
156: error: Unexpected keyword argument "y" for "DynamicAliasModel2" [call-arg]
163: error: Required dynamic aliases disallowed [pydantic-alias]
181: error: Untyped fields disallowed [pydantic-field]
185: error: Unexpected keyword argument "x" for "AliasGeneratorModel2" [call-arg]
186: error: Unexpected keyword argument "z" for "AliasGeneratorModel2" [call-arg]
189: error: Name "Missing" is not defined [name-defined]
219: error: Property "y" defined in "FrozenModel" is read-only [misc]
240: error: Incompatible types in assignment (expression has type "None", variable has type "int") [assignment]
241: error: Incompatible types in assignment (expression has type "None", variable has type "int") [assignment]
244: error: Incompatible types in assignment (expression has type "Set[Any]", variable has type "str") [assignment]
245: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment]
247: error: Argument "default_factory" to "Field" has incompatible type "int"; expected "Optional[Callable[[], Any]]" [arg-type]
250: error: Field default and default_factory cannot be specified together [pydantic-field]
260: error: Missing positional argument "self" in call to "instance_method" of "ModelWithAnnotatedValidator" [call-arg]
97: error: Missing named argument "y" for "construct" of "Model" [call-arg]
99: error: Argument "x" to "construct" of "Model" has incompatible type "str"; expected "int" [arg-type]
102: error: Argument "x" to "InheritingModel" has incompatible type "str"; expected "int" [arg-type]
103: error: Argument "x" to "Model" has incompatible type "str"; expected "int" [arg-type]
120: error: Argument "data" to "Response" has incompatible type "int"; expected "Model" [arg-type]
128: error: Argument "y" to "AliasModel" has incompatible type "int"; expected "str" [arg-type]
134: error: Required dynamic aliases disallowed [pydantic-alias]
138: error: Argument "z" to "DynamicAliasModel" has incompatible type "str"; expected "int" [arg-type]
149: error: Unexpected keyword argument "y" for "DynamicAliasModel2" [call-arg]
156: error: Required dynamic aliases disallowed [pydantic-alias]
174: error: Untyped fields disallowed [pydantic-field]
178: error: Unexpected keyword argument "x" for "AliasGeneratorModel2" [call-arg]
179: error: Unexpected keyword argument "z" for "AliasGeneratorModel2" [call-arg]
182: error: Name "Missing" is not defined [name-defined]
212: error: Property "y" defined in "FrozenModel" is read-only [misc]
233: error: Incompatible types in assignment (expression has type "None", variable has type "int") [assignment]
234: error: Incompatible types in assignment (expression has type "None", variable has type "int") [assignment]
237: error: Incompatible types in assignment (expression has type "Set[Any]", variable has type "str") [assignment]
238: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment]
240: error: Argument "default_factory" to "Field" has incompatible type "int"; expected "Optional[Callable[[], Any]]" [arg-type]
243: error: Field default and default_factory cannot be specified together [pydantic-field]
253: error: Missing positional argument "self" in call to "instance_method" of "ModelWithAnnotatedValidator" [call-arg]
tests/mypy/outputs/plugin-fail.txt
+13 -13
View File
@@ -16,17 +16,17 @@
87: error: Missing named argument "e" for "DefaultTestingModel" [call-arg]
91: error: Name "Undefined" is not defined [name-defined]
94: error: Missing named argument "undefined" for "UndefinedAnnotationModel" [call-arg]
101: error: Missing named argument "y" for "construct" of "Model" [call-arg]
103: error: Argument "x" to "construct" of "Model" has incompatible type "str"; expected "int" [arg-type]
156: error: Missing named argument "x" for "DynamicAliasModel2" [call-arg]
97: error: Missing named argument "y" for "construct" of "Model" [call-arg]
99: error: Argument "x" to "construct" of "Model" has incompatible type "str"; expected "int" [arg-type]
149: error: Missing named argument "x" for "DynamicAliasModel2" [call-arg]
168: error: Unused "type: ignore" comment
175: error: Unused "type: ignore" comment
182: error: Unused "type: ignore" comment
189: error: Name "Missing" is not defined [name-defined]
219: error: Property "y" defined in "FrozenModel" is read-only [misc]
240: error: Incompatible types in assignment (expression has type "None", variable has type "int") [assignment]
241: error: Incompatible types in assignment (expression has type "None", variable has type "int") [assignment]
244: error: Incompatible types in assignment (expression has type "Set[Any]", variable has type "str") [assignment]
245: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment]
247: error: Argument "default_factory" to "Field" has incompatible type "int"; expected "Optional[Callable[[], Any]]" [arg-type]
250: error: Field default and default_factory cannot be specified together [pydantic-field]
260: error: Missing positional argument "self" in call to "instance_method" of "ModelWithAnnotatedValidator" [call-arg]
182: error: Name "Missing" is not defined [name-defined]
212: error: Property "y" defined in "FrozenModel" is read-only [misc]
233: error: Incompatible types in assignment (expression has type "None", variable has type "int") [assignment]
234: error: Incompatible types in assignment (expression has type "None", variable has type "int") [assignment]
237: error: Incompatible types in assignment (expression has type "Set[Any]", variable has type "str") [assignment]
238: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment]
240: error: Argument "default_factory" to "Field" has incompatible type "int"; expected "Optional[Callable[[], Any]]" [arg-type]
243: error: Field default and default_factory cannot be specified together [pydantic-field]
253: error: Missing positional argument "self" in call to "instance_method" of "ModelWithAnnotatedValidator" [call-arg]
tests/mypy/test_mypy.py
-1
View File
@@ -46,7 +46,6 @@ cases = [
('pyproject-plugin-strict.toml', 'plugin_success.py', 'plugin-success-strict.txt'),
('pyproject-plugin-strict.toml', 'plugin_fail.py', 'plugin-fail-strict.txt'),
('pyproject-plugin-strict.toml', 'fail_defaults.py', 'fail_defaults.txt'),
('mypy-plugin-strict.ini', 'settings_config.py', None),
('mypy-plugin-strict.ini', 'plugin_default_factory.py', 'plugin_default_factory.txt'),
]
executable_modules = list({fname[:-3] for _, fname, out_fname in cases if out_fname is None})
tests/pyright/pyright_example.py
+1 -14
View File
@@ -1,13 +1,10 @@
"""
This file is used to test pyright's ability to check pydantic code.
In particular pydantic provides the `@__dataclass_transform__` for `BaseModel`
and all subclasses (including `BaseSettings`), see #2721.
"""
from typing import List
from pydantic import BaseModel, BaseSettings, Field
from pydantic import BaseModel, Field
class MyModel(BaseModel):
@@ -26,13 +23,3 @@ class Knight(BaseModel):
k = Knight() # pyright: ignore
class Settings(BaseSettings):
x: str
y: int
s1 = Settings.parse_obj({})
s2 = Settings() # pyright: ignore[reportGeneralTypeIssues]
tests/test_edge_cases.py
+6 -9
View File
@@ -9,7 +9,6 @@ import pytest
from pydantic import (
BaseModel,
BaseSettings,
Extra,
NoneStrBytes,
StrBytes,
@@ -1207,21 +1206,19 @@ def test_self():
}
@pytest.mark.parametrize('model', [BaseModel, BaseSettings])
def test_self_recursive(model):
class SubModel(model):
def test_self_recursive():
class SubModel(BaseModel):
self: int
class Model(model):
class Model(BaseModel):
sm: SubModel
m = Model.parse_obj({'sm': {'self': '123'}})
assert m.dict() == {'sm': {'self': 123}}
@pytest.mark.parametrize('model', [BaseModel, BaseSettings])
def test_nested_init(model):
class NestedModel(model):
def test_nested_init():
class NestedModel(BaseModel):
self: str
modified_number: int = 1
@@ -1229,7 +1226,7 @@ def test_nested_init(model):
super().__init__(**kwargs)
someinit.modified_number += 1
class TopModel(model):
class TopModel(BaseModel):
self: str
nest: NestedModel
tests/test_settings.py
-1280
View File
File diff suppressed because it is too large Load Diff