diff --git a/changes/847-samuelcolvin.rst b/changes/847-samuelcolvin.rst new file mode 100644 index 0000000..6a56668 --- /dev/null +++ b/changes/847-samuelcolvin.rst @@ -0,0 +1,2 @@ +**Breaking Change:** ``BaseSettings`` now uses the special ``env`` settings to define which environment variables to +read, not aliases. diff --git a/docs/examples/settings.py b/docs/examples/settings.py index 075a986..14cb2af 100644 --- a/docs/examples/settings.py +++ b/docs/examples/settings.py @@ -1,44 +1,57 @@ from typing import Set -from pydantic import BaseModel, DSN, BaseSettings, PyObject - +from devtools import debug +from pydantic import BaseModel, BaseSettings, PyObject, RedisDsn, PostgresDsn, Field class SubModel(BaseModel): foo = 'bar' apple = 1 - class Settings(BaseSettings): - redis_host = 'localhost' - redis_port = 6379 - redis_database = 0 - redis_password: str = None + auth_key: str + api_key: str = Field(..., env='my_api_key') - auth_key: str = ... + redis_dsn: RedisDsn = 'redis://user:pass@localhost:6379/1' + pg_dsn: PostgresDsn = 'postgres://user:pass@localhost:5432/foobar' - invoicing_cls: PyObject = 'path.to.Invoice' - - db_name = 'foobar' - db_user = 'postgres' - db_password: str = None - db_host = 'localhost' - db_port = '5432' - db_driver = 'postgres' - db_query: dict = None - dsn: DSN = None + special_function: PyObject = 'math.cos' # to override domains: - # export MY_PREFIX_DOMAINS = '["foo.com", "bar.com"]' + # export my_prefix_domains='["foo.com", "bar.com"]' domains: Set[str] = set() # to override more_settings: - # export MY_PREFIX_MORE_SETTINGS = '{"foo": "x", "apple": 1}' + # export my_prefix_more_settings='{"foo": "x", "apple": 1}' more_settings: SubModel = SubModel() class Config: - env_prefix = 'MY_PREFIX_' # defaults to 'APP_' + env_prefix = 'my_prefix_' # defaults to no prefix, e.g. "" fields = { 'auth_key': { - 'alias': 'my_api_key' + 'env': 'my_auth_key', + }, + 'redis_dsn': { + 'env': ['service_redis_dsn', 'redis_url'] } } + +""" +When calling with +my_auth_key=a \ +MY_API_KEY=b \ +my_prefix_domains='["foo.com", "bar.com"]' \ +python docs/examples/settings.py +""" +debug(Settings().dict()) +""" +docs/examples/settings.py:45 + Settings().dict(): { + 'auth_key': 'a', + 'api_key': 'b', + 'redis_dsn': , + 'pg_dsn': , + 'special_function': , + 'domains': {'bar.com', 'foo.com'}, + 'more_settings': {'foo': 'bar', 'apple': 1}, + } (dict) len=7 +""" diff --git a/docs/index.rst b/docs/index.rst index dd07f19..b46e0c5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1011,9 +1011,6 @@ Version for models based on ``@dataclass`` decorator: (This script is complete, it should run "as is") -.. _settings: - - Alias Generator ~~~~~~~~~~~~~~~ If data source field names do not match your code style (e. g. CamelCase fields), @@ -1023,6 +1020,7 @@ you can automatically generate aliases using ``alias_generator``: (This script is complete, it should run "as is") +.. _settings: Settings ........ @@ -1034,23 +1032,35 @@ environment variables or keyword arguments (e.g. in unit tests). (This script is complete, it should run "as is") -Here ``redis_port`` could be modified via ``export MY_PREFIX_REDIS_PORT=6380`` or ``auth_key`` by -``export my_api_key=6380``. By default, environment variables are treated as case-insensitive, so -``export my_prefix_redis_port=6380`` would work as well. -(Aliases are always sensitive to case, so ``export MY_API_KEY=6380`` would not work.) +The following rules apply when finding and interpreting environment variables: + +* When no custom environment variable name(s) are given, the environment variable name is built using the field + name and prefix, eg to override ``special_function`` use ``export my_prefix_special_function='foo.bar'``, the default + prefix is an empty string. aliases are ignored for building the environment variable name. +* Custom environment variable names can be set using with ``Config.fields.[field name].env`` or ``Field(..., env=...)``, + in the above example ``auth_key`` and ``api_key``'s environment variable setups are the equivalent. +* In these cases ``env`` can either be a string or a list of strings. When a list of strings order is important: + in the case of ``redis_dsn`` ``service_redis_dsn`` would take precedence over ``redis_url``. + +.. warning:: + + Since V1 *pydantic* does not consider field aliases when finding environment variables to populate settings + models, use ``env`` instead as described above. + + To aid the transition from aliases to ``env``, a warning will be raised when aliases are used on settings models + without a custom env var name. If you really mean to use aliases, either ignore the warning or set ``env`` to + suppress it. By default ``BaseSettings`` considers field values in the following priority (where 3. has the highest priority and overrides the other two): 1. The default values set in your ``Settings`` class. -2. Environment variables, e.g. ``MY_PREFIX_REDIS_PORT`` as described above. +2. Environment variables, e.g. ``my_prefix_special_function`` as described above. 3. Arguments passed to the ``Settings`` class on initialisation. -This behaviour can be changed by overriding the ``_build_values`` method on ``BaseSettings``. +Complex types like ``list``, ``set``, ``dict`` and sub-models can be set by using JSON environment variables. -Complex types like ``list``, ``set``, ``dict`` and submodels can be set by using JSON environment variables. - -Case-sensitivity can be turned on through the ``Config``: +Case-sensitivity can be turned on through ``Config``: .. literalinclude:: examples/settings_case_sensitive.py diff --git a/pydantic/env_settings.py b/pydantic/env_settings.py index 9528907..18e15fd 100644 --- a/pydantic/env_settings.py +++ b/pydantic/env_settings.py @@ -1,7 +1,10 @@ import os -from typing import Any, Dict, Optional, cast +import warnings +from typing import Any, Dict, Iterable, Mapping, Optional +from .fields import ModelField from .main import BaseModel, Extra +from .typing import display_as_type class SettingsError(ValueError): @@ -30,26 +33,26 @@ class BaseSettings(BaseModel): d: Dict[str, Optional[str]] = {} if self.__config__.case_sensitive: - env_vars = cast(Dict[str, str], os.environ) + env_vars: Mapping[str, str] = os.environ else: env_vars = {k.lower(): v for k, v in os.environ.items()} for field in self.__fields__.values(): - if field.has_alias: - env_name = field.alias - else: - env_name = self.__config__.env_prefix + field.name.upper() + env_val: Optional[str] = None + for env_name in field.field_info.extra['env_names']: # type: ignore + env_val = env_vars.get(env_name) + if env_val is not None: + break - env_name_ = env_name if self.__config__.case_sensitive else env_name.lower() - env_val = env_vars.get(env_name_, None) + if env_val is None: + continue - if env_val: - if field.is_complex(): - try: - env_val = self.__config__.json_loads(env_val) # type: ignore - except ValueError as e: - raise SettingsError(f'error parsing JSON for "{env_name}"') from e - d[field.alias] = env_val + if field.is_complex(): + try: + env_val = self.__config__.json_loads(env_val) # type: ignore + except ValueError as e: + raise SettingsError(f'error parsing JSON for "{env_name}"') from e + d[field.alias] = env_val return d class Config: @@ -59,4 +62,30 @@ class BaseSettings(BaseModel): arbitrary_types_allowed = True case_sensitive = False + @classmethod + def prepare_field(cls, field: ModelField) -> None: + if not field.field_info: + return + + env_names: Iterable[str] + env = field.field_info.extra.pop('env', None) + if env is None: + if field.has_alias: + warnings.warn( + 'aliases are no longer used by BaseSettings to define which environment variables to read. ' + 'Instead use the "env" field setting. See https://pydantic-docs.helpmanual.io/#settings', + DeprecationWarning, + ) + env_names = [cls.env_prefix + field.name] + elif isinstance(env, str): + env_names = {env} + elif isinstance(env, (list, set, tuple)): + env_names = env + else: + raise TypeError(f'invalid field env: {env!r} ({display_as_type(env)}); should be string, list or set') + + if not cls.case_sensitive: + env_names = type(env_names)(n.lower() for n in env_names) + field.field_info.extra['env_names'] = env_names + __config__: Config # type: ignore diff --git a/pydantic/fields.py b/pydantic/fields.py index 2e8077d..79e5876 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -229,6 +229,7 @@ class ModelField: self.post_validators: Optional['ValidatorsList'] = None self.parse_json: bool = False self.shape: int = SHAPE_SINGLETON + self.model_config.prepare_field(self) self.prepare() @classmethod diff --git a/pydantic/main.py b/pydantic/main.py index 72b83c2..ee11c28 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -73,7 +73,7 @@ class BaseConfig: json_encoders: Dict[AnyType, AnyCallable] = {} @classmethod - def get_field_info(cls, name: str) -> Dict[str, str]: + def get_field_info(cls, name: str) -> Dict[str, Any]: field_info = cls.fields.get(name) or {} if isinstance(field_info, str): field_info = {'alias': field_info} @@ -84,6 +84,13 @@ class BaseConfig: field_info['alias'] = alias return field_info + @classmethod + def prepare_field(cls, field: 'ModelField') -> None: + """ + Optional hook to check or modify fields during model creation. + """ + pass + def inherit_config(self_config: 'ConfigType', parent_config: 'ConfigType') -> 'ConfigType': if not self_config: diff --git a/tests/test_settings.py b/tests/test_settings.py index ace7dbd..c709099 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -10,19 +10,15 @@ from pydantic.env_settings import SettingsError class SimpleSettings(BaseSettings): apple: str - class Config: - env_prefix = 'APP_' - case_sensitive = True - def test_sub_env(env): - env.set('APP_APPLE', 'hello') + env.set('apple', 'hello') s = SimpleSettings() assert s.apple == 'hello' def test_sub_env_override(env): - env.set('APP_APPLE', 'hello') + env.set('apple', 'hello') s = SimpleSettings(apple='goodbye') assert s.apple == 'goodbye' @@ -33,20 +29,23 @@ def test_sub_env_missing(): assert exc_info.value.errors() == [{'loc': ('apple',), 'msg': 'field required', 'type': 'value_error.missing'}] -def test_other_setting(env): +def test_other_setting(): with pytest.raises(ValidationError): SimpleSettings(apple='a', foobar=42) -def test_env_with_aliass(env): +def test_with_prefix(env): class Settings(BaseSettings): - apple: str = ... + apple: str class Config: - fields = {'apple': 'BOOM'} + env_prefix = 'foobar_' - env.set('BOOM', 'hello') - assert Settings().apple == 'hello' + with pytest.raises(ValidationError): + Settings() + env.set('foobar_apple', 'has_prefix') + s = Settings() + assert s.apple == 'has_prefix' class DateModel(BaseModel): @@ -59,22 +58,18 @@ class ComplexSettings(BaseSettings): carrots: dict = {} date: DateModel = DateModel() - class Config: - env_prefix = 'APP_' - case_sensitive = True - def test_list(env): - env.set('APP_APPLES', '["russet", "granny smith"]') + env.set('apples', '["russet", "granny smith"]') s = ComplexSettings() assert s.apples == ['russet', 'granny smith'] assert s.date.pips is False def test_set_dict_model(env): - env.set('APP_BANANAS', '[1, 2, 3, 3]') - env.set('APP_CARROTS', '{"a": null, "b": 4}') - env.set('APP_DATE', '{"pips": true}') + env.set('bananas', '[1, 2, 3, 3]') + env.set('CARROTS', '{"a": null, "b": 4}') + env.set('daTE', '{"pips": true}') s = ComplexSettings() assert s.bananas == {1, 2, 3} assert s.carrots == {'a': None, 'b': 4} @@ -82,8 +77,8 @@ def test_set_dict_model(env): def test_invalid_json(env): - env.set('APP_APPLES', '["russet", "granny smith",]') - with pytest.raises(SettingsError): + env.set('apples', '["russet", "granny smith",]') + with pytest.raises(SettingsError, match='error parsing JSON for "apples"'): ComplexSettings() @@ -107,37 +102,145 @@ def test_non_class(env): assert s.foobar == 'xxx' -def test_alias_matches_name(env): +def test_env_str(env): + class Settings(BaseSettings): + apple: str = ... + + class Config: + fields = {'apple': {'env': 'BOOM'}} + + env.set('BOOM', 'hello') + assert Settings().apple == 'hello' + + +def test_env_list(env): class Settings(BaseSettings): foobar: str class Config: - fields = {'foobar': 'foobar'} + fields = {'foobar': {'env': ['different1', 'different2']}} - env.set('foobar', 'xxx') + env.set('different1', 'value 1') + env.set('different2', 'value 2') s = Settings() - assert s.foobar == 'xxx' + assert s.foobar == 'value 1' -def test_case_insensitive(env): +def test_env_list_field(env): class Settings(BaseSettings): - foo: str - bAR: str + foobar: str = Field(..., env='foobar_env_name') + + env.set('FOOBAR_ENV_NAME', 'env value') + s = Settings() + assert s.foobar == 'env value' + + +def test_env_list_last(env): + class Settings(BaseSettings): + foobar: str class Config: - env_prefix = 'APP_' - case_sensitive = False + fields = {'foobar': {'env': ['different2']}} - env.set('apP_foO', 'foo') - env.set('app_bar', 'bar') + env.set('different1', 'value 1') + env.set('different2', 'value 2') s = Settings() - assert s.foo == 'foo' - assert s.bAR == 'bar' + assert s.foobar == 'value 2' + assert Settings(foobar='abc').foobar == 'abc' + + +def test_env_inheritance(env): + class SettingsParent(BaseSettings): + foobar: str = 'parent default' + + class Config: + fields = {'foobar': {'env': 'different'}} + + class SettingsChild(SettingsParent): + foobar: str = 'child default' + + assert SettingsParent().foobar == 'parent default' + assert SettingsParent(foobar='abc').foobar == 'abc' + + assert SettingsChild().foobar == 'child default' + assert SettingsChild(foobar='abc').foobar == 'abc' + env.set('different', 'env value') + assert SettingsParent().foobar == 'env value' + assert SettingsParent(foobar='abc').foobar == 'abc' + assert SettingsChild().foobar == 'env value' + assert SettingsChild(foobar='abc').foobar == 'abc' + + +def test_env_inheritance_field(env): + class SettingsParent(BaseSettings): + foobar: str = Field('parent default', env='foobar_env') + + class SettingsChild(SettingsParent): + foobar: str = 'child default' + + assert SettingsParent().foobar == 'parent default' + assert SettingsParent(foobar='abc').foobar == 'abc' + + assert SettingsChild().foobar == 'child default' + assert SettingsChild(foobar='abc').foobar == 'abc' + env.set('foobar_env', 'env value') + assert SettingsParent().foobar == 'env value' + assert SettingsParent(foobar='abc').foobar == 'abc' + assert SettingsChild().foobar == 'child default' + assert SettingsChild(foobar='abc').foobar == 'abc' + + +def test_env_invalid(env): + with pytest.raises(TypeError, match=r'invalid field env: 123 \(int\); should be string, list or set'): + + class Settings(BaseSettings): + foobar: str + + class Config: + fields = {'foobar': {'env': 123}} + + +def test_env_field(env): + with pytest.raises(TypeError, match=r'invalid field env: 123 \(int\); should be string, list or set'): + + class Settings(BaseSettings): + foobar: str = Field(..., env=123) + + +def test_aliases_warning(env): + with pytest.warns(DeprecationWarning, match='aliases are no longer used by BaseSettings'): + + class Settings(BaseSettings): + foobar: str = 'default value' + + class Config: + fields = {'foobar': 'foobar_alias'} + + assert Settings().foobar == 'default value' + env.set('foobar_alias', 'xxx') + assert Settings().foobar == 'default value' + assert Settings(foobar_alias='42').foobar == '42' + + +def test_aliases_no_warning(env): + class Settings(BaseSettings): + foobar: str = 'default value' + + class Config: + fields = {'foobar': {'alias': 'foobar_alias', 'env': 'foobar_env'}} + + assert Settings().foobar == 'default value' + assert Settings(foobar_alias='42').foobar == '42' + env.set('foobar_alias', 'xxx') + assert Settings().foobar == 'default value' + env.set('foobar_env', 'xxx') + assert Settings().foobar == 'xxx' + assert Settings(foobar_alias='42').foobar == '42' def test_case_sensitive(monkeypatch): class Settings(BaseSettings): - foo: str = Field(..., alias='foo') + foo: str class Config: case_sensitive = True @@ -165,7 +268,7 @@ def test_nested_dataclass(env): assert s.n.bar == 'bar value' -def test_config_file_settings(env): +def test_env_takes_precedence(env): class Settings(BaseSettings): foo: int bar: str