change how env variables work with settings (#847)

* change how env variables work with settings, fix #721

* inheritance and alias warnings

* update docs

* tweak env_settings.py

changes/847-samuelcolvin.rst

@@ -0,0 +1,2 @@
+**Breaking Change:** ``BaseSettings`` now uses the special ``env`` settings to define which environment variables to
+read, not aliases.

docs/examples/settings.py

@@ -1,44 +1,57 @@
 from typing import Set
 
-from pydantic import BaseModel, DSN, BaseSettings, PyObject
-
+from devtools import debug
+from pydantic import BaseModel, BaseSettings, PyObject, RedisDsn, PostgresDsn, Field
 
 class SubModel(BaseModel):
     foo = 'bar'
     apple = 1
 
-
 class Settings(BaseSettings):
-    redis_host = 'localhost'
-    redis_port = 6379
-    redis_database = 0
-    redis_password: str = None
+    auth_key: str
+    api_key: str = Field(..., env='my_api_key')
 
-    auth_key: str = ...
+    redis_dsn: RedisDsn = 'redis://user:pass@localhost:6379/1'
+    pg_dsn: PostgresDsn = 'postgres://user:pass@localhost:5432/foobar'
 
-    invoicing_cls: PyObject = 'path.to.Invoice'
-
-    db_name = 'foobar'
-    db_user = 'postgres'
-    db_password: str = None
-    db_host = 'localhost'
-    db_port = '5432'
-    db_driver = 'postgres'
-    db_query: dict = None
-    dsn: DSN = None
+    special_function: PyObject = 'math.cos'
 
     # to override domains:
-    # export MY_PREFIX_DOMAINS = '["foo.com", "bar.com"]'
+    # export my_prefix_domains='["foo.com", "bar.com"]'
     domains: Set[str] = set()
 
     # to override more_settings:
-    # export MY_PREFIX_MORE_SETTINGS = '{"foo": "x", "apple": 1}'
+    # export my_prefix_more_settings='{"foo": "x", "apple": 1}'
     more_settings: SubModel = SubModel()
 
     class Config:
-        env_prefix = 'MY_PREFIX_'  # defaults to 'APP_'
+        env_prefix = 'my_prefix_'  # defaults to no prefix, e.g. ""
         fields = {
             'auth_key': {
-                'alias': 'my_api_key'
+                'env': 'my_auth_key',
+            },
+            'redis_dsn': {
+                'env': ['service_redis_dsn', 'redis_url']
             }
         }
+
+"""
+When calling with
+my_auth_key=a \
+MY_API_KEY=b \
+my_prefix_domains='["foo.com", "bar.com"]' \
+python docs/examples/settings.py 
+"""
+debug(Settings().dict())
+"""
+docs/examples/settings.py:45 <module>
+  Settings().dict(): {
+    'auth_key': 'a',
+    'api_key': 'b',
+    'redis_dsn': <RedisDsn('redis://user:pass@localhost:6379/1' scheme='redis' ...)>,
+    'pg_dsn': <PostgresDsn('postgres://user:pass@localhost:5432/foobar' scheme='postgres' ...)>,
+    'special_function': <built-in function cos>,
+    'domains': {'bar.com', 'foo.com'},
+    'more_settings': {'foo': 'bar', 'apple': 1},
+  } (dict) len=7
+"""

docs/index.rst

@@ -1011,9 +1011,6 @@ Version for models based on ``@dataclass`` decorator:
 
 (This script is complete, it should run "as is")
 
-.. _settings:
-
-
 Alias Generator
 ~~~~~~~~~~~~~~~
 If data source field names do not match your code style (e. g. CamelCase fields),
@@ -1023,6 +1020,7 @@ you can automatically generate aliases using ``alias_generator``:
 
 (This script is complete, it should run "as is")
 
+.. _settings:
 
 Settings
 ........
@@ -1034,23 +1032,35 @@ environment variables or keyword arguments (e.g. in unit tests).
 
 (This script is complete, it should run "as is")
 
-Here ``redis_port`` could be modified via ``export MY_PREFIX_REDIS_PORT=6380`` or ``auth_key`` by
-``export my_api_key=6380``. By default, environment variables are treated as case-insensitive, so
-``export my_prefix_redis_port=6380`` would work as well.
-(Aliases are always sensitive to case, so ``export MY_API_KEY=6380`` would not work.)
+The following rules apply when finding and interpreting environment variables:
+
+* When no custom environment variable name(s) are given, the environment variable name is built using the field
+  name and prefix, eg to override ``special_function`` use ``export my_prefix_special_function='foo.bar'``, the default
+  prefix is an empty string. aliases are ignored for building the environment variable name.
+* Custom environment variable names can be set using with ``Config.fields.[field name].env`` or ``Field(..., env=...)``,
+  in the above example ``auth_key`` and ``api_key``'s environment variable setups are the equivalent.
+* In these cases ``env`` can either be a string or a list of strings. When a list of strings order is important:
+  in the case of ``redis_dsn`` ``service_redis_dsn`` would take precedence over ``redis_url``.
+
+.. warning::
+
+   Since V1 *pydantic* does not consider field aliases when finding environment variables to populate settings
+   models, use ``env`` instead as described above.
+
+   To aid the transition from aliases to ``env``, a warning will be raised when aliases are used on settings models
+   without a custom env var name. If you really mean to use aliases, either ignore the warning or set ``env`` to
+   suppress it.
 
 By default ``BaseSettings`` considers field values in the following priority (where 3. has the highest priority
 and overrides the other two):
 
 1. The default values set in your ``Settings`` class.
-2. Environment variables, e.g. ``MY_PREFIX_REDIS_PORT`` as described above.
+2. Environment variables, e.g. ``my_prefix_special_function`` as described above.
 3. Arguments passed to the ``Settings`` class on initialisation.
 
-This behaviour can be changed by overriding the ``_build_values`` method on ``BaseSettings``.
+Complex types like ``list``, ``set``, ``dict`` and sub-models can be set by using JSON environment variables.
 
-Complex types like ``list``, ``set``, ``dict`` and submodels can be set by using JSON environment variables.
-
-Case-sensitivity can be turned on through the ``Config``:
+Case-sensitivity can be turned on through ``Config``:
 
 .. literalinclude:: examples/settings_case_sensitive.py
 

pydantic/env_settings.py

@@ -1,7 +1,10 @@
 import os
-from typing import Any, Dict, Optional, cast
+import warnings
+from typing import Any, Dict, Iterable, Mapping, Optional
 
+from .fields import ModelField
 from .main import BaseModel, Extra
+from .typing import display_as_type
 
 
 class SettingsError(ValueError):
@@ -30,26 +33,26 @@ class BaseSettings(BaseModel):
         d: Dict[str, Optional[str]] = {}
 
         if self.__config__.case_sensitive:
-            env_vars = cast(Dict[str, str], os.environ)
+            env_vars: Mapping[str, str] = os.environ
         else:
             env_vars = {k.lower(): v for k, v in os.environ.items()}
 
         for field in self.__fields__.values():
-            if field.has_alias:
-                env_name = field.alias
-            else:
-                env_name = self.__config__.env_prefix + field.name.upper()
+            env_val: Optional[str] = None
+            for env_name in field.field_info.extra['env_names']:  # type: ignore
+                env_val = env_vars.get(env_name)
+                if env_val is not None:
+                    break
 
-            env_name_ = env_name if self.__config__.case_sensitive else env_name.lower()
-            env_val = env_vars.get(env_name_, None)
+            if env_val is None:
+                continue
 
-            if env_val:
-                if field.is_complex():
-                    try:
-                        env_val = self.__config__.json_loads(env_val)  # type: ignore
-                    except ValueError as e:
-                        raise SettingsError(f'error parsing JSON for "{env_name}"') from e
-                d[field.alias] = env_val
+            if field.is_complex():
+                try:
+                    env_val = self.__config__.json_loads(env_val)  # type: ignore
+                except ValueError as e:
+                    raise SettingsError(f'error parsing JSON for "{env_name}"') from e
+            d[field.alias] = env_val
         return d
 
     class Config:
@@ -59,4 +62,30 @@ class BaseSettings(BaseModel):
         arbitrary_types_allowed = True
         case_sensitive = False
 
+        @classmethod
+        def prepare_field(cls, field: ModelField) -> None:
+            if not field.field_info:
+                return
+
+            env_names: Iterable[str]
+            env = field.field_info.extra.pop('env', None)
+            if env is None:
+                if field.has_alias:
+                    warnings.warn(
+                        'aliases are no longer used by BaseSettings to define which environment variables to read. '
+                        'Instead use the "env" field setting. See https://pydantic-docs.helpmanual.io/#settings',
+                        DeprecationWarning,
+                    )
+                env_names = [cls.env_prefix + field.name]
+            elif isinstance(env, str):
+                env_names = {env}
+            elif isinstance(env, (list, set, tuple)):
+                env_names = env
+            else:
+                raise TypeError(f'invalid field env: {env!r} ({display_as_type(env)}); should be string, list or set')
+
+            if not cls.case_sensitive:
+                env_names = type(env_names)(n.lower() for n in env_names)
+            field.field_info.extra['env_names'] = env_names
+
     __config__: Config  # type: ignore

pydantic/fields.py

@@ -229,6 +229,7 @@ class ModelField:
         self.post_validators: Optional['ValidatorsList'] = None
         self.parse_json: bool = False
         self.shape: int = SHAPE_SINGLETON
+        self.model_config.prepare_field(self)
         self.prepare()
 
     @classmethod

pydantic/main.py

@@ -73,7 +73,7 @@ class BaseConfig:
     json_encoders: Dict[AnyType, AnyCallable] = {}
 
     @classmethod
-    def get_field_info(cls, name: str) -> Dict[str, str]:
+    def get_field_info(cls, name: str) -> Dict[str, Any]:
         field_info = cls.fields.get(name) or {}
         if isinstance(field_info, str):
             field_info = {'alias': field_info}
@@ -84,6 +84,13 @@ class BaseConfig:
             field_info['alias'] = alias
         return field_info
 
+    @classmethod
+    def prepare_field(cls, field: 'ModelField') -> None:
+        """
+        Optional hook to check or modify fields during model creation.
+        """
+        pass
+
 
 def inherit_config(self_config: 'ConfigType', parent_config: 'ConfigType') -> 'ConfigType':
     if not self_config:

tests/test_settings.py

@@ -10,19 +10,15 @@ from pydantic.env_settings import SettingsError
 class SimpleSettings(BaseSettings):
     apple: str
 
-    class Config:
-        env_prefix = 'APP_'
-        case_sensitive = True
-
 
 def test_sub_env(env):
-    env.set('APP_APPLE', 'hello')
+    env.set('apple', 'hello')
     s = SimpleSettings()
     assert s.apple == 'hello'
 
 
 def test_sub_env_override(env):
-    env.set('APP_APPLE', 'hello')
+    env.set('apple', 'hello')
     s = SimpleSettings(apple='goodbye')
     assert s.apple == 'goodbye'
 
@@ -33,20 +29,23 @@ def test_sub_env_missing():
     assert exc_info.value.errors() == [{'loc': ('apple',), 'msg': 'field required', 'type': 'value_error.missing'}]
 
 
-def test_other_setting(env):
+def test_other_setting():
     with pytest.raises(ValidationError):
         SimpleSettings(apple='a', foobar=42)
 
 
-def test_env_with_aliass(env):
+def test_with_prefix(env):
     class Settings(BaseSettings):
-        apple: str = ...
+        apple: str
 
         class Config:
-            fields = {'apple': 'BOOM'}
+            env_prefix = 'foobar_'
 
-    env.set('BOOM', 'hello')
-    assert Settings().apple == 'hello'
+    with pytest.raises(ValidationError):
+        Settings()
+    env.set('foobar_apple', 'has_prefix')
+    s = Settings()
+    assert s.apple == 'has_prefix'
 
 
 class DateModel(BaseModel):
@@ -59,22 +58,18 @@ class ComplexSettings(BaseSettings):
     carrots: dict = {}
     date: DateModel = DateModel()
 
-    class Config:
-        env_prefix = 'APP_'
-        case_sensitive = True
-
 
 def test_list(env):
-    env.set('APP_APPLES', '["russet", "granny smith"]')
+    env.set('apples', '["russet", "granny smith"]')
     s = ComplexSettings()
     assert s.apples == ['russet', 'granny smith']
     assert s.date.pips is False
 
 
 def test_set_dict_model(env):
-    env.set('APP_BANANAS', '[1, 2, 3, 3]')
-    env.set('APP_CARROTS', '{"a": null, "b": 4}')
-    env.set('APP_DATE', '{"pips": true}')
+    env.set('bananas', '[1, 2, 3, 3]')
+    env.set('CARROTS', '{"a": null, "b": 4}')
+    env.set('daTE', '{"pips": true}')
     s = ComplexSettings()
     assert s.bananas == {1, 2, 3}
     assert s.carrots == {'a': None, 'b': 4}
@@ -82,8 +77,8 @@ def test_set_dict_model(env):
 
 
 def test_invalid_json(env):
-    env.set('APP_APPLES', '["russet", "granny smith",]')
-    with pytest.raises(SettingsError):
+    env.set('apples', '["russet", "granny smith",]')
+    with pytest.raises(SettingsError, match='error parsing JSON for "apples"'):
         ComplexSettings()
 
 
@@ -107,37 +102,145 @@ def test_non_class(env):
     assert s.foobar == 'xxx'
 
 
-def test_alias_matches_name(env):
+def test_env_str(env):
+    class Settings(BaseSettings):
+        apple: str = ...
+
+        class Config:
+            fields = {'apple': {'env': 'BOOM'}}
+
+    env.set('BOOM', 'hello')
+    assert Settings().apple == 'hello'
+
+
+def test_env_list(env):
     class Settings(BaseSettings):
         foobar: str
 
         class Config:
-            fields = {'foobar': 'foobar'}
+            fields = {'foobar': {'env': ['different1', 'different2']}}
 
-    env.set('foobar', 'xxx')
+    env.set('different1', 'value 1')
+    env.set('different2', 'value 2')
     s = Settings()
-    assert s.foobar == 'xxx'
+    assert s.foobar == 'value 1'
 
 
-def test_case_insensitive(env):
+def test_env_list_field(env):
     class Settings(BaseSettings):
-        foo: str
-        bAR: str
+        foobar: str = Field(..., env='foobar_env_name')
+
+    env.set('FOOBAR_ENV_NAME', 'env value')
+    s = Settings()
+    assert s.foobar == 'env value'
+
+
+def test_env_list_last(env):
+    class Settings(BaseSettings):
+        foobar: str
 
         class Config:
-            env_prefix = 'APP_'
-            case_sensitive = False
+            fields = {'foobar': {'env': ['different2']}}
 
-    env.set('apP_foO', 'foo')
-    env.set('app_bar', 'bar')
+    env.set('different1', 'value 1')
+    env.set('different2', 'value 2')
     s = Settings()
-    assert s.foo == 'foo'
-    assert s.bAR == 'bar'
+    assert s.foobar == 'value 2'
+    assert Settings(foobar='abc').foobar == 'abc'
+
+
+def test_env_inheritance(env):
+    class SettingsParent(BaseSettings):
+        foobar: str = 'parent default'
+
+        class Config:
+            fields = {'foobar': {'env': 'different'}}
+
+    class SettingsChild(SettingsParent):
+        foobar: str = 'child default'
+
+    assert SettingsParent().foobar == 'parent default'
+    assert SettingsParent(foobar='abc').foobar == 'abc'
+
+    assert SettingsChild().foobar == 'child default'
+    assert SettingsChild(foobar='abc').foobar == 'abc'
+    env.set('different', 'env value')
+    assert SettingsParent().foobar == 'env value'
+    assert SettingsParent(foobar='abc').foobar == 'abc'
+    assert SettingsChild().foobar == 'env value'
+    assert SettingsChild(foobar='abc').foobar == 'abc'
+
+
+def test_env_inheritance_field(env):
+    class SettingsParent(BaseSettings):
+        foobar: str = Field('parent default', env='foobar_env')
+
+    class SettingsChild(SettingsParent):
+        foobar: str = 'child default'
+
+    assert SettingsParent().foobar == 'parent default'
+    assert SettingsParent(foobar='abc').foobar == 'abc'
+
+    assert SettingsChild().foobar == 'child default'
+    assert SettingsChild(foobar='abc').foobar == 'abc'
+    env.set('foobar_env', 'env value')
+    assert SettingsParent().foobar == 'env value'
+    assert SettingsParent(foobar='abc').foobar == 'abc'
+    assert SettingsChild().foobar == 'child default'
+    assert SettingsChild(foobar='abc').foobar == 'abc'
+
+
+def test_env_invalid(env):
+    with pytest.raises(TypeError, match=r'invalid field env: 123 \(int\); should be string, list or set'):
+
+        class Settings(BaseSettings):
+            foobar: str
+
+            class Config:
+                fields = {'foobar': {'env': 123}}
+
+
+def test_env_field(env):
+    with pytest.raises(TypeError, match=r'invalid field env: 123 \(int\); should be string, list or set'):
+
+        class Settings(BaseSettings):
+            foobar: str = Field(..., env=123)
+
+
+def test_aliases_warning(env):
+    with pytest.warns(DeprecationWarning, match='aliases are no longer used by BaseSettings'):
+
+        class Settings(BaseSettings):
+            foobar: str = 'default value'
+
+            class Config:
+                fields = {'foobar': 'foobar_alias'}
+
+    assert Settings().foobar == 'default value'
+    env.set('foobar_alias', 'xxx')
+    assert Settings().foobar == 'default value'
+    assert Settings(foobar_alias='42').foobar == '42'
+
+
+def test_aliases_no_warning(env):
+    class Settings(BaseSettings):
+        foobar: str = 'default value'
+
+        class Config:
+            fields = {'foobar': {'alias': 'foobar_alias', 'env': 'foobar_env'}}
+
+    assert Settings().foobar == 'default value'
+    assert Settings(foobar_alias='42').foobar == '42'
+    env.set('foobar_alias', 'xxx')
+    assert Settings().foobar == 'default value'
+    env.set('foobar_env', 'xxx')
+    assert Settings().foobar == 'xxx'
+    assert Settings(foobar_alias='42').foobar == '42'
 
 
 def test_case_sensitive(monkeypatch):
     class Settings(BaseSettings):
-        foo: str = Field(..., alias='foo')
+        foo: str
 
         class Config:
             case_sensitive = True
@@ -165,7 +268,7 @@ def test_nested_dataclass(env):
     assert s.n.bar == 'bar value'
 
 
-def test_config_file_settings(env):
+def test_env_takes_precedence(env):
     class Settings(BaseSettings):
         foo: int
         bar: str