feat(settings): allow custom encoding for dotenv files (#1620)

closes #1615

changes/1615-PrettyWood.md

@@ -0,0 +1 @@
+Allow custom encoding for `dotenv` files.

docs/usage/settings.md

@@ -91,7 +91,8 @@ MY_VAR='Hello world'
 
 Once you have your `.env` file filled with variables, *pydantic* supports loading it in two ways:
 
-**1.** setting `env_file` on `Config` in a `BaseSettings` class:
+**1.** setting `env_file` (and `env_file_encoding` if you don't want the default encoding of your OS) on `Config`
+in a `BaseSettings` class:
 
 ```py
 class Settings(BaseSettings):
@@ -99,12 +100,14 @@ class Settings(BaseSettings):
 
     class Config:
         env_file = '.env'
+        env_file_encoding = 'utf-8'
 ```
 
-**2.** instantiating a `BaseSettings` derived class with the `_env_file` keyword argument:
+**2.** instantiating a `BaseSettings` derived class with the `_env_file` keyword argument
+(and the `_env_file_encoding` if needed):
 
 ```py
-settings = Settings(_env_file='prod.env')
+settings = Settings(_env_file='prod.env', _env_file_encoding='utf-8')
 ```
 
 In either case, the value of the passed argument can be any valid path or filename, either absolute or relative to the

pydantic/env_settings.py

@@ -23,14 +23,28 @@ class BaseSettings(BaseModel):
     Heroku and any 12 factor app design.
     """
 
-    def __init__(__pydantic_self__, _env_file: Union[Path, str, None] = env_file_sentinel, **values: Any) -> None:
+    def __init__(
+        __pydantic_self__,
+        _env_file: Union[Path, str, None] = env_file_sentinel,
+        _env_file_encoding: Optional[str] = None,
+        **values: Any,
+    ) -> None:
         # Uses something other than `self` the first arg to allow "self" as a settable attribute
-        super().__init__(**__pydantic_self__._build_values(values, _env_file=_env_file))
+        super().__init__(
+            **__pydantic_self__._build_values(values, _env_file=_env_file, _env_file_encoding=_env_file_encoding)
+        )
 
-    def _build_values(self, init_kwargs: Dict[str, Any], _env_file: Union[Path, str, None] = None) -> Dict[str, Any]:
-        return deep_update(self._build_environ(_env_file), init_kwargs)
+    def _build_values(
+        self,
+        init_kwargs: Dict[str, Any],
+        _env_file: Union[Path, str, None] = None,
+        _env_file_encoding: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        return deep_update(self._build_environ(_env_file, _env_file_encoding), init_kwargs)
 
-    def _build_environ(self, _env_file: Union[Path, str, None] = None) -> Dict[str, Optional[str]]:
+    def _build_environ(
+        self, _env_file: Union[Path, str, None] = None, _env_file_encoding: Optional[str] = None
+    ) -> Dict[str, Optional[str]]:
         """
         Build environment variables suitable for passing to the Model.
         """
@@ -42,10 +56,16 @@ class BaseSettings(BaseModel):
             env_vars = {k.lower(): v for k, v in os.environ.items()}
 
         env_file = _env_file if _env_file != env_file_sentinel else self.__config__.env_file
+        env_file_encoding = _env_file_encoding if _env_file_encoding is not None else self.__config__.env_file_encoding
         if env_file is not None:
             env_path = Path(env_file)
             if env_path.is_file():
-                env_vars = {**read_env_file(env_path, case_sensitive=self.__config__.case_sensitive), **env_vars}
+                env_vars = {
+                    **read_env_file(
+                        env_path, encoding=env_file_encoding, case_sensitive=self.__config__.case_sensitive
+                    ),
+                    **env_vars,
+                }
 
         for field in self.__fields__.values():
             env_val: Optional[str] = None
@@ -68,6 +88,7 @@ class BaseSettings(BaseModel):
     class Config:
         env_prefix = ''
         env_file = None
+        env_file_encoding = None
         validate_all = True
         extra = Extra.forbid
         arbitrary_types_allowed = True
@@ -102,13 +123,13 @@ class BaseSettings(BaseModel):
     __config__: Config  # type: ignore
 
 
-def read_env_file(file_path: Path, *, case_sensitive: bool = False) -> Dict[str, Optional[str]]:
+def read_env_file(file_path: Path, *, encoding: str = None, case_sensitive: bool = False) -> Dict[str, Optional[str]]:
     try:
         from dotenv import dotenv_values
     except ImportError as e:
         raise ImportError('python-dotenv is not installed, run `pip install pydantic[dotenv]`') from e
 
-    file_vars: Dict[str, Optional[str]] = dotenv_values(file_path)
+    file_vars: Dict[str, Optional[str]] = dotenv_values(file_path, encoding=encoding)
     if not case_sensitive:
         return {k.lower(): v for k, v in file_vars.items()}
     else:

tests/test_settings.py

@@ -320,7 +320,7 @@ def test_env_takes_precedence(env):
         foo: int
         bar: str
 
-        def _build_values(self, init_kwargs, _env_file):
+        def _build_values(self, init_kwargs, _env_file, _env_file_encoding):
             return {**init_kwargs, **self._build_environ()}
 
     env.set('BAR', 'env setting')
@@ -340,7 +340,7 @@ def test_config_file_settings_nornir(env):
         b: str
         c: str
 
-        def _build_values(self, init_kwargs, _env_file):
+        def _build_values(self, init_kwargs, _env_file, _env_file_encoding):
             config_settings = init_kwargs.pop('__config_settings__')
             return {**config_settings, **init_kwargs, **self._build_environ()}
 
@@ -430,6 +430,22 @@ export C="best string"
     assert s.c == 'best string'
 
 
+@pytest.mark.skipif(not dotenv, reason='python-dotenv not installed')
+def test_env_file_config_custom_encoding(tmp_path):
+    p = tmp_path / '.env'
+    p.write_text('pika=p!±@', encoding='latin-1')
+
+    class Settings(BaseSettings):
+        pika: str
+
+        class Config:
+            env_file = p
+            env_file_encoding = 'latin-1'
+
+    s = Settings()
+    assert s.pika == 'p!±@'
+
+
 @pytest.mark.skipif(not dotenv, reason='python-dotenv not installed')
 def test_env_file_none(tmp_path):
     p = tmp_path / '.env'
@@ -529,6 +545,21 @@ MY_VAR='Hello world'
     }
 
 
+@pytest.mark.skipif(not dotenv, reason='python-dotenv not installed')
+def test_env_file_custom_encoding(tmp_path):
+    p = tmp_path / '.env'
+    p.write_text('pika=p!±@', encoding='latin-1')
+
+    class Settings(BaseSettings):
+        pika: str
+
+    with pytest.raises(UnicodeDecodeError):
+        Settings(_env_file=str(p))
+
+    s = Settings(_env_file=str(p), _env_file_encoding='latin-1')
+    assert s.dict() == {'pika': 'p!±@'}
+
+
 @pytest.mark.skipif(dotenv, reason='python-dotenv is installed')
 def test_dotenv_not_installed(tmp_path):
     p = tmp_path / '.env'