Toml support for pydantic-mypy plugin config (#2908)

* add toml reader

* fix path

* skip configparser step

* fix quotes

* full pyproject.toml check

* add doc note

* cleaner formatting, raise ValueError for non-bool

* fix tests

* add bad config test case

* add changelog file.

* bump mypy to 0.902

* tweak change MD, fix formatting in requirements

* import check around toml

* switch to tomli for parsing to match mypy dependency

* import check around toml/tomli

* add note on tomli usage

* more succinct changelog entry

* fix quotes in changelog

* linting fixes, remove unnecessary stub install

* mypy checks on mypy plugin file

* wrongly placed pragma no cover

Co-authored-by: PrettyWood <em.jolibois@gmail.com>

changes/2908-jrwalk.md

@@ -0,0 +1 @@
+Make `pydantic-mypy` plugin compatible with `pyproject.toml` configuration, consistent with `mypy` changes. See the [doc](https://pydantic-docs.helpmanual.io/mypy_plugin/#configuring-the-plugin) for more information.

docs/mypy_plugin.md

@@ -138,3 +138,28 @@ init_typed = True
 warn_required_dynamic_aliases = True
 warn_untyped_fields = True
 ```
+
+As of `mypy>=0.900`, mypy config may also be included in the `pyproject.toml` file rather than `mypy.ini`.
+The same configuration as above would be:
+```toml
+[tool.mypy]
+plugins = [
+  "pydantic.mypy"
+]
+
+follow_imports = "silent"
+warn_redundant_casts = true
+warn_unused_ignores = true
+disallow_any_generics = true
+check_untyped_defs = true
+no_implicit_reexport = true
+
+# for strict mypy: (this is the tricky one :-))
+disallow_untyped_defs = true
+
+[tool.pydantic-mypy]
+init_forbid_extra = true
+init_typed = true
+warn_required_dynamic_aliases = true
+warn_untyped_fields = true
+```

pydantic/mypy.py

@@ -1,6 +1,17 @@
 from configparser import ConfigParser
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type as TypingType, Union
 
+try:
+    import toml
+except ImportError:  # pragma: no cover
+    # future-proofing for upcoming `mypy` releases which will switch dependencies
+    try:
+        import tomli as toml  # type: ignore
+    except ImportError:
+        import warnings
+
+        warnings.warn('No TOML parser installed, cannot read configuration from `pyproject.toml`.')
+        toml = None  # type: ignore
 from mypy.errorcodes import ErrorCode
 from mypy.nodes import (
     ARG_NAMED,
@@ -110,11 +121,21 @@ class PydanticPluginConfig:
     def __init__(self, options: Options) -> None:
         if options.config_file is None:  # pragma: no cover
             return
-        plugin_config = ConfigParser()
-        plugin_config.read(options.config_file)
-        for key in self.__slots__:
-            setting = plugin_config.getboolean(CONFIGFILE_KEY, key, fallback=False)
-            setattr(self, key, setting)
+
+        if toml and options.config_file.endswith('.toml'):
+            with open(options.config_file, 'r') as rf:
+                config = toml.load(rf).get('tool', {}).get('pydantic-mypy', {})
+            for key in self.__slots__:
+                setting = config.get(key, False)
+                if not isinstance(setting, bool):
+                    raise ValueError(f'Configuration value must be a boolean for key: {key}')
+                setattr(self, key, setting)
+        else:
+            plugin_config = ConfigParser()
+            plugin_config.read(options.config_file)
+            for key in self.__slots__:
+                setting = plugin_config.getboolean(CONFIGFILE_KEY, key, fallback=False)
+                setattr(self, key, setting)
 
 
 def from_orm_callback(ctx: MethodContext) -> Type:

tests/mypy/configs/pyproject-default.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["poetry>=0.12"]
+build_backend = "poetry.masonry.api"
+
+[tool.poetry]
+name = "test"
+version = "0.0.1"
+readme = "README.md"
+authors = [
+    "author@example.com"
+]
+
+[tool.poetry.dependencies]
+python = "*"
+
+[tool.pytest.ini_options]
+addopts = "-v -p no:warnings"
+
+[tool.mypy]
+follow_imports = "silent"
+strict_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+disallow_any_generics = true
+check_untyped_defs = true
+no_implicit_reexport = true
+disallow_untyped_defs = true

tests/mypy/configs/pyproject-plugin-bad-param.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["poetry>=0.12"]
+build_backend = "poetry.masonry.api"
+
+[tool.poetry]
+name = "test"
+version = "0.0.1"
+readme = "README.md"
+authors = [
+    "author@example.com"
+]
+
+[tool.poetry.dependencies]
+python = "*"
+
+[tool.pytest.ini_options]
+addopts = "-v -p no:warnings"
+
+[tool.mypy]
+plugins = [
+    "pydantic.mypy"
+]
+follow_imports = "silent"
+strict_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+disallow_any_generics = true
+check_untyped_defs = true
+no_implicit_reexport = true
+disallow_untyped_defs = true
+
+[tool.pydantic-mypy]
+init_forbid_extra = "foo"  # this will raise a ValueError for the config

tests/mypy/configs/pyproject-plugin-strict.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["poetry>=0.12"]
+build_backend = "poetry.masonry.api"
+
+[tool.poetry]
+name = "test"
+version = "0.0.1"
+readme = "README.md"
+authors = [
+    "author@example.com"
+]
+
+[tool.poetry.dependencies]
+python = "*"
+
+[tool.pytest.ini_options]
+addopts = "-v -p no:warnings"
+
+[tool.mypy]
+plugins = [
+    "pydantic.mypy"
+]
+follow_imports = "silent"
+strict_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+disallow_any_generics = true
+check_untyped_defs = true
+no_implicit_reexport = true
+disallow_untyped_defs = true
+
+[tool.pydantic-mypy]
+init_forbid_extra = true
+init_typed = true
+warn_required_dynamic_aliases = true
+warn_untyped_fields = true

tests/mypy/configs/pyproject-plugin.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["poetry>=0.12"]
+build_backend = "poetry.masonry.api"
+
+[tool.poetry]
+name = "test"
+version = "0.0.1"
+readme = "README.md"
+authors = [
+    "author@example.com"
+]
+
+[tool.poetry.dependencies]
+python = "*"
+
+[tool.pytest.ini_options]
+addopts = "-v -p no:warnings"
+
+[tool.mypy]
+plugins = [
+    "pydantic.mypy"
+]
+follow_imports = "silent"
+strict_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+disallow_any_generics = true
+check_untyped_defs = true
+no_implicit_reexport = true
+disallow_untyped_defs = true

tests/mypy/test_mypy.py

@@ -8,12 +8,12 @@ import pytest
 try:
     from mypy import api as mypy_api
 except ImportError:
-    mypy_api = None
+    mypy_api = None  # type: ignore
 
 try:
     import dotenv
 except ImportError:
-    dotenv = None
+    dotenv = None  # type: ignore
 
 # This ensures mypy can find the test files, no matter where tests are run from:
 os.chdir(Path(__file__).parent.parent.parent)
@@ -29,20 +29,29 @@ cases = [
     ('mypy-default.ini', 'fail3.py', 'fail3.txt'),
     ('mypy-default.ini', 'fail4.py', 'fail4.txt'),
     ('mypy-default.ini', 'plugin_success.py', 'plugin_success.txt'),
+    ('pyproject-default.toml', 'success.py', None),
+    ('pyproject-default.toml', 'fail1.py', 'fail1.txt'),
+    ('pyproject-default.toml', 'fail2.py', 'fail2.txt'),
+    ('pyproject-default.toml', 'fail3.py', 'fail3.txt'),
+    ('pyproject-default.toml', 'fail4.py', 'fail4.txt'),
+    ('pyproject-plugin.toml', 'plugin_success.py', None),
+    ('pyproject-plugin.toml', 'plugin_fail.py', 'plugin-fail.txt'),
+    ('pyproject-plugin-strict.toml', 'plugin_success.py', 'plugin-success-strict.txt'),
+    ('pyproject-plugin-strict.toml', 'plugin_fail.py', 'plugin-fail-strict.txt'),
 ]
 executable_modules = list({fname[:-3] for _, fname, out_fname in cases if out_fname is None})
 
 
 @pytest.mark.skipif(not (dotenv and mypy_api), reason='dotenv or mypy are not installed')
 @pytest.mark.parametrize('config_filename,python_filename,output_filename', cases)
-def test_mypy_results(config_filename, python_filename, output_filename):
+def test_mypy_results(config_filename: str, python_filename: str, output_filename: str) -> None:
     full_config_filename = f'tests/mypy/configs/{config_filename}'
     full_filename = f'tests/mypy/modules/{python_filename}'
     output_path = None if output_filename is None else Path(f'tests/mypy/outputs/{output_filename}')
 
     # Specifying a different cache dir for each configuration dramatically speeds up subsequent execution
     # It also prevents cache-invalidation-related bugs in the tests
-    cache_dir = f'.mypy_cache/test-{config_filename[:-4]}'
+    cache_dir = f'.mypy_cache/test-{os.path.splitext(config_filename)[0]}'
     command = [full_filename, '--config-file', full_config_filename, '--cache-dir', cache_dir, '--show-error-codes']
     print(f"\nExecuting: mypy {' '.join(command)}")  # makes it easier to debug as necessary
     actual_result = mypy_api.run(command)
@@ -66,15 +75,31 @@ def test_mypy_results(config_filename, python_filename, output_filename):
     assert actual_out == expected_out, actual_out
 
 
+@pytest.mark.skipif(not (dotenv and mypy_api), reason='dotenv or mypy are not installed')
+def test_bad_toml_config() -> None:
+    full_config_filename = 'tests/mypy/configs/pyproject-plugin-bad-param.toml'
+    full_filename = 'tests/mypy/modules/success.py'
+
+    # Specifying a different cache dir for each configuration dramatically speeds up subsequent execution
+    # It also prevents cache-invalidation-related bugs in the tests
+    cache_dir = '.mypy_cache/test-pyproject-plugin-bad-param'
+    command = [full_filename, '--config-file', full_config_filename, '--cache-dir', cache_dir, '--show-error-codes']
+    print(f"\nExecuting: mypy {' '.join(command)}")  # makes it easier to debug as necessary
+    with pytest.raises(ValueError) as e:
+        mypy_api.run(command)
+
+    assert str(e.value) == 'Configuration value must be a boolean for key: init_forbid_extra'
+
+
 @pytest.mark.parametrize('module', executable_modules)
-def test_success_cases_run(module):
+def test_success_cases_run(module: str) -> None:
     """
     Ensure the "success" files can actually be executed
     """
     importlib.import_module(f'tests.mypy.modules.{module}')
 
 
-def test_explicit_reexports():
+def test_explicit_reexports() -> None:
     from pydantic import __all__ as root_all
     from pydantic.main import __all__ as main
     from pydantic.networks import __all__ as networks

tests/requirements-linting.txt

@@ -4,6 +4,7 @@ flake8-quotes==3.3.0
 hypothesis==6.17.4
 isort==5.9.3
 mypy==0.910
+types-toml==0.1.5
 pycodestyle==2.7.0
 pyflakes==2.3.1
 twine==3.4.2