From 524e31ff5a704fea576785f9ef823e969fcff62e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 6 Nov 2018 04:35:15 -0500 Subject: [PATCH] Switch to tomlkit for parsing and writing - Update tomlkit to preserve inline comments when deleting elemeents Signed-off-by: Dan Ryan --- pipenv/project.py | 58 ++++++++++++++----- pipenv/vendor/tomlkit/items.py | 12 ++++ .../patches/vendor/tomlkit-update-items.patch | 23 ++++++++ 3 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 tasks/vendoring/patches/vendor/tomlkit-update-items.patch diff --git a/pipenv/project.py b/pipenv/project.py index a60271e9..1354eff0 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -17,6 +17,7 @@ import pipfile.api import six import vistir import toml +import tomlkit from .cmdparse import Script from .utils import ( @@ -154,6 +155,9 @@ class Project(object): self._requirements_location = None self._original_dir = os.path.abspath(os.curdir) self._which = which + self._build_system = { + "requires": ["setuptools", "wheel"] + } self.python_version = python_version # Hack to skip this during pipenv run, or -r. if ("run" not in sys.argv) and chdir: @@ -574,32 +578,55 @@ class Project(object): _pipfile_cache.clear() def _parse_pipfile(self, contents): + toml_encoder = toml.TomlPreserveInlineDictEncoder() + toml_encoder.get_empty_table() # If any outline tables are present... if ("[packages." in contents) or ("[dev-packages." in contents): - data = toml.loads(contents) + data = tomlkit.parse(contents) # Convert all outline tables to inline tables. for section in ("packages", "dev-packages"): for package in data.get(section, {}): # Convert things to inline tables — fancy :) if hasattr(data[section][package], "keys"): + table = tomlkit.inline_table() _data = data[section][package] - data[section][package] = toml.TomlDecoder().get_empty_inline_table() - data[section][package].update(_data) - toml_encoder = toml.TomlEncoder(preserve=True) + table.update(_data) + data[section][package] = table # We lose comments here, but it's for the best.) try: - return contoml.loads(toml.dumps(data, encoder=toml_encoder)) + return data except RuntimeError: - return toml.loads(toml.dumps(data, encoder=toml_encoder)) + return toml.loads(tomlkit.dumps(data, encoder=toml_encoder)) else: # Fallback to toml parser, for large files. try: - return contoml.loads(contents) + return tomlkit.loads(contents) except Exception: - return toml.loads(contents) + return toml.loads(contents, encoder=toml_encoder) + + def _read_pyproject(self): + pyproject = self.path_to("pyproject.toml") + if os.path.exists(pyproject): + self._pyproject = toml.load(pyproject) + build_system = self._pyproject.get("build-system", None) + if not os.path.exists(self.path_to("setup.py")): + if not build_system or not build_system.get("requires"): + build_system = { + "requires": ["setuptools>=38.2.5", "wheel"], + "build-backend": "setuptools.build_meta", + } + self._build_system = build_system + + @property + def build_requires(self): + return self._build_system.get("requires", []) + + @property + def build_backend(self): + return self._build_system.get("build-backend", None) @property def settings(self): @@ -833,16 +860,21 @@ class Project(object): if path is None: path = self.pipfile_location try: - formatted_data = contoml.dumps(data).rstrip() + formatted_data = tomlkit.dumps(data).rstrip() except Exception: + document = tomlkit.document() for section in ("packages", "dev-packages"): + document[section] = tomlkit.container.Table() for package in data.get(section, {}): # Convert things to inline tables — fancy :) - if hasattr(data[section][package], "keys"): _data = data[section][package] - data[section][package] = toml.TomlDecoder().get_empty_inline_table() - data[section][package].update(_data) - formatted_data = toml.dumps(data).rstrip() + if hasattr(_data, "keys"): + table = tomlkit.inline_table() + table.update(data) + document[section][package] = table + else: + document[section][package] = tomlkit.string(_data) + formatted_data = tomlkit.dumps(document).rstrip() if ( vistir.compat.Path(path).absolute() diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 781e2e98..5b2fd15f 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -789,6 +789,18 @@ class Table(Item, dict): return self def remove(self, key): # type: (Union[Key, str]) -> Table + data = [ + (k, v) for k, v in self._value.body + if getattr(k, "key", "") == getattr(key, "key", key) + and getattr(v, "trivia", None) + ] + indexed_keys = [(i, k) for i, k in enumerate(self.keys())] + target_idx = next(iter(i for i, k in indexed_keys if k == data[0][0].key), None) + if target_idx == 0: + self.comment(data[0][1].trivia.comment) + else: + target_key = self._value.body[target_idx - 1][0].key + self._value[target_key].comment(data[0][1].trivia.comment) self._value.remove(key) if isinstance(key, Key): diff --git a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch new file mode 100644 index 00000000..316a548d --- /dev/null +++ b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch @@ -0,0 +1,23 @@ +diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py +index 781e2e98..5b2fd15f 100644 +--- a/pipenv/vendor/tomlkit/items.py ++++ b/pipenv/vendor/tomlkit/items.py +@@ -789,6 +789,18 @@ class Table(Item, dict): + return self + + def remove(self, key): # type: (Union[Key, str]) -> Table ++ data = [ ++ (k, v) for k, v in self._value.body ++ if getattr(k, "key", "") == getattr(key, "key", key) ++ and getattr(v, "trivia", None) ++ ] ++ indexed_keys = [(i, k) for i, k in enumerate(self.keys())] ++ target_idx = next(iter(i for i, k in indexed_keys if k == data[0][0].key), None) ++ if target_idx == 0: ++ self.comment(data[0][1].trivia.comment) ++ else: ++ target_key = self._value.body[target_idx - 1][0].key ++ self._value[target_key].comment(data[0][1].trivia.comment) + self._value.remove(key) + + if isinstance(key, Key):