From 0971363d23258a97170ba54d77dcbdcab7647c31 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Thu, 8 Sep 2022 01:50:10 +0200 Subject: [PATCH 01/26] Replace pipfile.Pipfile.find with our own This is because pipfile is quite dormant. --- pipenv/project.py | 5 ++++- pipenv/utils/pipfile.py | 47 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 pipenv/utils/pipfile.py diff --git a/pipenv/project.py b/pipenv/project.py index f9d53ba3..66889227 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -436,12 +436,15 @@ class Project: @property def pipfile_location(self) -> str: + + from pipenv.utils.pipfile import find_pipfile + if self.s.PIPENV_PIPFILE: return self.s.PIPENV_PIPFILE if self._pipfile_location is None: try: - loc = pipfile.Pipfile.find(max_depth=self.s.PIPENV_MAX_DEPTH) + loc = find_pipfile(max_depth=self.s.PIPENV_MAX_DEPTH) except RuntimeError: loc = "Pipfile" self._pipfile_location = normalize_pipfile_path(loc) diff --git a/pipenv/utils/pipfile.py b/pipenv/utils/pipfile.py new file mode 100644 index 00000000..6354cdbc --- /dev/null +++ b/pipenv/utils/pipfile.py @@ -0,0 +1,47 @@ +import os + + +def walk_up(bottom): + """mimic os.walk, but walk 'up' instead of down the directory tree. + From: https://gist.github.com/zdavkeos/1098474 + """ + + bottom = os.path.realpath(bottom) + + # get files in current dir + try: + names = os.listdir(bottom) + except Exception: + return + + dirs, nondirs = [], [] + for name in names: + if os.path.isdir(os.path.join(bottom, name)): + dirs.append(name) + else: + nondirs.append(name) + + yield bottom, dirs, nondirs + + new_path = os.path.realpath(os.path.join(bottom, "..")) + + # see if we are at the top + if new_path == bottom: + return + + for x in walk_up(new_path): + yield x + + +def find_pipfile(max_depth=3): + """Returns the path of a Pipfile in parent directories.""" + i = 0 + for c, _, _ in walk_up(os.getcwd()): + i += 1 + + if i < max_depth: + if "Pipfile": + p = os.path.join(c, "Pipfile") + if os.path.isfile(p): + return p + raise RuntimeError("No Pipfile found!") From 79aeff5eba479d17e5d8742583a637ab10e45618 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Mon, 12 Sep 2022 23:14:05 +0200 Subject: [PATCH 02/26] Replace pipfile.Pipefile with plette.Pipefile Plette has a better code base and the same API. --- pipenv/core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index f6d012b6..428b8d66 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -14,7 +14,6 @@ from typing import Dict, List, Optional, Union from pipenv import environments, exceptions, pep508checker from pipenv._compat import decode_for_output, fix_utf8 -from pipenv.patched import pipfile from pipenv.patched.pip._internal.build_env import _get_runnable_pip from pipenv.patched.pip._internal.exceptions import PipError from pipenv.patched.pip._internal.network.session import PipSession @@ -49,7 +48,7 @@ from pipenv.utils.shell import ( system_which, ) from pipenv.utils.spinner import create_spinner -from pipenv.vendor import click, dotenv, vistir +from pipenv.vendor import click, dotenv, plette, vistir from pipenv.vendor.requirementslib.models.requirements import Requirement if MYPY_RUNNING: @@ -2832,7 +2831,7 @@ def do_check( ) sys.exit(1) # Load the pipfile. - p = pipfile.Pipfile.load(project.pipfile_location) + p = plette.Pipfile.load(project.pipfile_location) failed = False # Assert each specified requirement. for marker, specifier in p.data["_meta"]["requires"].items(): From 097e253d1dc9a15fe299187fe67aa5acca505347 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Mon, 12 Sep 2022 23:22:46 +0200 Subject: [PATCH 03/26] Remove patched pipefile and patch --- pipenv/patched/pipfile/LICENSE | 3 - pipenv/patched/pipfile/LICENSE.APACHE | 177 -------------- pipenv/patched/pipfile/LICENSE.BSD | 23 -- pipenv/patched/pipfile/__about__.py | 21 -- pipenv/patched/pipfile/__init__.py | 11 - pipenv/patched/pipfile/api.py | 230 ------------------ tasks/vendoring/patches/patched/pipfile.patch | 125 ---------- 7 files changed, 590 deletions(-) delete mode 100644 pipenv/patched/pipfile/LICENSE delete mode 100644 pipenv/patched/pipfile/LICENSE.APACHE delete mode 100644 pipenv/patched/pipfile/LICENSE.BSD delete mode 100644 pipenv/patched/pipfile/__about__.py delete mode 100644 pipenv/patched/pipfile/__init__.py delete mode 100644 pipenv/patched/pipfile/api.py delete mode 100644 tasks/vendoring/patches/patched/pipfile.patch diff --git a/pipenv/patched/pipfile/LICENSE b/pipenv/patched/pipfile/LICENSE deleted file mode 100644 index 6f62d44e..00000000 --- a/pipenv/patched/pipfile/LICENSE +++ /dev/null @@ -1,3 +0,0 @@ -This software is made available under the terms of *either* of the licenses -found in LICENSE.APACHE or LICENSE.BSD. Contributions to this software is made -under the terms of *both* these licenses. diff --git a/pipenv/patched/pipfile/LICENSE.APACHE b/pipenv/patched/pipfile/LICENSE.APACHE deleted file mode 100644 index 4947287f..00000000 --- a/pipenv/patched/pipfile/LICENSE.APACHE +++ /dev/null @@ -1,177 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/pipenv/patched/pipfile/LICENSE.BSD b/pipenv/patched/pipfile/LICENSE.BSD deleted file mode 100644 index 698fc43e..00000000 --- a/pipenv/patched/pipfile/LICENSE.BSD +++ /dev/null @@ -1,23 +0,0 @@ -Copyright (c) Kenneth Reitz and individual contributors. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/patched/pipfile/__about__.py b/pipenv/patched/pipfile/__about__.py deleted file mode 100644 index 3ba72191..00000000 --- a/pipenv/patched/pipfile/__about__.py +++ /dev/null @@ -1,21 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. -from __future__ import absolute_import, division, print_function - -__all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", -] - -__title__ = "pipfile" -__summary__ = "" -__uri__ = "https://github.com/pypa/pipfile" - -__version__ = "0.0.2" - -__author__ = "Kenneth Reitz and individual contributors" -__email__ = "me@kennethreitz.org" - -__license__ = "BSD or Apache License, Version 2.0" -__copyright__ = "Copyright 2017 %s" % __author__ diff --git a/pipenv/patched/pipfile/__init__.py b/pipenv/patched/pipfile/__init__.py deleted file mode 100644 index fddd4f90..00000000 --- a/pipenv/patched/pipfile/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. -from __future__ import absolute_import, division, print_function - -from .__about__ import ( - __author__, __copyright__, __email__, __license__, __summary__, __title__, - __uri__, __version__ -) - -from .api import load, Pipfile diff --git a/pipenv/patched/pipfile/api.py b/pipenv/patched/pipfile/api.py deleted file mode 100644 index ae8ee252..00000000 --- a/pipenv/patched/pipfile/api.py +++ /dev/null @@ -1,230 +0,0 @@ -import pipenv.vendor.toml as toml - -import codecs -import json -import hashlib -import platform -import pipenv.vendor.six as six -import sys -import os - - -DEFAULT_SOURCE = { - u'url': u'https://pypi.org/simple', - u'verify_ssl': True, - u'name': u'pypi', -} - - -def format_full_version(info): - version = '{0.major}.{0.minor}.{0.micro}'.format(info) - kind = info.releaselevel - if kind != 'final': - version += kind[0] + str(info.serial) - return version - - -def walk_up(bottom): - """mimic os.walk, but walk 'up' instead of down the directory tree. - From: https://gist.github.com/zdavkeos/1098474 - """ - - bottom = os.path.realpath(bottom) - - # get files in current dir - try: - names = os.listdir(bottom) - except Exception: - return - - dirs, nondirs = [], [] - for name in names: - if os.path.isdir(os.path.join(bottom, name)): - dirs.append(name) - else: - nondirs.append(name) - - yield bottom, dirs, nondirs - - new_path = os.path.realpath(os.path.join(bottom, '..')) - - # see if we are at the top - if new_path == bottom: - return - - for x in walk_up(new_path): - yield x - - -class PipfileParser(object): - def __init__(self, filename='Pipfile'): - self.filename = filename - self.sources = [] - self.groups = { - 'default': [], - 'develop': [] - } - self.group_stack = ['default'] - self.requirements = [] - - def __repr__(self): - return ' Date: Mon, 12 Sep 2022 23:23:05 +0200 Subject: [PATCH 04/26] Remove unused import --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 6d22c14c..faea4dd5 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -48,7 +48,7 @@ from pipenv.utils.shell import ( system_which, ) from pipenv.utils.spinner import create_spinner -from pipenv.vendor import click, dotenv, plette, vistir +from pipenv.vendor import click, plette, vistir from pipenv.vendor.requirementslib.models.requirements import Requirement if MYPY_RUNNING: From 91bcd859e70801531d48278e3ddf972d78b2c541 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Mon, 12 Sep 2022 23:43:26 +0200 Subject: [PATCH 05/26] Remove pipfile module usage project.py --- pipenv/project.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index 10820a42..cfd6ef17 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -13,8 +13,6 @@ import sys import urllib.parse from pathlib import Path -import pipfile -import pipfile.api import toml import tomlkit import vistir @@ -42,6 +40,7 @@ from pipenv.utils.shell import ( system_which, ) from pipenv.utils.toml import cleanup_toml, convert_toml_outline_tables +from pipenv.vendor import plette from pipenv.vendor.requirementslib.models.utils import get_default_pyproject_backend try: @@ -141,7 +140,10 @@ class Project: "verify_ssl": True, "name": "pypi", } - pipfile.api.DEFAULT_SOURCE = self.default_source + + plette.pipfiles.DEFAULT_SOURCE_TOML = ( + f"[[source]]\n{toml.dumps(self.default_source)}" + ) # Hack to skip this during pipenv run, or -r. if ("run" not in sys.argv) and chdir: @@ -554,7 +556,7 @@ class Project: @property def _lockfile(self): """Pipfile.lock divided by PyPI and external dependencies.""" - pfile = pipfile.load(self.pipfile_location, inject_env=False) + pfile = plette.Pipfile.load(self.pipfile_location) lockfile = json.loads(pfile.lock()) for section in ("default", "develop"): lock_section = lockfile.get(section, {}) @@ -1006,7 +1008,7 @@ class Project: def calculate_pipfile_hash(self): # Update the lockfile if it is out-of-date. - p = pipfile.load(self.pipfile_location, inject_env=False) + p = plette.Pipfile.load(self.pipfile_location, inject_env=False) return p.hash def ensure_proper_casing(self): From d9bd51d6695acfe6810f7cb50ee04687b0988480 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Mon, 12 Sep 2022 23:46:33 +0200 Subject: [PATCH 06/26] Remove undefined keyword for loading pipenv --- pipenv/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/project.py b/pipenv/project.py index cfd6ef17..fb4df7a1 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -1008,7 +1008,7 @@ class Project: def calculate_pipfile_hash(self): # Update the lockfile if it is out-of-date. - p = plette.Pipfile.load(self.pipfile_location, inject_env=False) + p = plette.Pipfile.load(self.pipfile_location) return p.hash def ensure_proper_casing(self): From 9e170e10afe7bd60595a686dbc911795d8ab48b9 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 13 Sep 2022 11:40:15 +0200 Subject: [PATCH 07/26] Remove more usage of pipfile in favor of plette --- pipenv/core.py | 4 ++-- pipenv/project.py | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index faea4dd5..e75f12c0 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1218,7 +1218,7 @@ def do_init( if (project.lockfile_exists and not ignore_pipfile) and not skip_lock: old_hash = project.get_lockfile_hash() new_hash = project.calculate_pipfile_hash() - if new_hash != old_hash: + if new_hash.value != old_hash: if deploy: click.secho( "Your Pipfile.lock ({}) is out of date. Expected: ({}).".format( @@ -1244,7 +1244,7 @@ def do_init( else: msg = fix_utf8("Pipfile.lock is corrupted, replaced with ({1})...") click.secho( - msg.format(old_hash[-6:], new_hash[-6:]), + msg.format(old_hash[-6:], new_hash.value[-6:]), fg="yellow", bold=True, err=True, diff --git a/pipenv/project.py b/pipenv/project.py index fb4df7a1..8f4a8986 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -556,14 +556,16 @@ class Project: @property def _lockfile(self): """Pipfile.lock divided by PyPI and external dependencies.""" - pfile = plette.Pipfile.load(self.pipfile_location) - lockfile = json.loads(pfile.lock()) + lockfile = plette.Lockfile.with_meta_from( + plette.Pipfile.load(open(self.pipfile_location)) + ) for section in ("default", "develop"): lock_section = lockfile.get(section, {}) for key in list(lock_section.keys()): norm_key = pep423_name(key) lockfile[section][norm_key] = lock_section.pop(key) - return lockfile + + return lockfile._data @property def _pipfile(self): @@ -1008,8 +1010,8 @@ class Project: def calculate_pipfile_hash(self): # Update the lockfile if it is out-of-date. - p = plette.Pipfile.load(self.pipfile_location) - return p.hash + p = plette.Pipfile.load(open(self.pipfile_location)) + return p.get_hash() def ensure_proper_casing(self): """Ensures proper casing of Pipfile packages""" From 8f4e44d0031f0fdaf04401447b6ac18881dfae92 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 13 Sep 2022 14:47:19 +0200 Subject: [PATCH 08/26] Fix usage of plette.Pipfile --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index e75f12c0..a21da79c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2781,7 +2781,7 @@ def do_check( ) sys.exit(1) # Load the pipfile. - p = plette.Pipfile.load(project.pipfile_location) + p = plette.Pipfile.load(open(project.pipfile_location)) failed = False # Assert each specified requirement. for marker, specifier in p.data["_meta"]["requires"].items(): From 2a6b6c946dfa2b7828d1b33f67c0992a96508a0f Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 13 Sep 2022 14:55:53 +0200 Subject: [PATCH 09/26] Fix failing tests with Plette.Pipfile This commit comes with a warning! This might be considered a behavior change. Since, plette uses a stricter boolean: \"true\" is not parsed the same way as "true" --- tests/integration/test_project.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 4addeeb3..58b1f507 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -5,10 +5,10 @@ from pathlib import Path import pytest -from pipenv.patched import pipfile from pipenv.project import Project from pipenv.utils.shell import temp_environ from pipenv.vendor.vistir.path import is_in_path, normalize_path +from pipenv.vendor.plette import Pipfile @pytest.mark.project @@ -30,7 +30,8 @@ pytz = "*" os.environ['TEST_HOST'] = 'localhost:5000' project = Project() assert project.sources[0]['url'] == 'https://localhost:5000/simple' - assert 'localhost:5000' not in str(pipfile.load(p.pipfile_path)) + assert 'localhost:5000' not in str(Pipfile.load(open(p.pipfile_path))) + print(str(Pipfile.load(open(p.pipfile_path)))) @pytest.mark.project @@ -47,7 +48,7 @@ name = "testindex" [[source]] url = "https://pypi.org/simple" -verify_ssl = "true" +verify_ssl = true name = "pypi" [packages] @@ -131,12 +132,12 @@ name = "testindex" [[source]] url = "https://pypi.org/simple" -verify_ssl = "true" +verify_ssl = true name = "pypi" [[source]] url = "https://pypi.python.org/simple" -verify_ssl = "true" +verify_ssl = true name = "legacy" [packages] From 47094209514d3d7879a767795a43547b5f620deb Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 13 Sep 2022 15:00:39 +0200 Subject: [PATCH 10/26] Remove test for PipfileParser Testing and development of plette parser will be done upstream. --- tests/unit/test_vendor.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/tests/unit/test_vendor.py b/tests/unit/test_vendor.py index 62f317b9..ee250452 100644 --- a/tests/unit/test_vendor.py +++ b/tests/unit/test_vendor.py @@ -9,41 +9,6 @@ import pytest import pytz import tomlkit -from pipfile.api import PipfileParser - - -class TestPipfileParser: - - def test_inject_environment_variables(self): - os.environ['PYTEST_PIPFILE_TEST'] = "XYZ" - p = PipfileParser() - - parsed_dict = p.inject_environment_variables({ - "a_string": "https://$PYTEST_PIPFILE_TEST@something.com", - "another_string": "https://${PYTEST_PIPFILE_TEST}@something.com", - "nested": { - "a_string": "https://$PYTEST_PIPFILE_TEST@something.com", - "another_string": "${PYTEST_PIPFILE_TEST}", - }, - "list": [ - { - "a_string": "https://$PYTEST_PIPFILE_TEST@something.com", - "another_string": "${PYTEST_PIPFILE_TEST}" - }, - {}, - ], - "bool": True, - "none": None, - }) - - assert parsed_dict["a_string"] == "https://XYZ@something.com" - assert parsed_dict["another_string"] == "https://XYZ@something.com" - assert parsed_dict["nested"]["another_string"] == "XYZ" - assert parsed_dict["list"][0]["a_string"] == "https://XYZ@something.com" - assert parsed_dict["list"][1] == {} - assert parsed_dict["bool"] is True - assert parsed_dict["none"] is None - @pytest.mark.parametrize('dt, content', [ ( # Date. From d1aea86f7bfd79a2ac7c924ecbd02f6b27f8d997 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 13 Sep 2022 15:31:10 +0200 Subject: [PATCH 11/26] Fix get_hash method --- pipenv/core.py | 2 +- pipenv/project.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index a21da79c..dc40da92 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1218,7 +1218,7 @@ def do_init( if (project.lockfile_exists and not ignore_pipfile) and not skip_lock: old_hash = project.get_lockfile_hash() new_hash = project.calculate_pipfile_hash() - if new_hash.value != old_hash: + if new_hash != old_hash: if deploy: click.secho( "Your Pipfile.lock ({}) is out of date. Expected: ({}).".format( diff --git a/pipenv/project.py b/pipenv/project.py index 8f4a8986..6f52b51d 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -1011,7 +1011,7 @@ class Project: def calculate_pipfile_hash(self): # Update the lockfile if it is out-of-date. p = plette.Pipfile.load(open(self.pipfile_location)) - return p.get_hash() + return p.get_hash().value def ensure_proper_casing(self): """Ensures proper casing of Pipfile packages""" From a9a73dba3da7bda1e4dc6452f8821a37c31e2873 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 13 Sep 2022 16:05:48 +0200 Subject: [PATCH 12/26] Fix check command --- pipenv/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index dc40da92..209523f4 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2782,9 +2782,10 @@ def do_check( sys.exit(1) # Load the pipfile. p = plette.Pipfile.load(open(project.pipfile_location)) + p = plette.Lockfile.with_meta_from(p) failed = False # Assert each specified requirement. - for marker, specifier in p.data["_meta"]["requires"].items(): + for marker, specifier in p._data["_meta"]["requires"].items(): if marker in results: try: assert results[marker] == specifier From db743ea0114352e387f7872f28c95f2afe0328ac Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 13 Sep 2022 16:20:06 +0200 Subject: [PATCH 13/26] Disable test that break with plette Unnamed sources will break with the following error: pipenv.vendor.plette.models.base.ValidationError: {'url': 'https://pypi.org/simple', 'verify_ssl': True} name: required field --- tests/integration/test_install_basic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index 7b21bde7..718a06b2 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -416,6 +416,7 @@ extras = ["socks"] assert 'colorama = "*"' in contents +@pytest.mark.skip(reason="Plette is more strict then pipfile ...") @pytest.mark.dev @pytest.mark.basic @pytest.mark.install From af06ef03f1269631310435afa7071f937d071b72 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 13 Sep 2022 16:24:42 +0200 Subject: [PATCH 14/26] Fix do_init with the new hash format --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 209523f4..cc88ad0f 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1244,7 +1244,7 @@ def do_init( else: msg = fix_utf8("Pipfile.lock is corrupted, replaced with ({1})...") click.secho( - msg.format(old_hash[-6:], new_hash.value[-6:]), + msg.format(old_hash[-6:], new_hash[-6:]), fg="yellow", bold=True, err=True, From 2805e75e6c3a9a2f996a10a4f22d68104c865816 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 13 Sep 2022 16:48:09 +0200 Subject: [PATCH 15/26] Disable another test which breaks with plette --- tests/integration/test_lock.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index c8def92f..ac688b78 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -527,6 +527,7 @@ def test_lockfile_with_empty_dict(pipenv_instance_pypi): assert p.lockfile['_meta'] +@pytest.mark.skip(reason="Plette is more strict then pipfile ...") @pytest.mark.lock @pytest.mark.install @pytest.mark.skip_lock From 8c2118360fd80a3c6705663ff8fb89467a08ad55 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 13 Sep 2022 23:46:36 +0200 Subject: [PATCH 16/26] Fix broken test with plette.Pipfile This tests actually required the private pypi, since fake-package doesn't exist on Pypi! --- tests/integration/test_install_markers.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index b2b244e6..b71ff799 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -4,16 +4,14 @@ import pytest from flaky import flaky -from pipenv.patched import pipfile from pipenv.project import Project +from pipenv.vendor.plette import Pipfile from pipenv.utils.shell import temp_environ -@flaky @pytest.mark.markers -def test_package_environment_markers(pipenv_instance_pypi): - - with pipenv_instance_pypi() as p: +def test_package_environment_markers(pipenv_instance_private_pypi): + with pipenv_instance_private_pypi() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -168,7 +166,7 @@ six = "*" assert lock_hash == project.calculate_pipfile_hash() # sanity check on pytest - assert 'PYPI_USERNAME' not in str(pipfile.load(p.pipfile_path)) + assert 'PYPI_USERNAME' not in str(Pipfile.load(p.pipfile_path)) assert c.returncode == 0 assert project.get_lockfile_hash() == project.calculate_pipfile_hash() From 1b8f6278dfa7bc8502a3ca84fdddd6438b41d27b Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Wed, 14 Sep 2022 00:29:02 +0200 Subject: [PATCH 17/26] Fix test_environment_variable_value_does_not_change_hash It seems the are was an assertion in this test that was wrong. I don't think pipenv install should modifiy the Pipfile when running `pipenv install`. --- tests/integration/test_install_markers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index b71ff799..774a53be 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -165,8 +165,9 @@ six = "*" assert lock_hash is not None assert lock_hash == project.calculate_pipfile_hash() - # sanity check on pytest - assert 'PYPI_USERNAME' not in str(Pipfile.load(p.pipfile_path)) + # This line seems wrong. Why would pipenv install expand the variable and + # write it to Pipfile? + # assert 'PYPI_USERNAME' not in str(Pipfile.load(open(p.pipfile_path))) assert c.returncode == 0 assert project.get_lockfile_hash() == project.calculate_pipfile_hash() From b44dfd1e0a16d5e94a193d46f516942e64aded78 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Wed, 14 Sep 2022 00:55:16 +0200 Subject: [PATCH 18/26] Use context manager when opening Pipfile --- pipenv/core.py | 1 - pipenv/project.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index cc88ad0f..1f4f6702 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1028,7 +1028,6 @@ def do_lock( pypi_mirror=None, ): """Executes the freeze functionality.""" - cached_lockfile = {} if not pre: pre = project.settings.get("allow_prereleases") diff --git a/pipenv/project.py b/pipenv/project.py index 6f52b51d..465e58e6 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -556,9 +556,8 @@ class Project: @property def _lockfile(self): """Pipfile.lock divided by PyPI and external dependencies.""" - lockfile = plette.Lockfile.with_meta_from( - plette.Pipfile.load(open(self.pipfile_location)) - ) + with open(self.pipfile_location) as pf: + lockfile = plette.Lockfile.with_meta_from(plette.Pipfile.load(pf)) for section in ("default", "develop"): lock_section = lockfile.get(section, {}) for key in list(lock_section.keys()): From 7ecea16e2513fb2fca5027bd58abf55e5eebfa5b Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Wed, 14 Sep 2022 01:01:11 +0200 Subject: [PATCH 19/26] Fix test_lock_updated_source The source must contain a name or Plette will fail. --- tests/integration/test_lock.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index ac688b78..6f33c26b 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -404,6 +404,7 @@ def test_lock_updated_source(pipenv_instance_private_pypi): contents = """ [[source]] url = "{url}/${{MY_ENV_VAR}}" +name = source_with_env_expanded [packages] requests = "==2.14.0" From a2130f739683936128fab80225e2626cb040aeaf Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Wed, 14 Sep 2022 01:17:59 +0200 Subject: [PATCH 20/26] Fix missing name for source in test_dev_lock_use_default_packages_as_constraint --- tests/integration/test_lock.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 6f33c26b..33a1b1ba 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -404,7 +404,8 @@ def test_lock_updated_source(pipenv_instance_private_pypi): contents = """ [[source]] url = "{url}/${{MY_ENV_VAR}}" -name = source_with_env_expanded +name = expanded +verify_ssl = false [packages] requests = "==2.14.0" From 8bbb4bdd51a4ba64f91d697d0972dfb7dbcf7acc Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Wed, 14 Sep 2022 08:33:07 +0200 Subject: [PATCH 21/26] Fix test_lock_updated_source This was broken because of wrong Pipfile format. Plette is strict! --- tests/integration/test_lock.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 33a1b1ba..c1a662c7 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -391,7 +391,7 @@ six = {version = "*", index = "testpypi"} pipenv-test-public-package = "*" """.strip() f.write(contents) - c = p.pipenv(f'install -v') + c = p.pipenv('install -v') assert c.returncode == 0 @@ -404,7 +404,7 @@ def test_lock_updated_source(pipenv_instance_private_pypi): contents = """ [[source]] url = "{url}/${{MY_ENV_VAR}}" -name = expanded +name = "expanded" verify_ssl = false [packages] @@ -422,6 +422,8 @@ requests = "==2.14.0" contents = """ [[source]] url = "{url}/simple" +name = "expanded" +verify_ssl = false [packages] requests = "==2.14.0" From 21b6eb732d36122e17e8c3b87e92a76ef92b95ab Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Thu, 15 Sep 2022 14:53:16 +0200 Subject: [PATCH 22/26] Yank skipped test - pipenv no longer supports unnamed index --- tests/integration/test_install_basic.py | 27 ------------------------- 1 file changed, 27 deletions(-) diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index 718a06b2..eeea6cd2 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -416,33 +416,6 @@ extras = ["socks"] assert 'colorama = "*"' in contents -@pytest.mark.skip(reason="Plette is more strict then pipfile ...") -@pytest.mark.dev -@pytest.mark.basic -@pytest.mark.install -@pytest.mark.needs_internet -def test_install_with_unnamed_source(pipenv_instance_pypi): - """Ensure that running `pipenv install` doesn't break with an unamed index""" - with pipenv_instance_pypi(chdir=True) as p: - with open(p.pipfile_path, "w") as f: - contents = """ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true - -[packages] -dataclasses-json = {version="*", index="pypi"} - """.strip() - f.write(contents) - c = p.pipenv("install") - assert c.returncode == 0 - - @pytest.mark.dev @pytest.mark.install def test_install_dev_use_default_constraints(pipenv_instance_private_pypi): From 129e821481f705f36b38655716b663ef81d15005 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Thu, 15 Sep 2022 14:54:32 +0200 Subject: [PATCH 23/26] Remove comment - Plette does no expand env vars on disk --- tests/integration/test_install_markers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index 774a53be..28c8a0b6 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -165,9 +165,6 @@ six = "*" assert lock_hash is not None assert lock_hash == project.calculate_pipfile_hash() - # This line seems wrong. Why would pipenv install expand the variable and - # write it to Pipfile? - # assert 'PYPI_USERNAME' not in str(Pipfile.load(open(p.pipfile_path))) assert c.returncode == 0 assert project.get_lockfile_hash() == project.calculate_pipfile_hash() From a23fee69b0c4bf66bc795635508f68412e8087a4 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Thu, 15 Sep 2022 14:55:42 +0200 Subject: [PATCH 24/26] Remove skipped test - Pipenv no longer lock with incomplete sources --- tests/integration/test_lock.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index c1a662c7..c3c8ae10 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -531,25 +531,6 @@ def test_lockfile_with_empty_dict(pipenv_instance_pypi): assert p.lockfile['_meta'] -@pytest.mark.skip(reason="Plette is more strict then pipfile ...") -@pytest.mark.lock -@pytest.mark.install -@pytest.mark.skip_lock -def test_lock_with_incomplete_source(pipenv_instance_private_pypi): - with pipenv_instance_private_pypi(chdir=True) as p: - with open(p.pipfile_path, 'w') as f: - f.write(""" -[[source]] -url = "{}" - -[packages] -six = "*" - """.format(p.index_url)) - c = p.pipenv('install') - assert c.returncode == 0 - assert p.lockfile['_meta']['sources'] - - @pytest.mark.lock @pytest.mark.install def test_lock_no_warnings(pipenv_instance_pypi, recwarn): @@ -559,7 +540,6 @@ def test_lock_no_warnings(pipenv_instance_pypi, recwarn): assert len(recwarn) == 0 - @pytest.mark.vcs @pytest.mark.lock def test_vcs_lock_respects_top_level_pins(pipenv_instance_private_pypi): From 3f679ad5ad54badbb2a50e31c539d4d2980efc7c Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Thu, 15 Sep 2022 15:04:24 +0200 Subject: [PATCH 25/26] Add news about migration to Plette --- news/5339.behavior.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 news/5339.behavior.rst diff --git a/news/5339.behavior.rst b/news/5339.behavior.rst new file mode 100644 index 00000000..d9a7bb95 --- /dev/null +++ b/news/5339.behavior.rst @@ -0,0 +1,17 @@ +Remove usage of pipfile module in favour of Plette. +pipfile is not actively maintained anymore. Plette is actively maintained, +and has stricter checking of the Pipefile and Pipefile.lock. As a result, +Pipefile with unnamed package indecies will fail to lock. If a Pipefile +was hand crafeted, and the source is anonymous an error will be thrown. +The solution is simple, add a name to your index, e.g, replace:: + + [[source]] + url = "https://pypi.acme.com/simple" + verify_ssl = true + +With:: + + [[source]] + url = "https://pypi.acme.com/simple" + verify_ssl = true + name = acmes_private_index From 9c774f73aa57b77769c20df71db9da9899fb4f1d Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Thu, 15 Sep 2022 22:22:23 +0200 Subject: [PATCH 26/26] Propagate the markers to new requirment in lock file This is needed since the upgrade of requirementlib. --- pipenv/project.py | 3 ++- pipenv/utils/resolver.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pipenv/project.py b/pipenv/project.py index ed246157..a566c509 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -1004,7 +1004,8 @@ class Project: def calculate_pipfile_hash(self): # Update the lockfile if it is out-of-date. - p = plette.Pipfile.load(open(self.pipfile_location)) + with open(self.pipfile_location) as pf: + p = plette.Pipfile.load(pf) return p.get_hash().value def ensure_proper_casing(self): diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 4a604a3f..585b00ac 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -405,6 +405,7 @@ class Resolver: hashes = resolver.collect_hashes(ireq) if resolver else [] new_req = Requirement.from_ireq(ireq) new_req = new_req.add_hashes(hashes) + new_req = new_req.merge_markers(req.markers) name, entry = new_req.pipfile_entry locked_deps[pep423_name(name)] = translate_markers(entry) click.echo(