parent 8ae44bc90b

author Dan Ryan <dan@danryan.co> 1554074378 -0400 committer Dan Ryan <dan@danryan.co> 1558982736 -0400 gpgsig -----BEGIN PGP SIGNATURE----- iQIzBAABCgAdFiEEb6jpcpb+5zzDideCsyDQTvvOpJUFAlzsMFAACgkQsyDQTvvO pJWZmRAAtge5wdprlLnKdWUYK5USZb7Uk3zJHi9UIvnO7nKP6UA/L2D/5nxZitvx pZI7cGG+8sLp2yZNtQZdW6LNM1jmRXgzdMLYQh/5zo5gbj6KLOw7erh0FU3L3uM6 wNBNM135Eqt7b+4a4C5TEK2UjwRxBHAsF+3ZzUy+UJQqgQEKxFVxW4gC4yxpfMtL jipE8ludwuOIM88ZJapmLpv2R6adQTxWZedTlmczdsy2/WKGHTCCpWs96PBbntdI pVBmoXfMhgZi+IuGR3iBYU0qS97vjJ8Te9tQZAaB9JGSqv3hHDWo1ht/rrG2RXzp 3i0Cf2vG4035EUh56VYE9FCC9m6Vu3U9iIR34BZG9K5+lDP7pmJmjT+GymEgMP0N GoP3LYUO+dJjMjaUEMsC6QIi6DAots3uk4lxIw3wcA4Im/N/i5xafsRj1Eu3UdBL wBDKMz/FQjH+tD+mnvTlzaxD5vdhhCdBu1gK59rjNMlzg8hz6EF61QbHCaQHd4UI VOGIa8ThLlLI3addxzq/McceAc+OsLJ9hm06jkjvvoIuKrHyE3DybdbYQC7uEwyw 2AvVuMDCPcciYQnkJhNTKmGvPcUDYD7cF91GKcUJKdPdyzDCeFEo5SUTKnLV4Cj2 VD2sdVUk4jnmYfE4pCFvHKYooxQDMKQk/VCKl1c9QHL9/ijCx+o= =b+H1 -----END PGP SIGNATURE----- Ensure resolver doesn't compare editable specifiers - Don't compare versions of editable dependencies when updating using `--keep-outdated` -- editable dependencies will now be updated to the latest version - Ensure we don't drop markers from the lockfile when versions are not updated - Fixes #3656 - Fixes #3659 Signed-off-by: Dan Ryan <dan@danryan.co> Add future import for print function Signed-off-by: Dan Ryan <dan@danryan.co> Handle all possible markers in lockfiles Signed-off-by: Dan Ryan <dan@danryan.co> Fix json import Signed-off-by: Dan Ryan <dan@danryan.co> point to correct reference for lockfile Signed-off-by: Dan Ryan <dan@danryan.co> Fix marker merging errors Signed-off-by: Dan Ryan <dan@danryan.co> Prevent automatically setting `editable=True` - Fixes #3647 Signed-off-by: Dan Ryan <dan@danryan.co> Add new feature toggle for VCS dependency resolution - Fixes #3577 Signed-off-by: Dan Ryan <dan@danryan.co> Fix syntax error Signed-off-by: Dan Ryan <dan@danryan.co> Use string for environment Signed-off-by: Dan Ryan <dan@danryan.co> Fix class name resolution for py27 Signed-off-by: Dan Ryan <dan@danryan.co> Write json files as unicode Signed-off-by: Dan Ryan <dan@danryan.co> Fix resolution with env var Signed-off-by: Dan Ryan <dan@danryan.co>
2026-06-05 22:50:18 +00:00 · 2019-03-31 19:19:38 -04:00
parent b3b6b1f8bf
commit f0e3bbaa79
10 changed files with 84 additions and 17 deletions
@@ -0,0 +1 @@
+Added a new environment variable, ``PIPENV_RESOLVE_VCS``, to toggle dependency resolution off for non-editable VCS, file, and URL based dependencies.
@@ -0,0 +1 @@
+Pipenv will no longer inadvertently set ``editable=True`` on all vcs dependencies.
@@ -0,0 +1,2 @@
+The ``--keep-outdated`` argument to ``pipenv install`` and ``pipenv lock`` will now drop specifier constraints when encountering editable dependencies.
+- In addition, ``--keep-outdated`` will retain specifiers that would otherwise be dropped from any entries that have not been updated.
@@ -1394,6 +1394,8 @@ def pip_install(
            src_dir = os.environ["PIP_SRC"]
            src = ["--src", os.environ["PIP_SRC"]]
        if not requirement.editable:
+            # Leave this off becauase old lockfiles don't have all deps included
+            # TODO: When can it be turned back on?
            no_deps = False

        if src_dir is not None:
@@ -1412,7 +1414,7 @@ def pip_install(
            prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir,
            delete=False
        )
-        line = "-e" if requirement.editable else ""
+        line = "-e " if requirement.editable else ""
        if requirement.editable or requirement.name is not None:
            name = requirement.name
            if requirement.extras:
@@ -1640,7 +1642,6 @@ def system_which(command, mult=False):
    return result


-
 def format_help(help):
    """Formats the help string."""
    help = help.replace("Options:", str(crayons.normal("Options:", bold=True)))
@@ -1784,8 +1785,8 @@ def do_py(system=False):
            ),
            err=True,
        )
-        return    
-    
+        return
+
    try:
        click.echo(which("python", allow_global=system))
    except AttributeError:
@@ -236,6 +236,15 @@ Default is to lock dependencies and update ``Pipfile.lock`` on each run.
 NOTE: This only affects the ``install`` and ``uninstall`` commands.
 """

+PIPENV_RESOLVE_VCS = _is_env_truthy(os.environ.get("PIPENV_RESOLVE_VCS", 'true'))
+"""Tells Pipenv whether to resolve all VCS dependencies in full.
+
+As of Pipenv 2018.11.26, only editable VCS dependencies were resolved in full.
+To retain this behavior and avoid handling any conflicts that arise from the new
+approach, you may set this to '0', 'off', or 'false'.
+"""
+
+
 PIPENV_PYUP_API_KEY = os.environ.get(
    "PIPENV_PYUP_API_KEY", "1ab8d58f-5122e025-83674263-bc1e79e0"
 )
@@ -70,7 +70,7 @@ def clean_requires_python(candidates):
    all_candidates = []
    py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3]))))
    for c in candidates:
-        if c.requires_python:
+        if getattr(c, "requires_python", None):
            # Old specifications had people setting this to single digits
            # which is effectively the same as '>=digit,<digit+1'
            if c.requires_python.isdigit():
@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, print_function
 import json
 import logging
 import os
@@ -106,9 +109,14 @@ class Entry(object):
        self.pipfile = project.parsed_pipfile.get(pipfile_section, {})
        self.lockfile = project.lockfile_content.get(section, {})
        self.pipfile_dict = self.pipfile.get(self.pipfile_name, {})
-        self.lockfile_dict = self.lockfile.get(name, entry_dict)
+        if self.dev and self.name in project.lockfile_content.get("default", {}):
+            self.lockfile_dict = project.lockfile_content["default"][name]
+        else:
+            self.lockfile_dict = self.lockfile.get(name, entry_dict)
        self.resolver = resolver
        self.reverse_deps = reverse_deps
+        self._original_markers = None
+        self._markers = None
        self._entry = None
        self._lockfile_entry = None
        self._pipfile_entry = None
@@ -232,6 +240,10 @@ class Entry(object):
                entry_extras.extend(list(self.lockfile_entry.extras))
            self._entry.req.extras = entry_extras
            self.entry_dict["extras"] = self.entry.extras
+        if self.original_markers and not self.markers:
+            original_markers = self.marker_to_str(self.original_markers)
+            self.markers = original_markers
+            self.entry_dict["markers"] = self.marker_to_str(original_markers)
        entry_hashes = set(self.entry.hashes)
        locked_hashes = set(self.lockfile_entry.hashes)
        if entry_hashes != locked_hashes and not self.is_updated:
@@ -248,6 +260,10 @@ class Entry(object):
            self._lockfile_entry = self.make_requirement(self.name, self.lockfile_dict)
        return self._lockfile_entry

+    @lockfile_entry.setter
+    def lockfile_entry(self, entry):
+        self._lockfile_entry = entry
+
    @property
    def pipfile_entry(self):
        if self._pipfile_entry is None:
@@ -359,6 +375,7 @@ class Entry(object):

    @property
    def updated_specifier(self):
+        # type: () -> str
        return self.entry.specifiers

    @property
@@ -373,7 +390,7 @@ class Entry(object):
        return None

    def validate_specifiers(self):
-        if self.is_in_pipfile:
+        if self.is_in_pipfile and not self.pipfile_entry.editable:
            return self.pipfile_entry.requirement.specifier.contains(self.updated_version)
        return True

@@ -550,8 +567,11 @@ class Entry(object):
                constraint.check_if_exists(False)
            except Exception:
                from pipenv.exceptions import DependencyConflict
+                from pipenv.environments import is_verbose
+                if is_verbose():
+                    print("Tried constraint: {0!r}".format(constraint), file=sys.stderr)
                msg = (
-                    "Cannot resolve conflicting version {0}{1} while {1}{2} is "
+                    "Cannot resolve conflicting version {0}{1} while {2}{3} is "
                    "locked.".format(
                        self.name, self.updated_specifier, self.old_name, self.old_specifiers
                    )
@@ -624,6 +644,7 @@ def clean_results(results, resolver, project, dev=False):

 def clean_outdated(results, resolver, project, dev=False):
    from pipenv.vendor.requirementslib.models.requirements import Requirement
+    from pipenv.environments import is_verbose
    if not project.lockfile_exists:
        return results
    lockfile = project.lockfile_content
@@ -31,7 +31,9 @@ import crayons
 import parse

 from . import environments
-from .exceptions import PipenvUsageError, ResolutionFailure, RequirementError, PipenvCmdError
+from .exceptions import (
+    PipenvUsageError, RequirementError, PipenvCmdError, ResolutionFailure
+)
 from .pep508checker import lookup
 from .vendor.urllib3 import util as urllib3_util

@@ -398,7 +400,6 @@ class Resolver(object):
    ):
        # type: (...) -> Tuple[Requirement, Dict[str, str], Dict[str, str]]
        from .vendor.requirementslib.models.requirements import Requirement
-        from .exceptions import ResolutionFailure
        if index_lookup is None:
            index_lookup = {}
        if markers_lookup is None:
@@ -444,6 +445,7 @@ class Resolver(object):
        # type: (Requirement, Optional["Resolver"]) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]]
        from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name
        from .vendor.requirementslib.models.requirements import Requirement
+        from requirementslib.utils import is_installable_dir
        constraints = set()  # type: Set[str]
        locked_deps = dict()  # type: Dict[str, Dict[str, Union[str, bool, List[str]]]]
        if (req.is_file_or_url or req.is_vcs) and not req.is_wheel:
@@ -463,7 +465,16 @@ class Resolver(object):
            setup_info = req.req.setup_info
            setup_info.get_info()
            locked_deps[pep423_name(name)] = entry
-            requirements = [v for v in getattr(setup_info, "requires", {}).values()]
+            requirements = []
+            # Allow users to toggle resolution off for non-editable VCS packages
+            # but leave it on for local, installable folders on the filesystem
+            if environments.PIPENV_RESOLVE_VCS or (
+                req.editable or parsed_line.is_wheel or (
+                    req.is_file_or_url and parsed_line.is_local and
+                    is_installable_dir(parsed_line.path)
+                )
+            ):
+                requirements = [v for v in getattr(setup_info, "requires", {}).values()]
            for r in requirements:
                if getattr(r, "url", None) and not getattr(r, "editable", False):
                    if r is not None:
@@ -1797,13 +1808,15 @@ def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None):

    # If a package is **PRESENT** in the pipfile but has no markers, make sure we
    # **NEVER** include markers in the lockfile
-    if "markers" in dep:
+    if "markers" in dep and dep.get("markers", "").strip():
        # First, handle the case where there is no top level dependency in the pipfile
        if not is_top_level:
-            try:
-                lockfile["markers"] = translate_markers(dep)["markers"]
-            except TypeError:
-                pass
+            translated = translate_markers(dep).get("markers", "").strip()
+            if translated:
+                try:
+                    lockfile["markers"] = translated
+                except TypeError:
+                    pass
        # otherwise make sure we are prioritizing whatever the pipfile says about the markers
        # If the pipfile says nothing, then we should put nothing in the lockfile
        else:
@@ -608,7 +608,7 @@ index 9b4b4c2..8875543 100644
 +    all_candidates = []
 +    py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3]))))
 +    for c in candidates:
-+        if c.requires_python:
+        if getattr(c, "requires_python", None):
 +            # Old specifications had people setting this to single digits
 +            # which is effectively the same as '>=digit,<digit+1'
 +            if c.requires_python.isdigit():
@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+
+import json
 import os
 import sys

@@ -5,6 +8,7 @@ import pytest

 from flaky import flaky
 from vistir.compat import Path
+from vistir.misc import to_text
 from pipenv.utils import temp_environ


@@ -122,6 +126,21 @@ def test_keep_outdated_doesnt_upgrade_pipfile_pins(PipenvInstance, pypi):
        assert p.lockfile["default"]["urllib3"]["version"] == "==1.21.1"


+def test_keep_outdated_keeps_markers_not_removed(PipenvInstance, pypi):
+    with PipenvInstance(chdir=True, pypi=pypi) as p:
+        c = p.pipenv("install tablib")
+        assert c.ok
+        lockfile = Path(p.lockfile_path)
+        lockfile_content = lockfile.read_text()
+        lockfile_json = json.loads(lockfile_content)
+        assert "tablib" in lockfile_json["default"]
+        lockfile_json["default"]["tablib"]["markers"] = "python_version >= '2.7'"
+        lockfile.write_text(to_text(json.dumps(lockfile_json)))
+        c = p.pipenv("lock --keep-outdated")
+        assert c.ok
+        assert p.lockfile["default"]["tablib"].get("markers", "") == "python_version >= '2.7'"
+
+
@pytest.mark.lock
@pytest.mark.keep_outdated
 def test_keep_outdated_doesnt_update_satisfied_constraints(PipenvInstance, pypi):
				`@@ -0,0 +1 @@`
				Added a new environment variable, ``PIPENV_RESOLVE_VCS``, to toggle dependency resolution off for non-editable VCS, file, and URL based dependencies.
				`@@ -0,0 +1 @@`
				Pipenv will no longer inadvertently set ``editable=True`` on all vcs dependencies.