parent 8ae44bc90b

author Dan Ryan <dan@danryan.co> 1554074378 -0400
committer Dan Ryan <dan@danryan.co> 1558982736 -0400
gpgsig -----BEGIN PGP SIGNATURE-----

 iQIzBAABCgAdFiEEb6jpcpb+5zzDideCsyDQTvvOpJUFAlzsMFAACgkQsyDQTvvO
 pJWZmRAAtge5wdprlLnKdWUYK5USZb7Uk3zJHi9UIvnO7nKP6UA/L2D/5nxZitvx
 pZI7cGG+8sLp2yZNtQZdW6LNM1jmRXgzdMLYQh/5zo5gbj6KLOw7erh0FU3L3uM6
 wNBNM135Eqt7b+4a4C5TEK2UjwRxBHAsF+3ZzUy+UJQqgQEKxFVxW4gC4yxpfMtL
 jipE8ludwuOIM88ZJapmLpv2R6adQTxWZedTlmczdsy2/WKGHTCCpWs96PBbntdI
 pVBmoXfMhgZi+IuGR3iBYU0qS97vjJ8Te9tQZAaB9JGSqv3hHDWo1ht/rrG2RXzp
 3i0Cf2vG4035EUh56VYE9FCC9m6Vu3U9iIR34BZG9K5+lDP7pmJmjT+GymEgMP0N
 GoP3LYUO+dJjMjaUEMsC6QIi6DAots3uk4lxIw3wcA4Im/N/i5xafsRj1Eu3UdBL
 wBDKMz/FQjH+tD+mnvTlzaxD5vdhhCdBu1gK59rjNMlzg8hz6EF61QbHCaQHd4UI
 VOGIa8ThLlLI3addxzq/McceAc+OsLJ9hm06jkjvvoIuKrHyE3DybdbYQC7uEwyw
 2AvVuMDCPcciYQnkJhNTKmGvPcUDYD7cF91GKcUJKdPdyzDCeFEo5SUTKnLV4Cj2
 VD2sdVUk4jnmYfE4pCFvHKYooxQDMKQk/VCKl1c9QHL9/ijCx+o=
 =b+H1
 -----END PGP SIGNATURE-----

Ensure resolver doesn't compare editable specifiers

- Don't compare versions of editable dependencies when updating using
  `--keep-outdated` -- editable dependencies will now be updated to
  the latest version
- Ensure we don't drop markers from the lockfile when versions are not
  updated
- Fixes #3656
- Fixes #3659

Signed-off-by: Dan Ryan <dan@danryan.co>

Add future import for print function

Signed-off-by: Dan Ryan <dan@danryan.co>

Handle all possible markers in lockfiles

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix json import

Signed-off-by: Dan Ryan <dan@danryan.co>

point to correct reference for lockfile

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix marker merging errors

Signed-off-by: Dan Ryan <dan@danryan.co>

Prevent automatically setting `editable=True`

- Fixes #3647

Signed-off-by: Dan Ryan <dan@danryan.co>

Add new feature toggle for VCS dependency resolution

- Fixes #3577

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix syntax error

Signed-off-by: Dan Ryan <dan@danryan.co>

Use string for environment

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix class name resolution for py27

Signed-off-by: Dan Ryan <dan@danryan.co>

Write json files as unicode

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix resolution with env var

Signed-off-by: Dan Ryan <dan@danryan.co>

news/3577.feature.rst

@@ -0,0 +1 @@
+Added a new environment variable, ``PIPENV_RESOLVE_VCS``, to toggle dependency resolution off for non-editable VCS, file, and URL based dependencies.

news/3647.bugfix.rst

@@ -0,0 +1 @@
+Pipenv will no longer inadvertently set ``editable=True`` on all vcs dependencies.

news/3656.bugfix.rst

@@ -0,0 +1,2 @@
+The ``--keep-outdated`` argument to ``pipenv install`` and ``pipenv lock`` will now drop specifier constraints when encountering editable dependencies.
+- In addition, ``--keep-outdated`` will retain specifiers that would otherwise be dropped from any entries that have not been updated.

pipenv/core.py

@@ -1394,6 +1394,8 @@ def pip_install(
             src_dir = os.environ["PIP_SRC"]
             src = ["--src", os.environ["PIP_SRC"]]
         if not requirement.editable:
+            # Leave this off becauase old lockfiles don't have all deps included
+            # TODO: When can it be turned back on?
             no_deps = False
 
         if src_dir is not None:
@@ -1412,7 +1414,7 @@ def pip_install(
             prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir,
             delete=False
         )
-        line = "-e" if requirement.editable else ""
+        line = "-e " if requirement.editable else ""
         if requirement.editable or requirement.name is not None:
             name = requirement.name
             if requirement.extras:
@@ -1640,7 +1642,6 @@ def system_which(command, mult=False):
     return result
 
 
-
 def format_help(help):
     """Formats the help string."""
     help = help.replace("Options:", str(crayons.normal("Options:", bold=True)))
@@ -1784,8 +1785,8 @@ def do_py(system=False):
             ),
             err=True,
         )
-        return    
+        return
-    
+
     try:
         click.echo(which("python", allow_global=system))
     except AttributeError:

pipenv/environments.py

@@ -236,6 +236,15 @@ Default is to lock dependencies and update ``Pipfile.lock`` on each run.
 NOTE: This only affects the ``install`` and ``uninstall`` commands.
 """
 
+PIPENV_RESOLVE_VCS = _is_env_truthy(os.environ.get("PIPENV_RESOLVE_VCS", 'true'))
+"""Tells Pipenv whether to resolve all VCS dependencies in full.
+
+As of Pipenv 2018.11.26, only editable VCS dependencies were resolved in full.
+To retain this behavior and avoid handling any conflicts that arise from the new
+approach, you may set this to '0', 'off', or 'false'.
+"""
+
+
 PIPENV_PYUP_API_KEY = os.environ.get(
     "PIPENV_PYUP_API_KEY", "1ab8d58f-5122e025-83674263-bc1e79e0"
 )

pipenv/patched/piptools/utils.py

@@ -70,7 +70,7 @@ def clean_requires_python(candidates):
     all_candidates = []
     py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3]))))
     for c in candidates:
-        if c.requires_python:
+        if getattr(c, "requires_python", None):
             # Old specifications had people setting this to single digits
             # which is effectively the same as '>=digit,<digit+1'
             if c.requires_python.isdigit():

pipenv/resolver.py

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, print_function
 import json
 import logging
 import os
@@ -106,9 +109,14 @@ class Entry(object):
         self.pipfile = project.parsed_pipfile.get(pipfile_section, {})
         self.lockfile = project.lockfile_content.get(section, {})
         self.pipfile_dict = self.pipfile.get(self.pipfile_name, {})
-        self.lockfile_dict = self.lockfile.get(name, entry_dict)
+        if self.dev and self.name in project.lockfile_content.get("default", {}):
+            self.lockfile_dict = project.lockfile_content["default"][name]
+        else:
+            self.lockfile_dict = self.lockfile.get(name, entry_dict)
         self.resolver = resolver
         self.reverse_deps = reverse_deps
+        self._original_markers = None
+        self._markers = None
         self._entry = None
         self._lockfile_entry = None
         self._pipfile_entry = None
@@ -232,6 +240,10 @@ class Entry(object):
                 entry_extras.extend(list(self.lockfile_entry.extras))
             self._entry.req.extras = entry_extras
             self.entry_dict["extras"] = self.entry.extras
+        if self.original_markers and not self.markers:
+            original_markers = self.marker_to_str(self.original_markers)
+            self.markers = original_markers
+            self.entry_dict["markers"] = self.marker_to_str(original_markers)
         entry_hashes = set(self.entry.hashes)
         locked_hashes = set(self.lockfile_entry.hashes)
         if entry_hashes != locked_hashes and not self.is_updated:
@@ -248,6 +260,10 @@ class Entry(object):
             self._lockfile_entry = self.make_requirement(self.name, self.lockfile_dict)
         return self._lockfile_entry
 
+    @lockfile_entry.setter
+    def lockfile_entry(self, entry):
+        self._lockfile_entry = entry
+
     @property
     def pipfile_entry(self):
         if self._pipfile_entry is None:
@@ -359,6 +375,7 @@ class Entry(object):
 
     @property
     def updated_specifier(self):
+        # type: () -> str
         return self.entry.specifiers
 
     @property
@@ -373,7 +390,7 @@ class Entry(object):
         return None
 
     def validate_specifiers(self):
-        if self.is_in_pipfile:
+        if self.is_in_pipfile and not self.pipfile_entry.editable:
             return self.pipfile_entry.requirement.specifier.contains(self.updated_version)
         return True
 
@@ -550,8 +567,11 @@ class Entry(object):
                 constraint.check_if_exists(False)
             except Exception:
                 from pipenv.exceptions import DependencyConflict
+                from pipenv.environments import is_verbose
+                if is_verbose():
+                    print("Tried constraint: {0!r}".format(constraint), file=sys.stderr)
                 msg = (
-                    "Cannot resolve conflicting version {0}{1} while {1}{2} is "
+                    "Cannot resolve conflicting version {0}{1} while {2}{3} is "
                     "locked.".format(
                         self.name, self.updated_specifier, self.old_name, self.old_specifiers
                     )
@@ -624,6 +644,7 @@ def clean_results(results, resolver, project, dev=False):
 
 def clean_outdated(results, resolver, project, dev=False):
     from pipenv.vendor.requirementslib.models.requirements import Requirement
+    from pipenv.environments import is_verbose
     if not project.lockfile_exists:
         return results
     lockfile = project.lockfile_content

pipenv/utils.py

@@ -31,7 +31,9 @@ import crayons
 import parse
 
 from . import environments
-from .exceptions import PipenvUsageError, ResolutionFailure, RequirementError, PipenvCmdError
+from .exceptions import (
+    PipenvUsageError, RequirementError, PipenvCmdError, ResolutionFailure
+)
 from .pep508checker import lookup
 from .vendor.urllib3 import util as urllib3_util
 
@@ -398,7 +400,6 @@ class Resolver(object):
     ):
         # type: (...) -> Tuple[Requirement, Dict[str, str], Dict[str, str]]
         from .vendor.requirementslib.models.requirements import Requirement
-        from .exceptions import ResolutionFailure
         if index_lookup is None:
             index_lookup = {}
         if markers_lookup is None:
@@ -444,6 +445,7 @@ class Resolver(object):
         # type: (Requirement, Optional["Resolver"]) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]]
         from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name
         from .vendor.requirementslib.models.requirements import Requirement
+        from requirementslib.utils import is_installable_dir
         constraints = set()  # type: Set[str]
         locked_deps = dict()  # type: Dict[str, Dict[str, Union[str, bool, List[str]]]]
         if (req.is_file_or_url or req.is_vcs) and not req.is_wheel:
@@ -463,7 +465,16 @@ class Resolver(object):
             setup_info = req.req.setup_info
             setup_info.get_info()
             locked_deps[pep423_name(name)] = entry
-            requirements = [v for v in getattr(setup_info, "requires", {}).values()]
+            requirements = []
+            # Allow users to toggle resolution off for non-editable VCS packages
+            # but leave it on for local, installable folders on the filesystem
+            if environments.PIPENV_RESOLVE_VCS or (
+                req.editable or parsed_line.is_wheel or (
+                    req.is_file_or_url and parsed_line.is_local and
+                    is_installable_dir(parsed_line.path)
+                )
+            ):
+                requirements = [v for v in getattr(setup_info, "requires", {}).values()]
             for r in requirements:
                 if getattr(r, "url", None) and not getattr(r, "editable", False):
                     if r is not None:
@@ -1797,13 +1808,15 @@ def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None):
 
     # If a package is **PRESENT** in the pipfile but has no markers, make sure we
     # **NEVER** include markers in the lockfile
-    if "markers" in dep:
+    if "markers" in dep and dep.get("markers", "").strip():
         # First, handle the case where there is no top level dependency in the pipfile
         if not is_top_level:
-            try:
+            translated = translate_markers(dep).get("markers", "").strip()
-                lockfile["markers"] = translate_markers(dep)["markers"]
+            if translated:
-            except TypeError:
+                try:
-                pass
+                    lockfile["markers"] = translated
+                except TypeError:
+                    pass
         # otherwise make sure we are prioritizing whatever the pipfile says about the markers
         # If the pipfile says nothing, then we should put nothing in the lockfile
         else:

tasks/vendoring/patches/patched/piptools.patch

@@ -608,7 +608,7 @@ index 9b4b4c2..8875543 100644
 +    all_candidates = []
 +    py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3]))))
 +    for c in candidates:
-+        if c.requires_python:
++        if getattr(c, "requires_python", None):
 +            # Old specifications had people setting this to single digits
 +            # which is effectively the same as '>=digit,<digit+1'
 +            if c.requires_python.isdigit():

tests/integration/test_lock.py

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+
+import json
 import os
 import sys
 
@@ -5,6 +8,7 @@ import pytest
 
 from flaky import flaky
 from vistir.compat import Path
+from vistir.misc import to_text
 from pipenv.utils import temp_environ
 
 
@@ -122,6 +126,21 @@ def test_keep_outdated_doesnt_upgrade_pipfile_pins(PipenvInstance, pypi):
         assert p.lockfile["default"]["urllib3"]["version"] == "==1.21.1"
 
 
+def test_keep_outdated_keeps_markers_not_removed(PipenvInstance, pypi):
+    with PipenvInstance(chdir=True, pypi=pypi) as p:
+        c = p.pipenv("install tablib")
+        assert c.ok
+        lockfile = Path(p.lockfile_path)
+        lockfile_content = lockfile.read_text()
+        lockfile_json = json.loads(lockfile_content)
+        assert "tablib" in lockfile_json["default"]
+        lockfile_json["default"]["tablib"]["markers"] = "python_version >= '2.7'"
+        lockfile.write_text(to_text(json.dumps(lockfile_json)))
+        c = p.pipenv("lock --keep-outdated")
+        assert c.ok
+        assert p.lockfile["default"]["tablib"].get("markers", "") == "python_version >= '2.7'"
+
+
 @pytest.mark.lock
 @pytest.mark.keep_outdated
 def test_keep_outdated_doesnt_update_satisfied_constraints(PipenvInstance, pypi):