Fix version comparisons in do_outdated

- Fix comparisons between post-release and non-post-release versions
- Update `pip_shims` to accommodate new pip versions
- Implement syntax changes in `environment.py` to account for new pip
  syntax
- Add `required` version output to `pipenv update --outdated`
- Add pipfile version pin information as well, if available

Signed-off-by: Dan Ryan <dan@danryan.co>

news/3766.bugfix.rst

@@ -0,0 +1 @@
+``pipenv update --outdated`` will now correctly handle comparisons between pre/post-releases and normal releases.

news/3766.vendor.rst

@@ -0,0 +1 @@
+Updated ``pip_shims`` to support ``--outdated`` with new pip versions.

pipenv/core.py

@@ -1798,7 +1798,9 @@ def do_py(system=False):
 def do_outdated(pypi_mirror=None):
     # TODO: Allow --skip-lock here?
     from .vendor.requirementslib.models.requirements import Requirement
+    from .vendor.requirementslib.models.utils import get_version
     from .vendor.packaging.utils import canonicalize_name
+    from .vendor.vistir.compat import Mapping
     from collections import namedtuple
 
     packages = {}
@@ -1810,6 +1812,7 @@ def do_outdated(pypi_mirror=None):
         (pkg.project_name, pkg.parsed_version, pkg.latest_version)
         for pkg in project.environment.get_outdated_packages()
     }
+    reverse_deps = project.environment.reverse_dependencies()
     for result in installed_packages:
         dep = Requirement.from_line(str(result.as_requirement()))
         packages.update(dep.as_pipfile())
@@ -1833,10 +1836,26 @@ def do_outdated(pypi_mirror=None):
             elif canonicalize_name(package) in outdated_packages:
                 skipped.append(outdated_packages[canonicalize_name(package)])
     for package, old_version, new_version in skipped:
-        click.echo(crayons.yellow(
-            "Skipped Update of Package {0!s}: {1!s} installed, {2!s} available.".format(
-                package, old_version, new_version
-            )), err=True
+        name_in_pipfile = project.get_package_name_in_pipfile(package)
+        pipfile_version_text = ""
+        required = ""
+        version = None
+        if name_in_pipfile:
+            version = get_version(project.packages[name_in_pipfile])
+            reverse_deps = reverse_deps.get(name_in_pipfile)
+            if isinstance(reverse_deps, Mapping) and "required" in reverse_deps:
+                required = " {0} required".format(reverse_deps["required"])
+            if version:
+                pipfile_version_text = " ({0} set in Pipfile)".format(version)
+            else:
+                pipfile_version_text = " (Unpinned in Pipfile)"
+        click.echo(
+            crayons.yellow(
+                "Skipped Update of Package {0!s}: {1!s} installed,{2!s}{3!s}, "
+                "{4!s} available.".format(
+                    package, old_version, required, pipfile_version_text, new_version
+                )
+            ), err=True
         )
     if not outdated:
         click.echo(crayons.green("All packages are up to date!", bold=True))

pipenv/environment.py

@@ -368,7 +368,9 @@ class Environment(object):
 
     @contextlib.contextmanager
     def get_finder(self, pre=False):
-        from .vendor.pip_shims.shims import Command, cmdoptions, index_group, PackageFinder
+        from .vendor.pip_shims.shims import (
+            Command, cmdoptions, index_group, PackageFinder, parse_version, pip_version
+        )
         from .environments import PIPENV_CACHE_DIR
         index_urls = [source.get("url") for source in self.sources]
 
@@ -387,25 +389,35 @@ class Environment(object):
         pip_options.cache_dir = PIPENV_CACHE_DIR
         pip_options.pre = self.pipfile.get("pre", pre)
         with pip_command._build_session(pip_options) as session:
-            finder = PackageFinder(
-                find_links=pip_options.find_links,
-                index_urls=index_urls, allow_all_prereleases=pip_options.pre,
-                trusted_hosts=pip_options.trusted_hosts,
-                process_dependency_links=pip_options.process_dependency_links,
-                session=session
-            )
+            finder_args = {
+                "find_links": pip_options.find_links,
+                "index_urls": index_urls,
+                "allow_all_prereleases": pip_options.pre,
+                "trusted_hosts": pip_options.trusted_hosts,
+                "session": session
+            }
+            if parse_version(pip_version) < parse_version("19.0"):
+                finder_args.update(
+                    {"process_dependency_links": pip_options.process_dependency_links}
+                )
+            finder = PackageFinder(**finder_args)
             yield finder
 
     def get_package_info(self, pre=False):
+        from .vendor.pip_shims.shims import pip_version, parse_version
         dependency_links = []
         packages = self.get_installed_packages()
         # This code is borrowed from pip's current implementation
-        for dist in packages:
-            if dist.has_metadata('dependency_links.txt'):
-                dependency_links.extend(dist.get_metadata_lines('dependency_links.txt'))
+        if parse_version(pip_version) < parse_version("19.0"):
+            for dist in packages:
+                if dist.has_metadata('dependency_links.txt'):
+                    dependency_links.extend(
+                        dist.get_metadata_lines('dependency_links.txt')
+                    )
 
         with self.get_finder() as finder:
-            finder.add_dependency_links(dependency_links)
+            if parse_version(pip_version) < parse_version("19.0"):
+                finder.add_dependency_links(dependency_links)
 
             for dist in packages:
                 typ = 'unknown'
@@ -433,7 +445,7 @@ class Environment(object):
     def get_outdated_packages(self, pre=False):
         return [
             pkg for pkg in self.get_package_info(pre=pre)
-            if pkg.latest_version._version > pkg.parsed_version._version
+            if pkg.latest_version._key > pkg.parsed_version._key
         ]
 
     @classmethod

pipenv/vendor/pip_shims/__init__.py

@@ -3,7 +3,7 @@ from __future__ import absolute_import
 
 import sys
 
-__version__ = '0.3.3'
+__version__ = "0.3.3"
 
 from . import shims
 

pipenv/vendor/pip_shims/shims.py

@@ -15,7 +15,7 @@ from six.moves import Callable  # type: ignore  # noqa  # isort:skip
 
 
 class _shims(object):
-    CURRENT_PIP_VERSION = "19.0.3"
+    CURRENT_PIP_VERSION = "19.1.1"
     BASE_IMPORT_PATH = os.environ.get("PIP_SHIMS_BASE_MODULE", "pip")
     path_info = namedtuple("PathInfo", "path start_version end_version")
 
@@ -141,14 +141,25 @@ class _shims(object):
             ),
             "Link": ("index.Link", "7.0.0", "9999"),
             "make_abstract_dist": (
-                ("operations.prepare.make_abstract_dist", "10.0.0", "9999"),
+                (
+                    "distributions.make_distribution_for_install_requirement",
+                    "19.1.2",
+                    "9999",
+                ),
+                ("operations.prepare.make_abstract_dist", "10.0.0", "19.1.1"),
                 ("req.req_set.make_abstract_dist", "7.0.0", "9.0.3"),
             ),
+            "make_distribution_for_install_requirement": (
+                "distributions.make_distribution_for_install_requirement",
+                "19.1.2",
+                "9999",
+            ),
             "make_option_group": (
                 ("cli.cmdoptions.make_option_group", "18.1", "9999"),
                 ("cmdoptions.make_option_group", "7.0.0", "18.0"),
             ),
             "PackageFinder": ("index.PackageFinder", "7.0.0", "9999"),
+            "CandidateEvaluator": ("index.CandidateEvaluator", "19.1", "9999"),
             "parse_requirements": ("req.req_file.parse_requirements", "7.0.0", "9999"),
             "path_to_url": ("download.path_to_url", "7.0.0", "9999"),
             "PipError": ("exceptions.PipError", "7.0.0", "9999"),
@@ -159,18 +170,44 @@ class _shims(object):
             ),
             "RequirementSet": ("req.req_set.RequirementSet", "7.0.0", "9999"),
             "RequirementTracker": ("req.req_tracker.RequirementTracker", "7.0.0", "9999"),
-            "Resolver": ("resolve.Resolver", "7.0.0", "9999"),
+            "Resolver": (
+                ("resolve.Resolver", "7.0.0", "19.1.1"),
+                ("legacy_resolve.Resolver", "19.1.2", "9999"),
+            ),
             "SafeFileCache": ("download.SafeFileCache", "7.0.0", "9999"),
             "UninstallPathSet": ("req.req_uninstall.UninstallPathSet", "7.0.0", "9999"),
             "url_to_path": ("download.url_to_path", "7.0.0", "9999"),
             "USER_CACHE_DIR": ("locations.USER_CACHE_DIR", "7.0.0", "9999"),
-            "VcsSupport": ("vcs.VcsSupport", "7.0.0", "9999"),
+            "VcsSupport": (
+                ("vcs.VcsSupport", "7.0.0", "19.1.1"),
+                ("vcs.versioncontrol.VcsSupport", "19.2", "9999"),
+            ),
             "Wheel": ("wheel.Wheel", "7.0.0", "9999"),
             "WheelCache": (
                 ("cache.WheelCache", "10.0.0", "9999"),
                 ("wheel.WheelCache", "7", "9.0.3"),
             ),
             "WheelBuilder": ("wheel.WheelBuilder", "7.0.0", "9999"),
+            "AbstractDistribution": (
+                "distributions.base.AbstractDistribution",
+                "19.1.2",
+                "9999",
+            ),
+            "InstalledDistribution": (
+                "distributions.installed.InstalledDistribution",
+                "19.1.2",
+                "9999",
+            ),
+            "SourceDistribution": (
+                ("req.req_set.IsSDist", "7.0.0", "9.0.3"),
+                ("operations.prepare.IsSDist", "10.0.0", "19.1.1"),
+                ("distributions.source.SourceDistribution", "19.1.2", "9999"),
+            ),
+            "WheelDistribution": (
+                "distributions.wheel.WheelDistribution",
+                "19.1.2",
+                "9999",
+            ),
             "PyPI": ("models.index.PyPI", "7.0.0", "9999"),
             "stdlib_pkgs": (
                 ("utils.compat.stdlib_pkgs", "18.1", "9999"),

tests/integration/conftest.py

@@ -194,11 +194,15 @@ WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh()
 class _Pipfile(object):
     def __init__(self, path):
         self.path = path
-        self.document = tomlkit.document()
-        self.document["source"] = tomlkit.aot()
-        self.document["requires"] = tomlkit.table()
-        self.document["packages"] = tomlkit.table()
-        self.document["dev_packages"] = tomlkit.table()
+        if self.path.exists():
+            self.loads()
+        else:
+            self.document = tomlkit.document()
+        self.document["source"] = self.document.get("source", tomlkit.aot())
+        self.document["requires"] = self.document.get("requires", tomlkit.table())
+        self.document["packages"] = self.document.get("packages", tomlkit.table())
+        self.document["dev_packages"] = self.document.get("dev_packages", tomlkit.table())
+        super(_Pipfile, self).__init__()
 
     def install(self, package, value, dev=False):
         section = "packages" if not dev else "dev_packages"
@@ -210,9 +214,20 @@ class _Pipfile(object):
             self.document[section][package] = value
         self.write()
 
+    def remove(self, package, dev=False):
+        section = "packages" if not dev else "dev_packages"
+        if not dev and package not in self.document[section]:
+            if package in self.document["dev_packages"]:
+                section = "dev_packages"
+        del self.document[section][package]
+        self.write()
+
     def add(self, package, value, dev=False):
         self.install(package, value, dev=dev)
 
+    def update(self, package, value, dev=False):
+        self.install(package, value, dev=dev)
+
     def loads(self):
         self.document = tomlkit.loads(self.path.read_text())
 

tests/integration/test_install_twists.py

@@ -372,3 +372,17 @@ def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, testsr
 
         c = p.pipenv('run python -c "import requests, flask, six, jinja2"')
         assert c.return_code == 0, c.err
+
+
+@pytest.mark.outdated
+def test_outdated_should_compare_postreleases_without_failing(PipenvInstance):
+    with PipenvInstance(chdir=True) as p:
+        c = p.pipenv("install Shapely==1.6.4")
+        assert c.return_code == 0
+        c = p.pipenv("update --outdated")
+        assert c.return_code == 0
+        assert "Skipped Update" in c.out
+        p._pipfile.update("shapely", "*")
+        c = p.pipenv("update --outdated")
+        assert c.return_code != 0
+        assert "out-of-date" in c.out