Merge branch 'bugfix/3148' of github.com:pypa/pipenv into bugfix/3148

Signed-off-by: Dan Ryan <dan@danryan.co>

news/3148.bugfix.rst

@@ -0,0 +1 @@
+Fixed resolution of direct-url dependencies in ``setup.py`` files to respect ``PEP-508`` style URL dependencies.

pipenv/_compat.py

@@ -5,9 +5,6 @@ Exposes a standard API that enables compatibility across python versions,
 operating systems, etc.
 """
 
-import functools
-import importlib
-import io
 import os
 import sys
 import warnings

pipenv/environment.py

@@ -11,15 +11,16 @@ import sys
 from distutils.sysconfig import get_python_lib
 from sysconfig import get_paths
 
+import itertools
 import pkg_resources
 import six
-import vistir
 
 import pipenv
 
-from cached_property import cached_property
+from .vendor.cached_property import cached_property
+import vistir
 
-from .utils import normalize_path
+from .utils import normalize_path, make_posix
 
 
 BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path)
@@ -149,7 +150,7 @@ class Environment(object):
         'stdlib': '/home/hawk/.pyenv/versions/3.7.1/lib/python3.7'}
         """
 
-        prefix = self.prefix.as_posix()
+        prefix = make_posix(self.prefix.as_posix())
         install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix'
         paths = get_paths(install_scheme, vars={
             'base': prefix,
@@ -158,8 +159,8 @@ class Environment(object):
         paths["PATH"] = paths["scripts"] + os.pathsep + os.defpath
         if "prefix" not in paths:
             paths["prefix"] = prefix
-        purelib = get_python_lib(plat_specific=0, prefix=prefix)
-        platlib = get_python_lib(plat_specific=1, prefix=prefix)
+        purelib = make_posix(get_python_lib(plat_specific=0, prefix=prefix))
+        platlib = make_posix(get_python_lib(plat_specific=1, prefix=prefix))
         if purelib == platlib:
             lib_dirs = purelib
         else:
@@ -180,7 +181,7 @@ class Environment(object):
     @property
     def python(self):
         """Path to the environment python"""
-        py = vistir.compat.Path(self.base_paths["scripts"]).joinpath("python").as_posix()
+        py = vistir.compat.Path(self.base_paths["scripts"]).joinpath("python").absolute().as_posix()
         if not py:
             return vistir.compat.Path(sys.executable).as_posix()
         return py
@@ -248,7 +249,10 @@ class Environment(object):
         """
 
         pkg_resources = self.safe_import("pkg_resources")
-        return pkg_resources.find_distributions(self.paths["libdirs"])
+        libdirs = self.base_paths["libdirs"].split(os.pathsep)
+        dists = (pkg_resources.find_distributions(libdir) for libdir in libdirs)
+        for dist in itertools.chain.from_iterable(dists):
+            yield dist
 
     def find_egg(self, egg_dist):
         """Find an egg by name in the given environment"""
@@ -276,13 +280,14 @@ class Environment(object):
         """Determine whether the supplied distribution is in the environment."""
         from .project import _normalized
         prefixes = [
-            _normalized(prefix) for prefix in self.base_paths["libdirs"]
-            if _normalized(self.prefix).startswith(_normalized(prefix))
+            _normalized(prefix) for prefix in self.base_paths["libdirs"].split(os.pathsep)
+            if _normalized(prefix).startswith(_normalized(self.prefix.as_posix()))
         ]
         location = self.locate_dist(dist)
         if not location:
             return False
-        return any(_normalized(location).startswith(prefix) for prefix in prefixes)
+        location = _normalized(make_posix(location))
+        return any(location.startswith(prefix) for prefix in prefixes)
 
     def get_installed_packages(self):
         """Returns all of the installed packages in a given environment"""
@@ -295,7 +300,7 @@ class Environment(object):
 
     @contextlib.contextmanager
     def get_finder(self, pre=False):
-        from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder
+        from .vendor.pip_shims.shims import Command, cmdoptions, index_group, PackageFinder
         from .environments import PIPENV_CACHE_DIR
         index_urls = [source.get("url") for source in self.sources]
 
@@ -616,10 +621,7 @@ class Environment(object):
                 monkey_patch.activate()
             pip_shims = self.safe_import("pip_shims")
             pathset_base = pip_shims.UninstallPathSet
-            import recursive_monkey_patch
-            recursive_monkey_patch.monkey_patch(
-                PatchedUninstaller, pathset_base
-            )
+            pathset_base._permitted = PatchedUninstaller._permitted
             dist = next(
                 iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())),
                 None

pipenv/project.py

@@ -20,7 +20,7 @@ from first import first
 import pipfile
 import pipfile.api
 
-from cached_property import cached_property
+from .vendor.cached_property import cached_property
 
 from .cmdparse import Script
 from .environment import Environment
@@ -29,6 +29,7 @@ from .environments import (
     PIPENV_PIPFILE, PIPENV_PYTHON, PIPENV_TEST_INDEX, PIPENV_VENV_IN_PROJECT,
     is_in_virtualenv
 )
+from .vendor.requirementslib.models.utils import get_default_pyproject_backend
 from .utils import (
     cleanup_toml, convert_toml_outline_tables, find_requirements,
     get_canonical_names, get_url_name, get_workon_home, is_editable,
@@ -529,8 +530,8 @@ class Project(object):
             if not os.path.exists(self.path_to("setup.py")):
                 if not build_system or not build_system.get("requires"):
                     build_system = {
-                        "requires": ["setuptools>=38.2.5", "wheel"],
-                        "build-backend": "setuptools.build_meta",
+                        "requires": ["setuptools>=40.8.0", "wheel"],
+                        "build-backend": get_default_pyproject_backend(),
                     }
                 self._build_system = build_system
 
@@ -540,7 +541,7 @@ class Project(object):
 
     @property
     def build_backend(self):
-        return self._build_system.get("build-backend", None)
+        return self._build_system.get("build-backend", get_default_pyproject_backend())
 
     @property
     def settings(self):
@@ -607,10 +608,8 @@ class Project(object):
 
     def _get_editable_packages(self, dev=False):
         section = "dev-packages" if dev else "packages"
-        # section = "{0}-editable".format(section)
         packages = {
             k: v
-            # for k, v in self._pipfile[section].items()
             for k, v in self.parsed_pipfile.get(section, {}).items()
             if is_editable(k) or is_editable(v)
         }
@@ -619,10 +618,8 @@ class Project(object):
     def _get_vcs_packages(self, dev=False):
         from pipenv.vendor.requirementslib.utils import is_vcs
         section = "dev-packages" if dev else "packages"
-        # section = "{0}-vcs".format(section)
         packages = {
             k: v
-            # for k, v in self._pipfile[section].items()
             for k, v in self.parsed_pipfile.get(section, {}).items()
             if is_vcs(v) or is_vcs(k)
         }

pipenv/utils.py

@@ -3,6 +3,7 @@ import contextlib
 import errno
 import logging
 import os
+import posixpath
 import re
 import shutil
 import stat
@@ -22,8 +23,8 @@ six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc"))
 six.add_move(six.MovedAttribute("Set", "collections", "collections.abc"))  # noqa
 from six.moves import Mapping, Sequence, Set
 from six.moves.urllib.parse import urlparse
-from vistir.compat import ResourceWarning, lru_cache
-from vistir.misc import fs_str
+from .vendor.vistir.compat import ResourceWarning, lru_cache
+from .vendor.vistir.misc import fs_str
 
 import crayons
 import parse
@@ -35,7 +36,7 @@ from .vendor.urllib3 import util as urllib3_util
 
 
 if environments.MYPY_RUNNING:
-    from typing import Tuple, Dict, Any, List, Union, Optional
+    from typing import Tuple, Dict, Any, List, Union, Optional, Text
     from .vendor.requirementslib.models.requirements import Requirement, Line
     from .project import Project
 
@@ -280,7 +281,7 @@ class Resolver(object):
     @staticmethod
     @lru_cache()
     def _get_pip_command():
-        from pip_shims.shims import Command
+        from .vendor.pip_shims.shims import Command
 
         class PipCommand(Command):
             """Needed for pip-tools."""
@@ -767,7 +768,7 @@ def create_spinner(text, nospin=None, spinner_name=None):
 
 
 def resolve(cmd, sp):
-    from .vendor import delegator
+    import delegator
     from .cmdparse import Script
     from .vendor.pexpect.exceptions import EOF, TIMEOUT
     from .vendor.vistir.compat import to_native_string
@@ -1486,7 +1487,7 @@ def handle_remove_readonly(func, path, exc):
         warnings.warn(default_warning_message.format(path), ResourceWarning)
         return
 
-    raise
+    raise exc
 
 
 def escape_cmd(cmd):
@@ -1769,7 +1770,22 @@ def add_to_set(original_set, element):
 
 def is_url_equal(url, other_url):
     # type: (str, str) -> bool
-    """Compare two urls by scheme, host, and path, ignoring auth"""
+    """
+    Compare two urls by scheme, host, and path, ignoring auth
+
+    :param str url: The initial URL to compare
+    :param str url: Second url to compare to the first
+    :return: Whether the URLs are equal without **auth**, **query**, and **fragment**
+    :rtype: bool
+
+    >>> is_url_equal("https://user:pass@mydomain.com/some/path?some_query",
+                     "https://user2:pass2@mydomain.com/some/path")
+    True
+
+    >>> is_url_equal("https://user:pass@mydomain.com/some/path?some_query",
+                 "https://mydomain.com/some?some_query")
+    False
+    """
     if not isinstance(url, six.string_types):
         raise TypeError("Expected string for url, received {0!r}".format(url))
     if not isinstance(other_url, six.string_types):
@@ -1781,6 +1797,34 @@ def is_url_equal(url, other_url):
     return unparsed == unparsed_other
 
 
+@lru_cache()
+def make_posix(path):
+    # type: (str) -> str
+    """
+    Convert a path with possible windows-style separators to a posix-style path
+    (with **/** separators instead of **\\** separators).
+
+    :param Text path: A path to convert.
+    :return: A converted posix-style path
+    :rtype: Text
+
+    >>> make_posix("c:/users/user/venvs/some_venv\\Lib\\site-packages")
+    "c:/users/user/venvs/some_venv/Lib/site-packages"
+
+    >>> make_posix("c:\\users\\user\\venvs\\some_venv")
+    "c:/users/user/venvs/some_venv"
+    """
+    if not isinstance(path, six.string_types):
+        raise TypeError("Expected a string for path, received {0!r}...".format(path))
+    starts_with_sep = path.startswith(os.path.sep)
+    separated = normalize_path(path).split(os.path.sep)
+    if isinstance(separated, (list, tuple)):
+        path = posixpath.join(*separated)
+        if starts_with_sep:
+            path = "/{0}".format(path)
+    return path
+
+
 def get_pipenv_dist(pkg="pipenv", pipenv_site=None):
     from .resolver import find_site_path
     pipenv_libdir = os.path.dirname(os.path.abspath(__file__))

pipenv/vendor/requirementslib/models/utils.py

@@ -366,7 +366,7 @@ def get_pyproject(path):
     if not pp_toml.exists():
         if not setup_py.exists():
             return None
-        requires = ["setuptools>=40.6", "wheel"]
+        requires = ["setuptools>=40.8", "wheel"]
         backend = get_default_pyproject_backend()
     else:
         pyproject_data = {}
@@ -375,10 +375,10 @@ def get_pyproject(path):
         build_system = pyproject_data.get("build-system", None)
         if build_system is None:
             if setup_py.exists():
-                requires = ["setuptools>=40.6", "wheel"]
+                requires = ["setuptools>=40.8", "wheel"]
                 backend = get_default_pyproject_backend()
             else:
-                requires = ["setuptools>=40.6", "wheel"]
+                requires = ["setuptools>=40.8", "wheel"]
                 backend = get_default_pyproject_backend()
             build_system = {
                 "requires": requires,
@@ -386,7 +386,7 @@ def get_pyproject(path):
             }
             pyproject_data["build_system"] = build_system
         else:
-            requires = build_system.get("requires", ["setuptools>=40.6", "wheel"])
+            requires = build_system.get("requires", ["setuptools>=40.8", "wheel"])
             backend = build_system.get("build-backend", get_default_pyproject_backend())
     return (requires, backend)
 

tests/integration/conftest.py

@@ -1,3 +1,4 @@
+# -*- coding=utf-8 -*-
 import json
 import os
 import sys
@@ -80,6 +81,10 @@ def pytest_runtest_setup(item):
         pytest.skip('requires github ssh')
     if item.get_marker('needs_hg') is not None and not WE_HAVE_HG:
         pytest.skip('requires mercurial')
+    if item.get_marker('skip_py27_win') is not None and (
+        sys.version_info[:2] <= (2, 7) and os.name == "nt"
+    ):
+        pytest.skip('must use python > 2.7 on windows')
 
 
 @pytest.fixture
@@ -114,6 +119,8 @@ def isolate(pathlib_tmpdir):
     os.environ["GIT_AUTHOR_EMAIL"] = fs_str("pipenv@pipenv.org")
     mkdir_p(os.path.join(home_dir, ".virtualenvs"))
     os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs"))
+    os.environ["HOME"] = home_dir
+    mkdir_p(os.path.join(home_dir, "projects"))
     # Ignore PIPENV_ACTIVE so that it works as under a bare environment.
     os.environ.pop("PIPENV_ACTIVE", None)
     os.environ.pop("VIRTUAL_ENV", None)
@@ -189,7 +196,7 @@ class _PipenvInstance(object):
     """An instance of a Pipenv Project..."""
     def __init__(
         self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None,
-        venv_root=None, ignore_virtualenvs=True, venv_in_project=True
+        venv_root=None, ignore_virtualenvs=True, venv_in_project=True, name=None
     ):
         self.pypi = pypi
         if ignore_virtualenvs:
@@ -203,13 +210,24 @@ class _PipenvInstance(object):
 
         self.original_dir = os.path.abspath(os.curdir)
         path = path if path else os.environ.get("PIPENV_PROJECT_DIR", None)
+        if name is not None:
+            path = Path(os.environ["HOME"]) / "projects" / name
+            path.mkdir(exist_ok=True)
         if not path:
-            self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-')
+            path = TemporaryDirectory(suffix='-project', prefix='pipenv-')
+        if isinstance(path, TemporaryDirectory):
+            self._path = path
             path = Path(self._path.name)
             try:
                 self.path = str(path.resolve())
             except OSError:
                 self.path = str(path.absolute())
+        elif isinstance(path, Path):
+            self._path = path
+            try:
+                self.path = str(path.resolve())
+            except OSError:
+                self.path = str(path.absolute())
         else:
             self._path = path
             self.path = path
@@ -240,7 +258,7 @@ class _PipenvInstance(object):
         if self.chdir:
             os.chdir(self.original_dir)
         self.path = None
-        if self._path:
+        if self._path and getattr(self._path, "cleanup", None):
             try:
                 self._path.cleanup()
             except OSError as e:

tests/integration/test_install_basic.py

@@ -461,11 +461,11 @@ version = "*"
 extras = ["socks"]
             """.strip()
             f.write(contents)
-        c = p.pipenv("install plette[validation]")
+        c = p.pipenv("install plette")
         assert c.return_code == 0
         with open(p.pipfile_path) as f:
             contents = f.read()
         assert "[packages.requests]" not in contents
         assert 'six = {version = "*"}' in contents
         assert 'requests = {version = "*"' in contents
-        assert 'plette = {' in contents
+        assert 'plette = "*"' in contents

tests/integration/test_install_twists.py

@@ -57,7 +57,7 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]}
         project.write_toml({"packages": {}, "dev-packages": {}})
         c = p.pipenv("install {0}".format(line))
         assert c.return_code == 0
-        assert "testpipenv" in p.pipfile["packages"], "{0}\n{1}\n\n{2}\n\n{3}".format(p.pipfile, Path(p.pipfile_path).read_text(), Path(os.getcwd()).joinpath("setup.py").read_text(), Path(os.path.join(os.getcwd(), "testpipenv.egg-info/PKG-INFO")).read_text())
+        assert "testpipenv" in p.pipfile["packages"]
         assert p.pipfile["packages"]["testpipenv"]["path"] == "."
         assert p.pipfile["packages"]["testpipenv"]["extras"] == ["dev"]
         assert "six" in p.lockfile["default"]

tests/integration/test_install_uri.py

@@ -1,5 +1,4 @@
-import os
-
+# -*- coding=utf-8 -*-
 import pytest
 
 from flaky import flaky
@@ -239,13 +238,15 @@ def test_get_vcs_refs(PipenvInstance, pip_src_dir):
 @pytest.mark.vcs
 @pytest.mark.install
 @pytest.mark.needs_internet
+@pytest.mark.skip_py27_win
 def test_vcs_entry_supersedes_non_vcs(PipenvInstance, pip_src_dir):
     """See issue #2181 -- non-editable VCS dep was specified, but not showing up
     in the lockfile -- due to not running pip install before locking and not locking
     the resolution graph of non-editable vcs dependencies.
     """
     with PipenvInstance(chdir=True) as p:
-        pyinstaller_path = p._pipfile.get_fixture_path("git/pyinstaller")
+        # pyinstaller_path = p._pipfile.get_fixture_path("git/pyinstaller")
+        pyinstaller_uri = "https://github.com/pyinstaller/pyinstaller.git"
         with open(p.pipfile_path, "w") as f:
             f.write(
                 """
@@ -257,7 +258,7 @@ name = "pypi"
 [packages]
 PyUpdater = "*"
 PyInstaller = {{ref = "develop", git = "{0}"}}
-            """.format(pyinstaller_path.as_uri()).strip()
+            """.format(pyinstaller_uri).strip()
             )
         c = p.pipenv("install")
         assert c.return_code == 0
@@ -268,7 +269,7 @@ PyInstaller = {{ref = "develop", git = "{0}"}}
         assert p.lockfile["default"]["pyinstaller"].get("ref") is not None
         assert (
             p.lockfile["default"]["pyinstaller"]["git"]
-            == pyinstaller_path.as_uri()
+            == pyinstaller_uri
         )
 
 

tests/integration/test_pipenv.py

@@ -5,12 +5,8 @@ XXX: Try our best to reduce tests in this file.
 
 import os
 
-from tempfile import mkdtemp
-
-import mock
 import pytest
 
-from pipenv._compat import Path
 from pipenv.project import Project
 from pipenv.utils import temp_environ
 from pipenv.vendor import delegator
@@ -93,22 +89,16 @@ def test_proper_names_unamanged_virtualenv(PipenvInstance, pypi):
 
 
 @pytest.mark.cli
-def test_directory_with_leading_dash(PipenvInstance):
-    def mocked_mkdtemp(suffix, prefix, dir):
-        if suffix == '-project':
-            prefix = '-dir-with-leading-dash'
-        return mkdtemp(suffix, prefix, dir)
-
-    with mock.patch('pipenv.vendor.vistir.compat.mkdtemp', side_effect=mocked_mkdtemp):
-        with temp_environ(), PipenvInstance(chdir=True) as p:
-            if "PIPENV_VENV_IN_PROJECT" in os.environ:
-                del os.environ['PIPENV_VENV_IN_PROJECT']
-            c = p.pipenv('--python python')
-            assert c.return_code == 0
-            c = p.pipenv('--venv')
-            assert c.return_code == 0
-            venv_path = c.out.strip()
-            assert os.path.isdir(venv_path)
-            # Manually clean up environment, since PipenvInstance assumes that
-            # the virutalenv is in the project directory.
-            p.pipenv('--rm')
+def test_directory_with_leading_dash(monkeypatch, PipenvInstance):
+    with temp_environ(), PipenvInstance(chdir=True, venv_in_project=False, name="-project-with-dash") as p:
+        if "PIPENV_VENV_IN_PROJECT" in os.environ:
+            del os.environ['PIPENV_VENV_IN_PROJECT']
+        c = p.pipenv('run pip freeze')
+        assert c.return_code == 0
+        c = p.pipenv('--venv')
+        assert c.return_code == 0
+        venv_path = c.out.strip()
+        assert os.path.isdir(venv_path)
+        # Manually clean up environment, since PipenvInstance assumes that
+        # the virutalenv is in the project directory.
+        p.pipenv('--rm')

tests/integration/test_project.py

@@ -171,11 +171,11 @@ def test_include_editable_packages(PipenvInstance, pypi, testsroot, pathlib_tmpd
 @pytest.mark.virtualenv
 def test_run_in_virtualenv_with_global_context(PipenvInstance, pypi, virtualenv):
     with PipenvInstance(chdir=True, pypi=pypi, venv_root=virtualenv.as_posix(), ignore_virtualenvs=False, venv_in_project=False) as p:
-        c = p.pipenv('run which pip')
+        c = p.pipenv('run pip freeze')
         assert c.return_code == 0
         assert 'Creating a virtualenv' not in c.err
         project = Project()
-        assert project.virtualenv_location == str(virtualenv)
+        assert project.virtualenv_location == virtualenv.as_posix()
         c = p.pipenv("run pip install click")
         assert c.return_code == 0
         assert "Courtesy Notice" in c.err
@@ -183,7 +183,7 @@ def test_run_in_virtualenv_with_global_context(PipenvInstance, pypi, virtualenv)
         assert c.return_code == 0
         c = p.pipenv('run python -c "import click;print(click.__file__)"')
         assert c.return_code == 0
-        assert c.out.strip().startswith(virtualenv.as_posix())
+        assert c.out.strip().startswith(str(virtualenv))
         c = p.pipenv("clean --dry-run")
         assert c.return_code == 0
         assert "click" in c.out
@@ -193,7 +193,7 @@ def test_run_in_virtualenv_with_global_context(PipenvInstance, pypi, virtualenv)
 @pytest.mark.virtualenv
 def test_run_in_virtualenv(PipenvInstance, pypi):
     with PipenvInstance(chdir=True, pypi=pypi) as p:
-        c = p.pipenv('run which pip')
+        c = p.pipenv('run pip freeze')
         assert c.return_code == 0
         assert 'Creating a virtualenv' in c.err
         project = Project()