Fix pep508 direct URL depedencies

- Fixes #3148

Signed-off-by: Dan Ryan <dan@danryan.co>

pipenv/core.py

@@ -722,24 +722,25 @@ def batch_install(deps_list, procs, failed_deps_queue,
                 os.environ["PIP_USER"] = vistir.compat.fs_str("0")
                 if "PYTHONHOME" in os.environ:
                     del os.environ["PYTHONHOME"]
-            if no_deps:
+            if not needs_deps:
                 link = getattr(dep.req, "link", None)
                 is_wheel = False
                 if link:
                     is_wheel = link.is_wheel
                 is_non_editable_vcs = (dep.is_vcs and not dep.editable)
-                no_deps = not (dep.is_file_or_url and not (is_wheel or dep.editable))
+                needs_deps = dep.is_file_or_url and not (is_wheel or dep.editable)
             c = pip_install(
                 dep,
                 ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]),
                 allow_global=allow_global,
-                no_deps=no_deps,
+                no_deps=not needs_deps,
                 block=any([dep.editable, dep.is_vcs, blocking]),
                 index=index,
                 requirements_dir=requirements_dir,
                 pypi_mirror=pypi_mirror,
                 trusted_hosts=trusted_hosts,
-                extra_indexes=extra_indexes
+                extra_indexes=extra_indexes,
+                use_pep517=not retry,
             )
             if dep.is_vcs or dep.editable:
                 c.block()
@@ -822,33 +823,33 @@ def do_install_dependencies(
     else:
         install_kwargs["nprocs"] = 1
 
-    with project.environment.activated():
-        batch_install(
-            deps_list, procs, failed_deps_queue, requirements_dir, **install_kwargs
+    # with project.environment.activated():
+    batch_install(
+        deps_list, procs, failed_deps_queue, requirements_dir, **install_kwargs
+    )
+
+    if not procs.empty():
+        _cleanup_procs(procs, concurrent, failed_deps_queue)
+
+    # Iterate over the hopefully-poorly-packaged dependencies…
+    if not failed_deps_queue.empty():
+        click.echo(
+            crayons.normal(fix_utf8("Installing initially failed dependencies…"), bold=True)
         )
-
-        if not procs.empty():
-            _cleanup_procs(procs, concurrent, failed_deps_queue)
-
-        # Iterate over the hopefully-poorly-packaged dependencies…
-        if not failed_deps_queue.empty():
-            click.echo(
-                crayons.normal(fix_utf8("Installing initially failed dependencies…"), bold=True)
-            )
-            retry_list = []
-            while not failed_deps_queue.empty():
-                failed_dep = failed_deps_queue.get()
-                retry_list.append(failed_dep)
-            install_kwargs.update({
-                "nprocs": 1,
-                "retry": False,
-                "blocking": True,
-            })
-            batch_install(
-                retry_list, procs, failed_deps_queue, requirements_dir, **install_kwargs
-            )
-        if not procs.empty():
-            _cleanup_procs(procs, False, failed_deps_queue, retry=False)
+        retry_list = []
+        while not failed_deps_queue.empty():
+            failed_dep = failed_deps_queue.get()
+            retry_list.append(failed_dep)
+        install_kwargs.update({
+            "nprocs": 1,
+            "retry": False,
+            "blocking": True,
+        })
+        batch_install(
+            retry_list, procs, failed_deps_queue, requirements_dir, **install_kwargs
+        )
+    if not procs.empty():
+        _cleanup_procs(procs, False, failed_deps_queue, retry=False)
 
 
 def convert_three_to_python(three, python):
@@ -1266,7 +1267,8 @@ def pip_install(
     requirements_dir=None,
     extra_indexes=None,
     pypi_mirror=None,
-    trusted_hosts=None
+    trusted_hosts=None,
+    use_pep517=True
 ):
     from pipenv.patched.notpip._internal import logger as piplogger
     from .utils import Mapping
@@ -1450,6 +1452,8 @@ def pip_install(
     if not ignore_hashes:
         pip_command.append("--require-hashes")
     pip_command.append("--no-build-isolation")
+    if not use_pep517:
+        pip_command.append("--no-use-pep517")
     if environments.is_verbose():
         click.echo("$ {0}".format(pip_command), err=True)
     cache_dir = vistir.compat.Path(PIPENV_CACHE_DIR)
@@ -1469,8 +1473,8 @@ def pip_install(
     cmd = Script.parse(pip_command)
     pip_command = cmd.cmdify()
     c = None
-    with project.environment.activated():
-        c = delegator.run(pip_command, block=block, env=pip_config)
+    # with project.environment.activated():
+    c = delegator.run(pip_command, block=block, env=pip_config)
     return c
 
 

pipenv/environment.py

@@ -487,9 +487,8 @@ class Environment(object):
             os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1")
             from .environments import PIPENV_USE_SYSTEM
             if self.is_venv:
-                if not PIPENV_USE_SYSTEM:
-                    os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"]
-                    os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix)
+                os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"]
+                os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix)
             else:
                 if not PIPENV_USE_SYSTEM and not os.environ.get("VIRTUAL_ENV"):
                     os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"]
@@ -502,7 +501,7 @@ class Environment(object):
             pep517_dir = os.path.join(os.path.dirname(pip_vendor.__file__), "pep517")
             site.addsitedir(pep517_dir)
             os.environ["PYTHONPATH"] = os.pathsep.join([
-                os.environ["PYTHONPATH"], pep517_dir
+                os.environ.get("PYTHONPATH", self.base_paths["PYTHONPATH"]), pep517_dir
             ])
             if include_extras:
                 site.addsitedir(parent_path)

pipenv/environments.py

@@ -303,3 +303,14 @@ def is_in_virtualenv():
 PIPENV_SPINNER_FAIL_TEXT = fix_utf8(u"✘ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}")
 
 PIPENV_SPINNER_OK_TEXT = fix_utf8(u"✔ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}")
+
+
+def is_type_checking():
+    try:
+        from typing import TYPE_CHECKING
+    except ImportError:
+        return False
+    return TYPE_CHECKING
+
+
+MYPY_RUNNING = is_type_checking()

pipenv/utils.py

@@ -35,6 +35,10 @@ from .exceptions import PipenvUsageError
 from .pep508checker import lookup
 
 
+if environments.MYPY_RUNNING:
+    from typing import Tuple, Dict, Any, List, Union
+
+
 logging.basicConfig(level=logging.ERROR)
 
 specifiers = [k for k in lookup.keys()]
@@ -228,69 +232,79 @@ def prepare_pip_source_args(sources, pip_args=None):
     return pip_args
 
 
-def resolve_separate(req):
-    """
-    Resolve a requirement that the normal resolver can't
+# def resolve_separate(req):
+#     # type: ('.vendor.requirementslib.requirements.Requirement') -> Tuple[Set[str], Dict[str, Dict[Any]]]
+#     """
+#     Resolve a requirement that the normal resolver can't
 
-    This includes non-editable urls to zip or tarballs, non-editable paths, etc.
-    """
+#     This includes non-editable urls to zip or tarballs, non-editable paths, etc.
+#     """
 
-    from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name
-    from .vendor.requirementslib.models.requirements import Requirement
-    constraints = set()
-    lockfile_update = {}
-    if req.is_file_or_url and not req.is_vcs:
-        setup_info = req.run_requires()
-        requirements = [v for v in setup_info.get("requires", {}).values()]
-        for r in requirements:
-            if getattr(r, "url", None) and not getattr(r, "editable", False):
-                requirement = Requirement.from_line(_requirement_to_str_lowercase_name(r))
-                constraint_update, child_lockfile = resolve_separate(requirement)
-                constraints |= constraint_update
-                lockfile_update.update(child_lockfile)
-                # for local packages with setup.py files and potential direct url deps:
-                if req.editable and requirement.is_direct_url:
-                    name, entry = requirement.pipfile_entry
-                    lockfile_update[name] = entry
-                continue
-            constraints.add(_requirement_to_str_lowercase_name(r))
-    return constraints, lockfile_update
+#     from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name
+#     from .vendor.requirementslib.models.requirements import Requirement
+#     constraints = set()
+#     lockfile_update = {}
+#     if req.is_file_or_url and not req.is_vcs:
+#         setup_info = req.run_requires()
+#         requirements = [v for v in setup_info.get("requires", {}).values()]
+#         for r in requirements:
+#             if getattr(r, "url", None) and not getattr(r, "editable", False):
+#                 requirement = Requirement.from_line(_requirement_to_str_lowercase_name(r))
+#                 constraint_update, child_lockfile = resolve_separate(requirement)
+#                 constraints |= constraint_update
+#                 lockfile_update.update(child_lockfile)
+#                 # for local packages with setup.py files and potential direct url deps:
+#                 if req.editable and requirement.is_direct_url:
+#                     name, entry = requirement.pipfile_entry
+#                     lockfile_update[name] = entry
+#                 continue
+#             constraints.add(_requirement_to_str_lowercase_name(r))
+#     return constraints, lockfile_update
 
 
-def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources):
-    from .vendor.requirementslib.models.requirements import Requirement
-    constraints = set()
-    skipped = {}
-    for dep in deps:
-        if not dep:
-            continue
-        url = None
-        indexes, trusted_hosts, remainder = parse_indexes(dep)
-        if indexes:
-            url = indexes[0]
-        dep = " ".join(remainder)
-        req = Requirement.from_line(dep)
-        if req.is_file_or_url and not req.is_vcs:
-            # TODO: This is a significant hack, should probably be reworked
-            constraint_update, lockfile_update = resolve_separate(req)
-            constraints |= constraint_update
-            name, entry = req.pipfile_entry
-            skipped[name] = entry
-            skipped.update(lockfile_update)
-            continue
-        constraints.add(req.constraint_line)
+# def get_resolver_metadata(
+#     deps,  # type: List[str]
+#     index_lookup,  # type: Dict[str, str]
+#     markers_lookup,  # type: Dict[str, str]
+#     project,  # type: '.project.Project'
+#     sources  # type: Dict[str, str]
+# ):
+#     # type: (...) -> Set()
+#     from .vendor.requirementslib.models.requirements import Requirement, Line
+#     constraints = set()
+#     skipped = {}
+#     for dep in deps:
+#         if not dep:
+#             continue
+#         url = None
+#         indexes, trusted_hosts, remainder = parse_indexes(dep)
+#         if indexes:
+#             url = indexes[0]
+#         dep = " ".join(remainder)
+#         line = Line(dep)
+#         if ((line.is_direct_url and line.is_vcs) or
+#                 (line.is_file or line.is_url and not (line.is_vcs and line.editable))):
+#             # TODO: This is a significant hack, should probably be reworked
+#             constraint_update, lockfile_update = resolve_separate(req)
+#             constraints |= constraint_update
+#             req = Requirement.from_line(dep)
+#             name, entry = req.pipfile_entry
+#             skipped[name] = entry
+#             skipped.update(lockfile_update)
+#             continue
+#         constraints.add(req.constraint_line)
 
-        if url:
-            source = first(
-                s for s in sources if s.get("url") and url.startswith(s["url"]))
-            if source:
-                index_lookup[req.name] = source.get("name")
-        # strip the marker and re-add it later after resolution
-        # but we will need a fallback in case resolution fails
-        # eg pypiwin32
-        if req.markers:
-            markers_lookup[req.name] = req.markers.replace('"', "'")
-    return constraints, skipped
+#         if url:
+#             source = first(
+#                 s for s in sources if s.get("url") and url.startswith(s["url"]))
+#             if source:
+#                 index_lookup[req.name] = source.get("name")
+#         # strip the marker and re-add it later after resolution
+#         # but we will need a fallback in case resolution fails
+#         # eg pypiwin32
+#         if req.markers:
+#             markers_lookup[req.name] = req.markers.replace('"', "'")
+#     return constraints, skipped
 
 
 class Resolver(object):
@@ -336,6 +350,85 @@ class Resolver(object):
         from pipenv.patched.piptools.scripts.compile import get_pip_command
         return get_pip_command()
 
+    @classmethod
+    def get_metadata(
+        cls,
+        deps,  # type: List[str]
+        index_lookup,  # type: Dict[str, str]
+        markers_lookup,  # type: Dict[str, str]
+        project,  # type: 'pipenv.project.Project'
+        sources  # type: Dict[str, str]
+    ):
+        # type: (...) -> Set()
+        constraints = set()
+        skipped = {}
+        for dep in deps:
+            if not dep:
+                continue
+            constraint_update, lockfile_update = cls.get_deps_from_line(
+                dep, index_lookup=index_lookup, markers_lookup=markers_lookup, sources=sources
+            )
+            constraints |= constraint_update
+            skipped.update(lockfile_update)
+        return constraints, skipped
+
+    @classmethod
+    def parse_line(cls, line, index_lookup=None, markers_lookup=None, sources=None):
+        from .vendor.requirementslib.models.requirements import Requirement
+        if sources is None:
+            sources = []  # type: List[Dict[str, Union[str, bool]]]
+        url = None
+        indexes, trusted_hosts, remainder = parse_indexes(line)
+        if indexes:
+            url = indexes[0]
+        line = " ".join(remainder)
+        req = Requirement.from_line(line)
+        if url:
+            source = first(
+                s for s in sources if s.get("url") and url.startswith(s["url"]))
+            if source and index_lookup is not None:
+                index_lookup[req.name] = source.get("name")
+        # strip the marker and re-add it later after resolution
+        # but we will need a fallback in case resolution fails
+        # eg pypiwin32
+        if req.markers and markers_lookup is not None:
+            markers_lookup[req.name] = req.markers.replace('"', "'")
+        return req
+
+    @classmethod
+    def get_deps_from_line(cls, line, index_lookup=None, markers_lookup=None, sources=None):
+        from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name
+        if sources is None:
+            sources = []  # type: List[Dict[str, Union[str, bool]]]
+        req = cls.parse_line(
+            line, index_lookup=index_lookup, markers_lookup=markers_lookup, sources=sources
+        )
+        parsed_line = req.line_instance
+        constraints = set()
+        locked_deps = {}
+        if ((parsed_line.is_direct_url and parsed_line.is_vcs) or
+                (parsed_line.is_file or parsed_line.is_url and not
+                (parsed_line.is_vcs and parsed_line.editable))
+        ):
+            # for local packages with setup.py files and potential direct url deps:
+            name, entry = req.pipfile_entry
+            # TODO: This might belong as a conditional include after we do the other logic in the for loop (line 427)
+            setup_info = parsed_line.setup_info
+            requirements = [v for v in setup_info.get_info().get("requires", {}).values()]
+            for r in requirements:
+                if getattr(r, "url", None) and not getattr(r, "editable", False):
+                    new_constraints, new_lock = cls.get_deps_from_line(
+                        _requirement_to_str_lowercase_name(r)
+                    )
+                    constraints |= new_constraints
+                    locked_deps.update(new_lock)
+                    continue
+                constraints.add(_requirement_to_str_lowercase_name(r))
+            locked_deps[name] = entry
+        else:
+            constraints.add(req.constraint_line)
+        return constraints, locked_deps
+
     @property
     def pip_command(self):
         if self._pip_command is None:
@@ -546,7 +639,7 @@ def actually_resolve_deps(
     warning_list = []
 
     with warnings.catch_warnings(record=True) as warning_list:
-        constraints, skipped = get_resolver_metadata(
+        constraints, skipped = Resolver.get_metadata(
             deps, index_lookup, markers_lookup, project, sources,
         )
         resolver = Resolver(constraints, req_dir, project, sources, clear=clear, pre=pre)
@@ -557,8 +650,11 @@ def actually_resolve_deps(
             is_url = url and not url.startswith("file:")
             path = v.get("path")
             if not is_url and not path:
-                path = pip_shims.shims.url_to_path(url)
-            if is_url or (path and os.path.exists(path) and not os.path.isdir(path)):
+                try:
+                    path = pip_shims.shims.url_to_path(url)
+                except AttributeError:
+                    path = None
+            if is_url or (path and os.path.exists(path) and not os.path.isdir(path)) or req.is_vcs:
                 existing = next(iter(req for req in resolved_tree if req.name == k), None)
                 if existing:
                     resolved_tree.remove(existing)
@@ -902,7 +998,7 @@ def resolve_deps(
                 entry.update(pf_entry)
                 if version is not None:
                     entry["version"] = version
-                if req.is_direct_url:
+                if req.line_instance.is_direct_url:
                     entry["file"] = req.req.uri
             if collected_hashes:
                 entry["hashes"] = sorted(set(collected_hashes))

pipenv/vendor/requirementslib/models/requirements.py

@@ -113,6 +113,7 @@ class Line(object):
         self._pyproject_backend = None  # type: Optional[str]
         self._wheel_kwargs = None  # type: Dict[str, str]
         self._vcsrepo = None  # type: Optional[VCSRepository]
+        self._setup_info = None  # type: Optional[SetupInfo]
         self._ref = None  # type: Optional[str]
         self._ireq = None  # type: Optional[InstallRequirement]
         self._src_root = None  # type: Optional[str]
@@ -384,6 +385,13 @@ class Line(object):
             return True
         return False
 
+    @property
+    def setup_info(self):
+        # type: () -> Optional[SetupInfo]
+        if self._setup_info is None:
+            self._setup_info = SetupInfo.from_ireq(self.ireq)
+        return self._setup_info
+
     def _get_vcsrepo(self):
         # type: () -> Optional[VCSRepository]
         from .vcs import VCSRepository
@@ -1717,6 +1725,7 @@ class Requirement(object):
     hashes = attr.ib(default=attr.Factory(list), converter=list)
     extras = attr.ib(default=attr.Factory(list))
     abstract_dep = attr.ib(default=None)
+    line_instance = attr.ib(default=None)  # type: Optional[Line]
     _ireq = None
 
     @name.default
@@ -1818,6 +1827,7 @@ class Requirement(object):
         if "--hash=" in line:
             hashes = line.split(" --hash=")
             line, hashes = hashes[0], hashes[1:]
+        line_instance = Line(line)
         editable = line.startswith("-e ")
         line = line.split(" ", 1)[1] if editable else line
         line, markers = split_markers_from_line(line)
@@ -1888,6 +1898,7 @@ class Requirement(object):
             "req": r,
             "markers": markers,
             "editable": editable,
+            "line_instance": line_instance
         }
         if extras:
             extras = sorted(dedup([extra.lower() for extra in extras]))
@@ -1956,6 +1967,7 @@ class Requirement(object):
         if any(key in _pipfile for key in ["hash", "hashes"]):
             args["hashes"] = _pipfile.get("hashes", [pipfile.get("hash")])
         cls_inst = cls(**args)
+        cls_inst.line_instance = Line(cls_inst.as_line())
         return cls_inst
 
     def as_line(

pipenv/vendor/requirementslib/models/setup_info.py

@@ -75,12 +75,14 @@ def _get_src_dir(root):
     if src:
         return src
     virtual_env = os.environ.get("VIRTUAL_ENV")
-    if virtual_env:
+    if virtual_env is not None:
         return os.path.join(virtual_env, "src")
     if not root:
         # Intentionally don't match pip's behavior here -- this is a temporary copy
-        root = create_tracked_tempdir(prefix="requirementslib-", suffix="-src")
-    return os.path.join(root, "src")
+        src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src")
+    else:
+        src_dir = os.path.join(root, "src")
+    return src_dir
 
 
 def ensure_reqs(reqs):

tests/integration/conftest.py

@@ -233,6 +233,9 @@ class _PipenvInstance(object):
     def __enter__(self):
         if self.chdir:
             os.chdir(self.path)
+        os.environ['PIPENV_PIPFILE'] = fs_str(self.pipfile_path)
+        c = delegator.run("pipenv run pip install /home/hawk/git/pip")
+        assert c.return_code == 0
         return self
 
     def __exit__(self, *args):