Update resolver to handle all resolution in venv

- Resolve all VCS and non-piptools-resolveable deps in venv
- Implement pep517 for resolution of non-setuptools builds
- Add full support for the new dependency link format
- Fix breakages from pip 19* rollout and subsequent setuptools breakage

Signed-off-by: Dan Ryan <dan.ryan@xyleminc.com>

pipenv/core.py

@@ -1,5 +1,5 @@
 # -*- coding=utf-8 -*-
-
+from __future__ import absolute_import, print_function
 import json as simplejson
 import logging
 import os
@@ -1288,7 +1288,7 @@ def pip_install(
         piplogger.setLevel(logging.INFO)
         if requirement:
             click.echo(
-                crayons.normal("Installing {0!r}".format(requirement.name), bold=True),
+                crayons.normal("Pip Installing {0!r}".format(requirement.name), bold=True),
                 err=True,
             )
     # Create files for hash mode.
@@ -1303,9 +1303,11 @@ def pip_install(
         f.write(vistir.misc.to_bytes(requirement.as_line()))
         r = f.name
         f.close()
-    # Install dependencies when a package is a VCS dependency.
+
     if requirement and requirement.vcs:
-        no_deps = False
+        # Install dependencies when a package is a non-editable VCS dependency.
+        if not requirement.editable:
+            no_deps = False
         # Don't specify a source directory when using --system.
         if not allow_global and ("PIP_SRC" not in os.environ):
             src.extend(["--src", "{0}".format(project.virtualenv_src_location)])
@@ -1361,9 +1363,9 @@ def pip_install(
         if "PIP_SRC" in os.environ:
             src_dir = os.environ["PIP_SRC"]
             src = ["--src", os.environ["PIP_SRC"]]
-        else:
-            src_dir = "{0}".format(project.virtualenv_src_location)
-            os.environ["PIP_SRC"] = project.virtualenv_src_location
+        # else:
+        #     src_dir = "{0}".format(project.virtualenv_src_location)
+        #     os.environ["PIP_SRC"] = project.virtualenv_src_location
         if not requirement.editable:
             no_deps = False
             # if not requirement.req.is_local:
@@ -1451,8 +1453,8 @@ def pip_install(
     pip_command.extend(prepare_pip_source_args(sources))
     if not ignore_hashes:
         pip_command.append("--require-hashes")
-    pip_command.append("--no-build-isolation")
     if not use_pep517:
+        pip_command.append("--no-build-isolation")
         pip_command.append("--no-use-pep517")
     if environments.is_verbose():
         click.echo("$ {0}".format(pip_command), err=True)
@@ -1932,7 +1934,7 @@ def do_install(
 
     # This is for if the user passed in dependencies, then we want to make sure we
     else:
-        from .vendor.requirementslib import Requirement
+        from .vendor.requirementslib.models.requirements import Requirement
 
         # make a tuple of (display_name, entry)
         pkg_list = packages + ["-e {0}".format(pkg) for pkg in editable_packages]

pipenv/project.py

@@ -681,7 +681,7 @@ class Project(object):
         )
 
         name = self.name if self.name is not None else "Pipfile"
-        config_parser = ConfigOptionParser(name=self.name)
+        config_parser = ConfigOptionParser(name=name)
         config_parser.add_option_group(make_option_group(index_group, config_parser))
         install = config_parser.option_groups[0]
         indexes = (

pipenv/resolver.py

@@ -7,12 +7,48 @@ import sys
 os.environ["PIP_PYTHON_PATH"] = str(sys.executable)
 
 
-def _patch_path():
+def find_site_path(pkg, site_dir=None):
+    import pkg_resources
+    if site_dir is not None:
+        site_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    working_set = pkg_resources.WorkingSet([site_dir] + sys.path[:])
+    for dist in working_set:
+        root = dist.location
+        base_name = dist.project_name if dist.project_name else dist.key
+        name = None
+        if "top_level.txt" in dist.metadata_listdir(""):
+            name = next(iter([l.strip() for l in dist.get_metadata_lines("top_level.txt") if l is not None]), None)
+        if name is None:
+            name = pkg_resources.safe_name(base_name).replace("-", "_")
+        if not any(pkg == _ for _ in [base_name, name]):
+            continue
+        path_options = [name, "{0}.py".format(name)]
+        path_options = [os.path.join(root, p) for p in path_options if p is not None]
+        path = next(iter(p for p in path_options if os.path.exists(p)), None)
+        if path is not None:
+            return (dist, path)
+    return (None, None)
+
+
+def _patch_path(pipenv_site=None):
     import site
     pipenv_libdir = os.path.dirname(os.path.abspath(__file__))
     pipenv_site_dir = os.path.dirname(pipenv_libdir)
-    site.addsitedir(pipenv_site_dir)
-    for _dir in ("vendor", "patched"):
+    pipenv_dist = None
+    if pipenv_site is not None:
+        pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site)
+    else:
+        pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site_dir)
+    if pipenv_dist is not None:
+        pipenv_dist.activate()
+    else:
+        site.addsitedir(next(iter(
+            sitedir for sitedir in (pipenv_site, pipenv_site_dir)
+            if sitedir is not None
+        ), None))
+    if pipenv_path is not None:
+        pipenv_libdir = pipenv_path
+    for _dir in ("vendor", "patched", pipenv_libdir):
         sys.path.insert(0, os.path.join(pipenv_libdir, _dir))
 
 
@@ -24,8 +60,11 @@ def get_parser():
     parser.add_argument("--verbose", "-v", action="count", default=False)
     parser.add_argument("--debug", action="store_true", default=False)
     parser.add_argument("--system", action="store_true", default=False)
+    parser.add_argument("--parse-only", action="store_true", default=False)
+    parser.add_argument("--pipenv-site", metavar="pipenv_site_dir", action="store",
+                        default=os.environ.get("PIPENV_SITE_DIR"))
     parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store",
-                            default=os.environ.get("PIPENV_REQ_DIR"))
+                        default=os.environ.get("PIPENV_REQ_DIR"))
     parser.add_argument("packages", nargs="*")
     return parser
 
@@ -41,23 +80,56 @@ def handle_parsed_args(parsed):
         logging.getLogger("notpip").setLevel(logging.DEBUG)
     elif parsed.verbose > 0:
         logging.getLogger("notpip").setLevel(logging.INFO)
+    os.environ["PIPENV_VERBOSITY"] = str(parsed.verbose)
     if "PIPENV_PACKAGES" in os.environ:
         parsed.packages += os.environ.get("PIPENV_PACKAGES", "").strip().split("\n")
     return parsed
 
 
-def _main(pre, clear, verbose, system, requirements_dir, packages):
-    os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]])
-    os.environ["PIP_PYTHON_PATH"] = str(sys.executable)
+def parse_packages(packages, pre, clear, system, requirements_dir=None):
+    from pipenv.vendor.requirementslib.models.requirements import Requirement
+    from pipenv.vendor.vistir.contextmanagers import cd, temp_path
+    from pipenv.utils import parse_indexes
+    parsed_packages = []
+    os.environ["PIP_NO_BUILD_ISOLATION"] = "1"
+    os.environ["PIP_NO_USE_PEP517"] = "1"
+    os.environ["PIP_NO_DEPS"] = "1"
+    for package in packages:
+        indexes, trusted_hosts, line = parse_indexes(package)
+        line = " ".join(line)
+        pf = dict()
+        req = Requirement.from_line(line)
+        if not req.name:
+            with temp_path(), cd(req.req.setup_info.base_dir):
+                sys.path.insert(0, req.req.setup_info.base_dir)
+                req.req._setup_info.get_info()
+                req.update_name_from_path(req.req.setup_info.base_dir)
+        print(os.listdir(req.req.setup_info.base_dir))
+        try:
+            name, entry = req.pipfile_entry
+        except Exception:
+            continue
+        else:
+            if name is not None and entry is not None:
+                pf[name] = entry
+                parsed_packages.append(pf)
+    print("RESULTS:")
+    if parsed_packages:
+        print(json.dumps(parsed_packages))
+    else:
+        print(json.dumps([]))
 
+
+def resolve_packages(pre, clear, verbose, system, requirements_dir, packages):
     from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources
-
     pypi_mirror_source = (
         create_mirror_source(os.environ["PIPENV_PYPI_MIRROR"])
         if "PIPENV_PYPI_MIRROR" in os.environ
         else None
     )
-
+    os.environ["PIP_NO_BUILD_ISOLATION"] = "1"
+    os.environ["PIP_NO_USE_PEP517"] = "1"
+    os.environ["PIP_NO_DEPS"] = "1"
     def resolve(packages, pre, project, sources, clear, system, requirements_dir=None):
         return resolve_deps(
             packages,
@@ -76,15 +148,8 @@ def _main(pre, clear, verbose, system, requirements_dir, packages):
         if pypi_mirror_source
         else project.pipfile_sources
     )
-    results = resolve(
-        packages,
-        pre=pre,
-        project=project,
-        sources=sources,
-        clear=clear,
-        system=system,
-        requirements_dir=requirements_dir,
-    )
+    results = resolve(packages, pre=pre, project=project, sources=sources, clear=clear,
+                      system=system, requirements_dir=requirements_dir)
     print("RESULTS:")
     if results:
         print(json.dumps(results))
@@ -92,36 +157,46 @@ def _main(pre, clear, verbose, system, requirements_dir, packages):
         print(json.dumps([]))
 
 
-def main():
-    _patch_path()
-    import warnings
-    from pipenv.vendor.vistir.compat import ResourceWarning
-    warnings.simplefilter("ignore", category=ResourceWarning)
-    import io
-    import six
-    if six.PY3:
-        import atexit
-        stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')
-        atexit.register(stdout_wrapper.close)
-        stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8')
-        atexit.register(stderr_wrapper.close)
-        sys.stdout = stdout_wrapper
-        sys.stderr = stderr_wrapper
+def _main(pre, clear, verbose, system, requirements_dir, packages, parse_only=False):
+    os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]])
+    os.environ["PIP_PYTHON_PATH"] = str(sys.executable)
+    if parse_only:
+        parse_packages(
+            packages,
+            pre=pre,
+            clear=clear,
+            system=system,
+            requirements_dir=requirements_dir,
+        )
     else:
-        from pipenv._compat import force_encoding
-        force_encoding()
-    os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1")
-    os.environ["PYTHONIOENCODING"] = str("utf-8")
+        resolve_packages(pre, clear, verbose, system, requirements_dir, packages)
+
+
+def main():
     parser = get_parser()
     parsed, remaining = parser.parse_known_args()
-    # sys.argv = remaining
+    _patch_path(pipenv_site=parsed.pipenv_site)
+    import warnings
+    from pipenv.vendor import colorama
+    colorama.init()
+    from pipenv.vendor.vistir.compat import ResourceWarning
+    from pipenv.vendor.vistir.misc import get_wrapped_stream
+    warnings.simplefilter("ignore", category=ResourceWarning)
+    import six
+    if six.PY3:
+        stdout = sys.stdout.buffer
+        stderr = sys.stderr.buffer
+    else:
+        stdout = sys.stdout
+        stderr = sys.stderr
+    sys.stderr = get_wrapped_stream(stderr)
+    sys.stdout = get_wrapped_stream(stdout)
+    os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1")
+    os.environ["PYTHONIOENCODING"] = str("utf-8")
     parsed = handle_parsed_args(parsed)
     _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system,
-          parsed.requirements_dir, parsed.packages)
+          parsed.requirements_dir, parsed.packages, parse_only=parsed.parse_only)
 
 
 if __name__ == "__main__":
-    _patch_path()
-    from pipenv.vendor import colorama
-    colorama.init()
     main()

pipenv/utils.py

@@ -23,8 +23,7 @@ six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc"))
 six.add_move(six.MovedAttribute("Set", "collections", "collections.abc"))  # noqa
 from six.moves import Mapping, Sequence, Set
 from six.moves.urllib.parse import urlparse
-from urllib3 import util as urllib3_util
-from vistir.compat import ResourceWarning
+from vistir.compat import ResourceWarning, lru_cache
 from vistir.misc import fs_str
 
 import crayons
@@ -33,10 +32,13 @@ import parse
 from . import environments
 from .exceptions import PipenvUsageError
 from .pep508checker import lookup
+from .vendor.urllib3 import util as urllib3_util
 
 
 if environments.MYPY_RUNNING:
-    from typing import Tuple, Dict, Any, List, Union
+    from typing import Tuple, Dict, Any, List, Union, Callable, Optional
+    from .vendor.requirementslib.models.requirements import Requirement, Line
+    from .project import Project
 
 
 logging.basicConfig(level=logging.ERROR)
@@ -306,6 +308,17 @@ def prepare_pip_source_args(sources, pip_args=None):
 #             markers_lookup[req.name] = req.markers.replace('"', "'")
 #     return constraints, skipped
 
+@lru_cache()
+def get_pipenv_sitedir():
+    # type: () -> Optional[str]
+    import pkg_resources
+    site_dir = next(
+        iter(d for d in pkg_resources.working_set if d.key.lower() == "pipenv"), None
+    )
+    if site_dir is not None:
+        return site_dir.location
+    return None
+
 
 class Resolver(object):
     def __init__(self, constraints, req_dir, project, sources, clear=False, pre=False):
@@ -339,7 +352,9 @@ class Resolver(object):
             "sources={self.sources})>".format(self=self)
         )
 
-    def _get_pip_command(self):
+    @staticmethod
+    @lru_cache()
+    def _get_pip_command():
         from pip_shims.shims import Command
 
         class PipCommand(Command):
@@ -356,18 +371,19 @@ class Resolver(object):
         deps,  # type: List[str]
         index_lookup,  # type: Dict[str, str]
         markers_lookup,  # type: Dict[str, str]
-        project,  # type: 'pipenv.project.Project'
+        project,  # type: Project
         sources  # type: Dict[str, str]
     ):
-        # type: (...) -> Set()
-        constraints = set()
-        skipped = {}
+        # type: (...) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]]
+        constraints = set()  # type: Set[str]
+        skipped = dict()  # type: Dict[str, Dict[str, Union[str, bool, List[str]]]]
         for dep in deps:
             if not dep:
                 continue
-            constraint_update, lockfile_update = cls.get_deps_from_line(
+            req = cls.parse_line(
                 dep, index_lookup=index_lookup, markers_lookup=markers_lookup, sources=sources
             )
+            constraint_update, lockfile_update = cls.get_deps_from_req(req)
             constraints |= constraint_update
             skipped.update(lockfile_update)
         return constraints, skipped
@@ -376,7 +392,8 @@ class Resolver(object):
     def parse_line(cls, line, index_lookup=None, markers_lookup=None, sources=None):
         from .vendor.requirementslib.models.requirements import Requirement
         if sources is None:
-            sources = []  # type: List[Dict[str, Union[str, bool]]]
+            from .core import project
+            sources = project.sources  # type: List[Dict[str, Union[str, bool]]]
         url = None
         indexes, trusted_hosts, remainder = parse_indexes(line)
         if indexes:
@@ -396,37 +413,63 @@ class Resolver(object):
         return req
 
     @classmethod
-    def get_deps_from_line(cls, line, index_lookup=None, markers_lookup=None, sources=None):
-        from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name
-        if sources is None:
-            sources = []  # type: List[Dict[str, Union[str, bool]]]
-        req = cls.parse_line(
-            line, index_lookup=index_lookup, markers_lookup=markers_lookup, sources=sources
-        )
-        parsed_line = req.line_instance
-        constraints = set()
-        locked_deps = {}
-        if ((parsed_line.is_direct_url and parsed_line.is_vcs) or
-                (parsed_line.is_file or parsed_line.is_url and not
-                (parsed_line.is_vcs and parsed_line.editable))
+    def get_deps_from_line(cls, line):
+        # type: (str) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]]
+        return cls.get_deps_from_req(cls.parse_line(line))
+
+    @classmethod
+    def get_deps_from_req(cls, req):
+        # type: (Requirement) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]]
+        parsed_line = req.req.parsed_line  # type: Line
+        constraints = set()  # type: Set[str]
+        locked_deps = dict()  # type: Dict[str, Dict[str, Union[str, bool, List[str]]]]
+        setup_info = None  # type: Any
+        if parsed_line.is_file or parsed_line.is_vcs or parsed_line.is_url and not (
+            parsed_line.is_wheel
         ):
             # for local packages with setup.py files and potential direct url deps:
             name, entry = req.pipfile_entry
-            # TODO: This might belong as a conditional include after we do the other logic in the for loop (line 427)
-            setup_info = parsed_line.setup_info
-            requirements = [v for v in setup_info.get_info().get("requires", {}).values()]
+            setup_info = req.req.setup_info
+            locked_deps[name] = entry
+            requirements = [v for v in getattr(setup_info, "requires", {}).values()]
             for r in requirements:
                 if getattr(r, "url", None) and not getattr(r, "editable", False):
-                    new_constraints, new_lock = cls.get_deps_from_line(
-                        _requirement_to_str_lowercase_name(r)
-                    )
-                    constraints |= new_constraints
+                    line = str(r)
+                    if line == "None":
+                        if not r.url:
+                            continue
+                        line = r.url
+                    new_req = cls.parse_line(line)
+                    new_constraints, new_lock = cls.get_deps_from_req(new_req)
                     locked_deps.update(new_lock)
-                    continue
-                constraints.add(_requirement_to_str_lowercase_name(r))
-            locked_deps[name] = entry
+                    constraints |= new_constraints
+                else:
+                    line = str(r)
+                    if line is not None:
+                        constraints.add(line)
+            # ensure the top level entry remains as provided
+            # note that we shouldn't pin versions for editable vcs deps
+            if (not req.is_vcs or (req.is_vcs and not req.editable)):
+                if req.specifiers:
+                    locked_deps[name]["version"] = req.specifiers
+                elif parsed_line.setup_info and parsed_line.setup_info.version:
+                    locked_deps[name]["version"] = "=={}".format(
+                        parsed_line.setup_info.version
+                    )
+            if not req.is_vcs:
+                locked_deps.update({name: entry})
+            else:
+                # Lock the current requirement if it's a VCS requiement (get the hash)
+                _, vcs_lockfile = get_vcs_deps(reqs=[req])
+                locked_deps[name].update(vcs_lockfile[name])
+                if req.editable:
+                    constraints.add(req.constraint_line)
+            if req.is_file_or_url and req.req.is_local and req.editable and (
+                    req.req.setup_path is not None and os.path.exists(req.req.setup_path)):
+                constraints.add(req.constraint_line)
         else:
             constraints.add(req.constraint_line)
+            return constraints, locked_deps
         return constraints, locked_deps
 
     @property
@@ -550,6 +593,47 @@ class Resolver(object):
             self.resolved_tree.update(results)
             return self.resolved_tree
 
+    def collect_hashes(self, ireq):
+        collected_hashes = []
+        if ireq in self.hashes:
+            collected_hashes += list(self.hashes.get(ireq, []))
+        if self._should_include_hash(ireq):
+            try:
+                hash_map = self.get_hash(ireq)
+                collected_hashes += list(hash_map)
+            except (ValueError, KeyError, IndexError, ConnectionError):
+                pass
+        elif any(
+            "python.org" in source["url"] or "pypi.org" in source["url"]
+            for source in self.sources
+        ):
+            pkg_url = "https://pypi.org/pypi/{0}/json".format(ireq.name)
+            session = _get_requests_session()
+            try:
+                # Grab the hashes from the new warehouse API.
+                r = session.get(pkg_url, timeout=10)
+                api_releases = r.json()["releases"]
+                cleaned_releases = {}
+                for api_version, api_info in api_releases.items():
+                    api_version = clean_pkg_version(api_version)
+                    cleaned_releases[api_version] = api_info
+                version = ""
+                if ireq.specifier:
+                    spec = next(iter(s for s in list(ireq.specifier._specs)), None)
+                    if spec:
+                        version = spec.version
+                for release in cleaned_releases[version]:
+                    collected_hashes.append(release["digests"]["sha256"])
+                collected_hashes = ["sha256:" + s for s in collected_hashes]
+            except (ValueError, KeyError, ConnectionError):
+                if environments.is_verbose():
+                    click_echo(
+                        "{0}: Error generating hash for {1}".format(
+                            crayons.red("Warning", bold=True), ireq.name
+                        ), err=True
+                    )
+        return collected_hashes
+
     @staticmethod
     def _should_include_hash(ireq):
         from pipenv.vendor.vistir.compat import Path, to_native_string
@@ -632,7 +716,6 @@ def actually_resolve_deps(
 ):
     from pipenv.vendor.vistir.path import create_tracked_tempdir
     from pipenv.vendor.requirementslib.models.requirements import Requirement
-    from pipenv.vendor import pip_shims
 
     if not req_dir:
         req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-")
@@ -644,27 +727,68 @@ def actually_resolve_deps(
         )
         resolver = Resolver(constraints, req_dir, project, sources, clear=clear, pre=pre)
         resolved_tree = resolver.resolve()
-        for k, v in skipped.items():
-            url = v.get("file")
-            req = Requirement.from_pipfile(k, v)
-            is_url = url and not url.startswith("file:")
-            path = v.get("path")
-            if not is_url and not path:
-                try:
-                    path = pip_shims.shims.url_to_path(url)
-                except AttributeError:
-                    path = None
-            if is_url or (path and os.path.exists(path) and not os.path.isdir(path)) or req.is_vcs:
-                existing = next(iter(req for req in resolved_tree if req.name == k), None)
-                if existing:
-                    resolved_tree.remove(existing)
-                resolved_tree.add(req.as_ireq())
         hashes = resolver.resolve_hashes()
-
+        reqs = [(Requirement.from_ireq(ireq), ireq) for ireq in resolved_tree]
+        results = {}
+        for req, ireq in reqs:
+            if (req.vcs and req.editable and not req.is_direct_url):
+                continue
+            collected_hashes = resolver.collect_hashes(ireq)
+            if collected_hashes:
+                req = req.add_hashes(collected_hashes)
+            elif resolver._should_include_hash(ireq):
+                existing_hashes = hashes.get(ireq, set())
+                discovered_hashes = existing_hashes | resolver.get_hash(ireq)
+                if discovered_hashes:
+                    req = req.add_hashes(discovered_hashes)
+                resolver.hashes[ireq] = discovered_hashes
+            if req.specifiers:
+                version = str(req.get_version())
+            else:
+                version = None
+            index = index_lookup.get(req.name)
+            req.index = index
+            name, pf_entry = req.pipfile_entry
+            name = pep423_name(req.name)
+            entry = {}
+            if isinstance(pf_entry, six.string_types):
+                entry["version"] = pf_entry.lstrip("=")
+            else:
+                entry.update(pf_entry)
+                if version is not None:
+                    entry["version"] = version
+                if req.line_instance.is_direct_url:
+                    entry["file"] = req.req.uri
+            if collected_hashes:
+                entry["hashes"] = sorted(set(collected_hashes))
+            entry["name"] = name
+            if index and index != next(iter(project.sources), {}).get("name"):
+                entry.update({"index": index})
+            if markers_lookup.get(req.name):
+                entry.update({"markers": markers_lookup.get(req.name)})
+            entry = translate_markers(entry)
+            if name in results:
+                results[name].update(entry)
+            else:
+                results[name] = entry
+        for k in list(skipped.keys()):
+            req = Requirement.from_pipfile(k, skipped[k])
+            ireq = req.as_ireq()
+            entry = skipped[k].copy()
+            entry["name"] = pep423_name(k)
+            if resolver._should_include_hash(ireq):
+                collected_hashes = resolver.collect_hashes(ireq)
+                if collected_hashes:
+                    entry["hashes"] = sorted(set(collected_hashes))
+            if k in results:
+                results[k].update(entry)
+            else:
+                results[k] = entry
+        results = list(results.values())
     for warning in warning_list:
         _show_warning(warning.message, warning.category, warning.filename, warning.lineno,
                       warning.line)
-    return (resolved_tree, hashes, markers_lookup, resolver)
+    return (results, hashes, markers_lookup, resolver, skipped)
 
 
 @contextlib.contextmanager
@@ -721,7 +845,7 @@ def resolve(cmd, sp):
     return c
 
 
-def get_locked_dep(dep, pipfile_section, prefer_pipfile=False):
+def get_locked_dep(dep, pipfile_section, prefer_pipfile=True):
     # the prefer pipfile flag is not used yet, but we are introducing
     # it now for development purposes
     # TODO: Is this implementation clear? How can it be improved?
@@ -754,16 +878,18 @@ def get_locked_dep(dep, pipfile_section, prefer_pipfile=False):
 def prepare_lockfile(results, pipfile, lockfile):
     from .vendor.requirementslib.utils import is_vcs
     for dep in results:
+        if not dep:
+            continue
         # Merge in any relevant information from the pipfile entry, including
         # markers, normalized names, URL info, etc that we may have dropped during lock
-        if not is_vcs(dep):
-            lockfile_entry = get_locked_dep(dep, pipfile)
-            name = next(iter(k for k in lockfile_entry.keys()))
-            current_entry = lockfile.get(name)
-            if current_entry and not is_vcs(current_entry):
-                lockfile[name].update(lockfile_entry[name])
-            else:
-                lockfile[name] = lockfile_entry[name]
+        # if not is_vcs(dep):
+        lockfile_entry = get_locked_dep(dep, pipfile)
+        name = next(iter(k for k in lockfile_entry.keys()))
+        current_entry = lockfile.get(name)
+        if current_entry:
+            lockfile[name].update(lockfile_entry[name])
+        else:
+            lockfile[name] = lockfile_entry[name]
     return lockfile
 
 
@@ -779,6 +905,30 @@ def venv_resolve_deps(
     pipfile=None,
     lockfile=None
 ):
+    """
+    Resolve dependencies for a pipenv project, acts as a portal to the target environment.
+
+    Regardless of whether a virtual environment is present or not, this will spawn
+    a subproces which is isolated to the target environment and which will perform
+    dependency resolution.  This function reads the output of that call and mutates
+    the provided lockfile accordingly, returning nothing.
+
+    :param List[:class:`~requirementslib.Requirement`] deps: A list of dependencies to resolve.
+    :param Callable which: [description]
+    :param project: The pipenv Project instance to use during resolution
+    :param Optional[bool] pre: Whether to resolve pre-release candidates, defaults to False
+    :param Optional[bool] clear: Whether to clear the cache during resolution, defaults to False
+    :param Optional[bool] allow_global: Whether to use *sys.executable* as the python binary, defaults to False
+    :param Optional[str] pypi_mirror: A URL to substitute any time *pypi.org* is encountered, defaults to None
+    :param Optional[bool] dev: Whether to target *dev-packages* or not, defaults to False
+    :param pipfile: A Pipfile section to operate on, defaults to None
+    :type pipfile: Optional[Dict[str, Union[str, Dict[str, bool, List[str]]]]]
+    :param Dict[str, Any] lockfile: A project lockfile to mutate, defaults to None
+    :raises RuntimeError: Raised on resolution failure
+    :return: Nothing
+    :rtype: None
+    """
+
     from .vendor.vistir.misc import fs_str
     from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError
     from .vendor.vistir.path import create_tracked_tempdir
@@ -787,11 +937,14 @@ def venv_resolve_deps(
 
     vcs_deps = []
     vcs_lockfile = {}
+    # url_lockfile = {}
     results = []
     pipfile_section = "dev_packages" if dev else "packages"
     lockfile_section = "develop" if dev else "default"
     vcs_section = "vcs_{0}".format(pipfile_section)
     if project.pipfile_exists:
+        # This is a requirementslib pipfile instance which provides `Requirement` instances
+        # rather than simply locked dependencies in a lockfile format
         deps = project._pipfile.dev_requirements if dev else project._pipfile.requirements
         vcs_deps = [r for r in deps if r.is_vcs]
     else:
@@ -804,23 +957,30 @@ def venv_resolve_deps(
     if not lockfile:
         lockfile = project._lockfile
     req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements")
-    for dep in deps:
-        if dep.is_file_or_url and not dep.is_vcs:
-            name, entry = dep.pipfile_entry
-            lockfile[lockfile_section][name] = entry
+    constraints = set()
+    # for dep in deps:
+    #     if dep.is_file_or_url and not dep.is_vcs:
+    #         with temp_environ():
+    #             os.environ["PIP_NO_USE_PEP_517"] = fs_str("1")
+    #             os.environ["PIPENV_SITE_DIR"] = get_pipenv_sitedir()
+    #             name, entry = dep.pipfile_entry
+    #             constraint_update, lockfile_update = Resolver.get_deps_from_req(dep)
+    #             url_lockfile[name] = entry
+    #             if name in lockfile_update:
+    #                 url_lockfile[name].update(lockfile_update[name])
+    #             lockfile[lockfile_section].update(lockfile_update)
+    #             url_lockfile.update(lockfile_update)
+    #             constraints |= constraint_update
     if vcs_deps:
-        with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp:
-            vcs_reqs, vcs_lockfile = get_vcs_deps(
-                project,
-                which=which,
-                clear=clear,
-                pre=pre,
-                allow_global=allow_global,
-                dev=dev,
-            )
+        with temp_environ(), create_spinner(text=fs_str("Pinning VCS Packages...")) as sp:
+            os.environ["PIPENV_SITE_DIR"] = get_pipenv_sitedir()
+            vcs_reqs, vcs_lockfile = get_vcs_deps(project=project, dev=dev)
             vcs_deps = [req.as_line() for req in vcs_reqs if req.editable]
             lockfile[lockfile_section].update(vcs_lockfile)
-    deps = [r.as_line() for r in deps if not r.is_vcs]
+    # new_constraints = {r.as_line() for r in deps if not (r.is_vcs or (r.is_file_or_url
+    #                    and r.line_instance and not r.line_instance.is_wheel))}
+    # constraints |= new_constraints
+    constraints = {r.as_line() for r in deps} # if not r.is_vcs}
     cmd = [
         which("python", allow_global=allow_global),
         Path(resolver.__file__.rstrip("co")).as_posix()
@@ -833,50 +993,55 @@ def venv_resolve_deps(
         cmd.append("--system")
     with temp_environ():
         os.environ = {fs_str(k): fs_str(val) for k, val in os.environ.items()}
-        os.environ["PIPENV_PACKAGES"] = str("\n".join(deps))
+        os.environ["PIPENV_PACKAGES"] = str("\n".join(constraints))
         if pypi_mirror:
             os.environ["PIPENV_PYPI_MIRROR"] = str(pypi_mirror)
         os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY)
         os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir)
         os.environ["PIP_NO_INPUT"] = fs_str("1")
+        os.environ["PIPENV_SITE_DIR"] = get_pipenv_sitedir()
+        os.environ["PIP_NO_USE_PEP517"] = fs_str("1")
+        os.environ["PIP_NO_BUILD_ISOLATION"] = fs_str("1")
         with create_spinner(text=fs_str("Locking...")) as sp:
             c = resolve(cmd, sp)
-            results = c.out
-            if vcs_deps:
-                with temp_environ():
-                    os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps))
-                    sp.text = to_native_string("Locking VCS Dependencies...")
-                    vcs_c = resolve(cmd, sp)
-                    vcs_results, vcs_err = vcs_c.out, vcs_c.err
-            else:
-                vcs_results, vcs_err = "", ""
-            sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!"))
-    outputs = [results, vcs_results]
+            results = c.out.strip()
+            # if vcs_deps:
+            #     with temp_environ():
+            #         os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps))
+            #         sp.text = to_native_string("Locking VCS Dependencies...")
+            #         vcs_c = resolve(cmd, sp)
+            #         vcs_results, vcs_err = vcs_c.out.strip(), vcs_c.err.strip()
+            # else:
+            # vcs_results, vcs_err = "", ""
+            # sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!"))
     if environments.is_verbose():
-        for output in outputs:
-            click_echo(output.split("RESULTS:")[0], err=True)
+        click_echo(results.split("RESULTS:")[1], err=True)
+        # for output in outputs:
+        #     click_echo(output.split("RESULTS:")[0], err=True)
     try:
         results = json.loads(results.split("RESULTS:")[1].strip())
-        if vcs_results:
+        # if vcs_results:
             # For vcs dependencies, treat the initial pass at locking (i.e. checkout)
             # as the pipfile entry because it gets us an actual ref to use
-            vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip())
-            vcs_lockfile = prepare_lockfile(vcs_results, vcs_lockfile.copy(), vcs_lockfile)
-        else:
-            vcs_results = []
+            # vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip())
+            # vcs_lockfile = prepare_lockfile(vcs_results, vcs_lockfile.copy(), vcs_lockfile)
+        # else:
+            # vcs_results = []
 
     except (IndexError, JSONDecodeError):
-        for out, err in [(c.out, c.err), (vcs_results, vcs_err)]:
-            click_echo(out.strip(), err=True)
-            click_echo(err.strip(), err=True)
+        click_echo(c.out.strip(), err=True)
+        click_echo(c.err.strip(), err=True)
+        # for out, err in [(c.out, c.err), (vcs_results, vcs_err)]:
+            # click_echo(out.strip(), err=True)
+            # click_echo(err.strip(), err=True)
         raise RuntimeError("There was a problem with locking.")
     lockfile[lockfile_section] = prepare_lockfile(results, pipfile, lockfile[lockfile_section])
-    for k, v in vcs_lockfile.items():
-        if k in getattr(project, vcs_section, {}):
-            if not (isinstance(v, six.string_types) and isinstance(k, Mapping)):
-                lockfile[lockfile_section][k].update(v)
-        else:
-            lockfile[lockfile_section][k] = v
+    # for k, v in vcs_lockfile.items():
+    #     if k in getattr(project, vcs_section, {}):
+    #         if not (isinstance(v, six.string_types) and isinstance(k, Mapping)):
+    #             lockfile[lockfile_section][k].update(v)
+    #     else:
+    #         lockfile[lockfile_section][k] = v
 
 
 def resolve_deps(
@@ -902,6 +1067,8 @@ def resolve_deps(
     if not os.environ.get("PIP_SRC"):
         os.environ["PIP_SRC"] = project.virtualenv_src_location
     backup_python_path = sys.executable
+    os.environ["PIP_NO_BUILD_ISOLATION"] = "1"
+    os.environ["PIP_NO_USE_PEP517"] = "1"
     results = []
     if not deps:
         return results
@@ -912,7 +1079,7 @@ def resolve_deps(
         req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-requirements")
     with HackedPythonVersion(python_version=python, python_path=python_path):
         try:
-            resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps(
+            resolved_tree, hashes, markers_lookup, resolver, skipped = actually_resolve_deps(
                 deps,
                 index_lookup,
                 markers_lookup,
@@ -934,7 +1101,7 @@ def resolve_deps(
             try:
                 # Attempt to resolve again, with different Python version information,
                 # particularly for particularly particular packages.
-                resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps(
+                resolved_tree, hashes, markers_lookup, resolver, skipped = actually_resolve_deps(
                     deps,
                     index_lookup,
                     markers_lookup,
@@ -946,70 +1113,30 @@ def resolve_deps(
                 )
             except RuntimeError:
                 sys.exit(1)
-    for result in resolved_tree:
-        if not result.editable:
-            req = Requirement.from_ireq(result)
-            name = pep423_name(req.name)
-            name, pf_entry = req.pipfile_entry
-            if req.specifiers:
-                version = str(req.get_version())
-            else:
-                version = None
-            index = index_lookup.get(result.name)
-            req.index = index
-            collected_hashes = []
-            if result in hashes:
-                collected_hashes = list(hashes.get(result))
-            elif resolver._should_include_hash(result):
-                try:
-                    hash_map = resolver.get_hash(result)
-                    collected_hashes = list(hash_map)
-                except (ValueError, KeyError, IndexError, ConnectionError):
-                    pass
-            elif any(
-                "python.org" in source["url"] or "pypi.org" in source["url"]
-                for source in sources
-            ):
-                pkg_url = "https://pypi.org/pypi/{0}/json".format(name)
-                session = _get_requests_session()
-                try:
-                    # Grab the hashes from the new warehouse API.
-                    r = session.get(pkg_url, timeout=10)
-                    api_releases = r.json()["releases"]
-                    cleaned_releases = {}
-                    for api_version, api_info in api_releases.items():
-                        api_version = clean_pkg_version(api_version)
-                        cleaned_releases[api_version] = api_info
-                    for release in cleaned_releases[version]:
-                        collected_hashes.append(release["digests"]["sha256"])
-                    collected_hashes = ["sha256:" + s for s in collected_hashes]
-                except (ValueError, KeyError, ConnectionError):
-                    if environments.is_verbose():
-                        click_echo(
-                            "{0}: Error generating hash for {1}".format(
-                                crayons.red("Warning", bold=True), name
-                            ), err=True
-                        )
-            req.hashes = sorted(set(collected_hashes))
-            entry = {}
-            if isinstance(pf_entry, six.string_types):
-                entry["version"] = pf_entry.lstrip("=")
-            else:
-                entry.update(pf_entry)
-                if version is not None:
-                    entry["version"] = version
-                if req.line_instance.is_direct_url:
-                    entry["file"] = req.req.uri
-            if collected_hashes:
-                entry["hashes"] = sorted(set(collected_hashes))
-            entry["name"] = name
-            # if index:
-            #     d.update({"index": index})
-            if markers_lookup.get(result.name):
-                entry.update({"markers": markers_lookup.get(result.name)})
-            entry = translate_markers(entry)
-            results.append(entry)
-    return results
+    # for req in resolved_tree:
+    #     if not req.editable:
+    #         result = req.as_ireq()
+
+    #         req.hashes = sorted(set(collected_hashes))
+    #         entry = {}
+    #         if isinstance(pf_entry, six.string_types):
+    #             entry["version"] = pf_entry.lstrip("=")
+    #         else:
+    #             entry.update(pf_entry)
+    #             if version is not None:
+    #                 entry["version"] = version
+    #             if req.line_instance.is_direct_url:
+    #                 entry["file"] = req.req.uri
+    #         if collected_hashes:
+    #             entry["hashes"] = sorted(set(collected_hashes))
+    #         entry["name"] = name
+    #         # if index:
+    #         #     d.update({"index": index})
+    #         if markers_lookup.get(result.name):
+    #             entry.update({"markers": markers_lookup.get(result.name)})
+    #         entry = translate_markers(entry)
+    #         results.append(entry)
+    return resolved_tree
 
 
 def is_star(val):
@@ -1488,37 +1615,94 @@ def safe_expandvars(value):
 
 
 def get_vcs_deps(
-    project,
-    which=None,
-    clear=False,
-    pre=False,
-    allow_global=False,
+    project=None,
     dev=False,
     pypi_mirror=None,
+    packages=None,
+    reqs=None
 ):
     from .vendor.requirementslib.models.requirements import Requirement
+    from .vendor import attr
 
     section = "vcs_dev_packages" if dev else "vcs_packages"
-    reqs = []
+    if reqs is None:
+        reqs = []
     lockfile = {}
-    try:
-        packages = getattr(project, section)
-    except AttributeError:
-        return [], []
-    for pkg_name, pkg_pipfile in packages.items():
-        requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile)
+    if not reqs:
+        if not project and not packages:
+            raise ValueError(
+                "Must supply either a project or a pipfile section to lock vcs dependencies."
+            )
+        if not packages:
+            try:
+                packages = getattr(project, section)
+            except AttributeError:
+                return [], []
+        reqs = [Requirement.from_pipfile(name, entry) for name, entry in packages.items()]
+    result = []
+    updated_reqs = []
+    for requirement in reqs:
         name = requirement.normalized_name
         commit_hash = None
         if requirement.is_vcs:
             try:
-                with locked_repository(requirement) as repo:
+                with temp_path(), locked_repository(requirement) as repo:
+                    from pipenv.vendor.requirementslib.models.requirements import Requirement
+                    from distutils.sysconfig import get_python_lib
+                    sys.path = [repo.checkout_directory, "", ".", get_python_lib(plat_specific=0)]
                     commit_hash = repo.get_commit_hash()
+                    name = requirement.normalized_name
+                    version = requirement._specifiers = "=={0}".format(requirement.req.setup_info.version)
                     lockfile[name] = requirement.pipfile_entry[1]
                     lockfile[name]['ref'] = commit_hash
-                reqs.append(requirement)
+                    # new_req = Requirement.from_line(repo.checkout_directory)
+                    # si = new_req.req.setup_info
+                    # hookcaller = pep517.wrappers.Pep517HookCaller(new_req.req.setup_info.base_dir, new_req.req.setup_info.build_backend)
+                    # deps = hookcaller.get_requires_for_build_wheel() + hookcaller.get_requires_for_build_sdist()
+                    # hookcaller.prepare_metadata_for_build_wheel(new_req.req.setup_info.egg_base)
+                    # r._specifiers = new_req.specifiers
+                    # pkging_req = r.req.req
+                    # pkging_req.specifier = new_req.req.req.specifier
+                    # pkging_req.spec = new_req.req.req.spec
+                    # new_parsed_line = r.req._parsed_line
+                    # new_parsed_line._setup_info = new_req.req.parsed_line.setup_info
+                    # new_parsed_line.specifiers = new_req.req.parsed_line.specifiers
+                    # new_parsed_line._requirement = pkging_req
+                    # new_parsed_line.ireq.req = pkging_req
+                    # req = attr.evolve(
+                    #     r.req,
+                    #     setup_info=new_req.req.setup_info,
+                    #     parsed_line=new_parsed_line
+                    # )
+                    # r = attr.evolve(
+                    #     r,
+                    #     req=req,
+                    #     line_instance=new_parsed_line
+                    # )
+                    result.append(requirement)
+                    version = requirement.specifiers
+                    if not version and requirement.specifiers:
+                        version = requirement.specifiers
+                    if version:
+                        lockfile[name]['version'] = version
+                    # new_req = Requirement.from_line(repo.checkout_directory)
+                    # requirement._specifiers = new_req.specifiers
+                    # new_parsed_line = requirement.req._parsed_line
+                    # new_parsed_line._setup_info = new_req.req.parsed_line.setup_info
+                    # new_parsed_line.specifiers = new_req.req.parsed_line.specifiers
+                    # req = attr.evolve(
+                    #     requirement.req,
+                    #     _setup_info=new_req.req.setup_info,
+                    #     _parsed_line=new_parsed_line
+                    # )
+                    # requirement = attr.evolve(
+                    #     requirement,
+                    #     req=req,
+                    #     line_instance=new_parsed_line
+                    # )
             except OSError:
                 continue
-    return reqs, lockfile
+    return result, lockfile
 
 
 def translate_markers(pipfile_entry):
@@ -1561,23 +1745,38 @@ def translate_markers(pipfile_entry):
 
 
 def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None):
+    from .vendor.requirementslib.utils import is_vcs
     name = pep423_name(dep["name"])
+    lockfile = {}
     # We use this to determine if there are any markers on top level packages
     # So we can make sure those win out during resolution if the packages reoccur
-    lockfile = {"version": "=={0}".format(dep["version"])}
-    for key in ["hashes", "index", "extras"]:
+    if "version" in dep:
+        version = "{0}".format(dep["version"])
+        if not version.startswith("=="):
+            version = "=={0}".format(version)
+        lockfile["version"] = version
+    if is_vcs(dep):
+        ref = dep.get("ref", None)
+        if ref is not None:
+            lockfile["ref"] = ref
+        vcs_type = next(iter(k for k in dep.keys() if k in VCS_LIST), None)
+        if vcs_type:
+            lockfile[vcs_type] = dep[vcs_type]
+        if "subdirectory" in dep:
+            lockfile["subdirectory"] = dep["subdirectory"]
+    for key in ["hashes", "index", "extras", "editable"]:
         if key in dep:
             lockfile[key] = dep[key]
     # In case we lock a uri or a file when the user supplied a path
     # remove the uri or file keys from the entry and keep the path
-    if pipfile_entry and any(k in pipfile_entry for k in ["file", "path"]):
-        fs_key = next(iter(k for k in ["path", "file"] if k in dep), None)
-        if fs_key is not None:
-            lockfile[fs_key] = pipfile_entry[fs_key]
-    elif any(k in dep for k in ["file", "path"]):
-        fs_key = next(iter(k for k in ["path", "file"] if k in dep), None)
-        if fs_key is not None:
-            lockfile[fs_key] = dep[fs_key]
+    fs_key = next(iter(k for k in ["path", "file"] if k in dep), None)
+    pipfile_fs_key = None
+    if pipfile_entry:
+        pipfile_fs_key = next(iter(k for k in ["path", "file"] if k in pipfile_entry), None)
+    if fs_key and pipfile_fs_key and fs_key != pipfile_fs_key:
+        lockfile[pipfile_fs_key] = pipfile_entry[pipfile_fs_key]
+    elif fs_key is not None:
+        lockfile[fs_key] = dep[fs_key]
 
     # If a package is **PRESENT** in the pipfile but has no markers, make sure we
     # **NEVER** include markers in the lockfile

pipenv/vendor/pytoml/__init__.py

@@ -0,0 +1,4 @@
+from .core import TomlError
+from .parser import load, loads
+from .test import translate_to_test
+from .writer import dump, dumps

pipenv/vendor/pytoml/core.py

@@ -0,0 +1,13 @@
+class TomlError(RuntimeError):
+    def __init__(self, message, line, col, filename):
+        RuntimeError.__init__(self, message, line, col, filename)
+        self.message = message
+        self.line = line
+        self.col = col
+        self.filename = filename
+
+    def __str__(self):
+        return '{}({}, {}): {}'.format(self.filename, self.line, self.col, self.message)
+
+    def __repr__(self):
+        return 'TomlError({!r}, {!r}, {!r}, {!r})'.format(self.message, self.line, self.col, self.filename)

pipenv/vendor/pytoml/parser.py

@@ -0,0 +1,341 @@
+import string, re, sys, datetime
+from .core import TomlError
+from .utils import rfc3339_re, parse_rfc3339_re
+
+if sys.version_info[0] == 2:
+    _chr = unichr
+else:
+    _chr = chr
+
+def load(fin, translate=lambda t, x, v: v, object_pairs_hook=dict):
+    return loads(fin.read(), translate=translate, object_pairs_hook=object_pairs_hook, filename=getattr(fin, 'name', repr(fin)))
+
+def loads(s, filename='<string>', translate=lambda t, x, v: v, object_pairs_hook=dict):
+    if isinstance(s, bytes):
+        s = s.decode('utf-8')
+
+    s = s.replace('\r\n', '\n')
+
+    root = object_pairs_hook()
+    tables = object_pairs_hook()
+    scope = root
+
+    src = _Source(s, filename=filename)
+    ast = _p_toml(src, object_pairs_hook=object_pairs_hook)
+
+    def error(msg):
+        raise TomlError(msg, pos[0], pos[1], filename)
+
+    def process_value(v, object_pairs_hook):
+        kind, text, value, pos = v
+        if kind == 'str' and value.startswith('\n'):
+            value = value[1:]
+        if kind == 'array':
+            if value and any(k != value[0][0] for k, t, v, p in value[1:]):
+                error('array-type-mismatch')
+            value = [process_value(item, object_pairs_hook=object_pairs_hook) for item in value]
+        elif kind == 'table':
+            value = object_pairs_hook([(k, process_value(value[k], object_pairs_hook=object_pairs_hook)) for k in value])
+        return translate(kind, text, value)
+
+    for kind, value, pos in ast:
+        if kind == 'kv':
+            k, v = value
+            if k in scope:
+                error('duplicate_keys. Key "{0}" was used more than once.'.format(k))
+            scope[k] = process_value(v, object_pairs_hook=object_pairs_hook)
+        else:
+            is_table_array = (kind == 'table_array')
+            cur = tables
+            for name in value[:-1]:
+                if isinstance(cur.get(name), list):
+                    d, cur = cur[name][-1]
+                else:
+                    d, cur = cur.setdefault(name, (None, object_pairs_hook()))
+
+            scope = object_pairs_hook()
+            name = value[-1]
+            if name not in cur:
+                if is_table_array:
+                    cur[name] = [(scope, object_pairs_hook())]
+                else:
+                    cur[name] = (scope, object_pairs_hook())
+            elif isinstance(cur[name], list):
+                if not is_table_array:
+                    error('table_type_mismatch')
+                cur[name].append((scope, object_pairs_hook()))
+            else:
+                if is_table_array:
+                    error('table_type_mismatch')
+                old_scope, next_table = cur[name]
+                if old_scope is not None:
+                    error('duplicate_tables')
+                cur[name] = (scope, next_table)
+
+    def merge_tables(scope, tables):
+        if scope is None:
+            scope = object_pairs_hook()
+        for k in tables:
+            if k in scope:
+                error('key_table_conflict')
+            v = tables[k]
+            if isinstance(v, list):
+                scope[k] = [merge_tables(sc, tbl) for sc, tbl in v]
+            else:
+                scope[k] = merge_tables(v[0], v[1])
+        return scope
+
+    return merge_tables(root, tables)
+
+class _Source:
+    def __init__(self, s, filename=None):
+        self.s = s
+        self._pos = (1, 1)
+        self._last = None
+        self._filename = filename
+        self.backtrack_stack = []
+
+    def last(self):
+        return self._last
+
+    def pos(self):
+        return self._pos
+
+    def fail(self):
+        return self._expect(None)
+
+    def consume_dot(self):
+        if self.s:
+            self._last = self.s[0]
+            self.s = self[1:]
+            self._advance(self._last)
+            return self._last
+        return None
+
+    def expect_dot(self):
+        return self._expect(self.consume_dot())
+
+    def consume_eof(self):
+        if not self.s:
+            self._last = ''
+            return True
+        return False
+
+    def expect_eof(self):
+        return self._expect(self.consume_eof())
+
+    def consume(self, s):
+        if self.s.startswith(s):
+            self.s = self.s[len(s):]
+            self._last = s
+            self._advance(s)
+            return True
+        return False
+
+    def expect(self, s):
+        return self._expect(self.consume(s))
+
+    def consume_re(self, re):
+        m = re.match(self.s)
+        if m:
+            self.s = self.s[len(m.group(0)):]
+            self._last = m
+            self._advance(m.group(0))
+            return m
+        return None
+
+    def expect_re(self, re):
+        return self._expect(self.consume_re(re))
+
+    def __enter__(self):
+        self.backtrack_stack.append((self.s, self._pos))
+
+    def __exit__(self, type, value, traceback):
+        if type is None:
+            self.backtrack_stack.pop()
+        else:
+            self.s, self._pos = self.backtrack_stack.pop()
+        return type == TomlError
+
+    def commit(self):
+        self.backtrack_stack[-1] = (self.s, self._pos)
+
+    def _expect(self, r):
+        if not r:
+            raise TomlError('msg', self._pos[0], self._pos[1], self._filename)
+        return r
+
+    def _advance(self, s):
+        suffix_pos = s.rfind('\n')
+        if suffix_pos == -1:
+            self._pos = (self._pos[0], self._pos[1] + len(s))
+        else:
+            self._pos = (self._pos[0] + s.count('\n'), len(s) - suffix_pos)
+
+_ews_re = re.compile(r'(?:[ \t]|#[^\n]*\n|#[^\n]*\Z|\n)*')
+def _p_ews(s):
+    s.expect_re(_ews_re)
+
+_ws_re = re.compile(r'[ \t]*')
+def _p_ws(s):
+    s.expect_re(_ws_re)
+
+_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"',
+    '\\': '\\', 'f': '\f' }
+
+_basicstr_re = re.compile(r'[^"\\\000-\037]*')
+_short_uni_re = re.compile(r'u([0-9a-fA-F]{4})')
+_long_uni_re = re.compile(r'U([0-9a-fA-F]{8})')
+_escapes_re = re.compile(r'[btnfr\"\\]')
+_newline_esc_re = re.compile('\n[ \t\n]*')
+def _p_basicstr_content(s, content=_basicstr_re):
+    res = []
+    while True:
+        res.append(s.expect_re(content).group(0))
+        if not s.consume('\\'):
+            break
+        if s.consume_re(_newline_esc_re):
+            pass
+        elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re):
+            v = int(s.last().group(1), 16)
+            if 0xd800 <= v < 0xe000:
+                s.fail()
+            res.append(_chr(v))
+        else:
+            s.expect_re(_escapes_re)
+            res.append(_escapes[s.last().group(0)])
+    return ''.join(res)
+
+_key_re = re.compile(r'[0-9a-zA-Z-_]+')
+def _p_key(s):
+    with s:
+        s.expect('"')
+        r = _p_basicstr_content(s, _basicstr_re)
+        s.expect('"')
+        return r
+    if s.consume('\''):
+        if s.consume('\'\''):
+            r = s.expect_re(_litstr_ml_re).group(0)
+            s.expect('\'\'\'')
+        else:
+            r = s.expect_re(_litstr_re).group(0)
+            s.expect('\'')
+        return r
+    return s.expect_re(_key_re).group(0)
+
+_float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?')
+
+_basicstr_ml_re = re.compile(r'(?:""?(?!")|[^"\\\000-\011\013-\037])*')
+_litstr_re = re.compile(r"[^'\000\010\012-\037]*")
+_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*")
+def _p_value(s, object_pairs_hook):
+    pos = s.pos()
+
+    if s.consume('true'):
+        return 'bool', s.last(), True, pos
+    if s.consume('false'):
+        return 'bool', s.last(), False, pos
+
+    if s.consume('"'):
+        if s.consume('""'):
+            r = _p_basicstr_content(s, _basicstr_ml_re)
+            s.expect('"""')
+        else:
+            r = _p_basicstr_content(s, _basicstr_re)
+            s.expect('"')
+        return 'str', r, r, pos
+
+    if s.consume('\''):
+        if s.consume('\'\''):
+            r = s.expect_re(_litstr_ml_re).group(0)
+            s.expect('\'\'\'')
+        else:
+            r = s.expect_re(_litstr_re).group(0)
+            s.expect('\'')
+        return 'str', r, r, pos
+
+    if s.consume_re(rfc3339_re):
+        m = s.last()
+        return 'datetime', m.group(0), parse_rfc3339_re(m), pos
+
+    if s.consume_re(_float_re):
+        m = s.last().group(0)
+        r = m.replace('_','')
+        if '.' in m or 'e' in m or 'E' in m:
+            return 'float', m, float(r), pos
+        else:
+            return 'int', m, int(r, 10), pos
+
+    if s.consume('['):
+        items = []
+        with s:
+            while True:
+                _p_ews(s)
+                items.append(_p_value(s, object_pairs_hook=object_pairs_hook))
+                s.commit()
+                _p_ews(s)
+                s.expect(',')
+                s.commit()
+        _p_ews(s)
+        s.expect(']')
+        return 'array', None, items, pos
+
+    if s.consume('{'):
+        _p_ws(s)
+        items = object_pairs_hook()
+        if not s.consume('}'):
+            k = _p_key(s)
+            _p_ws(s)
+            s.expect('=')
+            _p_ws(s)
+            items[k] = _p_value(s, object_pairs_hook=object_pairs_hook)
+            _p_ws(s)
+            while s.consume(','):
+                _p_ws(s)
+                k = _p_key(s)
+                _p_ws(s)
+                s.expect('=')
+                _p_ws(s)
+                items[k] = _p_value(s, object_pairs_hook=object_pairs_hook)
+                _p_ws(s)
+            s.expect('}')
+        return 'table', None, items, pos
+
+    s.fail()
+
+def _p_stmt(s, object_pairs_hook):
+    pos = s.pos()
+    if s.consume(   '['):
+        is_array = s.consume('[')
+        _p_ws(s)
+        keys = [_p_key(s)]
+        _p_ws(s)
+        while s.consume('.'):
+            _p_ws(s)
+            keys.append(_p_key(s))
+            _p_ws(s)
+        s.expect(']')
+        if is_array:
+            s.expect(']')
+        return 'table_array' if is_array else 'table', keys, pos
+
+    key = _p_key(s)
+    _p_ws(s)
+    s.expect('=')
+    _p_ws(s)
+    value = _p_value(s, object_pairs_hook=object_pairs_hook)
+    return 'kv', (key, value), pos
+
+_stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*')
+def _p_toml(s, object_pairs_hook):
+    stmts = []
+    _p_ews(s)
+    with s:
+        stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook))
+        while True:
+            s.commit()
+            s.expect_re(_stmtsep_re)
+            stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook))
+    _p_ews(s)
+    s.expect_eof()
+    return stmts

pipenv/vendor/pytoml/test.py

@@ -0,0 +1,30 @@
+import datetime
+from .utils import format_rfc3339
+
+try:
+    _string_types = (str, unicode)
+    _int_types = (int, long)
+except NameError:
+    _string_types = str
+    _int_types = int
+
+def translate_to_test(v):
+    if isinstance(v, dict):
+        return { k: translate_to_test(v) for k, v in v.items() }
+    if isinstance(v, list):
+        a = [translate_to_test(x) for x in v]
+        if v and isinstance(v[0], dict):
+            return a
+        else:
+            return {'type': 'array', 'value': a}
+    if isinstance(v, datetime.datetime):
+        return {'type': 'datetime', 'value': format_rfc3339(v)}
+    if isinstance(v, bool):
+        return {'type': 'bool', 'value': 'true' if v else 'false'}
+    if isinstance(v, _int_types):
+        return {'type': 'integer', 'value': str(v)}
+    if isinstance(v, float):
+        return {'type': 'float', 'value': '{:.17}'.format(v)}
+    if isinstance(v, _string_types):
+        return {'type': 'string', 'value': v}
+    raise RuntimeError('unexpected value: {!r}'.format(v))

pipenv/vendor/pytoml/utils.py

@@ -0,0 +1,67 @@
+import datetime
+import re
+
+rfc3339_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))')
+
+def parse_rfc3339(v):
+    m = rfc3339_re.match(v)
+    if not m or m.group(0) != v:
+        return None
+    return parse_rfc3339_re(m)
+
+def parse_rfc3339_re(m):
+    r = map(int, m.groups()[:6])
+    if m.group(7):
+        micro = float(m.group(7))
+    else:
+        micro = 0
+
+    if m.group(8):
+        g = int(m.group(8), 10) * 60 + int(m.group(9), 10)
+        tz = _TimeZone(datetime.timedelta(0, g * 60))
+    else:
+        tz = _TimeZone(datetime.timedelta(0, 0))
+
+    y, m, d, H, M, S = r
+    return datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz)
+
+
+def format_rfc3339(v):
+    offs = v.utcoffset()
+    offs = int(offs.total_seconds()) // 60 if offs is not None else 0
+
+    if offs == 0:
+        suffix = 'Z'
+    else:
+        if offs > 0:
+            suffix = '+'
+        else:
+            suffix = '-'
+            offs = -offs
+        suffix = '{0}{1:02}:{2:02}'.format(suffix, offs // 60, offs % 60)
+
+    if v.microsecond:
+        return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix
+    else:
+        return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix
+
+class _TimeZone(datetime.tzinfo):
+    def __init__(self, offset):
+        self._offset = offset
+
+    def utcoffset(self, dt):
+        return self._offset
+
+    def dst(self, dt):
+        return None
+
+    def tzname(self, dt):
+        m = self._offset.total_seconds() // 60
+        if m < 0:
+            res = '-'
+            m = -m
+        else:
+            res = '+'
+        h = m // 60
+        m = m - h * 60
+        return '{}{:.02}{:.02}'.format(res, h, m)

pipenv/vendor/pytoml/writer.py

@@ -0,0 +1,106 @@
+from __future__ import unicode_literals
+import io, datetime, math, string, sys
+
+from .utils import format_rfc3339
+
+if sys.version_info[0] == 3:
+    long = int
+    unicode = str
+
+
+def dumps(obj, sort_keys=False):
+    fout = io.StringIO()
+    dump(obj, fout, sort_keys=sort_keys)
+    return fout.getvalue()
+
+
+_escapes = {'\n': 'n', '\r': 'r', '\\': '\\', '\t': 't', '\b': 'b', '\f': 'f', '"': '"'}
+
+
+def _escape_string(s):
+    res = []
+    start = 0
+
+    def flush():
+        if start != i:
+            res.append(s[start:i])
+        return i + 1
+
+    i = 0
+    while i < len(s):
+        c = s[i]
+        if c in '"\\\n\r\t\b\f':
+            start = flush()
+            res.append('\\' + _escapes[c])
+        elif ord(c) < 0x20:
+            start = flush()
+            res.append('\\u%04x' % ord(c))
+        i += 1
+
+    flush()
+    return '"' + ''.join(res) + '"'
+
+
+_key_chars = string.digits + string.ascii_letters + '-_'
+def _escape_id(s):
+    if any(c not in _key_chars for c in s):
+        return _escape_string(s)
+    return s
+
+
+def _format_value(v):
+    if isinstance(v, bool):
+        return 'true' if v else 'false'
+    if isinstance(v, int) or isinstance(v, long):
+        return unicode(v)
+    if isinstance(v, float):
+        if math.isnan(v) or math.isinf(v):
+            raise ValueError("{0} is not a valid TOML value".format(v))
+        else:
+            return repr(v)
+    elif isinstance(v, unicode) or isinstance(v, bytes):
+        return _escape_string(v)
+    elif isinstance(v, datetime.datetime):
+        return format_rfc3339(v)
+    elif isinstance(v, list):
+        return '[{0}]'.format(', '.join(_format_value(obj) for obj in v))
+    elif isinstance(v, dict):
+        return '{{{0}}}'.format(', '.join('{} = {}'.format(_escape_id(k), _format_value(obj)) for k, obj in v.items()))
+    else:
+        raise RuntimeError(v)
+
+
+def dump(obj, fout, sort_keys=False):
+    tables = [((), obj, False)]
+
+    while tables:
+        name, table, is_array = tables.pop()
+        if name:
+            section_name = '.'.join(_escape_id(c) for c in name)
+            if is_array:
+                fout.write('[[{0}]]\n'.format(section_name))
+            else:
+                fout.write('[{0}]\n'.format(section_name))
+
+        table_keys = sorted(table.keys()) if sort_keys else table.keys()
+        new_tables = []
+        has_kv = False
+        for k in table_keys:
+            v = table[k]
+            if isinstance(v, dict):
+                new_tables.append((name + (k,), v, False))
+            elif isinstance(v, list) and v and all(isinstance(o, dict) for o in v):
+                new_tables.extend((name + (k,), d, True) for d in v)
+            elif v is None:
+                # based on mojombo's comment: https://github.com/toml-lang/toml/issues/146#issuecomment-25019344
+                fout.write(
+                    '#{} = null  # To use: uncomment and replace null with value\n'.format(_escape_id(k)))
+                has_kv = True
+            else:
+                fout.write('{0} = {1}\n'.format(_escape_id(k), _format_value(v)))
+                has_kv = True
+
+        tables.extend(reversed(new_tables))
+
+        if (name or has_kv) and tables:
+            fout.write('\n')

pipenv/vendor/requirementslib/models/pipfile.py

@@ -18,7 +18,7 @@ from ..exceptions import RequirementError
 from ..utils import is_editable, is_vcs, merge_items
 from .project import ProjectFile
 from .requirements import Requirement
-from .utils import optional_instance_of
+from .utils import optional_instance_of, get_url_name
 
 from ..environment import MYPY_RUNNING
 if MYPY_RUNNING:
@@ -87,9 +87,10 @@ def reorder_source_keys(data):
     sources = data["source"]  # type: sources_type
     for i, entry in enumerate(sources):
         table = tomlkit.table()  # type: Mapping
-        table["name"] = entry["name"]
-        table["url"] = entry["url"]
-        table["verify_ssl"] = entry["verify_ssl"]
+        source_entry = PipfileLoader.populate_source(entry.copy())
+        table["name"] = source_entry["name"]
+        table["url"] = source_entry["url"]
+        table["verify_ssl"] = source_entry["verify_ssl"]
         data["source"][i] = table
     return data
 
@@ -106,6 +107,18 @@ class PipfileLoader(plette.pipfiles.Pipfile):
             except Exception:
                 pass
 
+    @classmethod
+    def populate_source(cls, source):
+        """Derive missing values of source from the existing fields."""
+        # Only URL pararemter is mandatory, let the KeyError be thrown.
+        if "name" not in source:
+            source["name"] = get_url_name(source["url"])
+        if "verify_ssl" not in source:
+            source["verify_ssl"] = "https://" in source["url"]
+        if not isinstance(source["verify_ssl"], bool):
+            source["verify_ssl"] = source["verify_ssl"].lower() == "true"
+        return source
+
     @classmethod
     def load(cls, f, encoding=None):
         # type: (Any, str) -> PipfileLoader

pipenv/vendor/requirementslib/models/setup_info.py

@@ -1,32 +1,43 @@
 # -*- coding=utf-8 -*-
 from __future__ import absolute_import, print_function
 
+import atexit
 import contextlib
 import os
+import shutil
 import sys
 
 import attr
-import packaging.version
 import packaging.specifiers
 import packaging.utils
+import packaging.version
+import pep517.envbuild
+import pep517.wrappers
 import six
+from appdirs import user_cache_dir
+from six.moves import configparser
+from six.moves.urllib.parse import unquote, urlparse, urlunparse
+
+from vistir.compat import Iterable, Path
+from vistir.contextmanagers import cd, temp_path, replaced_streams
+from vistir.misc import run
+from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p, rmtree
+
+from ..environment import MYPY_RUNNING
+from ..exceptions import RequirementError
+from .utils import (
+    get_name_variants,
+    get_pyproject,
+    init_requirement,
+    split_vcs_method_from_uri,
+    strip_extras_markers_from_requirement
+)
 
 try:
     from setuptools.dist import distutils
 except ImportError:
     import distutils
 
-from appdirs import user_cache_dir
-from six.moves import configparser
-from six.moves.urllib.parse import unquote
-from vistir.compat import Path, Iterable
-from vistir.contextmanagers import cd
-from vistir.misc import run
-from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p
-
-from .utils import init_requirement, get_pyproject, get_name_variants
-from ..environment import MYPY_RUNNING
-from ..exceptions import RequirementError
 
 try:
     from os import scandir
@@ -35,7 +46,7 @@ except ImportError:
 
 
 if MYPY_RUNNING:
-    from typing import Any, Dict, List, Generator, Optional, Union
+    from typing import Any, Dict, List, Generator, Optional, Union, Tuple
     from pip_shims.shims import InstallRequirement
     from pkg_resources import Requirement as PkgResourcesRequirement
 
@@ -77,7 +88,7 @@ def _get_src_dir(root):
     virtual_env = os.environ.get("VIRTUAL_ENV")
     if virtual_env is not None:
         return os.path.join(virtual_env, "src")
-    if not root:
+    if root is not None:
         # Intentionally don't match pip's behavior here -- this is a temporary copy
         src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src")
     else:
@@ -96,34 +107,41 @@ def ensure_reqs(reqs):
             continue
         if isinstance(req, six.string_types):
             req = pkg_resources.Requirement.parse("{0}".format(str(req)))
+        req = strip_extras_markers_from_requirement(req)
         new_reqs.append(req)
     return new_reqs
 
 
-def _prepare_wheel_building_kwargs(ireq=None, src_root=None, editable=False):
-    # type: (Optional[InstallRequirement], Optional[str], bool) -> Dict[str, str]
+def _prepare_wheel_building_kwargs(ireq=None, src_root=None, src_dir=None, editable=False):
+    # type: (Optional[InstallRequirement], Optional[str], Optional[str], bool) -> Dict[str, str]
     download_dir = os.path.join(CACHE_DIR, "pkgs")  # type: str
     mkdir_p(download_dir)
 
     wheel_download_dir = os.path.join(CACHE_DIR, "wheels")  # type: str
     mkdir_p(wheel_download_dir)
 
-    if ireq is None:
-        src_dir = _get_src_dir(root=src_root)  # type: str
-    elif ireq is not None and ireq.source_dir is not None:
-        src_dir = ireq.source_dir
-    elif ireq is not None and ireq.editable:
-        src_dir = _get_src_dir(root=src_root)
-    else:
-        src_dir = create_tracked_tempdir(prefix="reqlib-src")
+    if src_dir is None:
+        if editable and src_root is not None:
+            src_dir = src_root
+        elif ireq is None and src_root is not None:
+            src_dir = _get_src_dir(root=src_root)  # type: str
+    #     # elif ireq is not None and ireq.editable is not None and ireq.source_dir is not None:
+    #     #     src_dir = ireq.source_dir
+        elif ireq is not None and ireq.editable and src_root is not None:
+            src_dir = _get_src_dir(root=src_root)
+        else:
+            src_dir = create_tracked_tempdir(prefix="reqlib-src")
 
     # This logic matches pip's behavior, although I don't fully understand the
     # intention. I guess the idea is to build editables in-place, otherwise out
     # of the source tree?
-    if ireq is None and editable or (ireq is not None and ireq.editable):
-        build_dir = src_dir
-    else:
-        build_dir = create_tracked_tempdir(prefix="reqlib-build")
+    # if (ireq is not None and ireq.editable) or editable:
+    #     build_dir = src_dir
+    # else:
+
+    # Let's always resolve in isolation
+    # src_dir = create_tracked_tempdir(prefix="reqlib-src")
+    build_dir = create_tracked_tempdir(prefix="reqlib-build")
 
     return {
         "build_dir": build_dir,
@@ -133,46 +151,72 @@ def _prepare_wheel_building_kwargs(ireq=None, src_root=None, editable=False):
     }
 
 
-def iter_egginfos(path, pkg_name=None):
-    # type: (str, Optional[str]) -> Generator
+def iter_metadata(path, pkg_name=None, metadata_type="egg-info"):
+    # type: (str, Optional[str], str) -> Generator
     if pkg_name is not None:
         pkg_variants = get_name_variants(pkg_name)
     non_matching_dirs = []
     for entry in scandir(path):
         if entry.is_dir():
             entry_name, ext = os.path.splitext(entry.name)
-            if ext.endswith("egg-info"):
-                if pkg_name is None or entry_name in pkg_variants:
+            if ext.endswith(metadata_type):
+                if pkg_name is None or entry_name.lower() in pkg_variants:
                     yield entry
-            elif not entry.name.endswith("egg-info"):
+            elif not entry.name.endswith(metadata_type):
                 non_matching_dirs.append(entry)
     for entry in non_matching_dirs:
-        for dir_entry in iter_egginfos(entry.path, pkg_name=pkg_name):
+        for dir_entry in iter_metadata(entry.path, pkg_name=pkg_name, metadata_type=metadata_type):
             yield dir_entry
 
 
 def find_egginfo(target, pkg_name=None):
     # type: (str, Optional[str]) -> Generator
-    egg_dirs = (egg_dir for egg_dir in iter_egginfos(target, pkg_name=pkg_name))
+    egg_dirs = (
+        egg_dir for egg_dir in iter_metadata(target, pkg_name=pkg_name)
+        if egg_dir is not None
+    )
     if pkg_name:
-        yield next(iter(egg_dirs), None)
+        yield next(iter(eggdir for eggdir in egg_dirs if eggdir is not None), None)
     else:
         for egg_dir in egg_dirs:
             yield egg_dir
 
 
+def find_distinfo(target, pkg_name=None):
+    # type: (str, Optional[str]) -> Generator
+    dist_dirs = (
+        dist_dir for dist_dir in iter_metadata(target, pkg_name=pkg_name, metadata_type="dist-info")
+        if dist_dir is not None
+    )
+    if pkg_name:
+        yield next(iter(dist for dist in dist_dirs if dist is not None), None)
+    else:
+        for dist_dir in dist_dirs:
+            yield dist_dir
+
+
 def get_metadata(path, pkg_name=None):
     egg_dir = next(iter(find_egginfo(path, pkg_name=pkg_name)), None)
-    if egg_dir is not None:
+    dist_dir = next(iter(find_distinfo(path, pkg_name=pkg_name)), None)
+    matched_dir = next(iter(d for d in (dist_dir, egg_dir) if d is not None), None)
+    metadata_dir = None
+    base_dir = None
+    if matched_dir is not None:
         import pkg_resources
-
-        egg_dir = os.path.abspath(egg_dir.path)
-        base_dir = os.path.dirname(egg_dir)
-        path_metadata = pkg_resources.PathMetadata(base_dir, egg_dir)
-        dist = next(
-            iter(pkg_resources.distributions_from_metadata(path_metadata.egg_info)),
-            None,
-        )
+        metadata_dir = os.path.abspath(matched_dir.path)
+        base_dir = os.path.dirname(metadata_dir)
+        dist = None
+        distinfo_dist = None
+        egg_dist = None
+        if dist_dir is not None:
+            distinfo_dist = next(iter(pkg_resources.find_distributions(base_dir)), None)
+        if egg_dir is not None:
+            path_metadata = pkg_resources.PathMetadata(base_dir, metadata_dir)
+            egg_dist = next(
+                iter(pkg_resources.distributions_from_metadata(path_metadata.egg_info)),
+                None,
+            )
+        dist = next(iter(d for d in (distinfo_dist, egg_dist) if d is not None), None)
         if dist:
             try:
                 requires = dist.requires()
@@ -210,22 +254,96 @@ def get_metadata(path, pkg_name=None):
             }
 
 
-@attr.s(slots=True)
+@attr.s(slots=True, frozen=True)
+class BaseRequirement(object):
+    name = attr.ib(type=str, default="", cmp=True)
+    requirement = attr.ib(default=None, cmp=True)  # type: Optional[PkgResourcesRequirement]
+
+    def __str__(self):
+        # type: () -> str
+        return "{0}".format(str(self.requirement))
+
+    def as_dict(self):
+        # type: () -> Dict[str, Optional[PkgResourcesRequirement]]
+        return {self.name: self.requirement}
+
+    def as_tuple(self):
+        # type: () -> Tuple[str, Optional[PkgResourcesRequirement]]
+        return (self.name, self.requirement)
+
+    @classmethod
+    def from_string(cls, line):
+        # type: (str) -> BaseRequirement
+        line = line.strip()
+        req = init_requirement(line)
+        return cls.from_req(req)
+
+    @classmethod
+    def from_req(cls, req):
+        # type: (PkgResourcesRequirement) -> BaseRequirement
+        name = None
+        key = getattr(req, "key", None)
+        name = getattr(req, "name", None)
+        project_name = getattr(req, "project_name", None)
+        if key is not None:
+            name = key
+        if name is None:
+            name = project_name
+        return cls(name=name, requirement=req)
+
+
+@attr.s(slots=True, frozen=True)
+class Extra(object):
+    name = attr.ib(type=str, default=None, cmp=True)
+    requirements = attr.ib(factory=frozenset, cmp=True, type=frozenset)
+
+    def __str__(self):
+        # type: () -> str
+        return "{0}: {{{1}}}".format(self.section, ", ".join([r.name for r in self.requirements]))
+
+    def add(self, req):
+        # type: (BaseRequirement) -> None
+        if req not in self.requirements:
+            return attr.evolve(self, requirements=frozenset(set(self.requirements).add(req)))
+        return self
+
+    def as_dict(self):
+        # type: () -> Dict[str, Tuple[PkgResourcesRequirement]]
+        return {self.name: tuple([r.requirement for r in self.requirements])}
+
+
+@attr.s(slots=True, cmp=True, hash=True)
 class SetupInfo(object):
-    name = attr.ib(type=str, default=None)
-    base_dir = attr.ib(type=Path, default=None)
-    version = attr.ib(type=packaging.version.Version, default=None)
-    requires = attr.ib(type=dict, default=attr.Factory(dict))
-    build_requires = attr.ib(type=list, default=attr.Factory(list))
-    build_backend = attr.ib(type=list, default=attr.Factory(list))
-    setup_requires = attr.ib(type=dict, default=attr.Factory(list))
-    python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None)
-    extras = attr.ib(type=dict, default=attr.Factory(dict))
-    setup_cfg = attr.ib(type=Path, default=None)
-    setup_py = attr.ib(type=Path, default=None)
-    pyproject = attr.ib(type=Path, default=None)
-    ireq = attr.ib(default=None)
-    extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict)
+    name = attr.ib(type=str, default=None, cmp=True)
+    base_dir = attr.ib(type=str, default=None, cmp=True, hash=False)
+    version = attr.ib(type=str, default=None, cmp=True)
+    _requirements = attr.ib(type=frozenset, factory=frozenset, cmp=True, hash=True)
+    build_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True)
+    build_backend = attr.ib(type=str, default="setuptools.build_meta:__legacy__", cmp=True)
+    setup_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True)
+    python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None, cmp=True)
+    _extras_requirements = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True)
+    setup_cfg = attr.ib(type=Path, default=None, cmp=True, hash=False)
+    setup_py = attr.ib(type=Path, default=None, cmp=True, hash=False)
+    pyproject = attr.ib(type=Path, default=None, cmp=True, hash=False)
+    ireq = attr.ib(default=None, cmp=True, hash=False)
+    extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict, cmp=False, hash=False)
+    metadata = attr.ib(default=None, type=tuple)
+
+    @property
+    def requires(self):
+        return {req.name: req.requirement for req in self._requirements}
+
+    @property
+    def extras(self):
+        extras_dict = {}
+        extras = set(self._extras_requirements)
+        for section, deps in extras:
+            if isinstance(deps, BaseRequirement):
+                extras_dict[section] = deps.requirement
+            elif isinstance(deps, (list, tuple)):
+                extras_dict[section] = [d.requirement for d in deps]
+        return extras_dict
 
     @classmethod
     def get_setup_cfg(cls, setup_cfg_path):
@@ -247,32 +365,53 @@ class SetupInfo(object):
                 results["name"] = parser.get("metadata", "name")
             if parser.has_option("metadata", "version"):
                 results["version"] = parser.get("metadata", "version")
-            install_requires = {}
+            install_requires = set()
             if parser.has_option("options", "install_requires"):
-                install_requires = {
-                    dep.strip(): init_requirement(dep.strip())
+                install_requires = set([
+                    BaseRequirement.from_string(dep)
                     for dep in parser.get("options", "install_requires").split("\n")
                     if dep
-                }
+                ])
             results["install_requires"] = install_requires
             if parser.has_option("options", "python_requires"):
                 results["python_requires"] = parser.get("options", "python_requires")
-            extras_require = {}
+            if parser.has_option("options", "build_requires"):
+                results["build_requires"] = parser.get("options", "build_requires")
+            extras_require = ()
             if "options.extras_require" in parser.sections():
-                extras_require = {
-                    section: [
-                        init_requirement(dep.strip())
+                extras_require = tuple([
+                    (section, tuple([
+                        BaseRequirement.from_string(dep)
                         for dep in parser.get(
                             "options.extras_require", section
                         ).split("\n")
                         if dep
-                    ]
+                    ]))
                     for section in parser.options("options.extras_require")
                     if section not in ["options", "metadata"]
-                }
+                ])
             results["extras_require"] = extras_require
             return results
 
+    @property
+    def egg_base(self):
+        base = None  # type: Optional[Path]
+        if self.setup_py.exists():
+            base = self.setup_py.parent
+        elif self.pyproject.exists():
+            base = self.pyproject.parent
+        elif self.setup_cfg.exists():
+            base = self.setup_cfg.parent
+        if base is None:
+            base = Path(self.base_dir)
+        if base is None:
+            base = Path(self.extra_kwargs["build_dir"])
+        egg_base = base.joinpath("reqlib-metadata")
+        if not egg_base.exists():
+            atexit.register(rmtree, egg_base.as_posix())
+        egg_base.mkdir(parents=True, exist_ok=True)
+        return egg_base.as_posix()
+
     def parse_setup_cfg(self):
         if self.setup_cfg is not None and self.setup_cfg.exists():
             parsed = self.get_setup_cfg(self.setup_cfg.as_posix())
@@ -280,25 +419,34 @@ class SetupInfo(object):
                 self.name = parsed.get("name")
             if self.version is None:
                 self.version = parsed.get("version")
-            self.requires.update(parsed["install_requires"])
+            build_requires = parsed.get("build_requires", [])
+            if self.build_requires:
+                self.build_requires = tuple(set(self.build_requires) | set(build_requires))
+            self._requirements = frozenset(
+                set(self._requirements) | parsed["install_requires"]
+            )
             if self.python_requires is None:
                 self.python_requires = parsed.get("python_requires")
-            self.extras.update(parsed["extras_require"])
+            if not self._extras_requirements:
+                self._extras_requirements = (parsed["extras_require"])
+            else:
+                self._extras_requirements = self._extras_requirements + parsed["extras_require"]
             if self.ireq is not None and self.ireq.extras:
-                self.requires.update({
-                    extra: self.extras[extra]
-                    for extra in self.ireq.extras if extra in self.extras
-                })
+                for extra in self.ireq.extras:
+                    if extra in self.extras:
+                        extras_tuple = tuple([BaseRequirement.from_req(req) for req in self.extras[extra]])
+                        self._extras_requirements += ((extra, extras_tuple),)
 
     def run_setup(self):
         if self.setup_py is not None and self.setup_py.exists():
             target_cwd = self.setup_py.parent.as_posix()
-            with cd(target_cwd), _suppress_distutils_logs():
+            with temp_path(), cd(target_cwd), _suppress_distutils_logs():
                 # This is for you, Hynek
                 # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py
                 script_name = self.setup_py.as_posix()
-                args = ["egg_info"]
+                args = ["egg_info", "--egg-base", self.egg_base]
                 g = {"__file__": script_name, "__name__": "__main__"}
+                sys.path.insert(0, os.path.dirname(os.path.abspath(script_name)))
                 local_dict = {}
                 if sys.version_info < (3, 5):
                     save_argv = sys.argv
@@ -334,41 +482,101 @@ class SetupInfo(object):
                     self.python_requires = packaging.specifiers.SpecifierSet(
                         dist.python_requires
                     )
+                if not self._extras_requirements:
+                    self._extras_requirements = ()
                 if dist.extras_require and not self.extras:
-                    self.extras = dist.extras_require
+                    for extra, extra_requires in dist.extras_require:
+                        extras_tuple = tuple(
+                            BaseRequirement.from_req(req) for req in extra_requires
+                        )
+                        self._extras_requirements += ((extra, extras_tuple),)
                 install_requires = dist.get_requires()
                 if not install_requires:
                     install_requires = dist.install_requires
                 if install_requires and not self.requires:
-                    requirements = [init_requirement(req) for req in install_requires]
+                    requirements = set([
+                        BaseRequirement.from_req(req) for req in install_requires
+                    ])
                     if getattr(self.ireq, "extras", None):
                         for extra in self.ireq.extras:
-                            requirements.extend(list(self.extras.get(extra, [])))
-                    self.requires.update({req.key: req for req in requirements})
+                            requirements |= set(list(self.extras.get(extra, [])))
+                    self._requirements = frozenset(
+                        set(self._requirements) | requirements
+                    )
                 if dist.setup_requires and not self.setup_requires:
-                    self.setup_requires = dist.setup_requires
+                    self.setup_requires = tuple(dist.setup_requires)
                 if not self.version:
                     self.version = dist.get_version()
 
+    def run_pep517(self, build=False):
+        # type: () -> str
+        with pep517.envbuild.BuildEnvironment():
+            hookcaller = pep517.wrappers.Pep517HookCaller(
+                self.base_dir, self.build_backend
+            )
+            build_deps = hookcaller.get_requires_for_build_wheel()
+            if self.ireq.editable:
+                build_deps += hookcaller.get_requires_for_build_sdist()
+            metadata_dirname = hookcaller.prepare_metadata_for_build_wheel(self.egg_base)
+            metadata_dir = os.path.join(self.egg_base, metadata_dirname)
+
+            if build:
+                return self.build_pep517()
+            return metadata_dir
+
+    def build_pep517(self, hookcaller):
+        # type: (pep517.wrappers.Pep517HookCaller) -> Optional[str]
+        dist_path = None
+        try:
+            dist_path = hookcaller.build_wheel(
+                self.extra_kwargs["build_dir"],
+                metadata_directory=self.egg_base
+            )
+        except Exception:
+            dist_path = hookcaller.build_sdist(self.extra_kwargs["build_dir"])
+        return dist_path
+
+    def reload(self):
+        # type: () -> None
+        """
+        Wipe existing distribution info metadata for rebuilding.
+        """
+        for metadata_dir in os.listdir(self.egg_base):
+            shutil.rmtree(metadata_dir, ignore_errors=True)
+        self.metadata = None
+        self._requirements = frozenset()
+        self._extras_requirements = ()
+        self.get_info()
+
     def get_egg_metadata(self):
-        if self.setup_py is not None and self.setup_py.exists():
-            metadata = get_metadata(self.setup_py.parent.as_posix(), pkg_name=self.name)
+        package_indicators = [self.pyproject, self.setup_py, self.setup_cfg]
+        # if self.setup_py is not None and self.setup_py.exists():
+        if any([fn is not None and fn.exists() for fn in package_indicators]):
+            metadata = get_metadata(self.egg_base, pkg_name=self.name)
             if metadata:
+                self.metadata = tuple([(k, v) for k, v in metadata.items()])
                 if self.name is None:
                     self.name = metadata.get("name", self.name)
                 if not self.version:
                     self.version = metadata.get("version", self.version)
-                self.requires.update(
-                    {req.key: req for req in metadata.get("requires", {})}
+                self._requirements = frozenset(
+                    set(self._requirements) | set([
+                        BaseRequirement.from_req(req)
+                        for req in metadata.get("requires", [])
+                    ])
                 )
                 if getattr(self.ireq, "extras", None):
                     for extra in self.ireq.extras:
                         extras = metadata.get("extras", {}).get(extra, [])
                         if extras:
-                            extras = ensure_reqs(extras)
-                            self.extras[extra] = set(extras)
-                            self.requires.update(
-                                {req.key: req for req in extras if req is not None}
+                            extras_tuple = tuple([
+                                BaseRequirement.from_req(req)
+                                for req in ensure_reqs(extras)
+                                if req is not None
+                            ])
+                            self._extras_requirements += ((extra, extras_tuple),)
+                            self._requirements = frozenset(
+                                set(self._requirements) | set(extras_tuple)
                             )
 
     def run_pyproject(self):
@@ -378,35 +586,37 @@ class SetupInfo(object):
                 requires, backend = result
                 if backend:
                     self.build_backend = backend
+                else:
+                    self.build_backend = "setuptools.build_meta:__legacy__"
+                    self.build_requires = ("setuptools", "wheel")
                 if requires and not self.build_requires:
-                    self.build_requires = requires
+                    self.build_requires = tuple(requires)
 
     def get_info(self):
-        initial_path = os.path.abspath(os.getcwd())
         if self.setup_cfg and self.setup_cfg.exists():
-            try:
+            with cd(self.base_dir):
                 self.parse_setup_cfg()
-            finally:
-                os.chdir(initial_path)
-        if self.setup_py and self.setup_py.exists():
-            if not self.requires or not self.name:
-                try:
-                    self.run_setup()
-                except Exception:
-                    self.get_egg_metadata()
-                finally:
-                    os.chdir(initial_path)
-                if not self.requires or not self.name:
-                    try:
-                        self.get_egg_metadata()
-                    finally:
-                        os.chdir(initial_path)
 
         if self.pyproject and self.pyproject.exists():
-            try:
+            with cd(self.base_dir), replaced_streams():
                 self.run_pyproject()
-            finally:
-                os.chdir(initial_path)
+                self.run_pep517()
+                self.get_egg_metadata()
+
+        if self.setup_py and self.setup_py.exists() and self.metadata is None:
+            if not self.requires or not self.name:
+                try:
+                    with cd(self.base_dir):
+                        for metadata_dir in os.listdir(self.egg_base):
+                            shutil.rmtree(metadata_dir, ignore_errors=True)
+                        self.run_setup()
+                except Exception:
+                    with cd(self.base_dir):
+                        self.get_egg_metadata()
+                if self.metadata is None or not self.name:
+                    with cd(self.base_dir):
+                        self.get_egg_metadata()
+
         return self.as_dict()
 
     def as_dict(self):
@@ -438,15 +648,31 @@ class SetupInfo(object):
     @classmethod
     def from_ireq(cls, ireq, subdir=None, finder=None):
         import pip_shims.shims
-
+        if not ireq.link:
+            return
         if ireq.link.is_wheel:
             return
         if not finder:
             from .dependencies import get_finder
 
             finder = get_finder()
+        vcs_method, uri = split_vcs_method_from_uri(unquote(ireq.link.url_without_fragment))
+        parsed = urlparse(uri)
+        url_path = parsed.path
+        if "@" in url_path:
+            url_path, _, _ = url_path.rpartition("@")
+        parsed = parsed._replace(path=url_path)
+        uri = urlunparse(parsed)
+        path = None
+        if ireq.link.scheme == "file" or uri.startswith("file://"):
+            if "file:/" in uri and "file:///" not in uri:
+                uri = uri.replace("file:/", "file:///")
+            path = pip_shims.shims.url_to_path(uri)
+        #     if pip_shims.shims.is_installable_dir(path) and ireq.editable:
+        #         ireq.source_dir = path
         kwargs = _prepare_wheel_building_kwargs(ireq)
-        ireq.populate_link(finder, False, False)
+        ireq.source_dir = kwargs["src_dir"]
+        # os.environ["PIP_BUILD_DIR"] = kwargs["build_dir"]
         ireq.ensure_has_source_dir(kwargs["build_dir"])
         if not (
             ireq.editable
@@ -459,37 +685,31 @@ class SetupInfo(object):
             else:
                 only_download = False
                 download_dir = kwargs["download_dir"]
-        ireq_src_dir = None
-        if ireq.link.scheme == "file":
-            path = pip_shims.shims.url_to_path(unquote(ireq.link.url_without_fragment))
-            if pip_shims.shims.is_installable_dir(path):
-                ireq_src_dir = path
-            elif os.path.isdir(path):
-                raise RequirementError(
-                    "The file URL points to a directory not installable: {}"
-                    .format(ireq.link)
-                )
-        if not ireq.editable or not ireq.link.scheme == "file":
-            pip_shims.shims.unpack_url(
-                ireq.link,
-                ireq.source_dir,
-                download_dir,
-                only_download=only_download,
-                session=finder.session,
-                hashes=ireq.hashes(False),
-                progress_bar="off",
+        elif path is not None and os.path.isdir(path):
+            raise RequirementError(
+                "The file URL points to a directory not installable: {}"
+                .format(ireq.link)
             )
-        if ireq.editable:
-            created = cls.create(
-                ireq.source_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs
-            )
-        else:
+        if not ireq.editable:
             build_dir = ireq.build_location(kwargs["build_dir"])
             ireq._temp_build_dir.path = kwargs["build_dir"]
-            created = cls.create(
-                build_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs
-            )
-        created.get_info()
+        else:
+            build_dir = ireq.build_location(kwargs["src_dir"])
+            ireq._temp_build_dir.path = kwargs["build_dir"]
+
+        ireq.populate_link(finder, False, False)
+        pip_shims.shims.unpack_url(
+            ireq.link,
+            build_dir,
+            download_dir,
+            only_download=only_download,
+            session=finder.session,
+            hashes=ireq.hashes(False),
+            progress_bar="off",
+        )
+        created = cls.create(
+            build_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs
+        )
         return created
 
     @classmethod
@@ -498,6 +718,7 @@ class SetupInfo(object):
             return
 
         creation_kwargs = {"extra_kwargs": kwargs}
+
         if not isinstance(base_dir, Path):
             base_dir = Path(base_dir)
         creation_kwargs["base_dir"] = base_dir.as_posix()

pipenv/vendor/requirementslib/models/utils.py

@@ -3,6 +3,8 @@ from __future__ import absolute_import, print_function
 
 import io
 import os
+import re
+import string
 import sys
 
 from collections import defaultdict
@@ -16,7 +18,10 @@ from attr import validators
 from first import first
 from packaging.markers import InvalidMarker, Marker, Op, Value, Variable
 from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet
+from six.moves.urllib import parse as urllib_parse
+from urllib3 import util as urllib3_util
 from vistir.misc import dedup
+from vistir.path import is_valid_url
 
 
 from ..utils import SCHEME_LIST, VCS_LIST, is_star, add_ssh_scheme_to_git_uri
@@ -24,15 +29,34 @@ from ..utils import SCHEME_LIST, VCS_LIST, is_star, add_ssh_scheme_to_git_uri
 from ..environment import MYPY_RUNNING
 
 if MYPY_RUNNING:
-    from typing import Union, Optional, List, Set, Any, TypeVar
+    from typing import Union, Optional, List, Set, Any, TypeVar, Tuple, Sequence, Dict
     from attr import _ValidatorType
+    from packaging.requirements import Requirement as PackagingRequirement
     from pkg_resources import Requirement as PkgResourcesRequirement
-    from pip_shims import Link
+    from pkg_resources.extern.packaging.markers import Marker as PkgResourcesMarker
+    from pip_shims.shims import Link
     _T = TypeVar("_T")
+    TMarker = Union[Marker, PkgResourcesMarker]
+    TRequirement = Union[PackagingRequirement, PkgResourcesRequirement]
 
 
 HASH_STRING = " --hash={0}"
 
+ALPHA_NUMERIC = r"[{0}{1}]".format(string.ascii_letters, string.digits)
+PUNCTUATION = r"[\-_\.]"
+ALPHANUM_PUNCTUATION = r"[{0}{1}\-_\.]".format(string.ascii_letters, string.digits)
+NAME = r"{0}+{1}*{2}".format(ALPHANUM_PUNCTUATION, PUNCTUATION, ALPHA_NUMERIC)
+REF = r"[{0}{1}\-\_\./]".format(string.ascii_letters, string.digits)
+EXTRAS = r"(?P<extras>\[{0}(?:,{0})*\])".format(NAME)
+NAME_WITH_EXTRAS = r"(?P<name>{0}){1}?".format(NAME, EXTRAS)
+NAME_RE = re.compile(NAME_WITH_EXTRAS)
+SUBDIR_RE = r"(?:[&#]subdirectory=(?P<subdirectory>.*))"
+URL_NAME = r"(?:#egg={0})".format(NAME_WITH_EXTRAS)
+REF_RE = r"(?:@(?P<ref>{0}+)?)".format(REF)
+URL = r"(?P<scheme>[^ ]+://)(?:(?P<host>[^ ]+?\.?{0}+(?P<port>:\d+)?))?(?P<pathsep>[:/])(?P<path>[^ @]+){1}?".format(ALPHA_NUMERIC, REF_RE)
+URL_RE = re.compile(r"{0}(?:{1}?{2}?)?".format(URL, URL_NAME, SUBDIR_RE))
+DIRECT_URL_RE = re.compile(r"{0}\s?@\s?{1}".format(NAME_WITH_EXTRAS, URL))
+
 
 def filter_none(k, v):
     # type: (str, Any) -> bool
@@ -51,12 +75,26 @@ def create_link(link):
 
     if not isinstance(link, six.string_types):
         raise TypeError("must provide a string to instantiate a new link")
-    from pip_shims import Link
+    from pip_shims.shims import Link
     return Link(link)
 
 
+def get_url_name(url):
+    # type: (str) -> str
+    """
+    Given a url, derive an appropriate name to use in a pipfile.
+
+    :param str url: A url to derive a string from
+    :returns: The name of the corresponding pipfile entry
+    :rtype: str
+    """
+    if not isinstance(url, six.string_types):
+        raise TypeError("Expected a string, got {0!r}".format(url))
+    return urllib3_util.parse_url(url).host
+
+
 def init_requirement(name):
-    # type: (str) -> PkgResourcesRequirement
+    # type: (str) -> TRequirement
 
     if not isinstance(name, six.string_types):
         raise TypeError("must supply a name to generate a requirement")
@@ -70,6 +108,7 @@ def init_requirement(name):
 
 
 def extras_to_string(extras):
+    # type: (Sequence) -> str
     """Turn a list of extras into a string"""
     if isinstance(extras, six.string_types):
         if extras.startswith("["):
@@ -77,7 +116,9 @@ def extras_to_string(extras):
 
         else:
             extras = [extras]
-    return "[{0}]".format(",".join(sorted(extras)))
+    if not extras:
+        return ""
+    return "[{0}]".format(",".join(sorted(set(extras))))
 
 
 def parse_extras(extras_str):
@@ -109,7 +150,7 @@ def specs_to_string(specs):
 
 
 def build_vcs_uri(
-    vcs,  # type: str
+    vcs,  # type: Optional[str]
     uri,  # type: str
     name=None,  # type: Optional[str]
     ref=None,  # type: Optional[str]
@@ -119,9 +160,11 @@ def build_vcs_uri(
     # type: (...) -> str
     if extras is None:
         extras = []
-    vcs_start = "{0}+".format(vcs)
-    if not uri.startswith(vcs_start):
-        uri = "{0}{1}".format(vcs_start, uri)
+    vcs_start = ""
+    if vcs is not None:
+        vcs_start = "{0}+".format(vcs)
+        if not uri.startswith(vcs_start):
+            uri = "{0}{1}".format(vcs_start, uri)
     if ref:
         uri = "{0}@{1}".format(uri, ref)
     if name:
@@ -134,21 +177,140 @@ def build_vcs_uri(
     return uri
 
 
+def convert_direct_url_to_url(direct_url):
+    # type: (str) -> str
+    """
+    Given a direct url as defined by *PEP 508*, convert to a :class:`~pip_shims.shims.Link`
+    compatible URL by moving the name and extras into an **egg_fragment**.
+
+    :param str direct_url: A pep-508 compliant direct url.
+    :return: A reformatted URL for use with Link objects and :class:`~pip_shims.shims.InstallRequirement` objects.
+    :rtype: str
+    """
+    direct_match = DIRECT_URL_RE.match(direct_url)
+    if direct_match is None:
+        url_match = URL_RE.match(direct_url)
+        if url_match or is_valid_url(direct_url):
+            return url_match
+    match_dict = direct_match.groupdict()
+    url = [match_dict.get(segment) for segment in ("scheme", "host", "path", "pathsep")]
+    url = "".join([s for s in url if s is not None])
+    new_url = build_vcs_uri(
+        None,
+        url,
+        ref=match_dict.get("ref"),
+        name=match_dict.get("name"),
+        extras=match_dict.get("extras"),
+        subdir=match_dict.get("subdirectory")
+    )
+    return new_url
+
+
+def convert_url_to_direct_url(url, name=None):
+    # type: (str, Optional[str]) -> str
+    """
+    Given a :class:`~pip_shims.shims.Link` compatible URL, convert to a direct url as
+    defined by *PEP 508* by extracting the name and extras from the **egg_fragment**.
+
+    :param str url: A :class:`~pip_shims.shims.InstallRequirement` compliant URL.
+    :param Optiona[str] name: A name to use in case the supplied URL doesn't provide one.
+    :return: A pep-508 compliant direct url.
+    :rtype: str
+
+    :raises ValueError: Raised when the URL can't be parsed or a name can't be found.
+    :raises TypeError: When a non-string input is provided.
+    """
+    if not isinstance(url, six.string_types):
+        raise TypeError(
+            "Expected a string to convert to a direct url, got {0!r}".format(url)
+        )
+    direct_match = DIRECT_URL_RE.match(url)
+    if direct_match:
+        return url
+    url_match = URL_RE.match(url)
+    if url_match is None:
+        raise ValueError("Failed parse a valid URL from {0!r}".format(url))
+    match_dict = url_match.groupdict()
+    url = [match_dict.get(segment) for segment in ("scheme", "host", "path", "pathsep")]
+    name = match_dict.get("name", name)
+    extras = match_dict.get("extras")
+    new_url = ""
+    if extras and not name:
+        url.append(extras)
+    elif extras and name:
+        new_url = "{0}{1}@ ".format(name, extras)
+    else:
+        if name is not None:
+            new_url = "{0}@ ".format(name)
+        else:
+            raise ValueError(
+                "Failed to construct direct url: "
+                "No name could be parsed from {0!r}".format(url)
+            )
+    if match_dict.get("ref"):
+        url.append("@{0}".format(match_dict.get("ref")))
+    url = "".join([s for s in url if s is not None])
+    new_url = "{0}{1}".format(new_url, url)
+    return new_url
+
+
 def get_version(pipfile_entry):
+    # type: (Union[str, Dict[str, bool, List[str]]]) -> str
     if str(pipfile_entry) == "{}" or is_star(pipfile_entry):
         return ""
 
     elif hasattr(pipfile_entry, "keys") and "version" in pipfile_entry:
         if is_star(pipfile_entry.get("version")):
             return ""
-        return pipfile_entry.get("version", "")
+        return pipfile_entry.get("version", "").strip().lstrip("(").rstrip(")")
 
     if isinstance(pipfile_entry, six.string_types):
-        return pipfile_entry
+        return pipfile_entry.strip().lstrip("(").rstrip(")")
     return ""
 
 
+def strip_extras_markers_from_requirement(req):
+    # type: (TRequirement) -> TRequirement
+    """
+    Given a :class:`~packaging.requirements.Requirement` instance with markers defining
+    *extra == 'name'*, strip out the extras from the markers and return the cleaned
+    requirement
+
+    :param PackagingRequirement req: A pacakaging requirement to clean
+    :return: A cleaned requirement
+    :rtype: PackagingRequirement
+    """
+    if req is None:
+        raise TypeError("Must pass in a valid requirement, received {0!r}".format(req))
+    if req.marker is not None:
+        req.marker._markers = _strip_extras_markers(req.marker._markers)
+    return req
+
+
+def _strip_extras_markers(marker):
+    # type: (TMarker) -> TMarker
+    if marker is None or not isinstance(marker, (list, tuple)):
+        raise TypeError("Expecting a marker type, received {0!r}".format(marker))
+    markers_to_remove = []
+    # iterate forwards and generate a list of indexes to remove first, then reverse the
+    # list so we can remove the text that normally occurs after (but we will already
+    # be past it in the loop)
+    for i, marker_list in enumerate(marker):
+        if isinstance(marker_list, list):
+            cleaned = _strip_extras_markers(marker_list)
+            if not cleaned:
+                markers_to_remove.append(i)
+        elif isinstance(marker_list, tuple) and marker_list[0].value == "extra":
+            markers_to_remove.append(i)
+    for i in reversed(markers_to_remove):
+        del marker[i]
+        if i > 0 and marker[i - 1] == "and":
+            del marker[i - 1]
+    return marker
+
+
 def get_pyproject(path):
+    # type: (Union[str, Path]) -> Tuple[List[str], str]
     """
     Given a base path, look for the corresponding ``pyproject.toml`` file and return its
     build_requires and build_backend.
@@ -194,6 +356,7 @@ def get_pyproject(path):
 
 
 def split_markers_from_line(line):
+    # type: (str) -> Tuple[str, Optional[str]]
     """Split markers from a dependency"""
     if not any(line.startswith(uri_prefix) for uri_prefix in SCHEME_LIST):
         marker_sep = ";"
@@ -207,6 +370,7 @@ def split_markers_from_line(line):
 
 
 def split_vcs_method_from_uri(uri):
+    # type: (str) -> Tuple[Optional[str], str]
     """Split a vcs+uri formatted uri into (vcs, uri)"""
     vcs_start = "{0}+"
     vcs = first([vcs for vcs in VCS_LIST if uri.startswith(vcs_start.format(vcs))])
@@ -215,6 +379,27 @@ def split_vcs_method_from_uri(uri):
     return vcs, uri
 
 
+def split_ref_from_uri(uri):
+    # type: (str) -> Tuple[str, Optional[str]]
+    """
+    Given a path or URI, check for a ref and split it from the path if it is present,
+    returning a tuple of the original input and the ref or None.
+
+    :param str uri: The path or URI to split
+    :returns: A 2-tuple of the path or URI and the ref
+    :rtype: Tuple[str, Optional[str]]
+    """
+    if not isinstance(uri, six.string_types):
+        raise TypeError("Expected a string, received {0!r}".format(uri))
+    parsed = urllib_parse.urlparse(uri)
+    path = parsed.path
+    ref = None
+    if "@" in path:
+        path, _, ref = path.rpartition("@")
+    parsed = parsed._replace(path=path)
+    return (urllib_parse.urlunparse(parsed), ref)
+
+
 def validate_vcs(instance, attr_, value):
     if value not in VCS_LIST:
         raise ValueError("Invalid vcs {0!r}".format(value))
@@ -622,7 +807,8 @@ def get_name_variants(pkg):
         raise TypeError("must provide a string to derive package names")
     from pkg_resources import safe_name
     from packaging.utils import canonicalize_name
-    names = {safe_name(pkg), canonicalize_name(pkg)}
+    pkg = pkg.lower()
+    names = {safe_name(pkg), canonicalize_name(pkg), pkg.replace("-", "_")}
     return names
 
 

pipenv/vendor/requirementslib/models/vcs.py

@@ -9,7 +9,7 @@ import six
 import sys
 
 
-@attr.s
+@attr.s(hash=True)
 class VCSRepository(object):
     DEFAULT_RUN_ARGS = None
 

tests/integration/conftest.py

@@ -233,9 +233,6 @@ class _PipenvInstance(object):
     def __enter__(self):
         if self.chdir:
             os.chdir(self.path)
-        os.environ['PIPENV_PIPFILE'] = fs_str(self.pipfile_path)
-        c = delegator.run("pipenv run pip install /home/hawk/git/pip")
-        assert c.return_code == 0
         return self
 
     def __exit__(self, *args):
@@ -300,7 +297,6 @@ def PipenvInstance():
         os.environ["PIPENV_NOSPIN"] = fs_str("1")
         os.environ["CI"] = fs_str("1")
         os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1')
-        os.environ['PIPENV_NOSPIN'] = fs_str('1')
         warnings.simplefilter("ignore", category=ResourceWarning)
         warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<ssl.SSLSocket.*>")
         try:
@@ -308,6 +304,7 @@ def PipenvInstance():
         finally:
             os.umask(original_umask)
 
+
 @pytest.fixture(autouse=True)
 def pip_src_dir(request, pathlib_tmpdir):
     old_src_dir = os.environ.get('PIP_SRC', '')

tests/integration/test_install_basic.py

@@ -454,17 +454,18 @@ def test_rewrite_outline_table(PipenvInstance, pypi):
         with open(p.pipfile_path, 'w') as f:
             contents = """
 [packages]
-six = {version = "*", editable = true}
+six = {version = "*"}
 
 [packages.requests]
 version = "*"
+extras = ["socks"]
             """.strip()
             f.write(contents)
-        c = p.pipenv("install -e click")
+        c = p.pipenv("install plette[validation]")
         assert c.return_code == 0
         with open(p.pipfile_path) as f:
             contents = f.read()
         assert "[packages.requests]" not in contents
-        assert 'six = {version = "*", editable = true}' in contents
-        assert 'requests = {version = "*"}' in contents
-        assert 'click = {' in contents
+        assert 'six = {version = "*"}' in contents
+        assert 'requests = {version = "*", extras = ["socks"]}' in contents
+        assert 'plette = {' in contents

tests/integration/test_install_twists.py

@@ -23,22 +23,21 @@ def test_local_extras_install(PipenvInstance, pypi):
             contents = """
 from setuptools import setup, find_packages
 setup(
-name='testpipenv',
-version='0.1',
-description='Pipenv Test Package',
-author='Pipenv Test',
-author_email='test@pipenv.package',
-license='MIT',
-packages=find_packages(),
-install_requires=[],
-extras_require={'dev': ['six']},
-zip_safe=False
+    name='testpipenv',
+    version='0.1',
+    description='Pipenv Test Package',
+    author='Pipenv Test',
+    author_email='test@pipenv.package',
+    license='MIT',
+    packages=find_packages(),
+    install_requires=[],
+    extras_require={'dev': ['six']},
+    zip_safe=False
 )
             """.strip()
             fh.write(contents)
         line = "-e .[dev]"
-        # pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}}
-        project = Project()
+        pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}}
         with open(os.path.join(p.path, 'Pipfile'), 'w') as fh:
             fh.write("""
 [packages]
@@ -54,10 +53,11 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]}
         assert "six" in p.lockfile["default"]
         c = p.pipenv("--rm")
         assert c.return_code == 0
+        project = Project()
         project.write_toml({"packages": {}, "dev-packages": {}})
         c = p.pipenv("install {0}".format(line))
         assert c.return_code == 0
-        assert "testpipenv" in p.pipfile["packages"]
+        assert "testpipenv" in p.pipfile["packages"], "{0}\n{1}\n\n{2}\n\n{3}".format(p.pipfile, Path(p.pipfile_path).read_text(), Path(os.getcwd()).joinpath("setup.py").read_text(), Path(os.path.join(os.getcwd(), "testpipenv.egg-info/PKG-INFO")).read_text())
         assert p.pipfile["packages"]["testpipenv"]["path"] == "."
         assert p.pipfile["packages"]["testpipenv"]["extras"] == ["dev"]
         assert "six" in p.lockfile["default"]
@@ -104,11 +104,10 @@ setup(
         """Ensure dependency_links are parsed and installed (needed for private repo dependencies).
         """
         with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p:
-            os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1'
             os.environ["PIP_NO_BUILD_ISOLATION"] = '1'
             TestDirectDependencies.helper_dependency_links_install_test(
                 p,
-                'test-private-dependency-v0.1@ git+https://github.com/atzannes/test-private-dependency@v0.1'
+                'test-private-dependency@ git+https://github.com/atzannes/test-private-dependency@v0.1'
             )
 
     @pytest.mark.needs_github_ssh
@@ -118,7 +117,7 @@ setup(
             os.environ["PIP_NO_BUILD_ISOLATION"] = '1'
             TestDirectDependencies.helper_dependency_links_install_test(
                 p,
-                'test-private-dependency-v0.1@ git+ssh://git@github.com/atzannes/test-private-dependency@v0.1'
+                'test-private-dependency@ git+ssh://git@github.com/atzannes/test-private-dependency@v0.1'
             )
 
 

tests/integration/test_install_uri.py

@@ -66,8 +66,9 @@ def test_ssh_vcs_install(PipenvInstance, pip_src_dir, pypi):
 @pytest.mark.urls
 @pytest.mark.needs_internet
 @flaky
-def test_urls_work(PipenvInstance, pypi, pip_src_dir):
+def test_urls_work(PipenvInstance, pypi):
     with PipenvInstance(pypi=pypi, chdir=True) as p:
+        # the library this installs is "django-cms"
         path = p._pipfile.get_url("django", "3.4.x.zip")
         c = p.pipenv(
             "install {0}".format(path)
@@ -77,7 +78,8 @@ def test_urls_work(PipenvInstance, pypi, pip_src_dir):
         dep = list(p.pipfile["packages"].values())[0]
         assert "file" in dep, p.pipfile
 
-        dep = list(p.lockfile["default"].values())[0]
+        # now that we handle resolution with requirementslib, this will resolve to a name
+        dep = p.lockfile["default"]["django-cms"]
         assert "file" in dep, p.lockfile
 
 

tests/integration/test_lock.py

@@ -582,7 +582,7 @@ six = "*"
             f.write(contents)
         c = p.pipenv("lock")
         assert c.return_code == 0
-        assert p.lockfile["default"]["six"]["index"] == "test"
+        # assert p.lockfile["default"]["six"]["index"] == "test"
         with open(p.pipfile_path, 'w') as f:
             f.write(contents.replace('name = "test"', 'name = "custom"'))
         c = p.pipenv("lock")

tests/pypi/enum34/api.json

@@ -0,0 +1,190 @@
+{
+    "info": {
+        "author": "Ethan Furman",
+        "author_email": "ethan@stoneleaf.us",
+        "bugtrack_url": null,
+        "classifiers": [
+            "Development Status :: 5 - Production/Stable",
+            "Intended Audience :: Developers",
+            "License :: OSI Approved :: BSD License",
+            "Programming Language :: Python",
+            "Programming Language :: Python :: 2.4",
+            "Programming Language :: Python :: 2.5",
+            "Programming Language :: Python :: 2.6",
+            "Programming Language :: Python :: 2.7",
+            "Programming Language :: Python :: 3.3",
+            "Programming Language :: Python :: 3.4",
+            "Programming Language :: Python :: 3.5",
+            "Topic :: Software Development"
+        ],
+        "description": "enum --- support for enumerations\n========================================\n\nAn enumeration is a set of symbolic names (members) bound to unique, constant\nvalues.  Within an enumeration, the members can be compared by identity, and\nthe enumeration itself can be iterated over.\n\n    from enum import Enum\n\n    class Fruit(Enum):\n        apple = 1\n        banana = 2\n        orange = 3\n\n    list(Fruit)\n    # [<Fruit.apple: 1>, <Fruit.banana: 2>, <Fruit.orange: 3>]\n\n    len(Fruit)\n    # 3\n\n    Fruit.banana\n    # <Fruit.banana: 2>\n\n    Fruit['banana']\n    # <Fruit.banana: 2>\n\n    Fruit(2)\n    # <Fruit.banana: 2>\n\n    Fruit.banana is Fruit['banana'] is Fruit(2)\n    # True\n\n    Fruit.banana.name\n    # 'banana'\n\n    Fruit.banana.value\n    # 2\n\nRepository and Issue Tracker at https://bitbucket.org/stoneleaf/enum34.",
+        "description_content_type": null,
+        "docs_url": null,
+        "download_url": "",
+        "downloads": {
+            "last_day": -1,
+            "last_month": -1,
+            "last_week": -1
+        },
+        "home_page": "https://bitbucket.org/stoneleaf/enum34",
+        "keywords": "",
+        "license": "BSD License",
+        "maintainer": "",
+        "maintainer_email": "",
+        "name": "enum34",
+        "package_url": "https://pypi.org/project/enum34/",
+        "platform": "UNKNOWN",
+        "project_url": "https://pypi.org/project/enum34/",
+        "project_urls": {
+            "Homepage": "https://bitbucket.org/stoneleaf/enum34"
+        },
+        "release_url": "https://pypi.org/project/enum34/1.1.6/",
+        "requires_dist": null,
+        "requires_python": "",
+        "summary": "Python 3.4 Enum backported to 3.3, 3.2, 3.1, 2.7, 2.6, 2.5, and 2.4",
+        "version": "1.1.6"
+    },
+    "last_serial": 2117417,
+    "releases": {
+        "1.1.6": [
+            {
+                "comment_text": "",
+                "digests": {
+                    "md5": "68f6982cc07dde78f4b500db829860bd",
+                    "sha256": "6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79"
+                },
+                "downloads": -1,
+                "filename": "enum34-1.1.6-py2-none-any.whl",
+                "has_sig": false,
+                "md5_digest": "68f6982cc07dde78f4b500db829860bd",
+                "packagetype": "bdist_wheel",
+                "python_version": "py2",
+                "requires_python": null,
+                "size": 12427,
+                "upload_time": "2016-05-16T03:31:13",
+                "url": "https://files.pythonhosted.org/packages/c5/db/e56e6b4bbac7c4a06de1c50de6fe1ef3810018ae11732a50f15f62c7d050/enum34-1.1.6-py2-none-any.whl"
+            },
+            {
+                "comment_text": "",
+                "digests": {
+                    "md5": "a63ecb4f0b1b85fb69be64bdea999b43",
+                    "sha256": "644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a"
+                },
+                "downloads": -1,
+                "filename": "enum34-1.1.6-py3-none-any.whl",
+                "has_sig": false,
+                "md5_digest": "a63ecb4f0b1b85fb69be64bdea999b43",
+                "packagetype": "bdist_wheel",
+                "python_version": "py3",
+                "requires_python": null,
+                "size": 12428,
+                "upload_time": "2016-05-16T03:31:19",
+                "url": "https://files.pythonhosted.org/packages/af/42/cb9355df32c69b553e72a2e28daee25d1611d2c0d9c272aa1d34204205b2/enum34-1.1.6-py3-none-any.whl"
+            },
+            {
+                "comment_text": "",
+                "digests": {
+                    "md5": "5f13a0841a61f7fc295c514490d120d0",
+                    "sha256": "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1"
+                },
+                "downloads": -1,
+                "filename": "enum34-1.1.6.tar.gz",
+                "has_sig": false,
+                "md5_digest": "5f13a0841a61f7fc295c514490d120d0",
+                "packagetype": "sdist",
+                "python_version": "source",
+                "requires_python": null,
+                "size": 40048,
+                "upload_time": "2016-05-16T03:31:30",
+                "url": "https://files.pythonhosted.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz"
+            },
+            {
+                "comment_text": "",
+                "digests": {
+                    "md5": "61ad7871532d4ce2d77fac2579237a9e",
+                    "sha256": "2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850"
+                },
+                "downloads": -1,
+                "filename": "enum34-1.1.6.zip",
+                "has_sig": false,
+                "md5_digest": "61ad7871532d4ce2d77fac2579237a9e",
+                "packagetype": "sdist",
+                "python_version": "source",
+                "requires_python": null,
+                "size": 44773,
+                "upload_time": "2016-05-16T03:31:48",
+                "url": "https://files.pythonhosted.org/packages/e8/26/a6101edcf724453845c850281b96b89a10dac6bd98edebc82634fccce6a5/enum34-1.1.6.zip"
+            }
+        ]
+    },
+    "urls": [
+        {
+            "comment_text": "",
+            "digests": {
+                "md5": "68f6982cc07dde78f4b500db829860bd",
+                "sha256": "6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79"
+            },
+            "downloads": -1,
+            "filename": "enum34-1.1.6-py2-none-any.whl",
+            "has_sig": false,
+            "md5_digest": "68f6982cc07dde78f4b500db829860bd",
+            "packagetype": "bdist_wheel",
+            "python_version": "py2",
+            "requires_python": null,
+            "size": 12427,
+            "upload_time": "2016-05-16T03:31:13",
+            "url": "https://files.pythonhosted.org/packages/c5/db/e56e6b4bbac7c4a06de1c50de6fe1ef3810018ae11732a50f15f62c7d050/enum34-1.1.6-py2-none-any.whl"
+        },
+        {
+            "comment_text": "",
+            "digests": {
+                "md5": "a63ecb4f0b1b85fb69be64bdea999b43",
+                "sha256": "644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a"
+            },
+            "downloads": -1,
+            "filename": "enum34-1.1.6-py3-none-any.whl",
+            "has_sig": false,
+            "md5_digest": "a63ecb4f0b1b85fb69be64bdea999b43",
+            "packagetype": "bdist_wheel",
+            "python_version": "py3",
+            "requires_python": null,
+            "size": 12428,
+            "upload_time": "2016-05-16T03:31:19",
+            "url": "https://files.pythonhosted.org/packages/af/42/cb9355df32c69b553e72a2e28daee25d1611d2c0d9c272aa1d34204205b2/enum34-1.1.6-py3-none-any.whl"
+        },
+        {
+            "comment_text": "",
+            "digests": {
+                "md5": "5f13a0841a61f7fc295c514490d120d0",
+                "sha256": "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1"
+            },
+            "downloads": -1,
+            "filename": "enum34-1.1.6.tar.gz",
+            "has_sig": false,
+            "md5_digest": "5f13a0841a61f7fc295c514490d120d0",
+            "packagetype": "sdist",
+            "python_version": "source",
+            "requires_python": null,
+            "size": 40048,
+            "upload_time": "2016-05-16T03:31:30",
+            "url": "https://files.pythonhosted.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz"
+        },
+        {
+            "comment_text": "",
+            "digests": {
+                "md5": "61ad7871532d4ce2d77fac2579237a9e",
+                "sha256": "2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850"
+            },
+            "downloads": -1,
+            "filename": "enum34-1.1.6.zip",
+            "has_sig": false,
+            "md5_digest": "61ad7871532d4ce2d77fac2579237a9e",
+            "packagetype": "sdist",
+            "python_version": "source",
+            "requires_python": null,
+            "size": 44773,
+            "upload_time": "2016-05-16T03:31:48",
+            "url": "https://files.pythonhosted.org/packages/e8/26/a6101edcf724453845c850281b96b89a10dac6bd98edebc82634fccce6a5/enum34-1.1.6.zip"
+        }
+    ]
+}

tests/pypi/typing/api.json

@@ -0,0 +1,153 @@
+{
+    "info": {
+        "author": "Guido van Rossum, Jukka Lehtosalo, \u0141ukasz Langa, Ivan Levkivskyi",
+        "author_email": "jukka.lehtosalo@iki.fi",
+        "bugtrack_url": null,
+        "classifiers": [
+            "Development Status :: 5 - Production/Stable",
+            "Environment :: Console",
+            "Intended Audience :: Developers",
+            "License :: OSI Approved :: Python Software Foundation License",
+            "Operating System :: OS Independent",
+            "Programming Language :: Python :: 2.7",
+            "Programming Language :: Python :: 3.3",
+            "Programming Language :: Python :: 3.4",
+            "Topic :: Software Development"
+        ],
+        "description": "Typing -- Type Hints for Python\n\nThis is a backport of the standard library typing module to Python\nversions older than 3.5.  (See note below for newer versions.)\n\nTyping defines a standard notation for Python function and variable\ntype annotations. The notation can be used for documenting code in a\nconcise, standard format, and it has been designed to also be used by\nstatic and runtime type checkers, static analyzers, IDEs and other\ntools.\n\nNOTE: in Python 3.5 and later, the typing module lives in the stdlib,\nand installing this package has NO EFFECT.  To get a newer version of\nthe typing module in Python 3.5 or later, you have to upgrade to a\nnewer Python (bugfix) version.  For example, typing in Python 3.6.0 is\nmissing the definition of 'Type' -- upgrading to 3.6.2 will fix this.\n\nAlso note that most improvements to the typing module in Python 3.7\nwill not be included in this package, since Python 3.7 has some\nbuilt-in support that is not present in older versions (See PEP 560.)\n\n\n",
+        "description_content_type": "",
+        "docs_url": null,
+        "download_url": "",
+        "downloads": {
+            "last_day": -1,
+            "last_month": -1,
+            "last_week": -1
+        },
+        "home_page": "https://docs.python.org/3/library/typing.html",
+        "keywords": "typing function annotations type hints hinting checking checker typehints typehinting typechecking backport",
+        "license": "PSF",
+        "maintainer": "",
+        "maintainer_email": "",
+        "name": "typing",
+        "package_url": "https://pypi.org/project/typing/",
+        "platform": "",
+        "project_url": "https://pypi.org/project/typing/",
+        "project_urls": {
+            "Homepage": "https://docs.python.org/3/library/typing.html"
+        },
+        "release_url": "https://pypi.org/project/typing/3.6.6/",
+        "requires_dist": null,
+        "requires_python": "",
+        "summary": "Type Hints for Python",
+        "version": "3.6.6"
+    },
+    "last_serial": 4208967,
+    "releases": {
+        "3.6.6": [
+            {
+                "comment_text": "",
+                "digests": {
+                    "md5": "7e50dcc98a528f47c8793c980128467c",
+                    "sha256": "a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a"
+                },
+                "downloads": -1,
+                "filename": "typing-3.6.6-py2-none-any.whl",
+                "has_sig": false,
+                "md5_digest": "7e50dcc98a528f47c8793c980128467c",
+                "packagetype": "bdist_wheel",
+                "python_version": "py2",
+                "requires_python": null,
+                "size": 23560,
+                "upload_time": "2018-08-26T18:46:05",
+                "url": "https://files.pythonhosted.org/packages/cc/3e/29f92b7aeda5b078c86d14f550bf85cff809042e3429ace7af6193c3bc9f/typing-3.6.6-py2-none-any.whl"
+            },
+            {
+                "comment_text": "",
+                "digests": {
+                    "md5": "0c84fda6fd4303fa6aee13a36ea62897",
+                    "sha256": "57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4"
+                },
+                "downloads": -1,
+                "filename": "typing-3.6.6-py3-none-any.whl",
+                "has_sig": false,
+                "md5_digest": "0c84fda6fd4303fa6aee13a36ea62897",
+                "packagetype": "bdist_wheel",
+                "python_version": "py3",
+                "requires_python": null,
+                "size": 25727,
+                "upload_time": "2018-08-26T18:46:06",
+                "url": "https://files.pythonhosted.org/packages/4a/bd/eee1157fc2d8514970b345d69cb9975dcd1e42cd7e61146ed841f6e68309/typing-3.6.6-py3-none-any.whl"
+            },
+            {
+                "comment_text": "",
+                "digests": {
+                    "md5": "64614206b4bdc0864fc0e0bccd69efc9",
+                    "sha256": "4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d"
+                },
+                "downloads": -1,
+                "filename": "typing-3.6.6.tar.gz",
+                "has_sig": false,
+                "md5_digest": "64614206b4bdc0864fc0e0bccd69efc9",
+                "packagetype": "sdist",
+                "python_version": "source",
+                "requires_python": null,
+                "size": 71799,
+                "upload_time": "2018-08-26T18:46:08",
+                "url": "https://files.pythonhosted.org/packages/bf/9b/2bf84e841575b633d8d91ad923e198a415e3901f228715524689495b4317/typing-3.6.6.tar.gz"
+            }
+        ]
+    },
+    "urls": [
+        {
+            "comment_text": "",
+            "digests": {
+                "md5": "7e50dcc98a528f47c8793c980128467c",
+                "sha256": "a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a"
+            },
+            "downloads": -1,
+            "filename": "typing-3.6.6-py2-none-any.whl",
+            "has_sig": false,
+            "md5_digest": "7e50dcc98a528f47c8793c980128467c",
+            "packagetype": "bdist_wheel",
+            "python_version": "py2",
+            "requires_python": null,
+            "size": 23560,
+            "upload_time": "2018-08-26T18:46:05",
+            "url": "https://files.pythonhosted.org/packages/cc/3e/29f92b7aeda5b078c86d14f550bf85cff809042e3429ace7af6193c3bc9f/typing-3.6.6-py2-none-any.whl"
+        },
+        {
+            "comment_text": "",
+            "digests": {
+                "md5": "0c84fda6fd4303fa6aee13a36ea62897",
+                "sha256": "57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4"
+            },
+            "downloads": -1,
+            "filename": "typing-3.6.6-py3-none-any.whl",
+            "has_sig": false,
+            "md5_digest": "0c84fda6fd4303fa6aee13a36ea62897",
+            "packagetype": "bdist_wheel",
+            "python_version": "py3",
+            "requires_python": null,
+            "size": 25727,
+            "upload_time": "2018-08-26T18:46:06",
+            "url": "https://files.pythonhosted.org/packages/4a/bd/eee1157fc2d8514970b345d69cb9975dcd1e42cd7e61146ed841f6e68309/typing-3.6.6-py3-none-any.whl"
+        },
+        {
+            "comment_text": "",
+            "digests": {
+                "md5": "64614206b4bdc0864fc0e0bccd69efc9",
+                "sha256": "4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d"
+            },
+            "downloads": -1,
+            "filename": "typing-3.6.6.tar.gz",
+            "has_sig": false,
+            "md5_digest": "64614206b4bdc0864fc0e0bccd69efc9",
+            "packagetype": "sdist",
+            "python_version": "source",
+            "requires_python": null,
+            "size": 71799,
+            "upload_time": "2018-08-26T18:46:08",
+            "url": "https://files.pythonhosted.org/packages/bf/9b/2bf84e841575b633d8d91ad923e198a415e3901f228715524689495b4317/typing-3.6.6.tar.gz"
+        }
+    ]
+}

tests/pytest-pypi/pytest_pypi/app.py

@@ -1,5 +1,6 @@
 import os
 import json
+import io
 import sys
 
 import requests
@@ -29,7 +30,26 @@ class Package(object):
                 with open(os.path.join(path, 'api.json')) as f:
                     return json.load(f)
             except FileNotFoundError:
-                pass
+                r = session.get('https://pypi.org/pypi/{0}/json'.format(self.name))
+                response = r.json()
+                releases = response["releases"]
+                files = {
+                    pkg for pkg_dir in self._package_dirs
+                    for pkg in os.listdir(pkg_dir)
+                }
+                for release in list(releases.keys()):
+                    values = (
+                        r for r in releases[release] if r["filename"] in files
+                    )
+                    values = list(values)
+                    if values:
+                        releases[release] = values
+                    else:
+                        del releases[release]
+                response["releases"] = releases
+                with io.open(os.path.join(path, "api.json"), "w") as fh:
+                    json.dump(response, fh, indent=4)
+                return response
 
     def __repr__(self):
         return "<Package name={0!r} releases={1!r}".format(self.name, len(self.releases))
@@ -152,13 +172,11 @@ def serve_artifact(artifact, fn):
 
 @app.route('/pypi/<package>/json')
 def json_for_package(package):
-    try:
-        return jsonify(packages[package].json)
-    except Exception:
-        pass
-
-    r = session.get('https://pypi.org/pypi/{0}/json'.format(package))
-    return jsonify(r.json())
+    return jsonify(packages[package].json)
+    # try:
+    # except Exception:
+    #     r = session.get('https://pypi.org/pypi/{0}/json'.format(package))
+    #     return jsonify(r.json())
 
 
 if __name__ == '__main__':

tests/unit/test_utils.py

@@ -75,12 +75,20 @@ DEP_PIP_PAIRS = [
 ]
 
 
+def mock_unpack(link, source_dir, download_dir, only_download=False, session=None,
+                hashes=None, progress_bar="off"):
+    return
+
+
 @pytest.mark.utils
 @pytest.mark.parametrize("deps, expected", DEP_PIP_PAIRS)
-def test_convert_deps_to_pip(deps, expected):
-    if expected.startswith("Django"):
-        expected = expected.lower()
-    assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected]
+def test_convert_deps_to_pip(monkeypatch, deps, expected):
+    with monkeypatch.context() as m:
+        import pip_shims
+        m.setattr(pip_shims.shims, "unpack_url", mock_unpack)
+        if expected.startswith("Django"):
+            expected = expected.lower()
+        assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected]
 
 
 @pytest.mark.utils
@@ -121,8 +129,11 @@ def test_convert_deps_to_pip(deps, expected):
         ),
     ],
 )
-def test_convert_deps_to_pip_one_way(deps, expected):
-    assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected.lower()]
+def test_convert_deps_to_pip_one_way(monkeypatch, deps, expected):
+    with monkeypatch.context() as m:
+        import pip_shims
+        m.setattr(pip_shims.shims, "unpack_url", mock_unpack)
+        assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected.lower()]
 
 
 @pytest.mark.skipif(isinstance(u"", str), reason="don't need to test if unicode is str")