Only resolve requirements with markers we can evaluate

Signed-off-by: Dan Ryan <dan@danryan.co>

news/3298.bugfix.rst

@@ -0,0 +1,3 @@
+- Fixed a bug which caused failures in warning reporting when running pipenv inside a virtualenv under some circumstances.
+
+Fixed a bug with package discovery when running ``pipenv clean``.

news/3298.feature.rst

@@ -0,0 +1,5 @@
+- Added full support for resolution of all dependency types including direct URLs, zip archives, tarballs, etc.
+
+Improved error handling and formatting.
+
+Introduced improved cross platform stream wrappers for better ``stdout`` and ``stderr`` consistency.

news/3298.vendor.rst

@@ -0,0 +1,23 @@
+- Updated vendored dependencies:
+
+  - **certifi**: ``2018.10.15`` => ``2018.11.29``
+  - **cached_property**: ``1.4.3`` => ``1.5.1``
+  - **colorama**: ``0.3.9`` => ``0.4.1``
+  - **idna**: ``2.7`` => ``2.8``
+  - **packaging**: ``18.0`` => ``19.0``
+  - **pathlib2**: ``2.3.2`` => ``2.3.3``
+  - **pep517**: ``(new)`` => ``0.5.0``
+  - **pipdeptree**: ``0.13.0`` => ``0.13.1``
+  - **pyparsing**: ``2.2.2`` => ``2.3.1``
+  - **python-dotenv**: ``0.9.1`` => ``0.10.1``
+  - **pythonfinder**: ``1.1.10`` => ``1.2.0``
+  - **pytoml**: ``(new)`` => ``0.1.20``
+  - **requests**: ``2.20.1`` => ``2.21.0``
+  - **requirementslib**: ``1.3.3`` => ``1.4.0``
+  - **shellingham**: ``1.2.7`` => ``1.2.8``
+  - **six**: ``1.11.0`` => ``1.12.0``
+  - **tomlkit**: ``0.5.2`` => ``0.5.3``
+  - **urllib3**: ``1.24`` => ``1.24.1``
+  - **vistir**: ``0.3.0`` => ``0.3.1``
+
+Removed vendored dependency **cursor**.

pipenv/utils.py

@@ -330,6 +330,7 @@ class Resolver(object):
     ):
         # type: (...) -> Tuple[Requirement, Dict[str, str], Dict[str, str]]
         from .vendor.requirementslib.models.requirements import Requirement
+        from .exceptions import ResolutionFailure
         if index_lookup is None:
             index_lookup = {}
         if markers_lookup is None:
@@ -342,7 +343,10 @@ class Resolver(object):
         if indexes:
             url = indexes[0]
         line = " ".join(remainder)
-        req = Requirement.from_line(line)
+        try:
+            req = Requirement.from_line(line)
+        except ValueError:
+            raise ResolutionFailure("Failed to resolve requirement from line: {0!s}".format(line))
         if url:
             index_lookup[req.normalized_name] = project.get_source(
                 url=url, refresh=True).get("name")
@@ -386,7 +390,14 @@ class Resolver(object):
                             continue
                         line = _requirement_to_str_lowercase_name(r)
                         new_req, _, _ = cls.parse_line(line)
-                    new_constraints, new_lock = cls.get_deps_from_req(new_req)
+                    if r.marker and not r.marker.evaluate():
+                        new_constraints = {}
+                        _, new_entry = req.pipfile_entry
+                        new_lock = {
+                            pep_423_name(new_req.normalized_name): new_entry
+                        }
+                    else:
+                        new_constraints, new_lock = cls.get_deps_from_req(new_req)
                     locked_deps.update(new_lock)
                     constraints |= new_constraints
                 else:

pipenv/vendor/requirementslib/models/setup_info.py

@@ -19,22 +19,21 @@ from distlib.wheel import Wheel
 from packaging.markers import Marker
 from six.moves import configparser
 from six.moves.urllib.parse import unquote, urlparse, urlunparse
-
 from vistir.compat import Iterable, Path, lru_cache
 from vistir.contextmanagers import cd, temp_path
 from vistir.misc import run
 from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p, rmtree
 
-from ..environment import MYPY_RUNNING
-from ..exceptions import RequirementError
 from .utils import (
+    get_default_pyproject_backend,
     get_name_variants,
     get_pyproject,
     init_requirement,
     split_vcs_method_from_uri,
     strip_extras_markers_from_requirement,
-    get_default_pyproject_backend
 )
+from ..environment import MYPY_RUNNING
+from ..exceptions import RequirementError
 
 try:
     from setuptools.dist import distutils
@@ -49,15 +48,32 @@ except ImportError:
 
 
 if MYPY_RUNNING:
-    from typing import Any, Dict, List, Generator, Optional, Union, Tuple, TypeVar, Text, Set, AnyStr
+    from typing import (
+        Any,
+        Dict,
+        List,
+        Generator,
+        Optional,
+        Union,
+        Tuple,
+        TypeVar,
+        Text,
+        Set,
+        AnyStr,
+    )
     from pip_shims.shims import InstallRequirement, PackageFinder
     from pkg_resources import (
-        PathMetadata, DistInfoDistribution, Requirement as PkgResourcesRequirement
+        PathMetadata,
+        DistInfoDistribution,
+        Requirement as PkgResourcesRequirement,
     )
     from packaging.requirements import Requirement as PackagingRequirement
+
     TRequirement = TypeVar("TRequirement")
-    RequirementType = TypeVar('RequirementType', covariant=True, bound=PackagingRequirement)
-    MarkerType = TypeVar('MarkerType', covariant=True, bound=Marker)
+    RequirementType = TypeVar(
+        "RequirementType", covariant=True, bound=PackagingRequirement
+    )
+    MarkerType = TypeVar("MarkerType", covariant=True, bound=Marker)
     STRING_TYPE = Union[str, bytes, Text]
     S = TypeVar("S", bytes, str, Text)
 
@@ -77,16 +93,37 @@ def pep517_subprocess_runner(cmd, cwd=None, extra_environ=None):
     if extra_environ:
         env.update(extra_environ)
 
-    run(cmd, cwd=cwd, env=env, block=True, combine_stderr=True, return_object=False,
-        write_to_stdout=False, nospin=True)
+    run(
+        cmd,
+        cwd=cwd,
+        env=env,
+        block=True,
+        combine_stderr=True,
+        return_object=False,
+        write_to_stdout=False,
+        nospin=True,
+    )
 
 
 class BuildEnv(pep517.envbuild.BuildEnvironment):
     def pip_install(self, reqs):
-        cmd = [sys.executable, '-m', 'pip', 'install', '--ignore-installed', '--prefix',
-               self.path] + list(reqs)
-        run(cmd, block=True, combine_stderr=True, return_object=False,
-            write_to_stdout=False, nospin=True)
+        cmd = [
+            sys.executable,
+            "-m",
+            "pip",
+            "install",
+            "--ignore-installed",
+            "--prefix",
+            self.path,
+        ] + list(reqs)
+        run(
+            cmd,
+            block=True,
+            combine_stderr=True,
+            return_object=False,
+            write_to_stdout=False,
+            nospin=True,
+        )
 
 
 class HookCaller(pep517.wrappers.Pep517HookCaller):
@@ -156,6 +193,7 @@ def _get_src_dir(root):
 def ensure_reqs(reqs):
     # type: (List[Union[S, PkgResourcesRequirement]]) -> List[PkgResourcesRequirement]
     import pkg_resources
+
     if not isinstance(reqs, Iterable):
         raise TypeError("Expecting an Iterable, got %r" % reqs)
     new_reqs = []
@@ -169,7 +207,9 @@ def ensure_reqs(reqs):
     return new_reqs
 
 
-def _prepare_wheel_building_kwargs(ireq=None, src_root=None, src_dir=None, editable=False):
+def _prepare_wheel_building_kwargs(
+    ireq=None, src_root=None, src_dir=None, editable=False
+):
     # type: (Optional[InstallRequirement], Optional[AnyStr], Optional[AnyStr], bool) -> Dict[AnyStr, AnyStr]
     download_dir = os.path.join(CACHE_DIR, "pkgs")  # type: STRING_TYPE
     mkdir_p(download_dir)
@@ -214,14 +254,17 @@ def iter_metadata(path, pkg_name=None, metadata_type="egg-info"):
             elif not entry.name.endswith(metadata_type):
                 non_matching_dirs.append(entry)
     for entry in non_matching_dirs:
-        for dir_entry in iter_metadata(entry.path, pkg_name=pkg_name, metadata_type=metadata_type):
+        for dir_entry in iter_metadata(
+            entry.path, pkg_name=pkg_name, metadata_type=metadata_type
+        ):
             yield dir_entry
 
 
 def find_egginfo(target, pkg_name=None):
     # type: (AnyStr, Optional[AnyStr]) -> Generator
     egg_dirs = (
-        egg_dir for egg_dir in iter_metadata(target, pkg_name=pkg_name)
+        egg_dir
+        for egg_dir in iter_metadata(target, pkg_name=pkg_name)
         if egg_dir is not None
     )
     if pkg_name:
@@ -234,7 +277,10 @@ def find_egginfo(target, pkg_name=None):
 def find_distinfo(target, pkg_name=None):
     # type: (AnyStr, Optional[AnyStr]) -> Generator
     dist_dirs = (
-        dist_dir for dist_dir in iter_metadata(target, pkg_name=pkg_name, metadata_type="dist-info")
+        dist_dir
+        for dist_dir in iter_metadata(
+            target, pkg_name=pkg_name, metadata_type="dist-info"
+        )
         if dist_dir is not None
     )
     if pkg_name:
@@ -260,6 +306,7 @@ def get_metadata(path, pkg_name=None, metadata_type=None):
     base_dir = None
     if matched_dir is not None:
         import pkg_resources
+
         metadata_dir = os.path.abspath(matched_dir.path)
         base_dir = os.path.dirname(metadata_dir)
         dist = None
@@ -320,12 +367,7 @@ def get_metadata_from_wheel(wheel_path):
             extras[extra].append(parsed_req)
         else:
             requires.append(parsed_req)
-    return {
-        "name": name,
-        "version": version,
-        "requires": requires,
-        "extras": extras
-    }
+    return {"name": name, "version": version, "requires": requires, "extras": extras}
 
 
 def get_metadata_from_dist(dist):
@@ -362,14 +404,16 @@ def get_metadata_from_dist(dist):
         "name": dist.project_name,
         "version": dist.version,
         "requires": requires,
-        "extras": extras
+        "extras": extras,
     }
 
 
 @attr.s(slots=True, frozen=True)
 class BaseRequirement(object):
     name = attr.ib(default="", cmp=True)  # type: STRING_TYPE
-    requirement = attr.ib(default=None, cmp=True)  # type: Optional[PkgResourcesRequirement]
+    requirement = attr.ib(
+        default=None, cmp=True
+    )  # type: Optional[PkgResourcesRequirement]
 
     def __str__(self):
         # type: () -> S
@@ -413,12 +457,16 @@ class Extra(object):
 
     def __str__(self):
         # type: () -> S
-        return "{0}: {{{1}}}".format(self.section, ", ".join([r.name for r in self.requirements]))
+        return "{0}: {{{1}}}".format(
+            self.section, ", ".join([r.name for r in self.requirements])
+        )
 
     def add(self, req):
         # type: (BaseRequirement) -> None
         if req not in self.requirements:
-            return attr.evolve(self, requirements=frozenset(set(self.requirements).add(req)))
+            return attr.evolve(
+                self, requirements=frozenset(set(self.requirements).add(req))
+            )
         return self
 
     def as_dict(self):
@@ -435,12 +483,16 @@ class SetupInfo(object):
     build_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True)
     build_backend = attr.ib(cmp=True)  # type: STRING_TYPE
     setup_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True)
-    python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None, cmp=True)
+    python_requires = attr.ib(
+        type=packaging.specifiers.SpecifierSet, default=None, cmp=True
+    )
     _extras_requirements = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True)
     setup_cfg = attr.ib(type=Path, default=None, cmp=True, hash=False)
     setup_py = attr.ib(type=Path, default=None, cmp=True, hash=False)
     pyproject = attr.ib(type=Path, default=None, cmp=True, hash=False)
-    ireq = attr.ib(default=None, cmp=True, hash=False)  # type: Optional[InstallRequirement]
+    ireq = attr.ib(
+        default=None, cmp=True, hash=False
+    )  # type: Optional[InstallRequirement]
     extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict, cmp=False, hash=False)
     metadata = attr.ib(default=None)  # type: Optional[Tuple[STRING_TYPE]]
 
@@ -452,7 +504,9 @@ class SetupInfo(object):
     @property
     def requires(self):
         # type: () -> Dict[S, RequirementType]
-        return {req.name: req.requirement for req in self._requirements}
+        return {
+            req.name: req.requirement for req in self._requirements
+        }
 
     @property
     def extras(self):
@@ -489,30 +543,34 @@ class SetupInfo(object):
                 results["version"] = parser.get("metadata", "version")
             install_requires = set()  # type: Set[BaseRequirement]
             if parser.has_option("options", "install_requires"):
-                install_requires = set([
-                    BaseRequirement.from_string(dep)
-                    for dep in parser.get("options", "install_requires").split("\n")
-                    if dep
-                ])
+                install_requires = set(
+                    [
+                        BaseRequirement.from_string(dep)
+                        for dep in parser.get("options", "install_requires").split("\n")
+                        if dep
+                    ]
+                )
             results["install_requires"] = install_requires
             if parser.has_option("options", "python_requires"):
                 results["python_requires"] = parser.get("options", "python_requires")
             if parser.has_option("options", "build_requires"):
                 results["build_requires"] = parser.get("options", "build_requires")
-            extras_require = ()
             if "options.extras_require" in parser.sections():
-                extras_require = tuple([
-                    (section, tuple([
-                        BaseRequirement.from_string(dep)
-                        for dep in parser.get(
-                            "options.extras_require", section
-                        ).split("\n")
-                        if dep
-                    ]))
-                    for section in parser.options("options.extras_require")
-                    if section not in ["options", "metadata"]
-                ])
-            results["extras_require"] = extras_require
+                extras_require_section = parser.options("options.extras_require")
+                extras = []
+                for section in extras_require_section:
+                    if section in ["options", "metadata"]:
+                        continue
+                    section_contents = parser.get("options.extras_require", section)
+                    section_list = section_contents.split("\n")
+                    section_extras = []
+                    for extra_name in section_list:
+                        if not extra_name or extra_name.startswith("#"):
+                            continue
+                        section_extras.append(BaseRequirement.from_string(extra_name))
+                    if section_extras:
+                        extras.append(tuple([section, tuple(section_extras)]))
+            results["extras_require"] = tuple(extras)
             return results
 
     @property
@@ -545,20 +603,26 @@ class SetupInfo(object):
                 self.version = parsed.get("version")
             build_requires = parsed.get("build_requires", [])
             if self.build_requires:
-                self.build_requires = tuple(set(self.build_requires) | set(build_requires))
+                self.build_requires = tuple(
+                    set(self.build_requires) | set(build_requires)
+                )
             self._requirements = frozenset(
                 set(self._requirements) | set(parsed["install_requires"])
             )
             if self.python_requires is None:
                 self.python_requires = parsed.get("python_requires")
             if not self._extras_requirements:
-                self._extras_requirements = (parsed["extras_require"])
+                self._extras_requirements = parsed["extras_require"]
             else:
-                self._extras_requirements = self._extras_requirements + parsed["extras_require"]
+                self._extras_requirements = (
+                    self._extras_requirements + parsed["extras_require"]
+                )
             if self.ireq is not None and self.ireq.extras:
                 for extra in self.ireq.extras:
                     if extra in self.extras:
-                        extras_tuple = tuple([BaseRequirement.from_req(req) for req in self.extras[extra]])
+                        extras_tuple = tuple(
+                            [BaseRequirement.from_req(req) for req in self.extras[extra]]
+                        )
                         self._extras_requirements += ((extra, extras_tuple),)
                         self._requirements = frozenset(
                             set(self._requirements) | set(list(extras_tuple))
@@ -585,16 +649,22 @@ class SetupInfo(object):
                     _setup_stop_after = "run"
                     sys.argv[0] = script_name
                     sys.argv[1:] = args
-                    with open(script_name, 'rb') as f:
+                    with open(script_name, "rb") as f:
                         if sys.version_info < (3, 5):
                             exec(f.read(), g, local_dict)
                         else:
                             exec(f.read(), g)
                 # We couldn't import everything needed to run setup
                 except NameError:
-                    python = os.environ.get('PIP_PYTHON_PATH', sys.executable)
-                    out, _ = run([python, "setup.py"] + args, cwd=target_cwd, block=True,
-                                 combine_stderr=False, return_object=False, nospin=True)
+                    python = os.environ.get("PIP_PYTHON_PATH", sys.executable)
+                    out, _ = run(
+                        [python, "setup.py"] + args,
+                        cwd=target_cwd,
+                        block=True,
+                        combine_stderr=False,
+                        return_object=False,
+                        nospin=True,
+                    )
                 finally:
                     _setup_stop_after = None
                     sys.argv = save_argv
@@ -622,15 +692,13 @@ class SetupInfo(object):
                 if not install_requires:
                     install_requires = dist.install_requires
                 if install_requires and not self.requires:
-                    requirements = set([
-                        BaseRequirement.from_req(req) for req in install_requires
-                    ])
+                    requirements = set(
+                        [BaseRequirement.from_req(req) for req in install_requires]
+                    )
                     if getattr(self.ireq, "extras", None):
                         for extra in self.ireq.extras:
                             requirements |= set(list(self.extras.get(extra, [])))
-                    self._requirements = frozenset(
-                        set(self._requirements) | requirements
-                    )
+                    self._requirements = frozenset(set(self._requirements) | requirements)
                 if dist.setup_requires and not self.setup_requires:
                     self.setup_requires = tuple(dist.setup_requires)
                 if not self.version:
@@ -647,15 +715,20 @@ class SetupInfo(object):
         # type: () -> S
         if not self.pyproject.exists():
             build_requires = ", ".join(['"{0}"'.format(r) for r in self.build_requires])
-            self.pyproject.write_text(u"""
+            self.pyproject.write_text(
+                u"""
 [build-system]
 requires = [{0}]
 build-backend = "{1}"
-            """.format(build_requires, self.build_backend).strip())
+            """.format(
+                    build_requires, self.build_backend
+                ).strip()
+            )
         return build_pep517(
-            self.base_dir, self.extra_kwargs["build_dir"],
+            self.base_dir,
+            self.extra_kwargs["build_dir"],
             config_settings=self.pep517_config,
-            dist_type="wheel"
+            dist_type="wheel",
         )
 
     # noinspection PyPackageRequirements
@@ -663,15 +736,20 @@ build-backend = "{1}"
         # type: () -> S
         if not self.pyproject.exists():
             build_requires = ", ".join(['"{0}"'.format(r) for r in self.build_requires])
-            self.pyproject.write_text(u"""
+            self.pyproject.write_text(
+                u"""
 [build-system]
 requires = [{0}]
 build-backend = "{1}"
-            """.format(build_requires, self.build_backend).strip())
+            """.format(
+                    build_requires, self.build_backend
+                ).strip()
+            )
         return build_pep517(
-            self.base_dir, self.extra_kwargs["build_dir"],
+            self.base_dir,
+            self.extra_kwargs["build_dir"],
             config_settings=self.pep517_config,
-            dist_type="sdist"
+            dist_type="sdist",
         )
 
     def build(self):
@@ -719,12 +797,17 @@ build-backend = "{1}"
         # if self.setup_py is not None and self.setup_py.exists():
         metadata_dirs = []
         if any([fn is not None and fn.exists() for fn in package_indicators]):
-            metadata_dirs = [self.extra_kwargs["build_dir"], self.egg_base, self.extra_kwargs["src_dir"]]
+            metadata_dirs = [
+                self.extra_kwargs["build_dir"],
+                self.egg_base,
+                self.extra_kwargs["src_dir"],
+            ]
         if metadata_dir is not None:
             metadata_dirs = [metadata_dir] + metadata_dirs
         metadata = [
             get_metadata(d, pkg_name=self.name, metadata_type=metadata_type)
-            for d in metadata_dirs if os.path.exists(d)
+            for d in metadata_dirs
+            if os.path.exists(d)
         ]
         metadata = next(iter(d for d in metadata if d), None)
         if metadata is not None:
@@ -749,20 +832,20 @@ build-backend = "{1}"
         if not self.version:
             self.version = metadata.get("version", self.version)
         self._requirements = frozenset(
-            set(self._requirements) | set([
-                BaseRequirement.from_req(req)
-                for req in metadata.get("requires", [])
-            ])
+            set(self._requirements)
+            | set([BaseRequirement.from_req(req) for req in metadata.get("requires", [])])
         )
         if getattr(self.ireq, "extras", None):
             for extra in self.ireq.extras:
                 extras = metadata.get("extras", {}).get(extra, [])
                 if extras:
-                    extras_tuple = tuple([
-                        BaseRequirement.from_req(req)
-                        for req in ensure_reqs(tuple(extras))
-                        if req is not None
-                    ])
+                    extras_tuple = tuple(
+                        [
+                            BaseRequirement.from_req(req)
+                            for req in ensure_reqs(tuple(extras))
+                            if req is not None
+                        ]
+                    )
                     self._extras_requirements += ((extra, extras_tuple),)
                     self._requirements = frozenset(
                         set(self._requirements) | set(extras_tuple)
@@ -839,6 +922,7 @@ build-backend = "{1}"
     def from_ireq(cls, ireq, subdir=None, finder=None):
         # type: (InstallRequirement, Optional[AnyStr], Optional[PackageFinder]) -> Optional[SetupInfo]
         import pip_shims.shims
+
         if not ireq.link:
             return
         if ireq.link.is_wheel:
@@ -875,8 +959,7 @@ build-backend = "{1}"
                 download_dir = kwargs["download_dir"]
         elif path is not None and os.path.isdir(path):
             raise RequirementError(
-                "The file URL points to a directory not installable: {}"
-                .format(ireq.link)
+                "The file URL points to a directory not installable: {}".format(ireq.link)
             )
         ireq.build_location(kwargs["build_dir"])
         src_dir = ireq.ensure_has_source_dir(kwargs["src_dir"])
@@ -892,9 +975,7 @@ build-backend = "{1}"
             hashes=ireq.hashes(False),
             progress_bar="off",
         )
-        created = cls.create(
-            src_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs
-        )
+        created = cls.create(src_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs)
         return created
 
     @classmethod