Requirementslib fixes

- Fixes #2344

Signed-off-by: Dan Ryan <dan@danryan.co>

pipenv/vendor/requirementslib/_compat.py

@@ -17,7 +17,10 @@ if six.PY2:
 
     class FileNotFoundError(IOError):
         pass
+
+
 else:
+
     class FileNotFoundError(FileNotFoundError):
         pass
 
@@ -57,5 +60,7 @@ get_installed_distributions = do_import(
 is_installable_file = do_import("utils.misc", "is_installable_file", old_path="utils")
 is_installable_dir = do_import("utils.misc", "is_installable_dir", old_path="utils")
 PyPI = do_import("models.index", "PyPI")
-make_abstract_dist = do_import("operations.prepare", "make_abstract_dist", old_path="req.req_set")
-VcsSupport = do_import('vcs', 'VcsSupport')
+make_abstract_dist = do_import(
+    "operations.prepare", "make_abstract_dist", old_path="req.req_set"
+)
+VcsSupport = do_import("vcs", "VcsSupport")

pipenv/vendor/requirementslib/models/lockfile.py

@@ -3,9 +3,7 @@ from __future__ import absolute_import
 import attr
 import json
 from .requirements import Requirement
-from .utils import (
-    optional_instance_of,
-)
+from .utils import optional_instance_of
 from .._compat import Path, FileNotFoundError
 
 

pipenv/vendor/requirementslib/models/markers.py

@@ -11,8 +11,12 @@ from ..exceptions import RequirementError
 class PipenvMarkers(BaseRequirement):
     """System-level requirements - see PEP508 for more detail"""
 
-    os_name = attr.ib(default=None, validator=attr.validators.optional(validate_markers))
-    sys_platform = attr.ib(default=None, validator=attr.validators.optional(validate_markers))
+    os_name = attr.ib(
+        default=None, validator=attr.validators.optional(validate_markers)
+    )
+    sys_platform = attr.ib(
+        default=None, validator=attr.validators.optional(validate_markers)
+    )
     platform_machine = attr.ib(
         default=None, validator=attr.validators.optional(validate_markers)
     )
@@ -59,7 +63,9 @@ class PipenvMarkers(BaseRequirement):
         try:
             marker = Marker(marker_string)
         except InvalidMarker:
-            raise RequirementError("Invalid requirement: Invalid marker %r" % marker_string)
+            raise RequirementError(
+                "Invalid requirement: Invalid marker %r" % marker_string
+            )
         marker_dict = {}
         for m in marker._markers:
             if isinstance(m, six.string_types):

pipenv/vendor/requirementslib/models/pipfile.py

@@ -15,9 +15,7 @@ class Source(object):
     #: URL to PyPI instance
     url = attr.ib(default="pypi")
     #: If False, skip SSL checks
-    verify_ssl = attr.ib(
-        default=True, validator=optional_instance_of(bool)
-    )
+    verify_ssl = attr.ib(default=True, validator=optional_instance_of(bool))
     #: human name to refer to this source (can be referenced in packages or dev-packages)
     name = attr.ib(default="")
 
@@ -27,13 +25,13 @@ class Source(object):
     @property
     def expanded(self):
         source_dict = attr.asdict(self).copy()
-        source_dict['url'] = os.path.expandvars(source_dict.get('url'))
+        source_dict["url"] = os.path.expandvars(source_dict.get("url"))
         return source_dict
 
 
 @attr.s
 class Section(object):
-    ALLOWED_NAMES = ('packages', 'dev-packages',)
+    ALLOWED_NAMES = ("packages", "dev-packages")
     #: Name of the pipfile section
     name = attr.ib(default="packages")
     #: A list of requirements that are contained by the section
@@ -63,7 +61,7 @@ class RequiresSection(object):
         requires = attr.asdict(self, filter=filter_none)
         if not requires:
             return {}
-        return {'requires': requires}
+        return {"requires": requires}
 
 
 @attr.s
@@ -72,7 +70,7 @@ class PipenvSection(object):
 
     def get_dict(self):
         if self.allow_prereleases:
-            return {'pipenv': attr.asdict(self)}
+            return {"pipenv": attr.asdict(self)}
         return {}
 
 
@@ -111,7 +109,7 @@ class Pipfile(object):
         _dict = {}
         for src in self.sources:
             _dict.update(src.get_dict())
-        return {'source': _dict} if _dict else {}
+        return {"source": _dict} if _dict else {}
 
     def get_sections(self):
         """Return a dictionary with both pipfile sections and requirements"""
@@ -131,7 +129,7 @@ class Pipfile(object):
 
     def get_dict(self):
         _dict = attr.asdict(self, recurse=False)
-        for k in ['path', 'pipfile_hash', 'sources', 'sections', 'requires', 'pipenv']:
+        for k in ["path", "pipfile_hash", "sources", "sections", "requires", "pipenv"]:
             if k in _dict:
                 _dict.pop(k)
         return _dict
@@ -153,25 +151,25 @@ class Pipfile(object):
     def load(cls, path):
         if not isinstance(path, Path):
             path = Path(path)
-        pipfile_path = path / 'Pipfile'
+        pipfile_path = path / "Pipfile"
         if not path.exists():
             raise FileNotFoundError("%s is not a valid project path!" % path)
         elif not pipfile_path.exists() or not pipfile_path.is_file():
             raise RequirementError("%s is not a valid Pipfile" % pipfile_path)
         pipfile_dict = toml.load(pipfile_path.as_posix())
         sections = [cls.get_section(pipfile_dict, s) for s in Section.ALLOWED_NAMES]
-        pipenv = pipfile_dict.get('pipenv', {})
-        requires = pipfile_dict.get('requires', {})
+        pipenv = pipfile_dict.get("pipenv", {})
+        requires = pipfile_dict.get("requires", {})
         creation_dict = {
-            'path': pipfile_path,
-            'sources': [Source(**src) for src in pipfile_dict.get('source', [])],
-            'sections': sections,
-            'scripts': pipfile_dict.get('scripts')
+            "path": pipfile_path,
+            "sources": [Source(**src) for src in pipfile_dict.get("source", [])],
+            "sections": sections,
+            "scripts": pipfile_dict.get("scripts"),
         }
         if requires:
-            creation_dict['requires'] = RequiresSection(**requires)
+            creation_dict["requires"] = RequiresSection(**requires)
         if pipenv:
-            creation_dict['pipenv'] = PipenvSection(**pipenv)
+            creation_dict["pipenv"] = PipenvSection(**pipenv)
         return cls(**creation_dict)
 
     @staticmethod

pipenv/vendor/requirementslib/models/requirements.py

@@ -113,7 +113,7 @@ class FileRequirement(BaseRequirement):
     name = attr.ib()
     req = attr.ib()
     _has_hashed_name = False
-    _uri_scheme = None
+    _uri_scheme = attr.ib(default=None)
 
     @classmethod
     def get_link_from_line(cls, line):
@@ -126,18 +126,26 @@ class FileRequirement(BaseRequirement):
         parsed_url = urllib_parse.urlsplit(vcs_line)
         vcs_type = None
         scheme = parsed_url.scheme
-        if '+' in parsed_url.scheme:
-            vcs_type, scheme = parsed_url.scheme.split('+')
-        if (scheme == 'file' or not scheme) and parsed_url.path and os.path.exists(parsed_url.path):
+        if "+" in parsed_url.scheme:
+            vcs_type, scheme = parsed_url.scheme.split("+")
+        if (
+            (scheme == "file" or not scheme)
+            and parsed_url.path
+            and os.path.exists(parsed_url.path)
+        ):
             path = Path(parsed_url.path).absolute().as_posix()
             uri = path_to_url(path)
             if not parsed_url.scheme:
                 relpath = get_converted_relative_path(path)
-            uri = '{0}#{1}'.format(uri, parsed_url.fragment) if parsed_url.fragment else uri
+            uri = (
+                "{0}#{1}".format(uri, parsed_url.fragment)
+                if parsed_url.fragment
+                else uri
+            )
         else:
             path = None
             uri = urllib_parse.urlunsplit((scheme,) + parsed_url[1:])
-        vcs_line = '{0}+{1}'.format(vcs_type, uri) if vcs_type else uri
+        vcs_line = "{0}+{1}".format(vcs_type, uri) if vcs_type else uri
         link = Link(vcs_line)
         if added_ssh_scheme:
             uri = strip_ssh_from_git_uri(uri)
@@ -158,8 +166,8 @@ class FileRequirement(BaseRequirement):
         if self.link and self.link.egg_fragment:
             return self.link.egg_fragment
         elif self.link and self.link.is_wheel:
-            return os.path.basename(Wheel(self.link.path).name)
-        if self._uri_scheme != "uri" and self.path and self.setup_path:
+            return Wheel(self.link.filename).name
+        if self._uri_scheme != "uri" and self.path and self.setup_path.exists():
             from distutils.core import run_setup
 
             try:
@@ -240,6 +248,7 @@ class FileRequirement(BaseRequirement):
         line = line.strip('"').strip("'")
         link = None
         path = None
+        uri_scheme = None
         editable = line.startswith("-e ")
         line = line.split(" ", 1)[1] if editable else line
         setup_path = None
@@ -251,34 +260,29 @@ class FileRequirement(BaseRequirement):
         if is_valid_url(line) and not is_installable_file(line):
             vcs_type, relpath, uri, link = cls.get_link_from_line(line)
         else:
+            parsed = urlparse(line)
             if is_valid_url(line):
-                parsed = urlparse(line)
                 vcs_type, relpath, uri, link = cls.get_link_from_line(line)
-                # link = Link("{0}".format(line))
+                uri_scheme = parsed.scheme
                 if parsed.scheme == "file":
-                    path = Path(relpath)
-                    setup_path = path / "setup.py"
-                    path = path.absolute().as_posix()
+                    path = parsed.path
+                    setup_path = Path(path) / "setup.py"
             else:
                 vcs_type, relpath, uri, link = cls.get_link_from_line(line)
-                path = Path(relpath)
+                path = Path(parsed.path)
                 setup_path = path / "setup.py"
                 path = path.as_posix()
-                # link = Link(unquote(_path.absolute().as_uri()))
-                # if _path.is_absolute() or _path.as_posix() == ".":
-                #     path = _path.as_posix()
-                # else:
-                #     path = get_converted_relative_path(line)
-        # print(link)
-        print(uri)
         arg_dict = {
             "path": path,
             "uri": link.url_without_fragment,
             "link": link,
             "editable": editable,
             "setup_path": setup_path,
+            "uri_scheme": uri_scheme,
         }
-        if link.egg_fragment:
+        if link and link.is_wheel:
+            arg_dict["name"] = Wheel(link.filename).name
+        elif link.egg_fragment:
             arg_dict["name"] = link.egg_fragment
         created = cls(**arg_dict)
         return created
@@ -286,14 +290,17 @@ class FileRequirement(BaseRequirement):
     @classmethod
     def from_pipfile(cls, name, pipfile):
         uri_key = first((k for k in ["uri", "file"] if k in pipfile))
-        uri = pipfile.get(uri_key, pipfile.get("path"))
-        if not uri_key:
-            abs_path = os.path.abspath(uri)
-            uri = path_to_url(abs_path) if os.path.exists(abs_path) else None
+        path = pipfile.get("path")
+        uri = pipfile.get(uri_key, path)
+        parsed = urlparse(uri)
+        if not parsed.scheme:
+            path = parsed.path
+            abs_path = Path(uri).absolute().as_posix()
+            uri = path_to_url(abs_path)
         link = Link(unquote(uri)) if uri else None
         arg_dict = {
             "name": name,
-            "path": pipfile.get("path"),
+            "path": path,
             "uri": unquote(link.url_without_fragment if link else uri),
             "editable": pipfile.get("editable"),
             "link": link,
@@ -302,7 +309,10 @@ class FileRequirement(BaseRequirement):
 
     @property
     def line_part(self):
-        seed = self.formatted_path or self.link.url or self.uri
+        if (self._uri_scheme and self._uri_scheme == 'file') or (self.link.is_artifact or self.link.is_wheel) and self.link.url:
+            seed = self.link.url_without_fragment or self.uri
+        else:
+            seed = self.formatted_path or self.link.url or self.uri
         # add egg fragments to remote artifacts (valid urls only)
         if not self._has_hashed_name and self.is_remote_artifact:
             seed += "#egg={0}".format(self.name)
@@ -313,20 +323,34 @@ class FileRequirement(BaseRequirement):
     def pipfile_part(self):
         pipfile_dict = {k: v for k, v in attr.asdict(self, filter=filter_none).items()}
         name = pipfile_dict.pop("name")
+        if '_uri_scheme' in pipfile_dict:
+            pipfile_dict.pop('_uri_scheme')
         if "setup_path" in pipfile_dict:
             pipfile_dict.pop("setup_path")
         req = self.req
         # For local paths and remote installable artifacts (zipfiles, etc)
-        if self.is_remote_artifact:
+        collision_keys = {'file', 'uri', 'path'}
+        if self._uri_scheme:
+            dict_key = self._uri_scheme
+            target_key = dict_key if dict_key in pipfile_dict else next((k for k in ('file', 'uri', 'path') if k in pipfile_dict), None)
+            if target_key:
+                winning_value = pipfile_dict.pop(target_key)
+                collisions = (k for k in collision_keys if k in pipfile_dict)
+                for key in collisions:
+                    pipfile_dict.pop(key)
+                pipfile_dict[dict_key] = winning_value
+        elif self.is_remote_artifact or self.link.is_artifact and (self._uri_scheme and self._uri_scheme == 'file'):
             dict_key = "file"
             # Look for uri first because file is a uri format and this is designed
             # to make sure we add file keys to the pipfile as a replacement of uri
-            target_keys = [k for k in pipfile_dict.keys() if k in ["uri", "path"]]
-            pipfile_dict[dict_key] = pipfile_dict.pop(first(target_keys))
-            if len(target_keys) > 1:
-                pipfile_dict.pop(target_keys[1])
+            target_key = next((k for k in ('file', 'uri', 'path') if k in pipfile_dict), None)
+            winning_value = pipfile_dict.pop(target_key)
+            key_to_remove = (k for k in collision_keys if k in pipfile_dict)
+            for key in key_to_remove:
+                pipfile_dict.pop(key)
+            pipfile_dict[dict_key] = winning_value
         else:
-            collisions = [key for key in ["path", "uri", "file"] if key in pipfile_dict]
+            collisions = [key for key in ["path", "file", "uri",] if key in pipfile_dict]
             if len(collisions) > 1:
                 for k in collisions[1:]:
                     pipfile_dict.pop(k)
@@ -362,14 +386,13 @@ class VCSRequirement(FileRequirement):
         split = urllib_parse.urlsplit(self.uri)
         scheme, rest = split[0], split[1:]
         vcs_type = ""
-        if '+' in scheme:
-            vcs_type, scheme = scheme.split('+', 1)
+        if "+" in scheme:
+            vcs_type, scheme = scheme.split("+", 1)
             vcs_type = "{0}+".format(vcs_type)
         new_uri = urllib_parse.urlunsplit((scheme,) + rest)
         new_uri = "{0}{1}".format(vcs_type, new_uri)
         self.uri = new_uri
 
-
     @link.default
     def get_link(self):
         return build_vcs_link(
@@ -459,18 +482,26 @@ class VCSRequirement(FileRequirement):
         parsed_url = urllib_parse.urlsplit(vcs_line)
         vcs_type = None
         scheme = parsed_url.scheme
-        if '+' in parsed_url.scheme:
-            vcs_type, scheme = parsed_url.scheme.split('+')
-        if (scheme == 'file' or not scheme) and parsed_url.path and os.path.exists(parsed_url.path):
+        if "+" in parsed_url.scheme:
+            vcs_type, scheme = parsed_url.scheme.split("+")
+        if (
+            (scheme == "file" or not scheme)
+            and parsed_url.path
+            and os.path.exists(parsed_url.path)
+        ):
             path = Path(parsed_url.path).absolute().as_posix()
             uri = path_to_url(path)
             if not parsed_url.scheme:
                 relpath = get_converted_relative_path(path)
-            uri = '{0}#{1}'.format(uri, parsed_url.fragment) if parsed_url.fragment else uri                
+            uri = (
+                "{0}#{1}".format(uri, parsed_url.fragment)
+                if parsed_url.fragment
+                else uri
+            )
         else:
             path = None
             uri = urllib_parse.urlunsplit((scheme,) + parsed_url[1:])
-        vcs_line = '{0}+{1}'.format(vcs_type, uri) if vcs_type else uri
+        vcs_line = "{0}+{1}".format(vcs_type, uri) if vcs_type else uri
         link = Link(vcs_line)
         name = link.egg_fragment
         uri = link.url_without_fragment
@@ -602,18 +633,17 @@ class Requirement(object):
         vcs = None
         # Installable local files and installable non-vcs urls are handled
         # as files, generally speaking
-        if (
-            is_installable_file(line)
-            or (is_valid_url(line) and not is_vcs(line))
-        ):
+        if is_installable_file(line) or (is_valid_url(line) and not is_vcs(line)):
             r = FileRequirement.from_line(line_with_prefix)
         elif is_vcs(line):
             r = VCSRequirement.from_line(line_with_prefix)
             vcs = r.vcs
-        elif line == '.' and not is_installable_file(line):
-            raise RequirementError('Error parsing requirement %s -- are you sure it is installable?' % line)
+        elif line == "." and not is_installable_file(line):
+            raise RequirementError(
+                "Error parsing requirement %s -- are you sure it is installable?" % line
+            )
         else:
-            specs = '!=<>~'
+            specs = "!=<>~"
             spec_matches = set(specs) & set(line)
             version = None
             name = line
@@ -624,7 +654,7 @@ class Requirement(object):
             if not extras:
                 name, extras = _strip_extras(name)
             if version:
-                name = '{0}{1}'.format(name, version)
+                name = "{0}{1}".format(name, version)
             r = NamedRequirement.from_line(line)
         if extras:
             extras = first(
@@ -745,7 +775,7 @@ class Requirement(object):
         if not self._ireq:
             ireq_line = self.as_line()
             if ireq_line.startswith("-e "):
-                ireq_line = ireq_line[len("-e "):]
+                ireq_line = ireq_line[len("-e ") :]
                 self._ireq = InstallRequirement.from_editable(ireq_line)
             else:
                 self._ireq = InstallRequirement.from_line(ireq_line)

pipenv/vendor/requirementslib/models/utils.py

@@ -7,11 +7,7 @@ from first import first
 from packaging.markers import Marker, InvalidMarker
 from packaging.specifiers import SpecifierSet, InvalidSpecifier
 from .._compat import Link
-from ..utils import (
-    SCHEME_LIST,
-    VCS_LIST,
-    is_star,
-)
+from ..utils import SCHEME_LIST, VCS_LIST, is_star
 
 
 HASH_STRING = " --hash={0}"
@@ -124,6 +120,7 @@ def validate_vcs(instance, attr_, value):
 
 
 def validate_path(instance, attr_, value):
+    return True
     if not os.path.exists(value):
         raise ValueError("Invalid path {0!r}", format(value))