Properly parse urls as Link objects

- Fixes #1121
- Adds a bunch of tests for pipenv.utils.get_requirement()

pipenv/utils.py

@@ -299,12 +299,13 @@ def get_requirement(dep):
         markers = None
     # Strip extras from the requirement so we can make a properly parseable req
     dep, extras = pip.req.req_install._strip_extras(dep)
+    matches_uri = any(dep.startswith(uri_prefix) for uri_prefix in SCHEME_LIST)
     # Only operate on local, existing, non-URI formatted paths
-    if (is_file(dep) and isinstance(dep, six.string_types) and
-            not any(dep.startswith(uri_prefix) for uri_prefix in SCHEME_LIST)):
+    if is_file(dep) and isinstance(dep, six.string_types) and not matches_uri:
         dep_path = Path(dep)
         # Only parse if it is a file or an installable dir
         if dep_path.is_file() or (dep_path.is_dir() and pip.utils.is_installable_dir(dep)):
+            # Create pip Link objects for obtaining fragments and spotting wheels
             dep_link = Link(dep_path.absolute().as_uri())
             if dep_path.is_dir() or dep_link.is_wheel or is_archive_file(dep_path.as_posix()):
                 if dep_path.is_absolute() or dep_path.as_posix() == '.':
@@ -312,13 +313,23 @@ def get_requirement(dep):
                 else:
                     path = get_converted_relative_path(dep)
                 dep = dep_link.egg_fragment if dep_link.egg_fragment else dep_link.url_without_fragment
+
+    elif is_valid_url(dep) and matches_uri:
+        dep_link = Link(dep)
+        # Parse the requirement using just the dependency name version from the egg fragment
+        # if possible. Then we can drop in the URI later.  This is how pip does it.
+        dep = dep_link.egg_fragment if dep_link.egg_fragment else dep_link.url_without_fragment
+        if dep_link.egg_fragment:
+            path = dep_link.url_without_fragment
     req = [r for r in requirements.parse(dep)][0]
     # If the result is a local file with a URI and we have a local path, unset the URI
     # and set the path instead
-    if path and not req.path:
+    if path and not req.path and not matches_uri:
         req.path = path
         req.uri = None
         req.local_file = True
+    elif matches_uri and path and not req.uri:
+        req.uri = path
     if markers:
         req.markers = markers
     if extras:

tests/test_utils.py

@@ -250,3 +250,22 @@ twine = "*"
     @pytest.mark.skipif(os.name == 'nt', reason='*nix file paths tested')
     def test_nix_normalize_drive(self, input_path, expected):
         assert pipenv.utils.normalize_drive(input_path) == expected
+
+    @pytest.mark.requirements
+    def test_get_requirements(self):
+        url_with_egg = pipenv.utils.get_requirement('https://github.com/IndustriaTech/django-user-clipboard/archive/0.6.1.zip#egg=django-user-clipboard==0.6.1')
+        assert url_with_egg.uri == 'https://github.com/IndustriaTech/django-user-clipboard/archive/0.6.1.zip'
+        assert url_with_egg.name == 'django-user-clipboard' and url_with_egg.specs == [('==', '0.6.1')]
+        url = pipenv.utils.get_requirement('https://github.com/kennethreitz/tablib/archive/0.12.1.zip')
+        assert url.uri == 'https://github.com/kennethreitz/tablib/archive/0.12.1.zip'
+        vcs_url = pipenv.utils.get_requirement('git+https://github.com/kennethreitz/tablib.git@master#egg=tablib')
+        assert vcs_url.vcs == 'git' and vcs_url.name == 'tablib' and vcs_url.revision == 'master'
+        assert vcs_url.uri == 'git+https://github.com/kennethreitz/tablib.git'
+        normal = pipenv.utils.get_requirement('tablib')
+        assert normal.name == 'tablib'
+        spec = pipenv.utils.get_requirement('tablib==0.12.1')
+        assert spec.name == 'tablib' and spec.specs == [('==', '0.12.1')]
+        extras_markers = pipenv.utils.get_requirement("requests[security]; os_name=='posix'")
+        assert extras_markers.extras == ['security']
+        assert extras_markers.name == 'requests'
+        assert extras_markers.markers == "os_name=='posix'"