Fix local and relative path installation

Summary of squashed commits:
* Handle relative paths more elegantly
* Wrapper around requirements.parse() for filesystem paths
* Undo previous hacks for simplicity
* Vendor pathlib2 for backwards compatibility
* Resolve relative paths for parsing, keep them relative in pipfile
* Add checks for empty req names and remote uris
* Add tests for local paths
* Fix test paramaterization
* Bugfixes for python27 and windows paths
* Fix windows tests
* Fix windows tests for python27 path encoding
* Re-vendor pathlib2 correctly
* Fix tests for windows paths
* Fix file and path checking
* Fix SCHEME_LIST rename
* Fix path resolution to check existence first
* Catch OSErrors for unpinned dependencies
* Last holdout of FILE_LIST conversion
* Fix for path resolution for unpinned packages
* Dont do path conversions on dictionaries
* Update docstring and comments

Signed-off-by: Dan Ryan <dan@danryan.co>

pipenv/cli.py

@@ -36,7 +36,7 @@ from .utils import (
     proper_case, pep423_name, split_vcs, resolve_deps, shellquote, is_vcs,
     python_version, suggest_package, find_windows_executable, is_file,
     prepare_pip_source_args, temp_environ, is_valid_url, download_file,
-    need_update_check, touch_update_stamp
+    get_requirement, need_update_check, touch_update_stamp
 )
 from .__version__ import __version__
 from . import pep508checker, progress
@@ -1352,7 +1352,7 @@ def pip_install(
             f.write(package_name)
 
     # Install dependencies when a package is a VCS dependency.
-    if [x for x in requirements.parse(package_name.split('--hash')[0].split('--trusted-host')[0])][0].vcs:
+    if get_requirement(package_name.split('--hash')[0].split('--trusted-host')[0]).vcs:
         no_deps = False
 
         # Don't specify a source directory when using --system.
@@ -1872,18 +1872,16 @@ def install(
 
         # pip install:
         with spinner():
-            # So that we still write the provided path to the pipfile but convert for installing
-            pip_dep_name = package_name
-            if is_file(package_name):
-                pip_dep_name = convert_path_to_uri(package_name)
-            c = pip_install(pip_dep_name, ignore_hashes=True, allow_global=system, no_deps=False, verbose=verbose, pre=pre)
+
+            c = pip_install(package_name, ignore_hashes=True, allow_global=system, no_deps=False, verbose=verbose, pre=pre)
 
             # Warn if --editable wasn't passed.
             try:
-                converted = convert_deps_from_pip(pip_dep_name)
+                converted = convert_deps_from_pip(package_name)
             except ValueError as e:
                 click.echo('{0}: {1}'.format(crayons.red('WARNING'), e))
                 sys.exit(1)
+
             key = [k for k in converted.keys()][0]
             if is_vcs(key) or is_vcs(converted[key]) and not converted[key].get('editable'):
                 click.echo(

pipenv/project.py

@@ -14,7 +14,7 @@ import toml
 from .utils import (
     mkdir_p, convert_deps_from_pip, pep423_name, recase_file,
     find_requirements, is_file, is_vcs, python_version, cleanup_toml,
-    convert_path_to_uri, convert_file_uri_to_path
+    is_installable_file, is_valid_url
 )
 from .environments import PIPENV_MAX_DEPTH, PIPENV_VENV_IN_PROJECT
 from .environments import PIPENV_VIRTUALENV, PIPENV_PIPFILE
@@ -67,10 +67,13 @@ class Project(object):
                     else:
                         ps.update({k: v})
                 else:
-                    if not is_file(v) and not is_file(k):
+                    if not (is_installable_file(k) or is_installable_file(v) or
+                            any(file_prefix in v for file_prefix in ['path', 'file'])):
                         ps.update({k: v})
             else:
-                if not is_vcs(k) and not is_file(k) and not is_vcs(v):
+                if not (any(is_vcs(i) for i in [k, v]) or
+                        any(is_installable_file(i) for i in [k, v]) or
+                        any(is_valid_url(i) for i in [k, v])):
                     ps.update({k: v})
         return ps
 
@@ -461,18 +464,8 @@ class Project(object):
         # Read and append Pipfile.
         p = self._pipfile
 
-        # Always use file:// URIs for local path installation
-        # But allow local relpaths in pipfile
-        pip_dep_name = package_name
-        pipfile_package_path = None
-        if is_file(package_name) and not package_name.startswith('file://'):
-            pip_dep_name = convert_path_to_uri(package_name)
-            pipfile_package_path = os.path.join('.', os.path.relpath(
-                convert_file_uri_to_path(pip_dep_name), start=self.project_directory)
-            )
-
         # Don't re-capitalize file URLs or VCSs.
-        converted = convert_deps_from_pip(pip_dep_name)
+        converted = convert_deps_from_pip(package_name)
         converted = converted[[k for k in converted.keys()][0]]
 
         if not (is_file(package_name) or is_vcs(converted) or 'path' in converted):
@@ -484,13 +477,10 @@ class Project(object):
         if key not in p:
             p[key] = {}
 
-        package = convert_deps_from_pip(pip_dep_name)
+        package = convert_deps_from_pip(package_name)
         package_name = [k for k in package.keys()][0]
 
         # Add the package to the group.
-        if pipfile_package_path and 'file' in package[package_name]:
-            del package[package_name]['file']
-            package[package_name]['path'] = pipfile_package_path
         p[key][package_name] = package[package_name]
 
         # Write Pipfile.

pipenv/utils.py

@@ -23,6 +23,10 @@ try:
     from urllib.parse import urlparse
 except ImportError:
     from urlparse import urlparse
+try:
+    from pathlib import Path
+except ImportError:
+    from pathlib2 import Path
 
 from distutils.spawn import find_executable
 from contextlib import contextmanager
@@ -41,7 +45,7 @@ specifiers = [k for k in lookup.keys()]
 
 # List of version control systems we support.
 VCS_LIST = ('git', 'svn', 'hg', 'bzr')
-SCHEME_LIST = ('http://', 'https://', 'ftp://', 'file:///')
+SCHEME_LIST = ('http://', 'https://', 'ftp://', 'file://')
 
 requests = requests.Session()
 
@@ -266,6 +270,35 @@ packages = [
 ]
 
 
+def get_requirement(dep):
+    """Pre-clean requirement strings passed to the requirements parser.
+    
+    Ensures that we can accept both local and relative paths, file and VCS URIs,
+    remote URIs, and package names, and that we pass only valid requirement strings
+    to the requirements parser. Performs necessary modifications to requirements
+    object if the user input was a local relative path.
+    """
+    path = None
+    # Only operate on local, existing, non-URI formatted paths
+    if (is_file(dep) and isinstance(dep, six.string_types) and
+            not any(dep.startswith(uri_prefix) for uri_prefix in SCHEME_LIST)):
+        dep_path = Path(dep)
+        # Only parse if it is a file or an installable dir
+        if dep_path.is_file() or (dep_path.is_dir() and pip.utils.is_installable_dir(dep)):
+            if dep_path.is_absolute():
+                path = dep
+            else:
+                path = get_converted_relative_path(dep)
+            dep = dep_path.resolve().as_uri()
+    req = [r for r in requirements.parse(dep)][0]
+    # If the result is a local file with a URI and we have a local path, unset the URI
+    # and set the path instead
+    if req.local_file and req.uri and not req.path and path:
+        req.path = path
+        req.uri = None
+    return req
+
+
 def cleanup_toml(tml):
     toml = tml.split('\n')
     new_toml = []
@@ -539,12 +572,11 @@ def convert_deps_from_pip(dep):
 
     dependency = {}
 
-    req = [r for r in requirements.parse(dep)][0]
+    req = get_requirement(dep)
     extras = {'extras': req.extras}
 
     # File installs.
-    if (req.uri or (os.path.isfile(req.path) if req.path else False) or
-            os.path.isfile(req.name)) and not req.vcs:
+    if (req.uri or req.path or (os.path.isfile(req.name) if req.name else False)) and not req.vcs:
         # Assign a package name to the file, last 7 of it's sha256 hex digest.
         if not req.uri and not req.path:
             req.path = os.path.abspath(req.name)
@@ -778,6 +810,17 @@ def is_vcs(pipfile_entry):
     return False
 
 
+def is_installable_file(path):
+    """Determine if a path can potentially be installed"""
+    if hasattr(path, 'keys') and any(key for key in path.keys() if key in ['file', 'path']):
+        path = urlparse(path['file']).path if 'file' in path else path['path']
+    if not isinstance(path, six.string_types) or path == '*':
+        return False
+    lookup_path = Path(path)
+    return lookup_path.is_file() or (lookup_path.is_dir() and
+            pip.utils.is_installable_dir(lookup_path.resolve().as_posix()))
+
+
 def is_file(package):
     """Determine if a package name is for a File dependency."""
     if hasattr(package, 'keys'):
@@ -885,21 +928,9 @@ def find_windows_executable(bin_path, exe_name):
     return find_executable(exe_name)
 
 
-def convert_path_to_uri(path):
-    """Given a path, return a file:// URI pointer"""
-    # Adapted and borrowed from pip
-    drive, path = os.path.splitdrive(os.path.abspath(path))
-    filepath = path.split(os.path.sep)
-    url = '/'.join(filepath)
-    if drive:
-        return 'file:///' + drive + url
-    return 'file://' + url
-
-
-def convert_file_uri_to_path(uri):
-    """Given a URI, return an OS path"""
-    path = urlparse(uri)
-    return os.path.abspath(os.path.join(path.netloc, path.path))
+def get_converted_relative_path(path, relative_to=os.curdir):
+    """Given a vague relative path, return the path relative to the given location"""
+    return os.path.join('.', os.path.relpath(path, start=relative_to))
 
 
 def walk_up(bottom):

tests/test_pipenv.py

@@ -7,7 +7,7 @@ import json
 import pytest
 
 from pipenv.cli import activate_virtualenv
-from pipenv.utils import temp_environ, get_windows_path
+from pipenv.utils import temp_environ, get_windows_path, mkdir_p
 from pipenv.vendor import toml
 from pipenv.vendor import delegator
 from pipenv.project import Project
@@ -843,6 +843,7 @@ requests = "==2.14.0"
 
             assert 'file' in dep or 'path' in dep
 
+
     @pytest.mark.install
     @pytest.mark.files
     @pytest.mark.urls
@@ -861,3 +862,25 @@ requests = "==2.14.0"
             # check Pipfile.lock
             assert 'requests' in p.lockfile['default']
             assert 'records' in p.lockfile['default']
+
+
+    @pytest.mark.install
+    @pytest.mark.files
+    def test_relative_paths(self):
+        file_name = 'tablib-0.12.1.tar.gz'
+        test_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)))
+        source_path = os.path.abspath(os.path.join(test_dir, 'test_artifacts', file_name))
+        
+        with PipenvInstance() as p:
+            artifact_dir = 'artifacts'
+            artifact_path = os.path.join(p.path, artifact_dir)
+            mkdir_p(artifact_path)
+            shutil.copy(source_path, os.path.join(artifact_path, file_name))
+            # Test installing a relative path in a subdirectory
+            c = p.pipenv('install {}/{}'.format(artifact_dir, file_name))
+            key = [k for k in p.pipfile['packages'].keys()][0]
+            dep = p.pipfile['packages'][key]
+
+            assert 'path' in dep
+            assert u'{}'.format(os.path.join('.', artifact_dir, file_name)) == u'{}'.format(dep['path'])
+            assert c.return_code == 0

tests/test_utils.py

@@ -182,6 +182,24 @@ class TestUtils:
         assert pipenv.utils.is_valid_url(url)
         assert pipenv.utils.is_valid_url(not_url) is False
 
+    @pytest.mark.parametrize('input_path, expected', [
+        ('artifacts/file.zip', './artifacts/file.zip'),
+        ('./artifacts/file.zip', './artifacts/file.zip'),
+        ('../otherproject/file.zip', './../otherproject/file.zip')
+    ])
+    @pytest.mark.skipif(os.name == 'nt', reason='Nix-based file paths tested')
+    def test_nix_converted_relative_path(self, input_path, expected):
+        assert pipenv.utils.get_converted_relative_path(input_path) == expected
+
+    @pytest.mark.parametrize('input_path, expected', [
+        ('artifacts/file.zip', '.\\artifacts\\file.zip'),
+        ('./artifacts/file.zip', '.\\artifacts\\file.zip'),
+        ('../otherproject/file.zip', '.\\..\\otherproject\\file.zip')
+    ])
+    @pytest.mark.skipif(os.name != 'nt', reason='Windows-based file paths tested')
+    def test_win_converted_relative_path(self, input_path, expected):
+        assert pipenv.utils.get_converted_relative_path(input_path) == expected
+
     def test_download_file(self):
         url = "https://github.com/kennethreitz/pipenv/blob/master/README.rst"
         output = "test_download.rst"