diff --git a/pipenv/utils.py b/pipenv/utils.py index 83a8782f..83a38027 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import errno import os import hashlib import tempfile @@ -32,7 +33,9 @@ from piptools.repositories.pypi import PyPIRepository from piptools.scripts.compile import get_pip_command from piptools import logging from piptools.exceptions import NoCandidateFound +from pip.download import is_archive_file from pip.exceptions import DistributionNotFound +from pip.index import Link from requests.exceptions import HTTPError, ConnectionError from .pep508checker import lookup @@ -61,6 +64,15 @@ def get_requirement(dep): :returns: :class:`requirements.Requirement` object """ path = None + uri = None + cleaned_uri = None + editable = False + dep_link = None + # check for editable dep / vcs dep + if dep.startswith('-e '): + editable = True + # Use the user supplied path as the written dependency + dep = dep.split(' ', 1)[1] # Split out markers if they are present - similar to how pip does it # See pip.req.req_install.InstallRequirement.from_line if not any(dep.startswith(uri_prefix) for uri_prefix in SCHEME_LIST): @@ -76,23 +88,44 @@ def get_requirement(dep): markers = None # Strip extras from the requirement so we can make a properly parseable req dep, extras = pip.req.req_install._strip_extras(dep) - # Only operate on local, existing, non-URI formatted paths - if (is_file(dep) and isinstance(dep, six.string_types) and - not any(dep.startswith(uri_prefix) for uri_prefix in SCHEME_LIST)): + # Only operate on local, existing, non-URI formatted paths which are installable + if is_installable_file(dep): dep_path = Path(dep) - # Only parse if it is a file or an installable dir - if dep_path.is_file() or (dep_path.is_dir() and pip.utils.is_installable_dir(dep)): - if dep_path.is_absolute() or dep_path.as_posix() == '.': - path = dep_path.as_posix() - else: - path = get_converted_relative_path(dep) - dep = dep_path.resolve().as_uri() + dep_link = Link(dep_path.absolute().as_uri()) + if dep_path.is_absolute() or dep_path.as_posix() == '.': + path = dep_path.as_posix() + else: + path = get_converted_relative_path(dep) + dep = dep_link.egg_fragment if dep_link.egg_fragment else dep_link.url_without_fragment + elif is_vcs(dep): + # Generate a Link object for parsing egg fragments + dep_link = Link(dep) + # Save the original path to store in the pipfile + uri = dep_link.url + # Construct the requirement using proper git+ssh:// replaced uris or names if available + cleaned_uri = clean_git_uri(dep) + dep = cleaned_uri + if editable: + dep = '-e {0}'.format(dep) req = [r for r in requirements.parse(dep)][0] + # if all we built was the requirement name and still need everything else + if req.name and not any([req.uri, req.path]): + if dep_link: + if dep_link.scheme.startswith('file') and path and not req.path: + req.path = path + req.local_file = True + req.uri = None + else: + req.uri = dep_link.url_without_fragment # If the result is a local file with a URI and we have a local path, unset the URI - # and set the path instead - if req.local_file and req.uri and not req.path and path: + # and set the path instead -- note that local files may have 'path' set by accident + elif req.local_file and path and not req.vcs: req.path = path req.uri = None + elif req.vcs and req.uri and cleaned_uri and uri != req.uri: + req.uri = strip_ssh_from_git_uri(req.uri) + req.line = strip_ssh_from_git_uri(req.line) + req.editable = editable if markers: req.markers = markers if extras: @@ -296,7 +329,7 @@ def resolve_deps(deps, which, which_pip, project, sources=None, verbose=False, p markers_lookup = {} python_path = which('python', allow_global=allow_global) - backup_python_path = shellquote(sys.executable) + backup_python_path = sys.executable results = [] @@ -387,7 +420,7 @@ def convert_deps_from_pip(dep): extras = {'extras': req.extras} # File installs. - if (req.uri or req.path or (os.path.isfile(req.name) if req.name else False)) and not req.vcs: + if (req.uri or req.path or is_installable_file(req.name)) and not req.vcs: # Assign a package name to the file, last 7 of it's sha256 hex digest. if not req.uri and not req.path: req.path = os.path.abspath(req.name) @@ -611,6 +644,22 @@ def is_required_version(version, specified_version): return True +def strip_ssh_from_git_uri(uri): + """Return git+ssh:// formatted URI to git+git@ format""" + if isinstance(uri, six.string_types): + uri = uri.replace('git+ssh://', 'git+') + return uri + + +def clean_git_uri(uri): + """Cleans VCS uris from pip format""" + if isinstance(uri, six.string_types): + # Add scheme for parsing purposes, this is also what pip does + if uri.startswith('git+') and '://' not in uri: + uri = uri.replace('git+', 'git+ssh://') + return uri + + def is_vcs(pipfile_entry): import requirements """Determine if dictionary entry from Pipfile is for a vcs dependency.""" @@ -618,17 +667,13 @@ def is_vcs(pipfile_entry): if hasattr(pipfile_entry, 'keys'): return any(key for key in pipfile_entry.keys() if key in VCS_LIST) elif isinstance(pipfile_entry, six.string_types): - # Add scheme for parsing purposes, this is also what pip does - if pipfile_entry.startswith('git+') and '://' not in pipfile_entry: - pipfile_entry = pipfile_entry.replace('git+', 'git+ssh://') - return bool(requirements.requirement.VCS_REGEX.match(pipfile_entry)) + return bool(requirements.requirement.VCS_REGEX.match(clean_git_uri(pipfile_entry))) return False def is_installable_file(path): - import pip - """Determine if a path can potentially be installed""" + import pip if hasattr(path, 'keys') and any(key for key in path.keys() if key in ['file', 'path']): path = urlparse(path['file']).path if 'file' in path else path['path'] if not isinstance(path, six.string_types) or path == '*': @@ -643,9 +688,15 @@ def is_installable_file(path): pass else: return False + if not os.path.exists(os.path.abspath(path)): + return False lookup_path = Path(path) - return lookup_path.is_file() or (lookup_path.is_dir() and - pip.utils.is_installable_dir(lookup_path.resolve().as_posix())) + absolute_path = '{0}'.format(lookup_path.absolute()) + if lookup_path.is_dir() and pip.utils.is_installable_dir(absolute_path): + return True + elif lookup_path.is_file() and is_archive_file(absolute_path): + return True + return False def is_file(package): diff --git a/tests/test_pipenv.py b/tests/test_pipenv.py index d48b0f56..f9ce2829 100644 --- a/tests/test_pipenv.py +++ b/tests/test_pipenv.py @@ -440,6 +440,30 @@ tablib = "<0.12" assert 'tablib' in p.lockfile['default'] + @pytest.mark.e + @pytest.mark.install + @pytest.mark.vcs + @pytest.mark.resolver + def test_editable_vcs_install_in_pipfile_with_dependency_resolution_doesnt_traceback(self): + # See https://github.com/pypa/pipenv/issues/1240 + with PipenvInstance() as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[packages] +pypa-docs-theme = {git = "https://github.com/pypa/pypa-docs-theme", editable = true} + +# This version of requests depends on idna<2.6, forcing dependency resolution +# failure +requests = "==2.16.0" +idna = "==2.6.0" + """.strip() + f.write(contents) + c = p.pipenv('install') + assert c.return_code == 1 + assert "Your dependencies could not be resolved" in c.err + assert 'Traceback' not in c.err or 'PermissionError' in c.err + + @pytest.mark.run @pytest.mark.install def test_multiprocess_bug_and_install(self): @@ -602,12 +626,12 @@ requests = {version = "*", os_name = "== 'splashwear'"} @pytest.mark.tablib def test_install_editable_git_tag(self): with PipenvInstance() as p: - c = p.pipenv('install -e git+git://github.com/kennethreitz/tablib.git@v0.12.1#egg=tablib') + c = p.pipenv('install -e git+https://github.com/kennethreitz/tablib.git@v0.12.1#egg=tablib') assert c.return_code == 0 assert 'tablib' in p.pipfile['packages'] assert 'tablib' in p.lockfile['default'] assert 'git' in p.lockfile['default']['tablib'] - assert p.lockfile['default']['tablib']['git'] == 'git://github.com/kennethreitz/tablib.git' + assert p.lockfile['default']['tablib']['git'] == 'https://github.com/kennethreitz/tablib.git' assert 'ref' in p.lockfile['default']['tablib'] @pytest.mark.run @@ -1102,3 +1126,17 @@ requests = "==2.14.0" assert 'path' in dep assert Path(os.path.join('.', artifact_dir, file_name)) == Path(dep['path']) assert c.return_code == 0 + + @pytest.mark.install + @pytest.mark.local_file + def test_install_local_file_collision(self): + with PipenvInstance() as p: + target_package = 'alembic' + fake_file = os.path.join(p.path, target_package) + with open(fake_file, 'w') as f: + f.write('') + c = p.pipenv('install {}'.format(target_package)) + assert c.return_code == 0 + assert target_package in p.pipfile['packages'] + assert p.pipfile['packages'][target_package] == '*' + assert target_package in p.lockfile['default'] diff --git a/tests/test_utils.py b/tests/test_utils.py index f570a631..f25b1936 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -250,3 +250,43 @@ twine = "*" @pytest.mark.skipif(os.name == 'nt', reason='*nix file paths tested') def test_nix_normalize_drive(self, input_path, expected): assert pipenv.utils.normalize_drive(input_path) == expected + + @pytest.mark.requirements + def test_get_requirements(self): + # Test eggs in URLs + url_with_egg = pipenv.utils.get_requirement('https://github.com/IndustriaTech/django-user-clipboard/archive/0.6.1.zip#egg=django-user-clipboard') + assert url_with_egg.uri == 'https://github.com/IndustriaTech/django-user-clipboard/archive/0.6.1.zip' + assert url_with_egg.name == 'django-user-clipboard' + + # Test URLs without eggs pointing at installable zipfiles + url = pipenv.utils.get_requirement('https://github.com/kennethreitz/tablib/archive/0.12.1.zip') + assert url.uri == 'https://github.com/kennethreitz/tablib/archive/0.12.1.zip' + + # Test VCS urls with refs and eggnames + vcs_url = pipenv.utils.get_requirement('git+https://github.com/kennethreitz/tablib.git@master#egg=tablib') + assert vcs_url.vcs == 'git' and vcs_url.name == 'tablib' and vcs_url.revision == 'master' + assert vcs_url.uri == 'git+https://github.com/kennethreitz/tablib.git' + + # Test normal package requirement + normal = pipenv.utils.get_requirement('tablib') + assert normal.name == 'tablib' + + # Pinned package requirement + spec = pipenv.utils.get_requirement('tablib==0.12.1') + assert spec.name == 'tablib' and spec.specs == [('==', '0.12.1')] + + # Test complex package with both extras and markers + extras_markers = pipenv.utils.get_requirement("requests[security]; os_name=='posix'") + assert extras_markers.extras == ['security'] + assert extras_markers.name == 'requests' + assert extras_markers.markers == "os_name=='posix'" + + # Test VCS uris get generated correctly, retain git+git@ if supplied that way, and are named according to egg fragment + git_reformat = pipenv.utils.get_requirement('-e git+git@github.com:pypa/pipenv.git#egg=pipenv') + assert git_reformat.uri == 'git+git@github.com:pypa/pipenv.git' + assert git_reformat.name == 'pipenv' + assert git_reformat.editable + # Previously VCS uris were being treated as local files, so make sure these are not handled that way + assert not git_reformat.local_file + # Test regression where VCS uris were being handled as paths rather than VCS entries + assert git_reformat.vcs == 'git'