Merge branch 'master' into update-piptools

.buildkite/pipeline.yml

@@ -1,4 +1,5 @@
 steps:
   - label: ":python:"
     commands:
+      # - make
       - ./run-tests.sh

CONTRIBUTING.md

@@ -3,8 +3,8 @@
 Before opening any issues or proposing any pull requests, please do the
 following:
 
-1. Read our [Contributor's Guide](http://docs.pipenv.org/en/latest/dev/contributing/).
-2. Understand our [development philosophy](http://docs.pipenv.org/en/latest/dev/philosophy/).
+1. Read our [Contributor's Guide](https://docs.pipenv.org/dev/contributing/).
+2. Understand our [development philosophy](https://docs.pipenv.org/dev/philosophy/).
 
 To get the greatest chance of helpful responses, please also observe the
 following additional notes.

Dockerfile

@@ -1,11 +1,7 @@
-FROM ubuntu:17.10
+FROM ubuntu:18.04
 
 # -- Install Pipenv:
-RUN apt-get update \
-  && apt-get install software-properties-common python-software-properties -y \
-  && add-apt-repository ppa:pypa/ppa -y \
-  && apt-get update \
-  && apt-get install git pipenv -y
+RUN apt update && apt install python3-pip -y && pip3 install pipenv
 
 ENV LC_ALL C.UTF-8
 ENV LANG C.UTF-8

Makefile

@@ -0,0 +1,2 @@
+test:
+	docker-compose up

docker-compose.yml

@@ -0,0 +1,7 @@
+version: '3'
+services:
+  pipenv-tests:
+    image: kennethreitz/pipenv-tests
+    command: bash /pipenv/run-tests.sh
+    volumes:
+      - .:/pipenv

pipenv/_compat.py

@@ -5,6 +5,7 @@ Exposes a standard API that enables compatibility across python versions,
 operating systems, etc.
 """
 import functools
+import importlib
 import io
 import os
 import six
@@ -50,6 +51,24 @@ if six.PY2:
     class ResourceWarning(Warning):
         pass
 
+# -*- coding=utf-8 -*-
+
+
+def pip_import(module_path, subimport=None, old_path=None):
+    internal = 'pip._internal.{0}'.format(module_path)
+    old_path = old_path or module_path
+    pip9 = 'pip.{0}'.format(old_path)
+    try:
+        _tmp = importlib.import_module(internal)
+    except ImportError:
+        _tmp = importlib.import_module(pip9)
+    if subimport:
+        return getattr(_tmp, subimport, _tmp)
+    return _tmp
+
+
+vcs = pip_import('vcs', 'VcsSupport')
+
 
 class TemporaryDirectory(object):
     """Create and return a temporary directory.  This has the same

pipenv/core.py

@@ -16,6 +16,7 @@ import crayons
 import dotenv
 import delegator
 from .vendor import pexpect
+from first import first
 import pipfile
 from blindspin import spinner
 from requests.packages import urllib3
@@ -46,9 +47,11 @@ from .utils import (
     is_star,
     rmtree,
     split_argument,
+    extract_uri_from_vcs_dep,
 )
 from ._compat import (
     TemporaryDirectory,
+    vcs
 )
 from .import pep508checker, progress
 from .environments import (
@@ -999,6 +1002,7 @@ def do_lock(
 ):
     """Executes the freeze functionality."""
     from notpip._vendor.distlib.markers import Evaluator
+    from .utils import get_vcs_deps
     allowed_marker_keys = ['markers'] + [k for k in Evaluator.allowed_values.keys()]
     cached_lockfile = {}
     if not pre:
@@ -1035,6 +1039,9 @@ def do_lock(
         if dev_package in project.packages:
             dev_packages[dev_package] = project.packages[dev_package]
     # Resolve dev-package dependencies, with pip-tools.
+    pip_freeze = delegator.run(
+        '{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=system)))
+    ).out
     deps = convert_deps_to_pip(
         dev_packages, project, r=False, include_index=True
     )
@@ -1066,24 +1073,14 @@ def do_lock(
             lockfile['develop'][dep['name']]['markers'] = dep['markers']
     # Add refs for VCS installs.
     # TODO: be smarter about this.
-    vcs_deps = convert_deps_to_pip(project.vcs_dev_packages, project, r=False)
-    pip_freeze = delegator.run(
-        '{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=system)))
-    ).out
-    if vcs_deps:
-        for line in pip_freeze.strip().split('\n'):
-            # if the line doesn't match a vcs dependency in the Pipfile,
-            # ignore it
-            if not any(dep in line for dep in vcs_deps):
-                continue
-
-            try:
-                installed = convert_deps_from_pip(line)
-                name = list(installed.keys())[0]
-                if is_vcs(installed[name]):
-                    lockfile['develop'].update(installed)
-            except IndexError:
-                pass
+    vcs_dev_lines, vcs_dev_lockfiles = get_vcs_deps(project, pip_freeze, which=which, verbose=verbose, clear=clear, pre=pre, allow_global=system, dev=True)
+    for lf in vcs_dev_lockfiles:
+        try:
+            name = first(lf.keys())
+        except AttributeError:
+            continue
+        if hasattr(lf[name], 'keys'):
+            lockfile['develop'].update(lf)
     if write:
         # Alert the user of progress.
         click.echo(
@@ -1132,23 +1129,15 @@ def do_lock(
             lockfile['default'][dep['name']]['markers'] = dep['markers']
     # Add refs for VCS installs.
     # TODO: be smarter about this.
-    vcs_deps = convert_deps_to_pip(project.vcs_packages, project, r=False)
-    if vcs_deps:
-        for line in pip_freeze.strip().split('\n'):
-            # if the line doesn't match a vcs dependency in the Pipfile,
-            # ignore it
-            if not any(dep in line for dep in vcs_deps):
-                continue
+    _vcs_deps, vcs_lockfiles = get_vcs_deps(project, pip_freeze, which=which, verbose=verbose, clear=clear, pre=pre, allow_global=system, dev=False)
+    for lf in vcs_lockfiles:
+        try:
+            name = first(lf.keys())
+        except AttributeError:
+            continue
+        if hasattr(lf[name], 'keys'):
+            lockfile['default'].update(lf)
 
-            try:
-                installed = convert_deps_from_pip(line)
-                name = list(installed.keys())[0]
-                if is_vcs(installed[name]):
-                    # Convert name to PEP 423 name.
-                    installed = {pep423_name(name): installed[name]}
-                    lockfile['default'].update(installed)
-            except IndexError:
-                pass
     # Support for --keep-outdated…
     if keep_outdated:
         for section_name, section in (

pipenv/patched/pew/pew.py

@@ -184,7 +184,8 @@ def fork_bash(env, cwd):
 
 def fork_cmder(env, cwd):
     shell_cmd = ['cmd']
-    cmderrc_path = r'%CMDER_ROOT%\vendor\init.bat'
+    escaped_cmder_root = os.environ['CMDER_ROOT'].replace(' ', '^ ')
+    cmderrc_path = r'{0}\vendor\init.bat'.format(escaped_cmder_root)
     if expandpath(cmderrc_path).exists():
         shell_cmd += ['/k', cmderrc_path]
     if cwd:

pipenv/patched/piptools/utils.py

@@ -60,6 +60,36 @@ def make_install_requirement(name, version, extras, markers, constraint=False):
             constraint=constraint)
 
 
+def _requirement_to_str_lowercase_name(requirement):
+    """
+    Formats a packaging.requirements.Requirement with a lowercase name.
+
+    This is simply a copy of
+    https://github.com/pypa/packaging/blob/16.8/packaging/requirements.py#L109-L124
+    modified to lowercase the dependency name.
+
+    Previously, we were invoking the original Requirement.__str__ method and
+    lowercasing the entire result, which would lowercase the name, *and* other,
+    important stuff that should not be lowercased (such as the marker). See
+    this issue for more information: https://github.com/pypa/pipenv/issues/2113.
+    """
+    parts = [requirement.name.lower()]
+
+    if requirement.extras:
+        parts.append("[{0}]".format(",".join(sorted(requirement.extras))))
+
+    if requirement.specifier:
+        parts.append(str(requirement.specifier))
+
+    if requirement.url:
+        parts.append("@ {0}".format(requirement.url))
+
+    if requirement.marker:
+        parts.append("; {0}".format(requirement.marker))
+
+    return "".join(parts)
+
+
 def format_requirement(ireq, marker=None):
     """
     Generic formatter for pretty printing InstallRequirements to the terminal
@@ -68,7 +98,7 @@ def format_requirement(ireq, marker=None):
     if ireq.editable:
         line = '-e {}'.format(ireq.link)
     else:
-        line = str(ireq.req).lower()
+        line = _requirement_to_str_lowercase_name(ireq.req)
 
     if marker:
         line = '{}; {}'.format(line, marker)

pipenv/project.py

@@ -497,41 +497,41 @@ class Project(object):
     def lockfile_content(self):
         return self.load_lockfile()
 
-    @property
-    def editable_packages(self):
+    def _get_editable_packages(self, dev=False):
+        section = 'dev-packages' if dev else 'packages'
         packages = {
             k: v
-            for k, v in self.parsed_pipfile.get('packages', {}).items()
+            for k, v in self.parsed_pipfile.get(section, {}).items()
             if is_editable(v)
         }
         return packages
 
-    @property
-    def editable_dev_packages(self):
+    def _get_vcs_packages(self, dev=False):
+        section = 'dev-packages' if dev else 'packages'
         packages = {
             k: v
-            for k, v in self.parsed_pipfile.get('dev-packages', {}).items()
-            if is_editable(v)
+            for k, v in self.parsed_pipfile.get(section, {}).items()
+            if is_vcs(v) or is_vcs(k)
         }
-        return packages
+        return packages or {}
+
+    @property
+    def editable_packages(self):
+        return self._get_editable_packages(dev=False)
+
+    @property
+    def editable_dev_packages(self):
+        return self._get_editable_packages(dev=True)
 
     @property
     def vcs_packages(self):
         """Returns a list of VCS packages, for not pip-tools to consume."""
-        ps = {}
-        for k, v in self.parsed_pipfile.get('packages', {}).items():
-            if is_vcs(v) or is_vcs(k):
-                ps.update({k: v})
-        return ps
+        return self._get_vcs_packages(dev=False)
 
     @property
     def vcs_dev_packages(self):
         """Returns a list of VCS packages, for not pip-tools to consume."""
-        ps = {}
-        for k, v in self.parsed_pipfile.get('dev-packages', {}).items():
-            if is_vcs(v) or is_vcs(k):
-                ps.update({k: v})
-        return ps
+        return self._get_vcs_packages(dev=True)
 
     @property
     def all_packages(self):

pipenv/utils.py

@@ -1358,3 +1358,71 @@ def safe_expandvars(value):
     if isinstance(value, six.string_types):
         return os.path.expandvars(value)
     return value
+
+
+def extract_uri_from_vcs_dep(dep):
+    valid_keys = VCS_LIST + ('uri', 'file')
+    if hasattr(dep, 'keys'):
+        return first(dep[k] for k in valid_keys if k in dep) or None
+    return None
+
+
+def install_or_update_vcs(vcs_obj, src_dir, name, rev=None):    
+    target_dir = os.path.join(src_dir, name)
+    target_rev = vcs_obj.make_rev_options(rev)
+    if not os.path.exists(target_dir):
+        vcs_obj.obtain(target_dir)
+    vcs_obj.update(target_dir, target_rev)
+    return vcs_obj.get_revision(target_dir)
+
+
+def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=False, pre=False, allow_global=False, dev=False):
+    from ._compat import vcs
+    section = 'vcs_dev_packages' if dev else 'vcs_packages'
+    lines = []
+    lockfiles = []
+    try:
+        packages = getattr(project, section)
+    except AttributeError:
+        return [], []
+    vcs_registry = vcs()
+    vcs_uri_map = {
+        extract_uri_from_vcs_dep(v): {'name': k, 'ref': v.get('ref')}
+        for k, v in packages.items()
+    }
+    for line in pip_freeze.strip().split('\n'):
+        # if the line doesn't match a vcs dependency in the Pipfile,
+        # ignore it
+        _vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line)
+        if not _vcs_match:
+            continue
+
+        pipfile_name = vcs_uri_map[_vcs_match]['name']
+        pipfile_rev = vcs_uri_map[_vcs_match]['ref']
+        src_dir = os.environ.get('PIP_SRC', os.path.join(project.virtualenv_location, 'src'))
+        mkdir_p(src_dir)
+        names = {pipfile_name}
+        _pip_uri = line.lstrip('-e ')
+        backend_name = str(_pip_uri.split('+', 1)[0])
+        backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)]
+        __vcs = backend(url=_pip_uri)
+
+        installed = convert_deps_from_pip(line)
+        if not hasattr(installed, 'keys'):
+            pass
+        lock_name = first(installed.keys())
+        names.add(lock_name)
+        locked_rev = None
+        for _name in names:
+            locked_rev = install_or_update_vcs(__vcs, src_dir, _name, rev=pipfile_rev)
+        if is_vcs(installed[lock_name]):
+            installed[lock_name]['ref'] = locked_rev
+            lockfiles.append({pipfile_name: installed[lock_name]})
+        pipfile_srcdir = os.path.join(src_dir, pipfile_name)
+        lockfile_srcdir = os.path.join(src_dir, lock_name)
+        lines.append(line)
+        if os.path.exists(pipfile_srcdir):
+            lockfiles.extend(venv_resolve_deps(['-e {0}'.format(pipfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global))
+        else:
+            lockfiles.extend(venv_resolve_deps(['-e {0}'.format(lockfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global))
+    return lines, lockfiles

tasks/vendoring/patches/patched/pew-cmder-root-space-escape-fix.patch

@@ -0,0 +1,14 @@
+diff --git a/pipenv/patched/pew/pew.py b/pipenv/patched/pew/pew.py
+index 2d3889a0..91f313c1 100644
+--- a/pipenv/patched/pew/pew.py
++++ b/pipenv/patched/pew/pew.py
+@@ -184,7 +184,8 @@ def fork_bash(env, cwd):
+ 
+ def fork_cmder(env, cwd):
+     shell_cmd = ['cmd']
+-    cmderrc_path = r'%CMDER_ROOT%\vendor\init.bat'
++    escaped_cmder_root = os.environ['CMDER_ROOT'].replace(' ', '^ ')
++    cmderrc_path = r'{0}\vendor\init.bat'.format(escaped_cmder_root)
+     if expandpath(cmderrc_path).exists():
+         shell_cmd += ['/k', cmderrc_path]
+     if cwd:

tests/integration/test_install_markers.py

@@ -33,6 +33,26 @@ tablib = {version = "*", markers="os_name=='splashwear'"}
         c = p.pipenv('run python -c "import tablib;"')
         assert c.return_code == 1
 
+@pytest.mark.markers
+@flaky
+def test_platform_python_implementation_marker(PipenvInstance, pypi):
+    """Markers should be converted during locking to help users who input this incorrectly
+    """
+    with PipenvInstance(pypi=pypi) as p:
+        with open(p.pipfile_path, 'w') as f:
+            contents = """
+[packages]
+depends-on-marked-package = "*"
+            """.strip()
+            f.write(contents)
+
+        c = p.pipenv('install')
+        assert c.return_code == 0
+
+        # depends-on-marked-package has an install_requires of 'pytz; platform_python_implementation=="CPython"'
+        # Verify that that marker shows up in our lockfile unaltered.
+        assert p.lockfile['default']['pytz']['markers'] == "platform_python_implementation == 'CPython'"
+
 
 @pytest.mark.run
 @pytest.mark.alt

tests/integration/test_install_uri.py

@@ -2,6 +2,10 @@ import pytest
 import os
 from flaky import flaky
 import delegator
+try:
+    from pathlib import Path
+except ImportError:
+    from pathlib2 import Path
 
 
 @pytest.mark.vcs
@@ -144,3 +148,25 @@ def test_install_local_vcs_not_in_lockfile(PipenvInstance, pip_src_dir):
         assert six_key in p.lockfile['default']
         # Make sure we didn't put six in the lockfile by accident as a vcs ref
         assert 'six' not in p.lockfile['default']
+
+
+@pytest.mark.vcs
+@pytest.mark.install
+@pytest.mark.needs_internet
+def test_get_vcs_refs(PipenvInstance, pip_src_dir):
+    with PipenvInstance(chdir=True) as p:
+        c = p.pipenv('install -e git+https://github.com/hynek/structlog.git@16.1.0#egg=structlog')
+        assert c.return_code == 0
+        assert 'structlog' in p.pipfile['packages']
+        assert 'structlog' in p.lockfile['default']
+        assert 'six' in p.lockfile['default']
+        assert p.lockfile['default']['structlog']['ref'] == 'a39f6906a268fb2f4c365042b31d0200468fb492'
+        pipfile = Path(p.pipfile_path)
+        new_content = pipfile.read_bytes().replace(b'16.1.0', b'18.1.0')
+        pipfile.write_bytes(new_content)
+        c = p.pipenv('lock')
+        assert c.return_code == 0
+        assert p.lockfile['default']['structlog']['ref'] == 'a73fbd3a9c3cafb11f43168582083f839b883034'
+        assert 'structlog' in p.pipfile['packages']
+        assert 'structlog' in p.lockfile['default']
+        assert 'six' in p.lockfile['default']