Merge branch 'master' into patch-2

Pipfile.lock

@@ -295,6 +295,8 @@
         },
         "pluggy": {
             "hashes": [
+                "sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c",
+                "sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5",
                 "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff"
             ],
             "version": "==0.6.0"

pipenv/core.py

@@ -117,7 +117,7 @@ if PIPENV_NOSPIN:
 
 def which(command, location=None, allow_global=False):
     if location is None:
-        location = project.virtualenv_location
+        location = project.virtualenv_location or os.environ.get('VIRTUAL_ENV')
     if not allow_global:
         if os.name == 'nt':
             p = find_windows_executable(
@@ -1060,7 +1060,7 @@ def do_lock(
     # TODO: be smarter about this.
     vcs_deps = convert_deps_to_pip(project.vcs_dev_packages, project, r=False)
     pip_freeze = delegator.run(
-        '{0} freeze'.format(escape_grouped_arguments(which_pip()))
+        '{0} -m pipenv.vendor.pip9 freeze'.format(escape_grouped_arguments(which('python', allow_global=system)))
     ).out
     if vcs_deps:
         for line in pip_freeze.strip().split('\n'):
@@ -1125,7 +1125,7 @@ def do_lock(
     # Add refs for VCS installs.
     # TODO: be smarter about this.
     vcs_deps = convert_deps_to_pip(project.vcs_packages, project, r=False)
-    pip_freeze = delegator.run('{0} freeze'.format(which_pip())).out
+    pip_freeze = delegator.run('{0} -m pipenv.vendor.pip9 freeze'.format(which('python', allow_global=system))).out
     for dep in vcs_deps:
         for line in pip_freeze.strip().split('\n'):
             try:
@@ -1228,8 +1228,8 @@ def do_purge(bare=False, downloads=False, allow_global=False, verbose=False):
         return
 
     freeze = delegator.run(
-        '{0} freeze'.format(
-            escape_grouped_arguments(which_pip(allow_global=allow_global))
+        '{0} -m pipenv.vendor.pip9 freeze'.format(
+            escape_grouped_arguments(which('python', allow_global=allow_global))
         )
     ).out
     # Remove comments from the output, if any.
@@ -1260,8 +1260,8 @@ def do_purge(bare=False, downloads=False, allow_global=False, verbose=False):
                 len(actually_installed)
             )
         )
-    command = '{0} uninstall {1} -y'.format(
-        escape_grouped_arguments(which_pip(allow_global=allow_global)),
+    command = '{0} -m pipenv.vendor.pip9 uninstall {1} -y'.format(
+        escape_grouped_arguments(which('python', allow_global=allow_global)),
         ' '.join(actually_installed),
     )
     if verbose:
@@ -1441,11 +1441,11 @@ def pip_install(
             install_reqs += ' --require-hashes'
         no_deps = '--no-deps' if no_deps else ''
         pre = '--pre' if pre else ''
-        quoted_pip = which_pip(allow_global=allow_global)
-        quoted_pip = escape_grouped_arguments(quoted_pip)
+        quoted_python = which('python', allow_global=allow_global)
+        quoted_python = escape_grouped_arguments(quoted_python)
         upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else ''
-        pip_command = '{0} install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format(
-            quoted_pip,
+        pip_command = '{0} -m pipenv.vendor.pip9 install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format(
+            quoted_python,
             install_reqs,
             ' '.join(prepare_pip_source_args([source])),
             no_deps,
@@ -1468,7 +1468,7 @@ def pip_install(
 def pip_download(package_name):
     for source in project.sources:
         cmd = '{0} download "{1}" -i {2} -d {3}'.format(
-            delegator.run(which_pip()),
+            escape_grouped_arguments(which_pip()),
             package_name,
             source['url'],
             project.download_location,
@@ -2048,8 +2048,8 @@ def do_uninstall(
         sys.exit(1)
     for package_name in package_names:
         click.echo(u'Un-installing {0}…'.format(crayons.green(package_name)))
-        cmd = '"{0}" uninstall {1} -y'.format(
-            which_pip(allow_global=system), package_name
+        cmd = '"{0}" -m pipenv.vendor.pip9 uninstall {1} -y'.format(
+            which('python', allow_global=system), package_name
         )
         if verbose:
             click.echo('$ {0}'.format(cmd))

pipenv/vendor/Makefile

@@ -0,0 +1,14 @@
+# Taken from pip: https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/src/pip/_vendor/Makefile
+all: clean vendor
+
+clean:
+	@# Delete vendored items
+	find . -maxdepth 1 -mindepth 1 -type d -exec rm -rf {} \;
+
+vendor:
+	@# Install vendored libraries
+	pip install -t . -r vendor.txt
+
+	@# Cleanup .egg-info directories
+	rm -rf *.egg-info
+	rm -rf *.dist-info

pipenv/vendor/vendor.txt

@@ -0,0 +1,30 @@
+# appdirs==1.4.4
+backports.shutil_get_terminal_size==1.0.0
+backports.weakref==1.0.post1
+blindspin==2.0.1
+click==6.7
+click-completion==0.2.1
+click-didyoumean==0.0.3
+colorama==0.3.9
+delegator.py==0.1.0
+docopt==0.6.2
+python-dotenv==0.8.2
+first==2.0.1
+iso8601==0.1.12
+jinja2==2.9.5
+markupsafe==1.0
+parse==1.8.0
+pathlib2==2.1.0
+pexpect==4.2.1
+pip==9.0.3
+pipdeptree==0.10.1
+pipreqs==0.4.8
+ptyprocess==0.5.1
+pytoml==0.1.14
+requests==2.11.1
+requirements-parser==0.2.0
+six==1.10.0
+semver==2.7.8
+shutilwhich==1.1.0
+toml==0.9.2
+yarg==0.1.9

setup.py

@@ -8,14 +8,19 @@ from shutil import rmtree
 from setuptools import find_packages, setup, Command
 
 here = os.path.abspath(os.path.dirname(__file__))
+
 with codecs.open(os.path.join(here, 'README.rst'), encoding='utf-8') as f:
     long_description = '\n' + f.read()
+
 about = {}
+
 with open(os.path.join(here, "pipenv", "__version__.py")) as f:
     exec (f.read(), about)
+
 if sys.argv[-1] == "publish":
     os.system("python setup.py sdist bdist_wheel upload")
     sys.exit()
+
 required = [
     'pip>=9.0.1',
     'certifi',

tasks/__init__.py

@@ -0,0 +1,8 @@
+# -*- coding=utf-8 -*-
+# Copyied from pip's vendoring process
+# see https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/tasks/__init__.py
+import invoke
+
+from . import vendoring
+
+ns = invoke.Collection(vendoring)

tasks/vendoring/__init__.py

@@ -0,0 +1,373 @@
+# -*- coding=utf-8 -*-
+""""Vendoring script, python 3.5 needed"""
+# Taken from pip
+# see https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/tasks/vendoring/__init__.py
+from pathlib import Path
+from pipenv.utils import TemporaryDirectory, mkdir_p
+import os
+import re
+import shutil
+import sys
+import invoke
+
+TASK_NAME = 'update'
+
+FILE_WHITE_LIST = (
+    'Makefile',
+    'vendor.txt',
+    '__init__.py',
+    'README.rst',
+    'LICENSE*',
+    'appdirs.py',
+    '*.LICENSE'
+)
+
+FLATTEN = (
+    'click-completion',
+    'delegator',
+    'docopt',
+    'first',
+    'parse',
+    'pathlib2',
+    'pipdeptree',
+    'semver',
+    'six',
+    'toml',
+)
+
+
+def drop_dir(path):
+    shutil.rmtree(str(path))
+
+
+def remove_all(paths):
+    for path in paths:
+        if path.is_dir():
+            drop_dir(path)
+        else:
+            path.unlink()
+
+
+def log(msg):
+    print('[vendoring.%s] %s' % (TASK_NAME, msg))
+
+
+def _get_vendor_dir(ctx):
+    git_root = ctx.run('git rev-parse --show-toplevel', hide=True).stdout
+    return Path(git_root.strip()) / 'pipenv' / 'vendor'
+
+
+def _get_patched_dir(ctx):
+    git_root = ctx.run('git rev-parse --show-toplevel', hide=True).stdout
+    return Path(git_root.strip()) / 'pipenv' / 'patched'
+
+
+def clean_vendor(ctx, vendor_dir):
+    # Old _vendor cleanup
+    remove_all(vendor_dir.glob('*.pyc'))
+    log('Cleaning %s' % vendor_dir)
+    for item in vendor_dir.iterdir():
+        if item.is_dir():
+            shutil.rmtree(str(item))
+        elif item.name not in FILE_WHITE_LIST:
+            item.unlink()
+        else:
+            log('Skipping %s' % item)
+
+
+def detect_vendored_libs(vendor_dir):
+    retval = []
+    for item in vendor_dir.iterdir():
+        if item.is_dir():
+            retval.append(item.name)
+        elif item.name.endswith(".pyi"):
+            continue
+        elif item.name not in FILE_WHITE_LIST:
+            retval.append(item.name[:-3])
+    return retval
+
+
+def rewrite_imports(package_dir, vendored_libs):
+    for item in package_dir.iterdir():
+        if item.is_dir():
+            rewrite_imports(item, vendored_libs)
+        elif item.name.endswith('.py'):
+            rewrite_file_imports(item, vendored_libs)
+
+
+def rewrite_file_imports(item, vendored_libs):
+    """Rewrite 'import xxx' and 'from xxx import' for vendored_libs"""
+    text = item.read_text(encoding='utf-8')
+    for lib in vendored_libs:
+        text = re.sub(
+            r'(\n\s*)import %s(\n\s*)' % lib,
+            r'\1from .vendor import %s\2' % lib,
+            text,
+        )
+        text = re.sub(
+            r'(\n\s*)from %s' % lib,
+            r'\1from .vendor.%s' % lib,
+            text,
+        )
+    item.write_text(text, encoding='utf-8')
+
+
+def apply_patch(ctx, patch_file_path):
+    log('Applying patch %s' % patch_file_path.name)
+    ctx.run('git apply --verbose %s' % patch_file_path)
+
+
+@invoke.task
+def update_safety(ctx):
+    ignore_subdeps = ['pip', 'pip-egg-info', 'bin']
+    ignore_files = ['pip-delete-this-directory.txt', 'PKG-INFO']
+    vendor_dir = _get_patched_dir(ctx)
+    log('Using vendor dir: %s' % vendor_dir)
+    log('Downloading safety package files...')
+    build_dir = vendor_dir / 'build'
+    download_dir = TemporaryDirectory(prefix='pipenv-', suffix='-safety')
+    if build_dir.exists() and build_dir.is_dir():
+        drop_dir(build_dir)
+
+    ctx.run(
+        'pip download -b {0} --no-binary=:all: --no-clean -d {1} safety pyyaml'.format(
+            str(build_dir), str(download_dir.name),
+        )
+    )
+    safety_dir = build_dir / 'safety'
+    yaml_build_dir = build_dir / 'pyyaml'
+    main_file = safety_dir / '__main__.py'
+    main_content = """
+from safety.cli import cli
+
+# Disable insecure warnings.
+import requests
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+
+cli(prog_name="safety")
+    """.strip()
+    with open(str(main_file), 'w') as fh:
+        fh.write(main_content)
+
+    with ctx.cd(str(safety_dir)):
+        ctx.run('pip install --no-compile --no-binary=:all: -t . .')
+        safety_dir = safety_dir.absolute()
+        yaml_dir = safety_dir / 'yaml'
+        if yaml_dir.exists():
+            version_choices = ['2', '3']
+            version_choices.remove(str(sys.version_info[0]))
+            mkdir_p(str(yaml_dir / 'yaml{0}'.format(sys.version_info[0])))
+            for fn in yaml_dir.glob('*.py'):
+                fn.rename(str(fn.parent.joinpath('yaml{0}'.format(sys.version_info[0]), fn.name)))
+            if version_choices[0] == '2':
+                lib = yaml_build_dir / 'lib' / 'yaml'
+            else:
+                lib = yaml_build_dir / 'lib3' / 'yaml'
+            shutil.copytree(str(lib.absolute()), str(yaml_dir / 'yaml{0}'.format(version_choices[0])))
+            yaml_init = yaml_dir / '__init__.py'
+            yaml_init.write_text("""
+import sys
+if sys.version_info[0] == 3:
+    from .yaml3 import *
+else:
+    from .yaml2 import *
+            """.strip())
+        requests_dir = safety_dir / 'requests'
+        cacert = vendor_dir / 'requests' / 'cacert.pem'
+        if not cacert.exists():
+            from pipenv.vendor import requests
+            cacert = Path(requests.certs.where())
+        target_cert = requests_dir / 'cacert.pem'
+        target_cert.write_bytes(cacert.read_bytes())
+        ctx.run("sed -i 's/r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers)/r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, verify=False)/g' {0}".format(str(safety_dir / 'safety' / 'safety.py')))
+        for egg in safety_dir.glob('*.egg-info'):
+            drop_dir(egg.absolute())
+        for dep in ignore_subdeps:
+            dep_dir = safety_dir / dep
+            if dep_dir.exists():
+                drop_dir(dep_dir)
+        for dep in ignore_files:
+            fn = safety_dir / dep
+            if fn.exists():
+                fn.unlink()
+    zip_name = '{0}/safety'.format(str(vendor_dir))
+    shutil.make_archive(zip_name, format='zip', root_dir=str(safety_dir), base_dir='./')
+    drop_dir(build_dir)
+    download_dir.cleanup()
+
+
+@invoke.task
+def get_licenses(ctx):
+    vendor_dir = _get_vendor_dir(ctx)
+    log('Using vendor dir: %s' % vendor_dir)
+    log('Downloading LICENSE files...')
+    build_dir = vendor_dir / 'build'
+    download_dir = TemporaryDirectory(prefix='pipenv-', suffix='-licenses')
+    if build_dir.exists() and build_dir.is_dir():
+        drop_dir(build_dir)
+
+    ctx.run(
+        'pip download -b {0} --no-binary=:all: --no-clean --no-deps -r {1}/vendor.txt -d {2}'.format(
+            str(build_dir), str(vendor_dir), str(download_dir.name),
+        )
+    )
+    for p in build_dir.glob('*/*LICENSE*'):
+        parent = p.parent
+        matches = [flat for flat in FLATTEN if parent.joinpath(flat).exists() or parent.name == flat]
+        egg_info_dir = [e for e in parent.glob('*.egg-info')]
+        if any(matches):
+            from pipenv.utils import pep423_name
+            pkg = pep423_name(matches[0]).lower()                        
+            pkg_name = pkg if parent.joinpath(pkg).exists() else parent.name.lower()
+            target_file = '{0}.LICENSE'.format(pkg_name)
+            target_file = vendor_dir / target_file
+        elif egg_info_dir:
+            egg_info_dir = egg_info_dir[0]
+            pkg_name = egg_info_dir.stem.lower()
+            target_file = vendor_dir / pkg_name / p.name.lower()
+            if '.' in pkg_name:
+                target_file = vendor_dir.joinpath(*pkg_name.split('.')) / p.name
+        else:
+            target_dir = vendor_dir / parent.name
+            if '.' in parent.name:
+                target_dir = vendor_dir.joinpath(*parent.name.split('.'))
+            target_file = target_dir / p.name.lower()
+        mkdir_p(str(target_file.parent.absolute()))
+        shutil.copyfile(str(p.absolute()), str(target_file.absolute()))
+    drop_dir(build_dir)
+    download_dir.cleanup()
+
+
+def get_patched(ctx):
+    log('Reinstalling patched libraries')
+    patched_dir = _get_patched_dir(ctx)
+    ctx.run(
+        'pip install -t {0} -r {0}/patched.txt --no-compile --no-deps'.format(
+            str(patched_dir),
+        )
+    )
+    remove_all(patched_dir.glob('*.dist_info'))
+    remove_all(patched_dir.glob('*.egg-info'))
+    # Cleanup setuptools unneeded parts
+    (patched_dir / 'easy_install.py').unlink()
+    drop_dir(patched_dir / 'setuptools')
+    drop_dir(patched_dir / 'pkg_resources' / '_vendor')
+    drop_dir(patched_dir / 'pkg_resources' / 'extern')
+
+    # Drop interpreter and OS specific msgpack libs.
+    # Pip will rely on the python-only fallback instead.
+    remove_all(patched_dir.glob('msgpack/*.so'))
+    drop_dir(patched_dir / 'bin')
+    drop_dir(patched_dir / 'tests')
+
+    # Detect the vendored packages/modules
+    vendored_libs = detect_vendored_libs(patched_dir)
+    log("Detected vendored libraries: %s" % ", ".join(vendored_libs))
+
+    # Global import rewrites
+    log("Rewriting all imports related to vendored libs")
+    for item in patched_dir.iterdir():
+        if item.is_dir():
+            rewrite_imports(item, vendored_libs)
+        elif item.name not in FILE_WHITE_LIST:
+            rewrite_file_imports(item, vendored_libs)
+
+    # Special cases: apply stored patches
+    log("Apply patches")
+    patch_dir = Path(__file__).parent / 'patches'
+    for patch in patch_dir.glob('*.patch'):
+        apply_patch(ctx, patch)
+
+
+def vendor(ctx, vendor_dir):
+    log('Reinstalling vendored libraries')
+    # We use --no-deps because we want to ensure that all of our dependencies
+    # are added to vendor.txt, this includes all dependencies recursively up
+    # the chain.
+    ctx.run(
+        'pip install -t {0} -r {0}/vendor.txt --no-compile --no-deps'.format(
+            str(vendor_dir),
+        )
+    )
+    remove_all(vendor_dir.glob('*.dist-info'))
+    remove_all(vendor_dir.glob('*.egg-info'))
+
+    # Cleanup setuptools unneeded parts
+    (vendor_dir / 'easy_install.py').unlink()
+    drop_dir(vendor_dir / 'setuptools')
+    drop_dir(vendor_dir / 'pkg_resources' / '_vendor')
+    drop_dir(vendor_dir / 'pkg_resources' / 'extern')
+
+    # Drop interpreter and OS specific msgpack libs.
+    # Pip will rely on the python-only fallback instead.
+    remove_all(vendor_dir.glob('msgpack/*.so'))
+    drop_dir(vendor_dir / 'bin')
+    drop_dir(vendor_dir / 'tests')
+
+    # Detect the vendored packages/modules
+    vendored_libs = detect_vendored_libs(vendor_dir)
+    log("Detected vendored libraries: %s" % ", ".join(vendored_libs))
+
+    # Global import rewrites
+    log("Rewriting all imports related to vendored libs")
+    for item in vendor_dir.iterdir():
+        if item.is_dir():
+            rewrite_imports(item, vendored_libs)
+        elif item.name not in FILE_WHITE_LIST:
+            rewrite_file_imports(item, vendored_libs)
+
+
+@invoke.task
+def rewrite_all_imports(ctx):
+    vendor_dir = _get_vendor_dir(ctx)
+    log('Using vendor dir: %s' % vendor_dir)
+    vendored_libs = detect_vendored_libs(vendor_dir)
+    log("Detected vendored libraries: %s" % ", ".join(vendored_libs))
+    log("Rewriting all imports related to vendored libs")
+    for item in vendor_dir.iterdir():
+        if item.is_dir():
+            rewrite_imports(item, vendored_libs)
+        elif item.name not in FILE_WHITE_LIST:
+            rewrite_file_imports(item, vendored_libs)
+
+
+@invoke.task
+def update_stubs(ctx):
+    vendor_dir = _get_vendor_dir(ctx)
+    vendored_libs = detect_vendored_libs(vendor_dir)
+
+    print("[vendoring.update_stubs] Add mypy stubs")
+
+    extra_stubs_needed = {
+        # Some projects need stubs other than a simple <name>.pyi
+        "six": ["six.__init__", "six.moves"],
+        # Some projects should not have stubs coz they're single file modules
+        "appdirs": [],
+    }
+
+    for lib in vendored_libs:
+        if lib not in extra_stubs_needed:
+            (vendor_dir / (lib + ".pyi")).write_text("from %s import *" % lib)
+            continue
+
+        for selector in extra_stubs_needed[lib]:
+            fname = selector.replace(".", os.sep) + ".pyi"
+            if selector.endswith(".__init__"):
+                selector = selector[:-9]
+
+            f_path = vendor_dir / fname
+            if not f_path.parent.exists():
+                f_path.parent.mkdir()
+            f_path.write_text("from %s import *" % selector)
+
+
+@invoke.task(name=TASK_NAME, post=[update_stubs])
+def main(ctx):
+    vendor_dir = _get_vendor_dir(ctx)
+    log('Using vendor dir: %s' % vendor_dir)
+    clean_vendor(ctx, vendor_dir)
+    vendor(ctx, vendor_dir)
+    get_licenses(ctx)
+    update_safety(ctx)
+    log('Revendoring complete')