diff --git a/Pipfile.lock b/Pipfile.lock index 29bc7eb0..deee6963 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -295,6 +295,8 @@ }, "pluggy": { "hashes": [ + "sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c", + "sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5", "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff" ], "version": "==0.6.0" diff --git a/pipenv/core.py b/pipenv/core.py index 5c065b06..848c530a 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -117,7 +117,7 @@ if PIPENV_NOSPIN: def which(command, location=None, allow_global=False): if location is None: - location = project.virtualenv_location + location = project.virtualenv_location or os.environ.get('VIRTUAL_ENV') if not allow_global: if os.name == 'nt': p = find_windows_executable( @@ -1060,7 +1060,7 @@ def do_lock( # TODO: be smarter about this. vcs_deps = convert_deps_to_pip(project.vcs_dev_packages, project, r=False) pip_freeze = delegator.run( - '{0} freeze'.format(escape_grouped_arguments(which_pip())) + '{0} -m pipenv.vendor.pip9 freeze'.format(escape_grouped_arguments(which('python', allow_global=system))) ).out if vcs_deps: for line in pip_freeze.strip().split('\n'): @@ -1125,7 +1125,7 @@ def do_lock( # Add refs for VCS installs. # TODO: be smarter about this. vcs_deps = convert_deps_to_pip(project.vcs_packages, project, r=False) - pip_freeze = delegator.run('{0} freeze'.format(which_pip())).out + pip_freeze = delegator.run('{0} -m pipenv.vendor.pip9 freeze'.format(which('python', allow_global=system))).out for dep in vcs_deps: for line in pip_freeze.strip().split('\n'): try: @@ -1228,8 +1228,8 @@ def do_purge(bare=False, downloads=False, allow_global=False, verbose=False): return freeze = delegator.run( - '{0} freeze'.format( - escape_grouped_arguments(which_pip(allow_global=allow_global)) + '{0} -m pipenv.vendor.pip9 freeze'.format( + escape_grouped_arguments(which('python', allow_global=allow_global)) ) ).out # Remove comments from the output, if any. @@ -1260,8 +1260,8 @@ def do_purge(bare=False, downloads=False, allow_global=False, verbose=False): len(actually_installed) ) ) - command = '{0} uninstall {1} -y'.format( - escape_grouped_arguments(which_pip(allow_global=allow_global)), + command = '{0} -m pipenv.vendor.pip9 uninstall {1} -y'.format( + escape_grouped_arguments(which('python', allow_global=allow_global)), ' '.join(actually_installed), ) if verbose: @@ -1441,11 +1441,11 @@ def pip_install( install_reqs += ' --require-hashes' no_deps = '--no-deps' if no_deps else '' pre = '--pre' if pre else '' - quoted_pip = which_pip(allow_global=allow_global) - quoted_pip = escape_grouped_arguments(quoted_pip) + quoted_python = which('python', allow_global=allow_global) + quoted_python = escape_grouped_arguments(quoted_python) upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else '' - pip_command = '{0} install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format( - quoted_pip, + pip_command = '{0} -m pipenv.vendor.pip9 install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format( + quoted_python, install_reqs, ' '.join(prepare_pip_source_args([source])), no_deps, @@ -1468,7 +1468,7 @@ def pip_install( def pip_download(package_name): for source in project.sources: cmd = '{0} download "{1}" -i {2} -d {3}'.format( - delegator.run(which_pip()), + escape_grouped_arguments(which_pip()), package_name, source['url'], project.download_location, @@ -2048,8 +2048,8 @@ def do_uninstall( sys.exit(1) for package_name in package_names: click.echo(u'Un-installing {0}…'.format(crayons.green(package_name))) - cmd = '"{0}" uninstall {1} -y'.format( - which_pip(allow_global=system), package_name + cmd = '"{0}" -m pipenv.vendor.pip9 uninstall {1} -y'.format( + which('python', allow_global=system), package_name ) if verbose: click.echo('$ {0}'.format(cmd)) diff --git a/pipenv/patched/safety.zip b/pipenv/patched/safety.zip index 6d09c24f..bef9faba 100644 Binary files a/pipenv/patched/safety.zip and b/pipenv/patched/safety.zip differ diff --git a/pipenv/vendor/Makefile b/pipenv/vendor/Makefile new file mode 100644 index 00000000..5c44fea4 --- /dev/null +++ b/pipenv/vendor/Makefile @@ -0,0 +1,14 @@ +# Taken from pip: https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/src/pip/_vendor/Makefile +all: clean vendor + +clean: + @# Delete vendored items + find . -maxdepth 1 -mindepth 1 -type d -exec rm -rf {} \; + +vendor: + @# Install vendored libraries + pip install -t . -r vendor.txt + + @# Cleanup .egg-info directories + rm -rf *.egg-info + rm -rf *.dist-info diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt new file mode 100644 index 00000000..90ac46d3 --- /dev/null +++ b/pipenv/vendor/vendor.txt @@ -0,0 +1,30 @@ +# appdirs==1.4.4 +backports.shutil_get_terminal_size==1.0.0 +backports.weakref==1.0.post1 +blindspin==2.0.1 +click==6.7 +click-completion==0.2.1 +click-didyoumean==0.0.3 +colorama==0.3.9 +delegator.py==0.1.0 +docopt==0.6.2 +python-dotenv==0.8.2 +first==2.0.1 +iso8601==0.1.12 +jinja2==2.9.5 +markupsafe==1.0 +parse==1.8.0 +pathlib2==2.1.0 +pexpect==4.2.1 +pip==9.0.3 +pipdeptree==0.10.1 +pipreqs==0.4.8 +ptyprocess==0.5.1 +pytoml==0.1.14 +requests==2.11.1 +requirements-parser==0.2.0 +six==1.10.0 +semver==2.7.8 +shutilwhich==1.1.0 +toml==0.9.2 +yarg==0.1.9 diff --git a/setup.py b/setup.py index f4ebf98b..996dc998 100644 --- a/setup.py +++ b/setup.py @@ -8,14 +8,19 @@ from shutil import rmtree from setuptools import find_packages, setup, Command here = os.path.abspath(os.path.dirname(__file__)) + with codecs.open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: long_description = '\n' + f.read() + about = {} + with open(os.path.join(here, "pipenv", "__version__.py")) as f: exec (f.read(), about) + if sys.argv[-1] == "publish": os.system("python setup.py sdist bdist_wheel upload") sys.exit() + required = [ 'pip>=9.0.1', 'certifi', diff --git a/tasks/__init__.py b/tasks/__init__.py new file mode 100644 index 00000000..5cb7fb29 --- /dev/null +++ b/tasks/__init__.py @@ -0,0 +1,8 @@ +# -*- coding=utf-8 -*- +# Copyied from pip's vendoring process +# see https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/tasks/__init__.py +import invoke + +from . import vendoring + +ns = invoke.Collection(vendoring) diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py new file mode 100644 index 00000000..4a77ecde --- /dev/null +++ b/tasks/vendoring/__init__.py @@ -0,0 +1,373 @@ +# -*- coding=utf-8 -*- +""""Vendoring script, python 3.5 needed""" +# Taken from pip +# see https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/tasks/vendoring/__init__.py +from pathlib import Path +from pipenv.utils import TemporaryDirectory, mkdir_p +import os +import re +import shutil +import sys +import invoke + +TASK_NAME = 'update' + +FILE_WHITE_LIST = ( + 'Makefile', + 'vendor.txt', + '__init__.py', + 'README.rst', + 'LICENSE*', + 'appdirs.py', + '*.LICENSE' +) + +FLATTEN = ( + 'click-completion', + 'delegator', + 'docopt', + 'first', + 'parse', + 'pathlib2', + 'pipdeptree', + 'semver', + 'six', + 'toml', +) + + +def drop_dir(path): + shutil.rmtree(str(path)) + + +def remove_all(paths): + for path in paths: + if path.is_dir(): + drop_dir(path) + else: + path.unlink() + + +def log(msg): + print('[vendoring.%s] %s' % (TASK_NAME, msg)) + + +def _get_vendor_dir(ctx): + git_root = ctx.run('git rev-parse --show-toplevel', hide=True).stdout + return Path(git_root.strip()) / 'pipenv' / 'vendor' + + +def _get_patched_dir(ctx): + git_root = ctx.run('git rev-parse --show-toplevel', hide=True).stdout + return Path(git_root.strip()) / 'pipenv' / 'patched' + + +def clean_vendor(ctx, vendor_dir): + # Old _vendor cleanup + remove_all(vendor_dir.glob('*.pyc')) + log('Cleaning %s' % vendor_dir) + for item in vendor_dir.iterdir(): + if item.is_dir(): + shutil.rmtree(str(item)) + elif item.name not in FILE_WHITE_LIST: + item.unlink() + else: + log('Skipping %s' % item) + + +def detect_vendored_libs(vendor_dir): + retval = [] + for item in vendor_dir.iterdir(): + if item.is_dir(): + retval.append(item.name) + elif item.name.endswith(".pyi"): + continue + elif item.name not in FILE_WHITE_LIST: + retval.append(item.name[:-3]) + return retval + + +def rewrite_imports(package_dir, vendored_libs): + for item in package_dir.iterdir(): + if item.is_dir(): + rewrite_imports(item, vendored_libs) + elif item.name.endswith('.py'): + rewrite_file_imports(item, vendored_libs) + + +def rewrite_file_imports(item, vendored_libs): + """Rewrite 'import xxx' and 'from xxx import' for vendored_libs""" + text = item.read_text(encoding='utf-8') + for lib in vendored_libs: + text = re.sub( + r'(\n\s*)import %s(\n\s*)' % lib, + r'\1from .vendor import %s\2' % lib, + text, + ) + text = re.sub( + r'(\n\s*)from %s' % lib, + r'\1from .vendor.%s' % lib, + text, + ) + item.write_text(text, encoding='utf-8') + + +def apply_patch(ctx, patch_file_path): + log('Applying patch %s' % patch_file_path.name) + ctx.run('git apply --verbose %s' % patch_file_path) + + +@invoke.task +def update_safety(ctx): + ignore_subdeps = ['pip', 'pip-egg-info', 'bin'] + ignore_files = ['pip-delete-this-directory.txt', 'PKG-INFO'] + vendor_dir = _get_patched_dir(ctx) + log('Using vendor dir: %s' % vendor_dir) + log('Downloading safety package files...') + build_dir = vendor_dir / 'build' + download_dir = TemporaryDirectory(prefix='pipenv-', suffix='-safety') + if build_dir.exists() and build_dir.is_dir(): + drop_dir(build_dir) + + ctx.run( + 'pip download -b {0} --no-binary=:all: --no-clean -d {1} safety pyyaml'.format( + str(build_dir), str(download_dir.name), + ) + ) + safety_dir = build_dir / 'safety' + yaml_build_dir = build_dir / 'pyyaml' + main_file = safety_dir / '__main__.py' + main_content = """ +from safety.cli import cli + +# Disable insecure warnings. +import requests +from requests.packages.urllib3.exceptions import InsecureRequestWarning +requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + +cli(prog_name="safety") + """.strip() + with open(str(main_file), 'w') as fh: + fh.write(main_content) + + with ctx.cd(str(safety_dir)): + ctx.run('pip install --no-compile --no-binary=:all: -t . .') + safety_dir = safety_dir.absolute() + yaml_dir = safety_dir / 'yaml' + if yaml_dir.exists(): + version_choices = ['2', '3'] + version_choices.remove(str(sys.version_info[0])) + mkdir_p(str(yaml_dir / 'yaml{0}'.format(sys.version_info[0]))) + for fn in yaml_dir.glob('*.py'): + fn.rename(str(fn.parent.joinpath('yaml{0}'.format(sys.version_info[0]), fn.name))) + if version_choices[0] == '2': + lib = yaml_build_dir / 'lib' / 'yaml' + else: + lib = yaml_build_dir / 'lib3' / 'yaml' + shutil.copytree(str(lib.absolute()), str(yaml_dir / 'yaml{0}'.format(version_choices[0]))) + yaml_init = yaml_dir / '__init__.py' + yaml_init.write_text(""" +import sys +if sys.version_info[0] == 3: + from .yaml3 import * +else: + from .yaml2 import * + """.strip()) + requests_dir = safety_dir / 'requests' + cacert = vendor_dir / 'requests' / 'cacert.pem' + if not cacert.exists(): + from pipenv.vendor import requests + cacert = Path(requests.certs.where()) + target_cert = requests_dir / 'cacert.pem' + target_cert.write_bytes(cacert.read_bytes()) + ctx.run("sed -i 's/r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers)/r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, verify=False)/g' {0}".format(str(safety_dir / 'safety' / 'safety.py'))) + for egg in safety_dir.glob('*.egg-info'): + drop_dir(egg.absolute()) + for dep in ignore_subdeps: + dep_dir = safety_dir / dep + if dep_dir.exists(): + drop_dir(dep_dir) + for dep in ignore_files: + fn = safety_dir / dep + if fn.exists(): + fn.unlink() + zip_name = '{0}/safety'.format(str(vendor_dir)) + shutil.make_archive(zip_name, format='zip', root_dir=str(safety_dir), base_dir='./') + drop_dir(build_dir) + download_dir.cleanup() + + +@invoke.task +def get_licenses(ctx): + vendor_dir = _get_vendor_dir(ctx) + log('Using vendor dir: %s' % vendor_dir) + log('Downloading LICENSE files...') + build_dir = vendor_dir / 'build' + download_dir = TemporaryDirectory(prefix='pipenv-', suffix='-licenses') + if build_dir.exists() and build_dir.is_dir(): + drop_dir(build_dir) + + ctx.run( + 'pip download -b {0} --no-binary=:all: --no-clean --no-deps -r {1}/vendor.txt -d {2}'.format( + str(build_dir), str(vendor_dir), str(download_dir.name), + ) + ) + for p in build_dir.glob('*/*LICENSE*'): + parent = p.parent + matches = [flat for flat in FLATTEN if parent.joinpath(flat).exists() or parent.name == flat] + egg_info_dir = [e for e in parent.glob('*.egg-info')] + if any(matches): + from pipenv.utils import pep423_name + pkg = pep423_name(matches[0]).lower() + pkg_name = pkg if parent.joinpath(pkg).exists() else parent.name.lower() + target_file = '{0}.LICENSE'.format(pkg_name) + target_file = vendor_dir / target_file + elif egg_info_dir: + egg_info_dir = egg_info_dir[0] + pkg_name = egg_info_dir.stem.lower() + target_file = vendor_dir / pkg_name / p.name.lower() + if '.' in pkg_name: + target_file = vendor_dir.joinpath(*pkg_name.split('.')) / p.name + else: + target_dir = vendor_dir / parent.name + if '.' in parent.name: + target_dir = vendor_dir.joinpath(*parent.name.split('.')) + target_file = target_dir / p.name.lower() + mkdir_p(str(target_file.parent.absolute())) + shutil.copyfile(str(p.absolute()), str(target_file.absolute())) + drop_dir(build_dir) + download_dir.cleanup() + + +def get_patched(ctx): + log('Reinstalling patched libraries') + patched_dir = _get_patched_dir(ctx) + ctx.run( + 'pip install -t {0} -r {0}/patched.txt --no-compile --no-deps'.format( + str(patched_dir), + ) + ) + remove_all(patched_dir.glob('*.dist_info')) + remove_all(patched_dir.glob('*.egg-info')) + # Cleanup setuptools unneeded parts + (patched_dir / 'easy_install.py').unlink() + drop_dir(patched_dir / 'setuptools') + drop_dir(patched_dir / 'pkg_resources' / '_vendor') + drop_dir(patched_dir / 'pkg_resources' / 'extern') + + # Drop interpreter and OS specific msgpack libs. + # Pip will rely on the python-only fallback instead. + remove_all(patched_dir.glob('msgpack/*.so')) + drop_dir(patched_dir / 'bin') + drop_dir(patched_dir / 'tests') + + # Detect the vendored packages/modules + vendored_libs = detect_vendored_libs(patched_dir) + log("Detected vendored libraries: %s" % ", ".join(vendored_libs)) + + # Global import rewrites + log("Rewriting all imports related to vendored libs") + for item in patched_dir.iterdir(): + if item.is_dir(): + rewrite_imports(item, vendored_libs) + elif item.name not in FILE_WHITE_LIST: + rewrite_file_imports(item, vendored_libs) + + # Special cases: apply stored patches + log("Apply patches") + patch_dir = Path(__file__).parent / 'patches' + for patch in patch_dir.glob('*.patch'): + apply_patch(ctx, patch) + + +def vendor(ctx, vendor_dir): + log('Reinstalling vendored libraries') + # We use --no-deps because we want to ensure that all of our dependencies + # are added to vendor.txt, this includes all dependencies recursively up + # the chain. + ctx.run( + 'pip install -t {0} -r {0}/vendor.txt --no-compile --no-deps'.format( + str(vendor_dir), + ) + ) + remove_all(vendor_dir.glob('*.dist-info')) + remove_all(vendor_dir.glob('*.egg-info')) + + # Cleanup setuptools unneeded parts + (vendor_dir / 'easy_install.py').unlink() + drop_dir(vendor_dir / 'setuptools') + drop_dir(vendor_dir / 'pkg_resources' / '_vendor') + drop_dir(vendor_dir / 'pkg_resources' / 'extern') + + # Drop interpreter and OS specific msgpack libs. + # Pip will rely on the python-only fallback instead. + remove_all(vendor_dir.glob('msgpack/*.so')) + drop_dir(vendor_dir / 'bin') + drop_dir(vendor_dir / 'tests') + + # Detect the vendored packages/modules + vendored_libs = detect_vendored_libs(vendor_dir) + log("Detected vendored libraries: %s" % ", ".join(vendored_libs)) + + # Global import rewrites + log("Rewriting all imports related to vendored libs") + for item in vendor_dir.iterdir(): + if item.is_dir(): + rewrite_imports(item, vendored_libs) + elif item.name not in FILE_WHITE_LIST: + rewrite_file_imports(item, vendored_libs) + + +@invoke.task +def rewrite_all_imports(ctx): + vendor_dir = _get_vendor_dir(ctx) + log('Using vendor dir: %s' % vendor_dir) + vendored_libs = detect_vendored_libs(vendor_dir) + log("Detected vendored libraries: %s" % ", ".join(vendored_libs)) + log("Rewriting all imports related to vendored libs") + for item in vendor_dir.iterdir(): + if item.is_dir(): + rewrite_imports(item, vendored_libs) + elif item.name not in FILE_WHITE_LIST: + rewrite_file_imports(item, vendored_libs) + + +@invoke.task +def update_stubs(ctx): + vendor_dir = _get_vendor_dir(ctx) + vendored_libs = detect_vendored_libs(vendor_dir) + + print("[vendoring.update_stubs] Add mypy stubs") + + extra_stubs_needed = { + # Some projects need stubs other than a simple .pyi + "six": ["six.__init__", "six.moves"], + # Some projects should not have stubs coz they're single file modules + "appdirs": [], + } + + for lib in vendored_libs: + if lib not in extra_stubs_needed: + (vendor_dir / (lib + ".pyi")).write_text("from %s import *" % lib) + continue + + for selector in extra_stubs_needed[lib]: + fname = selector.replace(".", os.sep) + ".pyi" + if selector.endswith(".__init__"): + selector = selector[:-9] + + f_path = vendor_dir / fname + if not f_path.parent.exists(): + f_path.parent.mkdir() + f_path.write_text("from %s import *" % selector) + + +@invoke.task(name=TASK_NAME, post=[update_stubs]) +def main(ctx): + vendor_dir = _get_vendor_dir(ctx) + log('Using vendor dir: %s' % vendor_dir) + clean_vendor(ctx, vendor_dir) + vendor(ctx, vendor_dir) + get_licenses(ctx) + update_safety(ctx) + log('Revendoring complete')