From 4dac1676579473e257f68686abdc94191dee237d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 22 Oct 2018 10:16:04 -0400 Subject: [PATCH] Vendor boltons Signed-off-by: Dan Ryan Update vendored dependencies Signed-off-by: Dan Ryan Fix file handle leaks - Fix #3020 - Fix #3088 - Patch delegator - Add weakref finalizer for tempfiles Signed-off-by: Dan Ryan Fix spinner handlers on windows Signed-off-by: Dan Ryan Fix spinner output and encoding issue Signed-off-by: Dan Ryan fix encoding Signed-off-by: Dan Ryan Fix unicode output on windows, fix tomlkit imports Signed-off-by: Dan Ryan Unvendor boltons, fix compatibility, update merge functionalities Signed-off-by: Dan Ryan Update pythonfinder, vistir version, requirementslib version Signed-off-by: Dan Ryan Fix vendoring script Signed-off-by: Dan Ryan Silence pip version checks Signed-off-by: Dan Ryan Add debugging to locking Signed-off-by: Dan Ryan --- news/{3020.feature => 3020.feature.rst} | 0 news/3088.bugfix.rst | 1 + news/3089.feature.rst | 1 + news/3089.vendor.rst | 11 + pipenv/__init__.py | 1 + pipenv/_compat.py | 6 +- pipenv/core.py | 535 +++++++++--------- pipenv/environments.py | 9 +- pipenv/project.py | 76 ++- pipenv/resolver.py | 86 +-- pipenv/utils.py | 195 +++---- pipenv/vendor/delegator.py | 17 +- pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/models/path.py | 1 - pipenv/vendor/requirementslib/__init__.py | 9 +- pipenv/vendor/requirementslib/exceptions.py | 2 + pipenv/vendor/requirementslib/models/cache.py | 16 +- .../vendor/requirementslib/models/lockfile.py | 26 + .../vendor/requirementslib/models/pipfile.py | 14 +- .../requirementslib/models/requirements.py | 48 +- .../requirementslib/models/resolvers.py | 11 +- pipenv/vendor/requirementslib/models/utils.py | 23 +- pipenv/vendor/requirementslib/utils.py | 283 ++++++++- pipenv/vendor/tomlkit/items.py | 2 +- pipenv/vendor/tomlkit/toml_char.py | 2 +- pipenv/vendor/vendor.txt | 6 +- pipenv/vendor/vistir/__init__.py | 7 +- pipenv/vendor/vistir/backports/tempfile.py | 6 +- pipenv/vendor/vistir/compat.py | 3 +- pipenv/vendor/vistir/contextmanagers.py | 26 +- pipenv/vendor/vistir/misc.py | 6 +- pipenv/vendor/vistir/path.py | 71 ++- pipenv/vendor/vistir/spin.py | 49 +- pipenv/vendor/vistir/termcolors.py | 4 +- pipenv/vendor/yaspin/core.py | 27 +- tasks/vendoring/__init__.py | 2 + .../vendor/delegator-close-filehandles.patch | 57 +- .../patches/vendor/vistir-imports.patch | 49 +- .../vendor/yaspin-signal-handling.patch | 62 ++ tests/integration/conftest.py | 138 ++++- tests/integration/test_lock.py | 4 +- tests/unit/test_utils.py | 11 +- 42 files changed, 1339 insertions(+), 566 deletions(-) rename news/{3020.feature => 3020.feature.rst} (100%) create mode 100644 news/3088.bugfix.rst create mode 100644 news/3089.feature.rst create mode 100644 news/3089.vendor.rst create mode 100644 tasks/vendoring/patches/vendor/yaspin-signal-handling.patch diff --git a/news/3020.feature b/news/3020.feature.rst similarity index 100% rename from news/3020.feature rename to news/3020.feature.rst diff --git a/news/3088.bugfix.rst b/news/3088.bugfix.rst new file mode 100644 index 00000000..b10c4b2b --- /dev/null +++ b/news/3088.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which caused ``Unexpected EOF`` errors to be thrown when PIP awaited input from users who put login credentials in their environment. diff --git a/news/3089.feature.rst b/news/3089.feature.rst new file mode 100644 index 00000000..47f280ee --- /dev/null +++ b/news/3089.feature.rst @@ -0,0 +1 @@ +Added windows-compatible spinner via upgraded ``vistir`` dependency. diff --git a/news/3089.vendor.rst b/news/3089.vendor.rst new file mode 100644 index 00000000..c57209f6 --- /dev/null +++ b/news/3089.vendor.rst @@ -0,0 +1,11 @@ +Updated vendored dependencies: + - ``certifi 2018.08.24 => 2018.10.15`` + - ``urllib3 1.23 => 1.24`` + - ``requests 2.19.1 => 2.20.0`` + - ``shellingham ``1.2.6 => 1.2.7`` + - ``tomlkit 0.4.4. => 0.4.6`` + - ``vistir 0.1.6 => 0.1.8`` + - ``pythonfinder 0.1.2 => 0.1.3`` + - ``requirementslib 1.1.9 => 1.1.10`` + - ``backports.functools_lru_cache 1.5.0 (new)`` + - ``cursor 1.2.0 (new)`` diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 471dcb94..1fea44d5 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -14,6 +14,7 @@ PIPENV_PATCHED = os.sep.join([PIPENV_ROOT, "patched"]) sys.path.insert(0, PIPENV_VENDOR) # Inject patched directory into system path. sys.path.insert(0, PIPENV_PATCHED) +os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" # Hack to make things work better. try: if "concurrency" in sys.modules: diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 558df3b8..223baec1 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -59,10 +59,10 @@ except ImportError: return False -if six.PY2: +from vistir.compat import ResourceWarning - class ResourceWarning(Warning): - pass + +warnings.filterwarnings("ignore", category=ResourceWarning) def pip_import(module_path, subimport=None, old_path=None): diff --git a/pipenv/core.py b/pipenv/core.py index ca37bd5c..82c11f1d 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -5,7 +5,6 @@ import os import sys import shutil import time -import tempfile import json as simplejson import click import click_completion @@ -13,10 +12,12 @@ import crayons import dotenv import delegator import pipfile -from blindspin import spinner import vistir +import warnings import six +import urllib3.util as urllib3_util + from .cmdparse import Script from .project import Project, SourceNotFound from .utils import ( @@ -24,8 +25,6 @@ from .utils import ( is_required_version, proper_case, pep423_name, - split_file, - merge_deps, venv_resolve_deps, escape_grouped_arguments, python_version, @@ -45,9 +44,7 @@ from .utils import ( from . import environments, pep508checker, progress from .environments import ( PIPENV_COLORBLIND, - PIPENV_NOSPIN, PIPENV_SHELL_FANCY, - PIPENV_TIMEOUT, PIPENV_SKIP_VALIDATION, PIPENV_HIDE_EMOJIS, PIPENV_YES, @@ -96,12 +93,36 @@ click_completion.init() # Disable colors, for the color blind and others who do not prefer colors. if PIPENV_COLORBLIND: crayons.disable() -# Disable spinner, for cleaner build logs (the unworthy). -if PIPENV_NOSPIN: - @contextlib.contextmanager # noqa: F811 - def spinner(): - yield + +UNICODE_TO_ASCII_TRANSLATION_MAP = { + 8230: u"...", + 8211: u"-" +} + + +def fix_utf8(text): + if not isinstance(text, six.string_types): + return text + if six.PY2: + text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) + return u"{0}".format(text) + + +@contextlib.contextmanager +def spinner(text=None, nospin=None, spinner_name=None): + if not text: + text = "Running..." + if not spinner_name: + spinner_name = environments.PIPENV_SPINNER + if nospin is None: + nospin = environments.PIPENV_NOSPIN + with vistir.spin.create_spinner( + spinner_name=spinner_name, + start_text=text, + nospin=nospin + ) as sp: + yield sp def which(command, location=None, allow_global=False): @@ -132,7 +153,7 @@ project = Project(which=which) def do_clear(): - click.echo(crayons.white("Clearing caches…", bold=True)) + click.echo(crayons.white(fix_utf8("Clearing caches…"), bold=True)) try: from pip._internal import locations except ImportError: # pip 9. @@ -161,7 +182,7 @@ def load_dot_env(): if os.path.isfile(dotenv_file): click.echo( - crayons.normal("Loading .env environment variables…", bold=True), + crayons.normal(fix_utf8("Loading .env environment variables…"), bold=True), err=True, ) else: @@ -295,7 +316,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): if project.requirements_exists and not skip_requirements: click.echo( crayons.normal( - u"requirements.txt found, instead of Pipfile! Converting…", + fix_utf8("requirements.txt found, instead of Pipfile! Converting…"), bold=True, ) ) @@ -317,7 +338,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): ) else: click.echo( - crayons.normal(u"Creating a Pipfile for this project…", bold=True), + crayons.normal(fix_utf8("Creating a Pipfile for this project…"), bold=True), err=True, ) # Create the pipfile if it doesn't exist. @@ -405,7 +426,7 @@ def ensure_python(three=None, python=None): u"{0}: Python {1} {2}".format( crayons.red("Warning", bold=True), crayons.blue(python), - u"was not found on your system…", + fix_utf8("was not found on your system…"), ), err=True, ) @@ -424,7 +445,7 @@ def ensure_python(three=None, python=None): except ValueError: abort() except PyenvError as e: - click.echo(u"Something went wrong…") + click.echo(fix_utf8("Something went wrong…")) click.echo(crayons.blue(e.err), err=True) abort() s = "{0} {1} {2}".format( @@ -443,14 +464,14 @@ def ensure_python(three=None, python=None): crayons.green(u"CPython {0}".format(version), bold=True), crayons.normal(u"with pyenv", bold=True), crayons.normal(u"(this may take a few minutes)"), - crayons.normal(u"…", bold=True), + crayons.normal(fix_utf8("…"), bold=True), ) ) with spinner(): try: c = pyenv.install(version) except PyenvError as e: - click.echo(u"Something went wrong…") + click.echo(fix_utf8("Something went wrong…")) click.echo(crayons.blue(e.err), err=True) # Print the results, in a beautiful blue… click.echo(crayons.blue(c.out), err=True) @@ -518,7 +539,7 @@ def ensure_virtualenv(three=None, python=None, site_packages=False, pypi_mirror= ): abort() click.echo( - crayons.normal(u"Removing existing virtualenv…", bold=True), err=True + crayons.normal(fix_utf8("Removing existing virtualenv…"), bold=True), err=True ) # Remove the virtualenv. cleanup_virtualenv(bare=True) @@ -665,26 +686,31 @@ def do_install_dependencies( If requirements is True, simply spits out a requirements format to stdout. """ - from .vendor.requirementslib.models.requirements import Requirement + from six.moves import queue def cleanup_procs(procs, concurrent): - for c in procs: - if concurrent: - c.block() + while not procs.empty(): + c = procs.get() + # if concurrent: + c.block() + failed = False + if c.return_code != 0: + failed = True if "Ignoring" in c.out: click.echo(crayons.yellow(c.out.strip())) elif environments.is_verbose(): - click.echo(crayons.blue(c.out or c.err)) + click.echo(crayons.blue(c.out.strip() or c.err.strip())) # The Installation failed… - if c.return_code != 0: + if failed: # Save the Failed Dependency for later. - failed_deps_list.append((c.dep, c.ignore_hash)) + dep = c.dep.copy() + failed_deps_list.append(dep) # Alert the user. click.echo( "{0} {1}! Will try again.".format( crayons.red("An error occurred while installing"), - crayons.green(c.dep.as_line()), - ) + crayons.green(dep.as_line()), + ), err=True ) if requirements: @@ -694,143 +720,138 @@ def do_install_dependencies( if skip_lock or only or not project.lockfile_exists: if not bare: click.echo( - crayons.normal(u"Installing dependencies from Pipfile…", bold=True) + crayons.normal(fix_utf8("Installing dependencies from Pipfile…"), bold=True) ) - lockfile = split_file(project._lockfile) + lockfile = project.get_or_create_lockfile() else: - with open(project.lockfile_location) as f: - lockfile = split_file(simplejson.load(f)) + lockfile = project.get_or_create_lockfile() if not bare: click.echo( crayons.normal( - u"Installing dependencies from Pipfile.lock ({0})…".format( + fix_utf8("Installing dependencies from Pipfile.lock ({0})…".format( lockfile["_meta"].get("hash", {}).get("sha256")[-6:] - ), + )), bold=True, ) ) # Allow pip to resolve dependencies when in skip-lock mode. no_deps = not skip_lock - deps_list, dev_deps_list = merge_deps( - lockfile, - project, - dev=dev, - requirements=requirements, - ignore_hashes=ignore_hashes, - blocking=blocking, - only=only, - ) failed_deps_list = [] + deps_list = list(lockfile.get_requirements(dev=dev, only=only)) if requirements: index_args = prepare_pip_source_args(project.sources) index_args = " ".join(index_args).replace(" -", "\n-") - deps_list = [dep for dep, ignore_hash, block in deps_list] - dev_deps_list = [dep for dep, ignore_hash, block in dev_deps_list] + deps = [ + req.as_line(sources=project.sources, include_hashes=False) for req in deps_list + ] # Output only default dependencies click.echo(index_args) - if not dev: - click.echo( - "\n".join(d.partition("--hash")[0].strip() for d in sorted(deps_list)) - ) - sys.exit(0) - # Output only dev dependencies - if dev: - click.echo( - "\n".join( - d.partition("--hash")[0].strip() for d in sorted(dev_deps_list) - ) - ) - sys.exit(0) - procs = [] - deps_list_bar = progress.bar( - deps_list, label=INSTALL_LABEL if os.name != "nt" else "" - ) - for dep, ignore_hash, block in deps_list_bar: - if len(procs) < PIPENV_MAX_SUBPROCESS: - # Use a specific index, if specified. - indexes, trusted_hosts, dep = parse_indexes(dep) - index = None - extra_indexes = [] - if indexes: - index = indexes[0] - if len(indexes) > 0: - extra_indexes = indexes[1:] - dep = Requirement.from_line(" ".join(dep)) - if index: - _index = None - try: - _index = project.find_source(index).get("name") - except SourceNotFound: - _index = None - dep.index = _index - dep._index = index - dep.extra_indexes = extra_indexes - # Install the module. - prev_no_deps_setting = no_deps - if dep.is_file_or_url and any( - dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] - ): - no_deps = False + click.echo( + "\n".join(sorted(deps)) + ) + sys.exit(0) + + procs = queue.Queue(maxsize=PIPENV_MAX_SUBPROCESS) + trusted_hosts = [] + + deps_list_bar = progress.bar(deps_list, width=32, + label=INSTALL_LABEL if os.name != "nt" else "") + + indexes = [] + for dep in deps_list_bar: + index = None + if dep.index: + index = project.find_source(dep.index) + indexes.append(index) + if not index.get("verify_ssl", False): + trusted_hosts.append(urllib3_util.parse_url(index.get("url")).host) + # Install the module. + is_artifact = False + if dep.is_file_or_url and any( + dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] + ): + is_artifact = True + + extra_indexes = [] + if not index and indexes: + index = next(iter(indexes)) + if len(indexes) > 1: + extra_indexes = indexes[1:] + with vistir.contextmanagers.temp_environ(): + if "PIP_USER" in os.environ: + del os.environ["PIP_USER"] c = pip_install( dep, - ignore_hashes=ignore_hash, + ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), allow_global=allow_global, - no_deps=no_deps, - block=block, + no_deps=False if is_artifact else no_deps, + block=any([dep.editable, blocking]), index=index, requirements_dir=requirements_dir, - extra_indexes=extra_indexes, pypi_mirror=pypi_mirror, - trusted_hosts=trusted_hosts + trusted_hosts=trusted_hosts, + extra_indexes=extra_indexes ) - c.dep = dep - c.ignore_hash = ignore_hash - c.index = index - c.extra_indexes = extra_indexes - procs.append(c) - no_deps = prev_no_deps_setting - if len(procs) >= PIPENV_MAX_SUBPROCESS or len(procs) == len(deps_list): - cleanup_procs(procs, concurrent) - procs = [] - cleanup_procs(procs, concurrent) + if procs.qsize() < PIPENV_MAX_SUBPROCESS: + c.dep = dep + procs.put(c) + + if procs.full() or procs.qsize() == len(deps_list): + cleanup_procs(procs, concurrent) + if not procs.empty(): + cleanup_procs(procs, concurrent) + # Iterate over the hopefully-poorly-packaged dependencies… if failed_deps_list: click.echo( - crayons.normal(u"Installing initially failed dependencies…", bold=True) + crayons.normal(fix_utf8("Installing initially failed dependencies…"), bold=True) ) - for dep, ignore_hash in progress.bar(failed_deps_list, label=INSTALL_LABEL2): + for dep in progress.bar(failed_deps_list, label=INSTALL_LABEL2): # Use a specific index, if specified. # Install the module. - prev_no_deps_setting = no_deps + is_artifact = False + index = None + if dep.index: + index = project.find_source(dep.index) if dep.is_file_or_url and any( dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] ): - no_deps = False - c = pip_install( - dep, - ignore_hashes=ignore_hash, - allow_global=allow_global, - no_deps=no_deps, - index=getattr(dep, "_index", None), - requirements_dir=requirements_dir, - extra_indexes=getattr(dep, "extra_indexes", None), - ) - no_deps = prev_no_deps_setting - # The Installation failed… - if c.return_code != 0: - # We echo both c.out and c.err because pip returns error details on out. - click.echo(crayons.blue(format_pip_output(c.out))) - click.echo(crayons.blue(format_pip_error(c.err)), err=True) - # Return the subprocess' return code. - sys.exit(c.return_code) - else: - click.echo( - "{0} {1}{2}".format( - crayons.green("Success installing"), - crayons.green(dep.name), - crayons.green("!"), - ) + is_artifact = True + extra_indexes = [] + if not index and indexes: + index = next(iter(indexes)) + if len(indexes) > 1: + extra_indexes = indexes[1:] + with vistir.contextmanagers.temp_environ(): + if "PIP_USER" in os.environ: + del os.environ["PIP_USER"] + c = pip_install( + dep, + ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), + allow_global=allow_global, + no_deps=True if is_artifact else no_deps, + index=index, + requirements_dir=requirements_dir, + pypi_mirror=pypi_mirror, + trusted_hosts=trusted_hosts, + extra_indexes=extra_indexes, + block=True ) + # The Installation failed… + if c.return_code != 0: + # We echo both c.out and c.err because pip returns error details on out. + click.echo(crayons.blue(format_pip_output(c.out))) + click.echo(crayons.blue(format_pip_error(c.err)), err=True) + # Return the subprocess' return code. + sys.exit(c.return_code) + else: + click.echo( + "{0} {1}{2}".format( + crayons.green("Success installing"), + crayons.green(dep.name), + crayons.green("!"), + ) + ) def convert_three_to_python(three, python): @@ -851,7 +872,7 @@ def convert_three_to_python(three, python): def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): """Creates a virtualenv.""" click.echo( - crayons.normal(u"Creating a virtualenv for this project…", bold=True), err=True + crayons.normal(fix_utf8("Creating a virtualenv for this project…"), bold=True), err=True ) click.echo( u"Pipfile: {0}".format(crayons.red(project.pipfile_location, bold=True)), @@ -865,14 +886,14 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): u"{0} {1} {3} {2}".format( crayons.normal("Using", bold=True), crayons.red(python, bold=True), - crayons.normal(u"to create virtualenv…", bold=True), + crayons.normal(fix_utf8("to create virtualenv…"), bold=True), crayons.green("({0})".format(python_version(python))), ), err=True, ) cmd = [ - sys.executable, + vistir.compat.Path(sys.executable).absolute().as_posix(), "-m", "virtualenv", "--prompt=({0}) ".format(project.name), @@ -883,7 +904,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): # Pass site-packages flag to virtualenv, if desired… if site_packages: click.echo( - crayons.normal(u"Making site-packages available…", bold=True), err=True + crayons.normal(fix_utf8("Making site-packages available…"), bold=True), err=True ) cmd.append("--system-site-packages") @@ -893,11 +914,12 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): pip_config = {} # Actually create the virtualenv. - with spinner(): - c = delegator.run(cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config) - c.block() + nospin = os.environ.get("PIPENV_ACTIVE", environments.PIPENV_NOSPIN) + c = vistir.misc.run(cmd, verbose=False, return_object=True, + spinner_name=environments.PIPENV_SPINNER, combine_stderr=False, + block=False, nospin=nospin, env=pip_config) click.echo(crayons.blue("{0}".format(c.out)), err=True) - if c.return_code != 0: + if c.returncode != 0: click.echo(crayons.blue("{0}".format(c.err)), err=True) click.echo( u"{0}: Failed to create virtual environment.".format( @@ -1019,9 +1041,9 @@ def do_lock( # Alert the user of progress. click.echo( u"{0} {1} {2}".format( - crayons.normal("Locking"), - crayons.red("[{0}]".format(settings["log_string"])), - crayons.normal("dependencies…"), + crayons.normal(u"Locking"), + crayons.red(u"[{0}]".format(settings["log_string"])), + crayons.normal(fix_utf8("dependencies…")), ), err=True, ) @@ -1125,7 +1147,7 @@ def do_purge(bare=False, downloads=False, allow_global=False): if downloads: if not bare: - click.echo(crayons.normal(u"Clearing out downloads directory…", bold=True)) + click.echo(crayons.normal(fix_utf8("Clearing out downloads directory…"), bold=True)) shutil.rmtree(project.download_location) return @@ -1154,7 +1176,7 @@ def do_purge(bare=False, downloads=False, allow_global=False): actually_installed.append(dep) if not bare: click.echo( - u"Found {0} installed package(s), purging…".format(len(actually_installed)) + fix_utf8("Found {0} installed package(s), purging…".format(len(actually_installed))) ) command = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=allow_global)), @@ -1185,7 +1207,6 @@ def do_init( """Executes the init functionality.""" from .environments import PIPENV_VIRTUALENV - cleanup_reqdir = False if not system: if not project.virtualenv_exists: try: @@ -1197,8 +1218,7 @@ def do_init( if not deploy: ensure_pipfile(system=system) if not requirements_dir: - cleanup_reqdir = True - requirements_dir = vistir.compat.TemporaryDirectory( + requirements_dir = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) # Write out the lockfile if it doesn't exist, but not if the Pipfile is being ignored @@ -1215,26 +1235,25 @@ def do_init( ) ) click.echo(crayons.normal("Aborting deploy.", bold=True), err=True) - requirements_dir.cleanup() sys.exit(1) elif (system or allow_global) and not (PIPENV_VIRTUALENV): click.echo( - crayons.red( - u"Pipfile.lock ({0}) out of date, but installation " - u"uses {1}… re-building lockfile must happen in " - u"isolation. Please rebuild lockfile in a virtualenv. " - u"Continuing anyway…".format( + crayons.red(fix_utf8( + "Pipfile.lock ({0}) out of date, but installation " + "uses {1}… re-building lockfile must happen in " + "isolation. Please rebuild lockfile in a virtualenv. " + "Continuing anyway…".format( crayons.white(old_hash[-6:]), crayons.white("--system") - ), + )), bold=True, ), err=True, ) else: if old_hash: - msg = u"Pipfile.lock ({1}) out of date, updating to ({0})…" + msg = fix_utf8("Pipfile.lock ({1}) out of date, updating to ({0})…") else: - msg = u"Pipfile.lock is corrupted, replaced with ({0})…" + msg = fix_utf8("Pipfile.lock is corrupted, replaced with ({0})…") click.echo( crayons.red(msg.format(old_hash[-6:], new_hash[-6:]), bold=True), err=True, @@ -1259,11 +1278,10 @@ def do_init( err=True, ) click.echo("See also: --deploy flag.", err=True) - requirements_dir.cleanup() sys.exit(1) else: click.echo( - crayons.normal(u"Pipfile.lock not found, creating…", bold=True), + crayons.normal(fix_utf8("Pipfile.lock not found, creating…"), bold=True), err=True, ) do_lock( @@ -1279,11 +1297,9 @@ def do_init( allow_global=allow_global, skip_lock=skip_lock, concurrent=concurrent, - requirements_dir=requirements_dir.name, + requirements_dir=requirements_dir, pypi_mirror=pypi_mirror, ) - if cleanup_reqdir: - requirements_dir.cleanup() # Hint the user what to do to activate the virtualenv. if not allow_global and not deploy and "PIPENV_ACTIVE" not in os.environ: @@ -1312,14 +1328,14 @@ def pip_install( trusted_hosts=None ): from notpip._internal import logger as piplogger + from .utils import Mapping from .vendor.urllib3.util import parse_url src = [] write_to_tmpfile = False if requirement: - editable_with_markers = requirement.editable and requirement.markers needs_hashes = not requirement.editable and not ignore_hashes and r is None - write_to_tmpfile = needs_hashes or editable_with_markers + write_to_tmpfile = needs_hashes if not trusted_hosts: trusted_hosts = [] @@ -1333,12 +1349,16 @@ def pip_install( ) # Create files for hash mode. if write_to_tmpfile: - with vistir.compat.NamedTemporaryFile( + if not requirements_dir: + requirements_dir = vistir.path.create_tracked_tempdir( + prefix="pipenv", suffix="requirements") + f = vistir.compat.NamedTemporaryFile( prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, delete=False - ) as f: - f.write(vistir.misc.to_bytes(requirement.as_line())) - r = f.name + ) + f.write(vistir.misc.to_bytes(requirement.as_line())) + r = f.name + f.close() # Install dependencies when a package is a VCS dependency. if requirement and requirement.vcs: no_deps = False @@ -1348,21 +1368,27 @@ def pip_install( # Try installing for each source in project.sources. if index: - try: - index_source = project.find_source(index) - index_source = index_source.copy() - except SourceNotFound: - src_name = project.src_name_from_url(index) - index_url = parse_url(index) - verify_ssl = index_url.host not in trusted_hosts - index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} + if isinstance(index, (Mapping, dict)): + index_source = index + else: + try: + index_source = project.find_source(index) + index_source = index_source.copy() + except SourceNotFound: + src_name = project.src_name_from_url(index) + index_url = parse_url(index) + verify_ssl = index_url.host not in trusted_hosts + index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} sources = [index_source.copy(),] if extra_indexes: if isinstance(extra_indexes, six.string_types): extra_indexes = [extra_indexes,] for idx in extra_indexes: + extra_src = None + if isinstance(idx, (Mapping, dict)): + extra_src = idx try: - extra_src = project.find_source(idx) + extra_src = project.find_source(idx) if not extra_src else extra_src except SourceNotFound: src_name = project.src_name_from_url(idx) src_url = parse_url(idx) @@ -1382,12 +1408,15 @@ def pip_install( for source in sources ] if (requirement and requirement.editable) and not r: - install_reqs = requirement.as_line(as_list=True) + line_kwargs = {"as_list": True} + if requirement.markers: + line_kwargs["include_markers"] = False + install_reqs = requirement.as_line(**line_kwargs) if requirement.editable and install_reqs[0].startswith("-e "): req, install_reqs = install_reqs[0], install_reqs[1:] editable_opt, req = req.split(" ", 1) install_reqs = [editable_opt, req] + install_reqs - if not any(item.startswith("--hash") for item in install_reqs): + if not all(item.startswith("--hash") for item in install_reqs): ignore_hashes = True elif r: install_reqs = ["-r", r] @@ -1396,7 +1425,7 @@ def pip_install( ignore_hashes = True else: ignore_hashes = True if not requirement.hashes else False - install_reqs = requirement.as_line(as_list=True) + install_reqs = [escape_cmd(r) for r in requirement.as_line(as_list=True)] pip_command = [which_pip(allow_global=allow_global), "install"] if pre: pip_command.append("--pre") @@ -1409,7 +1438,6 @@ def pip_install( pip_command.append("--upgrade-strategy=only-if-needed") if no_deps: pip_command.append("--no-deps") - install_reqs = [escape_cmd(req) for req in install_reqs] pip_command.extend(install_reqs) pip_command.extend(prepare_pip_source_args(sources)) if not ignore_hashes: @@ -1431,7 +1459,8 @@ def pip_install( pip_config.update( {"PIP_SRC": vistir.misc.fs_str(project.virtualenv_src_location)} ) - pip_command = Script.parse(pip_command).cmdify() + cmd = Script.parse(pip_command) + pip_command = cmd.cmdify() c = delegator.run(pip_command, block=block, env=pip_config) return c @@ -1623,9 +1652,9 @@ def ensure_lockfile(keep_outdated=False, pypi_mirror=None): if new_hash != old_hash: click.echo( crayons.red( - u"Pipfile.lock ({0}) out of date, updating to ({1})…".format( + fix_utf8("Pipfile.lock ({0}) out of date, updating to ({1})…".format( old_hash[-6:], new_hash[-6:] - ), + )), bold=True, ), err=True, @@ -1700,9 +1729,10 @@ def do_install( from .environments import PIPENV_VIRTUALENV, PIPENV_USE_SYSTEM from notpip._internal.exceptions import PipError - requirements_directory = vistir.compat.TemporaryDirectory( + requirements_directory = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) + warnings.filterwarnings("default", category=vistir.compat.ResourceWarning) if selective_upgrade: keep_outdated = True packages = packages if packages else [] @@ -1739,27 +1769,29 @@ def do_install( err=True, ) click.echo("See also: --deploy flag.", err=True) - requirements_directory.cleanup() sys.exit(1) # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True # Check if the file is remote or not if remote: - fd, temp_reqs = tempfile.mkstemp( - prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory.name - ) - requirements_url = requirements - # Download requirements file click.echo( crayons.normal( - u"Remote requirements file provided! Downloading…", bold=True + fix_utf8("Remote requirements file provided! Downloading…"), bold=True ), err=True, ) + fd = vistir.path.create_tracked_tempfile( + prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory + ) + temp_reqs = fd.name + requirements_url = requirements + # Download requirements file try: download_file(requirements, temp_reqs) except IOError: + fd.close() + os.unlink(temp_reqs) click.echo( crayons.red( u"Unable to find requirements file at {0}.".format( @@ -1768,8 +1800,9 @@ def do_install( ), err=True, ) - requirements_directory.cleanup() sys.exit(1) + finally: + fd.close() # Replace the url with the temporary requirements file requirements = temp_reqs remote = True @@ -1777,7 +1810,7 @@ def do_install( error, traceback = None, None click.echo( crayons.normal( - u"Requirements file provided! Importing into Pipfile…", bold=True + fix_utf8("Requirements file provided! Importing into Pipfile…"), bold=True ), err=True, ) @@ -1800,16 +1833,15 @@ def do_install( finally: # If requirements file was provided by remote url delete the temporary file if remote: - os.close(fd) # Close for windows to allow file cleanup. - os.remove(project.path_to(temp_reqs)) + fd.close() # Close for windows to allow file cleanup. + os.remove(temp_reqs) if error and traceback: click.echo(crayons.red(error)) click.echo(crayons.blue(str(traceback)), err=True) - requirements_directory.cleanup() sys.exit(1) if code: click.echo( - crayons.normal(u"Discovering imports from local codebase…", bold=True) + crayons.normal(fix_utf8("Discovering imports from local codebase…"), bold=True) ) for req in import_from_code(code): click.echo(" Found {0}!".format(crayons.green(req))) @@ -1886,17 +1918,21 @@ def do_install( for pkg_line in pkg_list: click.echo( crayons.normal( - u"Installing {0}…".format(crayons.green(pkg_line, bold=True)), + fix_utf8("Installing {0}…".format(crayons.green(pkg_line, bold=True))), bold=True, ) ) # pip install: - with spinner(): + with vistir.contextmanagers.temp_environ(), spinner(text="Installing...", + spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: + if "PIP_USER" in os.environ: + del os.environ["PIP_USER"] try: pkg_requirement = Requirement.from_line(pkg_line) except ValueError as e: - click.echo("{0}: {1}".format(crayons.red("WARNING"), e)) - requirements_directory.cleanup() + sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) sys.exit(1) if index_url: pkg_requirement.index = index_url @@ -1907,14 +1943,14 @@ def do_install( selective_upgrade=selective_upgrade, no_deps=False, pre=pre, - requirements_dir=requirements_directory.name, + requirements_dir=requirements_directory, index=index_url, extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) # Warn if --editable wasn't passed. if pkg_requirement.is_vcs and not pkg_requirement.editable: - click.echo( + sp.write_err( "{0}: You installed a VCS dependency in non-editable mode. " "This will work fine, but sub-dependencies will not be resolved by {1}." "\n To enable this sub-dependency functionality, specify that this dependency is editable." @@ -1923,37 +1959,34 @@ def do_install( crayons.red("$ pipenv lock"), ) ) - click.echo(crayons.blue(format_pip_output(c.out))) - # Ensure that package was successfully installed. - try: - assert c.return_code == 0 - except AssertionError: - click.echo( - "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(pkg_line) - ), - err=True, - ) - click.echo(crayons.blue(format_pip_error(c.err)), err=True) - if "setup.py egg_info" in c.err: - click.echo( - "This is likely caused by a bug in {0}. " - "Report this to its maintainers.".format( - crayons.green(pkg_requirement.name) + click.echo(crayons.blue(format_pip_output(c.out))) + # Ensure that package was successfully installed. + if c.return_code != 0: + sp.write_err(vistir.compat.fs_str( + "{0} An error occurred while installing {1}!".format( + crayons.red("Error: ", bold=True), crayons.green(pkg_line) ), - err=True, + )) + sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) + if "setup.py egg_info" in c.err: + sp.write_err(vistir.compat.fs_str( + "This is likely caused by a bug in {0}. " + "Report this to its maintainers.".format( + crayons.green(pkg_requirement.name) + ) + )) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sys.exit(1) + sp.write(vistir.compat.fs_str( + u"{0} {1} {2} {3}{4}".format( + crayons.normal(u"Adding", bold=True), + crayons.green(u"{0}".format(pkg_requirement.name), bold=True), + crayons.normal(u"to Pipfile's", bold=True), + crayons.red(u"[dev-packages]" if dev else u"[packages]", bold=True), + crayons.normal(fix_utf8("…"), bold=True), ) - requirements_directory.cleanup() - sys.exit(1) - click.echo( - "{0} {1} {2} {3}{4}".format( - crayons.normal("Adding", bold=True), - crayons.green(pkg_requirement.name, bold=True), - crayons.normal("to Pipfile's", bold=True), - crayons.red("[dev-packages]" if dev else "[packages]", bold=True), - crayons.normal("…", bold=True), - ) - ) + )) + sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Installation Succeeded")) # Add the package to the Pipfile. try: project.add_package_to_pipfile(pkg_requirement, dev) @@ -1975,7 +2008,6 @@ def do_install( pypi_mirror=pypi_mirror, skip_lock=skip_lock, ) - requirements_directory.cleanup() sys.exit(0) @@ -2009,7 +2041,7 @@ def do_uninstall( # Un-install all dependencies, if --all was provided. if all is True: click.echo( - crayons.normal(u"Un-installing all packages from virtualenv…", bold=True) + crayons.normal(fix_utf8("Un-installing all packages from virtualenv…"), bold=True) ) do_purge(allow_global=system) return @@ -2025,7 +2057,7 @@ def do_uninstall( return click.echo( crayons.normal( - u"Un-installing {0}…".format(crayons.red("[dev-packages]")), bold=True + fix_utf8("Un-installing {0}…".format(crayons.red("[dev-packages]"))), bold=True ) ) package_names = project.dev_packages.keys() @@ -2033,7 +2065,7 @@ def do_uninstall( click.echo(crayons.red("No package provided!"), err=True) return 1 for package_name in package_names: - click.echo(u"Un-installing {0}…".format(crayons.green(package_name))) + click.echo(fix_utf8("Un-installing {0}…".format(crayons.green(package_name)))) cmd = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=system)), package_name ) @@ -2055,7 +2087,7 @@ def do_uninstall( continue click.echo( - u"Removing {0} from Pipfile…".format(crayons.green(package_name)) + fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name))) ) # Remove package from both packages and dev-packages. project.remove_package_from_pipfile(package_name, dev=True) @@ -2076,7 +2108,7 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror from .shells import choose_shell shell = choose_shell() - click.echo("Launching subshell in virtual environment…", err=True) + click.echo(fix_utf8("Launching subshell in virtual environment…"), err=True) fork_args = (project.virtualenv_location, project.project_directory, shell_args) @@ -2087,9 +2119,9 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror try: shell.fork_compat(*fork_args) except (AttributeError, ImportError): - click.echo( - u"Compatibility mode not supported. " - u"Trying to continue as well-configured shell…", + click.echo(fix_utf8( + "Compatibility mode not supported. " + "Trying to continue as well-configured shell…"), err=True, ) shell.fork(*fork_args) @@ -2099,11 +2131,11 @@ def _inline_activate_virtualenv(): try: activate_this = which("activate_this.py") if not activate_this or not os.path.exists(activate_this): - click.echo( - u"{0}: activate_this.py not found. Your environment is most " - u"certainly not activated. Continuing anyway…" - u"".format(crayons.red("Warning", bold=True)), - err=True, + click.echo(fix_utf8( + "{0}: activate_this.py not found. Your environment is most " + "certainly not activated. Continuing anyway…").format( + crayons.red("Warning", bold=True) + ), err=True, ) return with open(activate_this) as f: @@ -2267,7 +2299,7 @@ def do_check( sys.exit(1) else: sys.exit(0) - click.echo(crayons.normal(u"Checking PEP 508 requirements…", bold=True)) + click.echo(crayons.normal(fix_utf8("Checking PEP 508 requirements…"), bold=True)) if system: python = system_which("python") else: @@ -2303,7 +2335,7 @@ def do_check( sys.exit(1) else: click.echo(crayons.green("Passed!")) - click.echo(crayons.normal(u"Checking installed package safety…", bold=True)) + click.echo(crayons.normal(fix_utf8("Checking installed package safety…"), bold=True)) path = pep508checker.__file__.rstrip("cdo") path = os.sep.join(__file__.split(os.sep)[:-1] + ["patched", "safety.zip"]) if not system: @@ -2501,7 +2533,7 @@ def do_sync( ) # Install everything. - requirements_dir = vistir.compat.TemporaryDirectory( + requirements_dir = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) do_init( @@ -2513,7 +2545,6 @@ def do_sync( deploy=deploy, system=system, ) - requirements_dir.cleanup() click.echo(crayons.green("All dependencies are now up-to-date!")) @@ -2548,7 +2579,7 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro else: click.echo( crayons.white( - "Uninstalling {0}…".format(repr(apparent_bad_package)), bold=True + fix_utf8("Uninstalling {0}…".format(repr(apparent_bad_package))), bold=True ) ) # Uninstall the package. diff --git a/pipenv/environments.py b/pipenv/environments.py index e6de4f0d..c5874136 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -1,7 +1,9 @@ +# -*- coding=utf-8 -*- + import os import sys from appdirs import user_cache_dir -from .vendor.vistir.misc import fs_str +from .vendor.vistir.misc import fs_str, to_text # HACK: avoid resolver.py uses the wrong byte code files. @@ -259,3 +261,8 @@ def is_verbose(threshold=1): def is_quiet(threshold=-1): return PIPENV_VERBOSITY <= threshold + + +PIPENV_SPINNER_FAIL_TEXT = fs_str(to_text(u"✘ {0}")) if not PIPENV_HIDE_EMOJIS else ("{0}") + +PIPENV_SPINNER_OK_TEXT = fs_str(to_text(u"✔ {0}")) if not PIPENV_HIDE_EMOJIS else ("{0}") diff --git a/pipenv/project.py b/pipenv/project.py index c2dc9668..280a6f8b 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -10,6 +10,7 @@ import hashlib import contoml from first import first from cached_property import cached_property +import operator import pipfile import pipfile.api import six @@ -144,7 +145,7 @@ class Project(object): self._lockfile_newlines = DEFAULT_NEWLINES self._requirements_location = None self._original_dir = os.path.abspath(os.curdir) - self.which = which + self._which = which self.python_version = python_version # Hack to skip this during pipenv run, or -r. if ("run" not in sys.argv) and chdir: @@ -678,6 +679,54 @@ class Project(object): data[u"requires"] = {"python_version": version[: len("2.7")]} self.write_toml(data, "Pipfile") + def get_or_create_lockfile(self): + from requirementslib.models.lockfile import Lockfile as Req_Lockfile + lockfile = None + try: + lockfile = Req_Lockfile.load(self.lockfile_location) + except OSError: + lockfile = Req_Lockfile(self.lockfile_content) + return lockfile + else: + if lockfile._lockfile is not None: + return lockfile + if self.lockfile_exists and self.lockfile_content: + from .vendor.plette.lockfiles import Lockfile + lockfile_dict = self.lockfile_content.copy() + sources = lockfile_dict["_meta"].get("sources", []) + if not sources: + sources = self.pipfile_sources + elif not isinstance(sources, list): + sources = [sources,] + lockfile_dict["_meta"]["sources"] = [ + { + "name": s["name"], + "url": s["url"], + "verify_ssl": ( + s["verify_ssl"] if isinstance(s["verify_ssl"], bool) else ( + True if s["verify_ssl"].lower() == "true" else False + ) + ) + } for s in sources + ] + _created_lockfile = Lockfile(lockfile_dict) + lockfile._lockfile = lockfile.projectfile.model = _created_lockfile + return lockfile + elif self.pipfile_exists: + from .vendor.plette.lockfiles import Lockfile, PIPFILE_SPEC_CURRENT + lockfile_dict = { + "_meta": { + "hash": {"sha256": self.calculate_pipfile_hash()}, + "pipfile-spec": PIPFILE_SPEC_CURRENT, + "sources": self.pipfile_sources, + "requires": self.parsed_pipfile.get("requires", {}) + }, + "default": self._lockfile["default"].copy(), + "develop": self._lockfile["develop"].copy() + } + lockfile._lockfile = Lockfile(lockfile_dict) + return lockfile + def write_toml(self, data, path=None): """Writes the given data structure out as TOML.""" if path is None: @@ -939,6 +988,8 @@ class Project(object): location = self.virtualenv_location if self.virtualenv_location else sys.prefix prefix = vistir.compat.Path(location).as_posix() scheme = sysconfig._get_default_scheme() + if not scheme: + scheme = "posix_prefix" if not sys.platform == "win32" else "nt" config = { "base": prefix, "installed_base": prefix, @@ -953,3 +1004,26 @@ class Project(object): if "prefix" not in paths: paths["prefix"] = prefix return paths + + @cached_property + def finders(self): + from .vendor.pythonfinder import Finder + finders = [ + Finder(path=self.env_paths["scripts"], global_search=gs, system=False) + for gs in (False, True) + ] + return finders + + @property + def finder(self): + return next(iter(self.finders), None) + + def which(self, search, as_path=True): + find = operator.methodcaller("which", search) + result = next(iter(filter(None, (find(finder) for finder in self.finders))), None) + if not result: + result = self._which(search) + else: + if as_path: + result = str(result.path) + return result diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 8c282b71..2854a93a 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -7,53 +7,52 @@ os.environ["PIP_PYTHON_PATH"] = sys.executable def _patch_path(): + import site pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + pipenv_site_dir = os.path.dirname(pipenv_libdir) + site.addsitedir(pipenv_site_dir) for _dir in ("vendor", "patched"): sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) - site_packages_dir = os.path.dirname(pipenv_libdir) - if site_packages_dir not in sys.path: - sys.path.append(site_packages_dir) + + +def get_parser(): + from argparse import ArgumentParser + parser = ArgumentParser("pipenvresolver") + parser.add_argument("--pre", action="store_true", default=False) + parser.add_argument("--clear", action="store_true", default=False) + parser.add_argument("--verbose", "-v", action="count", default=False) + parser.add_argument("--debug", action="store_true", default=False) + parser.add_argument("--system", action="store_true", default=False) + parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store", + default=os.environ.get("PIPENV_REQ_DIR")) + parser.add_argument("packages", nargs="*") + return parser def which(*args, **kwargs): return sys.executable -def main(): - do_pre = "--pre" in " ".join(sys.argv) - do_clear = "--clear" in " ".join(sys.argv) - is_verbose = "--verbose" in " ".join(sys.argv) - is_debug = "--debug" in " ".join(sys.argv) - system = "--system" in " ".join(sys.argv) - new_sys_argv = [] - for v in sys.argv: - if v.startswith("--"): - continue +def handle_parsed_args(parsed): + if parsed.debug: + parsed.verbose = max(parsed.verbose, 2) + if parsed.verbose > 1: + logging.getLogger("notpip").setLevel(logging.DEBUG) + elif parsed.verbose > 0: + logging.getLogger("notpip").setLevel(logging.INFO) + if "PIPENV_PACKAGES" in os.environ: + parsed.packages += os.environ["PIPENV_PACKAGES"].strip().split("\n") + return parsed - else: - new_sys_argv.append(v) - sys.argv = new_sys_argv +def main(pre, clear, verbose, system, requirements_dir, packages): os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) os.environ["PIP_PYTHON_PATH"] = sys.executable - verbosity = int(os.environ.get("PIPENV_VERBOSITY", 0)) - if is_debug: - verbosity = max(verbosity, 2) - elif is_verbose: - verbosity = max(verbosity, 1) - if verbosity > 1: # Shit's getting real at this point. - logging.getLogger("notpip").setLevel(logging.DEBUG) - elif verbosity > 0: - logging.getLogger("notpip").setLevel(logging.INFO) + import warnings + from pipenv.vendor.vistir.compat import ResourceWarning + warnings.filterwarnings("ignore", category=ResourceWarning) - if "PIPENV_PACKAGES" in os.environ: - packages = os.environ["PIPENV_PACKAGES"].strip().split("\n") - else: - packages = sys.argv[1:] - for i, package in enumerate(packages): - if package.startswith("--"): - del packages[i] from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources pypi_mirror_source = ( @@ -62,7 +61,7 @@ def main(): else None ) - def resolve(packages, pre, project, sources, clear, system): + def resolve(packages, pre, project, sources, clear, system, requirements_dir=None): return resolve_deps( packages, which, @@ -71,6 +70,7 @@ def main(): sources=sources, clear=clear, allow_global=system, + req_dir=requirements_dir ) from pipenv.core import project @@ -82,19 +82,31 @@ def main(): ) results = resolve( packages, - pre=do_pre, + pre=pre, project=project, sources=sources, - clear=do_clear, + clear=clear, system=system, + requirements_dir=requirements_dir, ) print("RESULTS:") if results: - print(json.dumps(results)) + import traceback + if isinstance(results, (Exception, traceback.types.TracebackType)): + sys.stderr.write(traceback.print_tb(results)) + sys.stderr.write(sys.exc_value()) + else: + print(json.dumps(results)) else: print(json.dumps([])) if __name__ == "__main__": + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" _patch_path() - main() + parser = get_parser() + parsed, remaining = parser.parse_known_intermixed_args() + sys.argv = remaining + parsed = handle_parsed_args(parsed) + main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.requirements_dir, + parsed.packages) diff --git a/pipenv/utils.py b/pipenv/utils.py index b965e46d..d8f2432b 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -16,6 +16,11 @@ from click import echo as click_echo from first import first from vistir.misc import fs_str +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +from six.moves import Mapping + +from vistir.compat import ResourceWarning + try: from weakref import finalize except ImportError: @@ -38,14 +43,8 @@ from contextlib import contextmanager from . import environments from .pep508checker import lookup -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) from six.moves.urllib.parse import urlparse -from six.moves import Mapping - -if six.PY2: - - class ResourceWarning(Warning): - pass +from urllib3 import util as urllib3_util specifiers = [k for k in lookup.keys()] @@ -127,20 +126,14 @@ def parse_python_version(output): def python_version(path_to_python): - import delegator + from .vendor.pythonfinder.utils import get_python_version if not path_to_python: return None try: - c = delegator.run([path_to_python, "--version"], block=False) + version = get_python_version(path_to_python) except Exception: return None - c.block() - version = parse_python_version(c.out.strip() or c.err.strip()) - try: - version = u"{major}.{minor}.{micro}".format(**version) - except TypeError: - return None return version @@ -194,7 +187,7 @@ def prepare_pip_source_args(sources, pip_args=None): # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): pip_args.extend( - ["--trusted-host", urlparse(sources[0]["url"]).hostname] + ["--trusted-host", urllib3_util.parse_url(sources[0]["url"]).host] ) # Add additional sources as extra indexes. if len(sources) > 1: @@ -203,7 +196,7 @@ def prepare_pip_source_args(sources, pip_args=None): # Trust the host if it's not verified. if not source.get("verify_ssl", True): pip_args.extend( - ["--trusted-host", urlparse(source["url"]).hostname] + ["--trusted-host", urllib3_util.parse_url(source["url"]).host] ) return pip_args @@ -228,7 +221,7 @@ def actually_resolve_deps( from pipenv.patched.piptools import logging as piptools_logging from pipenv.patched.piptools.exceptions import NoCandidateFound from .vendor.requirementslib.models.requirements import Requirement - from ._compat import TemporaryDirectory, NamedTemporaryFile + from .vendor.vistir.path import create_tracked_tempdir, create_tracked_tempfile class PipCommand(basecommand.Command): """Needed for pip-tools.""" @@ -236,10 +229,8 @@ def actually_resolve_deps( name = "PipCommand" constraints = [] - cleanup_req_dir = False if not req_dir: - req_dir = TemporaryDirectory(suffix="-requirements", prefix="pipenv-") - cleanup_req_dir = True + req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") for dep in deps: if not dep: continue @@ -267,26 +258,26 @@ def actually_resolve_deps( if sources: pip_args = prepare_pip_source_args(sources, pip_args) if environments.is_verbose(): - print("Using pip: {0}".format(" ".join(pip_args))) - with NamedTemporaryFile( + click_echo(crayons.blue("Using pip: {0}".format(" ".join(pip_args)))) + constraints_file = create_tracked_tempfile( mode="w", prefix="pipenv-", suffix="-constraints.txt", - dir=req_dir.name, + dir=req_dir, delete=False, - ) as f: - if sources: - requirementstxt_sources = " ".join(pip_args) if pip_args else "" - requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") - f.write(u"{0}\n".format(requirementstxt_sources)) - f.write(u"\n".join([_constraint for _constraint in constraints])) - constraints_file = f.name + ) + if sources: + requirementstxt_sources = " ".join(pip_args) if pip_args else "" + requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") + constraints_file.write(u"{0}\n".format(requirementstxt_sources)) + constraints_file.write(u"\n".join([_constraint for _constraint in constraints])) + constraints_file.close() pip_options, _ = pip_command.parser.parse_args(pip_args) pip_options.cache_dir = environments.PIPENV_CACHE_DIR session = pip_command._build_session(pip_options) pypi = PyPIRepository(pip_options=pip_options, use_json=False, session=session) constraints = parse_requirements( - constraints_file, finder=pypi.finder, session=pypi.session, options=pip_options + constraints_file.name, finder=pypi.finder, session=pypi.session, options=pip_options ) constraints = [c for c in constraints] if environments.is_verbose(): @@ -326,11 +317,7 @@ def actually_resolve_deps( "Please check your version specifier and version number. See PEP440 for more information." ) ) - if cleanup_req_dir: - req_dir.cleanup() raise RuntimeError - if cleanup_req_dir: - req_dir.cleanup() return (resolved_tree, hashes, markers_lookup, resolver) @@ -343,43 +330,72 @@ def venv_resolve_deps( allow_global=False, pypi_mirror=None, ): - from .vendor.vistir.misc import fs_str + from .vendor.vistir.misc import fs_str, run + from .vendor.vistir.compat import Path + from .vendor.vistir.path import create_tracked_tempdir + from .cmdparse import Script + from .core import spinner + from .vendor.pexpect.exceptions import EOF from .vendor import delegator from . import resolver import json if not deps: return [] - resolver = escape_grouped_arguments(resolver.__file__.rstrip("co")) - cmd = "{0} {1} {2} {3} {4}".format( - escape_grouped_arguments(which("python", allow_global=allow_global)), - resolver, - "--pre" if pre else "", - "--clear" if clear else "", - "--system" if allow_global else "", - ) + + req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") + + cmd = [ + which("python", allow_global=allow_global), + Path(resolver.__file__.rstrip("co")).as_posix() + ] + if pre: + cmd.append("--pre") + if clear: + cmd.append("--clear") + if allow_global: + cmd.append("--system") with temp_environ(): os.environ = {fs_str(k): fs_str(val) for k, val in os.environ.items()} os.environ["PIPENV_PACKAGES"] = str("\n".join(deps)) if pypi_mirror: os.environ["PIPENV_PYPI_MIRROR"] = str(pypi_mirror) os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) - c = delegator.run(cmd, block=True) - try: - assert c.return_code == 0 - except AssertionError: - if environments.is_verbose(): - click_echo(c.out, err=True) - click_echo(c.err, err=True) - else: - click_echo(c.err[(int(len(c.err) / 2) - 1):], err=True) - sys.exit(c.return_code) + os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) + os.environ["PIP_NO_INPUT"] = fs_str("1") + + out = "" + EOF.__module__ = "pexpect.exceptions" + with spinner(text=fs_str("Locking..."), spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: + c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) + _out = u"" + while True: + result = c.expect(u"\n", timeout=-1) + if result is EOF or result is None: + break + _out = c.out + out += _out + sp.text = fs_str("Locking... {0}".format(_out[:100])) + if environments.is_verbose(): + sp.write_err(_out.rstrip()) + c.block() + if c.return_code != 0: + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Locking Failed!" + )) + click_echo(c.err.strip(), err=True) + sys.exit(c.return_code) + else: + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) if environments.is_verbose(): click_echo(c.out.split("RESULTS:")[0], err=True) try: return json.loads(c.out.split("RESULTS:")[1].strip()) except IndexError: + click_echo(c.out.strip()) + click_echo(c.err.strip(), err=True) raise RuntimeError("There was a problem with locking.") @@ -392,13 +408,13 @@ def resolve_deps( clear=False, pre=False, allow_global=False, + req_dir=None ): """Given a list of dependencies, return a resolved list of dependencies, using pip-tools -- and their hashes, using the warehouse API / pip. """ from .patched.notpip._vendor.requests.exceptions import ConnectionError from .vendor.requirementslib.models.requirements import Requirement - from ._compat import TemporaryDirectory index_lookup = {} markers_lookup = {} @@ -408,7 +424,10 @@ def resolve_deps( if not deps: return results # First (proper) attempt: - req_dir = TemporaryDirectory(prefix="pipenv-", suffix="-requirements") + req_dir = req_dir if req_dir else os.environ.get("req_dir", None) + if not req_dir: + from .vendor.vistir.path import create_tracked_tempdir + req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-requirements") with HackedPythonVersion(python_version=python, python_path=python_path): try: resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( @@ -444,7 +463,6 @@ def resolve_deps( req_dir=req_dir, ) except RuntimeError: - req_dir.cleanup() sys.exit(1) for result in resolved_tree: if not result.editable: @@ -503,7 +521,6 @@ def resolve_deps( entry.update({"markers": markers_lookup.get(result.name)}) entry = translate_markers(entry) results.append(entry) - req_dir.cleanup() return results @@ -526,7 +543,6 @@ def is_pinned(val): def convert_deps_to_pip(deps, project=None, r=True, include_index=True): """"Converts a Pipfile-formatted dependency to a pip-formatted one.""" - from ._compat import NamedTemporaryFile from .vendor.requirementslib.models.requirements import Requirement dependencies = [] @@ -541,7 +557,8 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=True): return dependencies # Write requirements.txt to tmp directory. - f = NamedTemporaryFile(suffix="-requirements.txt", delete=False) + from .vendor.vistir.path import create_tracked_tempfile + f = create_tracked_tempfile(suffix="-requirements.txt", delete=False) f.write("\n".join(dependencies).encode("utf-8")) f.close() return f.name @@ -1054,54 +1071,6 @@ def escape_cmd(cmd): return cmd -@contextmanager -def atomic_open_for_write(target, binary=False, newline=None, encoding=None): - """Atomically open `target` for writing. - - This is based on Lektor's `atomic_open()` utility, but simplified a lot - to handle only writing, and skip many multi-process/thread edge cases - handled by Werkzeug. - - How this works: - - * Create a temp file (in the same directory of the actual target), and - yield for surrounding code to write to it. - * If some thing goes wrong, try to remove the temp file. The actual target - is not touched whatsoever. - * If everything goes well, close the temp file, and replace the actual - target with this new file. - """ - from ._compat import NamedTemporaryFile - - mode = "w+b" if binary else "w" - f = NamedTemporaryFile( - dir=os.path.dirname(target), - prefix=".__atomic-write", - mode=mode, - encoding=encoding, - newline=newline, - delete=False, - ) - # set permissions to 0644 - os.chmod(f.name, stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - try: - yield f - except BaseException: - f.close() - try: - os.remove(f.name) - except OSError: - pass - raise - else: - f.close() - try: - os.remove(target) # This is needed on Windows. - except OSError: - pass - os.rename(f.name, target) # No os.replace() on Python 2. - - def safe_expandvars(value): """Call os.path.expandvars if value is a string, otherwise do nothing. """ @@ -1127,8 +1096,8 @@ def get_vcs_deps( dev=False, pypi_mirror=None, ): - from ._compat import TemporaryDirectory, Path - import atexit + from .vendor.vistir.compat import Path + from .vendor.vistir.path import create_tracked_tempdir from .vendor.requirementslib.models.requirements import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" @@ -1144,8 +1113,7 @@ def get_vcs_deps( ) src_dir.mkdir(mode=0o775, exist_ok=True) else: - src_dir = TemporaryDirectory(prefix="pipenv-lock-dir") - atexit.register(src_dir.cleanup) + src_dir = create_tracked_tempdir(prefix="pipenv-lock-dir") for pkg_name, pkg_pipfile in packages.items(): requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) name = requirement.normalized_name @@ -1280,7 +1248,6 @@ def is_virtual_environment(path): @contextmanager def locked_repository(requirement): from .vendor.vistir.path import create_tracked_tempdir - from .vendor.vistir.misc import fs_str src_dir = create_tracked_tempdir(prefix="pipenv-src") if not requirement.is_vcs: return diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py index 3ffb2e31..56d12458 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py @@ -7,6 +7,8 @@ import locale import errno from pexpect.popen_spawn import PopenSpawn +import pexpect +pexpect.EOF.__module__ = "pexpect.exceptions" # Include `unicode` in STR_TYPES for Python 2.X try: @@ -110,7 +112,7 @@ class Command(object): if self.subprocess.before: result += self.subprocess.before - if self.subprocess.after: + if self.subprocess.after and self.subprocess.after is not pexpect.EOF: result += self.subprocess.after result += self.subprocess.read() @@ -206,7 +208,10 @@ class Command(object): if self.blocking: raise RuntimeError("expect can only be used on non-blocking commands.") - self.subprocess.expect(pattern=pattern, timeout=timeout) + try: + self.subprocess.expect(pattern=pattern, timeout=timeout) + except pexpect.EOF: + pass def send(self, s, end=os.linesep, signal=False): """Sends the given string or signal to std_in.""" @@ -249,8 +254,11 @@ class Command(object): self.subprocess.wait() else: self.subprocess.sendeof() - self.subprocess.wait() - self.subprocess.proc.stdout.close() + try: + self.subprocess.wait() + finally: + if self.subprocess.proc.stdout: + self.subprocess.proc.stdout.close() def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next @@ -272,7 +280,6 @@ class Command(object): c.run(block=False, cwd=cwd) if data: c.send(data) - c.subprocess.sendeof() c.block() return c diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 9ac6031c..91e9cabb 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.3' +__version__ = '1.1.3.post1' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index d54393b6..f39299a3 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -341,7 +341,6 @@ class SystemPath(object): self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) else: self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver] - print(ver) return ver @classmethod diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index e0bc6746..7b4b6376 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,6 +1,9 @@ # -*- coding=utf-8 -*- -__version__ = '1.1.10' +__version__ = '1.2.0' + +from .models.requirements import Requirement +from .models.lockfile import Lockfile +from .models.pipfile import Pipfile -from .exceptions import RequirementError -from .models import Requirement, Lockfile, Pipfile +__all__ = ["Lockfile", "Pipfile", "Requirement"] diff --git a/pipenv/vendor/requirementslib/exceptions.py b/pipenv/vendor/requirementslib/exceptions.py index 82578624..de8bf8ef 100644 --- a/pipenv/vendor/requirementslib/exceptions.py +++ b/pipenv/vendor/requirementslib/exceptions.py @@ -9,6 +9,8 @@ if six.PY2: def __init__(self, *args, **kwargs): self.errno = errno.EEXIST super(FileExistsError, self).__init__(*args, **kwargs) +else: + from six.moves.builtins import FileExistsError class RequirementError(Exception): diff --git a/pipenv/vendor/requirementslib/models/cache.py b/pipenv/vendor/requirementslib/models/cache.py index 16fc4ba8..71701090 100644 --- a/pipenv/vendor/requirementslib/models/cache.py +++ b/pipenv/vendor/requirementslib/models/cache.py @@ -5,21 +5,19 @@ import copy import hashlib import json import os -import six import sys import requests -import pip_shims import vistir from appdirs import user_cache_dir +from pip_shims.shims import FAVORITE_HASH, SafeFileCache from packaging.requirements import Requirement from .utils import as_tuple, key_from_req, lookup_table, get_pinned_version - -if six.PY2: - from ..exceptions import FileExistsError +from ..exceptions import FileExistsError +from ..utils import VCS_SUPPORT CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) @@ -189,7 +187,7 @@ class DependencyCache(object): for dep_name in self.cache[name][version_and_extras]) -class HashCache(pip_shims.SafeFileCache): +class HashCache(SafeFileCache): """Caches hashes of PyPI artifacts so we do not need to re-download them. Hashes are only cached when the URL appears to contain a hash in it and the @@ -206,7 +204,7 @@ class HashCache(pip_shims.SafeFileCache): def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - vcs = pip_shims.VcsSupport() + vcs = VCS_SUPPORT orig_scheme = location.scheme new_location = copy.deepcopy(location) if orig_scheme in vcs.all_schemes: @@ -223,11 +221,11 @@ class HashCache(pip_shims.SafeFileCache): return hash_value.decode('utf8') def _get_file_hash(self, location): - h = hashlib.new(pip_shims.FAVORITE_HASH) + h = hashlib.new(FAVORITE_HASH) with vistir.contextmanagers.open_file(location, self.session) as fp: for chunk in iter(lambda: fp.read(8096), b""): h.update(chunk) - return ":".join([pip_shims.FAVORITE_HASH, h.hexdigest()]) + return ":".join([FAVORITE_HASH, h.hexdigest()]) class _JSONCache(object): diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index 1997fc1f..3e482813 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -5,6 +5,7 @@ import copy import os import attr +import itertools import plette.lockfiles import six @@ -50,6 +51,31 @@ class Lockfile(object): def _get_lockfile(self): return self.projectfile.lockfile + @property + def lockfile(self): + return self._lockfile + + @property + def section_keys(self): + return ["default", "develop"] + + @property + def extended_keys(self): + return [k for k in itertools.product(self.section_keys, ["", "vcs", "editable"])] + + def get(self, k): + return self.__getitem__(k) + + def __contains__(self, k): + check_lockfile = k in self.extended_keys or self.lockfile.__contains__(k) + if check_lockfile: + return True + return super(Lockfile, self).__contains__(k) + + def __setitem__(self, k, v): + lockfile = self._lockfile + lockfile.__setitem__(k, v) + def __getitem__(self, k, *args, **kwargs): retval = None lockfile = self._lockfile diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 0d1c04c8..fe7743c2 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -75,11 +75,19 @@ class Pipfile(object): def get_deps(self, dev=False, only=True): deps = {} if dev: - deps.update(self.pipfile["dev-packages"]._data) + deps.update(self.pipfile._data["dev-packages"]) if only: return deps - deps = merge_items([deps, self.pipfile["packages"]._data]) - return deps + return merge_items([deps, self.pipfile._data["packages"]]) + + def get(self, k): + return self.__getitem__(k) + + def __contains__(self, k): + check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) + if check_pipfile: + return True + return super(Pipfile, self).__contains__(k) def __getitem__(self, k, *args, **kwargs): retval = None diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index db004869..3a029cbc 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -16,10 +16,7 @@ from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement from packaging.specifiers import Specifier, SpecifierSet from packaging.utils import canonicalize_name -from pip_shims.shims import ( - InstallRequirement, Link, Wheel, _strip_extras, parse_version, path_to_url, - url_to_path -) +from pip_shims.shims import _strip_extras, parse_version, path_to_url, url_to_path from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote from vistir.compat import FileNotFoundError, Path @@ -32,21 +29,16 @@ from vistir.path import ( from ..exceptions import RequirementError from ..utils import VCS_LIST, is_installable_file, is_vcs, ensure_setup_py from .baserequirement import BaseRequirement -from .dependencies import ( - AbstractDependency, find_all_matches, get_abstract_dependencies, - get_dependencies, get_finder -) from .markers import PipenvMarkers from .utils import ( HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string, filter_none, format_requirement, get_version, init_requirement, - is_pinned_requirement, make_install_requirement, optional_instance_of, - parse_extras, specs_to_string, split_markers_from_line, + is_pinned_requirement, make_install_requirement, optional_instance_of, parse_extras, + specs_to_string, split_markers_from_line, ireq_from_editable, ireq_from_line, split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path, - validate_specifiers, validate_vcs, normalize_name, + validate_specifiers, validate_vcs, normalize_name, create_link, Requirement as PkgResourcesRequirement ) -from .vcs import VCSRepository @attr.s @@ -128,7 +120,7 @@ class FileRequirement(BaseRequirement): editable = attr.ib(default=False, type=bool) extras = attr.ib(default=attr.Factory(list), type=list) uri = attr.ib(type=six.string_types) - link = attr.ib(type=Link) + link = attr.ib() name = attr.ib(type=six.string_types) req = attr.ib(type=PkgResourcesRequirement) _has_hashed_name = False @@ -166,6 +158,7 @@ class FileRequirement(BaseRequirement): See `https://bugs.python.org/issue23505#msg277350`. """ + # Git allows `git@github.com...` lines that are not really URIs. # Add "ssh://" so we can parse correctly, and restore afterwards. fixed_line = add_ssh_scheme_to_git_uri(line) @@ -176,7 +169,7 @@ class FileRequirement(BaseRequirement): p = Path(fixed_line).absolute() path = p.as_posix() uri = p.as_uri() - link = Link(uri) + link = create_link(uri) try: relpath = get_converted_relative_path(path) except ValueError: @@ -225,7 +218,7 @@ class FileRequirement(BaseRequirement): uri = strip_ssh_from_git_uri(original_uri) # Re-attach VCS prefix to build a Link. - link = Link( + link = create_link( urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme)) ) @@ -246,6 +239,7 @@ class FileRequirement(BaseRequirement): if self.link and self.link.egg_fragment: return self.link.egg_fragment elif self.link and self.link.is_wheel: + from pip_shims import Wheel return Wheel(self.link.filename).name if ( self._uri_scheme != "uri" @@ -263,7 +257,7 @@ class FileRequirement(BaseRequirement): except (FileNotFoundError, IOError) as e: dist = None except Exception as e: - from pip_shims.shims import InstallRequirement, make_abstract_dist + from pip_shims.shims import make_abstract_dist try: if not isinstance(Path, self.path): @@ -271,9 +265,9 @@ class FileRequirement(BaseRequirement): else: _path = self.path if self.editable: - _ireq = InstallRequirement.from_editable(_path.as_uri()) + _ireq = ireq_from_editable(_path.as_uri()) else: - _ireq = InstallRequirement.from_line(_path.as_posix()) + _ireq = ireq_from_line(_path.as_posix()) dist = make_abstract_dist(_ireq).get_dist() name = dist.project_name except (TypeError, ValueError, AttributeError) as e: @@ -286,7 +280,7 @@ class FileRequirement(BaseRequirement): self._has_hashed_name = True name = hashed_name if self.link and not self._has_hashed_name: - self.link = Link("{0}#egg={1}".format(self.link.url, name)) + self.link = create_link("{0}#egg={1}".format(self.link.url, name)) return name @link.default @@ -294,7 +288,7 @@ class FileRequirement(BaseRequirement): target = "{0}".format(self.uri) if hasattr(self, "name"): target = "{0}#egg={1}".format(target, self.name) - link = Link(target) + link = create_link(target) return link @req.default @@ -359,6 +353,7 @@ class FileRequirement(BaseRequirement): "uri_scheme": prefer, } if link and link.is_wheel: + from pip_shims import Wheel arg_dict["name"] = Wheel(link.filename).name elif link.egg_fragment: arg_dict["name"] = link.egg_fragment @@ -398,7 +393,7 @@ class FileRequirement(BaseRequirement): if not uri: uri = path_to_url(path) - link = Link(uri) + link = create_link(uri) arg_dict = { "name": name, @@ -588,6 +583,7 @@ class VCSRequirement(FileRequirement): return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) def get_vcs_repo(self, src_dir=None): + from .vcs import VCSRepository checkout_dir = self.get_checkout_dir(src_dir=src_dir) url = "{0}#egg={1}".format(self.vcs_uri, self.name) vcsrepo = VCSRepository( @@ -825,6 +821,7 @@ class Requirement(object): @classmethod def from_line(cls, line): + from pip_shims import InstallRequirement if isinstance(line, InstallRequirement): line = format_requirement(line) hashes = None @@ -1074,9 +1071,9 @@ class Requirement(object): if ireq_line.startswith("-e "): ireq_line = ireq_line[len("-e "):] with ensure_setup_py(self.req.path): - ireq = InstallRequirement.from_editable(ireq_line) + ireq = ireq_from_editable(ireq_line) else: - ireq = InstallRequirement.from_line(ireq_line) + ireq = ireq_from_line(ireq_line) if not getattr(ireq, "req", None): ireq.req = self.req.req else: @@ -1103,6 +1100,8 @@ class Requirement(object): :return: A set of requirement strings of the dependencies of this requirement. :rtype: set(str) """ + + from .dependencies import get_dependencies if not sources: sources = [{ 'name': 'pypi', @@ -1122,6 +1121,7 @@ class Requirement(object): :rtype: list[ :class:`~requirementslib.models.dependency.AbstractDependency` ] """ + from .dependencies import AbstractDependency, get_dependencies, get_abstract_dependencies if not self.abstract_dep: parent = getattr(self, 'parent', None) self.abstract_dep = AbstractDependency.from_requirement(self, parent=parent) @@ -1144,6 +1144,8 @@ class Requirement(object): :return: A list of Installation Candidates :rtype: list[ :class:`~pip._internal.index.InstallationCandidate` ] """ + + from .dependencies import get_finder, find_all_matches if not finder: finder = get_finder(sources=sources) return find_all_matches(finder, self.as_ireq()) diff --git a/pipenv/vendor/requirementslib/models/resolvers.py b/pipenv/vendor/requirementslib/models/resolvers.py index da6d0dda..1a239390 100644 --- a/pipenv/vendor/requirementslib/models/resolvers.py +++ b/pipenv/vendor/requirementslib/models/resolvers.py @@ -4,11 +4,10 @@ from contextlib import contextmanager import attr import six -from pip_shims.shims import VcsSupport, Wheel +from pip_shims.shims import Wheel -from ..utils import log +from ..utils import log, VCS_SUPPORT from .cache import HashCache -from .dependencies import AbstractDependency, find_all_matches, get_finder from .utils import format_requirement, is_pinned_requirement, version_from_ireq @@ -41,6 +40,7 @@ class DependencyResolver(object): @classmethod def create(cls, finder=None, allow_prereleases=False, get_all_hashes=True): if not finder: + from .dependencies import get_finder finder_args = [] if allow_prereleases: finder_args.append('--pre') @@ -140,6 +140,7 @@ class DependencyResolver(object): # Coerce input into AbstractDependency instances. # We accept str, Requirement, and AbstractDependency as input. + from .dependencies import AbstractDependency for dep in root_nodes: if isinstance(dep, six.string_types): dep = AbstractDependency.from_string(dep) @@ -183,6 +184,7 @@ class DependencyResolver(object): def get_hashes_for_one(self, ireq): if not self.finder: + from .dependencies import get_finder finder_args = [] if self.allow_prereleases: finder_args.append('--pre') @@ -191,7 +193,7 @@ class DependencyResolver(object): if ireq.editable: return set() - vcs = VcsSupport() + vcs = VCS_SUPPORT if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: return set() @@ -201,6 +203,7 @@ class DependencyResolver(object): matching_candidates = set() with self.allow_all_wheels(): + from .dependencies import find_all_matches matching_candidates = ( find_all_matches(self.finder, ireq, pre=self.allow_prereleases) ) diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 7350d0ac..3103580e 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -19,7 +19,7 @@ from packaging.requirements import Requirement as PackagingRequirement from pkg_resources import Requirement from vistir.misc import dedup -from pip_shims.shims import InstallRequirement, Link + from ..utils import SCHEME_LIST, VCS_LIST, is_star @@ -37,6 +37,21 @@ def optional_instance_of(cls): return validators.optional(validators.instance_of(cls)) +def create_link(link): + from pip_shims import Link + return Link(link) + + +def ireq_from_line(ireq): + from pip_shims import InstallRequirement + return InstallRequirement.from_line(ireq) + + +def ireq_from_editable(ireq): + from pip_shims import InstallRequirement + return InstallRequirement.from_editable(ireq) + + def init_requirement(name): req = Requirement.parse(name) req.vcs = None @@ -92,7 +107,7 @@ def build_vcs_link(vcs, uri, name=None, ref=None, subdirectory=None, extras=None uri = "{0}{1}".format(uri, extras) if subdirectory: uri = "{0}&subdirectory={1}".format(uri, subdirectory) - return Link(uri) + return create_link(uri) def get_version(pipfile_entry): @@ -443,11 +458,11 @@ def make_install_requirement(name, version, extras, markers, constraint=False): extras_string = "[{}]".format(",".join(sorted(extras))) if not markers: - return InstallRequirement.from_line( + return ireq_from_line( str('{}{}=={}'.format(name, extras_string, version)), constraint=constraint) else: - return InstallRequirement.from_line( + return ireq_from_line( str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))), constraint=constraint) diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index abc89831..085f3241 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -5,7 +5,6 @@ import contextlib import logging import os -import boltons.iterutils import six import tomlkit @@ -30,8 +29,10 @@ VCS_LIST = ("git", "svn", "hg", "bzr") VCS_SCHEMES = [] SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") +VCS_SUPPORT = VcsSupport() + if not VCS_SCHEMES: - VCS_SCHEMES = VcsSupport().all_schemes + VCS_SCHEMES = VCS_SUPPORT.all_schemes def setup_logger(): @@ -197,6 +198,127 @@ def ensure_setup_py(base_dir): setup_py.unlink() + +_UNSET = object() +_REMAP_EXIT = object() + + +# The following functionality is either borrowed or modified from the itertools module +# in the boltons library by Mahmoud Hashemi and distributed under the BSD license +# the text of which is included below: + +# (original text from https://github.com/mahmoud/boltons/blob/master/LICENSE) +# Copyright (c) 2013, Mahmoud Hashemi +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * The names of the contributors may not be used to endorse or +# promote products derived from this software without specific +# prior written permission. +# +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class PathAccessError(KeyError, IndexError, TypeError): + """An amalgamation of KeyError, IndexError, and TypeError, + representing what can occur when looking up a path in a nested + object. + """ + def __init__(self, exc, seg, path): + self.exc = exc + self.seg = seg + self.path = path + + def __repr__(self): + cn = self.__class__.__name__ + return '%s(%r, %r, %r)' % (cn, self.exc, self.seg, self.path) + + def __str__(self): + return ('could not access %r from path %r, got error: %r' + % (self.seg, self.path, self.exc)) + + +def get_path(root, path, default=_UNSET): + """Retrieve a value from a nested object via a tuple representing the + lookup path. + >>> root = {'a': {'b': {'c': [[1], [2], [3]]}}} + >>> get_path(root, ('a', 'b', 'c', 2, 0)) + 3 + The path format is intentionally consistent with that of + :func:`remap`. + One of get_path's chief aims is improved error messaging. EAFP is + great, but the error messages are not. + For instance, ``root['a']['b']['c'][2][1]`` gives back + ``IndexError: list index out of range`` + What went out of range where? get_path currently raises + ``PathAccessError: could not access 2 from path ('a', 'b', 'c', 2, + 1), got error: IndexError('list index out of range',)``, a + subclass of IndexError and KeyError. + You can also pass a default that covers the entire operation, + should the lookup fail at any level. + Args: + root: The target nesting of dictionaries, lists, or other + objects supporting ``__getitem__``. + path (tuple): A list of strings and integers to be successively + looked up within *root*. + default: The value to be returned should any + ``PathAccessError`` exceptions be raised. + """ + if isinstance(path, six.string_types): + path = path.split('.') + cur = root + try: + for seg in path: + try: + cur = cur[seg] + except (KeyError, IndexError) as exc: + raise PathAccessError(exc, seg, path) + except TypeError as exc: + # either string index in a list, or a parent that + # doesn't support indexing + try: + seg = int(seg) + cur = cur[seg] + except (ValueError, KeyError, IndexError, TypeError): + if not getattr(cur, "__iter__", None): + exc = TypeError('%r object is not indexable' + % type(cur).__name__) + raise PathAccessError(exc, seg, path) + except PathAccessError: + if default is _UNSET: + raise + return default + return cur + + +def default_visit(path, key, value): + return key, value + + +_orig_default_visit = default_visit + + # Modified from https://github.com/mahmoud/boltons/blob/master/boltons/iterutils.py def dict_path_enter(path, key, value): if isinstance(value, six.string_types): @@ -249,6 +371,157 @@ def dict_path_exit(path, key, old_parent, new_parent, new_items): return ret +def remap(root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, + **kwargs): + """The remap ("recursive map") function is used to traverse and + transform nested structures. Lists, tuples, sets, and dictionaries + are just a few of the data structures nested into heterogenous + tree-like structures that are so common in programming. + Unfortunately, Python's built-in ways to manipulate collections + are almost all flat. List comprehensions may be fast and succinct, + but they do not recurse, making it tedious to apply quick changes + or complex transforms to real-world data. + remap goes where list comprehensions cannot. + Here's an example of removing all Nones from some data: + >>> from pprint import pprint + >>> reviews = {'Star Trek': {'TNG': 10, 'DS9': 8.5, 'ENT': None}, + ... 'Babylon 5': 6, 'Dr. Who': None} + >>> pprint(remap(reviews, lambda p, k, v: v is not None)) + {'Babylon 5': 6, 'Star Trek': {'DS9': 8.5, 'TNG': 10}} + Notice how both Nones have been removed despite the nesting in the + dictionary. Not bad for a one-liner, and that's just the beginning. + See `this remap cookbook`_ for more delicious recipes. + .. _this remap cookbook: http://sedimental.org/remap.html + remap takes four main arguments: the object to traverse and three + optional callables which determine how the remapped object will be + created. + Args: + root: The target object to traverse. By default, remap + supports iterables like :class:`list`, :class:`tuple`, + :class:`dict`, and :class:`set`, but any object traversable by + *enter* will work. + visit (callable): This function is called on every item in + *root*. It must accept three positional arguments, *path*, + *key*, and *value*. *path* is simply a tuple of parents' + keys. *visit* should return the new key-value pair. It may + also return ``True`` as shorthand to keep the old item + unmodified, or ``False`` to drop the item from the new + structure. *visit* is called after *enter*, on the new parent. + The *visit* function is called for every item in root, + including duplicate items. For traversable values, it is + called on the new parent object, after all its children + have been visited. The default visit behavior simply + returns the key-value pair unmodified. + enter (callable): This function controls which items in *root* + are traversed. It accepts the same arguments as *visit*: the + path, the key, and the value of the current item. It returns a + pair of the blank new parent, and an iterator over the items + which should be visited. If ``False`` is returned instead of + an iterator, the value will not be traversed. + The *enter* function is only called once per unique value. The + default enter behavior support mappings, sequences, and + sets. Strings and all other iterables will not be traversed. + exit (callable): This function determines how to handle items + once they have been visited. It gets the same three + arguments as the other functions -- *path*, *key*, *value* + -- plus two more: the blank new parent object returned + from *enter*, and a list of the new items, as remapped by + *visit*. + Like *enter*, the *exit* function is only called once per + unique value. The default exit behavior is to simply add + all new items to the new parent, e.g., using + :meth:`list.extend` and :meth:`dict.update` to add to the + new parent. Immutable objects, such as a :class:`tuple` or + :class:`namedtuple`, must be recreated from scratch, but + use the same type as the new parent passed back from the + *enter* function. + reraise_visit (bool): A pragmatic convenience for the *visit* + callable. When set to ``False``, remap ignores any errors + raised by the *visit* callback. Items causing exceptions + are kept. See examples for more details. + remap is designed to cover the majority of cases with just the + *visit* callable. While passing in multiple callables is very + empowering, remap is designed so very few cases should require + passing more than one function. + When passing *enter* and *exit*, it's common and easiest to build + on the default behavior. Simply add ``from boltons.iterutils import + default_enter`` (or ``default_exit``), and have your enter/exit + function call the default behavior before or after your custom + logic. See `this example`_. + Duplicate and self-referential objects (aka reference loops) are + automatically handled internally, `as shown here`_. + .. _this example: http://sedimental.org/remap.html#sort_all_lists + .. _as shown here: http://sedimental.org/remap.html#corner_cases + """ + # TODO: improve argument formatting in sphinx doc + # TODO: enter() return (False, items) to continue traverse but cancel copy? + if not callable(visit): + raise TypeError('visit expected callable, not: %r' % visit) + if not callable(enter): + raise TypeError('enter expected callable, not: %r' % enter) + if not callable(exit): + raise TypeError('exit expected callable, not: %r' % exit) + reraise_visit = kwargs.pop('reraise_visit', True) + if kwargs: + raise TypeError('unexpected keyword arguments: %r' % kwargs.keys()) + + path, registry, stack = (), {}, [(None, root)] + new_items_stack = [] + while stack: + key, value = stack.pop() + id_value = id(value) + if key is _REMAP_EXIT: + key, new_parent, old_parent = value + id_value = id(old_parent) + path, new_items = new_items_stack.pop() + value = exit(path, key, old_parent, new_parent, new_items) + registry[id_value] = value + if not new_items_stack: + continue + elif id_value in registry: + value = registry[id_value] + else: + res = enter(path, key, value) + try: + new_parent, new_items = res + except TypeError: + # TODO: handle False? + raise TypeError('enter should return a tuple of (new_parent,' + ' items_iterator), not: %r' % res) + if new_items is not False: + # traverse unless False is explicitly passed + registry[id_value] = new_parent + new_items_stack.append((path, [])) + if value is not root: + path += (key,) + stack.append((_REMAP_EXIT, (key, new_parent, value))) + if new_items: + stack.extend(reversed(list(new_items))) + continue + if visit is _orig_default_visit: + # avoid function call overhead by inlining identity operation + visited_item = (key, value) + else: + try: + visited_item = visit(path, key, value) + except Exception: + if reraise_visit: + raise + visited_item = True + if visited_item is False: + continue # drop + elif visited_item is True: + visited_item = (key, value) + # TODO: typecheck? + # raise TypeError('expected (key, value) from visit(),' + # ' not: %r' % visited_item) + try: + new_items_stack[-1][1].append(visited_item) + except IndexError: + raise TypeError('expected remappable root, not: %r' % root) + return value + + def merge_items(target_list, sourced=False): if not sourced: target_list = [(id(t), t) for t in target_list] @@ -262,7 +535,7 @@ def merge_items(target_list, sourced=False): new_parent = ret try: - cur_val = boltons.iterutils.get_path(ret, path + (key,)) + cur_val = get_path(ret, path + (key,)) except KeyError as ke: pass else: @@ -279,9 +552,9 @@ def merge_items(target_list, sourced=False): source_map[path + (key,)] = t_name return True else: - remerge_visit = boltons.iterutils.default_visit + remerge_visit = default_visit - ret = boltons.iterutils.remap(target, enter=remerge_enter, visit=remerge_visit, + ret = remap(target, enter=remerge_enter, visit=remerge_visit, exit=remerge_exit) if not sourced: diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index abece020..781e2e98 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -18,7 +18,7 @@ from ._compat import unicode from ._utils import escape_string if PY2: - from functools32 import lru_cache + from pipenv.vendor.backports.functools_lru_cache import lru_cache else: from functools import lru_cache diff --git a/pipenv/vendor/tomlkit/toml_char.py b/pipenv/vendor/tomlkit/toml_char.py index 92f1a9c9..5164ea8b 100644 --- a/pipenv/vendor/tomlkit/toml_char.py +++ b/pipenv/vendor/tomlkit/toml_char.py @@ -4,7 +4,7 @@ from ._compat import PY2 from ._compat import unicode if PY2: - from functools32 import lru_cache + from pipenv.vendor.backports.functools_lru_cache import lru_cache else: from functools import lru_cache diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 8812d2ae..741ae319 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -21,13 +21,13 @@ pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.3 +pythonfinder==1.1.3.post1 requests==2.20.0 chardet==3.0.4 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.1.10 +requirementslib==1.2.0 attrs==18.2.0 distlib==0.2.8 packaging==18.0 @@ -41,7 +41,7 @@ semver==2.8.1 shutilwhich==1.1.0 toml==0.10.0 cached-property==1.4.3 -vistir==0.1.7 +vistir==0.2.0 pip-shims==0.3.1 ptyprocess==0.6.0 enum34==1.1.6 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index 881985d2..c8b253b8 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -11,11 +11,11 @@ from .contextmanagers import ( spinner, ) from .misc import load_path, partialclass, run, shell_escape -from .path import mkdir_p, rmtree, create_tracked_tempdir +from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile from .spin import VistirSpinner, create_spinner -__version__ = '0.1.8' +__version__ = '0.2.0' __all__ = [ @@ -36,5 +36,6 @@ __all__ = [ "spinner", "VistirSpinner", "create_spinner", - "create_tracked_tempdir" + "create_tracked_tempdir", + "create_tracked_tempfile", ] diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index 43470a6e..7b8066ee 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py @@ -175,6 +175,7 @@ def NamedTemporaryFile( prefix=None, dir=None, delete=True, + wrapper_class_override=None ): """Create and return a temporary file. Arguments: @@ -203,7 +204,10 @@ def NamedTemporaryFile( file = io.open( fd, mode, buffering=buffering, newline=newline, encoding=encoding ) - return _TemporaryFileWrapper(file, name, delete) + if wrapper_class_override is not None: + return wrapper_class_override(file, name, delete) + else: + return _TemporaryFileWrapper(file, name, delete) except BaseException: os.unlink(name) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index eab87908..ec3b65cb 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -33,9 +33,10 @@ else: from pathlib2 import Path from pipenv.vendor.backports.functools_lru_cache import lru_cache +from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile if sys.version_info < (3, 3): from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size - from .backports.tempfile import NamedTemporaryFile + NamedTemporaryFile = _NamedTemporaryFile else: from tempfile import NamedTemporaryFile from shutil import get_terminal_size diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 8f25e079..bcbf7541 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -231,12 +231,13 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None): @contextmanager -def open_file(link, session=None): +def open_file(link, session=None, stream=True): """ Open local or remote file for reading. :type link: pip._internal.index.Link or str :type session: requests.Session + :param bool stream: Try to stream if remote, default True :raises ValueError: If link points to a local directory. :return: a context manager to the opened file-like object """ @@ -255,11 +256,8 @@ def open_file(link, session=None): if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(link)) else: - try: - local_file = io.open(local_path, "rb") - yield local_file - finally: - local_file.close() + with io.open(local_path, "rb") as local_file: + yield local_file else: # Remote URL headers = {"Accept-Encoding": "identity"} @@ -267,8 +265,14 @@ def open_file(link, session=None): from requests import Session session = Session() - response = session.get(link, headers=headers, stream=True) - try: - yield response.raw - finally: - response.close() + with session.get(link, headers=headers, stream=stream) as resp: + try: + raw = getattr(resp, "raw", None) + result = raw if raw else resp + yield result + finally: + result.close() + if raw: + conn = getattr(raw, "_connection") + if conn is not None: + conn.close() diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index e2f85985..26382419 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -49,7 +49,7 @@ def _get_logger(name=None, level="ERROR"): formatter = logging.Formatter( "%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s" ) - handler = logging.StreamHandler() + handler = logging.StreamHandler(stream=sys.stderr) handler.setFormatter(formatter) logger.addHandler(handler) return logger @@ -157,7 +157,7 @@ def _create_subprocess( raise if not block: c.stdin.close() - log_level = "DEBUG" if verbose else "WARN" + log_level = "DEBUG" if verbose else "ERROR" logger = _get_logger(cmd._parts[0], level=log_level) output = [] err = [] @@ -199,7 +199,7 @@ def _create_subprocess( display_line = "{0}...".format(stdout_line[:display_limit]) if verbose: if spinner: - spinner.write(fs_str(display_line)) + spinner.write_err(fs_str(display_line)) else: logger.debug(display_line) if spinner: diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index ce7ecee0..d580aba2 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -15,7 +15,15 @@ import six from six.moves import urllib_parse from six.moves.urllib import request as urllib_request -from .compat import Path, _fs_encoding, TemporaryDirectory, ResourceWarning +from .backports.tempfile import _TemporaryFileWrapper +from .compat import ( + _NamedTemporaryFile, + Path, + ResourceWarning, + TemporaryDirectory, + _fs_encoding, + finalize, +) __all__ = [ @@ -28,6 +36,7 @@ __all__ = [ "mkdir_p", "ensure_mkdir_p", "create_tracked_tempdir", + "create_tracked_tempfile", "path_to_url", "rmtree", "safe_expandvars", @@ -37,6 +46,10 @@ __all__ = [ ] +if os.name == "nt" and six.PY34: + warnings.filterwarnings("ignore", category=DeprecationWarning, message="The Windows bytes API has been deprecated.*") + + def unicode_path(path): # Paths are supposed to be represented as unicode here if six.PY2 and not isinstance(path, six.text_type): @@ -52,9 +65,10 @@ def native_path(path): # once again thank you django... # https://github.com/django/django/blob/fc6b90b/django/utils/_os.py -if six.PY3 or os.name == 'nt': +if six.PY3 or os.name == "nt": abspathu = os.path.abspath else: + def abspathu(path): """ Version of os.path.abspath that uses the unicode representation @@ -74,6 +88,7 @@ def normalize_drive(path): always converted to uppercase because it seems to be preferred. """ from .misc import to_text + if os.name != "nt" or not isinstance(path, six.string_types): return path @@ -110,6 +125,7 @@ def url_to_path(url): Follows logic taken from pip's equivalent function """ from .misc import to_bytes + assert is_file_url(url), "Only file: urls can be converted to local paths" _, netloc, path, _, _ = urllib_parse.urlsplit(url) # Netlocs are UNC paths @@ -123,6 +139,7 @@ def url_to_path(url): def is_valid_url(url): """Checks if a given string is an url""" from .misc import to_text + if not url: return url pieces = urllib_parse.urlparse(to_text(url)) @@ -132,6 +149,7 @@ def is_valid_url(url): def is_file_url(url): """Returns true if the given url is a file url""" from .misc import to_text + if not url: return False if not isinstance(url, six.string_types): @@ -149,6 +167,7 @@ def is_readonly_path(fn): Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)` """ from .misc import to_bytes + fn = to_bytes(fn, encoding="utf-8") if os.path.exists(fn): return bool(os.stat(fn).st_mode & stat.S_IREAD) and not os.access(fn, os.W_OK) @@ -164,6 +183,7 @@ def mkdir_p(newdir, mode=0o777): """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ from .misc import to_bytes, to_text + newdir = to_bytes(newdir, "utf-8") if os.path.exists(newdir): if not os.path.isdir(newdir): @@ -176,7 +196,9 @@ def mkdir_p(newdir, mode=0o777): head, tail = os.path.split(to_bytes(newdir, encoding="utf-8")) # Make sure the tail doesn't point to the asame place as the head curdir = to_bytes(".", encoding="utf-8") - tail_and_head_match = os.path.relpath(tail, start=os.path.basename(head)) == curdir + tail_and_head_match = ( + os.path.relpath(tail, start=os.path.basename(head)) == curdir + ) if tail and not tail_and_head_match and not os.path.isdir(newdir): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): @@ -191,6 +213,7 @@ def mkdir_p(newdir, mode=0o777): def ensure_mkdir_p(mode=0o777): """Decorator to ensure `mkdir_p` is called to the function's return value. """ + def decorator(f): @functools.wraps(f) @@ -224,6 +247,19 @@ def create_tracked_tempdir(*args, **kwargs): return tempdir.name +def create_tracked_tempfile(*args, **kwargs): + """Create a tracked temporary file. + + This uses the `NamedTemporaryFile` construct, but does not remove the file + until the interpreter exits. + + The return value is the file object. + """ + + kwargs["wrapper_class_override"] = _TrackedTempfileWrapper + return _NamedTemporaryFile(*args, **kwargs) + + def set_write_bit(fn): """Set read-write permissions for the current user on the target path. Fail silently if the path doesn't exist. @@ -232,6 +268,7 @@ def set_write_bit(fn): """ from .misc import to_bytes, locale_encoding + fn = to_bytes(fn, encoding=locale_encoding) if not os.path.exists(fn): return @@ -253,6 +290,7 @@ def rmtree(directory, ignore_errors=False): """ from .misc import locale_encoding, to_bytes + directory = to_bytes(directory, encoding=locale_encoding) try: shutil.rmtree( @@ -278,10 +316,12 @@ def handle_remove_readonly(func, path, exc): This function will call check :func:`is_readonly_path` before attempting to call :func:`set_write_bit` on the target path and try again. """ + # Check for read-only attribute if six.PY2: from .compat import ResourceWarning from .misc import to_bytes + PERM_ERRORS = (errno.EACCES, errno.EPERM) default_warning_message = ( "Unable to remove file due to permissions restriction: {!r}" @@ -418,3 +458,28 @@ def safe_expandvars(value): if isinstance(value, six.string_types): return os.path.expandvars(value) return value + + +class _TrackedTempfileWrapper(_TemporaryFileWrapper): + def __init__(self, *args, **kwargs): + super(_TrackedTempfileWrapper, self).__init__(*args, **kwargs) + self._finalizer = finalize(self, self.cleanup) + + @classmethod + def _cleanup(cls, fileobj): + try: + fileobj.close() + finally: + os.unlink(fileobj.name) + + def cleanup(self): + if self._finalizer.detach(): + try: + self.close() + finally: + os.unlink(self.name) + else: + try: + self.close() + except OSError: + pass diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index d2cddd79..e4b4ba66 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -3,7 +3,7 @@ import os import signal import sys -from .termcolors import colored +from .termcolors import colored, COLORS from .compat import fs_str import cursor @@ -15,6 +15,7 @@ except ImportError: Spinners = None else: from yaspin.spinners import Spinners + from yaspin.constants import COLOR_MAP handler = None if yaspin and os.name == "nt": @@ -41,6 +42,16 @@ class DummySpinner(object): self.write_err(traceback) return False + def __getattr__(self, k): + try: + retval = super(DummySpinner, self).__getattribute__(k) + except AttributeError: + if k in COLOR_MAP.keys() or k.upper() in COLORS: + return self + raise + else: + return retval + def fail(self, exitcode=1, text=None): if text: self.write_err(text) @@ -125,6 +136,42 @@ class VistirSpinner(base_obj): ) return fn + def _register_signal_handlers(self): + # SIGKILL cannot be caught or ignored, and the receiving + # process cannot perform any clean-up upon receiving this + # signal. + try: + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) + except AttributeError: + pass + + for sig, sig_handler in self._sigmap.items(): + # A handler for a particular signal, once set, remains + # installed until it is explicitly reset. Store default + # signal handlers for subsequent reset at cleanup phase. + dfl_handler = signal.getsignal(sig) + self._dfl_sigmap[sig] = dfl_handler + + # ``signal.SIG_DFL`` and ``signal.SIG_IGN`` are also valid + # signal handlers and are not callables. + if callable(sig_handler): + # ``signal.signal`` accepts handler function which is + # called with two arguments: signal number and the + # interrupted stack frame. ``functools.partial`` solves + # the problem of passing spinner instance into the handler + # function. + sig_handler = functools.partial(sig_handler, spinner=self) + + signal.signal(sig, sig_handler) + + def _reset_signal_handlers(self): + for sig, sig_handler in self._dfl_sigmap.items(): + signal.signal(sig, sig_handler) + @staticmethod def _hide_cursor(): cursor.hide() diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py index 6f3ad32c..b8ccda8e 100644 --- a/pipenv/vendor/vistir/termcolors.py +++ b/pipenv/vendor/vistir/termcolors.py @@ -79,6 +79,7 @@ def colored(text, color=None, on_color=None, attrs=None): style = "BRIGHT" attrs.remove('bold') if color is not None: + color = color.upper() text = text = "%s%s%s%s%s" % ( getattr(colorama.Fore, color), getattr(colorama.Style, style), @@ -88,8 +89,9 @@ def colored(text, color=None, on_color=None, attrs=None): ) if on_color is not None: + on_color = on_color.upper() text = "%s%s%s%s" % ( - getattr(colorama.Back, color), + getattr(colorama.Back, on_color), text, colorama.Back.RESET, colorama.Style.NORMAL, diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index d01fb98e..06b8b621 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -16,6 +16,9 @@ import sys import threading import time +import colorama +import cursor + from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS @@ -23,6 +26,9 @@ from .helpers import to_unicode from .termcolor import colored +colorama.init() + + class Yaspin(object): """Implements a context manager that spawns a thread to write spinner frames into a tty (stdout) during @@ -369,11 +375,14 @@ class Yaspin(object): # SIGKILL cannot be caught or ignored, and the receiving # process cannot perform any clean-up upon receiving this # signal. - if signal.SIGKILL in self._sigmap.keys(): - raise ValueError( - "Trying to set handler for SIGKILL signal. " - "SIGKILL cannot be cought or ignored in POSIX systems." - ) + try: + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) + except AttributeError: + pass for sig, sig_handler in iteritems(self._sigmap): # A handler for a particular signal, once set, remains @@ -521,14 +530,12 @@ class Yaspin(object): @staticmethod def _hide_cursor(): - sys.stdout.write("\033[?25l") - sys.stdout.flush() + cursor.hide() @staticmethod def _show_cursor(): - sys.stdout.write("\033[?25h") - sys.stdout.flush() + cursor.show() @staticmethod def _clear_line(): - sys.stdout.write("\033[K") + sys.stdout.write(chr(27) + "[K") diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 3198c2d4..38ec032f 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -34,6 +34,7 @@ PY2_DOWNLOAD = ['enum34',] # from time to time, remove the no longer needed ones HARDCODED_LICENSE_URLS = { 'pytoml': 'https://github.com/avakar/pytoml/raw/master/LICENSE', + 'cursor': 'https://raw.githubusercontent.com/GijsTimmers/cursor/master/LICENSE', 'delegator.py': 'https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE', 'click-didyoumean': 'https://raw.githubusercontent.com/click-contrib/click-didyoumean/master/LICENSE', 'click-completion': 'https://raw.githubusercontent.com/click-contrib/click-completion/master/LICENSE', @@ -70,6 +71,7 @@ PATCHED_RENAMES = { LIBRARY_RENAMES = { 'pip': 'pipenv.patched.notpip', + "functools32": "pipenv.vendor.backports.functools_lru_cache", 'enum34': 'enum', } diff --git a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch index ac63825c..175efaa1 100644 --- a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch +++ b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch @@ -1,8 +1,26 @@ diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py -index 0c140cad..3ffb2e31 100644 +index d15aeb97..56d12458 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py -@@ -178,6 +178,7 @@ class Command(object): +@@ -7,6 +7,8 @@ import locale + import errno + + from pexpect.popen_spawn import PopenSpawn ++import pexpect ++pexpect.EOF.__module__ = "pexpect.exceptions" + + # Include `unicode` in STR_TYPES for Python 2.X + try: +@@ -110,7 +112,7 @@ class Command(object): + if self.subprocess.before: + result += self.subprocess.before + +- if self.subprocess.after: ++ if self.subprocess.after and self.subprocess.after is not pexpect.EOF: + result += self.subprocess.after + + result += self.subprocess.read() +@@ -178,6 +180,7 @@ class Command(object): # Use subprocess. if self.blocking: popen_kwargs = self._default_popen_kwargs.copy() @@ -10,11 +28,21 @@ index 0c140cad..3ffb2e31 100644 popen_kwargs["universal_newlines"] = not binary if cwd: popen_kwargs["cwd"] = cwd -@@ -233,18 +234,23 @@ class Command(object): - def block(self): +@@ -205,7 +208,10 @@ class Command(object): + if self.blocking: + raise RuntimeError("expect can only be used on non-blocking commands.") + +- self.subprocess.expect(pattern=pattern, timeout=timeout) ++ try: ++ self.subprocess.expect(pattern=pattern, timeout=timeout) ++ except pexpect.EOF: ++ pass + + def send(self, s, end=os.linesep, signal=False): + """Sends the given string or signal to std_in.""" +@@ -234,14 +240,25 @@ class Command(object): """Blocks until process is complete.""" if self._uses_subprocess: -- self.subprocess.stdin.close() # consume stdout and stderr - try: - stdout, stderr = self.subprocess.communicate() @@ -35,10 +63,21 @@ index 0c140cad..3ffb2e31 100644 + self.std_err.close() + self.subprocess.wait() else: - self.subprocess.sendeof() -- self.subprocess.proc.stdout.close() - self.subprocess.wait() -+ self.subprocess.proc.stdout.close() +- self.subprocess.wait() ++ self.subprocess.sendeof() ++ try: ++ self.subprocess.wait() ++ finally: ++ if self.subprocess.proc.stdout: ++ self.subprocess.proc.stdout.close() def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next +@@ -263,7 +280,6 @@ class Command(object): + c.run(block=False, cwd=cwd) + if data: + c.send(data) +- c.subprocess.sendeof() + c.block() + return c + diff --git a/tasks/vendoring/patches/vendor/vistir-imports.patch b/tasks/vendoring/patches/vendor/vistir-imports.patch index f93e7959..673efad8 100644 --- a/tasks/vendoring/patches/vendor/vistir-imports.patch +++ b/tasks/vendoring/patches/vendor/vistir-imports.patch @@ -1,25 +1,3 @@ -diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py -index 1f1b7a96..0c865fe6 100644 ---- a/pipenv/vendor/vistir/compat.py -+++ b/pipenv/vendor/vistir/compat.py -@@ -30,7 +30,7 @@ else: - from pathlib2 import Path - - if sys.version_info < (3, 3): -- from backports.shutil_get_terminal_size import get_terminal_size -+ from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size - from .backports.tempfile import NamedTemporaryFile - else: - from tempfile import NamedTemporaryFile -@@ -39,7 +39,7 @@ else: - try: - from weakref import finalize - except ImportError: -- from backports.weakref import finalize -+ from pipenv.vendor.backports.weakref import finalize - - try: - from functools import partialmethod diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index 483a479a..43470a6e 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py @@ -33,3 +11,30 @@ index 483a479a..43470a6e 100644 __all__ = ["finalize", "NamedTemporaryFile"] +diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py +index 9ae33fdc..ec3b65cb 100644 +--- a/pipenv/vendor/vistir/compat.py ++++ b/pipenv/vendor/vistir/compat.py +@@ -31,11 +31,11 @@ if sys.version_info >= (3, 5): + from functools import lru_cache + else: + from pathlib2 import Path +- from backports.functools_lru_cache import lru_cache ++ from pipenv.vendor.backports.functools_lru_cache import lru_cache + + from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile + if sys.version_info < (3, 3): +- from backports.shutil_get_terminal_size import get_terminal_size ++ from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size + NamedTemporaryFile = _NamedTemporaryFile + else: + from tempfile import NamedTemporaryFile +@@ -44,7 +44,7 @@ else: + try: + from weakref import finalize + except ImportError: +- from backports.weakref import finalize ++ from pipenv.vendor.backports.weakref import finalize + + try: + from functools import partialmethod diff --git a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch new file mode 100644 index 00000000..a1d27cd3 --- /dev/null +++ b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch @@ -0,0 +1,62 @@ +diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py +index d01fb98e..06b8b621 100644 +--- a/pipenv/vendor/yaspin/core.py ++++ b/pipenv/vendor/yaspin/core.py +@@ -16,6 +16,9 @@ import sys + import threading + import time + ++import colorama ++import cursor ++ + from .base_spinner import default_spinner + from .compat import PY2, basestring, builtin_str, bytes, iteritems, str + from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS +@@ -23,6 +26,9 @@ from .helpers import to_unicode + from .termcolor import colored + + ++colorama.init() ++ ++ + class Yaspin(object): + """Implements a context manager that spawns a thread + to write spinner frames into a tty (stdout) during +@@ -369,11 +375,14 @@ class Yaspin(object): + # SIGKILL cannot be caught or ignored, and the receiving + # process cannot perform any clean-up upon receiving this + # signal. +- if signal.SIGKILL in self._sigmap.keys(): +- raise ValueError( +- "Trying to set handler for SIGKILL signal. " +- "SIGKILL cannot be cought or ignored in POSIX systems." +- ) ++ try: ++ if signal.SIGKILL in self._sigmap.keys(): ++ raise ValueError( ++ "Trying to set handler for SIGKILL signal. " ++ "SIGKILL cannot be cought or ignored in POSIX systems." ++ ) ++ except AttributeError: ++ pass + + for sig, sig_handler in iteritems(self._sigmap): + # A handler for a particular signal, once set, remains +@@ -521,14 +530,12 @@ class Yaspin(object): + + @staticmethod + def _hide_cursor(): +- sys.stdout.write("\033[?25l") +- sys.stdout.flush() ++ cursor.hide() + + @staticmethod + def _show_cursor(): +- sys.stdout.write("\033[?25h") +- sys.stdout.flush() ++ cursor.show() + + @staticmethod + def _clear_line(): +- sys.stdout.write("\033[K") ++ sys.stdout.write(chr(27) + "[K") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 12f27342..7f797f99 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,5 +1,6 @@ import json import os +import sys import warnings import pytest @@ -7,13 +8,12 @@ import pytest from pipenv._compat import TemporaryDirectory, Path from pipenv.vendor import delegator from pipenv.vendor import requests -from pipenv.vendor import six from pipenv.vendor import toml from pytest_pypi.app import prepare_packages as prepare_pypi_packages +from vistir.compat import ResourceWarning -if six.PY2: - class ResourceWarning(Warning): - pass + +warnings.filterwarnings("default", category=ResourceWarning) HAS_WARNED_GITHUB = False @@ -25,8 +25,8 @@ def check_internet(): resp = requests.get('http://httpbin.org/ip', timeout=1.0) resp.raise_for_status() except Exception: - warnings.warn('Cannot connect to HTTPBin...', ResourceWarning) - warnings.warn('Will skip tests requiring Internet', ResourceWarning) + warnings.warn('Cannot connect to HTTPBin...', RuntimeWarning) + warnings.warn('Will skip tests requiring Internet', RuntimeWarning) return False return True @@ -46,10 +46,10 @@ def check_github_ssh(): global HAS_WARNED_GITHUB if not res and not HAS_WARNED_GITHUB: warnings.warn( - 'Cannot connect to GitHub via SSH', ResourceWarning + 'Cannot connect to GitHub via SSH', RuntimeWarning ) warnings.warn( - 'Will skip tests requiring SSH access to GitHub', ResourceWarning + 'Will skip tests requiring SSH access to GitHub', RuntimeWarning ) HAS_WARNED_GITHUB = True return res @@ -70,18 +70,109 @@ def pytest_runtest_setup(item): pytest.skip('requires github ssh') +@pytest.yield_fixture +def pathlib_tmpdir(request, tmpdir): + yield Path(str(tmpdir)) + tmpdir.remove(ignore_errors=True) + + +# Borrowed from pip's test runner filesystem isolation +@pytest.fixture(autouse=True) +def isolate(pathlib_tmpdir): + """ + Isolate our tests so that things like global configuration files and the + like do not affect our test results. + We use an autouse function scoped fixture because we want to ensure that + every test has it's own isolated home directory. + """ + warnings.filterwarnings("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") + + + # Create a directory to use as our home location. + home_dir = os.path.join(str(pathlib_tmpdir), "home") + os.environ["PIPENV_NOSPIN"] = "1" + os.makedirs(home_dir) + + # Create a directory to use as a fake root + fake_root = os.path.join(str(pathlib_tmpdir), "fake-root") + os.makedirs(fake_root) + + # if sys.platform == 'win32': + # # Note: this will only take effect in subprocesses... + # home_drive, home_path = os.path.splitdrive(home_dir) + # os.environ.update({ + # 'USERPROFILE': home_dir, + # 'HOMEDRIVE': home_drive, + # 'HOMEPATH': home_path, + # }) + # for env_var, sub_path in ( + # ('APPDATA', 'AppData/Roaming'), + # ('LOCALAPPDATA', 'AppData/Local'), + # ): + # path = os.path.join(home_dir, *sub_path.split('/')) + # os.environ[env_var] = path + # os.makedirs(path) + # else: + # # Set our home directory to our temporary directory, this should force + # # all of our relative configuration files to be read from here instead + # # of the user's actual $HOME directory. + # os.environ["HOME"] = home_dir + # # Isolate ourselves from XDG directories + # os.environ["XDG_DATA_HOME"] = os.path.join(home_dir, ".local", "share") + # os.environ["XDG_CONFIG_HOME"] = os.path.join(home_dir, ".config") + # os.environ["XDG_CACHE_HOME"] = os.path.join(home_dir, ".cache") + # os.environ["XDG_RUNTIME_DIR"] = os.path.join(home_dir, ".runtime") + # os.environ["XDG_DATA_DIRS"] = ":".join([ + # os.path.join(fake_root, "usr", "local", "share"), + # os.path.join(fake_root, "usr", "share"), + # ]) + # os.environ["XDG_CONFIG_DIRS"] = os.path.join(fake_root, "etc", "xdg") + + # Configure git, because without an author name/email git will complain + # and cause test failures. + os.environ["GIT_CONFIG_NOSYSTEM"] = "1" + os.environ["GIT_AUTHOR_NAME"] = "pipenv" + os.environ["GIT_AUTHOR_EMAIL"] = "pipenv@pipenv.org" + + # We want to disable the version check from running in the tests + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true" + workon_home = os.path.join(home_dir, ".virtualenvs") + os.makedirs(workon_home) + os.environ["WORKON_HOME"] = workon_home + project_dir = os.path.join(home_dir, "pipenv_project") + os.makedirs(project_dir) + os.environ["PIPENV_PROJECT_DIR"] = project_dir + os.environ["CI"] = "1" + + # Make sure tests don't share a requirements tracker. + os.environ.pop('PIP_REQ_TRACKER', None) + + # FIXME: Windows... + os.makedirs(os.path.join(home_dir, ".config", "git")) + with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: + fp.write( + b"[user]\n\tname = pipenv\n\temail = pipenv@pipenv.org\n" + ) + + class _PipenvInstance(object): """An instance of a Pipenv Project...""" - def __init__(self, pypi=None, pipfile=True, chdir=False): + def __init__(self, pypi=None, pipfile=True, chdir=False, path=None): self.pypi = pypi self.original_umask = os.umask(0o007) self.original_dir = os.path.abspath(os.curdir) - self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') - path = Path(self._path.name) - try: - self.path = str(path.resolve()) - except OSError: - self.path = str(path.absolute()) + path = os.environ.get("PIPENV_PROJECT_DIR", None) + if not path: + self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + path = Path(self._path.name) + try: + self.path = str(path.resolve()) + except OSError: + self.path = str(path.absolute()) + else: + self._path = None + self.path = path # set file creation perms self.pipfile_path = None self.chdir = chdir @@ -101,6 +192,7 @@ class _PipenvInstance(object): os.environ['PIPENV_DONT_USE_PYENV'] = '1' os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' os.environ['PIPENV_VENV_IN_PROJECT'] = '1' + os.environ['PIPENV_NOSPIN'] = '1' if self.chdir: os.chdir(self.path) return self @@ -110,13 +202,13 @@ class _PipenvInstance(object): if self.chdir: os.chdir(self.original_dir) self.path = None - try: - self._path.cleanup() - except OSError as e: - _warn_msg = warn_msg.format(e) - warnings.warn(_warn_msg, ResourceWarning) - finally: - os.umask(self.original_umask) + if self._path: + try: + self._path.cleanup() + except OSError as e: + _warn_msg = warn_msg.format(e) + warnings.warn(_warn_msg, ResourceWarning) + os.umask(self.original_umask) def pipenv(self, cmd, block=True): if self.pipfile_path: @@ -162,7 +254,7 @@ class _PipenvInstance(object): @pytest.fixture() def PipenvInstance(): - return _PipenvInstance + yield _PipenvInstance @pytest.fixture(scope='module') diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 1f1719d0..7743804d 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -37,9 +37,9 @@ flask = "==0.12.2" """.strip() f.write(contents) - req_list = ("requests==2.14.0") + req_list = ("requests==2.14.0",) - dev_req_list = ("flask==0.12.2") + dev_req_list = ("flask==0.12.2",) c = p.pipenv('lock -r') d = p.pipenv('lock -r -d') diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 98c85602..40977ede 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -4,6 +4,7 @@ import pytest from mock import patch, Mock from first import first import pipenv.utils +import pythonfinder.utils # Pipfile format <-> requirements.txt format. @@ -215,13 +216,13 @@ class TestUtils: ), ], ) - @patch("delegator.run") + # @patch(".vendor.pythonfinder.utils.get_python_version") def test_python_version_output_variants( - self, mocked_delegator, version_output, version + self, monkeypatch, version_output, version ): - run_ret = Mock() - run_ret.out = version_output - mocked_delegator.return_value = run_ret + def mock_version(path): + return version_output.split()[1] + monkeypatch.setattr("pipenv.vendor.pythonfinder.utils.get_python_version", mock_version) assert pipenv.utils.python_version("some/path") == version @pytest.mark.utils