diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 7968801c..95c62dc5 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -1,3 +1,22 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + phases: - template: phases/test.yml parameters: diff --git a/.vsts-ci/phases/run-tests.yml b/.vsts-ci/phases/run-tests.yml index 81f02875..fe35ad8d 100644 --- a/.vsts-ci/phases/run-tests.yml +++ b/.vsts-ci/phases/run-tests.yml @@ -11,10 +11,10 @@ steps: mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" mkdir -p "$WORKON_HOME" pip install certifi - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 + export GIT_SSL_CAINFO="$(python -m certifi)" + export LANG="C.UTF-8" export PIP_PROCESS_DEPENDENCY_LINKS="1" - echo "Path: $PATH" + echo "Path $PATH" echo "Installing Pipenv…" pip install -e "$(pwd)" --upgrade pipenv install --deploy --dev @@ -23,8 +23,8 @@ steps: - script: | # Fix Git SSL errors - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 + export GIT_SSL_CAINFO="$(python -m certifi)" + export LANG="C.UTF-8" export PIP_PROCESS_DEPENDENCY_LINKS="1" pipenv run pytest --junitxml=test-results.xml displayName: Run integration tests diff --git a/.vsts-ci/steps/run-tests.yml b/.vsts-ci/steps/run-tests.yml index cb24b092..a7b99a13 100644 --- a/.vsts-ci/steps/run-tests.yml +++ b/.vsts-ci/steps/run-tests.yml @@ -3,13 +3,16 @@ steps: # Fix Git SSL errors pip install certifi python -m certifi > cacert.txt - $env:GIT_SSL_CAINFO = $(Get-Content cacert.txt) + Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]$(Get-Content cacert.txt)" + $env:GIT_SSL_CAINFO="$(Get-Content cacert.txt)" # Shorten paths to get under MAX_PATH or else integration tests will fail # https://bugs.python.org/issue18199 - subst T: $env:TEMP - $env:TEMP = "T:\" - $env:TMP = "T:\" - D:\.venv\Scripts\pipenv run pytest -n 4 --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests + subst T: "$env:TEMP" + Write-Host "##vso[task.setvariable variable=TEMP]T:\" + $env:TEMP='T:\' + Write-Host "##vso[task.setvariable variable=TMP]T:\" + $env:TEMP='T:\' + D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests - task: PublishTestResults@2 diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index a397a23c..e423c3ab 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -1,3 +1,22 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + phases: - template: phases/test.yml parameters: diff --git a/news/3020.feature b/news/3020.feature.rst similarity index 100% rename from news/3020.feature rename to news/3020.feature.rst diff --git a/news/3088.bugfix.rst b/news/3088.bugfix.rst new file mode 100644 index 00000000..b10c4b2b --- /dev/null +++ b/news/3088.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which caused ``Unexpected EOF`` errors to be thrown when PIP awaited input from users who put login credentials in their environment. diff --git a/news/3089.feature.rst b/news/3089.feature.rst new file mode 100644 index 00000000..47f280ee --- /dev/null +++ b/news/3089.feature.rst @@ -0,0 +1 @@ +Added windows-compatible spinner via upgraded ``vistir`` dependency. diff --git a/news/3089.vendor.rst b/news/3089.vendor.rst new file mode 100644 index 00000000..c57209f6 --- /dev/null +++ b/news/3089.vendor.rst @@ -0,0 +1,11 @@ +Updated vendored dependencies: + - ``certifi 2018.08.24 => 2018.10.15`` + - ``urllib3 1.23 => 1.24`` + - ``requests 2.19.1 => 2.20.0`` + - ``shellingham ``1.2.6 => 1.2.7`` + - ``tomlkit 0.4.4. => 0.4.6`` + - ``vistir 0.1.6 => 0.1.8`` + - ``pythonfinder 0.1.2 => 0.1.3`` + - ``requirementslib 1.1.9 => 1.1.10`` + - ``backports.functools_lru_cache 1.5.0 (new)`` + - ``cursor 1.2.0 (new)`` diff --git a/news/3090.bugfix.rst b/news/3090.bugfix.rst new file mode 100644 index 00000000..af772c3d --- /dev/null +++ b/news/3090.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug in ``requirementslib`` which prevented successful installation from mercurial repositories. diff --git a/news/3094.bugfix.rst b/news/3094.bugfix.rst new file mode 100644 index 00000000..e17b2e32 --- /dev/null +++ b/news/3094.bugfix.rst @@ -0,0 +1 @@ +Fixed random resource warnings when using pyenv or any other subprocess calls. diff --git a/news/3102.bugfix.rst b/news/3102.bugfix.rst new file mode 100644 index 00000000..2224ac0b --- /dev/null +++ b/news/3102.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. diff --git a/news/3109.bugfix.rst b/news/3109.bugfix.rst new file mode 100644 index 00000000..af5718b6 --- /dev/null +++ b/news/3109.bugfix.rst @@ -0,0 +1 @@ +Fixed issues with broken subprocess calls leaking resource handles and causing random and sporadic failures. diff --git a/news/3113.bugfix.rst b/news/3113.bugfix.rst new file mode 100644 index 00000000..af43b87d --- /dev/null +++ b/news/3113.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue resolving virtualenv paths for users without ``platlib`` values on their systems. diff --git a/news/3114.bugfix.rst b/news/3114.bugfix.rst new file mode 100644 index 00000000..2224ac0b --- /dev/null +++ b/news/3114.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. diff --git a/news/3117.bugfix.rst b/news/3117.bugfix.rst new file mode 100644 index 00000000..2224ac0b --- /dev/null +++ b/news/3117.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. diff --git a/news/3121.bugfix.rst b/news/3121.bugfix.rst new file mode 100644 index 00000000..fb815c42 --- /dev/null +++ b/news/3121.bugfix.rst @@ -0,0 +1 @@ +Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. diff --git a/news/3121.vendor.rst b/news/3121.vendor.rst new file mode 100644 index 00000000..fb815c42 --- /dev/null +++ b/news/3121.vendor.rst @@ -0,0 +1 @@ +Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 471dcb94..6b8ddf66 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -1,11 +1,14 @@ +# -*- coding=utf-8 -*- # |~~\' |~~ # |__/||~~\|--|/~\\ / # | ||__/|__| |\/ # | + import os import sys -from .__version__ import __version__ +import warnings +from .__version__ import __version__ PIPENV_ROOT = os.path.dirname(os.path.realpath(__file__)) PIPENV_VENDOR = os.sep.join([PIPENV_ROOT, "vendor"]) @@ -14,12 +17,28 @@ PIPENV_PATCHED = os.sep.join([PIPENV_ROOT, "patched"]) sys.path.insert(0, PIPENV_VENDOR) # Inject patched directory into system path. sys.path.insert(0, PIPENV_PATCHED) + +from pipenv.vendor.urllib3.exceptions import DependencyWarning +from pipenv.vendor.vistir.compat import ResourceWarning, fs_str +warnings.filterwarnings("ignore", category=DependencyWarning) +warnings.filterwarnings("ignore", category=ResourceWarning) + +if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): + if sys.stdout.isatty() and sys.stderr.isatty(): + import io + sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') + sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') + +os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = fs_str("1") +os.environ["PIP_SHIMS_BASE_MODULE"] = fs_str("pipenv.patched.notpip") + # Hack to make things work better. try: if "concurrency" in sys.modules: del sys.modules["concurrency"] except Exception: pass + from .cli import cli from . import resolver diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 558df3b8..6cc4c732 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -11,6 +11,7 @@ import os import six import sys import warnings +import vistir from tempfile import _bin_openflags, gettempdir, _mkstemp_inner, mkdtemp from .utils import logging, rmtree @@ -59,10 +60,10 @@ except ImportError: return False -if six.PY2: +from vistir.compat import ResourceWarning - class ResourceWarning(Warning): - pass + +warnings.filterwarnings("ignore", category=ResourceWarning) def pip_import(module_path, subimport=None, old_path=None): @@ -300,3 +301,89 @@ def NamedTemporaryFile( os.unlink(name) os.close(fd) raise + + +def getpreferredencoding(): + import locale + # Borrowed from Invoke + # (see https://github.com/pyinvoke/invoke/blob/93af29d/invoke/runners.py#L881) + _encoding = locale.getpreferredencoding(False) + if six.PY2 and not sys.platform == "win32": + _default_encoding = locale.getdefaultlocale()[1] + if _default_encoding is not None: + _encoding = _default_encoding + return _encoding + + +DEFAULT_ENCODING = getpreferredencoding() + + +# From https://github.com/CarlFK/veyepar/blob/5c5de47/dj/scripts/fixunicode.py +# MIT LIcensed, thanks Carl! +def force_encoding(): + try: + stdout_isatty = sys.stdout.isatty + stderr_isatty = sys.stderr.isatty + except AttributeError: + return DEFAULT_ENCODING, DEFAULT_ENCODING + else: + if not (stdout_isatty() and stderr_isatty()): + return DEFAULT_ENCODING, DEFAULT_ENCODING + stdout_encoding = sys.stdout.encoding + stderr_encoding = sys.stderr.encoding + if sys.platform == "win32" and sys.version_info >= (3, 1): + return DEFAULT_ENCODING, DEFAULT_ENCODING + if stdout_encoding.lower() != "utf-8" or stderr_encoding.lower() != "utf-8": + + from ctypes import pythonapi, py_object, c_char_p + try: + PyFile_SetEncoding = pythonapi.PyFile_SetEncoding + except AttributeError: + return DEFAULT_ENCODING, DEFAULT_ENCODING + else: + PyFile_SetEncoding.argtypes = (py_object, c_char_p) + if stdout_encoding.lower() != "utf-8": + try: + was_set = PyFile_SetEncoding(sys.stdout, "utf-8") + except OSError: + was_set = False + if not was_set: + stdout_encoding = DEFAULT_ENCODING + else: + stdout_encoding = "utf-8" + + if stderr_encoding.lower() != "utf-8": + try: + was_set = PyFile_SetEncoding(sys.stderr, "utf-8") + except OSError: + was_set = False + if not was_set: + stderr_encoding = DEFAULT_ENCODING + else: + stderr_encoding = "utf-8" + + return stdout_encoding, stderr_encoding + + +OUT_ENCODING, ERR_ENCODING = force_encoding() + + +UNICODE_TO_ASCII_TRANSLATION_MAP = { + 8230: u"...", + 8211: u"-" +} + + +def decode_output(output): + if not isinstance(output, six.string_types): + return output + try: + output = output.encode(DEFAULT_ENCODING) + except (AttributeError, UnicodeDecodeError): + if six.PY2: + output = unicode.translate(vistir.misc.to_text(output), + UNICODE_TO_ASCII_TRANSLATION_MAP) + else: + output = output.translate(UNICODE_TO_ASCII_TRANSLATION_MAP) + output = output.decode(DEFAULT_ENCODING) + return output diff --git a/pipenv/core.py b/pipenv/core.py index ca37bd5c..59b9a29c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1,11 +1,11 @@ # -*- coding=utf-8 -*- + import contextlib import logging import os import sys import shutil import time -import tempfile import json as simplejson import click import click_completion @@ -13,10 +13,13 @@ import crayons import dotenv import delegator import pipfile -from blindspin import spinner import vistir +import warnings import six +import urllib3.util as urllib3_util +from functools import partial + from .cmdparse import Script from .project import Project, SourceNotFound from .utils import ( @@ -24,8 +27,6 @@ from .utils import ( is_required_version, proper_case, pep423_name, - split_file, - merge_deps, venv_resolve_deps, escape_grouped_arguments, python_version, @@ -45,9 +46,7 @@ from .utils import ( from . import environments, pep508checker, progress from .environments import ( PIPENV_COLORBLIND, - PIPENV_NOSPIN, PIPENV_SHELL_FANCY, - PIPENV_TIMEOUT, PIPENV_SKIP_VALIDATION, PIPENV_HIDE_EMOJIS, PIPENV_YES, @@ -96,12 +95,44 @@ click_completion.init() # Disable colors, for the color blind and others who do not prefer colors. if PIPENV_COLORBLIND: crayons.disable() -# Disable spinner, for cleaner build logs (the unworthy). -if PIPENV_NOSPIN: - @contextlib.contextmanager # noqa: F811 - def spinner(): - yield + +UNICODE_TO_ASCII_TRANSLATION_MAP = { + 8230: u"...", + 8211: u"-" +} + + +def fix_utf8(text): + if not isinstance(text, six.string_types): + return text + from ._compat import decode_output + try: + text = decode_output(text) + except UnicodeDecodeError: + if six.PY2: + text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) + return text + + +@contextlib.contextmanager +def _spinner(text=None, nospin=None, spinner_name=None): + if not text: + text = "Running..." + if not spinner_name: + spinner_name = environments.PIPENV_SPINNER + if nospin is None: + nospin = environments.PIPENV_NOSPIN + with vistir.spin.create_spinner( + spinner_name=spinner_name, + start_text=text, + nospin=nospin + ) as sp: + yield sp + + +spinner = partial(_spinner, text="Running...", nospin=environments.PIPENV_NOSPIN, + spinner_name=environments.PIPENV_SPINNER) def which(command, location=None, allow_global=False): @@ -132,7 +163,7 @@ project = Project(which=which) def do_clear(): - click.echo(crayons.white("Clearing caches…", bold=True)) + click.echo(crayons.white(fix_utf8("Clearing caches…"), bold=True)) try: from pip._internal import locations except ImportError: # pip 9. @@ -161,7 +192,7 @@ def load_dot_env(): if os.path.isfile(dotenv_file): click.echo( - crayons.normal("Loading .env environment variables…", bold=True), + crayons.normal(fix_utf8("Loading .env environment variables…"), bold=True), err=True, ) else: @@ -204,7 +235,7 @@ def cleanup_virtualenv(bare=True): def import_requirements(r=None, dev=False): from .patched.notpip._vendor import requests as pip_requests - from .patched.notpip._internal.req.req_file import parse_requirements + from .vendor.pip_shims.shims import parse_requirements # Parse requirements.txt file with Pip's parser. # Pip requires a `PipSession` which is a subclass of requests.Session. @@ -295,15 +326,22 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): if project.requirements_exists and not skip_requirements: click.echo( crayons.normal( - u"requirements.txt found, instead of Pipfile! Converting…", + fix_utf8("requirements.txt found, instead of Pipfile! Converting…"), bold=True, ) ) # Create a Pipfile… project.create_pipfile(python=python) - with spinner(): + with spinner(text=vistir.compat.fs_str("Importing requirements..."), + spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: # Import requirements.txt. - import_requirements() + try: + import_requirements() + except Exception: + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Failed...")) + else: + sp.ok(environments.PIPENV_SPINNER_FAIL_TEXT.format("Success!")) # Warn the user of side-effects. click.echo( u"{0}: Your {1} now contains pinned versions, if your {2} did. \n" @@ -317,7 +355,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): ) else: click.echo( - crayons.normal(u"Creating a Pipfile for this project…", bold=True), + crayons.normal(fix_utf8("Creating a Pipfile for this project…"), bold=True), err=True, ) # Create the pipfile if it doesn't exist. @@ -405,7 +443,7 @@ def ensure_python(three=None, python=None): u"{0}: Python {1} {2}".format( crayons.red("Warning", bold=True), crayons.blue(python), - u"was not found on your system…", + fix_utf8("was not found on your system…"), ), err=True, ) @@ -424,7 +462,7 @@ def ensure_python(three=None, python=None): except ValueError: abort() except PyenvError as e: - click.echo(u"Something went wrong…") + click.echo(fix_utf8("Something went wrong…")) click.echo(crayons.blue(e.err), err=True) abort() s = "{0} {1} {2}".format( @@ -443,17 +481,24 @@ def ensure_python(three=None, python=None): crayons.green(u"CPython {0}".format(version), bold=True), crayons.normal(u"with pyenv", bold=True), crayons.normal(u"(this may take a few minutes)"), - crayons.normal(u"…", bold=True), + crayons.normal(fix_utf8("…"), bold=True), ) ) - with spinner(): + with spinner(text=vistir.compat.fs_str("Installing python..."), + spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: try: c = pyenv.install(version) except PyenvError as e: - click.echo(u"Something went wrong…") + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Failed...") + ) + click.echo(fix_utf8("Something went wrong…"), err=True) click.echo(crayons.blue(e.err), err=True) + else: + environments.PIPENV_SPINNER_OK_TEXT.format("Success!") # Print the results, in a beautiful blue… - click.echo(crayons.blue(c.out), err=True) + click.echo(crayons.blue(c.out), err=True) # Find the newly installed Python, hopefully. version = str(version) path_to_python = find_a_system_python(version) @@ -518,7 +563,7 @@ def ensure_virtualenv(three=None, python=None, site_packages=False, pypi_mirror= ): abort() click.echo( - crayons.normal(u"Removing existing virtualenv…", bold=True), err=True + crayons.normal(fix_utf8("Removing existing virtualenv…"), bold=True), err=True ) # Remove the virtualenv. cleanup_virtualenv(bare=True) @@ -665,26 +710,31 @@ def do_install_dependencies( If requirements is True, simply spits out a requirements format to stdout. """ - from .vendor.requirementslib.models.requirements import Requirement + from six.moves import queue def cleanup_procs(procs, concurrent): - for c in procs: - if concurrent: - c.block() + while not procs.empty(): + c = procs.get() + # if concurrent: + c.block() + failed = False + if c.return_code != 0: + failed = True if "Ignoring" in c.out: click.echo(crayons.yellow(c.out.strip())) elif environments.is_verbose(): - click.echo(crayons.blue(c.out or c.err)) + click.echo(crayons.blue(c.out.strip() or c.err.strip())) # The Installation failed… - if c.return_code != 0: + if failed: # Save the Failed Dependency for later. - failed_deps_list.append((c.dep, c.ignore_hash)) + dep = c.dep.copy() + failed_deps_list.append(dep) # Alert the user. click.echo( "{0} {1}! Will try again.".format( crayons.red("An error occurred while installing"), - crayons.green(c.dep.as_line()), - ) + crayons.green(dep.as_line()), + ), err=True ) if requirements: @@ -694,143 +744,138 @@ def do_install_dependencies( if skip_lock or only or not project.lockfile_exists: if not bare: click.echo( - crayons.normal(u"Installing dependencies from Pipfile…", bold=True) + crayons.normal(fix_utf8("Installing dependencies from Pipfile…"), bold=True) ) - lockfile = split_file(project._lockfile) + lockfile = project.get_or_create_lockfile() else: - with open(project.lockfile_location) as f: - lockfile = split_file(simplejson.load(f)) + lockfile = project.get_or_create_lockfile() if not bare: click.echo( crayons.normal( - u"Installing dependencies from Pipfile.lock ({0})…".format( + fix_utf8("Installing dependencies from Pipfile.lock ({0})…".format( lockfile["_meta"].get("hash", {}).get("sha256")[-6:] - ), + )), bold=True, ) ) # Allow pip to resolve dependencies when in skip-lock mode. no_deps = not skip_lock - deps_list, dev_deps_list = merge_deps( - lockfile, - project, - dev=dev, - requirements=requirements, - ignore_hashes=ignore_hashes, - blocking=blocking, - only=only, - ) failed_deps_list = [] + deps_list = list(lockfile.get_requirements(dev=dev, only=only)) if requirements: index_args = prepare_pip_source_args(project.sources) index_args = " ".join(index_args).replace(" -", "\n-") - deps_list = [dep for dep, ignore_hash, block in deps_list] - dev_deps_list = [dep for dep, ignore_hash, block in dev_deps_list] + deps = [ + req.as_line(sources=project.sources, include_hashes=False) for req in deps_list + ] # Output only default dependencies click.echo(index_args) - if not dev: - click.echo( - "\n".join(d.partition("--hash")[0].strip() for d in sorted(deps_list)) - ) - sys.exit(0) - # Output only dev dependencies - if dev: - click.echo( - "\n".join( - d.partition("--hash")[0].strip() for d in sorted(dev_deps_list) - ) - ) - sys.exit(0) - procs = [] - deps_list_bar = progress.bar( - deps_list, label=INSTALL_LABEL if os.name != "nt" else "" - ) - for dep, ignore_hash, block in deps_list_bar: - if len(procs) < PIPENV_MAX_SUBPROCESS: - # Use a specific index, if specified. - indexes, trusted_hosts, dep = parse_indexes(dep) - index = None - extra_indexes = [] - if indexes: - index = indexes[0] - if len(indexes) > 0: - extra_indexes = indexes[1:] - dep = Requirement.from_line(" ".join(dep)) - if index: - _index = None - try: - _index = project.find_source(index).get("name") - except SourceNotFound: - _index = None - dep.index = _index - dep._index = index - dep.extra_indexes = extra_indexes - # Install the module. - prev_no_deps_setting = no_deps - if dep.is_file_or_url and any( - dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] - ): - no_deps = False + click.echo( + "\n".join(sorted(deps)) + ) + sys.exit(0) + + procs = queue.Queue(maxsize=PIPENV_MAX_SUBPROCESS) + trusted_hosts = [] + + deps_list_bar = progress.bar(deps_list, width=32, + label=INSTALL_LABEL if os.name != "nt" else "") + indexes = [] + for dep in deps_list_bar: + index = None + if dep.index: + index = project.find_source(dep.index) + indexes.append(index) + if not index.get("verify_ssl", False): + trusted_hosts.append(urllib3_util.parse_url(index.get("url")).host) + # Install the module. + is_artifact = False + if dep.is_file_or_url and any( + dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] + ): + is_artifact = True + + extra_indexes = [] + if not index and indexes: + index = next(iter(indexes)) + if len(indexes) > 1: + extra_indexes = indexes[1:] + with vistir.contextmanagers.temp_environ(): + if "PIP_USER" in os.environ: + del os.environ["PIP_USER"] c = pip_install( dep, - ignore_hashes=ignore_hash, + ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), allow_global=allow_global, - no_deps=no_deps, - block=block, + no_deps=False if is_artifact else no_deps, + block=any([dep.editable, blocking]), index=index, requirements_dir=requirements_dir, - extra_indexes=extra_indexes, pypi_mirror=pypi_mirror, - trusted_hosts=trusted_hosts + trusted_hosts=trusted_hosts, + extra_indexes=extra_indexes ) - c.dep = dep - c.ignore_hash = ignore_hash - c.index = index - c.extra_indexes = extra_indexes - procs.append(c) - no_deps = prev_no_deps_setting - if len(procs) >= PIPENV_MAX_SUBPROCESS or len(procs) == len(deps_list): - cleanup_procs(procs, concurrent) - procs = [] - cleanup_procs(procs, concurrent) + if procs.qsize() < PIPENV_MAX_SUBPROCESS: + c.dep = dep + procs.put(c) + + if procs.full() or procs.qsize() == len(deps_list): + cleanup_procs(procs, concurrent) + if not procs.empty(): + cleanup_procs(procs, concurrent) + # Iterate over the hopefully-poorly-packaged dependencies… if failed_deps_list: click.echo( - crayons.normal(u"Installing initially failed dependencies…", bold=True) + crayons.normal(fix_utf8("Installing initially failed dependencies…"), bold=True) ) - for dep, ignore_hash in progress.bar(failed_deps_list, label=INSTALL_LABEL2): + for dep in progress.bar(failed_deps_list, label=INSTALL_LABEL2): # Use a specific index, if specified. # Install the module. - prev_no_deps_setting = no_deps + is_artifact = False + index = None + if dep.index: + index = project.find_source(dep.index) if dep.is_file_or_url and any( dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] ): - no_deps = False - c = pip_install( - dep, - ignore_hashes=ignore_hash, - allow_global=allow_global, - no_deps=no_deps, - index=getattr(dep, "_index", None), - requirements_dir=requirements_dir, - extra_indexes=getattr(dep, "extra_indexes", None), - ) - no_deps = prev_no_deps_setting - # The Installation failed… - if c.return_code != 0: - # We echo both c.out and c.err because pip returns error details on out. - click.echo(crayons.blue(format_pip_output(c.out))) - click.echo(crayons.blue(format_pip_error(c.err)), err=True) - # Return the subprocess' return code. - sys.exit(c.return_code) - else: - click.echo( - "{0} {1}{2}".format( - crayons.green("Success installing"), - crayons.green(dep.name), - crayons.green("!"), - ) + is_artifact = True + extra_indexes = [] + if not index and indexes: + index = next(iter(indexes)) + if len(indexes) > 1: + extra_indexes = indexes[1:] + with vistir.contextmanagers.temp_environ(): + if "PIP_USER" in os.environ: + del os.environ["PIP_USER"] + c = pip_install( + dep, + ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), + allow_global=allow_global, + no_deps=True if is_artifact else no_deps, + index=index, + requirements_dir=requirements_dir, + pypi_mirror=pypi_mirror, + trusted_hosts=trusted_hosts, + extra_indexes=extra_indexes, + block=True ) + # The Installation failed… + if c.return_code != 0: + # We echo both c.out and c.err because pip returns error details on out. + click.echo(crayons.blue(format_pip_output(c.out))) + click.echo(crayons.blue(format_pip_error(c.err)), err=True) + # Return the subprocess' return code. + sys.exit(c.return_code) + else: + if environments.is_verbose(): + click.echo( + "{0} {1}{2}".format( + crayons.green("Success installing"), + crayons.green(dep.as_line(include_hashes=False)), + crayons.green("!"), + ), + ) def convert_three_to_python(three, python): @@ -851,7 +896,7 @@ def convert_three_to_python(three, python): def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): """Creates a virtualenv.""" click.echo( - crayons.normal(u"Creating a virtualenv for this project…", bold=True), err=True + crayons.normal(fix_utf8("Creating a virtualenv for this project…"), bold=True), err=True ) click.echo( u"Pipfile: {0}".format(crayons.red(project.pipfile_location, bold=True)), @@ -865,14 +910,14 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): u"{0} {1} {3} {2}".format( crayons.normal("Using", bold=True), crayons.red(python, bold=True), - crayons.normal(u"to create virtualenv…", bold=True), + crayons.normal(fix_utf8("to create virtualenv…"), bold=True), crayons.green("({0})".format(python_version(python))), ), err=True, ) cmd = [ - sys.executable, + vistir.compat.Path(sys.executable).absolute().as_posix(), "-m", "virtualenv", "--prompt=({0}) ".format(project.name), @@ -883,7 +928,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): # Pass site-packages flag to virtualenv, if desired… if site_packages: click.echo( - crayons.normal(u"Making site-packages available…", bold=True), err=True + crayons.normal(fix_utf8("Making site-packages available…"), bold=True), err=True ) cmd.append("--system-site-packages") @@ -893,11 +938,12 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): pip_config = {} # Actually create the virtualenv. - with spinner(): - c = delegator.run(cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config) - c.block() + nospin = os.environ.get("PIPENV_ACTIVE", environments.PIPENV_NOSPIN) + c = vistir.misc.run(cmd, verbose=False, return_object=True, + spinner_name=environments.PIPENV_SPINNER, combine_stderr=False, + block=False, nospin=nospin, env=pip_config) click.echo(crayons.blue("{0}".format(c.out)), err=True) - if c.return_code != 0: + if c.returncode != 0: click.echo(crayons.blue("{0}".format(c.err)), err=True) click.echo( u"{0}: Failed to create virtual environment.".format( @@ -992,9 +1038,6 @@ def do_lock( if dev_package in project.packages: dev_packages[dev_package] = project.packages[dev_package] # Resolve dev-package dependencies, with pip-tools. - pip_freeze = delegator.run( - "{0} freeze".format(escape_grouped_arguments(which_pip(allow_global=system))) - ).out sections = { "dev": { "packages": project.dev_packages, @@ -1019,9 +1062,9 @@ def do_lock( # Alert the user of progress. click.echo( u"{0} {1} {2}".format( - crayons.normal("Locking"), - crayons.red("[{0}]".format(settings["log_string"])), - crayons.normal("dependencies…"), + crayons.normal(u"Locking"), + crayons.red(u"[{0}]".format(settings["log_string"])), + crayons.normal(fix_utf8("dependencies…")), ), err=True, ) @@ -1050,7 +1093,6 @@ def do_lock( # TODO: be smarter about this. vcs_reqs, vcs_lockfile = get_vcs_deps( project, - pip_freeze, which=which, clear=clear, pre=pre, @@ -1125,7 +1167,7 @@ def do_purge(bare=False, downloads=False, allow_global=False): if downloads: if not bare: - click.echo(crayons.normal(u"Clearing out downloads directory…", bold=True)) + click.echo(crayons.normal(fix_utf8("Clearing out downloads directory…"), bold=True)) shutil.rmtree(project.download_location) return @@ -1154,7 +1196,7 @@ def do_purge(bare=False, downloads=False, allow_global=False): actually_installed.append(dep) if not bare: click.echo( - u"Found {0} installed package(s), purging…".format(len(actually_installed)) + fix_utf8("Found {0} installed package(s), purging…".format(len(actually_installed))) ) command = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=allow_global)), @@ -1185,7 +1227,6 @@ def do_init( """Executes the init functionality.""" from .environments import PIPENV_VIRTUALENV - cleanup_reqdir = False if not system: if not project.virtualenv_exists: try: @@ -1197,8 +1238,7 @@ def do_init( if not deploy: ensure_pipfile(system=system) if not requirements_dir: - cleanup_reqdir = True - requirements_dir = vistir.compat.TemporaryDirectory( + requirements_dir = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) # Write out the lockfile if it doesn't exist, but not if the Pipfile is being ignored @@ -1215,26 +1255,25 @@ def do_init( ) ) click.echo(crayons.normal("Aborting deploy.", bold=True), err=True) - requirements_dir.cleanup() sys.exit(1) elif (system or allow_global) and not (PIPENV_VIRTUALENV): click.echo( - crayons.red( - u"Pipfile.lock ({0}) out of date, but installation " - u"uses {1}… re-building lockfile must happen in " - u"isolation. Please rebuild lockfile in a virtualenv. " - u"Continuing anyway…".format( + crayons.red(fix_utf8( + "Pipfile.lock ({0}) out of date, but installation " + "uses {1}… re-building lockfile must happen in " + "isolation. Please rebuild lockfile in a virtualenv. " + "Continuing anyway…".format( crayons.white(old_hash[-6:]), crayons.white("--system") - ), + )), bold=True, ), err=True, ) else: if old_hash: - msg = u"Pipfile.lock ({1}) out of date, updating to ({0})…" + msg = fix_utf8("Pipfile.lock ({1}) out of date, updating to ({0})…") else: - msg = u"Pipfile.lock is corrupted, replaced with ({0})…" + msg = fix_utf8("Pipfile.lock is corrupted, replaced with ({0})…") click.echo( crayons.red(msg.format(old_hash[-6:], new_hash[-6:]), bold=True), err=True, @@ -1259,11 +1298,10 @@ def do_init( err=True, ) click.echo("See also: --deploy flag.", err=True) - requirements_dir.cleanup() sys.exit(1) else: click.echo( - crayons.normal(u"Pipfile.lock not found, creating…", bold=True), + crayons.normal(fix_utf8("Pipfile.lock not found, creating…"), bold=True), err=True, ) do_lock( @@ -1279,11 +1317,9 @@ def do_init( allow_global=allow_global, skip_lock=skip_lock, concurrent=concurrent, - requirements_dir=requirements_dir.name, + requirements_dir=requirements_dir, pypi_mirror=pypi_mirror, ) - if cleanup_reqdir: - requirements_dir.cleanup() # Hint the user what to do to activate the virtualenv. if not allow_global and not deploy and "PIPENV_ACTIVE" not in os.environ: @@ -1312,14 +1348,15 @@ def pip_install( trusted_hosts=None ): from notpip._internal import logger as piplogger + from .utils import Mapping from .vendor.urllib3.util import parse_url src = [] write_to_tmpfile = False if requirement: - editable_with_markers = requirement.editable and requirement.markers needs_hashes = not requirement.editable and not ignore_hashes and r is None - write_to_tmpfile = needs_hashes or editable_with_markers + has_subdir = requirement.is_vcs and requirement.req.subdirectory + write_to_tmpfile = needs_hashes or has_subdir if not trusted_hosts: trusted_hosts = [] @@ -1333,12 +1370,16 @@ def pip_install( ) # Create files for hash mode. if write_to_tmpfile: - with vistir.compat.NamedTemporaryFile( + if not requirements_dir: + requirements_dir = vistir.path.create_tracked_tempdir( + prefix="pipenv", suffix="requirements") + f = vistir.compat.NamedTemporaryFile( prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, delete=False - ) as f: - f.write(vistir.misc.to_bytes(requirement.as_line())) - r = f.name + ) + f.write(vistir.misc.to_bytes(requirement.as_line())) + r = f.name + f.close() # Install dependencies when a package is a VCS dependency. if requirement and requirement.vcs: no_deps = False @@ -1348,21 +1389,27 @@ def pip_install( # Try installing for each source in project.sources. if index: - try: - index_source = project.find_source(index) - index_source = index_source.copy() - except SourceNotFound: - src_name = project.src_name_from_url(index) - index_url = parse_url(index) - verify_ssl = index_url.host not in trusted_hosts - index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} + if isinstance(index, (Mapping, dict)): + index_source = index + else: + try: + index_source = project.find_source(index) + index_source = index_source.copy() + except SourceNotFound: + src_name = project.src_name_from_url(index) + index_url = parse_url(index) + verify_ssl = index_url.host not in trusted_hosts + index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} sources = [index_source.copy(),] if extra_indexes: if isinstance(extra_indexes, six.string_types): extra_indexes = [extra_indexes,] for idx in extra_indexes: + extra_src = None + if isinstance(idx, (Mapping, dict)): + extra_src = idx try: - extra_src = project.find_source(idx) + extra_src = project.find_source(idx) if not extra_src else extra_src except SourceNotFound: src_name = project.src_name_from_url(idx) src_url = parse_url(idx) @@ -1382,12 +1429,15 @@ def pip_install( for source in sources ] if (requirement and requirement.editable) and not r: - install_reqs = requirement.as_line(as_list=True) + line_kwargs = {"as_list": True} + if requirement.markers: + line_kwargs["include_markers"] = False + install_reqs = requirement.as_line(**line_kwargs) if requirement.editable and install_reqs[0].startswith("-e "): req, install_reqs = install_reqs[0], install_reqs[1:] editable_opt, req = req.split(" ", 1) install_reqs = [editable_opt, req] + install_reqs - if not any(item.startswith("--hash") for item in install_reqs): + if not all(item.startswith("--hash") for item in install_reqs): ignore_hashes = True elif r: install_reqs = ["-r", r] @@ -1396,7 +1446,7 @@ def pip_install( ignore_hashes = True else: ignore_hashes = True if not requirement.hashes else False - install_reqs = requirement.as_line(as_list=True) + install_reqs = [escape_cmd(r) for r in requirement.as_line(as_list=True)] pip_command = [which_pip(allow_global=allow_global), "install"] if pre: pip_command.append("--pre") @@ -1409,7 +1459,6 @@ def pip_install( pip_command.append("--upgrade-strategy=only-if-needed") if no_deps: pip_command.append("--no-deps") - install_reqs = [escape_cmd(req) for req in install_reqs] pip_command.extend(install_reqs) pip_command.extend(prepare_pip_source_args(sources)) if not ignore_hashes: @@ -1431,7 +1480,8 @@ def pip_install( pip_config.update( {"PIP_SRC": vistir.misc.fs_str(project.virtualenv_src_location)} ) - pip_command = Script.parse(pip_command).cmdify() + cmd = Script.parse(pip_command) + pip_command = cmd.cmdify() c = delegator.run(pip_command, block=block, env=pip_config) return c @@ -1476,22 +1526,34 @@ def which_pip(allow_global=False): def system_which(command, mult=False): """Emulates the system's which. Returns None if not found.""" _which = "which -a" if not os.name == "nt" else "where" - c = delegator.run("{0} {1}".format(_which, command)) + os.environ = { + vistir.compat.fs_str(k): vistir.compat.fs_str(val) + for k, val in os.environ.items() + } try: - # Which Not found… - if c.return_code == 127: - click.echo( - "{}: the {} system utility is required for Pipenv to find Python installations properly." - "\n Please install it.".format( - crayons.red("Warning", bold=True), crayons.red(_which) - ), - err=True, - ) - assert c.return_code == 0 - except AssertionError: - return None if not mult else [] - - result = c.out.strip() or c.err.strip() + c = delegator.run("{0} {1}".format(_which, command)) + try: + # Which Not found… + if c.return_code == 127: + click.echo( + "{}: the {} system utility is required for Pipenv to find Python installations properly." + "\n Please install it.".format( + crayons.red("Warning", bold=True), crayons.red(_which) + ), + err=True, + ) + assert c.return_code == 0 + except AssertionError: + return None if not mult else[] + except TypeError: + from .vendor.pythonfinder import Finder + finder = Finder() + result = finder.which(command) + if result: + return result.path.as_posix() + return + else: + result = c.out.strip() or c.err.strip() if mult: return result.split("\n") @@ -1623,9 +1685,9 @@ def ensure_lockfile(keep_outdated=False, pypi_mirror=None): if new_hash != old_hash: click.echo( crayons.red( - u"Pipfile.lock ({0}) out of date, updating to ({1})…".format( + fix_utf8("Pipfile.lock ({0}) out of date, updating to ({1})…".format( old_hash[-6:], new_hash[-6:] - ), + )), bold=True, ), err=True, @@ -1698,11 +1760,12 @@ def do_install( selective_upgrade=False, ): from .environments import PIPENV_VIRTUALENV, PIPENV_USE_SYSTEM - from notpip._internal.exceptions import PipError + from .vendor.pip_shims.shims import PipError - requirements_directory = vistir.compat.TemporaryDirectory( + requirements_directory = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) + warnings.filterwarnings("default", category=vistir.compat.ResourceWarning) if selective_upgrade: keep_outdated = True packages = packages if packages else [] @@ -1739,27 +1802,29 @@ def do_install( err=True, ) click.echo("See also: --deploy flag.", err=True) - requirements_directory.cleanup() sys.exit(1) # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True # Check if the file is remote or not if remote: - fd, temp_reqs = tempfile.mkstemp( - prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory.name - ) - requirements_url = requirements - # Download requirements file click.echo( crayons.normal( - u"Remote requirements file provided! Downloading…", bold=True + fix_utf8("Remote requirements file provided! Downloading…"), bold=True ), err=True, ) + fd = vistir.path.create_tracked_tempfile( + prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory + ) + temp_reqs = fd.name + requirements_url = requirements + # Download requirements file try: download_file(requirements, temp_reqs) except IOError: + fd.close() + os.unlink(temp_reqs) click.echo( crayons.red( u"Unable to find requirements file at {0}.".format( @@ -1768,8 +1833,9 @@ def do_install( ), err=True, ) - requirements_directory.cleanup() sys.exit(1) + finally: + fd.close() # Replace the url with the temporary requirements file requirements = temp_reqs remote = True @@ -1777,7 +1843,7 @@ def do_install( error, traceback = None, None click.echo( crayons.normal( - u"Requirements file provided! Importing into Pipfile…", bold=True + fix_utf8("Requirements file provided! Importing into Pipfile…"), bold=True ), err=True, ) @@ -1800,16 +1866,15 @@ def do_install( finally: # If requirements file was provided by remote url delete the temporary file if remote: - os.close(fd) # Close for windows to allow file cleanup. - os.remove(project.path_to(temp_reqs)) + fd.close() # Close for windows to allow file cleanup. + os.remove(temp_reqs) if error and traceback: click.echo(crayons.red(error)) click.echo(crayons.blue(str(traceback)), err=True) - requirements_directory.cleanup() sys.exit(1) if code: click.echo( - crayons.normal(u"Discovering imports from local codebase…", bold=True) + crayons.normal(fix_utf8("Discovering imports from local codebase…"), bold=True) ) for req in import_from_code(code): click.echo(" Found {0}!".format(crayons.green(req))) @@ -1886,17 +1951,21 @@ def do_install( for pkg_line in pkg_list: click.echo( crayons.normal( - u"Installing {0}…".format(crayons.green(pkg_line, bold=True)), + fix_utf8("Installing {0}…".format(crayons.green(pkg_line, bold=True))), bold=True, ) ) # pip install: - with spinner(): + with vistir.contextmanagers.temp_environ(), spinner(text="Installing...", + spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: + if "PIP_USER" in os.environ: + del os.environ["PIP_USER"] try: pkg_requirement = Requirement.from_line(pkg_line) except ValueError as e: - click.echo("{0}: {1}".format(crayons.red("WARNING"), e)) - requirements_directory.cleanup() + sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) sys.exit(1) if index_url: pkg_requirement.index = index_url @@ -1907,14 +1976,14 @@ def do_install( selective_upgrade=selective_upgrade, no_deps=False, pre=pre, - requirements_dir=requirements_directory.name, + requirements_dir=requirements_directory, index=index_url, extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) # Warn if --editable wasn't passed. if pkg_requirement.is_vcs and not pkg_requirement.editable: - click.echo( + sp.write_err( "{0}: You installed a VCS dependency in non-editable mode. " "This will work fine, but sub-dependencies will not be resolved by {1}." "\n To enable this sub-dependency functionality, specify that this dependency is editable." @@ -1923,37 +1992,34 @@ def do_install( crayons.red("$ pipenv lock"), ) ) - click.echo(crayons.blue(format_pip_output(c.out))) - # Ensure that package was successfully installed. - try: - assert c.return_code == 0 - except AssertionError: - click.echo( - "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(pkg_line) - ), - err=True, - ) - click.echo(crayons.blue(format_pip_error(c.err)), err=True) - if "setup.py egg_info" in c.err: - click.echo( - "This is likely caused by a bug in {0}. " - "Report this to its maintainers.".format( - crayons.green(pkg_requirement.name) + click.echo(crayons.blue(format_pip_output(c.out))) + # Ensure that package was successfully installed. + if c.return_code != 0: + sp.write_err(vistir.compat.fs_str( + "{0} An error occurred while installing {1}!".format( + crayons.red("Error: ", bold=True), crayons.green(pkg_line) ), - err=True, + )) + sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) + if "setup.py egg_info" in c.err: + sp.write_err(vistir.compat.fs_str( + "This is likely caused by a bug in {0}. " + "Report this to its maintainers.".format( + crayons.green(pkg_requirement.name) + ) + )) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sys.exit(1) + sp.write(vistir.compat.fs_str( + u"{0} {1} {2} {3}{4}".format( + crayons.normal(u"Adding", bold=True), + crayons.green(u"{0}".format(pkg_requirement.name), bold=True), + crayons.normal(u"to Pipfile's", bold=True), + crayons.red(u"[dev-packages]" if dev else u"[packages]", bold=True), + crayons.normal(fix_utf8("…"), bold=True), ) - requirements_directory.cleanup() - sys.exit(1) - click.echo( - "{0} {1} {2} {3}{4}".format( - crayons.normal("Adding", bold=True), - crayons.green(pkg_requirement.name, bold=True), - crayons.normal("to Pipfile's", bold=True), - crayons.red("[dev-packages]" if dev else "[packages]", bold=True), - crayons.normal("…", bold=True), - ) - ) + )) + sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Installation Succeeded")) # Add the package to the Pipfile. try: project.add_package_to_pipfile(pkg_requirement, dev) @@ -1975,7 +2041,6 @@ def do_install( pypi_mirror=pypi_mirror, skip_lock=skip_lock, ) - requirements_directory.cleanup() sys.exit(0) @@ -2009,7 +2074,7 @@ def do_uninstall( # Un-install all dependencies, if --all was provided. if all is True: click.echo( - crayons.normal(u"Un-installing all packages from virtualenv…", bold=True) + crayons.normal(fix_utf8("Un-installing all packages from virtualenv…"), bold=True) ) do_purge(allow_global=system) return @@ -2025,7 +2090,7 @@ def do_uninstall( return click.echo( crayons.normal( - u"Un-installing {0}…".format(crayons.red("[dev-packages]")), bold=True + fix_utf8("Un-installing {0}…".format(crayons.red("[dev-packages]"))), bold=True ) ) package_names = project.dev_packages.keys() @@ -2033,7 +2098,7 @@ def do_uninstall( click.echo(crayons.red("No package provided!"), err=True) return 1 for package_name in package_names: - click.echo(u"Un-installing {0}…".format(crayons.green(package_name))) + click.echo(fix_utf8("Un-installing {0}…".format(crayons.green(package_name)))) cmd = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=system)), package_name ) @@ -2055,7 +2120,7 @@ def do_uninstall( continue click.echo( - u"Removing {0} from Pipfile…".format(crayons.green(package_name)) + fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name))) ) # Remove package from both packages and dev-packages. project.remove_package_from_pipfile(package_name, dev=True) @@ -2076,7 +2141,7 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror from .shells import choose_shell shell = choose_shell() - click.echo("Launching subshell in virtual environment…", err=True) + click.echo(fix_utf8("Launching subshell in virtual environment…"), err=True) fork_args = (project.virtualenv_location, project.project_directory, shell_args) @@ -2087,9 +2152,9 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror try: shell.fork_compat(*fork_args) except (AttributeError, ImportError): - click.echo( - u"Compatibility mode not supported. " - u"Trying to continue as well-configured shell…", + click.echo(fix_utf8( + "Compatibility mode not supported. " + "Trying to continue as well-configured shell…"), err=True, ) shell.fork(*fork_args) @@ -2099,11 +2164,11 @@ def _inline_activate_virtualenv(): try: activate_this = which("activate_this.py") if not activate_this or not os.path.exists(activate_this): - click.echo( - u"{0}: activate_this.py not found. Your environment is most " - u"certainly not activated. Continuing anyway…" - u"".format(crayons.red("Warning", bold=True)), - err=True, + click.echo(fix_utf8( + "{0}: activate_this.py not found. Your environment is most " + "certainly not activated. Continuing anyway…").format( + crayons.red("Warning", bold=True) + ), err=True, ) return with open(activate_this) as f: @@ -2267,7 +2332,7 @@ def do_check( sys.exit(1) else: sys.exit(0) - click.echo(crayons.normal(u"Checking PEP 508 requirements…", bold=True)) + click.echo(crayons.normal(fix_utf8("Checking PEP 508 requirements…"), bold=True)) if system: python = system_which("python") else: @@ -2303,7 +2368,7 @@ def do_check( sys.exit(1) else: click.echo(crayons.green("Passed!")) - click.echo(crayons.normal(u"Checking installed package safety…", bold=True)) + click.echo(crayons.normal(fix_utf8("Checking installed package safety…"), bold=True)) path = pep508checker.__file__.rstrip("cdo") path = os.sep.join(__file__.split(os.sep)[:-1] + ["patched", "safety.zip"]) if not system: @@ -2501,7 +2566,7 @@ def do_sync( ) # Install everything. - requirements_dir = vistir.compat.TemporaryDirectory( + requirements_dir = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) do_init( @@ -2513,7 +2578,6 @@ def do_sync( deploy=deploy, system=system, ) - requirements_dir.cleanup() click.echo(crayons.green("All dependencies are now up-to-date!")) @@ -2548,7 +2612,7 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro else: click.echo( crayons.white( - "Uninstalling {0}…".format(repr(apparent_bad_package)), bold=True + fix_utf8("Uninstalling {0}…".format(repr(apparent_bad_package))), bold=True ) ) # Uninstall the package. diff --git a/pipenv/environments.py b/pipenv/environments.py index 45d5f720..cf1e1237 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -1,13 +1,17 @@ +# -*- coding=utf-8 -*- + import os import sys from appdirs import user_cache_dir -from .vendor.vistir.misc import fs_str +from .vendor.vistir.misc import fs_str, to_text # HACK: avoid resolver.py uses the wrong byte code files. # I hope I can remove this one day. os.environ["PYTHONDONTWRITEBYTECODE"] = fs_str("1") +PIPENV_IS_CI = bool("CI" in os.environ or "TF_BUILD" in os.environ) + # HACK: Prevent invalid shebangs with Homebrew-installed Python: # https://bugs.python.org/issue22490 os.environ.pop("__PYVENV_LAUNCHER__", None) @@ -68,7 +72,7 @@ PIPENV_HIDE_EMOJIS = bool(os.environ.get("PIPENV_HIDE_EMOJIS")) Default is to show emojis. This is automatically set on Windows. """ -if os.name == "nt": +if os.name == "nt" or PIPENV_IS_CI: PIPENV_HIDE_EMOJIS = True PIPENV_IGNORE_VIRTUALENVS = bool(os.environ.get("PIPENV_IGNORE_VIRTUALENVS")) @@ -94,7 +98,7 @@ Default is 3. See also ``PIPENV_NO_INHERIT``. PIPENV_MAX_RETRIES = int(os.environ.get( "PIPENV_MAX_RETRIES", - "1" if "CI" in os.environ else "0", + "1" if PIPENV_IS_CI else "0", )) """Specify how many retries Pipenv should attempt for network requests. @@ -128,9 +132,18 @@ PIPENV_NOSPIN = bool(os.environ.get("PIPENV_NOSPIN")) This can make the logs cleaner. Automatically set on Windows, and in CI environments. """ -if os.name == "nt" or "CI" in os.environ: +if PIPENV_IS_CI: PIPENV_NOSPIN = True +PIPENV_SPINNER = "dots" +"""Sets the default spinner type. + +Spinners are identitcal to the node.js spinners and can be found at +https://github.com/sindresorhus/cli-spinners +""" +if os.name == "nt": + PIPENV_SPINNER = "bouncingBar" + PIPENV_PIPFILE = os.environ.get("PIPENV_PIPFILE") """If set, this specifies a custom Pipfile location. @@ -258,3 +271,8 @@ def is_verbose(threshold=1): def is_quiet(threshold=-1): return PIPENV_VERBOSITY <= threshold + + +PIPENV_SPINNER_FAIL_TEXT = fs_str(to_text(u"✘ {0}")) if not PIPENV_HIDE_EMOJIS else ("{0}") + +PIPENV_SPINNER_OK_TEXT = fs_str(to_text(u"✔ {0}")) if not PIPENV_HIDE_EMOJIS else ("{0}") diff --git a/pipenv/patched/notpip/__init__.py b/pipenv/patched/notpip/__init__.py index 9227d0ea..ae265fa7 100644 --- a/pipenv/patched/notpip/__init__.py +++ b/pipenv/patched/notpip/__init__.py @@ -1 +1 @@ -__version__ = "18.0" +__version__ = "18.1" diff --git a/pipenv/patched/notpip/_internal/__init__.py b/pipenv/patched/notpip/_internal/__init__.py index dcd0937e..6d223928 100644 --- a/pipenv/patched/notpip/_internal/__init__.py +++ b/pipenv/patched/notpip/_internal/__init__.py @@ -4,7 +4,6 @@ from __future__ import absolute_import import locale import logging import os -import optparse import warnings import sys @@ -38,17 +37,12 @@ else: else: securetransport.inject_into_urllib3() -from pipenv.patched.notpip import __version__ -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.exceptions import CommandError, PipError -from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions, get_prog +from pipenv.patched.notpip._internal.cli.autocompletion import autocomplete +from pipenv.patched.notpip._internal.cli.main_parser import parse_command +from pipenv.patched.notpip._internal.commands import commands_dict +from pipenv.patched.notpip._internal.exceptions import PipError from pipenv.patched.notpip._internal.utils import deprecation from pipenv.patched.notpip._internal.vcs import git, mercurial, subversion, bazaar # noqa -from pipenv.patched.notpip._internal.baseparser import ( - ConfigOptionParser, UpdatingDefaultsHelpFormatter, -) -from pipenv.patched.notpip._internal.commands import get_summaries, get_similar_commands -from pipenv.patched.notpip._internal.commands import commands_dict from pipenv.patched.notpip._vendor.urllib3.exceptions import InsecureRequestWarning logger = logging.getLogger(__name__) @@ -57,232 +51,6 @@ logger = logging.getLogger(__name__) warnings.filterwarnings("ignore", category=InsecureRequestWarning) -def autocomplete(): - """Command and option completion for the main option parser (and options) - and its subcommands (and options). - - Enable by sourcing one of the completion shell scripts (bash, zsh or fish). - """ - # Don't complete if user hasn't sourced bash_completion file. - if 'PIP_AUTO_COMPLETE' not in os.environ: - return - cwords = os.environ['COMP_WORDS'].split()[1:] - cword = int(os.environ['COMP_CWORD']) - try: - current = cwords[cword - 1] - except IndexError: - current = '' - - subcommands = [cmd for cmd, summary in get_summaries()] - options = [] - # subcommand - try: - subcommand_name = [w for w in cwords if w in subcommands][0] - except IndexError: - subcommand_name = None - - parser = create_main_parser() - # subcommand options - if subcommand_name: - # special case: 'help' subcommand has no options - if subcommand_name == 'help': - sys.exit(1) - # special case: list locally installed dists for show and uninstall - should_list_installed = ( - subcommand_name in ['show', 'uninstall'] and - not current.startswith('-') - ) - if should_list_installed: - installed = [] - lc = current.lower() - for dist in get_installed_distributions(local_only=True): - if dist.key.startswith(lc) and dist.key not in cwords[1:]: - installed.append(dist.key) - # if there are no dists installed, fall back to option completion - if installed: - for dist in installed: - print(dist) - sys.exit(1) - - subcommand = commands_dict[subcommand_name]() - - for opt in subcommand.parser.option_list_all: - if opt.help != optparse.SUPPRESS_HELP: - for opt_str in opt._long_opts + opt._short_opts: - options.append((opt_str, opt.nargs)) - - # filter out previously specified options from available options - prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] - options = [(x, v) for (x, v) in options if x not in prev_opts] - # filter options by current input - options = [(k, v) for k, v in options if k.startswith(current)] - # get completion type given cwords and available subcommand options - completion_type = get_path_completion_type( - cwords, cword, subcommand.parser.option_list_all, - ) - # get completion files and directories if ``completion_type`` is - # ````, ```` or ```` - if completion_type: - options = auto_complete_paths(current, completion_type) - options = ((opt, 0) for opt in options) - for option in options: - opt_label = option[0] - # append '=' to options which require args - if option[1] and option[0][:2] == "--": - opt_label += '=' - print(opt_label) - else: - # show main parser options only when necessary - - opts = [i.option_list for i in parser.option_groups] - opts.append(parser.option_list) - opts = (o for it in opts for o in it) - if current.startswith('-'): - for opt in opts: - if opt.help != optparse.SUPPRESS_HELP: - subcommands += opt._long_opts + opt._short_opts - else: - # get completion type given cwords and all available options - completion_type = get_path_completion_type(cwords, cword, opts) - if completion_type: - subcommands = auto_complete_paths(current, completion_type) - - print(' '.join([x for x in subcommands if x.startswith(current)])) - sys.exit(1) - - -def get_path_completion_type(cwords, cword, opts): - """Get the type of path completion (``file``, ``dir``, ``path`` or None) - - :param cwords: same as the environmental variable ``COMP_WORDS`` - :param cword: same as the environmental variable ``COMP_CWORD`` - :param opts: The available options to check - :return: path completion type (``file``, ``dir``, ``path`` or None) - """ - if cword < 2 or not cwords[cword - 2].startswith('-'): - return - for opt in opts: - if opt.help == optparse.SUPPRESS_HELP: - continue - for o in str(opt).split('/'): - if cwords[cword - 2].split('=')[0] == o: - if any(x in ('path', 'file', 'dir') - for x in opt.metavar.split('/')): - return opt.metavar - - -def auto_complete_paths(current, completion_type): - """If ``completion_type`` is ``file`` or ``path``, list all regular files - and directories starting with ``current``; otherwise only list directories - starting with ``current``. - - :param current: The word to be completed - :param completion_type: path completion type(`file`, `path` or `dir`)i - :return: A generator of regular files and/or directories - """ - directory, filename = os.path.split(current) - current_path = os.path.abspath(directory) - # Don't complete paths if they can't be accessed - if not os.access(current_path, os.R_OK): - return - filename = os.path.normcase(filename) - # list all files that start with ``filename`` - file_list = (x for x in os.listdir(current_path) - if os.path.normcase(x).startswith(filename)) - for f in file_list: - opt = os.path.join(current_path, f) - comp_file = os.path.normcase(os.path.join(directory, f)) - # complete regular files when there is not ```` after option - # complete directories when there is ````, ```` or - # ````after option - if completion_type != 'dir' and os.path.isfile(opt): - yield comp_file - elif os.path.isdir(opt): - yield os.path.join(comp_file, '') - - -def create_main_parser(): - parser_kw = { - 'usage': '\n%prog [options]', - 'add_help_option': False, - 'formatter': UpdatingDefaultsHelpFormatter(), - 'name': 'global', - 'prog': get_prog(), - } - - parser = ConfigOptionParser(**parser_kw) - parser.disable_interspersed_args() - - pip_pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - parser.version = 'pip %s from %s (python %s)' % ( - __version__, pip_pkg_dir, sys.version[:3], - ) - - # add the general options - gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) - parser.add_option_group(gen_opts) - - parser.main = True # so the help formatter knows - - # create command listing for description - command_summaries = get_summaries() - description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries] - parser.description = '\n'.join(description) - - return parser - - -def parseopts(args): - parser = create_main_parser() - - # Note: parser calls disable_interspersed_args(), so the result of this - # call is to split the initial args into the general options before the - # subcommand and everything else. - # For example: - # args: ['--timeout=5', 'install', '--user', 'INITools'] - # general_options: ['--timeout==5'] - # args_else: ['install', '--user', 'INITools'] - general_options, args_else = parser.parse_args(args) - - # --version - if general_options.version: - sys.stdout.write(parser.version) - sys.stdout.write(os.linesep) - sys.exit() - - # pip || pip help -> print_help() - if not args_else or (args_else[0] == 'help' and len(args_else) == 1): - parser.print_help() - sys.exit() - - # the subcommand name - cmd_name = args_else[0] - - if cmd_name not in commands_dict: - guess = get_similar_commands(cmd_name) - - msg = ['unknown command "%s"' % cmd_name] - if guess: - msg.append('maybe you meant "%s"' % guess) - - raise CommandError(' - '.join(msg)) - - # all the args without the subcommand - cmd_args = args[:] - cmd_args.remove(cmd_name) - - return cmd_name, cmd_args - - -def check_isolated(args): - isolated = False - - if "--isolated" in args: - isolated = True - - return isolated - - def main(args=None): if args is None: args = sys.argv[1:] @@ -293,7 +61,7 @@ def main(args=None): autocomplete() try: - cmd_name, cmd_args = parseopts(args) + cmd_name, cmd_args = parse_command(args) except PipError as exc: sys.stderr.write("ERROR: %s" % exc) sys.stderr.write(os.linesep) @@ -306,5 +74,5 @@ def main(args=None): except locale.Error as e: # setlocale can apparently crash if locale are uninitialized logger.debug("Ignoring error %s when setting locale", e) - command = commands_dict[cmd_name](isolated=check_isolated(cmd_args)) + command = commands_dict[cmd_name](isolated=("--isolated" in cmd_args)) return command.main(cmd_args) diff --git a/pipenv/patched/notpip/_internal/build_env.py b/pipenv/patched/notpip/_internal/build_env.py index 1d351b5c..6d696fbd 100644 --- a/pipenv/patched/notpip/_internal/build_env.py +++ b/pipenv/patched/notpip/_internal/build_env.py @@ -7,6 +7,8 @@ import sys from distutils.sysconfig import get_python_lib from sysconfig import get_paths +from pipenv.patched.notpip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet + from pipenv.patched.notpip._internal.utils.misc import call_subprocess from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.utils.ui import open_spinner @@ -75,6 +77,20 @@ class BuildEnvironment(object): def cleanup(self): self._temp_dir.cleanup() + def missing_requirements(self, reqs): + """Return a list of the requirements from reqs that are not present + """ + missing = [] + with self: + ws = WorkingSet(os.environ["PYTHONPATH"].split(os.pathsep)) + for req in reqs: + try: + if ws.find(Requirement.parse(req)) is None: + missing.append(req) + except VersionConflict: + missing.append(req) + return missing + def install_requirements(self, finder, requirements, message): args = [ sys.executable, '-m', 'pip', 'install', '--ignore-installed', diff --git a/pipenv/patched/notpip/_internal/cache.py b/pipenv/patched/notpip/_internal/cache.py index a3a28fd8..d91b8170 100644 --- a/pipenv/patched/notpip/_internal/cache.py +++ b/pipenv/patched/notpip/_internal/cache.py @@ -8,9 +8,9 @@ import os from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal import index -from pipenv.patched.notpip._internal.compat import expanduser from pipenv.patched.notpip._internal.download import path_to_url +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.wheel import InvalidWheelFilename, Wheel @@ -22,7 +22,7 @@ class Cache(object): :param cache_dir: The root of the cache. - :param format_control: A pip.index.FormatControl object to limit + :param format_control: An object of FormatControl class to limit binaries being read from the cache. :param allowed_formats: which formats of files the cache should store. ('binary' and 'source' are the only allowed values) @@ -72,8 +72,8 @@ class Cache(object): return [] canonical_name = canonicalize_name(package_name) - formats = index.fmt_ctl_formats( - self.format_control, canonical_name + formats = self.format_control.get_allowed_formats( + canonical_name ) if not self.allowed_formats.intersection(formats): return [] @@ -101,7 +101,7 @@ class Cache(object): root = self.get_path_for_link(link) path = os.path.join(root, candidate) - return index.Link(path_to_url(path)) + return Link(path_to_url(path)) def cleanup(self): pass diff --git a/pipenv/patched/notpip/_internal/cli/__init__.py b/pipenv/patched/notpip/_internal/cli/__init__.py new file mode 100644 index 00000000..e589bb91 --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/__init__.py @@ -0,0 +1,4 @@ +"""Subpackage containing all of pip's command line interface related code +""" + +# This file intentionally does not import submodules diff --git a/pipenv/patched/notpip/_internal/cli/autocompletion.py b/pipenv/patched/notpip/_internal/cli/autocompletion.py new file mode 100644 index 00000000..15b560a1 --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/autocompletion.py @@ -0,0 +1,152 @@ +"""Logic that powers autocompletion installed by ``pip completion``. +""" + +import optparse +import os +import sys + +from pipenv.patched.notpip._internal.cli.main_parser import create_main_parser +from pipenv.patched.notpip._internal.commands import commands_dict, get_summaries +from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions + + +def autocomplete(): + """Entry Point for completion of main and subcommand options. + """ + # Don't complete if user hasn't sourced bash_completion file. + if 'PIP_AUTO_COMPLETE' not in os.environ: + return + cwords = os.environ['COMP_WORDS'].split()[1:] + cword = int(os.environ['COMP_CWORD']) + try: + current = cwords[cword - 1] + except IndexError: + current = '' + + subcommands = [cmd for cmd, summary in get_summaries()] + options = [] + # subcommand + try: + subcommand_name = [w for w in cwords if w in subcommands][0] + except IndexError: + subcommand_name = None + + parser = create_main_parser() + # subcommand options + if subcommand_name: + # special case: 'help' subcommand has no options + if subcommand_name == 'help': + sys.exit(1) + # special case: list locally installed dists for show and uninstall + should_list_installed = ( + subcommand_name in ['show', 'uninstall'] and + not current.startswith('-') + ) + if should_list_installed: + installed = [] + lc = current.lower() + for dist in get_installed_distributions(local_only=True): + if dist.key.startswith(lc) and dist.key not in cwords[1:]: + installed.append(dist.key) + # if there are no dists installed, fall back to option completion + if installed: + for dist in installed: + print(dist) + sys.exit(1) + + subcommand = commands_dict[subcommand_name]() + + for opt in subcommand.parser.option_list_all: + if opt.help != optparse.SUPPRESS_HELP: + for opt_str in opt._long_opts + opt._short_opts: + options.append((opt_str, opt.nargs)) + + # filter out previously specified options from available options + prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] + options = [(x, v) for (x, v) in options if x not in prev_opts] + # filter options by current input + options = [(k, v) for k, v in options if k.startswith(current)] + # get completion type given cwords and available subcommand options + completion_type = get_path_completion_type( + cwords, cword, subcommand.parser.option_list_all, + ) + # get completion files and directories if ``completion_type`` is + # ````, ```` or ```` + if completion_type: + options = auto_complete_paths(current, completion_type) + options = ((opt, 0) for opt in options) + for option in options: + opt_label = option[0] + # append '=' to options which require args + if option[1] and option[0][:2] == "--": + opt_label += '=' + print(opt_label) + else: + # show main parser options only when necessary + + opts = [i.option_list for i in parser.option_groups] + opts.append(parser.option_list) + opts = (o for it in opts for o in it) + if current.startswith('-'): + for opt in opts: + if opt.help != optparse.SUPPRESS_HELP: + subcommands += opt._long_opts + opt._short_opts + else: + # get completion type given cwords and all available options + completion_type = get_path_completion_type(cwords, cword, opts) + if completion_type: + subcommands = auto_complete_paths(current, completion_type) + + print(' '.join([x for x in subcommands if x.startswith(current)])) + sys.exit(1) + + +def get_path_completion_type(cwords, cword, opts): + """Get the type of path completion (``file``, ``dir``, ``path`` or None) + + :param cwords: same as the environmental variable ``COMP_WORDS`` + :param cword: same as the environmental variable ``COMP_CWORD`` + :param opts: The available options to check + :return: path completion type (``file``, ``dir``, ``path`` or None) + """ + if cword < 2 or not cwords[cword - 2].startswith('-'): + return + for opt in opts: + if opt.help == optparse.SUPPRESS_HELP: + continue + for o in str(opt).split('/'): + if cwords[cword - 2].split('=')[0] == o: + if not opt.metavar or any( + x in ('path', 'file', 'dir') + for x in opt.metavar.split('/')): + return opt.metavar + + +def auto_complete_paths(current, completion_type): + """If ``completion_type`` is ``file`` or ``path``, list all regular files + and directories starting with ``current``; otherwise only list directories + starting with ``current``. + + :param current: The word to be completed + :param completion_type: path completion type(`file`, `path` or `dir`)i + :return: A generator of regular files and/or directories + """ + directory, filename = os.path.split(current) + current_path = os.path.abspath(directory) + # Don't complete paths if they can't be accessed + if not os.access(current_path, os.R_OK): + return + filename = os.path.normcase(filename) + # list all files that start with ``filename`` + file_list = (x for x in os.listdir(current_path) + if os.path.normcase(x).startswith(filename)) + for f in file_list: + opt = os.path.join(current_path, f) + comp_file = os.path.normcase(os.path.join(directory, f)) + # complete regular files when there is not ```` after option + # complete directories when there is ````, ```` or + # ````after option + if completion_type != 'dir' and os.path.isfile(opt): + yield comp_file + elif os.path.isdir(opt): + yield os.path.join(comp_file, '') diff --git a/pipenv/patched/notpip/_internal/basecommand.py b/pipenv/patched/notpip/_internal/cli/base_command.py similarity index 92% rename from pipenv/patched/notpip/_internal/basecommand.py rename to pipenv/patched/notpip/_internal/cli/base_command.py index 60199d55..229831f2 100644 --- a/pipenv/patched/notpip/_internal/basecommand.py +++ b/pipenv/patched/notpip/_internal/cli/base_command.py @@ -7,10 +7,14 @@ import optparse import os import sys -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.baseparser import ( +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.parser import ( ConfigOptionParser, UpdatingDefaultsHelpFormatter, ) +from pipenv.patched.notpip._internal.cli.status_codes import ( + ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, + VIRTUALENV_NOT_FOUND, +) from pipenv.patched.notpip._internal.download import PipSession from pipenv.patched.notpip._internal.exceptions import ( BadCommand, CommandError, InstallationError, PreviousBuildDirError, @@ -18,12 +22,10 @@ from pipenv.patched.notpip._internal.exceptions import ( ) from pipenv.patched.notpip._internal.index import PackageFinder from pipenv.patched.notpip._internal.locations import running_under_virtualenv -from pipenv.patched.notpip._internal.req.req_file import parse_requirements -from pipenv.patched.notpip._internal.req.req_install import InstallRequirement -from pipenv.patched.notpip._internal.status_codes import ( - ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, - VIRTUALENV_NOT_FOUND, +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, ) +from pipenv.patched.notpip._internal.req.req_file import parse_requirements from pipenv.patched.notpip._internal.utils.logging import setup_logging from pipenv.patched.notpip._internal.utils.misc import get_prog, normalize_path from pipenv.patched.notpip._internal.utils.outdated import pip_version_check @@ -168,12 +170,14 @@ class Command(object): return UNKNOWN_ERROR finally: - # Check if we're using the latest version of pip available - skip_version_check = ( - options.disable_pip_version_check or - getattr(options, "no_index", False) + allow_version_check = ( + # Does this command have the index_group options? + hasattr(options, "no_index") and + # Is this command allowed to perform this check? + not (options.disable_pip_version_check or options.no_index) ) - if not skip_version_check: + # Check if we're using the latest version of pip available + if allow_version_check: session = self._build_session( options, retries=0, @@ -208,7 +212,7 @@ class RequirementCommand(Command): requirement_set.add_requirement(req_to_add) for req in args: - req_to_add = InstallRequirement.from_line( + req_to_add = install_req_from_line( req, None, isolated=options.isolated_mode, wheel_cache=wheel_cache ) @@ -216,7 +220,7 @@ class RequirementCommand(Command): requirement_set.add_requirement(req_to_add) for req in options.editables: - req_to_add = InstallRequirement.from_editable( + req_to_add = install_req_from_editable( req, isolated=options.isolated_mode, wheel_cache=wheel_cache diff --git a/pipenv/patched/notpip/_internal/cmdoptions.py b/pipenv/patched/notpip/_internal/cli/cmdoptions.py similarity index 81% rename from pipenv/patched/notpip/_internal/cmdoptions.py rename to pipenv/patched/notpip/_internal/cli/cmdoptions.py index c25e769f..a075a67e 100644 --- a/pipenv/patched/notpip/_internal/cmdoptions.py +++ b/pipenv/patched/notpip/_internal/cli/cmdoptions.py @@ -13,10 +13,9 @@ import warnings from functools import partial from optparse import SUPPRESS_HELP, Option, OptionGroup -from pipenv.patched.notpip._internal.index import ( - FormatControl, fmt_ctl_handle_mutual_exclude, fmt_ctl_no_binary, -) +from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.locations import USER_CACHE_DIR, src_prefix +from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.models.index import PyPI from pipenv.patched.notpip._internal.utils.hashes import STRONG_HASHES from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -53,13 +52,52 @@ def check_install_build_global(options, check_options=None): names = ["build_options", "global_options", "install_options"] if any(map(getname, names)): control = options.format_control - fmt_ctl_no_binary(control) + control.disallow_binaries() warnings.warn( 'Disabling all use of wheels due to the use of --build-options ' '/ --global-options / --install-options.', stacklevel=2, ) +def check_dist_restriction(options, check_target=False): + """Function for determining if custom platform options are allowed. + + :param options: The OptionParser options. + :param check_target: Whether or not to check if --target is being used. + """ + dist_restriction_set = any([ + options.python_version, + options.platform, + options.abi, + options.implementation, + ]) + + binary_only = FormatControl(set(), {':all:'}) + sdist_dependencies_allowed = ( + options.format_control != binary_only and + not options.ignore_dependencies + ) + + # Installations or downloads using dist restrictions must not combine + # source distributions and dist-specific wheels, as they are not + # gauranteed to be locally compatible. + if dist_restriction_set and sdist_dependencies_allowed: + raise CommandError( + "When restricting platform and interpreter constraints using " + "--python-version, --platform, --abi, or --implementation, " + "either --no-deps must be set, or --only-binary=:all: must be " + "set and --no-binary must not be set (or must be set to " + ":none:)." + ) + + if check_target: + if dist_restriction_set and not options.target_dir: + raise CommandError( + "Can not use any platform or abi specific options unless " + "installing via '--target'" + ) + + ########### # options # ########### @@ -365,24 +403,25 @@ def _get_format_control(values, option): def _handle_no_binary(option, opt_str, value, parser): - existing = getattr(parser.values, option.dest) - fmt_ctl_handle_mutual_exclude( + existing = _get_format_control(parser.values, option) + FormatControl.handle_mutual_excludes( value, existing.no_binary, existing.only_binary, ) def _handle_only_binary(option, opt_str, value, parser): - existing = getattr(parser.values, option.dest) - fmt_ctl_handle_mutual_exclude( + existing = _get_format_control(parser.values, option) + FormatControl.handle_mutual_excludes( value, existing.only_binary, existing.no_binary, ) def no_binary(): + format_control = FormatControl(set(), set()) return Option( "--no-binary", dest="format_control", action="callback", callback=_handle_no_binary, type="str", - default=FormatControl(set(), set()), + default=format_control, help="Do not use binary packages. Can be supplied multiple times, and " "each time adds to the existing value. Accepts either :all: to " "disable all binary packages, :none: to empty the set, or one or " @@ -393,10 +432,11 @@ def no_binary(): def only_binary(): + format_control = FormatControl(set(), set()) return Option( "--only-binary", dest="format_control", action="callback", callback=_handle_only_binary, type="str", - default=FormatControl(set(), set()), + default=format_control, help="Do not use source packages. Can be supplied multiple times, and " "each time adds to the existing value. Accepts either :all: to " "disable all source packages, :none: to empty the set, or one or " @@ -406,6 +446,61 @@ def only_binary(): ) +platform = partial( + Option, + '--platform', + dest='platform', + metavar='platform', + default=None, + help=("Only use wheels compatible with . " + "Defaults to the platform of the running system."), +) + + +python_version = partial( + Option, + '--python-version', + dest='python_version', + metavar='python_version', + default=None, + help=("Only use wheels compatible with Python " + "interpreter version . If not specified, then the " + "current system interpreter minor version is used. A major " + "version (e.g. '2') can be specified to match all " + "minor revs of that major version. A minor version " + "(e.g. '34') can also be specified."), +) + + +implementation = partial( + Option, + '--implementation', + dest='implementation', + metavar='implementation', + default=None, + help=("Only use wheels compatible with Python " + "implementation , e.g. 'pp', 'jy', 'cp', " + " or 'ip'. If not specified, then the current " + "interpreter implementation is used. Use 'py' to force " + "implementation-agnostic wheels."), +) + + +abi = partial( + Option, + '--abi', + dest='abi', + metavar='abi', + default=None, + help=("Only use wheels compatible with Python " + "abi , e.g. 'pypy_41'. If not specified, then the " + "current interpreter abi tag is used. Generally " + "you will need to specify --implementation, " + "--platform, and --python-version when using " + "this option."), +) + + def prefer_binary(): return Option( "--prefer-binary", @@ -501,7 +596,7 @@ no_clean = partial( '--no-clean', action='store_true', default=False, - help="Don't clean up build directories)." + help="Don't clean up build directories." ) # type: Any pre = partial( diff --git a/pipenv/patched/notpip/_internal/cli/main_parser.py b/pipenv/patched/notpip/_internal/cli/main_parser.py new file mode 100644 index 00000000..abe2f69e --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/main_parser.py @@ -0,0 +1,96 @@ +"""A single place for constructing and exposing the main parser +""" + +import os +import sys + +from pipenv.patched.notpip import __version__ +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.parser import ( + ConfigOptionParser, UpdatingDefaultsHelpFormatter, +) +from pipenv.patched.notpip._internal.commands import ( + commands_dict, get_similar_commands, get_summaries, +) +from pipenv.patched.notpip._internal.exceptions import CommandError +from pipenv.patched.notpip._internal.utils.misc import get_prog + +__all__ = ["create_main_parser", "parse_command"] + + +def create_main_parser(): + """Creates and returns the main parser for pip's CLI + """ + + parser_kw = { + 'usage': '\n%prog [options]', + 'add_help_option': False, + 'formatter': UpdatingDefaultsHelpFormatter(), + 'name': 'global', + 'prog': get_prog(), + } + + parser = ConfigOptionParser(**parser_kw) + parser.disable_interspersed_args() + + pip_pkg_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), "..", "..", + )) + parser.version = 'pip %s from %s (python %s)' % ( + __version__, pip_pkg_dir, sys.version[:3], + ) + + # add the general options + gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) + parser.add_option_group(gen_opts) + + parser.main = True # so the help formatter knows + + # create command listing for description + command_summaries = get_summaries() + description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries] + parser.description = '\n'.join(description) + + return parser + + +def parse_command(args): + parser = create_main_parser() + + # Note: parser calls disable_interspersed_args(), so the result of this + # call is to split the initial args into the general options before the + # subcommand and everything else. + # For example: + # args: ['--timeout=5', 'install', '--user', 'INITools'] + # general_options: ['--timeout==5'] + # args_else: ['install', '--user', 'INITools'] + general_options, args_else = parser.parse_args(args) + + # --version + if general_options.version: + sys.stdout.write(parser.version) + sys.stdout.write(os.linesep) + sys.exit() + + # pip || pip help -> print_help() + if not args_else or (args_else[0] == 'help' and len(args_else) == 1): + parser.print_help() + sys.exit() + + # the subcommand name + cmd_name = args_else[0] + + if cmd_name not in commands_dict: + guess = get_similar_commands(cmd_name) + + msg = ['unknown command "%s"' % cmd_name] + if guess: + msg.append('maybe you meant "%s"' % guess) + + raise CommandError(' - '.join(msg)) + + # all the args without the subcommand + cmd_args = args[:] + cmd_args.remove(cmd_name) + + return cmd_name, cmd_args diff --git a/pipenv/patched/notpip/_internal/baseparser.py b/pipenv/patched/notpip/_internal/cli/parser.py similarity index 88% rename from pipenv/patched/notpip/_internal/baseparser.py rename to pipenv/patched/notpip/_internal/cli/parser.py index f82093bf..2d2c8f4d 100644 --- a/pipenv/patched/notpip/_internal/baseparser.py +++ b/pipenv/patched/notpip/_internal/cli/parser.py @@ -9,8 +9,9 @@ from distutils.util import strtobool from pipenv.patched.notpip._vendor.six import string_types -from pipenv.patched.notpip._internal.compat import get_terminal_size +from pipenv.patched.notpip._internal.cli.status_codes import UNKNOWN_ERROR from pipenv.patched.notpip._internal.configuration import Configuration, ConfigurationError +from pipenv.patched.notpip._internal.utils.compat import get_terminal_size logger = logging.getLogger(__name__) @@ -192,7 +193,14 @@ class ConfigOptionParser(CustomOptionParser): continue if option.action in ('store_true', 'store_false', 'count'): - val = strtobool(val) + try: + val = strtobool(val) + except ValueError: + error_msg = invalid_config_error_message( + option.action, key, val + ) + self.error(error_msg) + elif option.action == 'append': val = val.split() val = [self.check_default(option, key, v) for v in val] @@ -225,7 +233,7 @@ class ConfigOptionParser(CustomOptionParser): try: self.config.load() except ConfigurationError as err: - self.exit(2, err.args[0]) + self.exit(UNKNOWN_ERROR, str(err)) defaults = self._update_defaults(self.defaults.copy()) # ours for option in self._get_all_options(): @@ -237,4 +245,17 @@ class ConfigOptionParser(CustomOptionParser): def error(self, msg): self.print_usage(sys.stderr) - self.exit(2, "%s\n" % msg) + self.exit(UNKNOWN_ERROR, "%s\n" % msg) + + +def invalid_config_error_message(action, key, val): + """Returns a better error message when invalid configuration option + is provided.""" + if action in ('store_true', 'store_false'): + return ("{0} is not a valid value for {1} option, " + "please specify a boolean value like yes/no, " + "true/false or 1/0 instead.").format(val, key) + + return ("{0} is not a valid value for {1} option, " + "please specify a numerical value like 1/0 " + "instead.").format(val, key) diff --git a/pipenv/patched/notpip/_internal/status_codes.py b/pipenv/patched/notpip/_internal/cli/status_codes.py similarity index 100% rename from pipenv/patched/notpip/_internal/status_codes.py rename to pipenv/patched/notpip/_internal/cli/status_codes.py diff --git a/pipenv/patched/notpip/_internal/commands/__init__.py b/pipenv/patched/notpip/_internal/commands/__init__.py index 140c4609..a403c6f9 100644 --- a/pipenv/patched/notpip/_internal/commands/__init__.py +++ b/pipenv/patched/notpip/_internal/commands/__init__.py @@ -21,7 +21,7 @@ from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import List, Type # noqa: F401 - from pipenv.patched.notpip._internal.basecommand import Command # noqa: F401 + from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 commands_order = [ InstallCommand, diff --git a/pipenv/patched/notpip/_internal/commands/check.py b/pipenv/patched/notpip/_internal/commands/check.py index cd5ffb5f..adf4f5e7 100644 --- a/pipenv/patched/notpip/_internal/commands/check.py +++ b/pipenv/patched/notpip/_internal/commands/check.py @@ -1,6 +1,6 @@ import logging -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.operations.check import ( check_package_set, create_package_set_from_installed, ) diff --git a/pipenv/patched/notpip/_internal/commands/completion.py b/pipenv/patched/notpip/_internal/commands/completion.py index f4c31c1b..cb8a11a7 100644 --- a/pipenv/patched/notpip/_internal/commands/completion.py +++ b/pipenv/patched/notpip/_internal/commands/completion.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import sys import textwrap -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.utils.misc import get_prog BASE_COMPLETION = """ diff --git a/pipenv/patched/notpip/_internal/commands/configuration.py b/pipenv/patched/notpip/_internal/commands/configuration.py index 090109c5..6c1dbdfd 100644 --- a/pipenv/patched/notpip/_internal/commands/configuration.py +++ b/pipenv/patched/notpip/_internal/commands/configuration.py @@ -2,11 +2,11 @@ import logging import os import subprocess -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import ERROR, SUCCESS from pipenv.patched.notpip._internal.configuration import Configuration, kinds from pipenv.patched.notpip._internal.exceptions import PipError from pipenv.patched.notpip._internal.locations import venv_config_file -from pipenv.patched.notpip._internal.status_codes import ERROR, SUCCESS from pipenv.patched.notpip._internal.utils.misc import get_prog logger = logging.getLogger(__name__) diff --git a/pipenv/patched/notpip/_internal/commands/download.py b/pipenv/patched/notpip/_internal/commands/download.py index 63d91b04..e5d87121 100644 --- a/pipenv/patched/notpip/_internal/commands/download.py +++ b/pipenv/patched/notpip/_internal/commands/download.py @@ -3,10 +3,8 @@ from __future__ import absolute_import import logging import os -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.basecommand import RequirementCommand -from pipenv.patched.notpip._internal.exceptions import CommandError -from pipenv.patched.notpip._internal.index import FormatControl +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer from pipenv.patched.notpip._internal.req import RequirementSet from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker @@ -69,52 +67,10 @@ class DownloadCommand(RequirementCommand): help=("Download packages into ."), ) - cmd_opts.add_option( - '--platform', - dest='platform', - metavar='platform', - default=None, - help=("Only download wheels compatible with . " - "Defaults to the platform of the running system."), - ) - - cmd_opts.add_option( - '--python-version', - dest='python_version', - metavar='python_version', - default=None, - help=("Only download wheels compatible with Python " - "interpreter version . If not specified, then the " - "current system interpreter minor version is used. A major " - "version (e.g. '2') can be specified to match all " - "minor revs of that major version. A minor version " - "(e.g. '34') can also be specified."), - ) - - cmd_opts.add_option( - '--implementation', - dest='implementation', - metavar='implementation', - default=None, - help=("Only download wheels compatible with Python " - "implementation , e.g. 'pp', 'jy', 'cp', " - " or 'ip'. If not specified, then the current " - "interpreter implementation is used. Use 'py' to force " - "implementation-agnostic wheels."), - ) - - cmd_opts.add_option( - '--abi', - dest='abi', - metavar='abi', - default=None, - help=("Only download wheels compatible with Python " - "abi , e.g. 'pypy_41'. If not specified, then the " - "current interpreter abi tag is used. Generally " - "you will need to specify --implementation, " - "--platform, and --python-version when using " - "this option."), - ) + cmd_opts.add_option(cmdoptions.platform()) + cmd_opts.add_option(cmdoptions.python_version()) + cmd_opts.add_option(cmdoptions.implementation()) + cmd_opts.add_option(cmdoptions.abi()) index_opts = cmdoptions.make_option_group( cmdoptions.index_group, @@ -135,25 +91,7 @@ class DownloadCommand(RequirementCommand): else: python_versions = None - dist_restriction_set = any([ - options.python_version, - options.platform, - options.abi, - options.implementation, - ]) - binary_only = FormatControl(set(), {':all:'}) - no_sdist_dependencies = ( - options.format_control != binary_only and - not options.ignore_dependencies - ) - if dist_restriction_set and no_sdist_dependencies: - raise CommandError( - "When restricting platform and interpreter constraints using " - "--python-version, --platform, --abi, or --implementation, " - "either --no-deps must be set, or --only-binary=:all: must be " - "set and --no-binary must not be set (or must be set to " - ":none:)." - ) + cmdoptions.check_dist_restriction(options) options.src_dir = os.path.abspath(options.src_dir) options.download_dir = normalize_path(options.download_dir) diff --git a/pipenv/patched/notpip/_internal/commands/freeze.py b/pipenv/patched/notpip/_internal/commands/freeze.py index 6959b6ab..343227ba 100644 --- a/pipenv/patched/notpip/_internal/commands/freeze.py +++ b/pipenv/patched/notpip/_internal/commands/freeze.py @@ -2,11 +2,11 @@ from __future__ import absolute_import import sys -from pipenv.patched.notpip._internal import index -from pipenv.patched.notpip._internal.basecommand import Command from pipenv.patched.notpip._internal.cache import WheelCache -from pipenv.patched.notpip._internal.compat import stdlib_pkgs +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.operations.freeze import freeze +from pipenv.patched.notpip._internal.utils.compat import stdlib_pkgs DEV_PKGS = {'pip', 'setuptools', 'distribute', 'wheel'} @@ -71,7 +71,7 @@ class FreezeCommand(Command): self.parser.insert_option_group(0, self.cmd_opts) def run(self, options, args): - format_control = index.FormatControl(set(), set()) + format_control = FormatControl(set(), set()) wheel_cache = WheelCache(options.cache_dir, format_control) skip = set(stdlib_pkgs) if not options.freeze_all: diff --git a/pipenv/patched/notpip/_internal/commands/hash.py b/pipenv/patched/notpip/_internal/commands/hash.py index a86574da..183f11ae 100644 --- a/pipenv/patched/notpip/_internal/commands/hash.py +++ b/pipenv/patched/notpip/_internal/commands/hash.py @@ -4,8 +4,8 @@ import hashlib import logging import sys -from pipenv.patched.notpip._internal.basecommand import Command -from pipenv.patched.notpip._internal.status_codes import ERROR +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import ERROR from pipenv.patched.notpip._internal.utils.hashes import FAVORITE_HASH, STRONG_HASHES from pipenv.patched.notpip._internal.utils.misc import read_chunks diff --git a/pipenv/patched/notpip/_internal/commands/help.py b/pipenv/patched/notpip/_internal/commands/help.py index aaf0f87e..f2c61965 100644 --- a/pipenv/patched/notpip/_internal/commands/help.py +++ b/pipenv/patched/notpip/_internal/commands/help.py @@ -1,6 +1,7 @@ from __future__ import absolute_import -from pipenv.patched.notpip._internal.basecommand import SUCCESS, Command +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import SUCCESS from pipenv.patched.notpip._internal.exceptions import CommandError diff --git a/pipenv/patched/notpip/_internal/commands/install.py b/pipenv/patched/notpip/_internal/commands/install.py index ebdf07d7..ddcb4759 100644 --- a/pipenv/patched/notpip/_internal/commands/install.py +++ b/pipenv/patched/notpip/_internal/commands/install.py @@ -9,9 +9,10 @@ from optparse import SUPPRESS_HELP from pipenv.patched.notpip._vendor import pkg_resources -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.basecommand import RequirementCommand from pipenv.patched.notpip._internal.cache import WheelCache +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand +from pipenv.patched.notpip._internal.cli.status_codes import ERROR from pipenv.patched.notpip._internal.exceptions import ( CommandError, InstallationError, PreviousBuildDirError, ) @@ -21,7 +22,6 @@ from pipenv.patched.notpip._internal.operations.prepare import RequirementPrepar from pipenv.patched.notpip._internal.req import RequirementSet, install_given_reqs from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker from pipenv.patched.notpip._internal.resolve import Resolver -from pipenv.patched.notpip._internal.status_codes import ERROR from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner from pipenv.patched.notpip._internal.utils.misc import ( ensure_dir, get_installed_version, @@ -83,6 +83,11 @@ class InstallCommand(RequirementCommand): '. Use --upgrade to replace existing packages in ' 'with new versions.' ) + cmd_opts.add_option(cmdoptions.platform()) + cmd_opts.add_option(cmdoptions.python_version()) + cmd_opts.add_option(cmdoptions.implementation()) + cmd_opts.add_option(cmdoptions.abi()) + cmd_opts.add_option( '--user', dest='use_user_site', @@ -204,7 +209,6 @@ class InstallCommand(RequirementCommand): def run(self, options, args): cmdoptions.check_install_build_global(options) - upgrade_strategy = "to-satisfy-only" if options.upgrade: upgrade_strategy = options.upgrade_strategy @@ -212,6 +216,13 @@ class InstallCommand(RequirementCommand): if options.build_dir: options.build_dir = os.path.abspath(options.build_dir) + cmdoptions.check_dist_restriction(options, check_target=True) + + if options.python_version: + python_versions = [options.python_version] + else: + python_versions = None + options.src_dir = os.path.abspath(options.src_dir) install_options = options.install_options or [] if options.use_user_site: @@ -246,7 +257,14 @@ class InstallCommand(RequirementCommand): global_options = options.global_options or [] with self._build_session(options) as session: - finder = self._build_package_finder(options, session) + finder = self._build_package_finder( + options=options, + session=session, + platform=options.platform, + python_versions=python_versions, + abi=options.abi, + implementation=options.implementation, + ) build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) @@ -266,6 +284,7 @@ class InstallCommand(RequirementCommand): ) as directory: requirement_set = RequirementSet( require_hashes=options.require_hashes, + check_supported_wheels=not options.target_dir, ) try: diff --git a/pipenv/patched/notpip/_internal/commands/list.py b/pipenv/patched/notpip/_internal/commands/list.py index 99aee99f..577c0b5f 100644 --- a/pipenv/patched/notpip/_internal/commands/list.py +++ b/pipenv/patched/notpip/_internal/commands/list.py @@ -6,8 +6,8 @@ import logging from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import zip_longest -from pipenv.patched.notpip._internal.basecommand import Command -from pipenv.patched.notpip._internal.cmdoptions import index_group, make_option_group +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.index import PackageFinder from pipenv.patched.notpip._internal.utils.misc import ( @@ -102,7 +102,9 @@ class ListCommand(Command): help='Include editable package from output.', default=True, ) - index_opts = make_option_group(index_group, self.parser) + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, self.parser + ) self.parser.insert_option_group(0, index_opts) self.parser.insert_option_group(0, cmd_opts) diff --git a/pipenv/patched/notpip/_internal/commands/search.py b/pipenv/patched/notpip/_internal/commands/search.py index ac111c14..986208f9 100644 --- a/pipenv/patched/notpip/_internal/commands/search.py +++ b/pipenv/patched/notpip/_internal/commands/search.py @@ -11,12 +11,12 @@ from pipenv.patched.notpip._vendor.packaging.version import parse as parse_versi # why we ignore the type on this import from pipenv.patched.notpip._vendor.six.moves import xmlrpc_client # type: ignore -from pipenv.patched.notpip._internal.basecommand import SUCCESS, Command -from pipenv.patched.notpip._internal.compat import get_terminal_size +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import NO_MATCHES_FOUND, SUCCESS from pipenv.patched.notpip._internal.download import PipXmlrpcTransport from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.models.index import PyPI -from pipenv.patched.notpip._internal.status_codes import NO_MATCHES_FOUND +from pipenv.patched.notpip._internal.utils.compat import get_terminal_size from pipenv.patched.notpip._internal.utils.logging import indent_log logger = logging.getLogger(__name__) diff --git a/pipenv/patched/notpip/_internal/commands/show.py b/pipenv/patched/notpip/_internal/commands/show.py index 8de6b6b8..3fd24482 100644 --- a/pipenv/patched/notpip/_internal/commands/show.py +++ b/pipenv/patched/notpip/_internal/commands/show.py @@ -7,8 +7,8 @@ from email.parser import FeedParser # type: ignore from pipenv.patched.notpip._vendor import pkg_resources from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal.basecommand import Command -from pipenv.patched.notpip._internal.status_codes import ERROR, SUCCESS +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import ERROR, SUCCESS logger = logging.getLogger(__name__) diff --git a/pipenv/patched/notpip/_internal/commands/uninstall.py b/pipenv/patched/notpip/_internal/commands/uninstall.py index 45a0eba5..cf6a511c 100644 --- a/pipenv/patched/notpip/_internal/commands/uninstall.py +++ b/pipenv/patched/notpip/_internal/commands/uninstall.py @@ -2,9 +2,10 @@ from __future__ import absolute_import from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.exceptions import InstallationError -from pipenv.patched.notpip._internal.req import InstallRequirement, parse_requirements +from pipenv.patched.notpip._internal.req import parse_requirements +from pipenv.patched.notpip._internal.req.constructors import install_req_from_line from pipenv.patched.notpip._internal.utils.misc import protect_pip_from_modification_on_windows @@ -47,7 +48,7 @@ class UninstallCommand(Command): with self._build_session(options) as session: reqs_to_uninstall = {} for name in args: - req = InstallRequirement.from_line( + req = install_req_from_line( name, isolated=options.isolated_mode, ) if req.name: diff --git a/pipenv/patched/notpip/_internal/commands/wheel.py b/pipenv/patched/notpip/_internal/commands/wheel.py index c04d58ed..08d695ab 100644 --- a/pipenv/patched/notpip/_internal/commands/wheel.py +++ b/pipenv/patched/notpip/_internal/commands/wheel.py @@ -4,9 +4,9 @@ from __future__ import absolute_import import logging import os -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.basecommand import RequirementCommand from pipenv.patched.notpip._internal.cache import WheelCache +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand from pipenv.patched.notpip._internal.exceptions import CommandError, PreviousBuildDirError from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer from pipenv.patched.notpip._internal.req import RequirementSet diff --git a/pipenv/patched/notpip/_internal/configuration.py b/pipenv/patched/notpip/_internal/configuration.py index 3df185f7..3c02c955 100644 --- a/pipenv/patched/notpip/_internal/configuration.py +++ b/pipenv/patched/notpip/_internal/configuration.py @@ -18,7 +18,9 @@ import os from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import configparser -from pipenv.patched.notpip._internal.exceptions import ConfigurationError +from pipenv.patched.notpip._internal.exceptions import ( + ConfigurationError, ConfigurationFileCouldNotBeLoaded, +) from pipenv.patched.notpip._internal.locations import ( legacy_config_file, new_config_file, running_under_virtualenv, site_config_files, venv_config_file, @@ -289,11 +291,16 @@ class Configuration(object): try: parser.read(fname) except UnicodeDecodeError: - raise ConfigurationError(( - "ERROR: " - "Configuration file contains invalid %s characters.\n" - "Please fix your configuration, located at %s\n" - ) % (locale.getpreferredencoding(False), fname)) + # See https://github.com/pypa/pip/issues/4963 + raise ConfigurationFileCouldNotBeLoaded( + reason="contains invalid {} characters".format( + locale.getpreferredencoding(False) + ), + fname=fname, + ) + except configparser.Error as error: + # See https://github.com/pypa/pip/issues/4893 + raise ConfigurationFileCouldNotBeLoaded(error=error) return parser def _load_environment_vars(self): diff --git a/pipenv/patched/notpip/_internal/download.py b/pipenv/patched/notpip/_internal/download.py index 06a45644..8f4c38f5 100644 --- a/pipenv/patched/notpip/_internal/download.py +++ b/pipenv/patched/notpip/_internal/download.py @@ -323,7 +323,7 @@ class InsecureHTTPAdapter(HTTPAdapter): conn.ca_certs = None -class PipSession(Session): +class PipSession(requests.Session): timeout = None @@ -753,7 +753,7 @@ def _copy_dist_from_dir(link_path, location): # build an sdist setup_py = 'setup.py' - sdist_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable)] + sdist_args = [sys.executable] sdist_args.append('-c') sdist_args.append(SETUPTOOLS_SHIM % setup_py) sdist_args.append('sdist') diff --git a/pipenv/patched/notpip/_internal/exceptions.py b/pipenv/patched/notpip/_internal/exceptions.py index 43595977..2eadcf28 100644 --- a/pipenv/patched/notpip/_internal/exceptions.py +++ b/pipenv/patched/notpip/_internal/exceptions.py @@ -247,3 +247,22 @@ class HashMismatch(HashError): class UnsupportedPythonVersion(InstallationError): """Unsupported python version according to Requires-Python package metadata.""" + + +class ConfigurationFileCouldNotBeLoaded(ConfigurationError): + """When there are errors while loading a configuration file + """ + + def __init__(self, reason="could not be loaded", fname=None, error=None): + super(ConfigurationFileCouldNotBeLoaded, self).__init__(error) + self.reason = reason + self.fname = fname + self.error = error + + def __str__(self): + if self.fname is not None: + message_part = " in {}.".format(self.fname) + else: + assert self.error is not None + message_part = ".\n{}\n".format(self.error.message) + return "Configuration file {}{}".format(self.reason, message_part) diff --git a/pipenv/patched/notpip/_internal/index.py b/pipenv/patched/notpip/_internal/index.py index 426880e9..b4b02373 100644 --- a/pipenv/patched/notpip/_internal/index.py +++ b/pipenv/patched/notpip/_internal/index.py @@ -20,24 +20,27 @@ from pipenv.patched.notpip._vendor.requests.exceptions import SSLError from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request -from pipenv.patched.notpip._internal.compat import ipaddress from pipenv.patched.notpip._internal.download import HAS_TLS, is_url, path_to_url, url_to_path from pipenv.patched.notpip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, InvalidWheelFilename, UnsupportedWheel, ) +from pipenv.patched.notpip._internal.models.candidate import InstallationCandidate +from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.models.index import PyPI +from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.pep425tags import get_supported +from pipenv.patched.notpip._internal.utils.compat import ipaddress from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( - ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, cached_property, normalize_path, - remove_auth_from_url, splitext, + ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, normalize_path, + remove_auth_from_url, ) from pipenv.patched.notpip._internal.utils.packaging import check_requires_python from pipenv.patched.notpip._internal.wheel import Wheel, wheel_ext -__all__ = ['FormatControl', 'fmt_ctl_handle_mutual_exclude', 'PackageFinder'] +__all__ = ['FormatControl', 'PackageFinder'] SECURE_ORIGINS = [ @@ -56,46 +59,120 @@ SECURE_ORIGINS = [ logger = logging.getLogger(__name__) -class InstallationCandidate(object): +def _get_content_type(url, session): + """Get the Content-Type of the given url, using a HEAD request""" + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) + if scheme not in {'http', 'https'}: + # FIXME: some warning or something? + # assertion error? + return '' - def __init__(self, project, version, location, requires_python=None): - self.project = project - self.version = parse_version(version) - self.location = location - self._key = (self.project, self.version, self.location) - self.requires_python = requires_python + resp = session.head(url, allow_redirects=True) + resp.raise_for_status() - def __repr__(self): - return "".format( - self.project, self.version, self.location, + return resp.headers.get("Content-Type", "") + + +def _handle_get_page_fail(link, reason, url, meth=None): + if meth is None: + meth = logger.debug + meth("Could not fetch URL %s: %s - skipping", link, reason) + + +def _get_html_page(link, session=None): + if session is None: + raise TypeError( + "_get_html_page() missing 1 required keyword argument: 'session'" ) - def __hash__(self): - return hash(self._key) + url = link.url + url = url.split('#', 1)[0] - def __lt__(self, other): - return self._compare(other, lambda s, o: s < o) + # Check for VCS schemes that do not support lookup as web pages. + from pipenv.patched.notpip._internal.vcs import VcsSupport + for scheme in VcsSupport.schemes: + if url.lower().startswith(scheme) and url[len(scheme)] in '+:': + logger.debug('Cannot look at %s URL %s', scheme, link) + return None - def __le__(self, other): - return self._compare(other, lambda s, o: s <= o) + try: + filename = link.filename + for bad_ext in ARCHIVE_EXTENSIONS: + if filename.endswith(bad_ext): + content_type = _get_content_type(url, session=session) + if content_type.lower().startswith('text/html'): + break + else: + logger.debug( + 'Skipping page %s because of Content-Type: %s', + link, + content_type, + ) + return - def __eq__(self, other): - return self._compare(other, lambda s, o: s == o) + logger.debug('Getting page %s', url) - def __ge__(self, other): - return self._compare(other, lambda s, o: s >= o) + # Tack index.html onto file:// URLs that point to directories + (scheme, netloc, path, params, query, fragment) = \ + urllib_parse.urlparse(url) + if (scheme == 'file' and + os.path.isdir(urllib_request.url2pathname(path))): + # add trailing slash if not present so urljoin doesn't trim + # final segment + if not url.endswith('/'): + url += '/' + url = urllib_parse.urljoin(url, 'index.html') + logger.debug(' file: URL is directory, getting %s', url) - def __gt__(self, other): - return self._compare(other, lambda s, o: s > o) + resp = session.get( + url, + headers={ + "Accept": "text/html", + # We don't want to blindly returned cached data for + # /simple/, because authors generally expecting that + # twine upload && pip install will function, but if + # they've done a pip install in the last ~10 minutes + # it won't. Thus by setting this to zero we will not + # blindly use any cached data, however the benefit of + # using max-age=0 instead of no-cache, is that we will + # still support conditional requests, so we will still + # minimize traffic sent in cases where the page hasn't + # changed at all, we will just always incur the round + # trip for the conditional GET now instead of only + # once per 10 minutes. + # For more information, please see pypa/pip#5670. + "Cache-Control": "max-age=0", + }, + ) + resp.raise_for_status() - def __ne__(self, other): - return self._compare(other, lambda s, o: s != o) + # The check for archives above only works if the url ends with + # something that looks like an archive. However that is not a + # requirement of an url. Unless we issue a HEAD request on every + # url we cannot know ahead of time for sure if something is HTML + # or not. However we can check after we've downloaded it. + content_type = resp.headers.get('Content-Type', 'unknown') + if not content_type.lower().startswith("text/html"): + logger.debug( + 'Skipping page %s because of Content-Type: %s', + link, + content_type, + ) + return - def _compare(self, other, method): - if not isinstance(other, InstallationCandidate): - return NotImplemented - - return method(self._key, other._key) + inst = HTMLPage(resp.content, resp.url, resp.headers) + except requests.HTTPError as exc: + _handle_get_page_fail(link, exc, url) + except SSLError as exc: + reason = "There was a problem confirming the ssl certificate: " + reason += str(exc) + _handle_get_page_fail(link, reason, url, meth=logger.info) + except requests.ConnectionError as exc: + _handle_get_page_fail(link, "connection error: %s" % exc, url) + except requests.Timeout: + _handle_get_page_fail(link, "timed out", url) + else: + return inst class PackageFinder(object): @@ -210,15 +287,15 @@ class PackageFinder(object): return "\n".join(lines) def add_dependency_links(self, links): - # # FIXME: this shouldn't be global list this, it should only - # # apply to requirements of the package that specifies the - # # dependency_links value - # # FIXME: also, we should track comes_from (i.e., use Link) + # FIXME: this shouldn't be global list this, it should only + # apply to requirements of the package that specifies the + # dependency_links value + # FIXME: also, we should track comes_from (i.e., use Link) if self.process_dependency_links: deprecated( "Dependency Links processing has been deprecated and will be " "removed in a future release.", - replacement=None, + replacement="PEP 508 URL dependencies", gone_in="18.2", issue=4187, ) @@ -242,6 +319,7 @@ class PackageFinder(object): return extras + @staticmethod def _sort_locations(locations, expand_dir=False): """ @@ -467,7 +545,7 @@ class PackageFinder(object): logger.debug('* %s', location) canonical_name = canonicalize_name(project_name) - formats = fmt_ctl_formats(self.format_control, canonical_name) + formats = self.format_control.get_allowed_formats(canonical_name) search = Search(project_name, canonical_name, formats) find_links_versions = self._package_versions( # We trust every directly linked archive in find_links @@ -483,7 +561,7 @@ class PackageFinder(object): continue with indent_log(): page_versions.extend( - self._package_versions(page.links, search) + self._package_versions(page.iter_links(), search) ) dependency_versions = self._package_versions( @@ -626,7 +704,9 @@ class PackageFinder(object): try: page = self._get_page(location) - except requests.HTTPError as e: + except requests.HTTPError: + continue + if page is None: continue yield page @@ -743,7 +823,7 @@ class PackageFinder(object): return InstallationCandidate(search.supplied, version, link, link.requires_python) def _get_page(self, link): - return HTMLPage.get_page(link, session=self.session) + return _get_html_page(link, session=self.session) def egg_info_matches( @@ -763,7 +843,7 @@ def egg_info_matches( return None if search_name is None: full_match = match.group(0) - return full_match[full_match.index('-'):] + return full_match.split('-', 1)[-1] name = match.group(0).lower() # To match the "safe" name that pkg_resources creates: name = name.replace('_', '-') @@ -775,377 +855,71 @@ def egg_info_matches( return None +def _determine_base_url(document, page_url): + """Determine the HTML document's base URL. + + This looks for a ```` tag in the HTML document. If present, its href + attribute denotes the base URL of anchor tags in the document. If there is + no such tag (or if it does not have a valid href attribute), the HTML + file's URL is used as the base URL. + + :param document: An HTML document representation. The current + implementation expects the result of ``html5lib.parse()``. + :param page_url: The URL of the HTML document. + """ + for base in document.findall(".//base"): + href = base.get("href") + if href is not None: + return href + return page_url + + +def _get_encoding_from_headers(headers): + """Determine if we have any encoding information in our headers. + """ + if headers and "Content-Type" in headers: + content_type, params = cgi.parse_header(headers["Content-Type"]) + if "charset" in params: + return params['charset'] + return None + + +_CLEAN_LINK_RE = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) + + +def _clean_link(url): + """Makes sure a link is fully encoded. That is, if a ' ' shows up in + the link, it will be rewritten to %20 (while not over-quoting + % or other characters).""" + return _CLEAN_LINK_RE.sub(lambda match: '%%%2x' % ord(match.group(0)), url) + + class HTMLPage(object): """Represents one page, along with its URL""" def __init__(self, content, url, headers=None): - # Determine if we have any encoding information in our headers - encoding = None - if headers and "Content-Type" in headers: - content_type, params = cgi.parse_header(headers["Content-Type"]) - - if "charset" in params: - encoding = params['charset'] - self.content = content - self.parsed = html5lib.parse( - self.content, - transport_encoding=encoding, - namespaceHTMLElements=False, - ) self.url = url self.headers = headers def __str__(self): return self.url - @classmethod - def get_page(cls, link, skip_archives=True, session=None): - if session is None: - raise TypeError( - "get_page() missing 1 required keyword argument: 'session'" - ) - - url = link.url - url = url.split('#', 1)[0] - - # Check for VCS schemes that do not support lookup as web pages. - from pipenv.patched.notpip._internal.vcs import VcsSupport - for scheme in VcsSupport.schemes: - if url.lower().startswith(scheme) and url[len(scheme)] in '+:': - logger.debug('Cannot look at %s URL %s', scheme, link) - return None - - try: - if skip_archives: - filename = link.filename - for bad_ext in ARCHIVE_EXTENSIONS: - if filename.endswith(bad_ext): - content_type = cls._get_content_type( - url, session=session, - ) - if content_type.lower().startswith('text/html'): - break - else: - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return - - logger.debug('Getting page %s', url) - - # Tack index.html onto file:// URLs that point to directories - (scheme, netloc, path, params, query, fragment) = \ - urllib_parse.urlparse(url) - if (scheme == 'file' and - os.path.isdir(urllib_request.url2pathname(path))): - # add trailing slash if not present so urljoin doesn't trim - # final segment - if not url.endswith('/'): - url += '/' - url = urllib_parse.urljoin(url, 'index.html') - logger.debug(' file: URL is directory, getting %s', url) - - resp = session.get( - url, - headers={ - "Accept": "text/html", - "Cache-Control": "max-age=600", - }, - ) - resp.raise_for_status() - - # The check for archives above only works if the url ends with - # something that looks like an archive. However that is not a - # requirement of an url. Unless we issue a HEAD request on every - # url we cannot know ahead of time for sure if something is HTML - # or not. However we can check after we've downloaded it. - content_type = resp.headers.get('Content-Type', 'unknown') - if not content_type.lower().startswith("text/html"): - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return - - inst = cls(resp.content, resp.url, resp.headers) - except requests.HTTPError as exc: - cls._handle_fail(link, exc, url) - except SSLError as exc: - reason = "There was a problem confirming the ssl certificate: " - reason += str(exc) - cls._handle_fail(link, reason, url, meth=logger.info) - except requests.ConnectionError as exc: - cls._handle_fail(link, "connection error: %s" % exc, url) - except requests.Timeout: - cls._handle_fail(link, "timed out", url) - else: - return inst - - @staticmethod - def _handle_fail(link, reason, url, meth=None): - if meth is None: - meth = logger.debug - - meth("Could not fetch URL %s: %s - skipping", link, reason) - - @staticmethod - def _get_content_type(url, session): - """Get the Content-Type of the given url, using a HEAD request""" - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) - if scheme not in {'http', 'https'}: - # FIXME: some warning or something? - # assertion error? - return '' - - resp = session.head(url, allow_redirects=True) - resp.raise_for_status() - - return resp.headers.get("Content-Type", "") - - @cached_property - def base_url(self): - bases = [ - x for x in self.parsed.findall(".//base") - if x.get("href") is not None - ] - if bases and bases[0].get("href"): - return bases[0].get("href") - else: - return self.url - - @property - def links(self): + def iter_links(self): """Yields all links in the page""" - for anchor in self.parsed.findall(".//a"): + document = html5lib.parse( + self.content, + transport_encoding=_get_encoding_from_headers(self.headers), + namespaceHTMLElements=False, + ) + base_url = _determine_base_url(document, self.url) + for anchor in document.findall(".//a"): if anchor.get("href"): href = anchor.get("href") - url = self.clean_link( - urllib_parse.urljoin(self.base_url, href) - ) + url = _clean_link(urllib_parse.urljoin(base_url, href)) pyrequire = anchor.get('data-requires-python') pyrequire = unescape(pyrequire) if pyrequire else None - yield Link(url, self, requires_python=pyrequire) - - _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) - - def clean_link(self, url): - """Makes sure a link is fully encoded. That is, if a ' ' shows up in - the link, it will be rewritten to %20 (while not over-quoting - % or other characters).""" - return self._clean_re.sub( - lambda match: '%%%2x' % ord(match.group(0)), url) - - -class Link(object): - - def __init__(self, url, comes_from=None, requires_python=None): - """ - Object representing a parsed link from https://pypi.org/simple/* - - url: - url of the resource pointed to (href of the link) - comes_from: - instance of HTMLPage where the link was found, or string. - requires_python: - String containing the `Requires-Python` metadata field, specified - in PEP 345. This may be specified by a data-requires-python - attribute in the HTML link tag, as described in PEP 503. - """ - - # url can be a UNC windows share - if url.startswith('\\\\'): - url = path_to_url(url) - - self.url = url - self.comes_from = comes_from - self.requires_python = requires_python if requires_python else None - - def __str__(self): - if self.requires_python: - rp = ' (requires-python:%s)' % self.requires_python - else: - rp = '' - if self.comes_from: - return '%s (from %s)%s' % (self.url, self.comes_from, rp) - else: - return str(self.url) - - def __repr__(self): - return '' % self - - def __eq__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url == other.url - - def __ne__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url != other.url - - def __lt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url < other.url - - def __le__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url <= other.url - - def __gt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url > other.url - - def __ge__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url >= other.url - - def __hash__(self): - return hash(self.url) - - @property - def filename(self): - _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) - name = posixpath.basename(path.rstrip('/')) or netloc - name = urllib_parse.unquote(name) - assert name, ('URL %r produced no filename' % self.url) - return name - - @property - def scheme(self): - return urllib_parse.urlsplit(self.url)[0] - - @property - def netloc(self): - return urllib_parse.urlsplit(self.url)[1] - - @property - def path(self): - return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) - - def splitext(self): - return splitext(posixpath.basename(self.path.rstrip('/'))) - - @property - def ext(self): - return self.splitext()[1] - - @property - def url_without_fragment(self): - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) - return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) - - _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') - - @property - def egg_fragment(self): - match = self._egg_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') - - @property - def subdirectory_fragment(self): - match = self._subdirectory_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _hash_re = re.compile( - r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' - ) - - @property - def hash(self): - match = self._hash_re.search(self.url) - if match: - return match.group(2) - return None - - @property - def hash_name(self): - match = self._hash_re.search(self.url) - if match: - return match.group(1) - return None - - @property - def show_url(self): - return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) - - @property - def is_wheel(self): - return self.ext == wheel_ext - - @property - def is_artifact(self): - """ - Determines if this points to an actual artifact (e.g. a tarball) or if - it points to an "abstract" thing like a path or a VCS location. - """ - from pipenv.patched.notpip._internal.vcs import vcs - - if self.scheme in vcs.all_schemes: - return False - - return True - - -FormatControl = namedtuple('FormatControl', 'no_binary only_binary') -"""This object has two fields, no_binary and only_binary. - -If a field is falsy, it isn't set. If it is {':all:'}, it should match all -packages except those listed in the other field. Only one field can be set -to {':all:'} at a time. The rest of the time exact package name matches -are listed, with any given package only showing up in one field at a time. -""" - - -def fmt_ctl_handle_mutual_exclude(value, target, other): - new = value.split(',') - while ':all:' in new: - other.clear() - target.clear() - target.add(':all:') - del new[:new.index(':all:') + 1] - if ':none:' not in new: - # Without a none, we want to discard everything as :all: covers it - return - for name in new: - if name == ':none:': - target.clear() - continue - name = canonicalize_name(name) - other.discard(name) - target.add(name) - - -def fmt_ctl_formats(fmt_ctl, canonical_name): - result = {"binary", "source"} - if canonical_name in fmt_ctl.only_binary: - result.discard('source') - elif canonical_name in fmt_ctl.no_binary: - result.discard('binary') - elif ':all:' in fmt_ctl.only_binary: - result.discard('source') - elif ':all:' in fmt_ctl.no_binary: - result.discard('binary') - return frozenset(result) - - -def fmt_ctl_no_binary(fmt_ctl): - fmt_ctl_handle_mutual_exclude( - ':all:', fmt_ctl.no_binary, fmt_ctl.only_binary, - ) + yield Link(url, self.url, requires_python=pyrequire) Search = namedtuple('Search', 'supplied canonical formats') diff --git a/pipenv/patched/notpip/_internal/locations.py b/pipenv/patched/notpip/_internal/locations.py index 29c6db79..3c7d5bd8 100644 --- a/pipenv/patched/notpip/_internal/locations.py +++ b/pipenv/patched/notpip/_internal/locations.py @@ -10,8 +10,8 @@ import sysconfig from distutils import sysconfig as distutils_sysconfig from distutils.command.install import SCHEME_KEYS # type: ignore -from pipenv.patched.notpip._internal.compat import WINDOWS, expanduser from pipenv.patched.notpip._internal.utils import appdirs +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser # Application Directories USER_CACHE_DIR = appdirs.user_cache_dir("pip") diff --git a/pipenv/patched/notpip/_internal/models/candidate.py b/pipenv/patched/notpip/_internal/models/candidate.py new file mode 100644 index 00000000..9627589e --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/candidate.py @@ -0,0 +1,24 @@ +from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version + +from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin + + +class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + + def __init__(self, project, version, location, requires_python=None): + self.project = project + self.version = parse_version(version) + self.location = location + self.requires_python = requires_python + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.location), + defining_class=InstallationCandidate + ) + + def __repr__(self): + return "".format( + self.project, self.version, self.location, + ) diff --git a/pipenv/patched/notpip/_internal/models/format_control.py b/pipenv/patched/notpip/_internal/models/format_control.py new file mode 100644 index 00000000..caad3cba --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/format_control.py @@ -0,0 +1,62 @@ +from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name + + +class FormatControl(object): + """A helper class for controlling formats from which packages are installed. + If a field is falsy, it isn't set. If it is {':all:'}, it should match all + packages except those listed in the other field. Only one field can be set + to {':all:'} at a time. The rest of the time exact package name matches + are listed, with any given package only showing up in one field at a time. + """ + def __init__(self, no_binary=None, only_binary=None): + self.no_binary = set() if no_binary is None else no_binary + self.only_binary = set() if only_binary is None else only_binary + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "{}({}, {})".format( + self.__class__.__name__, + self.no_binary, + self.only_binary + ) + + @staticmethod + def handle_mutual_excludes(value, target, other): + new = value.split(',') + while ':all:' in new: + other.clear() + target.clear() + target.add(':all:') + del new[:new.index(':all:') + 1] + # Without a none, we want to discard everything as :all: covers it + if ':none:' not in new: + return + for name in new: + if name == ':none:': + target.clear() + continue + name = canonicalize_name(name) + other.discard(name) + target.add(name) + + def get_allowed_formats(self, canonical_name): + result = {"binary", "source"} + if canonical_name in self.only_binary: + result.discard('source') + elif canonical_name in self.no_binary: + result.discard('binary') + elif ':all:' in self.only_binary: + result.discard('source') + elif ':all:' in self.no_binary: + result.discard('binary') + return frozenset(result) + + def disallow_binaries(self): + self.handle_mutual_excludes( + ':all:', self.no_binary, self.only_binary, + ) diff --git a/pipenv/patched/notpip/_internal/models/index.py b/pipenv/patched/notpip/_internal/models/index.py index f9e84894..0983fc9c 100644 --- a/pipenv/patched/notpip/_internal/models/index.py +++ b/pipenv/patched/notpip/_internal/models/index.py @@ -1,15 +1,29 @@ from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -class Index(object): - def __init__(self, url): +class PackageIndex(object): + """Represents a Package Index and provides easier access to endpoints + """ + + def __init__(self, url, file_storage_domain): + super(PackageIndex, self).__init__() self.url = url self.netloc = urllib_parse.urlsplit(url).netloc - self.simple_url = self.url_to_path('simple') - self.pypi_url = self.url_to_path('pypi') + self.simple_url = self._url_for_path('simple') + self.pypi_url = self._url_for_path('pypi') - def url_to_path(self, path): + # This is part of a temporary hack used to block installs of PyPI + # packages which depend on external urls only necessary until PyPI can + # block such packages themselves + self.file_storage_domain = file_storage_domain + + def _url_for_path(self, path): return urllib_parse.urljoin(self.url, path) -PyPI = Index('https://pypi.org/') +PyPI = PackageIndex( + 'https://pypi.org/', file_storage_domain='files.pythonhosted.org' +) +TestPyPI = PackageIndex( + 'https://test.pypi.org/', file_storage_domain='test-files.pythonhosted.org' +) diff --git a/pipenv/patched/notpip/_internal/models/link.py b/pipenv/patched/notpip/_internal/models/link.py new file mode 100644 index 00000000..686af1d0 --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/link.py @@ -0,0 +1,141 @@ +import posixpath +import re + +from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse + +from pipenv.patched.notpip._internal.download import path_to_url +from pipenv.patched.notpip._internal.utils.misc import splitext +from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin +from pipenv.patched.notpip._internal.wheel import wheel_ext + + +class Link(KeyBasedCompareMixin): + """Represents a parsed link from a Package Index's simple URL + """ + + def __init__(self, url, comes_from=None, requires_python=None): + """ + url: + url of the resource pointed to (href of the link) + comes_from: + instance of HTMLPage where the link was found, or string. + requires_python: + String containing the `Requires-Python` metadata field, specified + in PEP 345. This may be specified by a data-requires-python + attribute in the HTML link tag, as described in PEP 503. + """ + + # url can be a UNC windows share + if url.startswith('\\\\'): + url = path_to_url(url) + + self.url = url + self.comes_from = comes_from + self.requires_python = requires_python if requires_python else None + + super(Link, self).__init__( + key=(self.url), + defining_class=Link + ) + + def __str__(self): + if self.requires_python: + rp = ' (requires-python:%s)' % self.requires_python + else: + rp = '' + if self.comes_from: + return '%s (from %s)%s' % (self.url, self.comes_from, rp) + else: + return str(self.url) + + def __repr__(self): + return '' % self + + @property + def filename(self): + _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) + name = posixpath.basename(path.rstrip('/')) or netloc + name = urllib_parse.unquote(name) + assert name, ('URL %r produced no filename' % self.url) + return name + + @property + def scheme(self): + return urllib_parse.urlsplit(self.url)[0] + + @property + def netloc(self): + return urllib_parse.urlsplit(self.url)[1] + + @property + def path(self): + return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) + + def splitext(self): + return splitext(posixpath.basename(self.path.rstrip('/'))) + + @property + def ext(self): + return self.splitext()[1] + + @property + def url_without_fragment(self): + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) + return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) + + _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') + + @property + def egg_fragment(self): + match = self._egg_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') + + @property + def subdirectory_fragment(self): + match = self._subdirectory_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _hash_re = re.compile( + r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' + ) + + @property + def hash(self): + match = self._hash_re.search(self.url) + if match: + return match.group(2) + return None + + @property + def hash_name(self): + match = self._hash_re.search(self.url) + if match: + return match.group(1) + return None + + @property + def show_url(self): + return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) + + @property + def is_wheel(self): + return self.ext == wheel_ext + + @property + def is_artifact(self): + """ + Determines if this points to an actual artifact (e.g. a tarball) or if + it points to an "abstract" thing like a path or a VCS location. + """ + from pipenv.patched.notpip._internal.vcs import vcs + + if self.scheme in vcs.all_schemes: + return False + + return True diff --git a/pipenv/patched/notpip/_internal/operations/freeze.py b/pipenv/patched/notpip/_internal/operations/freeze.py index 532989bc..b18b98e4 100644 --- a/pipenv/patched/notpip/_internal/operations/freeze.py +++ b/pipenv/patched/notpip/_internal/operations/freeze.py @@ -10,11 +10,13 @@ from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError from pipenv.patched.notpip._internal.exceptions import InstallationError -from pipenv.patched.notpip._internal.req import InstallRequirement +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, +) from pipenv.patched.notpip._internal.req.req_file import COMMENT_RE from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.misc import ( - dist_is_editable, get_installed_distributions, + dist_is_editable, get_installed_distributions, make_vcs_requirement_url, ) logger = logging.getLogger(__name__) @@ -99,13 +101,13 @@ def freeze( line = line[2:].strip() else: line = line[len('--editable'):].strip().lstrip('=') - line_req = InstallRequirement.from_editable( + line_req = install_req_from_editable( line, isolated=isolated, wheel_cache=wheel_cache, ) else: - line_req = InstallRequirement.from_line( + line_req = install_req_from_line( COMMENT_RE.sub('', line).strip(), isolated=isolated, wheel_cache=wheel_cache, @@ -166,7 +168,13 @@ class FrozenRequirement(object): _date_re = re.compile(r'-(20\d\d\d\d\d\d)$') @classmethod - def from_dist(cls, dist, dependency_links): + def _init_args_from_dist(cls, dist, dependency_links): + """ + Compute and return arguments (req, editable, comments) to pass to + FrozenRequirement.__init__(). + + This method is for use in FrozenRequirement.from_dist(). + """ location = os.path.normcase(os.path.abspath(dist.location)) comments = [] from pipenv.patched.notpip._internal.vcs import vcs, get_src_requirement @@ -231,12 +239,15 @@ class FrozenRequirement(object): else: rev = '{%s}' % date_match.group(1) editable = True - req = '%s@%s#egg=%s' % ( - svn_location, - rev, - cls.egg_name(dist) - ) - return cls(dist.project_name, req, editable, comments) + egg_name = cls.egg_name(dist) + req = make_vcs_requirement_url(svn_location, rev, egg_name) + + return (req, editable, comments) + + @classmethod + def from_dist(cls, dist, dependency_links): + args = cls._init_args_from_dist(dist, dependency_links) + return cls(dist.project_name, *args) @staticmethod def egg_name(dist): diff --git a/pipenv/patched/notpip/_internal/operations/prepare.py b/pipenv/patched/notpip/_internal/operations/prepare.py index 9ebc3ebd..d61270b4 100644 --- a/pipenv/patched/notpip/_internal/operations/prepare.py +++ b/pipenv/patched/notpip/_internal/operations/prepare.py @@ -7,7 +7,6 @@ import os from pipenv.patched.notpip._vendor import pkg_resources, requests from pipenv.patched.notpip._internal.build_env import BuildEnvironment -from pipenv.patched.notpip._internal.compat import expanduser from pipenv.patched.notpip._internal.download import ( is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path, ) @@ -15,6 +14,7 @@ from pipenv.patched.notpip._internal.exceptions import ( DirectoryUrlHashUnsupported, HashUnpinned, InstallationError, PreviousBuildDirError, VcsHashUnsupported, ) +from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.hashes import MissingHashes from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import display_path, normalize_path, rmtree @@ -65,7 +65,7 @@ class DistAbstraction(object): """Return a setuptools Dist object.""" raise NotImplementedError(self.dist) - def prep_for_dist(self, finder): + def prep_for_dist(self, finder, build_isolation): """Ensure that we can get a Dist for this requirement.""" raise NotImplementedError(self.dist) @@ -93,36 +93,36 @@ class IsSDist(DistAbstraction): return dist def prep_for_dist(self, finder, build_isolation): - # Before calling "setup.py egg_info", we need to set-up the build - # environment. - build_requirements = self.req.get_pep_518_info() - should_isolate = build_isolation and build_requirements is not None + # Prepare for building. We need to: + # 1. Load pyproject.toml (if it exists) + # 2. Set up the build environment + + self.req.load_pyproject_toml() + should_isolate = self.req.use_pep517 and build_isolation if should_isolate: - # Haven't implemented PEP 517 yet, so spew a warning about it if - # build-requirements don't include setuptools and wheel. - missing_requirements = {'setuptools', 'wheel'} - { - pkg_resources.Requirement(r).key for r in build_requirements - } - if missing_requirements: + # Isolate in a BuildEnvironment and install the build-time + # requirements. + self.req.build_env = BuildEnvironment() + self.req.build_env.install_requirements( + finder, self.req.pyproject_requires, + "Installing build dependencies" + ) + missing = [] + if self.req.requirements_to_check: + check = self.req.requirements_to_check + missing = self.req.build_env.missing_requirements(check) + if missing: logger.warning( "Missing build requirements in pyproject.toml for %s.", self.req, ) logger.warning( - "This version of pip does not implement PEP 517 so it " - "cannot build a wheel without %s.", - " and ".join(map(repr, sorted(missing_requirements))) + "The project does not specify a build backend, and pip " + "cannot fall back to setuptools without %s.", + " and ".join(map(repr, sorted(missing))) ) - # Isolate in a BuildEnvironment and install the build-time - # requirements. - self.req.build_env = BuildEnvironment() - self.req.build_env.install_requirements( - finder, build_requirements, - "Installing build dependencies" - ) - try: self.req.run_egg_info() except (OSError, TypeError): @@ -136,7 +136,7 @@ class Installed(DistAbstraction): def dist(self, finder): return self.req.satisfied_by - def prep_for_dist(self, finder): + def prep_for_dist(self, finder, build_isolation): pass diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py index 4205f6e0..182c1c88 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py @@ -10,7 +10,12 @@ import sysconfig import warnings from collections import OrderedDict -import pipenv.patched.notpip._internal.utils.glibc +try: + import pipenv.patched.notpip._internal.utils.glibc +except ImportError: + import pipenv.patched.notpip.utils.glibc + +from pipenv.patched.notpip._internal.utils.compat import get_extension_suffixes logger = logging.getLogger(__name__) @@ -252,10 +257,9 @@ def get_supported(versions=None, noarch=False, platform=None, abis[0:0] = [abi] abi3s = set() - import imp - for suffix in imp.get_suffixes(): - if suffix[0].startswith('.abi'): - abi3s.add(suffix[0].split('.', 2)[1]) + for suffix in get_extension_suffixes(): + if suffix.startswith('.abi'): + abi3s.add(suffix.split('.', 2)[1]) abis.extend(sorted(list(abi3s))) diff --git a/pipenv/patched/notpip/_internal/pyproject.py b/pipenv/patched/notpip/_internal/pyproject.py new file mode 100644 index 00000000..a47e0f05 --- /dev/null +++ b/pipenv/patched/notpip/_internal/pyproject.py @@ -0,0 +1,144 @@ +from __future__ import absolute_import + +import io +import os + +from pipenv.patched.notpip._vendor import pytoml, six + +from pipenv.patched.notpip._internal.exceptions import InstallationError + + +def _is_list_of_str(obj): + return ( + isinstance(obj, list) and + all(isinstance(item, six.string_types) for item in obj) + ) + + +def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): + """Load the pyproject.toml file. + + Parameters: + use_pep517 - Has the user requested PEP 517 processing? None + means the user hasn't explicitly specified. + pyproject_toml - Location of the project's pyproject.toml file + setup_py - Location of the project's setup.py file + req_name - The name of the requirement we're processing (for + error reporting) + + Returns: + None if we should use the legacy code path, otherwise a tuple + ( + requirements from pyproject.toml, + name of PEP 517 backend, + requirements we should check are installed after setting + up the build environment + ) + """ + has_pyproject = os.path.isfile(pyproject_toml) + has_setup = os.path.isfile(setup_py) + + if has_pyproject: + with io.open(pyproject_toml, encoding="utf-8") as f: + pp_toml = pytoml.load(f) + build_system = pp_toml.get("build-system") + else: + build_system = None + + # The following cases must use PEP 517 + # We check for use_pep517 equalling False because that + # means the user explicitly requested --no-use-pep517 + if has_pyproject and not has_setup: + if use_pep517 is False: + raise InstallationError( + "Disabling PEP 517 processing is invalid: " + "project does not have a setup.py" + ) + use_pep517 = True + elif build_system and "build-backend" in build_system: + if use_pep517 is False: + raise InstallationError( + "Disabling PEP 517 processing is invalid: " + "project specifies a build backend of {} " + "in pyproject.toml".format( + build_system["build-backend"] + ) + ) + use_pep517 = True + + # If we haven't worked out whether to use PEP 517 yet, + # and the user hasn't explicitly stated a preference, + # we do so if the project has a pyproject.toml file. + elif use_pep517 is None: + use_pep517 = has_pyproject + + # At this point, we know whether we're going to use PEP 517. + assert use_pep517 is not None + + # If we're using the legacy code path, there is nothing further + # for us to do here. + if not use_pep517: + return None + + if build_system is None: + # Either the user has a pyproject.toml with no build-system + # section, or the user has no pyproject.toml, but has opted in + # explicitly via --use-pep517. + # In the absence of any explicit backend specification, we + # assume the setuptools backend, and require wheel and a version + # of setuptools that supports that backend. + build_system = { + "requires": ["setuptools>=38.2.5", "wheel"], + "build-backend": "setuptools.build_meta", + } + + # If we're using PEP 517, we have build system information (either + # from pyproject.toml, or defaulted by the code above). + # Note that at this point, we do not know if the user has actually + # specified a backend, though. + assert build_system is not None + + # Ensure that the build-system section in pyproject.toml conforms + # to PEP 518. + error_template = ( + "{package} has a pyproject.toml file that does not comply " + "with PEP 518: {reason}" + ) + + # Specifying the build-system table but not the requires key is invalid + if "requires" not in build_system: + raise InstallationError( + error_template.format(package=req_name, reason=( + "it has a 'build-system' table but not " + "'build-system.requires' which is mandatory in the table" + )) + ) + + # Error out if requires is not a list of strings + requires = build_system["requires"] + if not _is_list_of_str(requires): + raise InstallationError(error_template.format( + package=req_name, + reason="'build-system.requires' is not a list of strings.", + )) + + backend = build_system.get("build-backend") + check = [] + if backend is None: + # If the user didn't specify a backend, we assume they want to use + # the setuptools backend. But we can't be sure they have included + # a version of setuptools which supplies the backend, or wheel + # (which is neede by the backend) in their requirements. So we + # make a note to check that those requirements are present once + # we have set up the environment. + # TODO: Review this - it's quite a lot of work to check for a very + # specific case. The problem is, that case is potentially quite + # common - projects that adopted PEP 518 early for the ability to + # specify requirements to execute setup.py, but never considered + # needing to mention the build tools themselves. The original PEP + # 518 code had a similar check (but implemented in a different + # way). + backend = "setuptools.build_meta" + check = ["setuptools>=38.2.5", "wheel"] + + return (requires, backend, check) diff --git a/pipenv/patched/notpip/_internal/req/constructors.py b/pipenv/patched/notpip/_internal/req/constructors.py new file mode 100644 index 00000000..9fe28d88 --- /dev/null +++ b/pipenv/patched/notpip/_internal/req/constructors.py @@ -0,0 +1,298 @@ +"""Backing implementation for InstallRequirement's various constructors + +The idea here is that these formed a major chunk of InstallRequirement's size +so, moving them and support code dedicated to them outside of that class +helps creates for better understandability for the rest of the code. + +These are meant to be used elsewhere within pip to create instances of +InstallRequirement. +""" + +import logging +import os +import re +import traceback + +from pipenv.patched.notpip._vendor.packaging.markers import Marker +from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement +from pipenv.patched.notpip._vendor.packaging.specifiers import Specifier +from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError, parse_requirements + +from pipenv.patched.notpip._internal.download import ( + is_archive_file, is_url, path_to_url, url_to_path, +) +from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.models.index import PyPI, TestPyPI +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.req.req_install import InstallRequirement +from pipenv.patched.notpip._internal.utils.misc import is_installable_dir +from pipenv.patched.notpip._internal.vcs import vcs +from pipenv.patched.notpip._internal.wheel import Wheel + +__all__ = [ + "install_req_from_editable", "install_req_from_line", + "parse_editable" +] + +logger = logging.getLogger(__name__) +operators = Specifier._operators.keys() + + +def _strip_extras(path): + m = re.match(r'^(.+)(\[[^\]]+\])$', path) + extras = None + if m: + path_no_extras = m.group(1) + extras = m.group(2) + else: + path_no_extras = path + + return path_no_extras, extras + + +def parse_editable(editable_req): + """Parses an editable requirement into: + - a requirement name + - an URL + - extras + - editable options + Accepted requirements: + svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir + .[some_extra] + """ + + url = editable_req + + # If a file path is specified with extras, strip off the extras. + url_no_extras, extras = _strip_extras(url) + + if os.path.isdir(url_no_extras): + if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): + raise InstallationError( + "Directory %r is not installable. File 'setup.py' not found." % + url_no_extras + ) + # Treating it as code that has already been checked out + url_no_extras = path_to_url(url_no_extras) + + if url_no_extras.lower().startswith('file:'): + package_name = Link(url_no_extras).egg_fragment + if extras: + return ( + package_name, + url_no_extras, + Requirement("placeholder" + extras.lower()).extras, + ) + else: + return package_name, url_no_extras, None + + for version_control in vcs: + if url.lower().startswith('%s:' % version_control): + url = '%s+%s' % (version_control, url) + break + + if '+' not in url: + raise InstallationError( + '%s should either be a path to a local project or a VCS url ' + 'beginning with svn+, git+, hg+, or bzr+' % + editable_req + ) + + vc_type = url.split('+', 1)[0].lower() + + if not vcs.get_backend(vc_type): + error_message = 'For --editable=%s only ' % editable_req + \ + ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ + ' is currently supported' + raise InstallationError(error_message) + + package_name = Link(url).egg_fragment + if not package_name: + raise InstallationError( + "Could not detect requirement name for '%s', please specify one " + "with #egg=your_package_name" % editable_req + ) + return package_name, url, None + + +def deduce_helpful_msg(req): + """Returns helpful msg in case requirements file does not exist, + or cannot be parsed. + + :params req: Requirements file path + """ + msg = "" + if os.path.exists(req): + msg = " It does exist." + # Try to parse and check if it is a requirements file. + try: + with open(req, 'r') as fp: + # parse first line only + next(parse_requirements(fp.read())) + msg += " The argument you provided " + \ + "(%s) appears to be a" % (req) + \ + " requirements file. If that is the" + \ + " case, use the '-r' flag to install" + \ + " the packages specified within it." + except RequirementParseError: + logger.debug("Cannot parse '%s' as requirements \ + file" % (req), exc_info=1) + else: + msg += " File '%s' does not exist." % (req) + return msg + + +# ---- The actual constructors follow ---- + + +def install_req_from_editable( + editable_req, comes_from=None, isolated=False, options=None, + wheel_cache=None, constraint=False +): + name, url, extras_override = parse_editable(editable_req) + if url.startswith('file:'): + source_dir = url_to_path(url) + else: + source_dir = None + + if name is not None: + try: + req = Requirement(name) + except InvalidRequirement: + raise InstallationError("Invalid requirement: '%s'" % name) + else: + req = None + return InstallRequirement( + req, comes_from, source_dir=source_dir, + editable=True, + link=Link(url), + constraint=constraint, + isolated=isolated, + options=options if options else {}, + wheel_cache=wheel_cache, + extras=extras_override or (), + ) + + +def install_req_from_line( + name, comes_from=None, isolated=False, options=None, wheel_cache=None, + constraint=False +): + """Creates an InstallRequirement from a name, which might be a + requirement, directory containing 'setup.py', filename, or URL. + """ + if is_url(name): + marker_sep = '; ' + else: + marker_sep = ';' + if marker_sep in name: + name, markers = name.split(marker_sep, 1) + markers = markers.strip() + if not markers: + markers = None + else: + markers = Marker(markers) + else: + markers = None + name = name.strip() + req = None + path = os.path.normpath(os.path.abspath(name)) + link = None + extras = None + + if is_url(name): + link = Link(name) + else: + p, extras = _strip_extras(path) + looks_like_dir = os.path.isdir(p) and ( + os.path.sep in name or + (os.path.altsep is not None and os.path.altsep in name) or + name.startswith('.') + ) + if looks_like_dir: + if not is_installable_dir(p): + raise InstallationError( + "Directory %r is not installable. Neither 'setup.py' " + "nor 'pyproject.toml' found." % name + ) + link = Link(path_to_url(p)) + elif is_archive_file(p): + if not os.path.isfile(p): + logger.warning( + 'Requirement %r looks like a filename, but the ' + 'file does not exist', + name + ) + link = Link(path_to_url(p)) + + # it's a local file, dir, or url + if link: + # Handle relative file URLs + if link.scheme == 'file' and re.search(r'\.\./', link.url): + link = Link( + path_to_url(os.path.normpath(os.path.abspath(link.path)))) + # wheel file + if link.is_wheel: + wheel = Wheel(link.filename) # can raise InvalidWheelFilename + req = "%s==%s" % (wheel.name, wheel.version) + else: + # set the req to the egg fragment. when it's not there, this + # will become an 'unnamed' requirement + req = link.egg_fragment + + # a requirement specifier + else: + req = name + + if extras: + extras = Requirement("placeholder" + extras.lower()).extras + else: + extras = () + if req is not None: + try: + req = Requirement(req) + except InvalidRequirement: + if os.path.sep in req: + add_msg = "It looks like a path." + add_msg += deduce_helpful_msg(req) + elif '=' in req and not any(op in req for op in operators): + add_msg = "= is not a valid operator. Did you mean == ?" + else: + add_msg = traceback.format_exc() + raise InstallationError( + "Invalid requirement: '%s'\n%s" % (req, add_msg) + ) + + return InstallRequirement( + req, comes_from, link=link, markers=markers, + isolated=isolated, + options=options if options else {}, + wheel_cache=wheel_cache, + constraint=constraint, + extras=extras, + ) + + +def install_req_from_req( + req, comes_from=None, isolated=False, wheel_cache=None +): + try: + req = Requirement(req) + except InvalidRequirement: + raise InstallationError("Invalid requirement: '%s'" % req) + + domains_not_allowed = [ + PyPI.file_storage_domain, + TestPyPI.file_storage_domain, + ] + if req.url and comes_from.link.netloc in domains_not_allowed: + # Explicitly disallow pypi packages that depend on external urls + raise InstallationError( + "Packages installed from PyPI cannot depend on packages " + "which are not also hosted on PyPI.\n" + "%s depends on %s " % (comes_from.name, req) + ) + + return InstallRequirement( + req, comes_from, isolated=isolated, wheel_cache=wheel_cache + ) diff --git a/pipenv/patched/notpip/_internal/req/req_file.py b/pipenv/patched/notpip/_internal/req/req_file.py index 66a58022..5f23cd3a 100644 --- a/pipenv/patched/notpip/_internal/req/req_file.py +++ b/pipenv/patched/notpip/_internal/req/req_file.py @@ -13,10 +13,12 @@ import sys from pipenv.patched.notpip._vendor.six.moves import filterfalse from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -from pipenv.patched.notpip._internal import cmdoptions +from pipenv.patched.notpip._internal.cli import cmdoptions from pipenv.patched.notpip._internal.download import get_file_content from pipenv.patched.notpip._internal.exceptions import RequirementsFileParseError -from pipenv.patched.notpip._internal.req.req_install import InstallRequirement +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, +) __all__ = ['parse_requirements'] @@ -151,7 +153,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, for dest in SUPPORTED_OPTIONS_REQ_DEST: if dest in opts.__dict__ and opts.__dict__[dest]: req_options[dest] = opts.__dict__[dest] - yield InstallRequirement.from_line( + yield install_req_from_line( args_str, line_comes_from, constraint=constraint, isolated=isolated, options=req_options, wheel_cache=wheel_cache ) @@ -159,7 +161,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, # yield an editable requirement elif opts.editables: isolated = options.isolated_mode if options else False - yield InstallRequirement.from_editable( + yield install_req_from_editable( opts.editables[0], comes_from=line_comes_from, constraint=constraint, isolated=isolated, wheel_cache=wheel_cache ) diff --git a/pipenv/patched/notpip/_internal/req/req_install.py b/pipenv/patched/notpip/_internal/req/req_install.py index a9c642c0..3f32892c 100644 --- a/pipenv/patched/notpip/_internal/req/req_install.py +++ b/pipenv/patched/notpip/_internal/req/req_install.py @@ -1,66 +1,46 @@ from __future__ import absolute_import -import io import logging import os -import re import shutil import sys import sysconfig -import traceback import zipfile from distutils.util import change_root -from email.parser import FeedParser # type: ignore -from pipenv.patched.notpip._vendor import pkg_resources, pytoml, six -from pipenv.patched.notpip._vendor.packaging import specifiers -from pipenv.patched.notpip._vendor.packaging.markers import Marker -from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement +from pipenv.patched.notpip._vendor import pkg_resources, six +from pipenv.patched.notpip._vendor.packaging.requirements import Requirement from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.packaging.version import Version from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version -from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError, parse_requirements +from pipenv.patched.notpip._vendor.pep517.wrappers import Pep517HookCaller from pipenv.patched.notpip._internal import wheel from pipenv.patched.notpip._internal.build_env import NoOpBuildEnvironment -from pipenv.patched.notpip._internal.compat import native_str -from pipenv.patched.notpip._internal.download import ( - is_archive_file, is_url, path_to_url, url_to_path, -) from pipenv.patched.notpip._internal.exceptions import InstallationError from pipenv.patched.notpip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.pyproject import load_pyproject_toml from pipenv.patched.notpip._internal.req.req_uninstall import UninstallPathSet +from pipenv.patched.notpip._internal.utils.compat import native_str from pipenv.patched.notpip._internal.utils.hashes import Hashes from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( _make_build_dir, ask_path_exists, backup_dir, call_subprocess, display_path, dist_in_site_packages, dist_in_usersite, ensure_dir, - get_installed_version, is_installable_dir, read_text_file, rmtree, + get_installed_version, rmtree, ) +from pipenv.patched.notpip._internal.utils.packaging import get_metadata from pipenv.patched.notpip._internal.utils.setuptools_build import SETUPTOOLS_SHIM from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.utils.ui import open_spinner from pipenv.patched.notpip._internal.vcs import vcs -from pipenv.patched.notpip._internal.wheel import Wheel, move_wheel_files +from pipenv.patched.notpip._internal.wheel import move_wheel_files logger = logging.getLogger(__name__) -operators = specifiers.Specifier._operators.keys() - - -def _strip_extras(path): - m = re.match(r'^(.+)(\[[^\]]+\])$', path) - extras = None - if m: - path_no_extras = m.group(1) - extras = m.group(2) - else: - path_no_extras = path - - return path_no_extras, extras - class InstallRequirement(object): """ @@ -87,7 +67,6 @@ class InstallRequirement(object): if link is not None: self.link = self.original_link = link else: - from pipenv.patched.notpip._internal.index import Link self.link = self.original_link = req and req.url and Link(req.url) if extras: @@ -128,155 +107,32 @@ class InstallRequirement(object): self.isolated = isolated self.build_env = NoOpBuildEnvironment() - # Constructors - # TODO: Move these out of this class into custom methods. - @classmethod - def from_editable(cls, editable_req, comes_from=None, isolated=False, - options=None, wheel_cache=None, constraint=False): - from pipenv.patched.notpip._internal.index import Link + # The static build requirements (from pyproject.toml) + self.pyproject_requires = None - name, url, extras_override = parse_editable(editable_req) - if url.startswith('file:'): - source_dir = url_to_path(url) - else: - source_dir = None + # Build requirements that we will check are available + # TODO: We don't do this for --no-build-isolation. Should we? + self.requirements_to_check = [] - if name is not None: - try: - req = Requirement(name) - except InvalidRequirement: - raise InstallationError("Invalid requirement: '%s'" % name) - else: - req = None - return cls( - req, comes_from, source_dir=source_dir, - editable=True, - link=Link(url), - constraint=constraint, - isolated=isolated, - options=options if options else {}, - wheel_cache=wheel_cache, - extras=extras_override or (), - ) + # The PEP 517 backend we should use to build the project + self.pep517_backend = None - @classmethod - def from_req(cls, req, comes_from=None, isolated=False, wheel_cache=None): - try: - req = Requirement(req) - except InvalidRequirement: - raise InstallationError("Invalid requirement: '%s'" % req) - if req.url: - raise InstallationError( - "Direct url requirement (like %s) are not allowed for " - "dependencies" % req - ) - return cls(req, comes_from, isolated=isolated, wheel_cache=wheel_cache) - - @classmethod - def from_line( - cls, name, comes_from=None, isolated=False, options=None, - wheel_cache=None, constraint=False): - """Creates an InstallRequirement from a name, which might be a - requirement, directory containing 'setup.py', filename, or URL. - """ - from pipenv.patched.notpip._internal.index import Link - - if is_url(name): - marker_sep = '; ' - else: - marker_sep = ';' - if marker_sep in name: - name, markers = name.split(marker_sep, 1) - markers = markers.strip() - if not markers: - markers = None - else: - markers = Marker(markers) - else: - markers = None - name = name.strip() - req = None - path = os.path.normpath(os.path.abspath(name)) - link = None - extras = None - - if is_url(name): - link = Link(name) - else: - p, extras = _strip_extras(path) - looks_like_dir = os.path.isdir(p) and ( - os.path.sep in name or - (os.path.altsep is not None and os.path.altsep in name) or - name.startswith('.') - ) - if looks_like_dir: - if not is_installable_dir(p): - raise InstallationError( - "Directory %r is not installable. File 'setup.py' " - "not found." % name - ) - link = Link(path_to_url(p)) - elif is_archive_file(p): - if not os.path.isfile(p): - logger.warning( - 'Requirement %r looks like a filename, but the ' - 'file does not exist', - name - ) - link = Link(path_to_url(p)) - - # it's a local file, dir, or url - if link: - # Handle relative file URLs - if link.scheme == 'file' and re.search(r'\.\./', link.url): - link = Link( - path_to_url(os.path.normpath(os.path.abspath(link.path)))) - # wheel file - if link.is_wheel: - wheel = Wheel(link.filename) # can raise InvalidWheelFilename - req = "%s==%s" % (wheel.name, wheel.version) - else: - # set the req to the egg fragment. when it's not there, this - # will become an 'unnamed' requirement - req = link.egg_fragment - - # a requirement specifier - else: - req = name - - if extras: - extras = Requirement("placeholder" + extras.lower()).extras - else: - extras = () - if req is not None: - try: - req = Requirement(req) - except InvalidRequirement: - if os.path.sep in req: - add_msg = "It looks like a path." - add_msg += deduce_helpful_msg(req) - elif '=' in req and not any(op in req for op in operators): - add_msg = "= is not a valid operator. Did you mean == ?" - else: - add_msg = traceback.format_exc() - raise InstallationError( - "Invalid requirement: '%s'\n%s" % (req, add_msg)) - return cls( - req, comes_from, link=link, markers=markers, - isolated=isolated, - options=options if options else {}, - wheel_cache=wheel_cache, - constraint=constraint, - extras=extras, - ) + # Are we using PEP 517 for this requirement? + # After pyproject.toml has been loaded, the only valid values are True + # and False. Before loading, None is valid (meaning "use the default"). + # Setting an explicit value before loading pyproject.toml is supported, + # but after loading this flag should be treated as read only. + self.use_pep517 = None def __str__(self): if self.req: s = str(self.req) if self.link: s += ' from %s' % self.link.url + elif self.link: + s = self.link.url else: - s = self.link.url if self.link else None + s = '' if self.satisfied_by is not None: s += ' in %s' % display_path(self.satisfied_by.location) if self.comes_from: @@ -429,7 +285,7 @@ class InstallRequirement(object): package is not available until we run egg_info, so the build_location will return a temporary directory and store the _ideal_build_dir. - This is only called by self.egg_info_path to fix the temporary build + This is only called by self.run_egg_info to fix the temporary build directory. """ if self.source_dir is not None: @@ -557,48 +413,29 @@ class InstallRequirement(object): return pp_toml - def get_pep_518_info(self): - """Get PEP 518 build-time requirements. + def load_pyproject_toml(self): + """Load the pyproject.toml file. - Returns the list of the packages required to build the project, - specified as per PEP 518 within the package. If `pyproject.toml` is not - present, returns None to signify not using the same. + After calling this routine, all of the attributes related to PEP 517 + processing for this requirement have been set. In particular, the + use_pep517 attribute can be used to determine whether we should + follow the PEP 517 or legacy (setup.py) code path. """ - # If pyproject.toml does not exist, don't do anything. - if not os.path.isfile(self.pyproject_toml): - return None - - error_template = ( - "{package} has a pyproject.toml file that does not comply " - "with PEP 518: {reason}" + pep517_data = load_pyproject_toml( + self.use_pep517, + self.pyproject_toml, + self.setup_py, + str(self) ) - with io.open(self.pyproject_toml, encoding="utf-8") as f: - pp_toml = pytoml.load(f) - - # If there is no build-system table, just use setuptools and wheel. - if "build-system" not in pp_toml: - return ["setuptools", "wheel"] - - # Specifying the build-system table but not the requires key is invalid - build_system = pp_toml["build-system"] - if "requires" not in build_system: - raise InstallationError( - error_template.format(package=self, reason=( - "it has a 'build-system' table but not " - "'build-system.requires' which is mandatory in the table" - )) - ) - - # Error out if it's not a list of strings - requires = build_system["requires"] - if not _is_list_of_str(requires): - raise InstallationError(error_template.format( - package=self, - reason="'build-system.requires' is not a list of strings.", - )) - - return requires + if pep517_data is None: + self.use_pep517 = False + else: + self.use_pep517 = True + requires, backend, check = pep517_data + self.requirements_to_check = check + self.pyproject_requires = requires + self.pep517_backend = Pep517HookCaller(self.setup_py_dir, backend) def run_egg_info(self): assert self.source_dir @@ -615,7 +452,8 @@ class InstallRequirement(object): with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py - base_cmd = [os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', script] + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + base_cmd = [sys_executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] @@ -636,20 +474,20 @@ class InstallRequirement(object): command_desc='python setup.py egg_info') if not self.req: - if isinstance(parse_version(self.pkg_info()["Version"]), Version): + if isinstance(parse_version(self.metadata["Version"]), Version): op = "==" else: op = "===" self.req = Requirement( "".join([ - self.pkg_info()["Name"], + self.metadata["Name"], op, - self.pkg_info()["Version"], + self.metadata["Version"], ]) ) self._correct_build_location() else: - metadata_name = canonicalize_name(self.pkg_info()["Name"]) + metadata_name = canonicalize_name(self.metadata["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( 'Running setup.py (path:%s) egg_info for package %s ' @@ -659,19 +497,8 @@ class InstallRequirement(object): ) self.req = Requirement(metadata_name) - def egg_info_data(self, filename): - if self.satisfied_by is not None: - if not self.satisfied_by.has_metadata(filename): - return None - return self.satisfied_by.get_metadata(filename) - assert self.source_dir - filename = self.egg_info_path(filename) - if not os.path.exists(filename): - return None - data = read_text_file(filename) - return data - - def egg_info_path(self, filename): + @property + def egg_info_path(self): if self._egg_info_path is None: if self.editable: base = self.source_dir @@ -709,8 +536,7 @@ class InstallRequirement(object): if not filenames: raise InstallationError( - "Files/directories (from %s) not found in %s" - % (filename, base) + "Files/directories not found in %s" % base ) # if we have more than one match, we pick the toplevel one. This # can easily be the case if there is a dist folder which contains @@ -721,24 +547,18 @@ class InstallRequirement(object): (os.path.altsep and x.count(os.path.altsep) or 0) ) self._egg_info_path = os.path.join(base, filenames[0]) - return os.path.join(self._egg_info_path, filename) + return self._egg_info_path - def pkg_info(self): - p = FeedParser() - data = self.egg_info_data('PKG-INFO') - if not data: - logger.warning( - 'No PKG-INFO file found in %s', - display_path(self.egg_info_path('PKG-INFO')), - ) - p.feed(data or '') - return p.close() + @property + def metadata(self): + if not hasattr(self, '_metadata'): + self._metadata = get_metadata(self.get_dist()) - _requirements_section_re = re.compile(r'\[(.*?)\]') + return self._metadata def get_dist(self): """Return a pkg_resources.Distribution built from self.egg_info_path""" - egg_info = self.egg_info_path('').rstrip(os.path.sep) + egg_info = self.egg_info_path.rstrip(os.path.sep) base_dir = os.path.dirname(egg_info) metadata = pkg_resources.PathMetadata(base_dir, egg_info) dist_name = os.path.splitext(os.path.basename(egg_info))[0] @@ -750,7 +570,7 @@ class InstallRequirement(object): def assert_source_matches_version(self): assert self.source_dir - version = self.pkg_info()['version'] + version = self.metadata['version'] if self.req.specifier and version not in self.req.specifier: logger.warning( 'Requested %s, but installing version %s', @@ -794,10 +614,11 @@ class InstallRequirement(object): with indent_log(): # FIXME: should we do --install-headers here too? + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) with self.build_env: call_subprocess( [ - os.environ.get('PIP_PYTHON_PATH', sys.executable), + sys_executable, '-c', SETUPTOOLS_SHIM % self.setup_py ] + @@ -877,7 +698,7 @@ class InstallRequirement(object): def archive(self, build_dir): assert self.source_dir create_archive = True - archive_name = '%s-%s.zip' % (self.name, self.pkg_info()["version"]) + archive_name = '%s-%s.zip' % (self.name, self.metadata["version"]) archive_path = os.path.join(build_dir, archive_name) if os.path.exists(archive_path): response = ask_path_exists( @@ -1015,7 +836,8 @@ class InstallRequirement(object): def get_install_args(self, global_options, record_filename, root, prefix, pycompile): - install_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable), "-u"] + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + install_args = [sys_executable, "-u"] install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ @@ -1039,104 +861,3 @@ class InstallRequirement(object): py_ver_str, self.name)] return install_args - - -def parse_editable(editable_req): - """Parses an editable requirement into: - - a requirement name - - an URL - - extras - - editable options - Accepted requirements: - svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir - .[some_extra] - """ - - from pipenv.patched.notpip._internal.index import Link - - url = editable_req - - # If a file path is specified with extras, strip off the extras. - url_no_extras, extras = _strip_extras(url) - - if os.path.isdir(url_no_extras): - if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): - raise InstallationError( - "Directory %r is not installable. File 'setup.py' not found." % - url_no_extras - ) - # Treating it as code that has already been checked out - url_no_extras = path_to_url(url_no_extras) - - if url_no_extras.lower().startswith('file:'): - package_name = Link(url_no_extras).egg_fragment - if extras: - return ( - package_name, - url_no_extras, - Requirement("placeholder" + extras.lower()).extras, - ) - else: - return package_name, url_no_extras, None - - for version_control in vcs: - if url.lower().startswith('%s:' % version_control): - url = '%s+%s' % (version_control, url) - break - - if '+' not in url: - raise InstallationError( - '%s should either be a path to a local project or a VCS url ' - 'beginning with svn+, git+, hg+, or bzr+' % - editable_req - ) - - vc_type = url.split('+', 1)[0].lower() - - if not vcs.get_backend(vc_type): - error_message = 'For --editable=%s only ' % editable_req + \ - ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ - ' is currently supported' - raise InstallationError(error_message) - - package_name = Link(url).egg_fragment - if not package_name: - raise InstallationError( - "Could not detect requirement name for '%s', please specify one " - "with #egg=your_package_name" % editable_req - ) - return package_name, url, None - - -def deduce_helpful_msg(req): - """Returns helpful msg in case requirements file does not exist, - or cannot be parsed. - - :params req: Requirements file path - """ - msg = "" - if os.path.exists(req): - msg = " It does exist." - # Try to parse and check if it is a requirements file. - try: - with open(req, 'r') as fp: - # parse first line only - next(parse_requirements(fp.read())) - msg += " The argument you provided " + \ - "(%s) appears to be a" % (req) + \ - " requirements file. If that is the" + \ - " case, use the '-r' flag to install" + \ - " the packages specified within it." - except RequirementParseError: - logger.debug("Cannot parse '%s' as requirements \ - file" % (req), exc_info=1) - else: - msg += " File '%s' does not exist." % (req) - return msg - - -def _is_list_of_str(obj): - return ( - isinstance(obj, list) and - all(isinstance(item, six.string_types) for item in obj) - ) diff --git a/pipenv/patched/notpip/_internal/req/req_set.py b/pipenv/patched/notpip/_internal/req/req_set.py index 2c54c85a..a65851ff 100644 --- a/pipenv/patched/notpip/_internal/req/req_set.py +++ b/pipenv/patched/notpip/_internal/req/req_set.py @@ -12,19 +12,22 @@ logger = logging.getLogger(__name__) class RequirementSet(object): - def __init__(self, require_hashes=False, ignore_compatibility=True): + def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): """Create a RequirementSet. """ self.requirements = OrderedDict() self.require_hashes = require_hashes + self.check_supported_wheels = check_supported_wheels + if ignore_compatibility: + self.check_supported_wheels = False + self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False # Mapping of alias: real_name self.requirement_aliases = {} self.unnamed_requirements = [] self.successfully_downloaded = [] self.reqs_to_cleanup = [] - self.ignore_compatibility = ignore_compatibility def __str__(self): reqs = [req for req in self.requirements.values() @@ -56,17 +59,22 @@ class RequirementSet(object): requirement is applicable and has just been added. """ name = install_req.name + + # If the markers do not match, ignore this requirement. if not install_req.match_markers(extras_requested): - logger.info("Ignoring %s: markers '%s' don't match your " - "environment", install_req.name, - install_req.markers) + logger.info( + "Ignoring %s: markers '%s' don't match your environment", + name, install_req.markers, + ) return [], None - # This check has to come after we filter requirements with the - # environment markers. + # If the wheel is not supported, raise an error. + # Should check this after filtering out based on environment markers to + # allow specifying different wheels based on the environment/OS, in a + # single requirements file. if install_req.link and install_req.link.is_wheel: wheel = Wheel(install_req.link.filename) - if not wheel.supported() and not self.ignore_compatibility: + if self.check_supported_wheels and not wheel.supported(): raise InstallationError( "%s is not a supported wheel on this platform." % wheel.filename @@ -78,59 +86,73 @@ class RequirementSet(object): "a non direct req should have a parent" ) + # Unnamed requirements are scanned again and the requirement won't be + # added as a dependency until after scanning. if not name: # url or path requirement w/o an egg fragment self.unnamed_requirements.append(install_req) return [install_req], None - else: - try: - existing_req = self.get_requirement(name) - except KeyError: - existing_req = None - if (parent_req_name is None and existing_req and not - existing_req.constraint and - existing_req.extras == install_req.extras and not - existing_req.req.specifier == install_req.req.specifier): - raise InstallationError( - 'Double requirement given: %s (already in %s, name=%r)' - % (install_req, existing_req, name)) - if not existing_req: - # Add requirement - self.requirements[name] = install_req - # FIXME: what about other normalizations? E.g., _ vs. -? - if name.lower() != name: - self.requirement_aliases[name.lower()] = name - result = [install_req] - else: - # Assume there's no need to scan, and that we've already - # encountered this for scanning. - result = [] - if not install_req.constraint and existing_req.constraint: - if (install_req.link and not (existing_req.link and - install_req.link.path == existing_req.link.path)): - self.reqs_to_cleanup.append(install_req) - raise InstallationError( - "Could not satisfy constraints for '%s': " - "installation from path or url cannot be " - "constrained to a version" % name, - ) - # If we're now installing a constraint, mark the existing - # object for real installation. - existing_req.constraint = False - existing_req.extras = tuple( - sorted(set(existing_req.extras).union( - set(install_req.extras)))) - logger.debug("Setting %s extras to: %s", - existing_req, existing_req.extras) - # And now we need to scan this. - result = [existing_req] - # Canonicalise to the already-added object for the backref - # check below. - install_req = existing_req - # We return install_req here to allow for the caller to add it to - # the dependency information for the parent package. - return result, install_req + try: + existing_req = self.get_requirement(name) + except KeyError: + existing_req = None + + has_conflicting_requirement = ( + parent_req_name is None and + existing_req and + not existing_req.constraint and + existing_req.extras == install_req.extras and + existing_req.req.specifier != install_req.req.specifier + ) + if has_conflicting_requirement: + raise InstallationError( + "Double requirement given: %s (already in %s, name=%r)" + % (install_req, existing_req, name) + ) + + # When no existing requirement exists, add the requirement as a + # dependency and it will be scanned again after. + if not existing_req: + self.requirements[name] = install_req + # FIXME: what about other normalizations? E.g., _ vs. -? + if name.lower() != name: + self.requirement_aliases[name.lower()] = name + # We'd want to rescan this requirements later + return [install_req], install_req + + # Assume there's no need to scan, and that we've already + # encountered this for scanning. + if install_req.constraint or not existing_req.constraint: + return [], existing_req + + does_not_satisfy_constraint = ( + install_req.link and + not ( + existing_req.link and + install_req.link.path == existing_req.link.path + ) + ) + if does_not_satisfy_constraint: + self.reqs_to_cleanup.append(install_req) + raise InstallationError( + "Could not satisfy constraints for '%s': " + "installation from path or url cannot be " + "constrained to a version" % name, + ) + # If we're now installing a constraint, mark the existing + # object for real installation. + existing_req.constraint = False + existing_req.extras = tuple(sorted( + set(existing_req.extras) | set(install_req.extras) + )) + logger.debug( + "Setting %s extras to: %s", + existing_req, existing_req.extras, + ) + # Return the existing requirement for addition to the parent and + # scanning again. + return [existing_req], existing_req def has_requirement(self, project_name): name = project_name.lower() @@ -152,7 +174,7 @@ class RequirementSet(object): return self.requirements[name] if name in self.requirement_aliases: return self.requirements[self.requirement_aliases[name]] - # raise KeyError("No project with the name %r" % project_name) + pass def cleanup_files(self): """Clean up files, remove builds.""" diff --git a/pipenv/patched/notpip/_internal/req/req_uninstall.py b/pipenv/patched/notpip/_internal/req/req_uninstall.py index 3ccd3265..4cd15d84 100644 --- a/pipenv/patched/notpip/_internal/req/req_uninstall.py +++ b/pipenv/patched/notpip/_internal/req/req_uninstall.py @@ -9,9 +9,9 @@ import sysconfig from pipenv.patched.notpip._vendor import pkg_resources -from pipenv.patched.notpip._internal.compat import WINDOWS, cache_from_source, uses_pycache from pipenv.patched.notpip._internal.exceptions import UninstallationError from pipenv.patched.notpip._internal.locations import bin_py, bin_user +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, cache_from_source, uses_pycache from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( FakeFile, ask, dist_in_usersite, dist_is_local, egg_link_path, is_local, @@ -120,6 +120,8 @@ def compress_for_output_listing(paths): folders.add(os.path.dirname(path)) files.add(path) + _normcased_files = set(map(os.path.normcase, files)) + folders = compact(folders) # This walks the tree using os.walk to not miss extra folders @@ -130,8 +132,9 @@ def compress_for_output_listing(paths): if fname.endswith(".pyc"): continue - file_ = os.path.normcase(os.path.join(dirpath, fname)) - if os.path.isfile(file_) and file_ not in files: + file_ = os.path.join(dirpath, fname) + if (os.path.isfile(file_) and + os.path.normcase(file_) not in _normcased_files): # We are skipping this file. Add it to the set. will_skip.add(file_) diff --git a/pipenv/patched/notpip/_internal/resolve.py b/pipenv/patched/notpip/_internal/resolve.py index 461a2bb8..b0d096f9 100644 --- a/pipenv/patched/notpip/_internal/resolve.py +++ b/pipenv/patched/notpip/_internal/resolve.py @@ -18,7 +18,7 @@ from pipenv.patched.notpip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors, UnsupportedPythonVersion, ) -from pipenv.patched.notpip._internal.req.req_install import InstallRequirement +from pipenv.patched.notpip._internal.req.constructors import install_req_from_req from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import dist_in_usersite, ensure_dir from pipenv.patched.notpip._internal.utils.packaging import check_dist_requires_python @@ -249,9 +249,6 @@ class Resolver(object): # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) - if ignore_requires_python or self.ignore_requires_python: - self.ignore_compatibility = True - if req_to_install.constraint or req_to_install.prepared: return [] @@ -267,7 +264,7 @@ class Resolver(object): try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: - if self.ignore_compatibility: + if self.ignore_requires_python or self.ignore_compatibility: logger.warning(err.args[0]) else: raise @@ -281,7 +278,7 @@ class Resolver(object): more_reqs = [] def add_req(subreq, extras_requested): - sub_install_req = InstallRequirement.from_req( + sub_install_req = install_req_from_req( str(subreq), req_to_install, isolated=self.isolated, @@ -303,10 +300,10 @@ class Resolver(object): # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): + # 'unnamed' requirements will get added here available_requested = sorted( set(dist.extras) & set(req_to_install.extras) ) - # 'unnamed' requirements will get added here req_to_install.is_direct = True requirement_set.add_requirement( req_to_install, parent_req_name=None, @@ -338,7 +335,7 @@ class Resolver(object): for available in available_requested: if hasattr(dist, '_DistInfoDistribution__dep_map'): for req in dist._DistInfoDistribution__dep_map[available]: - req = InstallRequirement.from_req( + req = install_req_from_req( str(req), req_to_install, isolated=self.isolated, diff --git a/pipenv/patched/notpip/_internal/utils/appdirs.py b/pipenv/patched/notpip/_internal/utils/appdirs.py index 291de7a9..e8e14526 100644 --- a/pipenv/patched/notpip/_internal/utils/appdirs.py +++ b/pipenv/patched/notpip/_internal/utils/appdirs.py @@ -9,7 +9,7 @@ import sys from pipenv.patched.notpip._vendor.six import PY2, text_type -from pipenv.patched.notpip._internal.compat import WINDOWS, expanduser +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser def user_cache_dir(appname): diff --git a/pipenv/patched/notpip/_internal/compat.py b/pipenv/patched/notpip/_internal/utils/compat.py similarity index 96% rename from pipenv/patched/notpip/_internal/compat.py rename to pipenv/patched/notpip/_internal/utils/compat.py index 6e51e32a..483bfdc8 100644 --- a/pipenv/patched/notpip/_internal/compat.py +++ b/pipenv/patched/notpip/_internal/utils/compat.py @@ -25,6 +25,7 @@ except ImportError: __all__ = [ "ipaddress", "uses_pycache", "console_to_str", "native_str", "get_path_uid", "stdlib_pkgs", "WINDOWS", "samefile", "get_terminal_size", + "get_extension_suffixes", ] @@ -160,6 +161,18 @@ def get_path_uid(path): return file_uid +if sys.version_info >= (3, 4): + from importlib.machinery import EXTENSION_SUFFIXES + + def get_extension_suffixes(): + return EXTENSION_SUFFIXES +else: + from imp import get_suffixes + + def get_extension_suffixes(): + return [suffix[0] for suffix in get_suffixes()] + + def expanduser(path): """ Expand ~ and ~user constructions. diff --git a/pipenv/patched/notpip/_internal/utils/filesystem.py b/pipenv/patched/notpip/_internal/utils/filesystem.py index 91976486..e8d6a2bb 100644 --- a/pipenv/patched/notpip/_internal/utils/filesystem.py +++ b/pipenv/patched/notpip/_internal/utils/filesystem.py @@ -1,7 +1,7 @@ import os import os.path -from pipenv.patched.notpip._internal.compat import get_path_uid +from pipenv.patched.notpip._internal.utils.compat import get_path_uid def check_path_owner(path): diff --git a/pipenv/patched/notpip/_internal/utils/logging.py b/pipenv/patched/notpip/_internal/utils/logging.py index 257a6234..576c4fa0 100644 --- a/pipenv/patched/notpip/_internal/utils/logging.py +++ b/pipenv/patched/notpip/_internal/utils/logging.py @@ -5,7 +5,7 @@ import logging import logging.handlers import os -from pipenv.patched.notpip._internal.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.misc import ensure_dir try: diff --git a/pipenv/patched/notpip/_internal/utils/misc.py b/pipenv/patched/notpip/_internal/utils/misc.py index e254f3d4..45e5204c 100644 --- a/pipenv/patched/notpip/_internal/utils/misc.py +++ b/pipenv/patched/notpip/_internal/utils/misc.py @@ -26,14 +26,14 @@ from pipenv.patched.notpip._vendor.six import PY2 from pipenv.patched.notpip._vendor.six.moves import input from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -from pipenv.patched.notpip._internal.compat import ( - WINDOWS, console_to_str, expanduser, stdlib_pkgs, -) from pipenv.patched.notpip._internal.exceptions import CommandError, InstallationError from pipenv.patched.notpip._internal.locations import ( running_under_virtualenv, site_packages, user_site, virtualenv_no_global, write_delete_marker_file, ) +from pipenv.patched.notpip._internal.utils.compat import ( + WINDOWS, console_to_str, expanduser, stdlib_pkgs, +) if PY2: from io import BytesIO as StringIO @@ -96,7 +96,7 @@ def get_prog(): try: prog = os.path.basename(sys.argv[0]) if prog in ('__main__.py', '-c'): - return "%s -m pip" % os.environ.get('PIP_PYTHON_PATH', sys.executable) + return "%s -m pip" % sys.executable else: return prog except (AttributeError, TypeError, IndexError): @@ -187,12 +187,16 @@ def format_size(bytes): def is_installable_dir(path): - """Return True if `path` is a directory containing a setup.py file.""" + """Is path is a directory containing setup.py or pyproject.toml? + """ if not os.path.isdir(path): return False setup_py = os.path.join(path, 'setup.py') if os.path.isfile(setup_py): return True + pyproject_toml = os.path.join(path, 'pyproject.toml') + if os.path.isfile(pyproject_toml): + return True return False @@ -852,6 +856,44 @@ def enum(*sequential, **named): return type('Enum', (), enums) +def make_vcs_requirement_url(repo_url, rev, egg_project_name, subdir=None): + """ + Return the URL for a VCS requirement. + + Args: + repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). + """ + req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) + if subdir: + req += '&subdirectory={}'.format(subdir) + + return req + + +def split_auth_from_netloc(netloc): + """ + Parse out and remove the auth information from a netloc. + + Returns: (netloc, (username, password)). + """ + if '@' not in netloc: + return netloc, (None, None) + + # Split from the right because that's how urllib.parse.urlsplit() + # behaves if more than one @ is present (which can be checked using + # the password attribute of urlsplit()'s return value). + auth, netloc = netloc.rsplit('@', 1) + if ':' in auth: + # Split from the left because that's how urllib.parse.urlsplit() + # behaves if more than one : is present (which again can be checked + # using the password attribute of the return value) + user_pass = tuple(auth.split(':', 1)) + else: + user_pass = auth, None + + return netloc, user_pass + + def remove_auth_from_url(url): # Return a copy of url with 'username:password@' removed. # username/pass params are passed to subversion through flags @@ -859,12 +901,11 @@ def remove_auth_from_url(url): # parsed url purl = urllib_parse.urlsplit(url) - stripped_netloc = \ - purl.netloc.split('@')[-1] + netloc, user_pass = split_auth_from_netloc(purl.netloc) # stripped url url_pieces = ( - purl.scheme, stripped_netloc, purl.path, purl.query, purl.fragment + purl.scheme, netloc, purl.path, purl.query, purl.fragment ) surl = urllib_parse.urlunsplit(url_pieces) return surl diff --git a/pipenv/patched/notpip/_internal/utils/models.py b/pipenv/patched/notpip/_internal/utils/models.py new file mode 100644 index 00000000..d5cb80a7 --- /dev/null +++ b/pipenv/patched/notpip/_internal/utils/models.py @@ -0,0 +1,40 @@ +"""Utilities for defining models +""" + +import operator + + +class KeyBasedCompareMixin(object): + """Provides comparision capabilities that is based on a key + """ + + def __init__(self, key, defining_class): + self._compare_key = key + self._defining_class = defining_class + + def __hash__(self): + return hash(self._compare_key) + + def __lt__(self, other): + return self._compare(other, operator.__lt__) + + def __le__(self, other): + return self._compare(other, operator.__le__) + + def __gt__(self, other): + return self._compare(other, operator.__gt__) + + def __ge__(self, other): + return self._compare(other, operator.__ge__) + + def __eq__(self, other): + return self._compare(other, operator.__eq__) + + def __ne__(self, other): + return self._compare(other, operator.__ne__) + + def _compare(self, other, method): + if not isinstance(other, self._defining_class): + return NotImplemented + + return method(self._compare_key, other._compare_key) diff --git a/pipenv/patched/notpip/_internal/utils/outdated.py b/pipenv/patched/notpip/_internal/utils/outdated.py index 6133e6fd..f8b1fe04 100644 --- a/pipenv/patched/notpip/_internal/utils/outdated.py +++ b/pipenv/patched/notpip/_internal/utils/outdated.py @@ -9,8 +9,8 @@ import sys from pipenv.patched.notpip._vendor import lockfile, pkg_resources from pipenv.patched.notpip._vendor.packaging import version as packaging_version -from pipenv.patched.notpip._internal.compat import WINDOWS from pipenv.patched.notpip._internal.index import PackageFinder +from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner from pipenv.patched.notpip._internal.utils.misc import ensure_dir, get_installed_version @@ -22,16 +22,25 @@ logger = logging.getLogger(__name__) class SelfCheckState(object): def __init__(self, cache_dir): - self.statefile_path = os.path.join(cache_dir, "selfcheck.json") + self.state = {} + self.statefile_path = None - # Load the existing state - try: - with open(self.statefile_path) as statefile: - self.state = json.load(statefile)[sys.prefix] - except (IOError, ValueError, KeyError): - self.state = {} + # Try to load the existing state + if cache_dir: + self.statefile_path = os.path.join(cache_dir, "selfcheck.json") + try: + with open(self.statefile_path) as statefile: + self.state = json.load(statefile)[sys.prefix] + except (IOError, ValueError, KeyError): + # Explicitly suppressing exceptions, since we don't want to + # error out if the cache file is invalid. + pass def save(self, pypi_version, current_time): + # If we do not have a path to cache in, don't bother saving. + if not self.statefile_path: + return + # Check to make sure that we own the directory if not check_path_owner(os.path.dirname(self.statefile_path)): return diff --git a/pipenv/patched/notpip/_internal/utils/packaging.py b/pipenv/patched/notpip/_internal/utils/packaging.py index 13547743..d1e8ecaa 100644 --- a/pipenv/patched/notpip/_internal/utils/packaging.py +++ b/pipenv/patched/notpip/_internal/utils/packaging.py @@ -8,6 +8,7 @@ from pipenv.patched.notpip._vendor import pkg_resources from pipenv.patched.notpip._vendor.packaging import specifiers, version from pipenv.patched.notpip._internal import exceptions +from pipenv.patched.notpip._internal.utils.misc import display_path logger = logging.getLogger(__name__) @@ -35,22 +36,31 @@ def check_requires_python(requires_python): def get_metadata(dist): if (isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata('METADATA')): - return dist.get_metadata('METADATA') + metadata = dist.get_metadata('METADATA') elif dist.has_metadata('PKG-INFO'): - return dist.get_metadata('PKG-INFO') + metadata = dist.get_metadata('PKG-INFO') + else: + logger.warning("No metadata found in %s", display_path(dist.location)) + metadata = '' + + feed_parser = FeedParser() + feed_parser.feed(metadata) + return feed_parser.close() def check_dist_requires_python(dist, absorb=True): - metadata = get_metadata(dist) - feed_parser = FeedParser() - feed_parser.feed(metadata) - pkg_info_dict = feed_parser.close() + pkg_info_dict = get_metadata(dist) requires_python = pkg_info_dict.get('Requires-Python') - if not absorb: + if absorb: return requires_python try: if not check_requires_python(requires_python): - return requires_python + raise exceptions.UnsupportedPythonVersion( + "%s requires Python '%s' but the running Python is %s" % ( + dist.project_name, + requires_python, + '.'.join(map(str, sys.version_info[:3])),) + ) except specifiers.InvalidSpecifier as e: logger.warning( "Package %s has an invalid Requires-Python entry %s - %s", diff --git a/pipenv/patched/notpip/_internal/utils/temp_dir.py b/pipenv/patched/notpip/_internal/utils/temp_dir.py index ba472b0d..893dc975 100644 --- a/pipenv/patched/notpip/_internal/utils/temp_dir.py +++ b/pipenv/patched/notpip/_internal/utils/temp_dir.py @@ -3,8 +3,10 @@ from __future__ import absolute_import import logging import os.path import tempfile +import warnings from pipenv.patched.notpip._internal.utils.misc import rmtree +from pipenv.vendor.vistir.compat import finalize, ResourceWarning logger = logging.getLogger(__name__) @@ -45,6 +47,20 @@ class TempDirectory(object): self.path = path self.delete = delete self.kind = kind + self._finalizer = None + if path: + self._register_finalizer() + + def _register_finalizer(self): + if self.delete and self.path: + self._finalizer = finalize( + self, + self._cleanup, + self.path, + warn_message=None + ) + else: + self._finalizer = None def __repr__(self): return "<{} {!r}>".format(self.__class__.__name__, self.path) @@ -72,11 +88,27 @@ class TempDirectory(object): self.path = os.path.realpath( tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) ) + self._register_finalizer() logger.debug("Created temporary directory: {}".format(self.path)) + @classmethod + def _cleanup(cls, name, warn_message=None): + try: + rmtree(name) + except OSError: + pass + else: + if warn_message: + warnings.warn(warn_message, ResourceWarning) + def cleanup(self): """Remove the temporary directory created and reset state """ - if self.path is not None and os.path.exists(self.path): - rmtree(self.path) - self.path = None + if getattr(self._finalizer, "detach", None) and self._finalizer.detach(): + if os.path.exists(self.path): + try: + rmtree(self.path) + except OSError: + pass + else: + self.path = None diff --git a/pipenv/patched/notpip/_internal/utils/ui.py b/pipenv/patched/notpip/_internal/utils/ui.py index b96863cc..6eebd17d 100644 --- a/pipenv/patched/notpip/_internal/utils/ui.py +++ b/pipenv/patched/notpip/_internal/utils/ui.py @@ -15,7 +15,7 @@ from pipenv.patched.notpip._vendor.progress.bar import ( from pipenv.patched.notpip._vendor.progress.helpers import HIDE_CURSOR, SHOW_CURSOR, WritelnMixin from pipenv.patched.notpip._vendor.progress.spinner import Spinner -from pipenv.patched.notpip._internal.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.logging import get_indentation from pipenv.patched.notpip._internal.utils.misc import format_size from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/pipenv/patched/notpip/_internal/vcs/__init__.py b/pipenv/patched/notpip/_internal/vcs/__init__.py index 146f2829..5aeac633 100644 --- a/pipenv/patched/notpip/_internal/vcs/__init__.py +++ b/pipenv/patched/notpip/_internal/vcs/__init__.py @@ -17,7 +17,7 @@ from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Dict, Optional, Tuple # noqa: F401 - from pipenv.patched.notpip._internal.basecommand import Command # noqa: F401 + from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 __all__ = ['vcs', 'get_src_requirement'] @@ -200,12 +200,6 @@ class VersionControl(object): drive, tail = os.path.splitdrive(repo) return repo.startswith(os.path.sep) or drive - # See issue #1083 for why this method was introduced: - # https://github.com/pypa/pip/issues/1083 - def translate_egg_surname(self, surname): - # For example, Django has branches of the form "stable/1.7.x". - return surname.replace('/', '_') - def export(self, location): """ Export the repository at the url to the destination location @@ -213,51 +207,65 @@ class VersionControl(object): """ raise NotImplementedError - def get_url_rev(self, url): + def get_netloc_and_auth(self, netloc, scheme): """ - Returns the correct repository URL and revision by parsing the given - repository URL + Parse the repository URL's netloc, and return the new netloc to use + along with auth information. + + Args: + netloc: the original repository URL netloc. + scheme: the repository URL's scheme without the vcs prefix. + + This is mainly for the Subversion class to override, so that auth + information can be provided via the --username and --password options + instead of through the URL. For other subclasses like Git without + such an option, auth information must stay in the URL. + + Returns: (netloc, (username, password)). + """ + return netloc, (None, None) + + def get_url_rev_and_auth(self, url): + """ + Parse the repository URL to use, and return the URL, revision, + and auth info to use. + + Returns: (url, rev, (username, password)). """ - error_message = ( - "Sorry, '%s' is a malformed VCS url. " - "The format is +://, " - "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp" - ) - assert '+' in url, error_message % url - url = url.split('+', 1)[1] scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) + if '+' not in scheme: + raise ValueError( + "Sorry, {!r} is a malformed VCS url. " + "The format is +://, " + "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) + ) + # Remove the vcs prefix. + scheme = scheme.split('+', 1)[1] + netloc, user_pass = self.get_netloc_and_auth(netloc, scheme) rev = None if '@' in path: path, rev = path.rsplit('@', 1) url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev + return url, rev, user_pass - def get_url_rev_args(self, url): + def make_rev_args(self, username, password): """ - Return the URL and RevOptions "extra arguments" to use in obtain(), - as a tuple (url, extra_args). + Return the RevOptions "extra arguments" to use in obtain(). """ - return url, [] + return [] def get_url_rev_options(self, url): """ Return the URL and RevOptions object to use in obtain() and in some cases export(), as a tuple (url, rev_options). """ - url, rev = self.get_url_rev(url) - url, extra_args = self.get_url_rev_args(url) + url, rev, user_pass = self.get_url_rev_and_auth(url) + username, password = user_pass + extra_args = self.make_rev_args(username, password) rev_options = self.make_rev_options(rev, extra_args=extra_args) return url, rev_options - def get_info(self, location): - """ - Returns (url, revision), where both are strings - """ - assert not location.rstrip('/').endswith(self.dirname), \ - 'Bad directory: %s' % location - return self.get_url(location), self.get_revision(location) - def normalize_url(self, url): """ Normalize a URL for comparison by unquoting it and removing any @@ -291,7 +299,7 @@ class VersionControl(object): """ raise NotImplementedError - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): """ Update an already-existing repo to the given ``rev_options``. @@ -341,7 +349,7 @@ class VersionControl(object): self.repo_name, rev_display, ) - self.update(dest, rev_options) + self.update(dest, url, rev_options) else: logger.info('Skipping because already up-to-date.') return @@ -421,8 +429,6 @@ class VersionControl(object): def get_url(self, location): """ Return the url used at location - - This is used in get_info() and obtain(). """ raise NotImplementedError diff --git a/pipenv/patched/notpip/_internal/vcs/bazaar.py b/pipenv/patched/notpip/_internal/vcs/bazaar.py index b2664cd8..890448ed 100644 --- a/pipenv/patched/notpip/_internal/vcs/bazaar.py +++ b/pipenv/patched/notpip/_internal/vcs/bazaar.py @@ -6,7 +6,9 @@ import os from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import display_path, rmtree +from pipenv.patched.notpip._internal.utils.misc import ( + display_path, make_vcs_requirement_url, rmtree, +) from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.vcs import VersionControl, vcs @@ -62,16 +64,16 @@ class Bazaar(VersionControl): def switch(self, dest, url, rev_options): self.run_command(['switch', url], cwd=dest) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): cmd_args = ['pull', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def get_url_rev(self, url): + def get_url_rev_and_auth(self, url): # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it - url, rev = super(Bazaar, self).get_url_rev(url) + url, rev, user_pass = super(Bazaar, self).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'bzr+' + url - return url, rev + return url, rev, user_pass def get_url(self, location): urls = self.run_command(['info'], show_stdout=False, cwd=location) @@ -98,9 +100,9 @@ class Bazaar(VersionControl): return None if not repo.lower().startswith('bzr:'): repo = 'bzr+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] current_rev = self.get_revision(location) - return '%s@%s#egg=%s' % (repo, current_rev, egg_project_name) + egg_project_name = dist.egg_name().split('-', 1)[0] + return make_vcs_requirement_url(repo, current_rev, egg_project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/vcs/git.py b/pipenv/patched/notpip/_internal/vcs/git.py index ef2dd908..3db56144 100644 --- a/pipenv/patched/notpip/_internal/vcs/git.py +++ b/pipenv/patched/notpip/_internal/vcs/git.py @@ -8,9 +8,9 @@ from pipenv.patched.notpip._vendor.packaging.version import parse as parse_versi from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request -from pipenv.patched.notpip._internal.compat import samefile from pipenv.patched.notpip._internal.exceptions import BadCommand -from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.compat import samefile +from pipenv.patched.notpip._internal.utils.misc import display_path, make_vcs_requirement_url from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.vcs import VersionControl, vcs @@ -77,6 +77,20 @@ class Git(VersionControl): version = '.'.join(version.split('.')[:3]) return parse_version(version) + def get_branch(self, location): + """ + Return the current branch, or None if HEAD isn't at a branch + (e.g. detached HEAD). + """ + args = ['rev-parse', '--abbrev-ref', 'HEAD'] + output = self.run_command(args, show_stdout=False, cwd=location) + branch = output.strip() + + if branch == 'HEAD': + return None + + return branch + def export(self, location): """Export the Git repository at the url to the destination location""" if not location.endswith('/'): @@ -91,8 +105,8 @@ class Git(VersionControl): def get_revision_sha(self, dest, rev): """ - Return a commit hash for the given revision if it names a remote - branch or tag. Otherwise, return None. + Return (sha_or_none, is_branch), where sha_or_none is a commit hash + if the revision names a remote branch or tag, otherwise None. Args: dest: the repository directory. @@ -115,22 +129,30 @@ class Git(VersionControl): branch_ref = 'refs/remotes/origin/{}'.format(rev) tag_ref = 'refs/tags/{}'.format(rev) - return refs.get(branch_ref) or refs.get(tag_ref) + sha = refs.get(branch_ref) + if sha is not None: + return (sha, True) - def check_rev_options(self, dest, rev_options): - """Check the revision options before checkout. + sha = refs.get(tag_ref) - Returns a new RevOptions object for the SHA1 of the branch or tag - if found. + return (sha, False) + + def resolve_revision(self, dest, url, rev_options): + """ + Resolve a revision to a new RevOptions object with the SHA1 of the + branch, tag, or ref if found. Args: rev_options: a RevOptions object. """ rev = rev_options.arg_rev - sha = self.get_revision_sha(dest, rev) + sha, is_branch = self.get_revision_sha(dest, rev) if sha is not None: - return rev_options.make_new(sha) + rev_options = rev_options.make_new(sha) + rev_options.branch_name = rev if is_branch else None + + return rev_options # Do not show a warning for the common case of something that has # the form of a Git commit hash. @@ -139,6 +161,19 @@ class Git(VersionControl): "Did not find branch or tag '%s', assuming revision or ref.", rev, ) + + if not rev.startswith('refs/'): + return rev_options + + # If it looks like a ref, we have to fetch it explicitly. + self.run_command( + ['fetch', '-q', url] + rev_options.to_args(), + cwd=dest, + ) + # Change the revision to the SHA of the ref we fetched + sha = self.get_revision(dest, rev='FETCH_HEAD') + rev_options = rev_options.make_new(sha) + return rev_options def is_commit_id_equal(self, dest, name): @@ -164,20 +199,22 @@ class Git(VersionControl): if rev_options.rev: # Then a specific revision was requested. - rev_options = self.check_rev_options(dest, rev_options) - # Only do a checkout if the current commit id doesn't match - # the requested revision. - if not self.is_commit_id_equal(dest, rev_options.rev): - rev = rev_options.rev - # Only fetch the revision if it's a ref - if rev.startswith('refs/'): - self.run_command( - ['fetch', '-q', url] + rev_options.to_args(), - cwd=dest, - ) - # Change the revision to the SHA of the ref we fetched - rev = 'FETCH_HEAD' - self.run_command(['checkout', '-q', rev], cwd=dest) + rev_options = self.resolve_revision(dest, url, rev_options) + branch_name = getattr(rev_options, 'branch_name', None) + if branch_name is None: + # Only do a checkout if the current commit id doesn't match + # the requested revision. + if not self.is_commit_id_equal(dest, rev_options.rev): + cmd_args = ['checkout', '-q'] + rev_options.to_args() + self.run_command(cmd_args, cwd=dest) + elif self.get_branch(dest) != branch_name: + # Then a specific branch was requested, and that branch + # is not yet checked out. + track_branch = 'origin/{}'.format(branch_name) + cmd_args = [ + 'checkout', '-b', branch_name, '--track', track_branch, + ] + self.run_command(cmd_args, cwd=dest) #: repo may contain submodules self.update_submodules(dest) @@ -189,7 +226,7 @@ class Git(VersionControl): self.update_submodules(dest) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): # First fetch changes from the default remote if self.get_git_version() >= parse_version('1.9.0'): # fetch tags in addition to everything else @@ -197,7 +234,7 @@ class Git(VersionControl): else: self.run_command(['fetch', '-q'], cwd=dest) # Then reset to wanted revision (maybe even origin/master) - rev_options = self.check_rev_options(dest, rev_options) + rev_options = self.resolve_revision(dest, url, rev_options) cmd_args = ['reset', '--hard', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) #: update submodules @@ -218,9 +255,11 @@ class Git(VersionControl): url = found_remote.split(' ')[1] return url.strip() - def get_revision(self, location): + def get_revision(self, location, rev=None): + if rev is None: + rev = 'HEAD' current_rev = self.run_command( - ['rev-parse', 'HEAD'], show_stdout=False, cwd=location, + ['rev-parse', rev], show_stdout=False, cwd=location, ) return current_rev.strip() @@ -255,17 +294,15 @@ class Git(VersionControl): repo = self.get_url(location) if not repo.lower().startswith('git:'): repo = 'git+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None current_rev = self.get_revision(location) - req = '%s@%s#egg=%s' % (repo, current_rev, egg_project_name) - subdirectory = self._get_subdirectory(location) - if subdirectory: - req += '&subdirectory=' + subdirectory + egg_project_name = dist.egg_name().split('-', 1)[0] + subdir = self._get_subdirectory(location) + req = make_vcs_requirement_url(repo, current_rev, egg_project_name, + subdir=subdir) + return req - def get_url_rev(self, url): + def get_url_rev_and_auth(self, url): """ Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. That's required because although they use SSH they sometimes don't @@ -275,12 +312,12 @@ class Git(VersionControl): if '://' not in url: assert 'file:' not in url url = url.replace('git+', 'git+ssh://') - url, rev = super(Git, self).get_url_rev(url) + url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) url = url.replace('ssh://', '') else: - url, rev = super(Git, self).get_url_rev(url) + url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) - return url, rev + return url, rev, user_pass def update_submodules(self, location): if not os.path.exists(os.path.join(location, '.gitmodules')): diff --git a/pipenv/patched/notpip/_internal/vcs/mercurial.py b/pipenv/patched/notpip/_internal/vcs/mercurial.py index a143e765..d76d47f3 100644 --- a/pipenv/patched/notpip/_internal/vcs/mercurial.py +++ b/pipenv/patched/notpip/_internal/vcs/mercurial.py @@ -6,7 +6,7 @@ import os from pipenv.patched.notpip._vendor.six.moves import configparser from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.misc import display_path, make_vcs_requirement_url from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.vcs import VersionControl, vcs @@ -59,7 +59,7 @@ class Mercurial(VersionControl): cmd_args = ['update', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): self.run_command(['pull', '-q'], cwd=dest) cmd_args = ['update', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) @@ -88,11 +88,10 @@ class Mercurial(VersionControl): repo = self.get_url(location) if not repo.lower().startswith('hg:'): repo = 'hg+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None current_rev_hash = self.get_revision_hash(location) - return '%s@%s#egg=%s' % (repo, current_rev_hash, egg_project_name) + egg_project_name = dist.egg_name().split('-', 1)[0] + return make_vcs_requirement_url(repo, current_rev_hash, + egg_project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/vcs/subversion.py b/pipenv/patched/notpip/_internal/vcs/subversion.py index 5adbdaa3..f3c3db4d 100644 --- a/pipenv/patched/notpip/_internal/vcs/subversion.py +++ b/pipenv/patched/notpip/_internal/vcs/subversion.py @@ -4,17 +4,15 @@ import logging import os import re -from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse - -from pipenv.patched.notpip._internal.index import Link +from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.logging import indent_log -from pipenv.patched.notpip._internal.utils.misc import display_path, remove_auth_from_url, rmtree +from pipenv.patched.notpip._internal.utils.misc import ( + display_path, make_vcs_requirement_url, rmtree, split_auth_from_netloc, +) from pipenv.patched.notpip._internal.vcs import VersionControl, vcs _svn_xml_url_re = re.compile('url="([^"]+)"') _svn_rev_re = re.compile(r'committed-rev="(\d+)"') -_svn_url_re = re.compile(r'URL: (.+)') -_svn_revision_re = re.compile(r'Revision: (.+)') _svn_info_xml_rev_re = re.compile(r'\s*revision="(\d+)"') _svn_info_xml_url_re = re.compile(r'(.*)') @@ -31,34 +29,6 @@ class Subversion(VersionControl): def get_base_rev_args(self, rev): return ['-r', rev] - def get_info(self, location): - """Returns (url, revision), where both are strings""" - assert not location.rstrip('/').endswith(self.dirname), \ - 'Bad directory: %s' % location - output = self.run_command( - ['info', location], - show_stdout=False, - extra_environ={'LANG': 'C'}, - ) - match = _svn_url_re.search(output) - if not match: - logger.warning( - 'Cannot determine URL of svn checkout %s', - display_path(location), - ) - logger.debug('Output that cannot be parsed: \n%s', output) - return None, None - url = match.group(1).strip() - match = _svn_revision_re.search(output) - if not match: - logger.warning( - 'Cannot determine revision of svn checkout %s', - display_path(location), - ) - logger.debug('Output that cannot be parsed: \n%s', output) - return url, None - return url, match.group(1) - def export(self, location): """Export the svn repository at the url to the destination location""" url, rev_options = self.get_url_rev_options(self.url) @@ -87,7 +57,7 @@ class Subversion(VersionControl): cmd_args = ['switch'] + rev_options.to_args() + [url, dest] self.run_command(cmd_args) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): cmd_args = ['update'] + rev_options.to_args() + [dest] self.run_command(cmd_args) @@ -132,18 +102,34 @@ class Subversion(VersionControl): revision = max(revision, localrev) return revision - def get_url_rev(self, url): + def get_netloc_and_auth(self, netloc, scheme): + """ + This override allows the auth information to be passed to svn via the + --username and --password options instead of via the URL. + """ + if scheme == 'ssh': + # The --username and --password options can't be used for + # svn+ssh URLs, so keep the auth information in the URL. + return super(Subversion, self).get_netloc_and_auth( + netloc, scheme) + + return split_auth_from_netloc(netloc) + + def get_url_rev_and_auth(self, url): # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it - url, rev = super(Subversion, self).get_url_rev(url) + url, rev, user_pass = super(Subversion, self).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'svn+' + url - return url, rev + return url, rev, user_pass - def get_url_rev_args(self, url): - extra_args = get_rev_options_args(url) - url = remove_auth_from_url(url) + def make_rev_args(self, username, password): + extra_args = [] + if username: + extra_args += ['--username', username] + if password: + extra_args += ['--password', password] - return url, extra_args + return extra_args def get_url(self, location): # In cases where the source is in a subdirectory, not alongside @@ -213,42 +199,15 @@ class Subversion(VersionControl): repo = self.get_url(location) if repo is None: return None + repo = 'svn+' + repo + rev = self.get_revision(location) # FIXME: why not project name? egg_project_name = dist.egg_name().split('-', 1)[0] - rev = self.get_revision(location) - return 'svn+%s@%s#egg=%s' % (repo, rev, egg_project_name) + return make_vcs_requirement_url(repo, rev, egg_project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" return False -def get_rev_options_args(url): - """ - Return the extra arguments to pass to RevOptions. - """ - r = urllib_parse.urlsplit(url) - if hasattr(r, 'username'): - # >= Python-2.5 - username, password = r.username, r.password - else: - netloc = r[1] - if '@' in netloc: - auth = netloc.split('@')[0] - if ':' in auth: - username, password = auth.split(':', 1) - else: - username, password = auth, None - else: - username, password = None, None - - extra_args = [] - if username: - extra_args += ['--username', username] - if password: - extra_args += ['--password', password] - - return extra_args - - vcs.register(Subversion) diff --git a/pipenv/patched/notpip/_internal/wheel.py b/pipenv/patched/notpip/_internal/wheel.py index 14ec0014..6df5a3a3 100644 --- a/pipenv/patched/notpip/_internal/wheel.py +++ b/pipenv/patched/notpip/_internal/wheel.py @@ -167,7 +167,8 @@ def message_about_scripts_not_on_PATH(scripts): ] # If an executable sits with sys.executable, we don't warn for it. # This covers the case of venv invocations without activating the venv. - not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) + executable_loc = os.environ.get("PIP_PYTHON_PATH", sys.executable) + not_warn_dirs.append(os.path.normcase(os.path.dirname(executable_loc))) warn_for = { parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() if os.path.normcase(parent_dir) not in not_warn_dirs @@ -475,7 +476,7 @@ if __name__ == '__main__': if warn_script_location: msg = message_about_scripts_not_on_PATH(generated_console_scripts) if msg is not None: - logger.warn(msg) + logger.warning(msg) if len(gui) > 0: generated.extend( @@ -500,16 +501,19 @@ if __name__ == '__main__': with open_for_csv(temp_record, 'w+') as record_out: reader = csv.reader(record_in) writer = csv.writer(record_out) + outrows = [] for row in reader: row[0] = installed.pop(row[0], row[0]) if row[0] in changed: row[1], row[2] = rehash(row[0]) - writer.writerow(row) + outrows.append(tuple(row)) for f in generated: digest, length = rehash(f) - writer.writerow((normpath(f, lib_dir), digest, length)) + outrows.append((normpath(f, lib_dir), digest, length)) for f in installed: - writer.writerow((installed[f], '', '')) + outrows.append((installed[f], '', '')) + for row in sorted(outrows): + writer.writerow(row) shutil.move(temp_record, record) @@ -664,8 +668,9 @@ class WheelBuilder(object): # isolating. Currently, it breaks Python in virtualenvs, because it # relies on site.py to find parts of the standard library outside the # virtualenv. + executable_loc = os.environ.get('PIP_PYTHON_PATH', sys.executable) return [ - os.environ.get('PIP_PYTHON_PATH', sys.executable), '-u', '-c', + executable_loc, '-u', '-c', SETUPTOOLS_SHIM % req.setup_py ] + list(self.global_options) @@ -710,6 +715,7 @@ class WheelBuilder(object): :return: True if all the wheels built correctly. """ from pipenv.patched.notpip._internal import index + from pipenv.patched.notpip._internal.models.link import Link building_is_possible = self._wheel_dir or ( autobuilding and self.wheel_cache.cache_dir @@ -717,6 +723,7 @@ class WheelBuilder(object): assert building_is_possible buildset = [] + format_control = self.finder.format_control for req in requirements: if req.constraint: continue @@ -740,8 +747,7 @@ class WheelBuilder(object): if index.egg_info_matches(base, None, link) is None: # E.g. local directory. Build wheel just for this run. ephem_cache = True - if "binary" not in index.fmt_ctl_formats( - self.finder.format_control, + if "binary" not in format_control.get_allowed_formats( canonicalize_name(req.name)): logger.info( "Skipping bdist_wheel for %s, due to binaries " @@ -802,7 +808,7 @@ class WheelBuilder(object): self.preparer.build_dir ) # Update the link for this. - req.link = index.Link(path_to_url(wheel_file)) + req.link = Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_url( diff --git a/pipenv/patched/notpip/_vendor/certifi/__init__.py b/pipenv/patched/notpip/_vendor/certifi/__init__.py index 0c4963ef..aa329fbb 100644 --- a/pipenv/patched/notpip/_vendor/certifi/__init__.py +++ b/pipenv/patched/notpip/_vendor/certifi/__init__.py @@ -1,3 +1,3 @@ from .core import where, old_where -__version__ = "2018.04.16" +__version__ = "2018.08.24" diff --git a/pipenv/patched/notpip/_vendor/certifi/__main__.py b/pipenv/patched/notpip/_vendor/certifi/__main__.py index 5f1da0dd..983ed0f9 100644 --- a/pipenv/patched/notpip/_vendor/certifi/__main__.py +++ b/pipenv/patched/notpip/_vendor/certifi/__main__.py @@ -1,2 +1,2 @@ -from certifi import where +from pipenv.patched.notpip._vendor.certifi import where print(where()) diff --git a/pipenv/patched/notpip/_vendor/certifi/cacert.pem b/pipenv/patched/notpip/_vendor/certifi/cacert.pem index 2713f541..85de024e 100644 --- a/pipenv/patched/notpip/_vendor/certifi/cacert.pem +++ b/pipenv/patched/notpip/_vendor/certifi/cacert.pem @@ -3692,169 +3692,6 @@ lSTAGiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7Sof TUwJCA3sS61kFyjndc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR -----END CERTIFICATE----- -# Issuer: CN=Certplus Root CA G1 O=Certplus -# Subject: CN=Certplus Root CA G1 O=Certplus -# Label: "Certplus Root CA G1" -# Serial: 1491911565779898356709731176965615564637713 -# MD5 Fingerprint: 7f:09:9c:f7:d9:b9:5c:69:69:56:d5:37:3e:14:0d:42 -# SHA1 Fingerprint: 22:fd:d0:b7:fd:a2:4e:0d:ac:49:2c:a0:ac:a6:7b:6a:1f:e3:f7:66 -# SHA256 Fingerprint: 15:2a:40:2b:fc:df:2c:d5:48:05:4d:22:75:b3:9c:7f:ca:3e:c0:97:80:78:b0:f0:ea:76:e5:61:a6:c7:43:3e ------BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgISESBVg+QtPlRWhS2DN7cs3EYRMA0GCSqGSIb3DQEBDQUA -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBa -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB -ANpQh7bauKk+nWT6VjOaVj0W5QOVsjQcmm1iBdTYj+eJZJ+622SLZOZ5KmHNr49a -iZFluVj8tANfkT8tEBXgfs+8/H9DZ6itXjYj2JizTfNDnjl8KvzsiNWI7nC9hRYt -6kuJPKNxQv4c/dMcLRC4hlTqQ7jbxofaqK6AJc96Jh2qkbBIb6613p7Y1/oA/caP -0FG7Yn2ksYyy/yARujVjBYZHYEMzkPZHogNPlk2dT8Hq6pyi/jQu3rfKG3akt62f -6ajUeD94/vI4CTYd0hYCyOwqaK/1jpTvLRN6HkJKHRUxrgwEV/xhc/MxVoYxgKDE -EW4wduOU8F8ExKyHcomYxZ3MVwia9Az8fXoFOvpHgDm2z4QTd28n6v+WZxcIbekN -1iNQMLAVdBM+5S//Ds3EC0pd8NgAM0lm66EYfFkuPSi5YXHLtaW6uOrc4nBvCGrc -h2c0798wct3zyT8j/zXhviEpIDCB5BmlIOklynMxdCm+4kLV87ImZsdo/Rmz5yCT -mehd4F6H50boJZwKKSTUzViGUkAksnsPmBIgJPaQbEfIDbsYIC7Z/fyL8inqh3SV -4EJQeIQEQWGw9CEjjy3LKCHyamz0GqbFFLQ3ZU+V/YDI+HLlJWvEYLF7bY5KinPO -WftwenMGE9nTdDckQQoRb5fc5+R+ob0V8rqHDz1oihYHAgMBAAGjYzBhMA4GA1Ud -DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSowcCbkahDFXxd -Bie0KlHYlwuBsTAfBgNVHSMEGDAWgBSowcCbkahDFXxdBie0KlHYlwuBsTANBgkq -hkiG9w0BAQ0FAAOCAgEAnFZvAX7RvUz1isbwJh/k4DgYzDLDKTudQSk0YcbX8ACh -66Ryj5QXvBMsdbRX7gp8CXrc1cqh0DQT+Hern+X+2B50ioUHj3/MeXrKls3N/U/7 -/SMNkPX0XtPGYX2eEeAC7gkE2Qfdpoq3DIMku4NQkv5gdRE+2J2winq14J2by5BS -S7CTKtQ+FjPlnsZlFT5kOwQ/2wyPX1wdaR+v8+khjPPvl/aatxm2hHSco1S1cE5j -2FddUyGbQJJD+tZ3VTNPZNX70Cxqjm0lpu+F6ALEUz65noe8zDUa3qHpimOHZR4R -Kttjd5cUvpoUmRGywO6wT/gUITJDT5+rosuoD6o7BlXGEilXCNQ314cnrUlZp5Gr -RHpejXDbl85IULFzk/bwg2D5zfHhMf1bfHEhYxQUqq/F3pN+aLHsIqKqkHWetUNy -6mSjhEv9DKgma3GX7lZjZuhCVPnHHd/Qj1vfyDBviP4NxDMcU6ij/UgQ8uQKTuEV -V/xuZDDCVRHc6qnNSlSsKWNEz0pAoNZoWRsz+e86i9sgktxChL8Bq4fA1SCC28a5 -g4VCXA9DO2pJNdWY9BW/+mGBDAkgGNLQFwzLSABQ6XaCjGTXOqAHVcweMcDvOrRl -++O/QmueD6i9a5jc2NvLi6Td11n0bt3+qsOR0C5CB8AMTVPNJLFMWx5R9N/pkvo= ------END CERTIFICATE----- - -# Issuer: CN=Certplus Root CA G2 O=Certplus -# Subject: CN=Certplus Root CA G2 O=Certplus -# Label: "Certplus Root CA G2" -# Serial: 1492087096131536844209563509228951875861589 -# MD5 Fingerprint: a7:ee:c4:78:2d:1b:ee:2d:b9:29:ce:d6:a7:96:32:31 -# SHA1 Fingerprint: 4f:65:8e:1f:e9:06:d8:28:02:e9:54:47:41:c9:54:25:5d:69:cc:1a -# SHA256 Fingerprint: 6c:c0:50:41:e6:44:5e:74:69:6c:4c:fb:c9:f8:0f:54:3b:7e:ab:bb:44:b4:ce:6f:78:7c:6a:99:71:c4:2f:17 ------BEGIN CERTIFICATE----- -MIICHDCCAaKgAwIBAgISESDZkc6uo+jF5//pAq/Pc7xVMAoGCCqGSM49BAMDMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBaMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjB2MBAGByqGSM49AgEGBSuBBAAiA2IABM0PW1aC3/BFGtat -93nwHcmsltaeTpwftEIRyoa/bfuFo8XlGVzX7qY/aWfYeOKmycTbLXku54uNAm8x -Ik0G42ByRZ0OQneezs/lf4WbGOT8zC5y0xaTTsqZY1yhBSpsBqNjMGEwDgYDVR0P -AQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNqDYwJ5jtpMxjwj -FNiPwyCrKGBZMB8GA1UdIwQYMBaAFNqDYwJ5jtpMxjwjFNiPwyCrKGBZMAoGCCqG -SM49BAMDA2gAMGUCMHD+sAvZ94OX7PNVHdTcswYO/jOYnYs5kGuUIe22113WTNch -p+e/IQ8rzfcq3IUHnQIxAIYUFuXcsGXCwI4Un78kFmjlvPl5adytRSv3tjFzzAal -U5ORGpOucGpnutee5WEaXw== ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G1 O=OpenTrust -# Subject: CN=OpenTrust Root CA G1 O=OpenTrust -# Label: "OpenTrust Root CA G1" -# Serial: 1492036577811947013770400127034825178844775 -# MD5 Fingerprint: 76:00:cc:81:29:cd:55:5e:88:6a:7a:2e:f7:4d:39:da -# SHA1 Fingerprint: 79:91:e8:34:f7:e2:ee:dd:08:95:01:52:e9:55:2d:14:e9:58:d5:7e -# SHA256 Fingerprint: 56:c7:71:28:d9:8c:18:d9:1b:4c:fd:ff:bc:25:ee:91:03:d4:75:8e:a2:ab:ad:82:6a:90:f3:45:7d:46:0e:b4 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESCzkFU5fX82bWTCp59rY45nMA0GCSqGSIb3DQEBCwUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcxMB4XDTE0MDUyNjA4NDU1MFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQD4eUbalsUwXopxAy1wpLuwxQjczeY1wICkES3d5oeuXT2R0odsN7faYp6b -wiTXj/HbpqbfRm9RpnHLPhsxZ2L3EVs0J9V5ToybWL0iEA1cJwzdMOWo010hOHQX -/uMftk87ay3bfWAfjH1MBcLrARYVmBSO0ZB3Ij/swjm4eTrwSSTilZHcYTSSjFR0 -77F9jAHiOH3BX2pfJLKOYheteSCtqx234LSWSE9mQxAGFiQD4eCcjsZGT44ameGP -uY4zbGneWK2gDqdkVBFpRGZPTBKnjix9xNRbxQA0MMHZmf4yzgeEtE7NCv82TWLx -p2NX5Ntqp66/K7nJ5rInieV+mhxNaMbBGN4zK1FGSxyO9z0M+Yo0FMT7MzUj8czx -Kselu7Cizv5Ta01BG2Yospb6p64KTrk5M0ScdMGTHPjgniQlQ/GbI4Kq3ywgsNw2 -TgOzfALU5nsaqocTvz6hdLubDuHAk5/XpGbKuxs74zD0M1mKB3IDVedzagMxbm+W -G+Oin6+Sx+31QrclTDsTBM8clq8cIqPQqwWyTBIjUtz9GVsnnB47ev1CI9sjgBPw -vFEVVJSmdz7QdFG9URQIOTfLHzSpMJ1ShC5VkLG631UAC9hWLbFJSXKAqWLXwPYY -EQRVzXR7z2FwefR7LFxckvzluFqrTJOVoSfupb7PcSNCupt2LQIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUl0YhVyE1 -2jZVx/PxN3DlCPaTKbYwHwYDVR0jBBgwFoAUl0YhVyE12jZVx/PxN3DlCPaTKbYw -DQYJKoZIhvcNAQELBQADggIBAB3dAmB84DWn5ph76kTOZ0BP8pNuZtQ5iSas000E -PLuHIT839HEl2ku6q5aCgZG27dmxpGWX4m9kWaSW7mDKHyP7Rbr/jyTwyqkxf3kf -gLMtMrpkZ2CvuVnN35pJ06iCsfmYlIrM4LvgBBuZYLFGZdwIorJGnkSI6pN+VxbS -FXJfLkur1J1juONI5f6ELlgKn0Md/rcYkoZDSw6cMoYsYPXpSOqV7XAp8dUv/TW0 -V8/bhUiZucJvbI/NeJWsZCj9VrDDb8O+WVLhX4SPgPL0DTatdrOjteFkdjpY3H1P -XlZs5VVZV6Xf8YpmMIzUUmI4d7S+KNfKNsSbBfD4Fdvb8e80nR14SohWZ25g/4/I -i+GOvUKpMwpZQhISKvqxnUOOBZuZ2mKtVzazHbYNeS2WuOvyDEsMpZTGMKcmGS3t -TAZQMPH9WD25SxdfGbRqhFS0OE85og2WaMMolP3tLR9Ka0OWLpABEPs4poEL0L91 -09S5zvE/bw4cHjdx5RiHdRk/ULlepEU0rbDK5uUTdg8xFKmOLZTW1YVNcxVPS/Ky -Pu1svf0OnWZzsD2097+o4BGkxK51CUpjAEggpsadCwmKtODmzj7HPiY46SvepghJ -AwSQiumPv+i2tCqjI40cHLI5kqiPAlxAOXXUc0ECd97N4EOH1uS6SsNsEn/+KuYj -1oxx ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G2 O=OpenTrust -# Subject: CN=OpenTrust Root CA G2 O=OpenTrust -# Label: "OpenTrust Root CA G2" -# Serial: 1492012448042702096986875987676935573415441 -# MD5 Fingerprint: 57:24:b6:59:24:6b:ae:c8:fe:1c:0c:20:f2:c0:4e:eb -# SHA1 Fingerprint: 79:5f:88:60:c5:ab:7c:3d:92:e6:cb:f4:8d:e1:45:cd:11:ef:60:0b -# SHA256 Fingerprint: 27:99:58:29:fe:6a:75:15:c1:bf:e8:48:f9:c4:76:1d:b1:6c:22:59:29:25:7b:f4:0d:08:94:f2:9e:a8:ba:f2 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESChaRu/vbm9UpaPI+hIvyYRMA0GCSqGSIb3DQEBDQUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcyMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQDMtlelM5QQgTJT32F+D3Y5z1zCU3UdSXqWON2ic2rxb95eolq5cSG+Ntmh -/LzubKh8NBpxGuga2F8ORAbtp+Dz0mEL4DKiltE48MLaARf85KxP6O6JHnSrT78e -CbY2albz4e6WiWYkBuTNQjpK3eCasMSCRbP+yatcfD7J6xcvDH1urqWPyKwlCm/6 -1UWY0jUJ9gNDlP7ZvyCVeYCYitmJNbtRG6Q3ffyZO6v/v6wNj0OxmXsWEH4db0fE -FY8ElggGQgT4hNYdvJGmQr5J1WqIP7wtUdGejeBSzFfdNTVY27SPJIjki9/ca1TS -gSuyzpJLHB9G+h3Ykst2Z7UJmQnlrBcUVXDGPKBWCgOz3GIZ38i1MH/1PCZ1Eb3X -G7OHngevZXHloM8apwkQHZOJZlvoPGIytbU6bumFAYueQ4xncyhZW+vj3CzMpSZy -YhK05pyDRPZRpOLAeiRXyg6lPzq1O4vldu5w5pLeFlwoW5cZJ5L+epJUzpM5ChaH -vGOz9bGTXOBut9Dq+WIyiET7vycotjCVXRIouZW+j1MY5aIYFuJWpLIsEPUdN6b4 -t/bQWVyJ98LVtZR00dX+G7bw5tYee9I8y6jj9RjzIR9u701oBnstXW5DiabA+aC/ -gh7PU3+06yzbXfZqfUAkBXKJOAGTy3HCOV0GEfZvePg3DTmEJwIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUajn6QiL3 -5okATV59M4PLuG53hq8wHwYDVR0jBBgwFoAUajn6QiL35okATV59M4PLuG53hq8w -DQYJKoZIhvcNAQENBQADggIBAJjLq0A85TMCl38th6aP1F5Kr7ge57tx+4BkJamz -Gj5oXScmp7oq4fBXgwpkTx4idBvpkF/wrM//T2h6OKQQbA2xx6R3gBi2oihEdqc0 -nXGEL8pZ0keImUEiyTCYYW49qKgFbdEfwFFEVn8nNQLdXpgKQuswv42hm1GqO+qT -RmTFAHneIWv2V6CG1wZy7HBGS4tz3aAhdT7cHcCP009zHIXZ/n9iyJVvttN7jLpT -wm+bREx50B1ws9efAvSyB7DH5fitIw6mVskpEndI2S9G/Tvw/HRwkqWOOAgfZDC2 -t0v7NqwQjqBSM2OdAzVWxWm9xiNaJ5T2pBL4LTM8oValX9YZ6e18CL13zSdkzJTa -TkZQh+D5wVOAHrut+0dSixv9ovneDiK3PTNZbNTe9ZUGMg1RGUFcPk8G97krgCf2 -o6p6fAbhQ8MTOWIaNr3gKC6UAuQpLmBVrkA9sHSSXvAgZJY/X0VdiLWK2gKgW0VU -3jg9CcCoSmVGFvyqv1ROTVu+OEO3KMqLM6oaJbolXCkvW0pujOotnCr2BXbgd5eA -iN1nE28daCSLT7d0geX0YJ96Vdc+N9oWaz53rK4YcJUIeSkDiv7BO7M/Gg+kO14f -WKGVyasvc0rQLW6aWQ9VGHgtPFGml4vmu7JwqkwR3v98KzfUetF3NI/n+UL3PIEM -S1IK ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G3 O=OpenTrust -# Subject: CN=OpenTrust Root CA G3 O=OpenTrust -# Label: "OpenTrust Root CA G3" -# Serial: 1492104908271485653071219941864171170455615 -# MD5 Fingerprint: 21:37:b4:17:16:92:7b:67:46:70:a9:96:d7:a8:13:24 -# SHA1 Fingerprint: 6e:26:64:f3:56:bf:34:55:bf:d1:93:3f:7c:01:de:d8:13:da:8a:a6 -# SHA256 Fingerprint: b7:c3:62:31:70:6e:81:07:8c:36:7c:b8:96:19:8f:1e:32:08:dd:92:69:49:dd:8f:57:09:a4:10:f7:5b:62:92 ------BEGIN CERTIFICATE----- -MIICITCCAaagAwIBAgISESDm+Ez8JLC+BUCs2oMbNGA/MAoGCCqGSM49BAMDMEAx -CzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5U -cnVzdCBSb290IENBIEczMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAwMFow -QDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwUT3Bl -blRydXN0IFJvb3QgQ0EgRzMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARK7liuTcpm -3gY6oxH84Bjwbhy6LTAMidnW7ptzg6kjFYwvWYpa3RTqnVkrQ7cG7DK2uu5Bta1d -oYXM6h0UZqNnfkbilPPntlahFVmhTzeXuSIevRHr9LIfXsMUmuXZl5mjYzBhMA4G -A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRHd8MUi2I5 -DMlv4VBN0BBY3JWIbTAfBgNVHSMEGDAWgBRHd8MUi2I5DMlv4VBN0BBY3JWIbTAK -BggqhkjOPQQDAwNpADBmAjEAj6jcnboMBBf6Fek9LykBl7+BFjNAk2z8+e2AcG+q -j9uEwov1NcoG3GRvaBbhj5G5AjEA2Euly8LQCGzpGPta3U1fJAuwACEl74+nBCZx -4nxp5V2a+EEfOzmTk51V6s2N8fvB ------END CERTIFICATE----- - # Issuer: CN=ISRG Root X1 O=Internet Security Research Group # Subject: CN=ISRG Root X1 O=Internet Security Research Group # Label: "ISRG Root X1" @@ -4398,3 +4235,66 @@ MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUW8pe5d7SgarNqC1kUbbZcpuX ytRrJPOwPYdGWBrssd9v+1a6cGvHOMzosYxPD/fxZ3YOg9AeUY8CMD32IygmTMZg h5Mmm7I1HrrW9zzRHM76JTymGoEVW/MSD2zuZYrJh6j5B+BimoxcSg== -----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Label: "GlobalSign Root CA - R6" +# Serial: 1417766617973444989252670301619537 +# MD5 Fingerprint: 4f:dd:07:e4:d4:22:64:39:1e:0c:37:42:ea:d1:c6:ae +# SHA1 Fingerprint: 80:94:64:0e:b5:a7:a1:ca:11:9c:1f:dd:d5:9f:81:02:63:a7:fb:d1 +# SHA256 Fingerprint: 2c:ab:ea:fe:37:d0:6c:a2:2a:ba:73:91:c0:03:3d:25:98:29:52:c4:53:64:73:49:76:3a:3a:b5:ad:6c:cf:69 +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg +MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh +bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx +MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET +MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI +xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k +ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD +aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw +LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw +1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX +k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2 +SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h +bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n +WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY +rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce +MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu +bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN +nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt +Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61 +55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj +vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf +cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz +oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp +nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs +pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v +JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R +8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 +5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= +-----END CERTIFICATE----- + +# Issuer: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Subject: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Label: "OISTE WISeKey Global Root GC CA" +# Serial: 44084345621038548146064804565436152554 +# MD5 Fingerprint: a9:d6:b9:2d:2f:93:64:f8:a5:69:ca:91:e9:68:07:23 +# SHA1 Fingerprint: e0:11:84:5e:34:de:be:88:81:b9:9c:f6:16:26:d1:96:1f:c3:b9:31 +# SHA256 Fingerprint: 85:60:f9:1c:36:24:da:ba:95:70:b5:fe:a0:db:e3:6f:f1:1a:83:23:be:94:86:85:4f:b3:f3:4a:55:71:19:8d +-----BEGIN CERTIFICATE----- +MIICaTCCAe+gAwIBAgIQISpWDK7aDKtARb8roi066jAKBggqhkjOPQQDAzBtMQsw +CQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUgRm91 +bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9iYWwg +Um9vdCBHQyBDQTAeFw0xNzA1MDkwOTQ4MzRaFw00MjA1MDkwOTU4MzNaMG0xCzAJ +BgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBGb3Vu +ZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2JhbCBS +b290IEdDIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAETOlQwMYPchi82PG6s4ni +eUqjFqdrVCTbUf/q9Akkwwsin8tqJ4KBDdLArzHkdIJuyiXZjHWd8dvQmqJLIX4W +p2OQ0jnUsYd4XxiWD1AbNTcPasbc2RNNpI6QN+a9WzGRo1QwUjAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUSIcUrOPDnpBgOtfKie7T +rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV +57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg +Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 +-----END CERTIFICATE----- diff --git a/pipenv/patched/notpip/_vendor/packaging/__about__.py b/pipenv/patched/notpip/_vendor/packaging/__about__.py index 4255c5b5..21fc6ce3 100644 --- a/pipenv/patched/notpip/_vendor/packaging/__about__.py +++ b/pipenv/patched/notpip/_vendor/packaging/__about__.py @@ -12,10 +12,10 @@ __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "17.1" +__version__ = "18.0" __author__ = "Donald Stufft and individual contributors" __email__ = "donald@stufft.io" __license__ = "BSD or Apache License, Version 2.0" -__copyright__ = "Copyright 2014-2016 %s" % __author__ +__copyright__ = "Copyright 2014-2018 %s" % __author__ diff --git a/pipenv/patched/notpip/_vendor/packaging/requirements.py b/pipenv/patched/notpip/_vendor/packaging/requirements.py index 0da25914..5ec5d74a 100644 --- a/pipenv/patched/notpip/_vendor/packaging/requirements.py +++ b/pipenv/patched/notpip/_vendor/packaging/requirements.py @@ -92,16 +92,16 @@ class Requirement(object): try: req = REQUIREMENT.parseString(requirement_string) except ParseException as e: - raise InvalidRequirement( - "Invalid requirement, parse error at \"{0!r}\"".format( - requirement_string[e.loc:e.loc + 8])) + raise InvalidRequirement("Parse error at \"{0!r}\": {1}".format( + requirement_string[e.loc:e.loc + 8], e.msg + )) self.name = req.name if req.url: parsed_url = urlparse.urlparse(req.url) if not (parsed_url.scheme and parsed_url.netloc) or ( not parsed_url.scheme and not parsed_url.netloc): - raise InvalidRequirement("Invalid URL given") + raise InvalidRequirement("Invalid URL: {0}".format(req.url)) self.url = req.url else: self.url = None diff --git a/pipenv/patched/notpip/_vendor/packaging/specifiers.py b/pipenv/patched/notpip/_vendor/packaging/specifiers.py index 9b6353f0..4c798999 100644 --- a/pipenv/patched/notpip/_vendor/packaging/specifiers.py +++ b/pipenv/patched/notpip/_vendor/packaging/specifiers.py @@ -503,7 +503,7 @@ class Specifier(_IndividualSpecifier): return False # Ensure that we do not allow a local version of the version mentioned - # in the specifier, which is techincally greater than, to match. + # in the specifier, which is technically greater than, to match. if prospective.local is not None: if Version(prospective.base_version) == Version(spec.base_version): return False diff --git a/pipenv/vendor/pathlib2.LICENSE.rst b/pipenv/patched/notpip/_vendor/pep517/LICENSE similarity index 83% rename from pipenv/vendor/pathlib2.LICENSE.rst rename to pipenv/patched/notpip/_vendor/pep517/LICENSE index ddb51b8a..b0ae9dbc 100644 --- a/pipenv/vendor/pathlib2.LICENSE.rst +++ b/pipenv/patched/notpip/_vendor/pep517/LICENSE @@ -1,7 +1,6 @@ The MIT License (MIT) -Copyright (c) 2014 Matthias C. M. Troffaes -Copyright (c) 2012-2014 Antoine Pitrou and contributors +Copyright (c) 2017 Thomas Kluyver Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -10,14 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/patched/notpip/_vendor/pep517/__init__.py b/pipenv/patched/notpip/_vendor/pep517/__init__.py new file mode 100644 index 00000000..8beedea4 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/__init__.py @@ -0,0 +1,4 @@ +"""Wrappers to build Python packages using PEP 517 hooks +""" + +__version__ = '0.2' diff --git a/pipenv/patched/notpip/_vendor/pep517/_in_process.py b/pipenv/patched/notpip/_vendor/pep517/_in_process.py new file mode 100644 index 00000000..baa14d38 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/_in_process.py @@ -0,0 +1,182 @@ +"""This is invoked in a subprocess to call the build backend hooks. + +It expects: +- Command line args: hook_name, control_dir +- Environment variable: PEP517_BUILD_BACKEND=entry.point:spec +- control_dir/input.json: + - {"kwargs": {...}} + +Results: +- control_dir/output.json + - {"return_val": ...} +""" +from glob import glob +from importlib import import_module +import os +from os.path import join as pjoin +import re +import shutil +import sys + +# This is run as a script, not a module, so it can't do a relative import +import compat + +def _build_backend(): + """Find and load the build backend""" + ep = os.environ['PEP517_BUILD_BACKEND'] + mod_path, _, obj_path = ep.partition(':') + obj = import_module(mod_path) + if obj_path: + for path_part in obj_path.split('.'): + obj = getattr(obj, path_part) + return obj + +def get_requires_for_build_wheel(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_wheel + except AttributeError: + return [] + else: + return hook(config_settings) + +def prepare_metadata_for_build_wheel(metadata_directory, config_settings): + """Invoke optional prepare_metadata_for_build_wheel + + Implements a fallback by building a wheel if the hook isn't defined. + """ + backend = _build_backend() + try: + hook = backend.prepare_metadata_for_build_wheel + except AttributeError: + return _get_wheel_metadata_from_wheel(backend, metadata_directory, + config_settings) + else: + return hook(metadata_directory, config_settings) + +WHEEL_BUILT_MARKER = 'PEP517_ALREADY_BUILT_WHEEL' + +def _dist_info_files(whl_zip): + """Identify the .dist-info folder inside a wheel ZipFile.""" + res = [] + for path in whl_zip.namelist(): + m = re.match(r'[^/\\]+-[^/\\]+\.dist-info/', path) + if m: + res.append(path) + if res: + return res + raise Exception("No .dist-info folder found in wheel") + +def _get_wheel_metadata_from_wheel(backend, metadata_directory, config_settings): + """Build a wheel and extract the metadata from it. + + Fallback for when the build backend does not define the 'get_wheel_metadata' + hook. + """ + from zipfile import ZipFile + whl_basename = backend.build_wheel(metadata_directory, config_settings) + with open(os.path.join(metadata_directory, WHEEL_BUILT_MARKER), 'wb'): + pass # Touch marker file + + whl_file = os.path.join(metadata_directory, whl_basename) + with ZipFile(whl_file) as zipf: + dist_info = _dist_info_files(zipf) + zipf.extractall(path=metadata_directory, members=dist_info) + return dist_info[0].split('/')[0] + +def _find_already_built_wheel(metadata_directory): + """Check for a wheel already built during the get_wheel_metadata hook. + """ + if not metadata_directory: + return None + metadata_parent = os.path.dirname(metadata_directory) + if not os.path.isfile(pjoin(metadata_parent, WHEEL_BUILT_MARKER)): + return None + + whl_files = glob(os.path.join(metadata_parent, '*.whl')) + if not whl_files: + print('Found wheel built marker, but no .whl files') + return None + if len(whl_files) > 1: + print('Found multiple .whl files; unspecified behaviour. ' + 'Will call build_wheel.') + return None + + # Exactly one .whl file + return whl_files[0] + +def build_wheel(wheel_directory, config_settings, metadata_directory=None): + """Invoke the mandatory build_wheel hook. + + If a wheel was already built in the prepare_metadata_for_build_wheel fallback, this + will copy it rather than rebuilding the wheel. + """ + prebuilt_whl = _find_already_built_wheel(metadata_directory) + if prebuilt_whl: + shutil.copy2(prebuilt_whl, wheel_directory) + return os.path.basename(prebuilt_whl) + + return _build_backend().build_wheel(wheel_directory, config_settings, + metadata_directory) + + +def get_requires_for_build_sdist(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_sdist + except AttributeError: + return [] + else: + return hook(config_settings) + +class _DummyException(Exception): + """Nothing should ever raise this exception""" + +class GotUnsupportedOperation(Exception): + """For internal use when backend raises UnsupportedOperation""" + +def build_sdist(sdist_directory, config_settings): + """Invoke the mandatory build_sdist hook.""" + backend = _build_backend() + try: + return backend.build_sdist(sdist_directory, config_settings) + except getattr(backend, 'UnsupportedOperation', _DummyException): + raise GotUnsupportedOperation + +HOOK_NAMES = { + 'get_requires_for_build_wheel', + 'prepare_metadata_for_build_wheel', + 'build_wheel', + 'get_requires_for_build_sdist', + 'build_sdist', +} + +def main(): + if len(sys.argv) < 3: + sys.exit("Needs args: hook_name, control_dir") + hook_name = sys.argv[1] + control_dir = sys.argv[2] + if hook_name not in HOOK_NAMES: + sys.exit("Unknown hook: %s" % hook_name) + hook = globals()[hook_name] + + hook_input = compat.read_json(pjoin(control_dir, 'input.json')) + + json_out = {'unsupported': False, 'return_val': None} + try: + json_out['return_val'] = hook(**hook_input['kwargs']) + except GotUnsupportedOperation: + json_out['unsupported'] = True + + compat.write_json(json_out, pjoin(control_dir, 'output.json'), indent=2) + +if __name__ == '__main__': + main() diff --git a/pipenv/patched/notpip/_vendor/pep517/check.py b/pipenv/patched/notpip/_vendor/pep517/check.py new file mode 100644 index 00000000..3dffd2e0 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/check.py @@ -0,0 +1,194 @@ +"""Check a project and backend by attempting to build using PEP 517 hooks. +""" +import argparse +import logging +import os +from os.path import isfile, join as pjoin +from pipenv.patched.notpip._vendor.pytoml import TomlError, load as toml_load +import shutil +from subprocess import CalledProcessError +import sys +import tarfile +from tempfile import mkdtemp +import zipfile + +from .colorlog import enable_colourful_output +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + +def check_build_sdist(hooks): + with BuildEnvironment() as env: + try: + env.pip_install(hooks.build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_sdist({}) + log.info('Got build requires: %s', reqs) + except: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build sdist in %s', td) + try: + try: + filename = hooks.build_sdist(td, {}) + log.info('build_sdist returned %r', filename) + except: + log.info('Failure in build_sdist', exc_info=True) + return False + + if not filename.endswith('.tar.gz'): + log.error("Filename %s doesn't have .tar.gz extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if tarfile.is_tarfile(path): + log.info("Output file is a tar file") + else: + log.error("Output file is not a tar file") + return False + + finally: + shutil.rmtree(td) + + return True + +def check_build_wheel(hooks): + with BuildEnvironment() as env: + try: + env.pip_install(hooks.build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_wheel({}) + log.info('Got build requires: %s', reqs) + except: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build wheel in %s', td) + try: + try: + filename = hooks.build_wheel(td, {}) + log.info('build_wheel returned %r', filename) + except: + log.info('Failure in build_wheel', exc_info=True) + return False + + if not filename.endswith('.whl'): + log.error("Filename %s doesn't have .whl extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if zipfile.is_zipfile(path): + log.info("Output file is a zip file") + else: + log.error("Output file is not a zip file") + return False + + finally: + shutil.rmtree(td) + + return True + + +def check(source_dir): + pyproject = pjoin(source_dir, 'pyproject.toml') + if isfile(pyproject): + log.info('Found pyproject.toml') + else: + log.error('Missing pyproject.toml') + return False + + try: + with open(pyproject) as f: + pyproject_data = toml_load(f) + # Ensure the mandatory data can be loaded + buildsys = pyproject_data['build-system'] + requires = buildsys['requires'] + backend = buildsys['build-backend'] + log.info('Loaded pyproject.toml') + except (TomlError, KeyError): + log.error("Invalid pyproject.toml", exc_info=True) + return False + + hooks = Pep517HookCaller(source_dir, backend) + + sdist_ok = check_build_sdist(hooks) + wheel_ok = check_build_wheel(hooks) + + if not sdist_ok: + log.warning('Sdist checks failed; scroll up to see') + if not wheel_ok: + log.warning('Wheel checks failed') + + return sdist_ok + + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument('source_dir', + help="A directory containing pyproject.toml") + args = ap.parse_args(argv) + + enable_colourful_output() + + ok = check(args.source_dir) + + if ok: + print(ansi('Checks passed', 'green')) + else: + print(ansi('Checks failed', 'red')) + sys.exit(1) + +ansi_codes = { + 'reset': '\x1b[0m', + 'bold': '\x1b[1m', + 'red': '\x1b[31m', + 'green': '\x1b[32m', +} +def ansi(s, attr): + if os.name != 'nt' and sys.stdout.isatty(): + return ansi_codes[attr] + str(s) + ansi_codes['reset'] + else: + return str(s) + +if __name__ == '__main__': + main() diff --git a/pipenv/patched/notpip/_vendor/pep517/colorlog.py b/pipenv/patched/notpip/_vendor/pep517/colorlog.py new file mode 100644 index 00000000..26cf7480 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/colorlog.py @@ -0,0 +1,110 @@ +"""Nicer log formatting with colours. + +Code copied from Tornado, Apache licensed. +""" +# Copyright 2012 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys + +try: + import curses +except ImportError: + curses = None + +def _stderr_supports_color(): + color = False + if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): + try: + curses.setupterm() + if curses.tigetnum("colors") > 0: + color = True + except Exception: + pass + return color + +class LogFormatter(logging.Formatter): + """Log formatter with colour support + """ + DEFAULT_COLORS = { + logging.INFO: 2, # Green + logging.WARNING: 3, # Yellow + logging.ERROR: 1, # Red + logging.CRITICAL: 1, + } + + def __init__(self, color=True, datefmt=None): + r""" + :arg bool color: Enables color support. + :arg string fmt: Log message format. + It will be applied to the attributes dict of log records. The + text between ``%(color)s`` and ``%(end_color)s`` will be colored + depending on the level if color support is on. + :arg dict colors: color mappings from logging level to terminal color + code + :arg string datefmt: Datetime format. + Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. + .. versionchanged:: 3.2 + Added ``fmt`` and ``datefmt`` arguments. + """ + logging.Formatter.__init__(self, datefmt=datefmt) + self._colors = {} + if color and _stderr_supports_color(): + # The curses module has some str/bytes confusion in + # python3. Until version 3.2.3, most methods return + # bytes, but only accept strings. In addition, we want to + # output these strings with the logging module, which + # works with unicode strings. The explicit calls to + # unicode() below are harmless in python2 but will do the + # right conversion in python 3. + fg_color = (curses.tigetstr("setaf") or + curses.tigetstr("setf") or "") + if (3, 0) < sys.version_info < (3, 2, 3): + fg_color = str(fg_color, "ascii") + + for levelno, code in self.DEFAULT_COLORS.items(): + self._colors[levelno] = str(curses.tparm(fg_color, code), "ascii") + self._normal = str(curses.tigetstr("sgr0"), "ascii") + + scr = curses.initscr() + self.termwidth = scr.getmaxyx()[1] + curses.endwin() + else: + self._normal = '' + # Default width is usually 80, but too wide is worse than too narrow + self.termwidth = 70 + + def formatMessage(self, record): + l = len(record.message) + right_text = '{initial}-{name}'.format(initial=record.levelname[0], + name=record.name) + if l + len(right_text) < self.termwidth: + space = ' ' * (self.termwidth - (l + len(right_text))) + else: + space = ' ' + + if record.levelno in self._colors: + start_color = self._colors[record.levelno] + end_color = self._normal + else: + start_color = end_color = '' + + return record.message + space + start_color + right_text + end_color + +def enable_colourful_output(level=logging.INFO): + handler = logging.StreamHandler() + handler.setFormatter(LogFormatter()) + logging.root.addHandler(handler) + logging.root.setLevel(level) diff --git a/pipenv/patched/notpip/_vendor/pep517/compat.py b/pipenv/patched/notpip/_vendor/pep517/compat.py new file mode 100644 index 00000000..01c66fc7 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/compat.py @@ -0,0 +1,23 @@ +"""Handle reading and writing JSON in UTF-8, on Python 3 and 2.""" +import json +import sys + +if sys.version_info[0] >= 3: + # Python 3 + def write_json(obj, path, **kwargs): + with open(path, 'w', encoding='utf-8') as f: + json.dump(obj, f, **kwargs) + + def read_json(path): + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + +else: + # Python 2 + def write_json(obj, path, **kwargs): + with open(path, 'wb') as f: + json.dump(obj, f, encoding='utf-8', **kwargs) + + def read_json(path): + with open(path, 'rb') as f: + return json.load(f) diff --git a/pipenv/patched/notpip/_vendor/pep517/envbuild.py b/pipenv/patched/notpip/_vendor/pep517/envbuild.py new file mode 100644 index 00000000..c54d3585 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/envbuild.py @@ -0,0 +1,150 @@ +"""Build wheels/sdists by installing build deps to a temporary environment. +""" + +import os +import logging +from pipenv.patched.notpip._vendor import pytoml +import shutil +from subprocess import check_call +import sys +from sysconfig import get_paths +from tempfile import mkdtemp + +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + +def _load_pyproject(source_dir): + with open(os.path.join(source_dir, 'pyproject.toml')) as f: + pyproject_data = pytoml.load(f) + buildsys = pyproject_data['build-system'] + return buildsys['requires'], buildsys['build-backend'] + + +class BuildEnvironment(object): + """Context manager to install build deps in a simple temporary environment + + Based on code I wrote for pip, which is MIT licensed. + """ + # Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file) + # + # Permission is hereby granted, free of charge, to any person obtaining + # a copy of this software and associated documentation files (the + # "Software"), to deal in the Software without restriction, including + # without limitation the rights to use, copy, modify, merge, publish, + # distribute, sublicense, and/or sell copies of the Software, and to + # permit persons to whom the Software is furnished to do so, subject to + # the following conditions: + # + # The above copyright notice and this permission notice shall be + # included in all copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + path = None + + def __init__(self, cleanup=True): + self._cleanup = cleanup + + def __enter__(self): + self.path = mkdtemp(prefix='pep517-build-env-') + log.info('Temporary build environment: %s', self.path) + + self.save_path = os.environ.get('PATH', None) + self.save_pythonpath = os.environ.get('PYTHONPATH', None) + + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + install_dirs = get_paths(install_scheme, vars={ + 'base': self.path, + 'platbase': self.path, + }) + + scripts = install_dirs['scripts'] + if self.save_path: + os.environ['PATH'] = scripts + os.pathsep + self.save_path + else: + os.environ['PATH'] = scripts + os.pathsep + os.defpath + + if install_dirs['purelib'] == install_dirs['platlib']: + lib_dirs = install_dirs['purelib'] + else: + lib_dirs = install_dirs['purelib'] + os.pathsep + \ + install_dirs['platlib'] + if self.save_pythonpath: + os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ + self.save_pythonpath + else: + os.environ['PYTHONPATH'] = lib_dirs + + return self + + def pip_install(self, reqs): + """Install dependencies into this env by calling pip in a subprocess""" + if not reqs: + return + log.info('Calling pip to install %s', reqs) + check_call([sys.executable, '-m', 'pip', 'install', '--ignore-installed', + '--prefix', self.path] + list(reqs)) + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._cleanup and (self.path is not None) and os.path.isdir(self.path): + shutil.rmtree(self.path) + + if self.save_path is None: + os.environ.pop('PATH', None) + else: + os.environ['PATH'] = self.save_path + + if self.save_pythonpath is None: + os.environ.pop('PYTHONPATH', None) + else: + os.environ['PYTHONPATH'] = self.save_pythonpath + +def build_wheel(source_dir, wheel_dir, config_settings=None): + """Build a wheel from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str wheel_dir: Target directory to create wheel in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_wheel(config_settings) + env.pip_install(reqs) + return hooks.build_wheel(wheel_dir, config_settings) + + +def build_sdist(source_dir, sdist_dir, config_settings=None): + """Build an sdist from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str sdist_dir: Target directory to place sdist in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_sdist(config_settings) + env.pip_install(reqs) + return hooks.build_sdist(sdist_dir, config_settings) diff --git a/pipenv/patched/notpip/_vendor/pep517/wrappers.py b/pipenv/patched/notpip/_vendor/pep517/wrappers.py new file mode 100644 index 00000000..28260f32 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/wrappers.py @@ -0,0 +1,134 @@ +from contextlib import contextmanager +import os +from os.path import dirname, abspath, join as pjoin +import shutil +from subprocess import check_call +import sys +from tempfile import mkdtemp + +from . import compat + +_in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.py') + +@contextmanager +def tempdir(): + td = mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + +class UnsupportedOperation(Exception): + """May be raised by build_sdist if the backend indicates that it can't.""" + +class Pep517HookCaller(object): + """A wrapper around a source directory to be built with a PEP 517 backend. + + source_dir : The path to the source directory, containing pyproject.toml. + backend : The build backend spec, as per PEP 517, from pyproject.toml. + """ + def __init__(self, source_dir, build_backend): + self.source_dir = abspath(source_dir) + self.build_backend = build_backend + + def get_requires_for_build_wheel(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["wheel >= 0.25", "setuptools"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_wheel', { + 'config_settings': config_settings + }) + + def prepare_metadata_for_build_wheel(self, metadata_directory, config_settings=None): + """Prepare a *.dist-info folder with metadata for this project. + + Returns the name of the newly created folder. + + If the build backend defines a hook with this name, it will be called + in a subprocess. If not, the backend will be asked to build a wheel, + and the dist-info extracted from that. + """ + return self._call_hook('prepare_metadata_for_build_wheel', { + 'metadata_directory': abspath(metadata_directory), + 'config_settings': config_settings, + }) + + def build_wheel(self, wheel_directory, config_settings=None, metadata_directory=None): + """Build a wheel from this project. + + Returns the name of the newly created file. + + In general, this will call the 'build_wheel' hook in the backend. + However, if that was previously called by + 'prepare_metadata_for_build_wheel', and the same metadata_directory is + used, the previously built wheel will be copied to wheel_directory. + """ + if metadata_directory is not None: + metadata_directory = abspath(metadata_directory) + return self._call_hook('build_wheel', { + 'wheel_directory': abspath(wheel_directory), + 'config_settings': config_settings, + 'metadata_directory': metadata_directory, + }) + + def get_requires_for_build_sdist(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["setuptools >= 26"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_sdist', { + 'config_settings': config_settings + }) + + def build_sdist(self, sdist_directory, config_settings=None): + """Build an sdist from this project. + + Returns the name of the newly created file. + + This calls the 'build_sdist' backend hook in a subprocess. + """ + return self._call_hook('build_sdist', { + 'sdist_directory': abspath(sdist_directory), + 'config_settings': config_settings, + }) + + + def _call_hook(self, hook_name, kwargs): + env = os.environ.copy() + + # On Python 2, pytoml returns Unicode values (which is correct) but the + # environment passed to check_call needs to contain string values. We + # convert here by encoding using ASCII (the backend can only contain + # letters, digits and _, . and : characters, and will be used as a + # Python identifier, so non-ASCII content is wrong on Python 2 in + # any case). + if sys.version_info[0] == 2: + build_backend = self.build_backend.encode('ASCII') + else: + build_backend = self.build_backend + + env['PEP517_BUILD_BACKEND'] = build_backend + with tempdir() as td: + compat.write_json({'kwargs': kwargs}, pjoin(td, 'input.json'), + indent=2) + + # Run the hook in a subprocess + check_call([sys.executable, _in_proc_script, hook_name, td], + cwd=self.source_dir, env=env) + + data = compat.read_json(pjoin(td, 'output.json')) + if data.get('unsupported'): + raise UnsupportedOperation + return data['return_val'] + diff --git a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py index ed57821d..ac893b66 100644 --- a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py +++ b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py @@ -47,6 +47,11 @@ except ImportError: # Python 3.2 compatibility import imp as _imp +try: + FileExistsError +except NameError: + FileExistsError = OSError + from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import urllib, map, filter @@ -78,8 +83,11 @@ __import__('pipenv.patched.notpip._vendor.packaging.requirements') __import__('pipenv.patched.notpip._vendor.packaging.markers') -if (3, 0) < sys.version_info < (3, 3): - raise RuntimeError("Python 3.3 or later is required") +__metaclass__ = type + + +if (3, 0) < sys.version_info < (3, 4): + raise RuntimeError("Python 3.4 or later is required") if six.PY2: # Those builtin exceptions are only defined in Python 3 @@ -537,7 +545,7 @@ class IResourceProvider(IMetadataProvider): """List of resource names in the directory (like ``os.listdir()``)""" -class WorkingSet(object): +class WorkingSet: """A collection of active distributions on sys.path (or a similar list)""" def __init__(self, entries=None): @@ -637,13 +645,12 @@ class WorkingSet(object): distributions in the working set, otherwise only ones matching both `group` and `name` are yielded (in distribution order). """ - for dist in self: - entries = dist.get_entry_map(group) - if name is None: - for ep in entries.values(): - yield ep - elif name in entries: - yield entries[name] + return ( + entry + for dist in self + for entry in dist.get_entry_map(group).values() + if name is None or name == entry.name + ) def run_script(self, requires, script_name): """Locate distribution for `requires` and run `script_name` script""" @@ -944,7 +951,7 @@ class _ReqExtras(dict): return not req.marker or any(extra_evals) -class Environment(object): +class Environment: """Searchable snapshot of distributions on a search path""" def __init__( @@ -959,7 +966,7 @@ class Environment(object): `platform` is an optional string specifying the name of the platform that platform-specific distributions must be compatible with. If unspecified, it defaults to the current platform. `python` is an - optional string naming the desired version of Python (e.g. ``'3.3'``); + optional string naming the desired version of Python (e.g. ``'3.6'``); it defaults to the current version. You may explicitly set `platform` (and/or `python`) to ``None`` if you @@ -2087,7 +2094,12 @@ def _handle_ns(packageName, path_item): importer = get_importer(path_item) if importer is None: return None - loader = importer.find_module(packageName) + + # capture warnings due to #1111 + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + loader = importer.find_module(packageName) + if loader is None: return None module = sys.modules.get(packageName) @@ -2132,12 +2144,13 @@ def _rebuild_mod_path(orig_path, package_name, module): parts = path_parts[:-module_parts] return safe_sys_path_index(_normalize_cached(os.sep.join(parts))) - if not isinstance(orig_path, list): - # Is this behavior useful when module.__path__ is not a list? - return + new_path = sorted(orig_path, key=position_in_sys_path) + new_path = [_normalize_cached(p) for p in new_path] - orig_path.sort(key=position_in_sys_path) - module.__path__[:] = [_normalize_cached(p) for p in orig_path] + if isinstance(module.__path__, list): + module.__path__[:] = new_path + else: + module.__path__ = new_path def declare_namespace(packageName): @@ -2148,9 +2161,10 @@ def declare_namespace(packageName): if packageName in _namespace_packages: return - path, parent = sys.path, None - if '.' in packageName: - parent = '.'.join(packageName.split('.')[:-1]) + path = sys.path + parent, _, _ = packageName.rpartition('.') + + if parent: declare_namespace(parent) if parent not in _namespace_packages: __import__(parent) @@ -2161,7 +2175,7 @@ def declare_namespace(packageName): # Track what packages are namespaces, so when new path items are added, # they can be updated - _namespace_packages.setdefault(parent, []).append(packageName) + _namespace_packages.setdefault(parent or None, []).append(packageName) _namespace_packages.setdefault(packageName, []) for path_item in path: @@ -2279,7 +2293,7 @@ EGG_NAME = re.compile( ).match -class EntryPoint(object): +class EntryPoint: """Object representing an advertised importable object""" def __init__(self, name, module_name, attrs=(), extras=(), dist=None): @@ -2433,7 +2447,7 @@ def _version_from_file(lines): return safe_version(value.strip()) or None -class Distribution(object): +class Distribution: """Wrap an actual or potential sys.path entry w/metadata""" PKG_INFO = 'PKG-INFO' @@ -3027,7 +3041,10 @@ def _bypass_ensure_directory(path): dirname, filename = split(path) if dirname and filename and not isdir(dirname): _bypass_ensure_directory(dirname) - mkdir(dirname, 0o755) + try: + mkdir(dirname, 0o755) + except FileExistsError: + pass def split_sections(s): diff --git a/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py b/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py index 331a51bb..3a44fa1c 100644 --- a/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py +++ b/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py @@ -2,6 +2,8 @@ import os import errno import sys +from pipenv.patched.notpip._vendor import six + def _makedirs_31(path, exist_ok=False): try: @@ -15,8 +17,7 @@ def _makedirs_31(path, exist_ok=False): # and exists_ok considerations are disentangled. # See https://github.com/pypa/setuptools/pull/1083#issuecomment-315168663 needs_makedirs = ( - sys.version_info < (3, 2, 5) or - (3, 3) <= sys.version_info < (3, 3, 6) or + six.PY2 or (3, 4) <= sys.version_info < (3, 4, 1) ) makedirs = _makedirs_31 if needs_makedirs else os.makedirs diff --git a/pipenv/patched/notpip/_vendor/pyparsing.py b/pipenv/patched/notpip/_vendor/pyparsing.py index b5ba3d5f..455d1151 100644 --- a/pipenv/patched/notpip/_vendor/pyparsing.py +++ b/pipenv/patched/notpip/_vendor/pyparsing.py @@ -1,6 +1,6 @@ # module pyparsing.py # -# Copyright (c) 2003-2016 Paul T. McGuire +# Copyright (c) 2003-2018 Paul T. McGuire # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -25,6 +25,7 @@ __doc__ = \ """ pyparsing module - Classes and methods to define and execute parsing grammars +============================================================================= The pyparsing module is an alternative approach to creating and executing simple grammars, vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you @@ -58,10 +59,23 @@ The pyparsing module handles some of the problems that are typically vexing when - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) - quoted strings - embedded comments + + +Getting Started - +----------------- +Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing +classes inherit from. Use the docstrings for examples of how to: + - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes + - construct character word-group expressions using the L{Word} class + - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes + - use L{'+'}, L{'|'}, L{'^'}, and L{'&'} operators to combine simple expressions into more complex ones + - associate names with your parsed results using L{ParserElement.setResultsName} + - find some helpful expression short-cuts like L{delimitedList} and L{oneOf} + - find more useful common expressions in the L{pyparsing_common} namespace class """ -__version__ = "2.2.0" -__versionTime__ = "06 Mar 2017 02:06 UTC" +__version__ = "2.2.1" +__versionTime__ = "18 Sep 2018 00:49 UTC" __author__ = "Paul McGuire " import string @@ -82,6 +96,15 @@ try: except ImportError: from threading import RLock +try: + # Python 3 + from collections.abc import Iterable + from collections.abc import MutableMapping +except ImportError: + # Python 2.7 + from collections import Iterable + from collections import MutableMapping + try: from collections import OrderedDict as _OrderedDict except ImportError: @@ -940,7 +963,7 @@ class ParseResults(object): def __dir__(self): return (dir(type(self)) + list(self.keys())) -collections.MutableMapping.register(ParseResults) +MutableMapping.register(ParseResults) def col (loc,strg): """Returns current column within a string, counting newlines as line separators. @@ -1025,11 +1048,11 @@ def _trim_arity(func, maxargs=2): # special handling for Python 3.5.0 - extra deep call stack by 1 offset = -3 if system_version == (3,5,0) else -2 frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset] - return [(frame_summary.filename, frame_summary.lineno)] + return [frame_summary[:2]] def extract_tb(tb, limit=0): frames = traceback.extract_tb(tb, limit=limit) frame_summary = frames[-1] - return [(frame_summary.filename, frame_summary.lineno)] + return [frame_summary[:2]] else: extract_stack = traceback.extract_stack extract_tb = traceback.extract_tb @@ -1374,7 +1397,7 @@ class ParserElement(object): else: preloc = loc tokensStart = preloc - if self.mayIndexError or loc >= len(instring): + if self.mayIndexError or preloc >= len(instring): try: loc,tokens = self.parseImpl( instring, preloc, doActions ) except IndexError: @@ -1408,7 +1431,6 @@ class ParserElement(object): self.resultsName, asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), modal=self.modalResults ) - if debugging: #~ print ("Matched",self,"->",retTokens.asList()) if (self.debugActions[1] ): @@ -3242,7 +3264,7 @@ class ParseExpression(ParserElement): if isinstance( exprs, basestring ): self.exprs = [ ParserElement._literalStringClass( exprs ) ] - elif isinstance( exprs, collections.Iterable ): + elif isinstance( exprs, Iterable ): exprs = list(exprs) # if sequence of strings provided, wrap with Literal if all(isinstance(expr, basestring) for expr in exprs): @@ -4393,7 +4415,7 @@ def traceParseAction(f): @traceParseAction def remove_duplicate_chars(tokens): - return ''.join(sorted(set(''.join(tokens))) + return ''.join(sorted(set(''.join(tokens)))) wds = OneOrMore(wd).setParseAction(remove_duplicate_chars) print(wds.parseString("slkdjs sld sldd sdlf sdljf")) @@ -4583,7 +4605,7 @@ def oneOf( strs, caseless=False, useRegex=True ): symbols = [] if isinstance(strs,basestring): symbols = strs.split() - elif isinstance(strs, collections.Iterable): + elif isinstance(strs, Iterable): symbols = list(strs) else: warnings.warn("Invalid argument to oneOf, expected string or iterable", @@ -4734,7 +4756,7 @@ stringEnd = StringEnd().setName("stringEnd") _escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1]) _escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16))) _escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8))) -_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(printables, excludeChars=r'\]', exact=1) | Regex(r"\w", re.UNICODE) +_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1) _charRange = Group(_singleChar + Suppress("-") + _singleChar) _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" diff --git a/pipenv/patched/notpip/_vendor/pytoml/parser.py b/pipenv/patched/notpip/_vendor/pytoml/parser.py index e03a03fb..9f94e923 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/parser.py +++ b/pipenv/patched/notpip/_vendor/pytoml/parser.py @@ -223,8 +223,8 @@ _float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]? _datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') _basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') -_litstr_re = re.compile(r"[^'\000-\037]*") -_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\011\013-\037]))*") +_litstr_re = re.compile(r"[^'\000\010\012-\037]*") +_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*") def _p_value(s, object_pairs_hook): pos = s.pos() diff --git a/pipenv/patched/notpip/_vendor/requests/__init__.py b/pipenv/patched/notpip/_vendor/requests/__init__.py index 8e7576c1..af3d4b1d 100644 --- a/pipenv/patched/notpip/_vendor/requests/__init__.py +++ b/pipenv/patched/notpip/_vendor/requests/__init__.py @@ -91,7 +91,7 @@ except (AssertionError, ValueError): RequestsDependencyWarning) # Attempt to enable urllib3's SNI support, if possible -from pipenv.patched.notpip._internal.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.compat import WINDOWS if not WINDOWS: try: from pipenv.patched.notpip._vendor.urllib3.contrib import pyopenssl diff --git a/pipenv/patched/notpip/_vendor/vendor.txt b/pipenv/patched/notpip/_vendor/vendor.txt index b9854e9a..9389dd94 100644 --- a/pipenv/patched/notpip/_vendor/vendor.txt +++ b/pipenv/patched/notpip/_vendor/vendor.txt @@ -9,14 +9,15 @@ msgpack-python==0.5.6 lockfile==0.12.2 progress==1.4 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -packaging==17.1 -pyparsing==2.2.0 -pytoml==0.1.16 +packaging==18.0 +pep517==0.2 +pyparsing==2.2.1 +pytoml==0.1.19 retrying==1.3.3 requests==2.19.1 chardet==3.0.4 idna==2.7 urllib3==1.23 - certifi==2018.4.16 -setuptools==39.2.0 + certifi==2018.8.24 +setuptools==40.4.3 webencodings==0.5.1 diff --git a/pipenv/patched/patched.txt b/pipenv/patched/patched.txt index 3ad4c1e9..f73ed5a6 100644 --- a/pipenv/patched/patched.txt +++ b/pipenv/patched/patched.txt @@ -4,4 +4,4 @@ crayons==0.1.2 pipfile==0.0.2 pip-tools==3.1.0 prettytoml==0.3 -pip==18.0 +pip==18.1 diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py index de9b4353..c466ef04 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py @@ -1,49 +1,55 @@ # -*- coding=utf-8 -*- -import importlib -from pip_shims import pip_version -import pkg_resources +__all__ = [ + "InstallRequirement", + "parse_requirements", + "RequirementSet", + "user_cache_dir", + "FAVORITE_HASH", + "is_file_url", + "url_to_path", + "PackageFinder", + "FormatControl", + "Wheel", + "Command", + "cmdoptions", + "get_installed_distributions", + "PyPI", + "SafeFileCache", + "InstallationError", + "parse_version", + "pip_version", + "install_req_from_editable", + "install_req_from_line", + "user_cache_dir" +] -def do_import(module_path, subimport=None, old_path=None, vendored_name=None): - old_path = old_path or module_path - prefix = vendored_name if vendored_name else "pip" - prefixes = ["{0}._internal".format(prefix), "{0}".format(prefix)] - paths = [module_path, old_path] - search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] - package = subimport if subimport else None - for to_import in search_order: - if not subimport: - to_import, _, package = to_import.rpartition(".") - try: - imported = importlib.import_module(to_import) - except ImportError: - continue - else: - return getattr(imported, package) - - -InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name="notpip") -parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name="notpip") -RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name="notpip") -user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name="notpip") -FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name="notpip") -is_file_url = do_import('download', 'is_file_url', vendored_name="notpip") -url_to_path = do_import('download', 'url_to_path', vendored_name="notpip") -PackageFinder = do_import('index', 'PackageFinder', vendored_name="notpip") -FormatControl = do_import('index', 'FormatControl', vendored_name="notpip") -Wheel = do_import('wheel', 'Wheel', vendored_name="notpip") -Command = do_import('cli.base_command', 'Command', old_path='basecommand', vendored_name="notpip") -cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions', vendored_name="notpip") -get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils', vendored_name="notpip") -PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') -SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') -InstallationError = do_import('exceptions', 'InstallationError', vendored_name='notpip') +from pipenv.vendor.appdirs import user_cache_dir +from pip_shims.shims import ( + InstallRequirement, + parse_requirements, + RequirementSet, + FAVORITE_HASH, + is_file_url, + url_to_path, + PackageFinder, + FormatControl, + Wheel, + Command, + cmdoptions, + get_installed_distributions, + PyPI, + SafeFileCache, + InstallationError, + parse_version, + pip_version, +) # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if pkg_resources.parse_version(pip_version) < pkg_resources.parse_version('18.1'): +if parse_version(pip_version) < parse_version('18.1'): install_req_from_line = InstallRequirement.from_line install_req_from_editable = InstallRequirement.from_editable else: - install_req_from_line = do_import('req.constructors', 'install_req_from_line', vendored_name="notpip") - install_req_from_editable = do_import('req.constructors', 'install_req_from_editable', vendored_name="notpip") - + from pip_shims.shims import ( + install_req_from_editable, install_req_from_line + ) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index c1871670..2a0743a3 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -19,11 +19,10 @@ from .._compat import ( InstallRequirement, SafeFileCache ) -os.environ["PIP_SHIMS_BASE_MODULE"] = str("notpip") +os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") from pip_shims.shims import do_import, VcsSupport, WheelCache from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet, Specifier -from packaging.markers import Op, Value, Variable, Marker InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver @@ -31,7 +30,7 @@ from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR from ..exceptions import NoCandidateFound from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, - make_install_requirement, clean_requires_python) + make_install_requirement, clean_requires_python) from .base import BaseRepository try: @@ -243,6 +242,7 @@ class PyPIRepository(BaseRepository): dist = None ireq.isolated = False ireq._wheel_cache = wheel_cache + try: from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer except ImportError: @@ -295,7 +295,18 @@ class PyPIRepository(BaseRepository): resolver = PipResolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() + + cleanup_fn = getattr(reqset, "cleanup_files", None) + if cleanup_fn is not None: + try: + cleanup_fn() + except OSError: + pass + + if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): + if ireq.editable: + self._source_dir = TemporaryDirectory(fs_str("source")) + ireq.ensure_has_source_dir(self.source_dir) if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # Collect setup_requires info from local eggs. diff --git a/pipenv/project.py b/pipenv/project.py index c2dc9668..d25dba9a 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -10,6 +10,7 @@ import hashlib import contoml from first import first from cached_property import cached_property +import operator import pipfile import pipfile.api import six @@ -82,6 +83,8 @@ class _LockFileEncoder(json.JSONEncoder): if isinstance(obj, (ContainerElement, TokenElement)): return obj.primitive_value + elif isinstance(obj, vistir.compat.Path): + obj = obj.as_posix() return super(_LockFileEncoder, self).default(obj) def encode(self, obj): @@ -144,7 +147,7 @@ class Project(object): self._lockfile_newlines = DEFAULT_NEWLINES self._requirements_location = None self._original_dir = os.path.abspath(os.curdir) - self.which = which + self._which = which self.python_version = python_version # Hack to skip this during pipenv run, or -r. if ("run" not in sys.argv) and chdir: @@ -638,8 +641,9 @@ class Project(object): def create_pipfile(self, python=None): """Creates the Pipfile, filled with juicy defaults.""" - from .patched.notpip._internal import ConfigOptionParser - from .patched.notpip._internal.cmdoptions import make_option_group, index_group + from .vendor.pip_shims.shims import ( + ConfigOptionParser, make_option_group, index_group + ) config_parser = ConfigOptionParser(name=self.name) config_parser.add_option_group(make_option_group(index_group, config_parser)) @@ -678,6 +682,54 @@ class Project(object): data[u"requires"] = {"python_version": version[: len("2.7")]} self.write_toml(data, "Pipfile") + def get_or_create_lockfile(self): + from requirementslib.models.lockfile import Lockfile as Req_Lockfile + lockfile = None + try: + lockfile = Req_Lockfile.load(self.lockfile_location) + except OSError: + lockfile = Req_Lockfile(self.lockfile_content) + return lockfile + else: + if lockfile._lockfile is not None: + return lockfile + if self.lockfile_exists and self.lockfile_content: + from .vendor.plette.lockfiles import Lockfile + lockfile_dict = self.lockfile_content.copy() + sources = lockfile_dict["_meta"].get("sources", []) + if not sources: + sources = self.pipfile_sources + elif not isinstance(sources, list): + sources = [sources,] + lockfile_dict["_meta"]["sources"] = [ + { + "name": s["name"], + "url": s["url"], + "verify_ssl": ( + s["verify_ssl"] if isinstance(s["verify_ssl"], bool) else ( + True if s["verify_ssl"].lower() == "true" else False + ) + ) + } for s in sources + ] + _created_lockfile = Lockfile(lockfile_dict) + lockfile._lockfile = lockfile.projectfile.model = _created_lockfile + return lockfile + elif self.pipfile_exists: + from .vendor.plette.lockfiles import Lockfile, PIPFILE_SPEC_CURRENT + lockfile_dict = { + "_meta": { + "hash": {"sha256": self.calculate_pipfile_hash()}, + "pipfile-spec": PIPFILE_SPEC_CURRENT, + "sources": self.pipfile_sources, + "requires": self.parsed_pipfile.get("requires", {}) + }, + "default": self._lockfile["default"].copy(), + "develop": self._lockfile["develop"].copy() + } + lockfile._lockfile = Lockfile(lockfile_dict) + return lockfile + def write_toml(self, data, path=None): """Writes the given data structure out as TOML.""" if path is None: @@ -935,21 +987,66 @@ class Project(object): @property def env_paths(self): - import sysconfig location = self.virtualenv_location if self.virtualenv_location else sys.prefix - prefix = vistir.compat.Path(location).as_posix() - scheme = sysconfig._get_default_scheme() - config = { - "base": prefix, - "installed_base": prefix, - "platbase": prefix, - "installed_platbase": prefix - } - config.update(self._pyversion) + prefix = vistir.compat.Path(location) + import importlib + try: + _virtualenv = importlib.import_module("virtualenv") + except ImportError: + with vistir.contextmanagers.temp_path(): + from string import Formatter + formatter = Formatter() + import sysconfig + if getattr(sys, "real_prefix", None): + scheme = sysconfig._get_default_scheme() + sysconfig._INSTALL_SCHEMES["posix_prefix"]["purelib"] + if not scheme: + scheme = "posix_prefix" if not sys.platform == "win32" else "nt" + is_purelib = "purelib" in sysconfig._INSTALL_SCHEMES[scheme] + lib_key = "purelib" if is_purelib else "platlib" + lib = sysconfig._INSTALL_SCHEMES[scheme][lib_key] + fields = [field for _, field, _, _ in formatter.parse() if field] + config = { + "py_version_short": self._pyversion, + } + for field in fields: + if field not in config: + config[field] = prefix + sys.path = [ + os.path.join(sysconfig._INSTALL_SCHEMES[scheme][lib_key], "site-packages"), + ] + sys.path + six.reload_module(importlib) + _virtualenv = importlib.import_module("virtualenv") + home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix()) paths = { - k: v.format(**config) - for k, v in sysconfig._INSTALL_SCHEMES[scheme].items() + "lib": lib, + "include": inc, + "scripts": bin_, + "purelib": lib, + "prefix": home, + "base": home } - if "prefix" not in paths: - paths["prefix"] = prefix return paths + + @cached_property + def finders(self): + from .vendor.pythonfinder import Finder + finders = [ + Finder(path=self.env_paths["scripts"], global_search=gs, system=False) + for gs in (False, True) + ] + return finders + + @property + def finder(self): + return next(iter(self.finders), None) + + def which(self, search, as_path=True): + find = operator.methodcaller("which", search) + result = next(iter(filter(None, (find(finder) for finder in self.finders))), None) + if not result: + result = self._which(search) + else: + if as_path: + result = str(result.path) + return result diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 6526d990..9ef46878 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -3,57 +3,52 @@ import sys import json import logging -os.environ["PIP_PYTHON_PATH"] = sys.executable +os.environ["PIP_PYTHON_PATH"] = str(sys.executable) def _patch_path(): + import site pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + pipenv_site_dir = os.path.dirname(pipenv_libdir) + site.addsitedir(pipenv_site_dir) for _dir in ("vendor", "patched"): sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) - site_packages_dir = os.path.dirname(pipenv_libdir) - if site_packages_dir not in sys.path: - sys.path.append(site_packages_dir) + + +def get_parser(): + from argparse import ArgumentParser + parser = ArgumentParser("pipenv-resolver") + parser.add_argument("--pre", action="store_true", default=False) + parser.add_argument("--clear", action="store_true", default=False) + parser.add_argument("--verbose", "-v", action="count", default=False) + parser.add_argument("--debug", action="store_true", default=False) + parser.add_argument("--system", action="store_true", default=False) + parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store", + default=os.environ.get("PIPENV_REQ_DIR")) + parser.add_argument("packages", nargs="*") + return parser def which(*args, **kwargs): return sys.executable -def main(): - do_pre = "--pre" in " ".join(sys.argv) - do_clear = "--clear" in " ".join(sys.argv) - is_verbose = "--verbose" in " ".join(sys.argv) - is_debug = "--debug" in " ".join(sys.argv) - system = "--system" in " ".join(sys.argv) - new_sys_argv = [] - for v in sys.argv: - if v.startswith("--"): - continue - - else: - new_sys_argv.append(v) - sys.argv = new_sys_argv - - os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) - os.environ["PIP_PYTHON_PATH"] = sys.executable - - verbosity = int(os.environ.get("PIPENV_VERBOSITY", 0)) - if is_debug: - verbosity = max(verbosity, 2) - elif is_verbose: - verbosity = max(verbosity, 1) - if verbosity > 1: # Shit's getting real at this point. +def handle_parsed_args(parsed): + if parsed.debug: + parsed.verbose = max(parsed.verbose, 2) + if parsed.verbose > 1: logging.getLogger("notpip").setLevel(logging.DEBUG) - elif verbosity > 0: + elif parsed.verbose > 0: logging.getLogger("notpip").setLevel(logging.INFO) - if "PIPENV_PACKAGES" in os.environ: - packages = os.environ["PIPENV_PACKAGES"].strip().split("\n") - else: - packages = sys.argv[1:] - for i, package in enumerate(packages): - if package.startswith("--"): - del packages[i] + parsed.packages += os.environ.get("PIPENV_PACKAGES", "").strip().split("\n") + return parsed + + +def _main(pre, clear, verbose, system, requirements_dir, packages): + os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) + os.environ["PIP_PYTHON_PATH"] = str(sys.executable) + from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources pypi_mirror_source = ( @@ -62,7 +57,7 @@ def main(): else None ) - def resolve(packages, pre, project, sources, clear, system): + def resolve(packages, pre, project, sources, clear, system, requirements_dir=None): return resolve_deps( packages, which, @@ -71,23 +66,23 @@ def main(): sources=sources, clear=clear, allow_global=system, + req_dir=requirements_dir ) from pipenv.core import project - sources = ( replace_pypi_sources(project.pipfile_sources, pypi_mirror_source) if pypi_mirror_source else project.pipfile_sources ) - print("using sources: %s" % sources) results = resolve( packages, - pre=do_pre, + pre=pre, project=project, sources=sources, - clear=do_clear, + clear=clear, system=system, + requirements_dir=requirements_dir, ) print("RESULTS:") if results: @@ -96,6 +91,31 @@ def main(): print(json.dumps([])) +def main(): + _patch_path() + import warnings + from pipenv.vendor.vistir.compat import ResourceWarning + warnings.simplefilter("ignore", category=ResourceWarning) + import io + import six + if six.PY3: + sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') + sys.stderr = io.TextIOWrapper(sys.stderr.buffer,encoding='utf8') + else: + from pipenv._compat import force_encoding + force_encoding() + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") + os.environ["PYTHONIOENCODING"] = str("utf-8") + parser = get_parser() + parsed, remaining = parser.parse_known_args() + # sys.argv = remaining + parsed = handle_parsed_args(parsed) + _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, + parsed.requirements_dir, parsed.packages) + + if __name__ == "__main__": _patch_path() + from pipenv.vendor import colorama + colorama.init() main() diff --git a/pipenv/test_script.py b/pipenv/test_script.py new file mode 100644 index 00000000..d599ded6 --- /dev/null +++ b/pipenv/test_script.py @@ -0,0 +1,32 @@ +# -*- coding=utf-8 -*- + +import os +import sys + + +def _patch_path(): + import site + pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + pipenv_site_dir = os.path.dirname(pipenv_libdir) + site.addsitedir(pipenv_site_dir) + for _dir in ("vendor", "patched"): + sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) + + +def test_install(): + from pipenv.vendor.vistir.contextmanagers import cd + from pipenv.vendor.click.testing import CliRunner + runner = CliRunner() + with cd("/tmp/test"): + from pipenv.core import do_lock + locked = do_lock(system=False, clear=False, pre=False, keep_outdated=False, + write=True, pypi_mirror=None) + # result = runner.invoke(cli, ["lock", "--verbose"]) + # print(result.output) + # print(result.exit_code) + print(locked) + + +if __name__ == "__main__": + _patch_path() + test_install() diff --git a/pipenv/utils.py b/pipenv/utils.py index b965e46d..c0979863 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -16,6 +16,11 @@ from click import echo as click_echo from first import first from vistir.misc import fs_str +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +from six.moves import Mapping + +from vistir.compat import ResourceWarning + try: from weakref import finalize except ImportError: @@ -38,14 +43,8 @@ from contextlib import contextmanager from . import environments from .pep508checker import lookup -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) from six.moves.urllib.parse import urlparse -from six.moves import Mapping - -if six.PY2: - - class ResourceWarning(Warning): - pass +from urllib3 import util as urllib3_util specifiers = [k for k in lookup.keys()] @@ -127,20 +126,14 @@ def parse_python_version(output): def python_version(path_to_python): - import delegator + from .vendor.pythonfinder.utils import get_python_version if not path_to_python: return None try: - c = delegator.run([path_to_python, "--version"], block=False) + version = get_python_version(path_to_python) except Exception: return None - c.block() - version = parse_python_version(c.out.strip() or c.err.strip()) - try: - version = u"{major}.{minor}.{micro}".format(**version) - except TypeError: - return None return version @@ -194,7 +187,7 @@ def prepare_pip_source_args(sources, pip_args=None): # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): pip_args.extend( - ["--trusted-host", urlparse(sources[0]["url"]).hostname] + ["--trusted-host", urllib3_util.parse_url(sources[0]["url"]).host] ) # Add additional sources as extra indexes. if len(sources) > 1: @@ -203,7 +196,7 @@ def prepare_pip_source_args(sources, pip_args=None): # Trust the host if it's not verified. if not source.get("verify_ssl", True): pip_args.extend( - ["--trusted-host", urlparse(source["url"]).hostname] + ["--trusted-host", urllib3_util.parse_url(source["url"]).host] ) return pip_args @@ -218,28 +211,26 @@ def actually_resolve_deps( pre, req_dir=None, ): - from .patched.notpip._internal import basecommand - from .patched.notpip._internal.req import parse_requirements - from .patched.notpip._internal.exceptions import DistributionNotFound - from .patched.notpip._vendor.requests.exceptions import HTTPError + from .vendor.pip_shims.shims import ( + Command, parse_requirements, DistributionNotFound + ) + from .vendor.requests.exceptions import HTTPError from pipenv.patched.piptools.resolver import Resolver from pipenv.patched.piptools.repositories.pypi import PyPIRepository from pipenv.patched.piptools.scripts.compile import get_pip_command from pipenv.patched.piptools import logging as piptools_logging from pipenv.patched.piptools.exceptions import NoCandidateFound from .vendor.requirementslib.models.requirements import Requirement - from ._compat import TemporaryDirectory, NamedTemporaryFile + from .vendor.vistir.path import create_tracked_tempdir, create_tracked_tempfile - class PipCommand(basecommand.Command): + class PipCommand(Command): """Needed for pip-tools.""" name = "PipCommand" constraints = [] - cleanup_req_dir = False if not req_dir: - req_dir = TemporaryDirectory(suffix="-requirements", prefix="pipenv-") - cleanup_req_dir = True + req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") for dep in deps: if not dep: continue @@ -267,26 +258,26 @@ def actually_resolve_deps( if sources: pip_args = prepare_pip_source_args(sources, pip_args) if environments.is_verbose(): - print("Using pip: {0}".format(" ".join(pip_args))) - with NamedTemporaryFile( + click_echo(crayons.blue("Using pip: {0}".format(" ".join(pip_args))), err=True) + constraints_file = create_tracked_tempfile( mode="w", prefix="pipenv-", suffix="-constraints.txt", - dir=req_dir.name, + dir=req_dir, delete=False, - ) as f: - if sources: - requirementstxt_sources = " ".join(pip_args) if pip_args else "" - requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") - f.write(u"{0}\n".format(requirementstxt_sources)) - f.write(u"\n".join([_constraint for _constraint in constraints])) - constraints_file = f.name + ) + if sources: + requirementstxt_sources = " ".join(pip_args) if pip_args else "" + requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") + constraints_file.write(u"{0}\n".format(requirementstxt_sources)) + constraints_file.write(u"\n".join([_constraint for _constraint in constraints])) + constraints_file.close() pip_options, _ = pip_command.parser.parse_args(pip_args) pip_options.cache_dir = environments.PIPENV_CACHE_DIR session = pip_command._build_session(pip_options) pypi = PyPIRepository(pip_options=pip_options, use_json=False, session=session) constraints = parse_requirements( - constraints_file, finder=pypi.finder, session=pypi.session, options=pip_options + constraints_file.name, finder=pypi.finder, session=pypi.session, options=pip_options ) constraints = [c for c in constraints] if environments.is_verbose(): @@ -324,13 +315,9 @@ def actually_resolve_deps( click_echo( crayons.blue( "Please check your version specifier and version number. See PEP440 for more information." - ) + ), err=True ) - if cleanup_req_dir: - req_dir.cleanup() raise RuntimeError - if cleanup_req_dir: - req_dir.cleanup() return (resolved_tree, hashes, markers_lookup, resolver) @@ -344,42 +331,80 @@ def venv_resolve_deps( pypi_mirror=None, ): from .vendor.vistir.misc import fs_str + from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError + from .vendor.vistir.path import create_tracked_tempdir + from .cmdparse import Script + from .core import spinner + from .vendor.pexpect.exceptions import EOF, TIMEOUT from .vendor import delegator from . import resolver + from ._compat import decode_output import json if not deps: return [] - resolver = escape_grouped_arguments(resolver.__file__.rstrip("co")) - cmd = "{0} {1} {2} {3} {4}".format( - escape_grouped_arguments(which("python", allow_global=allow_global)), - resolver, - "--pre" if pre else "", - "--clear" if clear else "", - "--system" if allow_global else "", - ) + + req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") + cmd = [ + which("python", allow_global=allow_global), + Path(resolver.__file__.rstrip("co")).as_posix() + ] + if pre: + cmd.append("--pre") + if clear: + cmd.append("--clear") + if allow_global: + cmd.append("--system") with temp_environ(): os.environ = {fs_str(k): fs_str(val) for k, val in os.environ.items()} os.environ["PIPENV_PACKAGES"] = str("\n".join(deps)) if pypi_mirror: os.environ["PIPENV_PYPI_MIRROR"] = str(pypi_mirror) os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) - c = delegator.run(cmd, block=True) - try: - assert c.return_code == 0 - except AssertionError: - if environments.is_verbose(): - click_echo(c.out, err=True) - click_echo(c.err, err=True) - else: - click_echo(c.err[(int(len(c.err) / 2) - 1):], err=True) - sys.exit(c.return_code) + os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) + os.environ["PIP_NO_INPUT"] = fs_str("1") + out = to_native_string("") + EOF.__module__ = "pexpect.exceptions" + with spinner(text=fs_str("Locking..."), spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: + c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) + _out = decode_output("") + result = None + while True: + try: + result = c.expect(u"\n", timeout=-1) + except (EOF, TIMEOUT): + pass + if result is None: + break + _out = c.subprocess.before + if _out is not None: + _out = decode_output("{0}".format(_out)) + out += _out + sp.text = to_native_string("{0}".format(_out[:100])) + if environments.is_verbose(): + if _out is not None: + sp._hide_cursor() + sp.write(_out.rstrip()) + sp._show_cursor() + c.block() + if c.return_code != 0: + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Locking Failed!" + )) + click_echo(c.out.strip(), err=True) + click_echo(c.err.strip(), err=True) + sys.exit(c.return_code) + else: + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) if environments.is_verbose(): click_echo(c.out.split("RESULTS:")[0], err=True) try: return json.loads(c.out.split("RESULTS:")[1].strip()) - except IndexError: + except (IndexError, JSONDecodeError): + click_echo(c.out.strip(), err=True) + click_echo(c.err.strip(), err=True) raise RuntimeError("There was a problem with locking.") @@ -392,23 +417,28 @@ def resolve_deps( clear=False, pre=False, allow_global=False, + req_dir=None ): """Given a list of dependencies, return a resolved list of dependencies, using pip-tools -- and their hashes, using the warehouse API / pip. """ - from .patched.notpip._vendor.requests.exceptions import ConnectionError + from .vendor.requests.exceptions import ConnectionError from .vendor.requirementslib.models.requirements import Requirement - from ._compat import TemporaryDirectory index_lookup = {} markers_lookup = {} python_path = which("python", allow_global=allow_global) + if not os.environ.get("PIP_SRC"): + os.environ["PIP_SRC"] = project.virtualenv_src_location backup_python_path = sys.executable results = [] if not deps: return results # First (proper) attempt: - req_dir = TemporaryDirectory(prefix="pipenv-", suffix="-requirements") + req_dir = req_dir if req_dir else os.environ.get("req_dir", None) + if not req_dir: + from .vendor.vistir.path import create_tracked_tempdir + req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-requirements") with HackedPythonVersion(python_version=python, python_path=python_path): try: resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( @@ -444,7 +474,6 @@ def resolve_deps( req_dir=req_dir, ) except RuntimeError: - req_dir.cleanup() sys.exit(1) for result in resolved_tree: if not result.editable: @@ -478,7 +507,7 @@ def resolve_deps( click_echo( "{0}: Error generating hash for {1}".format( crayons.red("Warning", bold=True), name - ) + ), err=True ) # # Collect un-collectable hashes (should work with devpi). # try: @@ -503,7 +532,6 @@ def resolve_deps( entry.update({"markers": markers_lookup.get(result.name)}) entry = translate_markers(entry) results.append(entry) - req_dir.cleanup() return results @@ -526,7 +554,6 @@ def is_pinned(val): def convert_deps_to_pip(deps, project=None, r=True, include_index=True): """"Converts a Pipfile-formatted dependency to a pip-formatted one.""" - from ._compat import NamedTemporaryFile from .vendor.requirementslib.models.requirements import Requirement dependencies = [] @@ -541,7 +568,8 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=True): return dependencies # Write requirements.txt to tmp directory. - f = NamedTemporaryFile(suffix="-requirements.txt", delete=False) + from .vendor.vistir.path import create_tracked_tempfile + f = create_tracked_tempfile(suffix="-requirements.txt", delete=False) f.write("\n".join(dependencies).encode("utf-8")) f.close() return f.name @@ -610,9 +638,8 @@ def is_editable(pipfile_entry): def is_installable_file(path): """Determine if a path can potentially be installed""" - from .patched.notpip._internal.utils.misc import is_installable_dir + from .vendor.pip_shims.shims import is_installable_dir, is_archive_file from .patched.notpip._internal.utils.packaging import specifiers - from .patched.notpip._internal.download import is_archive_file from ._compat import Path if hasattr(path, "keys") and any( @@ -664,7 +691,7 @@ def is_file(package): def pep440_version(version): """Normalize version to PEP 440 standards""" - from .patched.notpip._internal.index import parse_version + from .vendor.pip_shims.shims import parse_version # Use pip built-in version parser. return str(parse_version(version)) @@ -1054,54 +1081,6 @@ def escape_cmd(cmd): return cmd -@contextmanager -def atomic_open_for_write(target, binary=False, newline=None, encoding=None): - """Atomically open `target` for writing. - - This is based on Lektor's `atomic_open()` utility, but simplified a lot - to handle only writing, and skip many multi-process/thread edge cases - handled by Werkzeug. - - How this works: - - * Create a temp file (in the same directory of the actual target), and - yield for surrounding code to write to it. - * If some thing goes wrong, try to remove the temp file. The actual target - is not touched whatsoever. - * If everything goes well, close the temp file, and replace the actual - target with this new file. - """ - from ._compat import NamedTemporaryFile - - mode = "w+b" if binary else "w" - f = NamedTemporaryFile( - dir=os.path.dirname(target), - prefix=".__atomic-write", - mode=mode, - encoding=encoding, - newline=newline, - delete=False, - ) - # set permissions to 0644 - os.chmod(f.name, stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - try: - yield f - except BaseException: - f.close() - try: - os.remove(f.name) - except OSError: - pass - raise - else: - f.close() - try: - os.remove(target) # This is needed on Windows. - except OSError: - pass - os.rename(f.name, target) # No os.replace() on Python 2. - - def safe_expandvars(value): """Call os.path.expandvars if value is a string, otherwise do nothing. """ @@ -1119,7 +1098,6 @@ def extract_uri_from_vcs_dep(dep): def get_vcs_deps( project, - pip_freeze=None, which=None, clear=False, pre=False, @@ -1127,8 +1105,8 @@ def get_vcs_deps( dev=False, pypi_mirror=None, ): - from ._compat import TemporaryDirectory, Path - import atexit + from .vendor.vistir.compat import Path + from .vendor.vistir.path import create_tracked_tempdir from .vendor.requirementslib.models.requirements import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" @@ -1138,24 +1116,19 @@ def get_vcs_deps( packages = getattr(project, section) except AttributeError: return [], [] - if os.environ.get("PIP_SRC"): - src_dir = Path( - os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src")) - ) - src_dir.mkdir(mode=0o775, exist_ok=True) - else: - src_dir = TemporaryDirectory(prefix="pipenv-lock-dir") - atexit.register(src_dir.cleanup) for pkg_name, pkg_pipfile in packages.items(): requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) name = requirement.normalized_name commit_hash = None if requirement.is_vcs: - with locked_repository(requirement) as repo: - commit_hash = repo.get_commit_hash() - lockfile[name] = requirement.pipfile_entry[1] - lockfile[name]['ref'] = commit_hash - reqs.append(requirement) + try: + with locked_repository(requirement) as repo: + commit_hash = repo.get_commit_hash() + lockfile[name] = requirement.pipfile_entry[1] + lockfile[name]['ref'] = commit_hash + reqs.append(requirement) + except OSError: + continue return reqs, lockfile @@ -1172,7 +1145,7 @@ def translate_markers(pipfile_entry): """ if not isinstance(pipfile_entry, Mapping): raise TypeError("Entry is not a pipfile formatted mapping.") - from notpip._vendor.distlib.markers import DEFAULT_CONTEXT as marker_context + from .vendor.distlib.markers import DEFAULT_CONTEXT as marker_context from .vendor.packaging.markers import Marker from .vendor.vistir.misc import dedup @@ -1280,12 +1253,11 @@ def is_virtual_environment(path): @contextmanager def locked_repository(requirement): from .vendor.vistir.path import create_tracked_tempdir - from .vendor.vistir.misc import fs_str - src_dir = create_tracked_tempdir(prefix="pipenv-src") if not requirement.is_vcs: return original_base = os.environ.pop("PIP_SHIMS_BASE_MODULE", None) os.environ["PIP_SHIMS_BASE_MODULE"] = fs_str("pipenv.patched.notpip") + src_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-src") try: with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo: yield repo @@ -1329,3 +1301,43 @@ def parse_indexes(line): indexes = index + extra_indexes trusted_hosts = args.trusted_host if args.trusted_host else [] return indexes, trusted_hosts, remainder + + +def fix_venv_site(venv_lib_dir): + # From https://github.com/pypa/pip/blob/master/tests/lib/venv.py#L84 + # Prevent accidental inclusions of site packages during virtualenv operations + from .vendor.vistir.compat import Path + import compileall + site_py = Path(venv_lib_dir).joinpath('site.py').as_posix() + with open(site_py) as fp: + site_contents = fp.read() + for pattern, replace in ( + ( + # Ensure enabling user site does not result in adding + # the real site-packages' directory to `sys.path`. + ( + '\ndef virtual_addsitepackages(known_paths):\n' + ), + ( + '\ndef virtual_addsitepackages(known_paths):\n' + ' return known_paths\n' + ), + ), + ( + # Fix sites ordering: user site must be added before system. + ( + '\n paths_in_sys = addsitepackages(paths_in_sys)' + '\n paths_in_sys = addusersitepackages(paths_in_sys)\n' + ), + ( + '\n paths_in_sys = addusersitepackages(paths_in_sys)' + '\n paths_in_sys = addsitepackages(paths_in_sys)\n' + ), + ), + ): + if pattern in site_contents and replace not in site_contents: + site_contents = site_contents.replace(pattern, replace) + with open(site_py, 'w') as fp: + fp.write(site_contents) + # Make sure bytecode is up-to-date too. + assert compileall.compile_file(str(site_py), quiet=1, force=True) diff --git a/pipenv/vendor/backports/__init__.py b/pipenv/vendor/backports/__init__.py index 3cd30963..0c64b4c1 100644 --- a/pipenv/vendor/backports/__init__.py +++ b/pipenv/vendor/backports/__init__.py @@ -1,7 +1,5 @@ -# See https://pypi.python.org/pypi/backports - -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) from . import weakref from . import enum from . import shutil_get_terminal_size +from . import functools_lru_cache diff --git a/pipenv/vendor/backports/functools_lru_cache.LICENSE b/pipenv/vendor/backports/functools_lru_cache.LICENSE new file mode 100644 index 00000000..5e795a61 --- /dev/null +++ b/pipenv/vendor/backports/functools_lru_cache.LICENSE @@ -0,0 +1,7 @@ +Copyright Jason R. Coombs + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pipenv/vendor/backports/functools_lru_cache.py b/pipenv/vendor/backports/functools_lru_cache.py new file mode 100644 index 00000000..707c6c76 --- /dev/null +++ b/pipenv/vendor/backports/functools_lru_cache.py @@ -0,0 +1,184 @@ +from __future__ import absolute_import + +import functools +from collections import namedtuple +from threading import RLock + +_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) + + +@functools.wraps(functools.update_wrapper) +def update_wrapper(wrapper, + wrapped, + assigned = functools.WRAPPER_ASSIGNMENTS, + updated = functools.WRAPPER_UPDATES): + """ + Patch two bugs in functools.update_wrapper. + """ + # workaround for http://bugs.python.org/issue3445 + assigned = tuple(attr for attr in assigned if hasattr(wrapped, attr)) + wrapper = functools.update_wrapper(wrapper, wrapped, assigned, updated) + # workaround for https://bugs.python.org/issue17482 + wrapper.__wrapped__ = wrapped + return wrapper + + +class _HashedSeq(list): + __slots__ = 'hashvalue' + + def __init__(self, tup, hash=hash): + self[:] = tup + self.hashvalue = hash(tup) + + def __hash__(self): + return self.hashvalue + + +def _make_key(args, kwds, typed, + kwd_mark=(object(),), + fasttypes=set([int, str, frozenset, type(None)]), + sorted=sorted, tuple=tuple, type=type, len=len): + 'Make a cache key from optionally typed positional and keyword arguments' + key = args + if kwds: + sorted_items = sorted(kwds.items()) + key += kwd_mark + for item in sorted_items: + key += item + if typed: + key += tuple(type(v) for v in args) + if kwds: + key += tuple(type(v) for k, v in sorted_items) + elif len(key) == 1 and type(key[0]) in fasttypes: + return key[0] + return _HashedSeq(key) + + +def lru_cache(maxsize=100, typed=False): + """Least-recently-used cache decorator. + + If *maxsize* is set to None, the LRU features are disabled and the cache + can grow without bound. + + If *typed* is True, arguments of different types will be cached separately. + For example, f(3.0) and f(3) will be treated as distinct calls with + distinct results. + + Arguments to the cached function must be hashable. + + View the cache statistics named tuple (hits, misses, maxsize, currsize) with + f.cache_info(). Clear the cache and statistics with f.cache_clear(). + Access the underlying function with f.__wrapped__. + + See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used + + """ + + # Users should only access the lru_cache through its public API: + # cache_info, cache_clear, and f.__wrapped__ + # The internals of the lru_cache are encapsulated for thread safety and + # to allow the implementation to change (including a possible C version). + + def decorating_function(user_function): + + cache = dict() + stats = [0, 0] # make statistics updateable non-locally + HITS, MISSES = 0, 1 # names for the stats fields + make_key = _make_key + cache_get = cache.get # bound method to lookup key or return None + _len = len # localize the global len() function + lock = RLock() # because linkedlist updates aren't threadsafe + root = [] # root of the circular doubly linked list + root[:] = [root, root, None, None] # initialize by pointing to self + nonlocal_root = [root] # make updateable non-locally + PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields + + if maxsize == 0: + + def wrapper(*args, **kwds): + # no caching, just do a statistics update after a successful call + result = user_function(*args, **kwds) + stats[MISSES] += 1 + return result + + elif maxsize is None: + + def wrapper(*args, **kwds): + # simple caching without ordering or size limit + key = make_key(args, kwds, typed) + result = cache_get(key, root) # root used here as a unique not-found sentinel + if result is not root: + stats[HITS] += 1 + return result + result = user_function(*args, **kwds) + cache[key] = result + stats[MISSES] += 1 + return result + + else: + + def wrapper(*args, **kwds): + # size limited caching that tracks accesses by recency + key = make_key(args, kwds, typed) if kwds or typed else args + with lock: + link = cache_get(key) + if link is not None: + # record recent use of the key by moving it to the front of the list + root, = nonlocal_root + link_prev, link_next, key, result = link + link_prev[NEXT] = link_next + link_next[PREV] = link_prev + last = root[PREV] + last[NEXT] = root[PREV] = link + link[PREV] = last + link[NEXT] = root + stats[HITS] += 1 + return result + result = user_function(*args, **kwds) + with lock: + root, = nonlocal_root + if key in cache: + # getting here means that this same key was added to the + # cache while the lock was released. since the link + # update is already done, we need only return the + # computed result and update the count of misses. + pass + elif _len(cache) >= maxsize: + # use the old root to store the new key and result + oldroot = root + oldroot[KEY] = key + oldroot[RESULT] = result + # empty the oldest link and make it the new root + root = nonlocal_root[0] = oldroot[NEXT] + oldkey = root[KEY] + root[KEY] = root[RESULT] = None + # now update the cache dictionary for the new links + del cache[oldkey] + cache[key] = oldroot + else: + # put result in a new link at the front of the list + last = root[PREV] + link = [last, root, key, result] + last[NEXT] = root[PREV] = cache[key] = link + stats[MISSES] += 1 + return result + + def cache_info(): + """Report cache statistics""" + with lock: + return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache)) + + def cache_clear(): + """Clear the cache and cache statistics""" + with lock: + cache.clear() + root = nonlocal_root[0] + root[:] = [root, root, None, None] + stats[:] = [0, 0] + + wrapper.__wrapped__ = user_function + wrapper.cache_info = cache_info + wrapper.cache_clear = cache_clear + return update_wrapper(wrapper, user_function) + + return decorating_function diff --git a/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py b/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py index fa12816e..cfcbdf66 100644 --- a/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py +++ b/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py @@ -1,9 +1,11 @@ -__title__ = "shutil_backports" -__version__ = "0.1.0" +"""A backport of the get_terminal_size function from Python 3.3's shutil.""" + +__title__ = "backports.shutil_get_terminal_size" +__version__ = "1.0.0" __license__ = "MIT" __author__ = "Christopher Rosell" __copyright__ = "Copyright 2014 Christopher Rosell" __all__ = ["get_terminal_size"] -from .get_terminal_size import * +from .get_terminal_size import get_terminal_size diff --git a/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py b/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py index f1336e58..28c96da8 100644 --- a/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py +++ b/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py @@ -39,7 +39,7 @@ try: except Exception: pass - return columns, lines + return terminal_size(columns, lines) except ImportError: import fcntl @@ -52,7 +52,7 @@ except ImportError: except Exception: columns = lines = 0 - return columns, lines + return terminal_size(columns, lines) def get_terminal_size(fallback=(80, 24)): @@ -74,7 +74,7 @@ def get_terminal_size(fallback=(80, 24)): The value returned is a named tuple of type os.terminal_size. """ - # Attempt to use the environment first + # Try the environment first try: columns = int(os.environ["COLUMNS"]) except (KeyError, ValueError): @@ -88,13 +88,14 @@ def get_terminal_size(fallback=(80, 24)): # Only query if necessary if columns <= 0 or lines <= 0: try: - columns, lines = _get_terminal_size(sys.__stdout__.fileno()) + size = _get_terminal_size(sys.__stdout__.fileno()) except (NameError, OSError): - pass + size = terminal_size(*fallback) - # Use fallback as last resort - if columns <= 0 and lines <= 0: - columns, lines = fallback + if columns <= 0: + columns = size.columns + if lines <= 0: + lines = size.lines return terminal_size(columns, lines) diff --git a/pipenv/vendor/certifi/__init__.py b/pipenv/vendor/certifi/__init__.py index aa329fbb..50f2e130 100644 --- a/pipenv/vendor/certifi/__init__.py +++ b/pipenv/vendor/certifi/__init__.py @@ -1,3 +1,3 @@ from .core import where, old_where -__version__ = "2018.08.24" +__version__ = "2018.10.15" diff --git a/pipenv/vendor/certifi/cacert.pem b/pipenv/vendor/certifi/cacert.pem index 85de024e..e75d85b3 100644 --- a/pipenv/vendor/certifi/cacert.pem +++ b/pipenv/vendor/certifi/cacert.pem @@ -326,36 +326,6 @@ OCiNUW7dFGdTbHFcJoRNdVq2fmBWqU2t+5sel/MN2dKXVHfaPRK34B7vCAas+YWH QMAJKOSLakhT2+zNVVXxxvjpoixMptEmX36vWkzaH6byHCx+rgIW0lbQL1dTR+iS -----END CERTIFICATE----- -# Issuer: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Subject: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Label: "Visa eCommerce Root" -# Serial: 25952180776285836048024890241505565794 -# MD5 Fingerprint: fc:11:b8:d8:08:93:30:00:6d:23:f9:7e:eb:52:1e:02 -# SHA1 Fingerprint: 70:17:9b:86:8c:00:a4:fa:60:91:52:22:3f:9f:3e:32:bd:e0:05:62 -# SHA256 Fingerprint: 69:fa:c9:bd:55:fb:0a:c7:8d:53:bb:ee:5c:f1:d5:97:98:9f:d0:aa:ab:20:a2:51:51:bd:f1:73:3e:e7:d1:22 ------BEGIN CERTIFICATE----- -MIIDojCCAoqgAwIBAgIQE4Y1TR0/BvLB+WUF1ZAcYjANBgkqhkiG9w0BAQUFADBr -MQswCQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRl -cm5hdGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNv -bW1lcmNlIFJvb3QwHhcNMDIwNjI2MDIxODM2WhcNMjIwNjI0MDAxNjEyWjBrMQsw -CQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRlcm5h -dGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNvbW1l -cmNlIFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvV95WHm6h -2mCxlCfLF9sHP4CFT8icttD0b0/Pmdjh28JIXDqsOTPHH2qLJj0rNfVIsZHBAk4E -lpF7sDPwsRROEW+1QK8bRaVK7362rPKgH1g/EkZgPI2h4H3PVz4zHvtH8aoVlwdV -ZqW1LS7YgFmypw23RuwhY/81q6UCzyr0TP579ZRdhE2o8mCP2w4lPJ9zcc+U30rq -299yOIzzlr3xF7zSujtFWsan9sYXiwGd/BmoKoMWuDpI/k4+oKsGGelT84ATB+0t -vz8KPFUgOSwsAGl0lUq8ILKpeeUYiZGo3BxN77t+Nwtd/jmliFKMAGzsGHxBvfaL -dXe6YJ2E5/4tAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD -AgEGMB0GA1UdDgQWBBQVOIMPPyw/cDMezUb+B4wg4NfDtzANBgkqhkiG9w0BAQUF -AAOCAQEAX/FBfXxcCLkr4NWSR/pnXKUTwwMhmytMiUbPWU3J/qVAtmPN3XEolWcR -zCSs00Rsca4BIGsDoo8Ytyk6feUWYFN4PMCvFYP3j1IzJL1kk5fui/fbGKhtcbP3 -LBfQdCVp9/5rPJS+TUtBjE7ic9DjkCJzQ83z7+pzzkWKsKZJ/0x9nXGIxHYdkFsd -7v3M9+79YKWxehZx0RbQfBI8bGmX265fOZpwLwU8GUYEmSA20GBuYQa7FkKMcPcw -++DbZqMAAb3mLNqRX6BGi01qnD093QVG/na/oAo85ADmJ7f/hC3euiInlhBx6yLt -398znM/jra6O1I7mT1GvFpLgXPYHDw== ------END CERTIFICATE----- - # Issuer: CN=AAA Certificate Services O=Comodo CA Limited # Subject: CN=AAA Certificate Services O=Comodo CA Limited # Label: "Comodo AAA Services root" diff --git a/pipenv/vendor/cursor/LICENSE b/pipenv/vendor/cursor/LICENSE new file mode 100644 index 00000000..00023c80 --- /dev/null +++ b/pipenv/vendor/cursor/LICENSE @@ -0,0 +1,5 @@ +This work is licensed under the Creative Commons +Attribution-ShareAlike 2.5 International License. To view a copy of +this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or +send a letter to Creative Commons, PO Box 1866, Mountain View, +CA 94042, USA. diff --git a/pipenv/vendor/cursor/__init__.py b/pipenv/vendor/cursor/__init__.py new file mode 100644 index 00000000..76a4f671 --- /dev/null +++ b/pipenv/vendor/cursor/__init__.py @@ -0,0 +1,4 @@ +from .cursor import hide, show, HiddenCursor + +__all__ = ["hide", "show", "HiddenCursor"] + diff --git a/pipenv/vendor/cursor/cursor.py b/pipenv/vendor/cursor/cursor.py new file mode 100644 index 00000000..e4407c02 --- /dev/null +++ b/pipenv/vendor/cursor/cursor.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +## Author: James Spencer: http://stackoverflow.com/users/1375885/james-spencer +## Packager: Gijs TImmers: https://github.com/GijsTimmers + +## Based on James Spencer's answer on StackOverflow: +## http://stackoverflow.com/questions/5174810/how-to-turn-off-blinking-cursor-in-command-window + +## Licence: CC-BY-SA-2.5 +## http://creativecommons.org/licenses/by-sa/2.5/ + +## This work is licensed under the Creative Commons +## Attribution-ShareAlike 2.5 International License. To view a copy of +## this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or +## send a letter to Creative Commons, PO Box 1866, Mountain View, +## CA 94042, USA. + +import sys +import os + +if os.name == 'nt': + import ctypes + + class _CursorInfo(ctypes.Structure): + _fields_ = [("size", ctypes.c_int), + ("visible", ctypes.c_byte)] + +def hide(stream=sys.stdout): + if os.name == 'nt': + ci = _CursorInfo() + handle = ctypes.windll.kernel32.GetStdHandle(-11) + ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) + ci.visible = False + ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) + elif os.name == 'posix': + stream.write("\033[?25l") + stream.flush() + +def show(stream=sys.stdout): + if os.name == 'nt': + ci = _CursorInfo() + handle = ctypes.windll.kernel32.GetStdHandle(-11) + ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) + ci.visible = True + ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) + elif os.name == 'posix': + stream.write("\033[?25h") + stream.flush() + +class HiddenCursor(object): + def __init__(self, stream=sys.stdout): + self._stream = stream + def __enter__(self): + hide(stream=self._stream) + def __exit__(self, type, value, traceback): + show(stream=self._stream) \ No newline at end of file diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py index d15aeb97..cf6f91c8 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py @@ -7,6 +7,8 @@ import locale import errno from pexpect.popen_spawn import PopenSpawn +import pexpect +pexpect.EOF.__module__ = "pexpect.exceptions" # Include `unicode` in STR_TYPES for Python 2.X try: @@ -110,8 +112,11 @@ class Command(object): if self.subprocess.before: result += self.subprocess.before - if self.subprocess.after: - result += self.subprocess.after + if self.subprocess.after and self.subprocess.after is not pexpect.EOF: + try: + result += self.subprocess.after + except (pexpect.EOF, pexpect.TIMEOUT): + pass result += self.subprocess.read() return result @@ -178,6 +183,7 @@ class Command(object): # Use subprocess. if self.blocking: popen_kwargs = self._default_popen_kwargs.copy() + del popen_kwargs["stdin"] popen_kwargs["universal_newlines"] = not binary if cwd: popen_kwargs["cwd"] = cwd @@ -205,7 +211,10 @@ class Command(object): if self.blocking: raise RuntimeError("expect can only be used on non-blocking commands.") - self.subprocess.expect(pattern=pattern, timeout=timeout) + try: + self.subprocess.expect(pattern=pattern, timeout=timeout) + except pexpect.EOF: + pass def send(self, s, end=os.linesep, signal=False): """Sends the given string or signal to std_in.""" @@ -234,14 +243,25 @@ class Command(object): """Blocks until process is complete.""" if self._uses_subprocess: # consume stdout and stderr - try: - stdout, stderr = self.subprocess.communicate() - self.__out = stdout - self.__err = stderr - except ValueError: - pass # Don't read from finished subprocesses. + if self.blocking: + try: + stdout, stderr = self.subprocess.communicate() + self.__out = stdout + self.__err = stderr + except ValueError: + pass # Don't read from finished subprocesses. + else: + self.subprocess.stdin.close() + self.std_out.close() + self.std_err.close() + self.subprocess.wait() else: - self.subprocess.wait() + self.subprocess.sendeof() + try: + self.subprocess.wait() + finally: + if self.subprocess.proc.stdout: + self.subprocess.proc.stdout.close() def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next @@ -263,7 +283,6 @@ class Command(object): c.run(block=False, cwd=cwd) if data: c.send(data) - c.subprocess.sendeof() c.block() return c diff --git a/pipenv/vendor/modutil.LICENSE b/pipenv/vendor/modutil.LICENSE deleted file mode 100644 index f680f071..00000000 --- a/pipenv/vendor/modutil.LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2018, Brett Cannon -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/vendor/parse.py b/pipenv/vendor/parse.py index ba58decb..7f9f0786 100644 --- a/pipenv/vendor/parse.py +++ b/pipenv/vendor/parse.py @@ -186,6 +186,19 @@ And messing about with alignment: Note that the "center" alignment does not test to make sure the value is centered - it just strips leading and trailing whitespace. +Width and precision may be used to restrict the size of matched text +from the input. Width specifies a minimum size and precision specifies +a maximum. For example: + +>>> parse('{:.2}{:.2}', 'look') # specifying precision + +>>> parse('{:4}{:4}', 'look at that') # specifying width + +>>> parse('{:4}{:.4}', 'look at that') # specifying both + +>>> parse('{:2d}{:2d}', '0440') # parsing two contiguous numbers + + Some notes for the date and time types: - the presence of the time part is optional (including ISO 8601, starting @@ -329,6 +342,9 @@ the pattern, the actual match represents the shortest successful match for **Version history (in brief)**: +- 1.9.0 We now honor precision and width specifiers when parsing numbers + and strings, allowing parsing of concatenated elements of fixed width + (thanks Julia Signell) - 1.8.4 Add LICENSE file at request of packagers. Correct handling of AM/PM to follow most common interpretation. Correct parsing of hexadecimal that looks like a binary prefix. @@ -389,7 +405,7 @@ See the end of the source file for the license of use. ''' from __future__ import absolute_import -__version__ = '1.8.4' +__version__ = '1.9.0' # yes, I now have two problems import re @@ -977,7 +993,11 @@ class Parser(object): self._group_index += 2 self._type_conversions[group] = lambda s, m: float(s) elif type == 'd': - s = r'\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|0[oO][0-7]+' + if format.get('width'): + width = '{1,%s}' % int(format['width']) + else: + width = '+' + s = '\\d{w}|0[xX][0-9a-fA-F]{w}|0[bB][01]{w}|0[oO][0-7]{w}'.format(w=width) self._type_conversions[group] = int_convert(10) elif type == 'ti': s = r'(\d{4}-\d\d-\d\d)((\s+|T)%s)?(Z|\s*[-+]\d\d:?\d\d)?' % \ @@ -1038,6 +1058,13 @@ class Parser(object): elif type: s = r'\%s+' % type + elif format.get('precision'): + if format.get('width'): + s = '.{%s,%s}?' % (format['width'], format['precision']) + else: + s = '.{1,%s}?' % format['precision'] + elif format.get('width'): + s = '.{%s,}?' % format['width'] else: s = '.+?' @@ -1053,8 +1080,6 @@ class Parser(object): if not fill: fill = '0' s = '%s*' % fill + s - elif format['zero']: - s = '0*' + s # allow numbers to be prefixed with a sign s = r'[-+ ]?' + s diff --git a/pipenv/vendor/passa/internals/dependencies.py b/pipenv/vendor/passa/internals/dependencies.py index 53b19b17..358cc33b 100644 --- a/pipenv/vendor/passa/internals/dependencies.py +++ b/pipenv/vendor/passa/internals/dependencies.py @@ -154,6 +154,7 @@ def _get_dependencies_from_json(ireq, sources): return dependencies except Exception as e: print("unable to read dependencies via {0} ({1})".format(url, e)) + session.close() return diff --git a/pipenv/vendor/pip_shims/__init__.py b/pipenv/vendor/pip_shims/__init__.py index 70fb0d58..22bf130f 100644 --- a/pipenv/vendor/pip_shims/__init__.py +++ b/pipenv/vendor/pip_shims/__init__.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import sys -__version__ = '0.3.1' +__version__ = '0.3.2' from . import shims diff --git a/pipenv/vendor/pip_shims/shims.py b/pipenv/vendor/pip_shims/shims.py index 7b81a608..34de6906 100644 --- a/pipenv/vendor/pip_shims/shims.py +++ b/pipenv/vendor/pip_shims/shims.py @@ -43,6 +43,9 @@ class _shims(object): "pip_shims.utils": utils } self.pip_version = getattr(self._modules["pip"], "__version__") + version_types = ["post", "pre", "dev", "rc"] + if any(post in self.pip_version.rsplit(".")[-1] for post in version_types): + self.pip_version, _, _ = self.pip_version.rpartition(".") self.parsed_pip_version = self._parse(self.pip_version) self._contextmanagers = ("RequirementTracker",) self._moves = { @@ -140,6 +143,7 @@ class _shims(object): ("wheel.WheelCache", "7", "9.0.3") ), "WheelBuilder": ("wheel.WheelBuilder", "7.0.0", "9999"), + "PyPI": ("models.index.PyPI", "7.0.0", "9999"), } def _ensure_methods(self, cls, classname, *methods): diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 672724b4..60192826 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.2' +__version__ = '1.1.6' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/cli.py b/pipenv/vendor/pythonfinder/cli.py index b5aa7da3..6e7980fe 100644 --- a/pipenv/vendor/pythonfinder/cli.py +++ b/pipenv/vendor/pythonfinder/cli.py @@ -17,7 +17,7 @@ from .pythonfinder import Finder @click.option( "--version", is_flag=True, default=False, help="Display PythonFinder version." ) -@click.option("--ignore-unsupported/--no-unsupported", is_flag=True, default=True, help="Ignore unsupported python versions.") +@click.option("--ignore-unsupported/--no-unsupported", is_flag=True, default=True, envvar="PYTHONFINDER_IGNORE_UNSUPPORTED", help="Ignore unsupported python versions.") @click.version_option(prog_name='pyfinder', version=__version__) @click.pass_context def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsupported=True): @@ -30,40 +30,44 @@ def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsup sys.exit(0) finder = Finder(ignore_unsupported=ignore_unsupported) if findall: - versions = finder.find_all_python_versions() + versions = [v for v in finder.find_all_python_versions()] if versions: click.secho("Found python at the following locations:", fg="green") for v in versions: py = v.py_version + comes_from = getattr(py, "comes_from", None) + if comes_from is not None: + comes_from_path = getattr(comes_from, "path", v.path) + else: + comes_from_path = v.path click.secho( - "Python: {py.version!s} ({py.architecture!s}) @ {py.comes_from.path!s}".format( - py=py + "{py.name!s}: {py.version!s} ({py.architecture!s}) @ {comes_from!s}".format( + py=py, comes_from=comes_from_path ), fg="yellow", ) + sys.exit(0) else: click.secho( "ERROR: No valid python versions found! Check your path and try again.", fg="red", ) if find: - if any([find.startswith("{0}".format(n)) for n in range(10)]): - found = finder.find_python_version(find.strip()) - else: - found = finder.system_path.python_executables + click.secho("Searching for python: {0!s}".format(find.strip()), fg="yellow") + found = finder.find_python_version(find.strip()) if found: - click.echo("Found Python Version: {0}".format(found), color="white") + click.secho("Found python at the following locations:", fg="green") sys.exit(0) else: - click.echo("Failed to find matching executable...") + click.secho("Failed to find matching executable...", fg="yellow") sys.exit(1) elif which: found = finder.system_path.which(which.strip()) if found: - click.echo("Found Executable: {0}".format(found), color="white") + click.secho("Found Executable: {0}".format(found), fg="white") sys.exit(0) else: - click.echo("Failed to find matching executable...") + click.secho("Failed to find matching executable...", fg="yellow") sys.exit(1) else: click.echo("Please provide a command", color="red") diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index 2cdb5fd9..7c69b9fc 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -15,3 +15,6 @@ if sys.maxsize > 2 ** 32: IS_64BIT_OS = platform.machine() == "AMD64" else: IS_64BIT_OS = False + + +IGNORE_UNSUPPORTED = bool(os.environ.get("PYTHONFINDER_IGNORE_UNSUPPORTED", False)) diff --git a/pipenv/vendor/pythonfinder/models/mixins.py b/pipenv/vendor/pythonfinder/models/mixins.py index 8cbd45df..7d406548 100644 --- a/pipenv/vendor/pythonfinder/models/mixins.py +++ b/pipenv/vendor/pythonfinder/models/mixins.py @@ -2,12 +2,14 @@ from __future__ import absolute_import, unicode_literals import abc +import attr import operator import six -from ..utils import KNOWN_EXTS, unnest +from ..utils import ensure_path, KNOWN_EXTS, unnest +@attr.s class BasePath(object): def which(self, name): """Search in this path for an executable. @@ -33,7 +35,14 @@ class BasePath(object): return found def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search for a specific python version on the path. Return all copies @@ -44,6 +53,7 @@ class BasePath(object): :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ @@ -52,7 +62,14 @@ class BasePath(object): "find_all_python_versions" if self.is_dir else "find_python_version" ) sub_finder = operator.methodcaller( - call_method, major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + call_method, + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, ) if not self.is_dir: return sub_finder(self) @@ -61,7 +78,14 @@ class BasePath(object): return [c for c in sorted(path_filter, key=version_sort, reverse=True)] def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search or self for the specified Python version and return the first match. @@ -72,6 +96,7 @@ class BasePath(object): :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ @@ -83,12 +108,13 @@ class BasePath(object): pre=pre, dev=dev, arch=arch, + name=name, ) is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") if not self.is_dir: - if self.is_python and self.as_python and version_matcher(self.as_python): - return self + if self.is_python and self.as_python and version_matcher(self.py_version): + return attr.evolve(self) return finder = ( (child, child.as_python) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index c90d9be3..20b2c196 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -10,6 +10,7 @@ from collections import defaultdict from itertools import chain import attr +import six from cached_property import cached_property @@ -19,8 +20,12 @@ from .mixins import BasePath from ..environment import PYENV_INSTALLED, PYENV_ROOT from ..exceptions import InvalidPythonVersion from ..utils import ( - ensure_path, filter_pythons, looks_like_python, optional_instance_of, - path_is_known_executable, unnest + ensure_path, + filter_pythons, + looks_like_python, + optional_instance_of, + path_is_known_executable, + unnest, ) from .python import PythonVersion @@ -75,7 +80,7 @@ class SystemPath(object): if entry not in self._version_dict[version]: self._version_dict[version].append(entry) continue - if isinstance(entry, VersionPath): + if type(entry).__name__ == "VersionPath": for path in entry.paths.values(): if path not in self._version_dict[version] and path.is_python: self._version_dict[version].append(path) @@ -130,17 +135,16 @@ class SystemPath(object): pyenv_index = self.path_order.index(last_pyenv) except ValueError: return - self.pyenv_finder = PyenvFinder.create(root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported) - # paths = (v.paths.values() for v in self.pyenv_finder.versions.values()) - root_paths = ( - p for path in self.pyenv_finder.expanded_paths for p in path if p.is_root + self.pyenv_finder = PyenvFinder.create( + root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported ) + root_paths = [p for p in self.pyenv_finder.roots] before_path = self.path_order[: pyenv_index + 1] after_path = self.path_order[pyenv_index + 2 :] self.path_order = ( - before_path + [p.path.as_posix() for p in root_paths] + after_path + before_path + [p.as_posix() for p in root_paths] + after_path ) - self.paths.update({p.path: p for p in root_paths}) + self.paths.update(self.pyenv_finder.roots) self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): @@ -155,7 +159,9 @@ class SystemPath(object): def get_path(self, path): path = ensure_path(path) - _path = self.paths.get(path.as_posix()) + _path = self.paths.get(path) + if not _path: + _path = self.paths.get(path.as_posix()) if not _path and path.as_posix() in self.path_order: _path = PathEntry.create( path=path.absolute(), is_root=True, only_python=self.only_python @@ -163,6 +169,14 @@ class SystemPath(object): self.paths[path.as_posix()] = _path return _path + def _get_paths(self): + return (self.get_path(k) for k in self.path_order) + + @cached_property + def path_entries(self): + paths = self._get_paths() + return paths + def find_all(self, executable): """Search the path for an executable. Return all copies. @@ -171,8 +185,8 @@ class SystemPath(object): :returns: List[PathEntry] """ sub_which = operator.methodcaller("which", name=executable) - filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) - return [f for f in filtered] + filtered = (sub_which(self.get_path(k)) for k in self.path_order) + return list(filtered) def which(self, executable): """Search for an executable on the path. @@ -182,11 +196,39 @@ class SystemPath(object): :returns: :class:`~pythonfinder.models.PathEntry` object. """ sub_which = operator.methodcaller("which", name=executable) - filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) - return next((f for f in filtered), None) + filtered = (sub_which(self.get_path(k)) for k in self.path_order) + return next(iter(f for f in filtered if f is not None), None) + + def _filter_paths(self, finder): + return ( + pth for pth in unnest(finder(p) for p in self.path_entries if p is not None) + if pth is not None + ) + + def _get_all_pythons(self, finder): + paths = {p.path.as_posix(): p for p in self._filter_paths(finder)} + paths.update(self.python_executables) + return (p for p in paths.values() if p is not None) + + def get_pythons(self, finder): + sort_key = operator.attrgetter("as_python.version_sort") + return ( + k for k in sorted( + (p for p in self._filter_paths(finder) if p.is_python), + key=sort_key, + reverse=True + ) if k is not None + ) def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search for a specific python version on the path. Return all copies @@ -197,32 +239,46 @@ class SystemPath(object): :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ sub_finder = operator.methodcaller( "find_all_python_versions", - major, + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, + name=name, ) + alternate_sub_finder = None + if major and not (minor or patch or pre or dev or arch or name): + alternate_sub_finder = operator.methodcaller( + "find_all_python_versions", + major=None, + name=major + ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version - paths = (self.get_path(k) for k in self.path_order) - path_filter = filter( - None, unnest((sub_finder(p) for p in paths if p is not None)) - ) - version_sort = operator.attrgetter("as_python.version_sort") - return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + values = list(self.get_pythons(sub_finder)) + if not values and alternate_sub_finder is not None: + values = list(self.get_pythons(alternate_sub_finder)) + return values def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search for a specific python version on the path. @@ -233,10 +289,24 @@ class SystemPath(object): :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. :rtype: :class:`~pythonfinder.models.PathEntry` """ + if isinstance(major, six.string_types) and not minor and not patch: + # Only proceed if this is in the format "x.y.z" or similar + if major.count(".") > 0 and major[0].isdigit(): + version = major.split(".", 2) + if len(version) > 3: + major, minor, patch, rest = version + elif len(version) == 3: + major, minor, patch = version + else: + major, minor = version + else: + name = "{0!s}".format(major) + major = None sub_finder = operator.methodcaller( "find_python_version", major, @@ -245,7 +315,15 @@ class SystemPath(object): pre=pre, dev=dev, arch=arch, + name=name, ) + alternate_sub_finder = None + if major and not (minor or patch or pre or dev or arch or name): + alternate_sub_finder = operator.methodcaller( + "find_all_python_versions", + major=None, + name=major + ) if major and minor and patch: _tuple_pre = pre if pre is not None else False _tuple_dev = dev if dev is not None else False @@ -255,12 +333,9 @@ class SystemPath(object): windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version - paths = (self.get_path(k) for k in self.path_order) - path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) - version_sort = operator.attrgetter("as_python.version_sort") - ver = next( - (c for c in sorted(path_filter, key=version_sort, reverse=True)), None - ) + ver = next(iter(self.get_pythons(sub_finder)), None) + if not ver and alternate_sub_finder is not None: + ver = next(iter(self.get_pythons(alternate_sub_finder)), None) if ver: if ver.as_python.version_tuple[:5] in self.python_version_dict: self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) @@ -269,23 +344,29 @@ class SystemPath(object): return ver @classmethod - def create(cls, path=None, system=False, only_python=False, global_search=True, ignore_unsupported=False): + def create( + cls, + path=None, + system=False, + only_python=False, + global_search=True, + ignore_unsupported=True, + ): """Create a new :class:`pythonfinder.models.SystemPath` instance. :param path: Search path to prepend when searching, defaults to None :param path: str, optional - :param system: Whether to use the running python by default instead of searching, defaults to False - :param system: bool, optional - :param only_python: Whether to search only for python executables, defaults to False - :param only_python: bool, optional - :param ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True - :param ignore_unsupported: bool, optional + :param bool system: Whether to use the running python by default instead of searching, defaults to False + :param bool only_python: Whether to search only for python executables, defaults to False + :param bool ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True :return: A new :class:`pythonfinder.models.SystemPath` instance. :rtype: :class:`pythonfinder.models.SystemPath` """ path_entries = defaultdict(PathEntry) paths = [] + if ignore_unsupported: + os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = fs_str("1") if global_search: paths = os.environ.get("PATH").split(os.pathsep) if path: @@ -316,7 +397,8 @@ class PathEntry(BasePath): _children = attr.ib(default=attr.Factory(dict)) is_root = attr.ib(default=True) only_python = attr.ib(default=False) - py_version = attr.ib(default=None) + name = attr.ib() + py_version = attr.ib() pythons = attr.ib() def __str__(self): @@ -329,17 +411,46 @@ class PathEntry(BasePath): children = self.path.iterdir() return children + def _gen_children(self): + pass_name = self.name != self.path.name + pass_args = {"is_root": False, "only_python": self.only_python} + if pass_name: + pass_args["name"] = self.name + + if not self.is_dir: + yield (self.path.as_posix(), copy.deepcopy(self)) + elif self.is_root: + for child in self._filter_children(): + yield (child.as_posix(), PathEntry.create(path=child, **pass_args)) + return + @cached_property def children(self): - if not self._children and self.is_dir and self.is_root: - self._children = { - child.as_posix(): PathEntry.create(path=child, is_root=False) - for child in self._filter_children() - } - elif not self.is_dir: - self._children = {self.path.as_posix(): self} + if not self._children: + children = {} + for child_key, child_val in self._gen_children(): + children[child_key] = child_val + self._children = children return self._children + @name.default + def get_name(self): + return self.path.name + + @py_version.default + def get_py_version(self): + from ..environment import IGNORE_UNSUPPORTED + if self.is_dir: + return None + if self.is_python: + from .python import PythonVersion + try: + py_version = PythonVersion.from_path(path=self, name=self.name) + except InvalidPythonVersion: + py_version = None + return py_version + return + @pythons.default def get_pythons(self): pythons = defaultdict() @@ -351,56 +462,62 @@ class PathEntry(BasePath): else: if self.is_python: _path = ensure_path(self.path) - pythons[_path.as_posix()] = copy.deepcopy(self) + pythons[_path.as_posix()] = self return pythons @cached_property def as_python(self): + py_version = None + if self.py_version: + return self.py_version if not self.is_dir and self.is_python: - if not self.py_version: - try: - from .python import PythonVersion - - self.py_version = PythonVersion.from_path(self.path) - except (ValueError, InvalidPythonVersion): - self.py_version = None - return self.py_version + try: + from .python import PythonVersion + py_version = PythonVersion.from_path(path=attr.evolve(self), name=self.name) + except (ValueError, InvalidPythonVersion): + py_version = None + return py_version @classmethod - def create(cls, path, is_root=False, only_python=False, pythons=None): + def create(cls, path, is_root=False, only_python=False, pythons=None, name=None): """Helper method for creating new :class:`pythonfinder.models.PathEntry` instances. - :param path: Path to the specified location. - :type path: str - :param is_root: Whether this is a root from the environment PATH variable, defaults to False - :param is_root: bool, optional - :param only_python: Whether to search only for python executables, defaults to False - :param only_python: bool, optional - :param pythons: A dictionary of existing python objects (usually from a finder), defaults to None - :param pythons: dict, optional + :param str path: Path to the specified location. + :param bool is_root: Whether this is a root from the environment PATH variable, defaults to False + :param bool only_python: Whether to search only for python executables, defaults to False + :param dict pythons: A dictionary of existing python objects (usually from a finder), defaults to None + :param str name: Name of the python version, e.g. ``anaconda3-5.3.0`` :return: A new instance of the class. :rtype: :class:`pythonfinder.models.PathEntry` """ target = ensure_path(path) - creation_args = {"path": target, "is_root": is_root, "only_python": only_python} + guessed_name = False + if not name: + guessed_name = True + name = target.name + creation_args = {"path": target, "is_root": is_root, "only_python": only_python, "name": name} if pythons: creation_args["pythons"] = pythons _new = cls(**creation_args) if pythons and only_python: children = {} + child_creation_args = { + "is_root": False, + "only_python": only_python + } + if not guessed_name: + child_creation_args["name"] = name for pth, python in pythons.items(): pth = ensure_path(pth) children[pth.as_posix()] = PathEntry( - path=pth, is_root=False, only_python=only_python, py_version=python + py_version=python, + path=pth, + **child_creation_args ) _new._children = children return _new - @cached_property - def name(self): - return self.path.name - @cached_property def is_dir(self): try: @@ -416,28 +533,5 @@ class PathEntry(BasePath): @cached_property def is_python(self): return self.is_executable and ( - self.py_version or looks_like_python(self.path.name) + looks_like_python(self.path.name) ) - - -@attr.s -class VersionPath(SystemPath): - base = attr.ib(default=None, validator=optional_instance_of(Path)) - - @classmethod - def create(cls, path, only_python=True, pythons=None): - """Accepts a path to a base python version directory. - - Generates the pyenv version listings for it""" - path = ensure_path(path) - path_entries = defaultdict(PathEntry) - if not path.name.lower() in ["scripts", "bin"]: - bin_name = "Scripts" if os.name == "nt" else "bin" - bin_dir = path / bin_name - else: - bin_dir = path - current_entry = PathEntry.create( - bin_dir, is_root=True, only_python=True, pythons=pythons - ) - path_entries[bin_dir.as_posix()] = current_entry - return cls(base=bin_dir, paths=path_entries) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 527c5f0a..1595a963 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, print_function import logging +import operator from collections import defaultdict @@ -10,9 +11,15 @@ import sysconfig from vistir.compat import Path -from ..utils import ensure_path, optional_instance_of, get_python_version, filter_pythons -from .mixins import BaseFinder -from .path import VersionPath +from ..utils import ( + ensure_path, + optional_instance_of, + get_python_version, + filter_pythons, + unnest, +) +from .mixins import BaseFinder, BasePath +from .path import SystemPath, PathEntry from .python import PythonVersion @@ -20,51 +27,66 @@ logger = logging.getLogger(__name__) @attr.s -class PyenvFinder(BaseFinder): +class PyenvFinder(BaseFinder, BasePath): root = attr.ib(default=None, validator=optional_instance_of(Path)) - # ignore_unsupported should come before versions, because its value is used - # in versions's default initializer. - ignore_unsupported = attr.ib(default=False) + #: ignore_unsupported should come before versions, because its value is used + #: in versions's default initializer. + ignore_unsupported = attr.ib(default=True) + paths = attr.ib(default=attr.Factory(list)) + roots = attr.ib(default=attr.Factory(defaultdict)) versions = attr.ib() pythons = attr.ib() + @property + def expanded_paths(self): + return ( + path for path in unnest(p for p in self.versions.values()) + if path is not None + ) + @classmethod - def version_from_bin_dir(cls, base_dir): - pythons = [py for py in filter_pythons(base_dir)] + def version_from_bin_dir(cls, base_dir, name=None): py_version = None - for py in pythons: - version = get_python_version(py.as_posix()) - try: - py_version = PythonVersion.parse(version) - except Exception: - continue - if py_version: - return py_version - return + version_path = PathEntry.create( + path=base_dir.absolute().as_posix(), + only_python=True, + name=base_dir.parent.name, + ) + py_version = next(iter(version_path.find_all_python_versions()), None) + return py_version @versions.default def get_versions(self): - versions = defaultdict(VersionPath) + versions = defaultdict() bin_ = sysconfig._INSTALL_SCHEMES[sysconfig._get_default_scheme()]["scripts"] for p in self.root.glob("versions/*"): - if p.parent.name == "envs": + if p.parent.name == "envs" or p.name == "envs": continue + bin_dir = Path(bin_.format(base=p.as_posix())) + version_path = None + if bin_dir.exists(): + version_path = PathEntry.create( + path=bin_dir.absolute().as_posix(), + only_python=False, + name=p.name, + is_root=True, + ) + version = None try: version = PythonVersion.parse(p.name) except ValueError: - bin_dir = Path(bin_.format(base=p.as_posix())) - if bin_dir.exists(): - version = self.version_from_bin_dir(bin_dir) - if not version: - if not self.ignore_unsupported: - raise - continue + entry = next(iter(version_path.find_all_python_versions()), None) + if not entry: + if self.ignore_unsupported: + continue + raise + else: + version = entry.py_version.as_dict() except Exception: if not self.ignore_unsupported: raise logger.warning( - 'Unsupported Python version %r, ignoring...', - p.name, exc_info=True + "Unsupported Python version %r, ignoring...", p.name, exc_info=True ) continue if not version: @@ -75,24 +97,128 @@ class PyenvFinder(BaseFinder): version.get("patch"), version.get("is_prerelease"), version.get("is_devrelease"), - version.get("is_debug") - ) - versions[version_tuple] = VersionPath.create( - path=p.resolve(), only_python=True + version.get("is_debug"), ) + self.roots[p] = version_path + versions[version_tuple] = version_path + self.paths.append(version_path) return versions @pythons.default def get_pythons(self): pythons = defaultdict() - for v in self.versions.values(): - for p in v.paths.values(): - _path = ensure_path(p.path) - if p.is_python: - pythons[_path] = p + for p in self.paths: + pythons.update(p.pythons) return pythons @classmethod - def create(cls, root, ignore_unsupported=False): + def create(cls, root, ignore_unsupported=True): root = ensure_path(root) return cls(root=root, ignore_unsupported=ignore_unsupported) + + def find_all_python_versions( + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, + ): + """Search for a specific python version on the path. Return all copies + + :param major: Major python version to search for. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` + :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. + :rtype: List[:class:`~pythonfinder.models.PathEntry`] + """ + + version_matcher = operator.methodcaller( + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + ) + py = operator.attrgetter("as_python") + pythons = ( + py_ver for py_ver in (py(p) for p in self.pythons.values() if p is not None) + if py_ver is not None + ) + # pythons = filter(None, [p.as_python for p in self.pythons.values()]) + matching_versions = filter(lambda py: version_matcher(py), pythons) + version_sort = operator.attrgetter("version_sort") + return sorted(matching_versions, key=version_sort, reverse=True) + + def find_python_version( + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, + ): + """Search or self for the specified Python version and return the first match. + + :param major: Major version number. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` + :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. + """ + + version_matcher = operator.methodcaller( + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + ) + pythons = filter(None, [p.as_python for p in self.pythons.values()]) + matching_versions = filter(lambda py: version_matcher(py), pythons) + version_sort = operator.attrgetter("version_sort") + return next(iter(c for c in sorted(matching_versions, key=version_sort, reverse=True)), None) + + +@attr.s +class VersionPath(SystemPath): + base = attr.ib(default=None, validator=optional_instance_of(Path)) + name = attr.ib(default=None) + + @classmethod + def create(cls, path, only_python=True, pythons=None, name=None): + """Accepts a path to a base python version directory. + + Generates the pyenv version listings for it""" + path = ensure_path(path) + path_entries = defaultdict(PathEntry) + bin_ = sysconfig._INSTALL_SCHEMES[sysconfig._get_default_scheme()]["scripts"] + if path.as_posix().endswith(Path(bin_).name): + path = path.parent + bin_dir = ensure_path(bin_.format(base=path.as_posix())) + if not name: + name = path.name + current_entry = PathEntry.create( + bin_dir, is_root=True, only_python=True, pythons=pythons, name=name + ) + path_entries[bin_dir.as_posix()] = current_entry + return cls(name=name, base=bin_dir, paths=path_entries) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index c71b9d9b..ec99afe7 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -13,7 +13,11 @@ from packaging.version import parse as parse_version from ..environment import SYSTEM_ARCH from ..utils import ( - _filter_none, ensure_path, get_python_version, optional_instance_of + _filter_none, + ensure_path, + get_python_version, + optional_instance_of, + ensure_path, ) @@ -30,6 +34,7 @@ class PythonVersion(object): architecture = attr.ib(default=None) comes_from = attr.ib(default=None) executable = attr.ib(default=None) + name = attr.ib(default=None) @property def version_sort(self): @@ -65,22 +70,37 @@ class PythonVersion(object): self.patch, self.is_prerelease, self.is_devrelease, - self.is_debug + self.is_debug, ) def matches( - self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None, debug=False + self, + major=None, + minor=None, + patch=None, + pre=False, + dev=False, + arch=None, + debug=False, + name=None, ): - if arch and arch.isdigit(): - arch = "{0}bit".format(arch) + if arch: + own_arch = self.get_architecture() + if arch.isdigit(): + arch = "{0}bit".format(arch) return ( (major is None or self.major == major) and (minor is None or self.minor == minor) and (patch is None or self.patch == patch) and (pre is None or self.is_prerelease == pre) and (dev is None or self.is_devrelease == dev) - and (arch is None or self.architecture == arch) + and (arch is None or own_arch == arch) and (debug is None or self.is_debug == debug) + and ( + name is None + or (name and self.name) + and (self.name == name or self.name.startswith(name)) + ) ) def as_major(self): @@ -93,6 +113,18 @@ class PythonVersion(object): self_dict.update({"patch": None}) return self.create(**self_dict) + def as_dict(self): + return { + "major": self.major, + "minor": self.minor, + "patch": self.patch, + "is_prerelease": self.is_prerelease, + "is_postrelease": self.is_postrelease, + "is_devrelease": self.is_devrelease, + "is_debug": self.is_debug, + "version": self.version, + } + @classmethod def parse(cls, version): """Parse a valid version string into a dictionary @@ -138,8 +170,15 @@ class PythonVersion(object): "version": version, } + def get_architecture(self): + if self.architecture: + return self.architecture + arch, _ = platform.architecture(path.path.as_posix()) + self.architecture = arch + return self.architecture + @classmethod - def from_path(cls, path): + def from_path(cls, path, name=None): """Parses a python version from a system path. Raises: @@ -147,29 +186,33 @@ class PythonVersion(object): :param path: A string or :class:`~pythonfinder.models.path.PathEntry` :type path: str or :class:`~pythonfinder.models.path.PathEntry` instance - :param launcher_entry: A python launcher environment object. + :param str name: Name of the python distribution in question :return: An instance of a PythonVersion. :rtype: :class:`~pythonfinder.models.python.PythonVersion` """ from .path import PathEntry + from ..environment import IGNORE_UNSUPPORTED if not isinstance(path, PathEntry): - path = PathEntry.create(path, is_root=False, only_python=True) - if not path.is_python: + path = PathEntry.create(path, is_root=False, only_python=True, name=name) + if not path.is_python and not IGNORE_UNSUPPORTED: raise ValueError("Not a valid python path: %s" % path.path) return - py_version = get_python_version(str(path.path)) + py_version = get_python_version(path.path.as_posix()) instance_dict = cls.parse(py_version) - if not isinstance(instance_dict.get("version"), Version): + if not isinstance(instance_dict.get("version"), Version) and not IGNORE_UNSUPPORTED: raise ValueError("Not a valid python path: %s" % path.path) return - architecture, _ = platform.architecture(path.path.as_posix()) - instance_dict.update({"comes_from": path, "architecture": architecture}) + if not name: + name = path.name + instance_dict.update( + {"comes_from": path, "name": name} + ) return cls(**instance_dict) @classmethod - def from_windows_launcher(cls, launcher_entry): + def from_windows_launcher(cls, launcher_entry, name=None): """Create a new PythonVersion instance from a Windows Launcher Entry :param launcher_entry: A python launcher environment object. @@ -193,12 +236,14 @@ class PythonVersion(object): launcher_entry.info, "sys_architecture", SYSTEM_ARCH ), "executable": exe_path, + "name": name } ) py_version = cls.create(**creation_dict) - comes_from = PathEntry.create(exe_path, only_python=True) + comes_from = PathEntry.create(exe_path, only_python=True, name=name) comes_from.py_version = copy.deepcopy(py_version) py_version.comes_from = comes_from + py_version.name = comes_from.name return py_version @classmethod diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index fcb4d42a..e47bcc2c 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -22,7 +22,14 @@ class WindowsFinder(BaseFinder): pythons = attr.ib() def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): version_matcher = operator.methodcaller( "matches", @@ -32,6 +39,7 @@ class WindowsFinder(BaseFinder): pre=pre, dev=dev, arch=arch, + name=name, ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -40,13 +48,26 @@ class WindowsFinder(BaseFinder): return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): return next( ( v for v in self.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=None, ) ), None, @@ -60,7 +81,7 @@ class WindowsFinder(BaseFinder): env_versions = pep514env.findall() path = None for version_object in env_versions: - install_path = getattr(version_object.info, 'install_path', None) + install_path = getattr(version_object.info, "install_path", None) if install_path is None: continue path = ensure_path(install_path.__getattr__("")) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index e965bb51..854cc8e7 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -4,16 +4,17 @@ import os import six import operator from .models import SystemPath +from vistir.compat import lru_cache class Finder(object): - def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=False): + def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=True): """Finder A cross-platform Finder for locating python and other executables. Searches for python and other specified binaries starting in `path`, if supplied, but searching the bin path of `sys.executable` if `system=True`, and then searching in the `os.environ['PATH']` if `global_search=True`. When `global_search` - is `False`, this search operation is restricted to the allowed locations of + is `False`, this search operation is restricted to the allowed locations of `path` and `system`. :param path: A bin-directory search location, defaults to None @@ -34,6 +35,14 @@ class Finder(object): self._system_path = None self._windows_finder = None + def __hash__(self): + return hash( + (self.path_prepend, self.system, self.global_search, self.ignore_unsupported) + ) + + def __eq__(self, other): + return self.__hash__() == other.__hash__() + @property def system_path(self): if not self._system_path: @@ -56,8 +65,9 @@ class Finder(object): def which(self, exe): return self.system_path.which(exe) + @lru_cache(maxsize=128) def find_python_version( - self, major, minor=None, patch=None, pre=None, dev=None, arch=None + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): from .models import PythonVersion @@ -69,12 +79,24 @@ class Finder(object): and patch is None ): if arch is None and "-" in major: - major, arch = major.rsplit("-", 1) - if not arch.isdigit(): - major = "{0}-{1}".format(major, arch) + orig_string = "{0!s}".format(major) + major, _, arch = major.rpartition("-") + if arch.startswith("x"): + arch = arch.lstrip("x") + if arch.lower().endswith("bit"): + arch = arch.lower().replace("bit", "") + if not (arch.isdigit() and (int(arch) & int(arch) - 1) == 0): + major = orig_string + arch = None else: arch = "{0}bit".format(arch) - version_dict = PythonVersion.parse(major) + try: + version_dict = PythonVersion.parse(major) + except ValueError: + if name is None: + name = "{0!s}".format(major) + major = None + version_dict = {} major = version_dict.get("major", major) minor = version_dict.get("minor", minor) patch = version_dict.get("patch", patch) @@ -83,16 +105,17 @@ class Finder(object): arch = version_dict.get("architecture", arch) if arch is None else arch if os.name == "nt": match = self.windows_finder.find_python_version( - major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) if match: return match return self.system_path.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) + @lru_cache(maxsize=128) def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): version_sort = operator.attrgetter("as_python.version_sort") python_version_dict = getattr(self.system_path, "python_version_dict") @@ -109,7 +132,7 @@ class Finder(object): paths = sorted(paths, key=version_sort, reverse=True) return paths versions = self.system_path.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) if not isinstance(versions, list): versions = [versions] diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index dced9eab..ca07b42f 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function +import itertools import locale import os import subprocess @@ -16,10 +17,18 @@ import vistir from .exceptions import InvalidPythonVersion +try: + from functools import lru_cache +except ImportError: + from backports.functools_lru_cache import lru_cache + +six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) +from six.moves import Iterable + PYTHON_IMPLEMENTATIONS = ( "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", - "stackless", "activepython" + "stackless", "activepython", "micropython" ) RULES_BASE = ["*{0}", "*{0}?", "*{0}?.?", "*{0}?.?m"] RULES = [rule.format(impl) for impl in PYTHON_IMPLEMENTATIONS for rule in RULES_BASE] @@ -29,17 +38,28 @@ KNOWN_EXTS = KNOWN_EXTS | set( filter(None, os.environ.get("PATHEXT", "").split(os.pathsep)) ) +MATCH_RULES = [] +for rule in RULES: + MATCH_RULES.extend( + [ + "{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) + for ext in KNOWN_EXTS + ] + ) + +@lru_cache(maxsize=128) def get_python_version(path): """Get python version string using subprocess from a given path.""" version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] try: - out, _ = vistir.misc.run(version_cmd, block=True, nospin=True) + c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, + combine_stderr=False) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) - if not out: + if not c.out: raise InvalidPythonVersion("%s is not a valid python path" % path) - return out.strip() + return c.out.strip() def optional_instance_of(cls): @@ -54,6 +74,7 @@ def path_is_executable(path): return os.access(str(path), os.X_OK) +@lru_cache(maxsize=1024) def path_is_known_executable(path): return ( path_is_executable(path) @@ -62,24 +83,19 @@ def path_is_known_executable(path): ) +@lru_cache(maxsize=1024) def looks_like_python(name): - match_rules = [] - for rule in RULES: - match_rules.extend( - [ - "{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) - for ext in KNOWN_EXTS - ] - ) if not any(name.lower().startswith(py_name) for py_name in PYTHON_IMPLEMENTATIONS): return False - return any(fnmatch(name, rule) for rule in match_rules) + return any(fnmatch(name, rule) for rule in MATCH_RULES) +@lru_cache(maxsize=128) def path_is_python(path): return path_is_executable(path) and looks_like_python(path.name) +@lru_cache(maxsize=1024) def ensure_path(path): """Given a path (either a string or a Path object), expand variables and return a Path object. @@ -90,13 +106,9 @@ def ensure_path(path): """ if isinstance(path, vistir.compat.Path): - path = path.as_posix() + return path path = vistir.compat.Path(os.path.expandvars(path)) - try: - path = path.resolve() - except OSError: - path = path.absolute() - return path + return path.absolute() def _filter_none(k, v): @@ -105,6 +117,7 @@ def _filter_none(k, v): return False +@lru_cache(maxsize=128) def filter_pythons(path): """Return all valid pythons in a given path""" if not isinstance(path, vistir.compat.Path): @@ -114,7 +127,21 @@ def filter_pythons(path): return filter(lambda x: path_is_python(x), path.iterdir()) +# def unnest(item): +# if isinstance(next((i for i in item), None), (list, tuple)): +# return chain(*filter(None, item)) +# return chain(filter(None, item)) + + def unnest(item): - if isinstance(next((i for i in item), None), (list, tuple)): - return chain(*filter(None, item)) - return chain(filter(None, item)) + if isinstance(item, Iterable) and not isinstance(item, six.string_types): + item, target = itertools.tee(item, 2) + else: + target = item + for el in target: + if isinstance(el, Iterable) and not isinstance(el, six.string_types): + el, el_copy = itertools.tee(el, 2) + for sub in unnest(el_copy): + yield sub + else: + yield el diff --git a/pipenv/vendor/requests/LICENSE b/pipenv/vendor/requests/LICENSE index 2e68b82e..841c6023 100644 --- a/pipenv/vendor/requests/LICENSE +++ b/pipenv/vendor/requests/LICENSE @@ -4,7 +4,7 @@ Copyright 2018 Kenneth Reitz you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pipenv/vendor/requests/__init__.py b/pipenv/vendor/requests/__init__.py index a5b3c9c3..bc168ee5 100644 --- a/pipenv/vendor/requests/__init__.py +++ b/pipenv/vendor/requests/__init__.py @@ -22,7 +22,7 @@ usage: ... or POST: >>> payload = dict(key1='value1', key2='value2') - >>> r = requests.post('http://httpbin.org/post', data=payload) + >>> r = requests.post('https://httpbin.org/post', data=payload) >>> print(r.text) { ... @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.23 + # urllib3 >= 1.21.1, <= 1.24 assert major == 1 assert minor >= 21 - assert minor <= 23 + assert minor <= 24 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] @@ -79,14 +79,14 @@ def _check_cryptography(cryptography_version): return if cryptography_version < [1, 3, 4]: - warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warning = 'Old version of cryptography ({}) may cause slowdown.'.format(cryptography_version) warnings.warn(warning, RequestsDependencyWarning) # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) except (AssertionError, ValueError): - warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported " + warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported " "version!".format(urllib3.__version__, chardet.__version__), RequestsDependencyWarning) @@ -123,12 +123,7 @@ from .exceptions import ( # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/pipenv/vendor/requests/__version__.py b/pipenv/vendor/requests/__version__.py index ef61ec0f..be8a45fe 100644 --- a/pipenv/vendor/requests/__version__.py +++ b/pipenv/vendor/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.19.1' -__build__ = 0x021901 +__version__ = '2.20.0' +__build__ = 0x022000 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' diff --git a/pipenv/vendor/requests/adapters.py b/pipenv/vendor/requests/adapters.py index a4b02842..fa4d9b3c 100644 --- a/pipenv/vendor/requests/adapters.py +++ b/pipenv/vendor/requests/adapters.py @@ -26,6 +26,7 @@ from urllib3.exceptions import ProtocolError from urllib3.exceptions import ReadTimeoutError from urllib3.exceptions import SSLError as _SSLError from urllib3.exceptions import ResponseError +from urllib3.exceptions import LocationValueError from .models import Response from .compat import urlparse, basestring @@ -35,7 +36,8 @@ from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema, InvalidProxyURL) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL, + InvalidURL) from .auth import _basic_auth_str try: @@ -127,8 +129,7 @@ class HTTPAdapter(BaseAdapter): self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in - self.__attrs__) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): # Can't handle by adding 'proxy_manager' to self.__attrs__ because @@ -224,7 +225,7 @@ class HTTPAdapter(BaseAdapter): if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {0}".format(cert_loc)) + "invalid path: {}".format(cert_loc)) conn.cert_reqs = 'CERT_REQUIRED' @@ -246,10 +247,10 @@ class HTTPAdapter(BaseAdapter): conn.key_file = None if conn.cert_file and not os.path.exists(conn.cert_file): raise IOError("Could not find the TLS certificate file, " - "invalid path: {0}".format(conn.cert_file)) + "invalid path: {}".format(conn.cert_file)) if conn.key_file and not os.path.exists(conn.key_file): raise IOError("Could not find the TLS key file, " - "invalid path: {0}".format(conn.key_file)) + "invalid path: {}".format(conn.key_file)) def build_response(self, req, resp): """Builds a :class:`Response ` object from a urllib3 @@ -378,7 +379,7 @@ class HTTPAdapter(BaseAdapter): when subclassing the :class:`HTTPAdapter `. - :param proxies: The url of the proxy being used for this request. + :param proxy: The url of the proxy being used for this request. :rtype: dict """ headers = {} @@ -407,7 +408,10 @@ class HTTPAdapter(BaseAdapter): :rtype: requests.Response """ - conn = self.get_connection(request.url, proxies) + try: + conn = self.get_connection(request.url, proxies) + except LocationValueError as e: + raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) @@ -421,7 +425,7 @@ class HTTPAdapter(BaseAdapter): timeout = TimeoutSauce(connect=connect, read=read) except ValueError as e: # this may raise a string formatting error. - err = ("Invalid timeout {0}. Pass a (connect, read) " + err = ("Invalid timeout {}. Pass a (connect, read) " "timeout tuple, or a single float to set " "both timeouts to the same value".format(timeout)) raise ValueError(err) @@ -471,11 +475,10 @@ class HTTPAdapter(BaseAdapter): # Receive the response from the server try: - # For Python 2.7+ versions, use buffering of HTTP - # responses + # For Python 2.7, use buffering of HTTP responses r = low_conn.getresponse(buffering=True) except TypeError: - # For compatibility with Python 2.6 versions and back + # For compatibility with Python 3.3+ r = low_conn.getresponse() resp = HTTPResponse.from_httplib( diff --git a/pipenv/vendor/requests/api.py b/pipenv/vendor/requests/api.py index a2cc84d7..abada96d 100644 --- a/pipenv/vendor/requests/api.py +++ b/pipenv/vendor/requests/api.py @@ -18,8 +18,10 @@ def request(method, url, **kwargs): :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. @@ -47,7 +49,7 @@ def request(method, url, **kwargs): Usage:: >>> import requests - >>> req = requests.request('GET', 'http://httpbin.org/get') + >>> req = requests.request('GET', 'https://httpbin.org/get') """ @@ -62,7 +64,8 @@ def get(url, params=None, **kwargs): r"""Sends a GET request. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response @@ -102,7 +105,8 @@ def post(url, data=None, json=None, **kwargs): r"""Sends a POST request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -116,7 +120,8 @@ def put(url, data=None, **kwargs): r"""Sends a PUT request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -130,7 +135,8 @@ def patch(url, data=None, **kwargs): r"""Sends a PATCH request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object diff --git a/pipenv/vendor/requests/auth.py b/pipenv/vendor/requests/auth.py index 4ae45947..bdde51c7 100644 --- a/pipenv/vendor/requests/auth.py +++ b/pipenv/vendor/requests/auth.py @@ -38,7 +38,7 @@ def _basic_auth_str(username, password): if not isinstance(username, basestring): warnings.warn( "Non-string usernames will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(username), category=DeprecationWarning, @@ -48,7 +48,7 @@ def _basic_auth_str(username, password): if not isinstance(password, basestring): warnings.warn( "Non-string passwords will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(password), category=DeprecationWarning, diff --git a/pipenv/vendor/requests/compat.py b/pipenv/vendor/requests/compat.py index 6b9c6fac..c44b35ef 100644 --- a/pipenv/vendor/requests/compat.py +++ b/pipenv/vendor/requests/compat.py @@ -43,9 +43,8 @@ if is_py2: import cookielib from Cookie import Morsel from StringIO import StringIO - from collections import Callable, Mapping, MutableMapping + from collections import Callable, Mapping, MutableMapping, OrderedDict - from urllib3.packages.ordered_dict import OrderedDict builtin_str = str bytes = str diff --git a/pipenv/vendor/requests/cookies.py b/pipenv/vendor/requests/cookies.py index 50883a84..56fccd9c 100644 --- a/pipenv/vendor/requests/cookies.py +++ b/pipenv/vendor/requests/cookies.py @@ -444,20 +444,21 @@ def create_cookie(name, value, **kwargs): By default, the pair of `name` and `value` will be set for the domain '' and sent on every request (this is sometimes called a "supercookie"). """ - result = dict( - version=0, - name=name, - value=value, - port=None, - domain='', - path='/', - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False,) + result = { + 'version': 0, + 'name': name, + 'value': value, + 'port': None, + 'domain': '', + 'path': '/', + 'secure': False, + 'expires': None, + 'discard': True, + 'comment': None, + 'comment_url': None, + 'rest': {'HttpOnly': None}, + 'rfc2109': False, + } badargs = set(kwargs) - set(result) if badargs: @@ -511,6 +512,7 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): :param cookiejar: (optional) A cookiejar to add the cookies to. :param overwrite: (optional) If False, will not replace cookies already in the jar with new ones. + :rtype: CookieJar """ if cookiejar is None: cookiejar = RequestsCookieJar() @@ -529,6 +531,7 @@ def merge_cookies(cookiejar, cookies): :param cookiejar: CookieJar object to add the cookies to. :param cookies: Dictionary or CookieJar object to be added. + :rtype: CookieJar """ if not isinstance(cookiejar, cookielib.CookieJar): raise ValueError('You can only merge into CookieJar') diff --git a/pipenv/vendor/requests/help.py b/pipenv/vendor/requests/help.py index 06e06b2a..e53d35ef 100644 --- a/pipenv/vendor/requests/help.py +++ b/pipenv/vendor/requests/help.py @@ -89,8 +89,7 @@ def info(): 'version': getattr(idna, '__version__', ''), } - # OPENSSL_VERSION_NUMBER doesn't exist in the Python 2.6 ssl module. - system_ssl = getattr(ssl, 'OPENSSL_VERSION_NUMBER', None) + system_ssl = ssl.OPENSSL_VERSION_NUMBER system_ssl_info = { 'version': '%x' % system_ssl if system_ssl is not None else '' } diff --git a/pipenv/vendor/requests/hooks.py b/pipenv/vendor/requests/hooks.py index 32b32de7..7a51f212 100644 --- a/pipenv/vendor/requests/hooks.py +++ b/pipenv/vendor/requests/hooks.py @@ -15,14 +15,14 @@ HOOKS = ['response'] def default_hooks(): - return dict((event, []) for event in HOOKS) + return {event: [] for event in HOOKS} # TODO: response is the only one def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" - hooks = hooks or dict() + hooks = hooks or {} hooks = hooks.get(key) if hooks: if hasattr(hooks, '__call__'): diff --git a/pipenv/vendor/requests/models.py b/pipenv/vendor/requests/models.py index 3d0e1f42..3dded57e 100644 --- a/pipenv/vendor/requests/models.py +++ b/pipenv/vendor/requests/models.py @@ -204,9 +204,13 @@ class Request(RequestHooksMixin): :param url: URL to send. :param headers: dictionary of headers to send. :param files: dictionary of {filename: fileobject} files to multipart upload. - :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place. + :param data: the body to attach to the request. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param json: json for the body to attach to the request (if files or data is not specified). - :param params: dictionary of URL parameters to append to the URL. + :param params: URL parameters to append to the URL. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param auth: Auth handler or (user, pass) tuple. :param cookies: dictionary or CookieJar of cookies to attach to this request. :param hooks: dictionary of callback hooks, for internal usage. @@ -214,7 +218,7 @@ class Request(RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> req.prepare() """ @@ -274,7 +278,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> r = req.prepare() @@ -648,10 +652,7 @@ class Response(object): if not self._content_consumed: self.content - return dict( - (attr, getattr(self, attr, None)) - for attr in self.__attrs__ - ) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): diff --git a/pipenv/vendor/requests/sessions.py b/pipenv/vendor/requests/sessions.py index ba135268..a448bd83 100644 --- a/pipenv/vendor/requests/sessions.py +++ b/pipenv/vendor/requests/sessions.py @@ -115,6 +115,22 @@ class SessionRedirectMixin(object): return to_native_string(location, 'utf8') return None + def should_strip_auth(self, old_url, new_url): + """Decide whether Authorization header should be removed when redirecting""" + old_parsed = urlparse(old_url) + new_parsed = urlparse(new_url) + if old_parsed.hostname != new_parsed.hostname: + return True + # Special case: allow http -> https redirect when using the standard + # ports. This isn't specified by RFC 7235, but is kept to avoid + # breaking backwards compatibility with older versions of requests + # that allowed any redirects on the same host. + if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) + and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): + return False + # Standard case: root URI must match + return old_parsed.port != new_parsed.port or old_parsed.scheme != new_parsed.scheme + def resolve_redirects(self, resp, req, stream=False, timeout=None, verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): """Receives a Response. Returns a generator of Responses or Requests.""" @@ -236,14 +252,10 @@ class SessionRedirectMixin(object): headers = prepared_request.headers url = prepared_request.url - if 'Authorization' in headers: + if 'Authorization' in headers and self.should_strip_auth(response.request.url, url): # If we get redirected to a new host, we should strip out any # authentication headers. - original_parsed = urlparse(response.request.url) - redirect_parsed = urlparse(url) - - if (original_parsed.hostname != redirect_parsed.hostname): - del headers['Authorization'] + del headers['Authorization'] # .netrc might have more auth for us on our new host. new_auth = get_netrc_auth(url) if self.trust_env else None @@ -299,7 +311,7 @@ class SessionRedirectMixin(object): """ method = prepared_request.method - # http://tools.ietf.org/html/rfc7231#section-6.4.4 + # https://tools.ietf.org/html/rfc7231#section-6.4.4 if response.status_code == codes.see_other and method != 'HEAD': method = 'GET' @@ -325,13 +337,13 @@ class Session(SessionRedirectMixin): >>> import requests >>> s = requests.Session() - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') Or as a context manager:: >>> with requests.Session() as s: - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') """ @@ -453,8 +465,8 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary, bytes, or file-like object to send - in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the @@ -550,7 +562,8 @@ class Session(SessionRedirectMixin): r"""Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response @@ -562,7 +575,8 @@ class Session(SessionRedirectMixin): r"""Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -573,7 +587,8 @@ class Session(SessionRedirectMixin): r"""Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -723,7 +738,7 @@ class Session(SessionRedirectMixin): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = {attr: getattr(self, attr, None) for attr in self.__attrs__} return state def __setstate__(self, state): @@ -735,7 +750,12 @@ def session(): """ Returns a :class:`Session` for context-management. + .. deprecated:: 1.0.0 + + This method has been deprecated since version 1.0.0 and is only kept for + backwards compatibility. New code should use :class:`~requests.sessions.Session` + to create a session. This may be removed at a future date. + :rtype: Session """ - return Session() diff --git a/pipenv/vendor/requests/status_codes.py b/pipenv/vendor/requests/status_codes.py index ff462c6c..813e8c4e 100644 --- a/pipenv/vendor/requests/status_codes.py +++ b/pipenv/vendor/requests/status_codes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -""" +r""" The ``codes`` object defines a mapping from common names for HTTP statuses to their numerical codes, accessible either as attributes or as dictionary items. diff --git a/pipenv/vendor/requests/utils.py b/pipenv/vendor/requests/utils.py index 431f6be0..0ce7fe11 100644 --- a/pipenv/vendor/requests/utils.py +++ b/pipenv/vendor/requests/utils.py @@ -173,10 +173,10 @@ def get_netrc_auth(url, raise_errors=False): for f in NETRC_FILES: try: - loc = os.path.expanduser('~/{0}'.format(f)) + loc = os.path.expanduser('~/{}'.format(f)) except KeyError: # os.path.expanduser can fail when $HOME is undefined and - # getpwuid fails. See http://bugs.python.org/issue20164 & + # getpwuid fails. See https://bugs.python.org/issue20164 & # https://github.com/requests/requests/issues/1846 return @@ -466,7 +466,7 @@ def _parse_content_type_header(header): if index_of_equals != -1: key = param[:index_of_equals].strip(items_to_strip) value = param[index_of_equals + 1:].strip(items_to_strip) - params_dict[key] = value + params_dict[key.lower()] = value return content_type, params_dict @@ -706,6 +706,10 @@ def should_bypass_proxies(url, no_proxy): no_proxy = get_proxy('no_proxy') parsed = urlparse(url) + if parsed.hostname is None: + # URLs don't always have hostnames, e.g. file:/// urls. + return True + if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the hostname, both with and without the port. @@ -725,7 +729,7 @@ def should_bypass_proxies(url, no_proxy): else: host_with_port = parsed.hostname if parsed.port: - host_with_port += ':{0}'.format(parsed.port) + host_with_port += ':{}'.format(parsed.port) for host in no_proxy: if parsed.hostname.endswith(host) or host_with_port.endswith(host): @@ -733,13 +737,8 @@ def should_bypass_proxies(url, no_proxy): # to apply the proxies on this URL. return True - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - # The proxy_bypass function is incredibly buggy on OS X in early versions - # of Python 2.6, so allow this call to fail. Only catch the specific - # exceptions we've seen, though: this call failing in other ways can reveal - # legitimate problems. with set_environ('no_proxy', no_proxy_arg): + # parsed.hostname can be `None` in cases such as a file URI. try: bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 1f3a2fcb..8ceccd79 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,6 +1,13 @@ # -*- coding=utf-8 -*- -__version__ = '1.1.9' +__version__ = '1.2.2' +import logging -from .exceptions import RequirementError -from .models import Requirement, Lockfile, Pipfile +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +from .models.requirements import Requirement +from .models.lockfile import Lockfile +from .models.pipfile import Pipfile + +__all__ = ["Lockfile", "Pipfile", "Requirement"] diff --git a/pipenv/vendor/requirementslib/exceptions.py b/pipenv/vendor/requirementslib/exceptions.py index 82578624..de8bf8ef 100644 --- a/pipenv/vendor/requirementslib/exceptions.py +++ b/pipenv/vendor/requirementslib/exceptions.py @@ -9,6 +9,8 @@ if six.PY2: def __init__(self, *args, **kwargs): self.errno = errno.EEXIST super(FileExistsError, self).__init__(*args, **kwargs) +else: + from six.moves.builtins import FileExistsError class RequirementError(Exception): diff --git a/pipenv/vendor/requirementslib/models/__init__.py b/pipenv/vendor/requirementslib/models/__init__.py index 8d12da45..99e519d6 100644 --- a/pipenv/vendor/requirementslib/models/__init__.py +++ b/pipenv/vendor/requirementslib/models/__init__.py @@ -2,10 +2,9 @@ from __future__ import absolute_import -__all__ = ["Requirement", "Lockfile", "Pipfile", "DependencyResolver"] +__all__ = ["Requirement", "Lockfile", "Pipfile"] from .requirements import Requirement from .lockfile import Lockfile from .pipfile import Pipfile -from .resolvers import DependencyResolver diff --git a/pipenv/vendor/requirementslib/models/cache.py b/pipenv/vendor/requirementslib/models/cache.py index 16fc4ba8..71701090 100644 --- a/pipenv/vendor/requirementslib/models/cache.py +++ b/pipenv/vendor/requirementslib/models/cache.py @@ -5,21 +5,19 @@ import copy import hashlib import json import os -import six import sys import requests -import pip_shims import vistir from appdirs import user_cache_dir +from pip_shims.shims import FAVORITE_HASH, SafeFileCache from packaging.requirements import Requirement from .utils import as_tuple, key_from_req, lookup_table, get_pinned_version - -if six.PY2: - from ..exceptions import FileExistsError +from ..exceptions import FileExistsError +from ..utils import VCS_SUPPORT CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) @@ -189,7 +187,7 @@ class DependencyCache(object): for dep_name in self.cache[name][version_and_extras]) -class HashCache(pip_shims.SafeFileCache): +class HashCache(SafeFileCache): """Caches hashes of PyPI artifacts so we do not need to re-download them. Hashes are only cached when the URL appears to contain a hash in it and the @@ -206,7 +204,7 @@ class HashCache(pip_shims.SafeFileCache): def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - vcs = pip_shims.VcsSupport() + vcs = VCS_SUPPORT orig_scheme = location.scheme new_location = copy.deepcopy(location) if orig_scheme in vcs.all_schemes: @@ -223,11 +221,11 @@ class HashCache(pip_shims.SafeFileCache): return hash_value.decode('utf8') def _get_file_hash(self, location): - h = hashlib.new(pip_shims.FAVORITE_HASH) + h = hashlib.new(FAVORITE_HASH) with vistir.contextmanagers.open_file(location, self.session) as fp: for chunk in iter(lambda: fp.read(8096), b""): h.update(chunk) - return ":".join([pip_shims.FAVORITE_HASH, h.hexdigest()]) + return ":".join([FAVORITE_HASH, h.hexdigest()]) class _JSONCache(object): diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index ae643517..c3df222f 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -9,6 +9,7 @@ import attr import packaging.markers import packaging.version import requests +import warnings from first import first from packaging.utils import canonicalize_name @@ -17,9 +18,10 @@ from pip_shims import ( FormatControl, InstallRequirement, PackageFinder, RequirementPreparer, RequirementSet, RequirementTracker, Resolver, WheelCache, pip_version ) -from vistir.compat import JSONDecodeError, TemporaryDirectory, fs_str +from vistir.compat import JSONDecodeError, fs_str, ResourceWarning from vistir.contextmanagers import cd, temp_environ from vistir.misc import partialclass +from vistir.path import create_tracked_tempdir from ..utils import get_pip_command, prepare_pip_source_args, _ensure_dir from .cache import CACHE_DIR, DependencyCache @@ -52,6 +54,8 @@ def find_all_matches(finder, ireq, pre=False): :return: A list of matching candidates. :rtype: list[:class:`~pip._internal.index.InstallationCandidate`] """ + + candidates = clean_requires_python(finder.find_all_candidates(ireq.name)) versions = {candidate.version for candidate in candidates} allowed_versions = _get_filtered_versions(ireq, versions, pre) @@ -340,9 +344,12 @@ def get_dependencies_from_json(ireq): def gen(ireq): info = None - info = session.get( - "https://pypi.org/pypi/{0}/{1}/json".format(ireq.req.name, version) - ).json()["info"] + try: + info = session.get( + "https://pypi.org/pypi/{0}/{1}/json".format(ireq.req.name, version) + ).json()["info"] + finally: + session.close() requires_dist = info.get("requires_dist", info.get("requires")) if not requires_dist: # The API can return None for this. return @@ -580,12 +587,12 @@ def start_resolver(finder=None, wheel_cache=None): download_dir = PKGS_DOWNLOAD_DIR _ensure_dir(download_dir) - _build_dir = TemporaryDirectory(fs_str("build")) - _source_dir = TemporaryDirectory(fs_str("source")) + _build_dir = create_tracked_tempdir(fs_str("build")) + _source_dir = create_tracked_tempdir(fs_str("source")) preparer = partialclass( RequirementPreparer, - build_dir=_build_dir.name, - src_dir=_source_dir.name, + build_dir=_build_dir, + src_dir=_source_dir, download_dir=download_dir, wheel_download_dir=WHEEL_DOWNLOAD_DIR, progress_bar="off", @@ -604,13 +611,16 @@ def start_resolver(finder=None, wheel_cache=None): wheel_cache=wheel_cache, use_user_site=False, ) - if packaging.version.parse(pip_version) >= packaging.version.parse('18'): - with RequirementTracker() as req_tracker: - preparer = preparer(req_tracker=req_tracker) + try: + if packaging.version.parse(pip_version) >= packaging.version.parse('18'): + with RequirementTracker() as req_tracker: + preparer = preparer(req_tracker=req_tracker) + yield resolver(preparer=preparer) + else: + preparer = preparer() yield resolver(preparer=preparer) - else: - preparer = preparer() - yield resolver(preparer=preparer) + finally: + finder.session.close() def get_grouped_dependencies(constraints): diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index f9ca97b8..3e482813 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -1,17 +1,21 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import json +import copy import os +import attr +import itertools import plette.lockfiles import six -from vistir.compat import Path -from vistir.contextmanagers import atomic_open_for_write +from vistir.compat import Path, FileNotFoundError +from .project import ProjectFile from .requirements import Requirement +from .utils import optional_instance_of +from ..utils import is_vcs, is_editable, merge_items DEFAULT_NEWLINES = u"\n" @@ -22,83 +26,218 @@ def preferred_newlines(f): return DEFAULT_NEWLINES -class Lockfile(plette.lockfiles.Lockfile): - def __init__(self, *args, **kwargs): - path = kwargs.pop("path", None) - self._requirements = kwargs.pop("requirements", []) - self._dev_requirements = kwargs.pop("dev_requirements", []) - self.path = Path(path) if path else None - self.newlines = u"\n" - super(Lockfile, self).__init__(*args, **kwargs) +is_lockfile = optional_instance_of(plette.lockfiles.Lockfile) +is_projectfile = optional_instance_of(ProjectFile) + + +@attr.s(slots=True) +class Lockfile(object): + path = attr.ib(validator=optional_instance_of(Path), type=Path) + _requirements = attr.ib(default=attr.Factory(list), type=list) + _dev_requirements = attr.ib(default=attr.Factory(list), type=list) + projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) + _lockfile = attr.ib(validator=is_lockfile, type=plette.lockfiles.Lockfile) + newlines = attr.ib(default=DEFAULT_NEWLINES, type=six.text_type) + + @path.default + def _get_path(self): + return Path(os.curdir).absolute() + + @projectfile.default + def _get_projectfile(self): + return self.load_projectfile(self.path) + + @_lockfile.default + def _get_lockfile(self): + return self.projectfile.lockfile + + @property + def lockfile(self): + return self._lockfile + + @property + def section_keys(self): + return ["default", "develop"] + + @property + def extended_keys(self): + return [k for k in itertools.product(self.section_keys, ["", "vcs", "editable"])] + + def get(self, k): + return self.__getitem__(k) + + def __contains__(self, k): + check_lockfile = k in self.extended_keys or self.lockfile.__contains__(k) + if check_lockfile: + return True + return super(Lockfile, self).__contains__(k) + + def __setitem__(self, k, v): + lockfile = self._lockfile + lockfile.__setitem__(k, v) + + def __getitem__(self, k, *args, **kwargs): + retval = None + lockfile = self._lockfile + section = None + pkg_type = None + try: + retval = lockfile[k] + except KeyError: + if "-" in k: + section, _, pkg_type = k.rpartition("-") + vals = getattr(lockfile.get(section, {}), "_data", {}) + if pkg_type == "vcs": + retval = {k: v for k, v in vals.items() if is_vcs(v)} + elif pkg_type == "editable": + retval = {k: v for k, v in vals.items() if is_editable(v)} + if retval is None: + raise + else: + retval = getattr(retval, "_data", retval) + return retval + + def __getattr__(self, k, *args, **kwargs): + retval = None + lockfile = super(Lockfile, self).__getattribute__("_lockfile") + try: + return super(Lockfile, self).__getattribute__(k) + except AttributeError: + retval = getattr(lockfile, k, None) + if retval is not None: + return retval + return super(Lockfile, self).__getattribute__(k, *args, **kwargs) + + def get_deps(self, dev=False, only=True): + deps = {} + if dev: + deps.update(self.develop._data) + if only: + return deps + deps = merge_items([deps, self.default._data]) + return deps @classmethod - def load(cls, path): + def read_projectfile(cls, path): + """Read the specified project file and provide an interface for writing/updating. + + :param str path: Path to the target file. + :return: A project file with the model and location for interaction + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + + pf = ProjectFile.read( + path, + plette.lockfiles.Lockfile, + invalid_ok=True + ) + return pf + + @classmethod + def load_projectfile(cls, path, create=True): + """Given a path, load or create the necessary lockfile. + + :param str path: Path to the project root or lockfile + :param bool create: Whether to create the lockfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the lockfile doesn't exist and ``create=False`` + :return: A project file instance for the supplied project + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + if not path: path = os.curdir path = Path(path).absolute() - if path.is_dir(): - path = path / "Pipfile.lock" - elif path.name == "Pipfile": - path = path.parent / "Pipfile.lock" - if not path.exists(): - raise OSError("Path does not exist: %s" % path) - return cls.create(path.parent, lockfile_name=path.name) + project_path = path if path.is_dir() else path.parent + lockfile_path = project_path / "Pipfile.lock" + if not project_path.exists(): + raise OSError("Project does not exist: %s" % project_path.as_posix()) + elif not lockfile_path.exists() and not create: + raise FileNotFoundError("Lockfile does not exist: %s" % lockfile_path.as_posix()) + projectfile = cls.read_projectfile(lockfile_path.as_posix()) + return projectfile @classmethod - def create(cls, project_path, lockfile_name="Pipfile.lock"): - """Create a new lockfile instance + def load(cls, path, create=True): + """Create a new lockfile instance. :param project_path: Path to project root - :type project_path: str or :class:`~pathlib.Path` - :returns: List[:class:`~requirementslib.Requirement`] objects + :type project_path: str or :class:`pathlib.Path` + :param str lockfile_name: Name of the lockfile in the project root directory + :param pipfile_path: Path to the project pipfile + :type pipfile_path: :class:`pathlib.Path` + :returns: A new lockfile representing the supplied project paths + :rtype: :class:`~requirementslib.models.lockfile.Lockfile` """ - if not isinstance(project_path, Path): - project_path = Path(project_path) - lockfile_path = project_path / lockfile_name - with lockfile_path.open(encoding="utf-8") as f: - lockfile = super(Lockfile, cls).load(f) - lockfile.newlines = preferred_newlines(f) - lockfile.path = lockfile_path - return lockfile + projectfile = cls.load_projectfile(path, create=create) + lockfile_path = Path(projectfile.location) + creation_args = { + "projectfile": projectfile, + "lockfile": projectfile.model, + "newlines": projectfile.line_ending, + "path": lockfile_path + } + return cls(**creation_args) - def get_requirements(self, dev=False): - section = self.develop if dev else self.default - for k in section.keys(): - yield Requirement.from_pipfile(k, section[k]._data) + @classmethod + def create(cls, path, create=True): + return cls.load(path, create=create) + + @property + def develop(self): + return self._lockfile.develop + + @property + def default(self): + return self._lockfile.default + + def get_requirements(self, dev=True, only=False): + """Produces a generator which generates requirements from the desired section. + + :param bool dev: Indicates whether to use dev requirements, defaults to False + :return: Requirements from the relevant the relevant pipfile + :rtype: :class:`~requirementslib.models.requirements.Requirement` + """ + + deps = self.get_deps(dev=dev, only=only) + for k, v in deps.items(): + yield Requirement.from_pipfile(k, v) @property def dev_requirements(self): if not self._dev_requirements: - self._dev_requirements = list(self.get_requirements(dev=True)) + self._dev_requirements = list(self.get_requirements(dev=True, only=True)) return self._dev_requirements @property def requirements(self): if not self._requirements: - self._requirements = list(self.get_requirements(dev=False)) + self._requirements = list(self.get_requirements(dev=False, only=True)) return self._requirements @property def dev_requirements_list(self): - return [{name: entry._data} for name, entry in self.develop.items()] + return [{name: entry._data} for name, entry in self._lockfile.develop.items()] @property def requirements_list(self): - return [{name: entry._data} for name, entry in self.develop.items()] + return [{name: entry._data} for name, entry in self._lockfile.default.items()] def write(self): - open_kwargs = {"newline": self.newlines} - with atomic_open_for_write(self.path.as_posix(), **open_kwargs) as f: - super(Lockfile, self).dump(f, encoding="utf-8") + self.projectfile.model = copy.deepcopy(self._lockfile) + self.projectfile.write() def as_requirements(self, include_hashes=False, dev=False): """Returns a list of requirements in pip-style format""" lines = [] section = self.dev_requirements if dev else self.requirements for req in section: - r = req.as_line() - if not include_hashes: - r = r.split("--hash", 1)[0] + kwargs = { + "include_hashes": include_hashes, + } + if req.editable: + kwargs["include_markers"] = False + r = req.as_line(**kwargs) lines.append(r.strip()) return lines diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 3a6f5b1e..fe7743c2 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -1,64 +1,214 @@ # -*- coding: utf-8 -*- -from vistir.compat import Path + +from __future__ import absolute_import, unicode_literals, print_function + +import attr +import copy +import os + +import tomlkit + +from vistir.compat import Path, FileNotFoundError from .requirements import Requirement +from .project import ProjectFile +from .utils import optional_instance_of from ..exceptions import RequirementError +from ..utils import is_vcs, is_editable, merge_items import plette.pipfiles -class Pipfile(plette.pipfiles.Pipfile): +is_pipfile = optional_instance_of(plette.pipfiles.Pipfile) +is_path = optional_instance_of(Path) +is_projectfile = optional_instance_of(ProjectFile) + + +class PipfileLoader(plette.pipfiles.Pipfile): + @classmethod + def validate(cls, data): + for key, klass in plette.pipfiles.PIPFILE_SECTIONS.items(): + if key not in data or key == "source": + continue + klass.validate(data[key]) + + @classmethod + def load(cls, f, encoding=None): + content = f.read() + if encoding is not None: + content = content.decode(encoding) + _data = tomlkit.loads(content) + if "source" not in _data: + # HACK: There is no good way to prepend a section to an existing + # TOML document, but there's no good way to copy non-structural + # content from one TOML document to another either. Modify the + # TOML content directly, and load the new in-memory document. + sep = "" if content.startswith("\n") else "\n" + content = plette.pipfiles.DEFAULT_SOURCE_TOML + sep + content + data = tomlkit.loads(content) + return cls(data) + + +@attr.s(slots=True) +class Pipfile(object): + path = attr.ib(validator=is_path, type=Path) + projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) + _pipfile = attr.ib(type=plette.pipfiles.Pipfile) + requirements = attr.ib(default=attr.Factory(list), type=list) + dev_requirements = attr.ib(default=attr.Factory(list), type=list) + + @path.default + def _get_path(self): + return Path(os.curdir).absolute() + + @projectfile.default + def _get_projectfile(self): + return self.load_projectfile(os.curdir, create=False) + + @_pipfile.default + def _get_pipfile(self): + return self.projectfile.model + + @property + def pipfile(self): + return self._pipfile + + def get_deps(self, dev=False, only=True): + deps = {} + if dev: + deps.update(self.pipfile._data["dev-packages"]) + if only: + return deps + return merge_items([deps, self.pipfile._data["packages"]]) + + def get(self, k): + return self.__getitem__(k) + + def __contains__(self, k): + check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) + if check_pipfile: + return True + return super(Pipfile, self).__contains__(k) + + def __getitem__(self, k, *args, **kwargs): + retval = None + pipfile = self._pipfile + section = None + pkg_type = None + try: + retval = pipfile[k] + except KeyError: + if "-" in k: + section, _, pkg_type = k.rpartition("-") + vals = getattr(pipfile.get(section, {}), "_data", {}) + if pkg_type == "vcs": + retval = {k: v for k, v in vals.items() if is_vcs(v)} + elif pkg_type == "editable": + retval = {k: v for k, v in vals.items() if is_editable(v)} + if retval is None: + raise + else: + retval = getattr(retval, "_data", retval) + return retval + + def __getattr__(self, k, *args, **kwargs): + retval = None + pipfile = super(Pipfile, self).__getattribute__("_pipfile") + try: + retval = super(Pipfile, self).__getattribute__(k) + except AttributeError: + retval = getattr(pipfile, k, None) + if retval is not None: + return retval + return super(Pipfile, self).__getattribute__(k, *args, **kwargs) @property def requires_python(self): - return self.requires.requires_python + return self._pipfile.requires.requires_python @property def allow_prereleases(self): - return self.get("pipenv", {}).get("allow_prereleases", False) + return self._pipfile.get("pipenv", {}).get("allow_prereleases", False) @classmethod - def load(cls, path): + def read_projectfile(cls, path): + """Read the specified project file and provide an interface for writing/updating. + + :param str path: Path to the target file. + :return: A project file with the model and location for interaction + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + pf = ProjectFile.read( + path, + PipfileLoader, + invalid_ok=True + ) + return pf + + @classmethod + def load_projectfile(cls, path, create=False): + """Given a path, load or create the necessary pipfile. + + :param str path: Path to the project root or pipfile + :param bool create: Whether to create the pipfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` + :return: A project file instance for the supplied project + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + if not path: + raise RuntimeError("Must pass a path to classmethod 'Pipfile.load'") if not isinstance(path, Path): - path = Path(path) - pipfile_path = path / "Pipfile" - if not path.exists(): + path = Path(path).absolute() + pipfile_path = path if path.name == "Pipfile" else path.joinpath("Pipfile") + project_path = pipfile_path.parent + if not project_path.exists(): raise FileNotFoundError("%s is not a valid project path!" % path) elif not pipfile_path.exists() or not pipfile_path.is_file(): - raise RequirementError("%s is not a valid Pipfile" % pipfile_path) - with pipfile_path.open(encoding="utf-8") as fp: - pipfile = super(Pipfile, cls).load(fp) - pipfile.dev_requirements = [ - Requirement.from_pipfile(k, v) for k, v in pipfile.get("dev-packages", {}).items() - ] - pipfile.requirements = [ - Requirement.from_pipfile(k, v) for k, v in pipfile.get("packages", {}).items() - ] - pipfile.path = pipfile_path - return pipfile + if not create: + raise RequirementError("%s is not a valid Pipfile" % pipfile_path) + return cls.read_projectfile(pipfile_path.as_posix()) - # def resolve(self): - # It would be nice to still use this api someday - # option_sources = [s.expanded for s in self.sources] - # pip_args = [] - # if self.pipenv.allow_prereleases: - # pip_args.append('--pre') - # pip_options = get_pip_options(pip_args, sources=option_sources) - # finder = get_finder(sources=option_sources, pip_options=pip_options) - # resolver = DependencyResolver.create(finder=finder, allow_prereleases=self.pipenv.allow_prereleases) - # pkg_dict = {} - # for pkg in self.dev_packages.requirements + self.packages.requirements: - # pkg_dict[pkg.name] = pkg - # resolver.resolve(list(pkg_dict.values())) - # return resolver + @classmethod + def load(cls, path, create=False): + """Given a path, load or create the necessary pipfile. + + :param str path: Path to the project root or pipfile + :param bool create: Whether to create the pipfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` + :return: A pipfile instance pointing at the supplied project + :rtype:: class:`~requirementslib.models.pipfile.Pipfile` + """ + + projectfile = cls.load_projectfile(path, create=create) + pipfile = projectfile.model + dev_requirements = [ + Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("dev-packages", {}).items() + ] + requirements = [ + Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("packages", {}).items() + ] + creation_args = { + "projectfile": projectfile, + "pipfile": pipfile, + "dev_requirements": dev_requirements, + "requirements": requirements, + "path": Path(projectfile.location) + } + return cls(**creation_args) + + def write(self): + self.projectfile.model = copy.deepcopy(self._pipfile) + self.projectfile.write() @property def dev_packages(self, as_requirements=True): if as_requirements: return self.dev_requirements - return self.get('dev-packages', {}) + return self._pipfile.get('dev-packages', {}) @property def packages(self, as_requirements=True): if as_requirements: return self.requirements - return self.get('packages', {}) + return self._pipfile.get('packages', {}) diff --git a/pipenv/vendor/requirementslib/models/project.py b/pipenv/vendor/requirementslib/models/project.py new file mode 100644 index 00000000..f6e037d6 --- /dev/null +++ b/pipenv/vendor/requirementslib/models/project.py @@ -0,0 +1,241 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, unicode_literals + +import collections +import io +import os + +import attr +import packaging.markers +import packaging.utils +import plette +import plette.models +import six +import tomlkit + + +SectionDifference = collections.namedtuple("SectionDifference", [ + "inthis", "inthat", +]) +FileDifference = collections.namedtuple("FileDifference", [ + "default", "develop", +]) + + +def _are_pipfile_entries_equal(a, b): + a = {k: v for k, v in a.items() if k not in ("markers", "hashes", "hash")} + b = {k: v for k, v in b.items() if k not in ("markers", "hashes", "hash")} + if a != b: + return False + try: + marker_eval_a = packaging.markers.Marker(a["markers"]).evaluate() + except (AttributeError, KeyError, TypeError, ValueError): + marker_eval_a = True + try: + marker_eval_b = packaging.markers.Marker(b["markers"]).evaluate() + except (AttributeError, KeyError, TypeError, ValueError): + marker_eval_b = True + return marker_eval_a == marker_eval_b + + +DEFAULT_NEWLINES = "\n" + + +def preferred_newlines(f): + if isinstance(f.newlines, six.text_type): + return f.newlines + return DEFAULT_NEWLINES + + +@attr.s +class ProjectFile(object): + """A file in the Pipfile project. + """ + location = attr.ib() + line_ending = attr.ib() + model = attr.ib() + + @classmethod + def read(cls, location, model_cls, invalid_ok=False): + try: + with io.open(location, encoding="utf-8") as f: + model = model_cls.load(f) + line_ending = preferred_newlines(f) + except Exception: + if not invalid_ok: + raise + model = None + line_ending = DEFAULT_NEWLINES + return cls(location=location, line_ending=line_ending, model=model) + + def write(self): + kwargs = {"encoding": "utf-8", "newline": self.line_ending} + with io.open(self.location, "w", **kwargs) as f: + self.model.dump(f) + + def dumps(self): + strio = six.StringIO() + self.model.dump(strio) + return strio.getvalue() + + +@attr.s +class Project(object): + + root = attr.ib() + _p = attr.ib(init=False) + _l = attr.ib(init=False) + + def __attrs_post_init__(self): + self.root = root = os.path.abspath(self.root) + self._p = ProjectFile.read( + os.path.join(root, "Pipfile"), + plette.Pipfile, + ) + self._l = ProjectFile.read( + os.path.join(root, "Pipfile.lock"), + plette.Lockfile, + invalid_ok=True, + ) + + @property + def pipfile(self): + return self._p.model + + @property + def pipfile_location(self): + return self._p.location + + @property + def lockfile(self): + return self._l.model + + @property + def lockfile_location(self): + return self._l.location + + @lockfile.setter + def lockfile(self, new): + self._l.model = new + + def is_synced(self): + return self.lockfile and self.lockfile.is_up_to_date(self.pipfile) + + def _get_pipfile_section(self, develop, insert=True): + name = "dev-packages" if develop else "packages" + try: + section = self.pipfile[name] + except KeyError: + section = plette.models.PackageCollection(tomlkit.table()) + if insert: + self.pipfile[name] = section + return section + + def contains_key_in_pipfile(self, key): + sections = [ + self._get_pipfile_section(develop=False, insert=False), + self._get_pipfile_section(develop=True, insert=False), + ] + return any( + (packaging.utils.canonicalize_name(name) == + packaging.utils.canonicalize_name(key)) + for section in sections + for name in section + ) + + def add_line_to_pipfile(self, line, develop): + from requirementslib import Requirement + requirement = Requirement.from_line(line) + section = self._get_pipfile_section(develop=develop) + key = requirement.normalized_name + entry = next(iter(requirement.as_pipfile().values())) + if isinstance(entry, dict): + # HACK: TOMLKit prefers to expand tables by default, but we + # always want inline tables here. Also tomlkit.inline_table + # does not have `update()`. + table = tomlkit.inline_table() + for k, v in entry.items(): + table[k] = v + entry = table + section[key] = entry + + def remove_keys_from_pipfile(self, keys, default, develop): + keys = {packaging.utils.canonicalize_name(key) for key in keys} + sections = [] + if default: + sections.append(self._get_pipfile_section( + develop=False, insert=False, + )) + if develop: + sections.append(self._get_pipfile_section( + develop=True, insert=False, + )) + for section in sections: + removals = set() + for name in section: + if packaging.utils.canonicalize_name(name) in keys: + removals.add(name) + for key in removals: + del section._data[key] + + def remove_keys_from_lockfile(self, keys): + keys = {packaging.utils.canonicalize_name(key) for key in keys} + removed = False + for section_name in ("default", "develop"): + try: + section = self.lockfile[section_name] + except KeyError: + continue + removals = set() + for name in section: + if packaging.utils.canonicalize_name(name) in keys: + removals.add(name) + removed = removed or bool(removals) + for key in removals: + del section._data[key] + + if removed: + # HACK: The lock file no longer represents the Pipfile at this + # point. Set the hash to an arbitrary invalid value. + self.lockfile.meta.hash = plette.models.Hash({"__invalid__": ""}) + + def difference_lockfile(self, lockfile): + """Generate a difference between the current and given lockfiles. + + Returns a 2-tuple containing differences in default in develop + sections. + + Each element is a 2-tuple of dicts. The first, `inthis`, contains + entries only present in the current lockfile; the second, `inthat`, + contains entries only present in the given one. + + If a key exists in both this and that, but the values differ, the key + is present in both dicts, pointing to values from each file. + """ + diff_data = { + "default": SectionDifference({}, {}), + "develop": SectionDifference({}, {}), + } + for section_name, section_diff in diff_data.items(): + try: + this = self.lockfile[section_name]._data + except (KeyError, TypeError): + this = {} + try: + that = lockfile[section_name]._data + except (KeyError, TypeError): + that = {} + for key, this_value in this.items(): + try: + that_value = that[key] + except KeyError: + section_diff.inthis[key] = this_value + continue + if not _are_pipfile_entries_equal(this_value, that_value): + section_diff.inthis[key] = this_value + section_diff.inthat[key] = that_value + for key, that_value in that.items(): + if key not in this: + section_diff.inthat[key] = that_value + return FileDifference(**diff_data) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 248ca777..08eb5618 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- + from __future__ import absolute_import -import atexit import collections import hashlib import os @@ -9,49 +9,43 @@ import os from contextlib import contextmanager import attr +import six from first import first from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement from packaging.specifiers import Specifier, SpecifierSet from packaging.utils import canonicalize_name -from pip_shims.shims import ( - InstallRequirement, Link, Wheel, _strip_extras, parse_version, path_to_url, - url_to_path -) +from pip_shims.shims import _strip_extras, parse_version, path_to_url, url_to_path, Link from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote -from vistir.compat import FileNotFoundError, Path, TemporaryDirectory +from vistir.compat import FileNotFoundError, Path from vistir.misc import dedup from vistir.path import ( create_tracked_tempdir, get_converted_relative_path, is_file_url, - is_valid_url, mkdir_p + is_valid_url ) from ..exceptions import RequirementError from ..utils import VCS_LIST, is_installable_file, is_vcs, ensure_setup_py from .baserequirement import BaseRequirement -from .dependencies import ( - AbstractDependency, find_all_matches, get_abstract_dependencies, - get_dependencies, get_finder -) from .markers import PipenvMarkers from .utils import ( HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string, filter_none, format_requirement, get_version, init_requirement, - is_pinned_requirement, make_install_requirement, optional_instance_of, - parse_extras, specs_to_string, split_markers_from_line, + is_pinned_requirement, make_install_requirement, optional_instance_of, parse_extras, + specs_to_string, split_markers_from_line, ireq_from_editable, ireq_from_line, split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path, - validate_specifiers, validate_vcs + validate_specifiers, validate_vcs, normalize_name, create_link, + Requirement as PkgResourcesRequirement ) -from .vcs import VCSRepository -@attr.s +@attr.s(slots=True) class NamedRequirement(BaseRequirement): name = attr.ib() version = attr.ib(validator=attr.validators.optional(validate_specifiers)) - req = attr.ib() + req = attr.ib(type=PkgResourcesRequirement) extras = attr.ib(default=attr.Factory(list)) editable = attr.ib(default=False) @@ -99,7 +93,7 @@ class NamedRequirement(BaseRequirement): # FIXME: This should actually be canonicalized but for now we have to # simply lowercase it and replace underscores, since full canonicalization # also replaces dots and that doesn't actually work when querying the index - return "{0}".format(self.name.lower().replace("_", "-")) + return "{0}".format(normalize_name(self.name)) @property def pipfile_part(self): @@ -115,21 +109,28 @@ LinkInfo = collections.namedtuple( ) -@attr.s +@attr.s(slots=True) class FileRequirement(BaseRequirement): """File requirements for tar.gz installable files or wheels or setup.py containing directories.""" + #: Path to the relevant `setup.py` location setup_path = attr.ib(default=None) + #: path to hit - without any of the VCS prefixes (like git+ / http+ / etc) path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) - # : path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - editable = attr.ib(default=None) + #: Whether the package is editable + editable = attr.ib(default=False) + #: Extras if applicable extras = attr.ib(default=attr.Factory(list)) + #: URI of the package uri = attr.ib() + #: Link object representing the package to clone link = attr.ib() + _has_hashed_name = attr.ib(default=False) + #: Package name name = attr.ib() + #: A :class:`~pkg_resources.Requirement` isntance req = attr.ib() - _has_hashed_name = False _uri_scheme = attr.ib(default=None) @classmethod @@ -164,6 +165,7 @@ class FileRequirement(BaseRequirement): See `https://bugs.python.org/issue23505#msg277350`. """ + # Git allows `git@github.com...` lines that are not really URIs. # Add "ssh://" so we can parse correctly, and restore afterwards. fixed_line = add_ssh_scheme_to_git_uri(line) @@ -174,7 +176,7 @@ class FileRequirement(BaseRequirement): p = Path(fixed_line).absolute() path = p.as_posix() uri = p.as_uri() - link = Link(uri) + link = create_link(uri) try: relpath = get_converted_relative_path(path) except ValueError: @@ -223,7 +225,7 @@ class FileRequirement(BaseRequirement): uri = strip_ssh_from_git_uri(original_uri) # Re-attach VCS prefix to build a Link. - link = Link( + link = create_link( urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme)) ) @@ -244,6 +246,7 @@ class FileRequirement(BaseRequirement): if self.link and self.link.egg_fragment: return self.link.egg_fragment elif self.link and self.link.is_wheel: + from pip_shims import Wheel return Wheel(self.link.filename).name if ( self._uri_scheme != "uri" @@ -261,7 +264,7 @@ class FileRequirement(BaseRequirement): except (FileNotFoundError, IOError) as e: dist = None except Exception as e: - from pip_shims.shims import InstallRequirement, make_abstract_dist + from pip_shims.shims import make_abstract_dist try: if not isinstance(Path, self.path): @@ -269,9 +272,9 @@ class FileRequirement(BaseRequirement): else: _path = self.path if self.editable: - _ireq = InstallRequirement.from_editable(_path.as_uri()) + _ireq = ireq_from_editable(_path.as_uri()) else: - _ireq = InstallRequirement.from_line(_path.as_posix()) + _ireq = ireq_from_line(_path.as_posix()) dist = make_abstract_dist(_ireq).get_dist() name = dist.project_name except (TypeError, ValueError, AttributeError) as e: @@ -284,7 +287,7 @@ class FileRequirement(BaseRequirement): self._has_hashed_name = True name = hashed_name if self.link and not self._has_hashed_name: - self.link = Link("{0}#egg={1}".format(self.link.url, name)) + self.link = create_link("{0}#egg={1}".format(self.link.url, name)) return name @link.default @@ -292,12 +295,12 @@ class FileRequirement(BaseRequirement): target = "{0}".format(self.uri) if hasattr(self, "name"): target = "{0}#egg={1}".format(target, self.name) - link = Link(target) + link = create_link(target) return link @req.default def get_requirement(self): - req = init_requirement(canonicalize_name(self.name)) + req = init_requirement(normalize_name(self.name)) req.editable = False req.line = self.link.url_without_fragment if self.path and self.link and self.link.scheme.startswith("file"): @@ -357,6 +360,7 @@ class FileRequirement(BaseRequirement): "uri_scheme": prefer, } if link and link.is_wheel: + from pip_shims import Wheel arg_dict["name"] = Wheel(link.filename).name elif link.egg_fragment: arg_dict["name"] = link.egg_fragment @@ -396,7 +400,7 @@ class FileRequirement(BaseRequirement): if not uri: uri = path_to_url(path) - link = Link(uri) + link = create_link(uri) arg_dict = { "name": name, @@ -473,14 +477,19 @@ class FileRequirement(BaseRequirement): return {name: pipfile_dict} -@attr.s +@attr.s(slots=True) class VCSRequirement(FileRequirement): + #: Whether the repository is editable editable = attr.ib(default=None) + #: URI for the repository uri = attr.ib(default=None) + #: path to the repository, if it's local path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) + #: vcs type, i.e. git/hg/svn vcs = attr.ib(validator=attr.validators.optional(validate_vcs), default=None) - # : vcs reference name (branch / commit / tag) + #: vcs reference name (branch / commit / tag) ref = attr.ib(default=None) + #: Subdirectory to use for installation if applicable subdirectory = attr.ib(default=None) _repo = attr.ib(default=None) _base_line = attr.ib(default=None) @@ -586,17 +595,30 @@ class VCSRequirement(FileRequirement): return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) def get_vcs_repo(self, src_dir=None): + from .vcs import VCSRepository checkout_dir = self.get_checkout_dir(src_dir=src_dir) - url = "{0}#egg={1}".format(self.vcs_uri, self.name) + link = build_vcs_link( + self.vcs, + self.uri, + name=self.name, + ref=self.ref, + subdirectory=self.subdirectory, + extras=self.extras + ) vcsrepo = VCSRepository( - url=url, + url=link.url, name=self.name, ref=self.ref if self.ref else None, checkout_directory=checkout_dir, - vcs_type=self.vcs + vcs_type=self.vcs, + subdirectory=self.subdirectory ) if not self.is_local: vcsrepo.obtain() + if self.subdirectory: + self.setup_path = os.path.join(checkout_dir, self.subdirectory, "setup.py") + else: + self.setup_path = os.path.join(checkout_dir, "setup.py") return vcsrepo def get_commit_hash(self): @@ -614,15 +636,15 @@ class VCSRequirement(FileRequirement): if not self.is_local and ref is not None: self.repo.checkout_ref(ref) repo_hash = self.repo.get_commit_hash() + self.req.revision = repo_hash return repo_hash @contextmanager def locked_vcs_repo(self, src_dir=None): + if not src_dir: + src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") vcsrepo = self.get_vcs_repo(src_dir=src_dir) - if self.ref and not self.is_local: - vcsrepo.checkout_ref(self.ref) - self.ref = self.get_commit_hash() - self.req.revision = self.ref + self.req.revision = vcsrepo.get_commit_hash() # Remove potential ref in the end of uri after ref is parsed if "@" in self.link.show_url and "@" in self.uri: @@ -731,7 +753,7 @@ class VCSRequirement(FileRequirement): @property def pipfile_part(self): - excludes = ["_repo", "_base_line"] + excludes = ["_repo", "_base_line", "setup_path"] filter_func = lambda k, v: bool(v) is True and k.name not in excludes pipfile_dict = attr.asdict(self, filter=filter_func).copy() if "vcs" in pipfile_dict: @@ -823,6 +845,7 @@ class Requirement(object): @classmethod def from_line(cls, line): + from pip_shims import InstallRequirement if isinstance(line, InstallRequirement): line = format_requirement(line) hashes = None @@ -948,7 +971,8 @@ class Requirement(object): cls_inst.req.req.line = cls_inst.as_line() return cls_inst - def as_line(self, sources=None, include_hashes=True, include_extras=True, as_list=False): + def as_line(self, sources=None, include_hashes=True, include_extras=True, + include_markers=True, as_list=False): """Format this requirement as a line in requirements.txt. If ``sources`` provided, it should be an sequence of mappings, containing @@ -967,7 +991,7 @@ class Requirement(object): self.req.line_part, self.extras_as_pip if include_extras else "", self.specifiers if include_specifiers else "", - self.markers_as_pip, + self.markers_as_pip if include_markers else "", ] if as_list: # This is used for passing to a subprocess call @@ -1070,10 +1094,10 @@ class Requirement(object): if self.editable or self.req.editable: if ireq_line.startswith("-e "): ireq_line = ireq_line[len("-e "):] - with ensure_setup_py(self.req.path): - ireq = InstallRequirement.from_editable(ireq_line) + with ensure_setup_py(self.req.setup_path): + ireq = ireq_from_editable(ireq_line) else: - ireq = InstallRequirement.from_line(ireq_line) + ireq = ireq_from_line(ireq_line) if not getattr(ireq, "req", None): ireq.req = self.req.req else: @@ -1100,6 +1124,8 @@ class Requirement(object): :return: A set of requirement strings of the dependencies of this requirement. :rtype: set(str) """ + + from .dependencies import get_dependencies if not sources: sources = [{ 'name': 'pypi', @@ -1119,6 +1145,7 @@ class Requirement(object): :rtype: list[ :class:`~requirementslib.models.dependency.AbstractDependency` ] """ + from .dependencies import AbstractDependency, get_dependencies, get_abstract_dependencies if not self.abstract_dep: parent = getattr(self, 'parent', None) self.abstract_dep = AbstractDependency.from_requirement(self, parent=parent) @@ -1141,6 +1168,8 @@ class Requirement(object): :return: A list of Installation Candidates :rtype: list[ :class:`~pip._internal.index.InstallationCandidate` ] """ + + from .dependencies import get_finder, find_all_matches if not finder: finder = get_finder(sources=sources) return find_all_matches(finder, self.as_ireq()) diff --git a/pipenv/vendor/requirementslib/models/resolvers.py b/pipenv/vendor/requirementslib/models/resolvers.py index da6d0dda..1a239390 100644 --- a/pipenv/vendor/requirementslib/models/resolvers.py +++ b/pipenv/vendor/requirementslib/models/resolvers.py @@ -4,11 +4,10 @@ from contextlib import contextmanager import attr import six -from pip_shims.shims import VcsSupport, Wheel +from pip_shims.shims import Wheel -from ..utils import log +from ..utils import log, VCS_SUPPORT from .cache import HashCache -from .dependencies import AbstractDependency, find_all_matches, get_finder from .utils import format_requirement, is_pinned_requirement, version_from_ireq @@ -41,6 +40,7 @@ class DependencyResolver(object): @classmethod def create(cls, finder=None, allow_prereleases=False, get_all_hashes=True): if not finder: + from .dependencies import get_finder finder_args = [] if allow_prereleases: finder_args.append('--pre') @@ -140,6 +140,7 @@ class DependencyResolver(object): # Coerce input into AbstractDependency instances. # We accept str, Requirement, and AbstractDependency as input. + from .dependencies import AbstractDependency for dep in root_nodes: if isinstance(dep, six.string_types): dep = AbstractDependency.from_string(dep) @@ -183,6 +184,7 @@ class DependencyResolver(object): def get_hashes_for_one(self, ireq): if not self.finder: + from .dependencies import get_finder finder_args = [] if self.allow_prereleases: finder_args.append('--pre') @@ -191,7 +193,7 @@ class DependencyResolver(object): if ireq.editable: return set() - vcs = VcsSupport() + vcs = VCS_SUPPORT if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: return set() @@ -201,6 +203,7 @@ class DependencyResolver(object): matching_candidates = set() with self.allow_all_wheels(): + from .dependencies import find_all_matches matching_candidates = ( find_all_matches(self.finder, ireq, pre=self.allow_prereleases) ) diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index cba63295..d72542c4 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -19,7 +19,7 @@ from packaging.requirements import Requirement as PackagingRequirement from pkg_resources import Requirement from vistir.misc import dedup -from pip_shims.shims import InstallRequirement, Link + from ..utils import SCHEME_LIST, VCS_LIST, is_star @@ -37,6 +37,21 @@ def optional_instance_of(cls): return validators.optional(validators.instance_of(cls)) +def create_link(link): + from pip_shims import Link + return Link(link) + + +def ireq_from_line(ireq): + from pip_shims import InstallRequirement + return InstallRequirement.from_line(ireq) + + +def ireq_from_editable(ireq): + from pip_shims import InstallRequirement + return InstallRequirement.from_editable(ireq) + + def init_requirement(name): req = Requirement.parse(name) req.vcs = None @@ -92,7 +107,7 @@ def build_vcs_link(vcs, uri, name=None, ref=None, subdirectory=None, extras=None uri = "{0}{1}".format(uri, extras) if subdirectory: uri = "{0}&subdirectory={1}".format(uri, subdirectory) - return Link(uri) + return create_link(uri) def get_version(pipfile_entry): @@ -117,7 +132,7 @@ def strip_ssh_from_git_uri(uri): def add_ssh_scheme_to_git_uri(uri): - """Cleans VCS uris from pip format""" + """Cleans VCS uris from pipenv.patched.notpip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: @@ -443,11 +458,11 @@ def make_install_requirement(name, version, extras, markers, constraint=False): extras_string = "[{}]".format(",".join(sorted(extras))) if not markers: - return InstallRequirement.from_line( + return ireq_from_line( str('{}{}=={}'.format(name, extras_string, version)), constraint=constraint) else: - return InstallRequirement.from_line( + return ireq_from_line( str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))), constraint=constraint) @@ -508,3 +523,15 @@ def fix_requires_python_marker(requires_python): ]) marker_to_add = PackagingRequirement('fakepkg; {0}'.format(marker_str)).marker return marker_to_add + + +def normalize_name(pkg): + """Given a package name, return its normalized, non-canonicalized form. + + :param str pkg: The name of a package + :return: A normalized package name + :rtype: str + """ + + assert isinstance(pkg, six.string_types) + return pkg.replace("_", "-").lower() diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index fb2e6bc3..4efb9bd3 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -1,7 +1,6 @@ # -*- coding=utf-8 -*- import attr from pip_shims import VcsSupport, parse_version, pip_version -import vistir import os @@ -14,6 +13,7 @@ class VCSRepository(object): name = attr.ib() checkout_directory = attr.ib() vcs_type = attr.ib() + subdirectory = attr.ib(default=None) commit_sha = attr.ib(default=None) ref = attr.ib(default=None) repo_instance = attr.ib() @@ -31,35 +31,31 @@ class VCSRepository(object): return url.startswith("file") def obtain(self): - if not os.path.exists(self.checkout_directory): + if (os.path.exists(self.checkout_directory) and not + self.repo_instance.is_repository_directory(self.checkout_directory)): + self.repo_instance.unpack(self.checkout_directory) + elif not os.path.exists(self.checkout_directory): self.repo_instance.obtain(self.checkout_directory) - if self.ref: - self.checkout_ref(self.ref) - self.commit_sha = self.get_commit_hash(self.ref) else: - if not self.commit_sha: - self.commit_sha = self.get_commit_hash() + if self.ref: + self.checkout_ref(self.ref) + if not self.commit_sha: + self.commit_sha = self.get_commit_hash() def checkout_ref(self, ref): if not self.repo_instance.is_commit_id_equal( - self.checkout_directory, self.get_commit_hash(ref) + self.checkout_directory, self.get_commit_hash() ) and not self.repo_instance.is_commit_id_equal(self.checkout_directory, ref): if not self.is_local: self.update(ref) def update(self, ref): target_ref = self.repo_instance.make_rev_options(ref) - sha = self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) - target_rev = target_ref.make_new(sha) if parse_version(pip_version) > parse_version("18.0"): self.repo_instance.update(self.checkout_directory, self.url, target_ref) else: self.repo_instance.update(self.checkout_directory, target_ref) - self.commit_hash = self.get_commit_hash(ref) + self.commit_sha = self.get_commit_hash() def get_commit_hash(self, ref=None): - if ref: - target_ref = self.repo_instance.make_rev_options(ref) - return self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) - # return self.repo_instance.get_revision(self.checkout_directory) return self.repo_instance.get_revision(self.checkout_directory) diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index b490d3cf..75a05ce0 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -6,8 +6,14 @@ import logging import os import six +import sys import tomlkit +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) +from six.moves import Mapping, Sequence, Set, ItemsView from six.moves.urllib.parse import urlparse, urlsplit from pip_shims.shims import ( @@ -22,14 +28,16 @@ VCS_LIST = ("git", "svn", "hg", "bzr") VCS_SCHEMES = [] SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") +VCS_SUPPORT = VcsSupport() + if not VCS_SCHEMES: - VCS_SCHEMES = VcsSupport().all_schemes + VCS_SCHEMES = VCS_SUPPORT.all_schemes def setup_logger(): logger = logging.getLogger("requirementslib") loglevel = logging.DEBUG - handler = logging.StreamHandler() + handler = logging.StreamHandler(stream=sys.stderr) handler.setLevel(loglevel) logger.addHandler(handler) logger.setLevel(loglevel) @@ -66,6 +74,12 @@ def is_vcs(pipfile_entry): return False +def is_editable(pipfile_entry): + if isinstance(pipfile_entry, Mapping): + return pipfile_entry.get("editable", False) is True + return False + + def multi_split(s, split): """Splits on multiple given separators.""" for r in split: @@ -181,3 +195,367 @@ def ensure_setup_py(base_dir): finally: if is_new: setup_py.unlink() + + + +_UNSET = object() +_REMAP_EXIT = object() + + +# The following functionality is either borrowed or modified from the itertools module +# in the boltons library by Mahmoud Hashemi and distributed under the BSD license +# the text of which is included below: + +# (original text from https://github.com/mahmoud/boltons/blob/master/LICENSE) +# Copyright (c) 2013, Mahmoud Hashemi +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * The names of the contributors may not be used to endorse or +# promote products derived from this software without specific +# prior written permission. +# +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class PathAccessError(KeyError, IndexError, TypeError): + """An amalgamation of KeyError, IndexError, and TypeError, + representing what can occur when looking up a path in a nested + object. + """ + def __init__(self, exc, seg, path): + self.exc = exc + self.seg = seg + self.path = path + + def __repr__(self): + cn = self.__class__.__name__ + return '%s(%r, %r, %r)' % (cn, self.exc, self.seg, self.path) + + def __str__(self): + return ('could not access %r from path %r, got error: %r' + % (self.seg, self.path, self.exc)) + + +def get_path(root, path, default=_UNSET): + """Retrieve a value from a nested object via a tuple representing the + lookup path. + >>> root = {'a': {'b': {'c': [[1], [2], [3]]}}} + >>> get_path(root, ('a', 'b', 'c', 2, 0)) + 3 + The path format is intentionally consistent with that of + :func:`remap`. + One of get_path's chief aims is improved error messaging. EAFP is + great, but the error messages are not. + For instance, ``root['a']['b']['c'][2][1]`` gives back + ``IndexError: list index out of range`` + What went out of range where? get_path currently raises + ``PathAccessError: could not access 2 from path ('a', 'b', 'c', 2, + 1), got error: IndexError('list index out of range',)``, a + subclass of IndexError and KeyError. + You can also pass a default that covers the entire operation, + should the lookup fail at any level. + Args: + root: The target nesting of dictionaries, lists, or other + objects supporting ``__getitem__``. + path (tuple): A list of strings and integers to be successively + looked up within *root*. + default: The value to be returned should any + ``PathAccessError`` exceptions be raised. + """ + if isinstance(path, six.string_types): + path = path.split('.') + cur = root + try: + for seg in path: + try: + cur = cur[seg] + except (KeyError, IndexError) as exc: + raise PathAccessError(exc, seg, path) + except TypeError as exc: + # either string index in a list, or a parent that + # doesn't support indexing + try: + seg = int(seg) + cur = cur[seg] + except (ValueError, KeyError, IndexError, TypeError): + if not getattr(cur, "__iter__", None): + exc = TypeError('%r object is not indexable' + % type(cur).__name__) + raise PathAccessError(exc, seg, path) + except PathAccessError: + if default is _UNSET: + raise + return default + return cur + + +def default_visit(path, key, value): + return key, value + + +_orig_default_visit = default_visit + + +# Modified from https://github.com/mahmoud/boltons/blob/master/boltons/iterutils.py +def dict_path_enter(path, key, value): + if isinstance(value, six.string_types): + return value, False + elif isinstance(value, (Mapping, dict)): + return value.__class__(), ItemsView(value) + elif isinstance(value, tomlkit.items.Array): + return value.__class__([], value.trivia), enumerate(value) + elif isinstance(value, (Sequence, list)): + return value.__class__(), enumerate(value) + elif isinstance(value, (Set, set)): + return value.__class__(), enumerate(value) + else: + return value, False + + +def dict_path_exit(path, key, old_parent, new_parent, new_items): + ret = new_parent + if isinstance(new_parent, (Mapping, dict)): + vals = dict(new_items) + try: + new_parent.update(new_items) + except AttributeError: + # Handle toml containers specifically + try: + new_parent.update(vals) + # Now use default fallback if needed + except AttributeError: + ret = new_parent.__class__(vals) + elif isinstance(new_parent, tomlkit.items.Array): + vals = tomlkit.items.item([v for i, v in new_items]) + try: + new_parent._value.extend(vals._value) + except AttributeError: + ret = tomlkit.items.item(vals) + elif isinstance(new_parent, (Sequence, list)): + vals = [v for i, v in new_items] + try: + new_parent.extend(vals) + except AttributeError: + ret = new_parent.__class__(vals) # tuples + elif isinstance(new_parent, (Set, set)): + vals = [v for i, v in new_items] + try: + new_parent.update(vals) + except AttributeError: + ret = new_parent.__class__(vals) # frozensets + else: + raise RuntimeError('unexpected iterable type: %r' % type(new_parent)) + return ret + + +def remap(root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, + **kwargs): + """The remap ("recursive map") function is used to traverse and + transform nested structures. Lists, tuples, sets, and dictionaries + are just a few of the data structures nested into heterogenous + tree-like structures that are so common in programming. + Unfortunately, Python's built-in ways to manipulate collections + are almost all flat. List comprehensions may be fast and succinct, + but they do not recurse, making it tedious to apply quick changes + or complex transforms to real-world data. + remap goes where list comprehensions cannot. + Here's an example of removing all Nones from some data: + >>> from pprint import pprint + >>> reviews = {'Star Trek': {'TNG': 10, 'DS9': 8.5, 'ENT': None}, + ... 'Babylon 5': 6, 'Dr. Who': None} + >>> pprint(remap(reviews, lambda p, k, v: v is not None)) + {'Babylon 5': 6, 'Star Trek': {'DS9': 8.5, 'TNG': 10}} + Notice how both Nones have been removed despite the nesting in the + dictionary. Not bad for a one-liner, and that's just the beginning. + See `this remap cookbook`_ for more delicious recipes. + .. _this remap cookbook: http://sedimental.org/remap.html + remap takes four main arguments: the object to traverse and three + optional callables which determine how the remapped object will be + created. + Args: + root: The target object to traverse. By default, remap + supports iterables like :class:`list`, :class:`tuple`, + :class:`dict`, and :class:`set`, but any object traversable by + *enter* will work. + visit (callable): This function is called on every item in + *root*. It must accept three positional arguments, *path*, + *key*, and *value*. *path* is simply a tuple of parents' + keys. *visit* should return the new key-value pair. It may + also return ``True`` as shorthand to keep the old item + unmodified, or ``False`` to drop the item from the new + structure. *visit* is called after *enter*, on the new parent. + The *visit* function is called for every item in root, + including duplicate items. For traversable values, it is + called on the new parent object, after all its children + have been visited. The default visit behavior simply + returns the key-value pair unmodified. + enter (callable): This function controls which items in *root* + are traversed. It accepts the same arguments as *visit*: the + path, the key, and the value of the current item. It returns a + pair of the blank new parent, and an iterator over the items + which should be visited. If ``False`` is returned instead of + an iterator, the value will not be traversed. + The *enter* function is only called once per unique value. The + default enter behavior support mappings, sequences, and + sets. Strings and all other iterables will not be traversed. + exit (callable): This function determines how to handle items + once they have been visited. It gets the same three + arguments as the other functions -- *path*, *key*, *value* + -- plus two more: the blank new parent object returned + from *enter*, and a list of the new items, as remapped by + *visit*. + Like *enter*, the *exit* function is only called once per + unique value. The default exit behavior is to simply add + all new items to the new parent, e.g., using + :meth:`list.extend` and :meth:`dict.update` to add to the + new parent. Immutable objects, such as a :class:`tuple` or + :class:`namedtuple`, must be recreated from scratch, but + use the same type as the new parent passed back from the + *enter* function. + reraise_visit (bool): A pragmatic convenience for the *visit* + callable. When set to ``False``, remap ignores any errors + raised by the *visit* callback. Items causing exceptions + are kept. See examples for more details. + remap is designed to cover the majority of cases with just the + *visit* callable. While passing in multiple callables is very + empowering, remap is designed so very few cases should require + passing more than one function. + When passing *enter* and *exit*, it's common and easiest to build + on the default behavior. Simply add ``from boltons.iterutils import + default_enter`` (or ``default_exit``), and have your enter/exit + function call the default behavior before or after your custom + logic. See `this example`_. + Duplicate and self-referential objects (aka reference loops) are + automatically handled internally, `as shown here`_. + .. _this example: http://sedimental.org/remap.html#sort_all_lists + .. _as shown here: http://sedimental.org/remap.html#corner_cases + """ + # TODO: improve argument formatting in sphinx doc + # TODO: enter() return (False, items) to continue traverse but cancel copy? + if not callable(visit): + raise TypeError('visit expected callable, not: %r' % visit) + if not callable(enter): + raise TypeError('enter expected callable, not: %r' % enter) + if not callable(exit): + raise TypeError('exit expected callable, not: %r' % exit) + reraise_visit = kwargs.pop('reraise_visit', True) + if kwargs: + raise TypeError('unexpected keyword arguments: %r' % kwargs.keys()) + + path, registry, stack = (), {}, [(None, root)] + new_items_stack = [] + while stack: + key, value = stack.pop() + id_value = id(value) + if key is _REMAP_EXIT: + key, new_parent, old_parent = value + id_value = id(old_parent) + path, new_items = new_items_stack.pop() + value = exit(path, key, old_parent, new_parent, new_items) + registry[id_value] = value + if not new_items_stack: + continue + elif id_value in registry: + value = registry[id_value] + else: + res = enter(path, key, value) + try: + new_parent, new_items = res + except TypeError: + # TODO: handle False? + raise TypeError('enter should return a tuple of (new_parent,' + ' items_iterator), not: %r' % res) + if new_items is not False: + # traverse unless False is explicitly passed + registry[id_value] = new_parent + new_items_stack.append((path, [])) + if value is not root: + path += (key,) + stack.append((_REMAP_EXIT, (key, new_parent, value))) + if new_items: + stack.extend(reversed(list(new_items))) + continue + if visit is _orig_default_visit: + # avoid function call overhead by inlining identity operation + visited_item = (key, value) + else: + try: + visited_item = visit(path, key, value) + except Exception: + if reraise_visit: + raise + visited_item = True + if visited_item is False: + continue # drop + elif visited_item is True: + visited_item = (key, value) + # TODO: typecheck? + # raise TypeError('expected (key, value) from visit(),' + # ' not: %r' % visited_item) + try: + new_items_stack[-1][1].append(visited_item) + except IndexError: + raise TypeError('expected remappable root, not: %r' % root) + return value + + +def merge_items(target_list, sourced=False): + if not sourced: + target_list = [(id(t), t) for t in target_list] + + ret = None + source_map = {} + + def remerge_enter(path, key, value): + new_parent, new_items = dict_path_enter(path, key, value) + if ret and not path and key is None: + new_parent = ret + + try: + cur_val = get_path(ret, path + (key,)) + except KeyError as ke: + pass + else: + new_parent = cur_val + + return new_parent, new_items + + def remerge_exit(path, key, old_parent, new_parent, new_items): + return dict_path_exit(path, key, old_parent, new_parent, new_items) + + for t_name, target in target_list: + if sourced: + def remerge_visit(path, key, value): + source_map[path + (key,)] = t_name + return True + else: + remerge_visit = default_visit + + ret = remap(target, enter=remerge_enter, visit=remerge_visit, + exit=remerge_exit) + + if not sourced: + return ret + return ret, source_map diff --git a/pipenv/vendor/resolvelib/LICENSE b/pipenv/vendor/resolvelib/LICENSE new file mode 100644 index 00000000..b9077766 --- /dev/null +++ b/pipenv/vendor/resolvelib/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2018, Tzu-ping Chung + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/pipenv/vendor/resolvelib/__init__.py b/pipenv/vendor/resolvelib/__init__.py new file mode 100644 index 00000000..e0e37434 --- /dev/null +++ b/pipenv/vendor/resolvelib/__init__.py @@ -0,0 +1,16 @@ +__all__ = [ + '__version__', + 'AbstractProvider', 'BaseReporter', 'Resolver', + 'NoVersionsAvailable', 'RequirementsConflicted', + 'ResolutionError', 'ResolutionImpossible', 'ResolutionTooDeep', +] + +__version__ = '0.2.2' + + +from .providers import AbstractProvider +from .reporters import BaseReporter +from .resolvers import ( + NoVersionsAvailable, RequirementsConflicted, + Resolver, ResolutionError, ResolutionImpossible, ResolutionTooDeep, +) diff --git a/pipenv/vendor/resolvelib/providers.py b/pipenv/vendor/resolvelib/providers.py new file mode 100644 index 00000000..515c0db4 --- /dev/null +++ b/pipenv/vendor/resolvelib/providers.py @@ -0,0 +1,78 @@ +class AbstractProvider(object): + """Delegate class to provide requirment interface for the resolver. + """ + def identify(self, dependency): + """Given a dependency, return an identifier for it. + + This is used in many places to identify the dependency, e.g. whether + two requirements should have their specifier parts merged, whether + two specifications would conflict with each other (because they the + same name but different versions). + """ + raise NotImplementedError + + def get_preference(self, resolution, candidates, information): + """Produce a sort key for given specification based on preference. + + The preference is defined as "I think this requirement should be + resolved first". The lower the return value is, the more preferred + this group of arguments is. + + :param resolution: Currently pinned candidate, or `None`. + :param candidates: A list of possible candidates. + :param information: A list of requirement information. + + Each information instance is a named tuple with two entries: + + * `requirement` specifies a requirement contributing to the current + candidate list + * `parent` specifies the candidate that provids (dependend on) the + requirement, or `None` to indicate a root requirement. + + The preference could depend on a various of issues, including (not + necessarily in this order): + + * Is this package pinned in the current resolution result? + * How relaxed is the requirement? Stricter ones should probably be + worked on first? (I don't know, actually.) + * How many possibilities are there to satisfy this requirement? Those + with few left should likely be worked on first, I guess? + * Are there any known conflicts for this requirement? We should + probably work on those with the most known conflicts. + + A sortable value should be returned (this will be used as the `key` + parameter of the built-in sorting function). The smaller the value is, + the more preferred this specification is (i.e. the sorting function + is called with `reverse=False`). + """ + raise NotImplementedError + + def find_matches(self, requirement): + """Find all possible candidates that satisfy a requirement. + + This should try to get candidates based on the requirement's type. + For VCS, local, and archive requirements, the one-and-only match is + returned, and for a "named" requirement, the index(es) should be + consulted to find concrete candidates for this requirement. + + The returned candidates should be sorted by reversed preference, e.g. + the latest should be LAST. This is done so list-popping can be as + efficient as possible. + """ + raise NotImplementedError + + def is_satisfied_by(self, requirement, candidate): + """Whether the given requirement can be satisfied by a candidate. + + A boolean should be retuened to indicate whether `candidate` is a + viable solution to the requirement. + """ + raise NotImplementedError + + def get_dependencies(self, candidate): + """Get dependencies of a candidate. + + This should return a collection of requirements that `candidate` + specifies as its dependencies. + """ + raise NotImplementedError diff --git a/pipenv/vendor/resolvelib/reporters.py b/pipenv/vendor/resolvelib/reporters.py new file mode 100644 index 00000000..c723031f --- /dev/null +++ b/pipenv/vendor/resolvelib/reporters.py @@ -0,0 +1,23 @@ +class BaseReporter(object): + """Delegate class to provider progress reporting for the resolver. + """ + def starting(self): + """Called before the resolution actually starts. + """ + + def starting_round(self, index): + """Called before each round of resolution starts. + + The index is zero-based. + """ + + def ending_round(self, index, state): + """Called before each round of resolution ends. + + This is NOT called if the resolution ends at this round. Use `ending` + if you want to report finalization. The index is zero-based. + """ + + def ending(self, state): + """Called before the resolution ends successfully. + """ diff --git a/pipenv/vendor/resolvelib/resolvers.py b/pipenv/vendor/resolvelib/resolvers.py new file mode 100644 index 00000000..9c69628e --- /dev/null +++ b/pipenv/vendor/resolvelib/resolvers.py @@ -0,0 +1,287 @@ +import collections + +from .structs import DirectedGraph + + +RequirementInformation = collections.namedtuple('RequirementInformation', [ + 'requirement', 'parent', +]) + + +class NoVersionsAvailable(Exception): + def __init__(self, requirement, parent): + super(NoVersionsAvailable, self).__init__() + self.requirement = requirement + self.parent = parent + + +class RequirementsConflicted(Exception): + def __init__(self, criterion): + super(RequirementsConflicted, self).__init__() + self.criterion = criterion + + +class Criterion(object): + """Internal representation of possible resolution results of a package. + + This holds two attributes: + + * `information` is a collection of `RequirementInformation` pairs. Each + pair is a requirement contributing to this criterion, and the candidate + that provides the requirement. + * `candidates` is a collection containing all possible candidates deducted + from the union of contributing requirements. It should never be empty. + """ + def __init__(self, candidates, information): + self.candidates = candidates + self.information = information + + @classmethod + def from_requirement(cls, provider, requirement, parent): + """Build an instance from a requirement. + """ + candidates = provider.find_matches(requirement) + if not candidates: + raise NoVersionsAvailable(requirement, parent) + return cls( + candidates=candidates, + information=[RequirementInformation(requirement, parent)], + ) + + def iter_requirement(self): + return (i.requirement for i in self.information) + + def iter_parent(self): + return (i.parent for i in self.information) + + def merged_with(self, provider, requirement, parent): + """Build a new instance from this and a new requirement. + """ + infos = list(self.information) + infos.append(RequirementInformation(requirement, parent)) + candidates = [ + c for c in self.candidates + if provider.is_satisfied_by(requirement, c) + ] + if not candidates: + raise RequirementsConflicted(self) + return type(self)(candidates, infos) + + +class ResolutionError(Exception): + pass + + +class ResolutionImpossible(ResolutionError): + def __init__(self, requirements): + super(ResolutionImpossible, self).__init__() + self.requirements = requirements + + +class ResolutionTooDeep(ResolutionError): + def __init__(self, round_count): + super(ResolutionTooDeep, self).__init__(round_count) + self.round_count = round_count + + +# Resolution state in a round. +State = collections.namedtuple('State', 'mapping graph') + + +class Resolution(object): + """Stateful resolution object. + + This is designed as a one-off object that holds information to kick start + the resolution process, and holds the results afterwards. + """ + def __init__(self, provider, reporter): + self._p = provider + self._r = reporter + self._criteria = {} + self._states = [] + + @property + def state(self): + try: + return self._states[-1] + except IndexError: + raise AttributeError('state') + + def _push_new_state(self): + """Push a new state into history. + + This new state will be used to hold resolution results of the next + coming round. + """ + try: + base = self._states[-1] + except IndexError: + graph = DirectedGraph() + graph.add(None) # Sentinel as root dependencies' parent. + state = State(mapping={}, graph=graph) + else: + state = State( + mapping=base.mapping.copy(), + graph=base.graph.copy(), + ) + self._states.append(state) + + def _contribute_to_criteria(self, name, requirement, parent): + try: + crit = self._criteria[name] + except KeyError: + crit = Criterion.from_requirement(self._p, requirement, parent) + else: + crit = crit.merged_with(self._p, requirement, parent) + self._criteria[name] = crit + + def _get_criterion_item_preference(self, item): + name, criterion = item + try: + pinned = self.state.mapping[name] + except (IndexError, KeyError): + pinned = None + return self._p.get_preference( + pinned, criterion.candidates, criterion.information, + ) + + def _is_current_pin_satisfying(self, name, criterion): + try: + current_pin = self.state.mapping[name] + except KeyError: + return False + return all( + self._p.is_satisfied_by(r, current_pin) + for r in criterion.iter_requirement() + ) + + def _check_pinnability(self, candidate, dependencies): + backup = self._criteria.copy() + contributed = set() + try: + for subdep in dependencies: + key = self._p.identify(subdep) + self._contribute_to_criteria(key, subdep, parent=candidate) + contributed.add(key) + except RequirementsConflicted: + self._criteria = backup + return None + return contributed + + def _pin_candidate(self, name, criterion, candidate, child_names): + try: + self.state.graph.remove(name) + except KeyError: + pass + self.state.mapping[name] = candidate + self.state.graph.add(name) + for parent in criterion.iter_parent(): + parent_name = None if parent is None else self._p.identify(parent) + try: + self.state.graph.connect(parent_name, name) + except KeyError: + # Parent is not yet pinned. Skip now; this edge will be + # connected when the parent is being pinned. + pass + for child_name in child_names: + try: + self.state.graph.connect(name, child_name) + except KeyError: + # Child is not yet pinned. Skip now; this edge will be + # connected when the child is being pinned. + pass + + def _pin_criteria(self): + criterion_names = [name for name, _ in sorted( + self._criteria.items(), + key=self._get_criterion_item_preference, + )] + for name in criterion_names: + # Any pin may modify any criterion during the loop. Criteria are + # replaced, not updated in-place, so we need to read this value + # in the loop instead of outside. (sarugaku/resolvelib#5) + criterion = self._criteria[name] + + if self._is_current_pin_satisfying(name, criterion): + # If the current pin already works, just use it. + continue + candidates = list(criterion.candidates) + while candidates: + candidate = candidates.pop() + dependencies = self._p.get_dependencies(candidate) + child_names = self._check_pinnability(candidate, dependencies) + if child_names is None: + continue + self._pin_candidate(name, criterion, candidate, child_names) + break + else: # All candidates tried, nothing works. Give up. (?) + raise ResolutionImpossible(list(criterion.iter_requirement())) + + def resolve(self, requirements, max_rounds): + if self._states: + raise RuntimeError('already resolved') + + for requirement in requirements: + try: + name = self._p.identify(requirement) + self._contribute_to_criteria(name, requirement, parent=None) + except RequirementsConflicted as e: + # If initial requirements conflict, nothing would ever work. + raise ResolutionImpossible(e.requirements + [requirement]) + + last = None + self._r.starting() + + for round_index in range(max_rounds): + self._r.starting_round(round_index) + + self._push_new_state() + self._pin_criteria() + + curr = self.state + if last is not None and len(curr.mapping) == len(last.mapping): + # Nothing new added. Done! Remove the duplicated entry. + del self._states[-1] + self._r.ending(last) + return + last = curr + + self._r.ending_round(round_index, curr) + + raise ResolutionTooDeep(max_rounds) + + +class Resolver(object): + """The thing that performs the actual resolution work. + """ + def __init__(self, provider, reporter): + self.provider = provider + self.reporter = reporter + + def resolve(self, requirements, max_rounds=20): + """Take a collection of constraints, spit out the resolution result. + + The return value is a representation to the final resolution result. It + is a tuple subclass with two public members: + + * `mapping`: A dict of resolved candidates. Each key is an identifier + of a requirement (as returned by the provider's `identify` method), + and the value is the resolved candidate. + * `graph`: A `DirectedGraph` instance representing the dependency tree. + The vertices are keys of `mapping`, and each edge represents *why* + a particular package is included. A special vertex `None` is + included to represent parents of user-supplied requirements. + + The following exceptions may be raised if a resolution cannot be found: + + * `NoVersionsAvailable`: A requirement has no available candidates. + * `ResolutionImpossible`: A resolution cannot be found for the given + combination of requirements. + * `ResolutionTooDeep`: The dependency tree is too deeply nested and + the resolver gave up. This is usually caused by a circular + dependency, but you can try to resolve this by increasing the + `max_rounds` argument. + """ + resolution = Resolution(self.provider, self.reporter) + resolution.resolve(requirements, max_rounds=max_rounds) + return resolution.state diff --git a/pipenv/vendor/resolvelib/structs.py b/pipenv/vendor/resolvelib/structs.py new file mode 100644 index 00000000..97bd0095 --- /dev/null +++ b/pipenv/vendor/resolvelib/structs.py @@ -0,0 +1,67 @@ +class DirectedGraph(object): + """A graph structure with directed edges. + """ + def __init__(self): + self._vertices = set() + self._forwards = {} # -> Set[] + self._backwards = {} # -> Set[] + + def __iter__(self): + return iter(self._vertices) + + def __len__(self): + return len(self._vertices) + + def __contains__(self, key): + return key in self._vertices + + def copy(self): + """Return a shallow copy of this graph. + """ + other = DirectedGraph() + other._vertices = set(self._vertices) + other._forwards = {k: set(v) for k, v in self._forwards.items()} + other._backwards = {k: set(v) for k, v in self._backwards.items()} + return other + + def add(self, key): + """Add a new vertex to the graph. + """ + if key in self._vertices: + raise ValueError('vertex exists') + self._vertices.add(key) + self._forwards[key] = set() + self._backwards[key] = set() + + def remove(self, key): + """Remove a vertex from the graph, disconnecting all edges from/to it. + """ + self._vertices.remove(key) + for f in self._forwards.pop(key): + self._backwards[f].remove(key) + for t in self._backwards.pop(key): + self._forwards[t].remove(key) + + def connected(self, f, t): + return f in self._backwards[t] and t in self._forwards[f] + + def connect(self, f, t): + """Connect two existing vertices. + + Nothing happens if the vertices are already connected. + """ + if t not in self._vertices: + raise KeyError(t) + self._forwards[f].add(t) + self._backwards[t].add(f) + + def iter_edges(self): + for f, children in self._forwards.items(): + for t in children: + yield f, t + + def iter_children(self, key): + return iter(self._forwards[key]) + + def iter_parents(self, key): + return iter(self._backwards[key]) diff --git a/pipenv/vendor/shellingham/__init__.py b/pipenv/vendor/shellingham/__init__.py index 90c00abb..576c4224 100644 --- a/pipenv/vendor/shellingham/__init__.py +++ b/pipenv/vendor/shellingham/__init__.py @@ -4,7 +4,7 @@ import os from ._core import ShellDetectionFailure -__version__ = '1.2.6' +__version__ = '1.2.7' def detect_shell(pid=None, max_depth=6): diff --git a/pipenv/vendor/shellingham/posix/ps.py b/pipenv/vendor/shellingham/posix/ps.py index ab4c2a9e..4a155ed5 100644 --- a/pipenv/vendor/shellingham/posix/ps.py +++ b/pipenv/vendor/shellingham/posix/ps.py @@ -21,6 +21,12 @@ def get_process_mapping(): if e.errno != errno.ENOENT: raise raise PsNotAvailable('ps not found') + except subprocess.CalledProcessError as e: + # `ps` can return 1 if the process list is completely empty. + # (sarugaku/shellingham#15) + if not e.output.strip(): + return {} + raise if not isinstance(output, str): encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() output = output.decode(encoding) @@ -28,9 +34,9 @@ def get_process_mapping(): for line in output.split('\n'): try: pid, ppid, args = line.strip().split(None, 2) + processes[pid] = Process( + args=tuple(shlex.split(args)), pid=pid, ppid=ppid, + ) except ValueError: continue - processes[pid] = Process( - args=tuple(shlex.split(args)), pid=pid, ppid=ppid, - ) return processes diff --git a/pipenv/vendor/toml.LICENSE b/pipenv/vendor/toml.LICENSE deleted file mode 100644 index d8b406c9..00000000 --- a/pipenv/vendor/toml.LICENSE +++ /dev/null @@ -1,26 +0,0 @@ -The MIT License - -Copyright 2013-2017 Uiri Noyb -Copyright 2015-2016 Julien Enselme -Copyright 2016 Google Inc. -Copyright 2017 Samuel Vasko -Copyright 2017 Nate Prewitt -Copyright 2017 Jack Evans - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file diff --git a/pipenv/vendor/toml.py b/pipenv/vendor/toml.py deleted file mode 100644 index dac39883..00000000 --- a/pipenv/vendor/toml.py +++ /dev/null @@ -1,1039 +0,0 @@ -"""Python module which parses and emits TOML. - -Released under the MIT license. -""" -import re -import io -import datetime -from os import linesep -import sys - -__version__ = "0.9.6" -_spec_ = "0.4.0" - - -class TomlDecodeError(Exception): - """Base toml Exception / Error.""" - pass - - -class TomlTz(datetime.tzinfo): - def __init__(self, toml_offset): - if toml_offset == "Z": - self._raw_offset = "+00:00" - else: - self._raw_offset = toml_offset - self._sign = -1 if self._raw_offset[0] == '-' else 1 - self._hours = int(self._raw_offset[1:3]) - self._minutes = int(self._raw_offset[4:6]) - - def tzname(self, dt): - return "UTC" + self._raw_offset - - def utcoffset(self, dt): - return self._sign * datetime.timedelta(hours=self._hours, - minutes=self._minutes) - - def dst(self, dt): - return datetime.timedelta(0) - - -class InlineTableDict(object): - """Sentinel subclass of dict for inline tables.""" - - -def _get_empty_inline_table(_dict): - class DynamicInlineTableDict(_dict, InlineTableDict): - """Concrete sentinel subclass for inline tables. - It is a subclass of _dict which is passed in dynamically at load time - It is also a subclass of InlineTableDict - """ - - return DynamicInlineTableDict() - - -try: - _range = xrange -except NameError: - unicode = str - _range = range - basestring = str - unichr = chr - -try: - FNFError = FileNotFoundError -except NameError: - FNFError = IOError - - -def load(f, _dict=dict): - """Parses named file or files as toml and returns a dictionary - - Args: - f: Path to the file to open, array of files to read into single dict - or a file descriptor - _dict: (optional) Specifies the class of the returned toml dictionary - - Returns: - Parsed toml file represented as a dictionary - - Raises: - TypeError -- When f is invalid type - TomlDecodeError: Error while decoding toml - IOError / FileNotFoundError -- When an array with no valid (existing) - (Python 2 / Python 3) file paths is passed - """ - - if isinstance(f, basestring): - with io.open(f, encoding='utf-8') as ffile: - return loads(ffile.read(), _dict) - elif isinstance(f, list): - from os import path as op - from warnings import warn - if not [path for path in f if op.exists(path)]: - error_msg = "Load expects a list to contain filenames only." - error_msg += linesep - error_msg += ("The list needs to contain the path of at least one " - "existing file.") - raise FNFError(error_msg) - d = _dict() - for l in f: - if op.exists(l): - d.update(load(l)) - else: - warn("Non-existent filename in list with at least one valid " - "filename") - return d - else: - try: - return loads(f.read(), _dict) - except AttributeError: - raise TypeError("You can only load a file descriptor, filename or " - "list") - - -_groupname_re = re.compile(r'^[A-Za-z0-9_-]+$') - - -def loads(s, _dict=dict): - """Parses string as toml - - Args: - s: String to be parsed - _dict: (optional) Specifies the class of the returned toml dictionary - - Returns: - Parsed toml file represented as a dictionary - - Raises: - TypeError: When a non-string is passed - TomlDecodeError: Error while decoding toml - """ - - implicitgroups = [] - retval = _dict() - currentlevel = retval - if not isinstance(s, basestring): - raise TypeError("Expecting something like a string") - - if not isinstance(s, unicode): - s = s.decode('utf8') - - sl = list(s) - openarr = 0 - openstring = False - openstrchar = "" - multilinestr = False - arrayoftables = False - beginline = True - keygroup = False - keyname = 0 - for i, item in enumerate(sl): - if item == '\r' and sl[i + 1] == '\n': - sl[i] = ' ' - continue - if keyname: - if item == '\n': - raise TomlDecodeError("Key name found without value." - " Reached end of line.") - if openstring: - if item == openstrchar: - keyname = 2 - openstring = False - openstrchar = "" - continue - elif keyname == 1: - if item.isspace(): - keyname = 2 - continue - elif item.isalnum() or item == '_' or item == '-': - continue - elif keyname == 2 and item.isspace(): - continue - if item == '=': - keyname = 0 - else: - raise TomlDecodeError("Found invalid character in key name: '" + - item + "'. Try quoting the key name.") - if item == "'" and openstrchar != '"': - k = 1 - try: - while sl[i - k] == "'": - k += 1 - if k == 3: - break - except IndexError: - pass - if k == 3: - multilinestr = not multilinestr - openstring = multilinestr - else: - openstring = not openstring - if openstring: - openstrchar = "'" - else: - openstrchar = "" - if item == '"' and openstrchar != "'": - oddbackslash = False - k = 1 - tripquote = False - try: - while sl[i - k] == '"': - k += 1 - if k == 3: - tripquote = True - break - if k == 1 or (k == 3 and tripquote): - while sl[i - k] == '\\': - oddbackslash = not oddbackslash - k += 1 - except IndexError: - pass - if not oddbackslash: - if tripquote: - multilinestr = not multilinestr - openstring = multilinestr - else: - openstring = not openstring - if openstring: - openstrchar = '"' - else: - openstrchar = "" - if item == '#' and (not openstring and not keygroup and - not arrayoftables): - j = i - try: - while sl[j] != '\n': - sl[j] = ' ' - j += 1 - except IndexError: - break - if item == '[' and (not openstring and not keygroup and - not arrayoftables): - if beginline: - if len(sl) > i + 1 and sl[i + 1] == '[': - arrayoftables = True - else: - keygroup = True - else: - openarr += 1 - if item == ']' and not openstring: - if keygroup: - keygroup = False - elif arrayoftables: - if sl[i - 1] == ']': - arrayoftables = False - else: - openarr -= 1 - if item == '\n': - if openstring or multilinestr: - if not multilinestr: - raise TomlDecodeError("Unbalanced quotes") - if ((sl[i - 1] == "'" or sl[i - 1] == '"') and ( - sl[i - 2] == sl[i - 1])): - sl[i] = sl[i - 1] - if sl[i - 3] == sl[i - 1]: - sl[i - 3] = ' ' - elif openarr: - sl[i] = ' ' - else: - beginline = True - elif beginline and sl[i] != ' ' and sl[i] != '\t': - beginline = False - if not keygroup and not arrayoftables: - if sl[i] == '=': - raise TomlDecodeError("Found empty keyname. ") - keyname = 1 - s = ''.join(sl) - s = s.split('\n') - multikey = None - multilinestr = "" - multibackslash = False - for line in s: - if not multilinestr or multibackslash or '\n' not in multilinestr: - line = line.strip() - if line == "" and (not multikey or multibackslash): - continue - if multikey: - if multibackslash: - multilinestr += line - else: - multilinestr += line - multibackslash = False - if len(line) > 2 and (line[-1] == multilinestr[0] and - line[-2] == multilinestr[0] and - line[-3] == multilinestr[0]): - try: - value, vtype = _load_value(multilinestr, _dict) - except ValueError as err: - raise TomlDecodeError(str(err)) - currentlevel[multikey] = value - multikey = None - multilinestr = "" - else: - k = len(multilinestr) - 1 - while k > -1 and multilinestr[k] == '\\': - multibackslash = not multibackslash - k -= 1 - if multibackslash: - multilinestr = multilinestr[:-1] - else: - multilinestr += "\n" - continue - if line[0] == '[': - arrayoftables = False - if len(line) == 1: - raise TomlDecodeError("Opening key group bracket on line by " - "itself.") - if line[1] == '[': - arrayoftables = True - line = line[2:] - splitstr = ']]' - else: - line = line[1:] - splitstr = ']' - i = 1 - quotesplits = _get_split_on_quotes(line) - quoted = False - for quotesplit in quotesplits: - if not quoted and splitstr in quotesplit: - break - i += quotesplit.count(splitstr) - quoted = not quoted - line = line.split(splitstr, i) - if len(line) < i + 1 or line[-1].strip() != "": - raise TomlDecodeError("Key group not on a line by itself.") - groups = splitstr.join(line[:-1]).split('.') - i = 0 - while i < len(groups): - groups[i] = groups[i].strip() - if len(groups[i]) > 0 and (groups[i][0] == '"' or - groups[i][0] == "'"): - groupstr = groups[i] - j = i + 1 - while not groupstr[0] == groupstr[-1]: - j += 1 - if j > len(groups) + 2: - raise TomlDecodeError("Invalid group name '" + - groupstr + "' Something " + - "went wrong.") - groupstr = '.'.join(groups[i:j]).strip() - groups[i] = groupstr[1:-1] - groups[i + 1:j] = [] - else: - if not _groupname_re.match(groups[i]): - raise TomlDecodeError("Invalid group name '" + - groups[i] + "'. Try quoting it.") - i += 1 - currentlevel = retval - for i in _range(len(groups)): - group = groups[i] - if group == "": - raise TomlDecodeError("Can't have a keygroup with an empty " - "name") - try: - currentlevel[group] - if i == len(groups) - 1: - if group in implicitgroups: - implicitgroups.remove(group) - if arrayoftables: - raise TomlDecodeError("An implicitly defined " - "table can't be an array") - elif arrayoftables: - currentlevel[group].append(_dict()) - else: - raise TomlDecodeError("What? " + group + - " already exists?" + - str(currentlevel)) - except TypeError: - currentlevel = currentlevel[-1] - try: - currentlevel[group] - except KeyError: - currentlevel[group] = _dict() - if i == len(groups) - 1 and arrayoftables: - currentlevel[group] = [_dict()] - except KeyError: - if i != len(groups) - 1: - implicitgroups.append(group) - currentlevel[group] = _dict() - if i == len(groups) - 1 and arrayoftables: - currentlevel[group] = [_dict()] - currentlevel = currentlevel[group] - if arrayoftables: - try: - currentlevel = currentlevel[-1] - except KeyError: - pass - elif line[0] == "{": - if line[-1] != "}": - raise TomlDecodeError("Line breaks are not allowed in inline" - "objects") - try: - _load_inline_object(line, currentlevel, _dict, multikey, - multibackslash) - except ValueError as err: - raise TomlDecodeError(str(err)) - elif "=" in line: - try: - ret = _load_line(line, currentlevel, _dict, multikey, - multibackslash) - except ValueError as err: - raise TomlDecodeError(str(err)) - if ret is not None: - multikey, multilinestr, multibackslash = ret - return retval - - -def _load_inline_object(line, currentlevel, _dict, multikey=False, - multibackslash=False): - candidate_groups = line[1:-1].split(",") - groups = [] - if len(candidate_groups) == 1 and not candidate_groups[0].strip(): - candidate_groups.pop() - while len(candidate_groups) > 0: - candidate_group = candidate_groups.pop(0) - try: - _, value = candidate_group.split('=', 1) - except ValueError: - raise ValueError("Invalid inline table encountered") - value = value.strip() - if ((value[0] == value[-1] and value[0] in ('"', "'")) or ( - value[0] in '-0123456789' or - value in ('true', 'false') or - (value[0] == "[" and value[-1] == "]") or - (value[0] == '{' and value[-1] == '}'))): - groups.append(candidate_group) - elif len(candidate_groups) > 0: - candidate_groups[0] = candidate_group + "," + candidate_groups[0] - else: - raise ValueError("Invalid inline table value encountered") - for group in groups: - status = _load_line(group, currentlevel, _dict, multikey, - multibackslash) - if status is not None: - break - - -# Matches a TOML number, which allows underscores for readability -_number_with_underscores = re.compile('([0-9])(_([0-9]))*') - - -def _strictly_valid_num(n): - n = n.strip() - if not n: - return False - if n[0] == '_': - return False - if n[-1] == '_': - return False - if "_." in n or "._" in n: - return False - if len(n) == 1: - return True - if n[0] == '0' and n[1] != '.': - return False - if n[0] == '+' or n[0] == '-': - n = n[1:] - if n[0] == '0' and n[1] != '.': - return False - if '__' in n: - return False - return True - - -def _get_split_on_quotes(line): - doublequotesplits = line.split('"') - quoted = False - quotesplits = [] - if len(doublequotesplits) > 1 and "'" in doublequotesplits[0]: - singlequotesplits = doublequotesplits[0].split("'") - doublequotesplits = doublequotesplits[1:] - while len(singlequotesplits) % 2 == 0 and len(doublequotesplits): - singlequotesplits[-1] += '"' + doublequotesplits[0] - doublequotesplits = doublequotesplits[1:] - if "'" in singlequotesplits[-1]: - singlequotesplits = (singlequotesplits[:-1] + - singlequotesplits[-1].split("'")) - quotesplits += singlequotesplits - for doublequotesplit in doublequotesplits: - if quoted: - quotesplits.append(doublequotesplit) - else: - quotesplits += doublequotesplit.split("'") - quoted = not quoted - return quotesplits - - -def _load_line(line, currentlevel, _dict, multikey, multibackslash): - i = 1 - quotesplits = _get_split_on_quotes(line) - quoted = False - for quotesplit in quotesplits: - if not quoted and '=' in quotesplit: - break - i += quotesplit.count('=') - quoted = not quoted - pair = line.split('=', i) - strictly_valid = _strictly_valid_num(pair[-1]) - if _number_with_underscores.match(pair[-1]): - pair[-1] = pair[-1].replace('_', '') - while len(pair[-1]) and (pair[-1][0] != ' ' and pair[-1][0] != '\t' and - pair[-1][0] != "'" and pair[-1][0] != '"' and - pair[-1][0] != '[' and pair[-1][0] != '{' and - pair[-1] != 'true' and pair[-1] != 'false'): - try: - float(pair[-1]) - break - except ValueError: - pass - if _load_date(pair[-1]) is not None: - break - i += 1 - prev_val = pair[-1] - pair = line.split('=', i) - if prev_val == pair[-1]: - raise ValueError("Invalid date or number") - if strictly_valid: - strictly_valid = _strictly_valid_num(pair[-1]) - pair = ['='.join(pair[:-1]).strip(), pair[-1].strip()] - if (pair[0][0] == '"' or pair[0][0] == "'") and \ - (pair[0][-1] == '"' or pair[0][-1] == "'"): - pair[0] = pair[0][1:-1] - if len(pair[1]) > 2 and ((pair[1][0] == '"' or pair[1][0] == "'") and - pair[1][1] == pair[1][0] and - pair[1][2] == pair[1][0] and - not (len(pair[1]) > 5 and - pair[1][-1] == pair[1][0] and - pair[1][-2] == pair[1][0] and - pair[1][-3] == pair[1][0])): - k = len(pair[1]) - 1 - while k > -1 and pair[1][k] == '\\': - multibackslash = not multibackslash - k -= 1 - if multibackslash: - multilinestr = pair[1][:-1] - else: - multilinestr = pair[1] + "\n" - multikey = pair[0] - else: - value, vtype = _load_value(pair[1], _dict, strictly_valid) - try: - currentlevel[pair[0]] - raise ValueError("Duplicate keys!") - except KeyError: - if multikey: - return multikey, multilinestr, multibackslash - else: - currentlevel[pair[0]] = value - - -def _load_date(val): - microsecond = 0 - tz = None - try: - if len(val) > 19: - if val[19] == '.': - if val[-1].upper() == 'Z': - subsecondval = val[20:-1] - tzval = "Z" - else: - subsecondvalandtz = val[20:] - if '+' in subsecondvalandtz: - splitpoint = subsecondvalandtz.index('+') - subsecondval = subsecondvalandtz[:splitpoint] - tzval = subsecondvalandtz[splitpoint:] - elif '-' in subsecondvalandtz: - splitpoint = subsecondvalandtz.index('-') - subsecondval = subsecondvalandtz[:splitpoint] - tzval = subsecondvalandtz[splitpoint:] - tz = TomlTz(tzval) - microsecond = int(int(subsecondval) * - (10 ** (6 - len(subsecondval)))) - else: - tz = TomlTz(val[19:]) - except ValueError: - tz = None - if "-" not in val[1:]: - return None - try: - d = datetime.datetime( - int(val[:4]), int(val[5:7]), - int(val[8:10]), int(val[11:13]), - int(val[14:16]), int(val[17:19]), microsecond, tz) - except ValueError: - return None - return d - - -def _load_unicode_escapes(v, hexbytes, prefix): - skip = False - i = len(v) - 1 - while i > -1 and v[i] == '\\': - skip = not skip - i -= 1 - for hx in hexbytes: - if skip: - skip = False - i = len(hx) - 1 - while i > -1 and hx[i] == '\\': - skip = not skip - i -= 1 - v += prefix - v += hx - continue - hxb = "" - i = 0 - hxblen = 4 - if prefix == "\\U": - hxblen = 8 - hxb = ''.join(hx[i:i + hxblen]).lower() - if hxb.strip('0123456789abcdef'): - raise ValueError("Invalid escape sequence: " + hxb) - if hxb[0] == "d" and hxb[1].strip('01234567'): - raise ValueError("Invalid escape sequence: " + hxb + - ". Only scalar unicode points are allowed.") - v += unichr(int(hxb, 16)) - v += unicode(hx[len(hxb):]) - return v - - -# Unescape TOML string values. - -# content after the \ -_escapes = ['0', 'b', 'f', 'n', 'r', 't', '"'] -# What it should be replaced by -_escapedchars = ['\0', '\b', '\f', '\n', '\r', '\t', '\"'] -# Used for substitution -_escape_to_escapedchars = dict(zip(_escapes, _escapedchars)) - - -def _unescape(v): - """Unescape characters in a TOML string.""" - i = 0 - backslash = False - while i < len(v): - if backslash: - backslash = False - if v[i] in _escapes: - v = v[:i - 1] + _escape_to_escapedchars[v[i]] + v[i + 1:] - elif v[i] == '\\': - v = v[:i - 1] + v[i:] - elif v[i] == 'u' or v[i] == 'U': - i += 1 - else: - raise ValueError("Reserved escape sequence used") - continue - elif v[i] == '\\': - backslash = True - i += 1 - return v - - -def _load_value(v, _dict, strictly_valid=True): - if not v: - raise ValueError("Empty value is invalid") - if v == 'true': - return (True, "bool") - elif v == 'false': - return (False, "bool") - elif v[0] == '"': - testv = v[1:].split('"') - triplequote = False - triplequotecount = 0 - if len(testv) > 1 and testv[0] == '' and testv[1] == '': - testv = testv[2:] - triplequote = True - closed = False - for tv in testv: - if tv == '': - if triplequote: - triplequotecount += 1 - else: - closed = True - else: - oddbackslash = False - try: - i = -1 - j = tv[i] - while j == '\\': - oddbackslash = not oddbackslash - i -= 1 - j = tv[i] - except IndexError: - pass - if not oddbackslash: - if closed: - raise ValueError("Stuff after closed string. WTF?") - else: - if not triplequote or triplequotecount > 1: - closed = True - else: - triplequotecount = 0 - escapeseqs = v.split('\\')[1:] - backslash = False - for i in escapeseqs: - if i == '': - backslash = not backslash - else: - if i[0] not in _escapes and (i[0] != 'u' and i[0] != 'U' and - not backslash): - raise ValueError("Reserved escape sequence used") - if backslash: - backslash = False - for prefix in ["\\u", "\\U"]: - if prefix in v: - hexbytes = v.split(prefix) - v = _load_unicode_escapes(hexbytes[0], hexbytes[1:], prefix) - v = _unescape(v) - if len(v) > 1 and v[1] == '"' and (len(v) < 3 or v[1] == v[2]): - v = v[2:-2] - return (v[1:-1], "str") - elif v[0] == "'": - if v[1] == "'" and (len(v) < 3 or v[1] == v[2]): - v = v[2:-2] - return (v[1:-1], "str") - elif v[0] == '[': - return (_load_array(v, _dict), "array") - elif v[0] == '{': - inline_object = _get_empty_inline_table(_dict) - _load_inline_object(v, inline_object, _dict) - return (inline_object, "inline_object") - else: - parsed_date = _load_date(v) - if parsed_date is not None: - return (parsed_date, "date") - if not strictly_valid: - raise ValueError("Weirdness with leading zeroes or " - "underscores in your number.") - itype = "int" - neg = False - if v[0] == '-': - neg = True - v = v[1:] - elif v[0] == '+': - v = v[1:] - v = v.replace('_', '') - if '.' in v or 'e' in v or 'E' in v: - if '.' in v and v.split('.', 1)[1] == '': - raise ValueError("This float is missing digits after " - "the point") - if v[0] not in '0123456789': - raise ValueError("This float doesn't have a leading digit") - v = float(v) - itype = "float" - else: - v = int(v) - if neg: - return (0 - v, itype) - return (v, itype) - - -def _bounded_string(s): - if len(s) == 0: - return True - if s[-1] != s[0]: - return False - i = -2 - backslash = False - while len(s) + i > 0: - if s[i] == "\\": - backslash = not backslash - i -= 1 - else: - break - return not backslash - - -def _load_array(a, _dict): - atype = None - retval = [] - a = a.strip() - if '[' not in a[1:-1] or "" != a[1:-1].split('[')[0].strip(): - strarray = False - tmpa = a[1:-1].strip() - if tmpa != '' and (tmpa[0] == '"' or tmpa[0] == "'"): - strarray = True - if not a[1:-1].strip().startswith('{'): - a = a[1:-1].split(',') - else: - # a is an inline object, we must find the matching parenthesis - # to define groups - new_a = [] - start_group_index = 1 - end_group_index = 2 - in_str = False - while end_group_index < len(a[1:]): - if a[end_group_index] == '"' or a[end_group_index] == "'": - if in_str: - backslash_index = end_group_index - 1 - while (backslash_index > -1 and - a[backslash_index] == '\\'): - in_str = not in_str - backslash_index -= 1 - in_str = not in_str - if in_str or a[end_group_index] != '}': - end_group_index += 1 - continue - - # Increase end_group_index by 1 to get the closing bracket - end_group_index += 1 - new_a.append(a[start_group_index:end_group_index]) - - # The next start index is at least after the closing bracket, a - # closing bracket can be followed by a comma since we are in - # an array. - start_group_index = end_group_index + 1 - while (start_group_index < len(a[1:]) and - a[start_group_index] != '{'): - start_group_index += 1 - end_group_index = start_group_index + 1 - a = new_a - b = 0 - if strarray: - while b < len(a) - 1: - ab = a[b].strip() - while (not _bounded_string(ab) or - (len(ab) > 2 and - ab[0] == ab[1] == ab[2] and - ab[-2] != ab[0] and - ab[-3] != ab[0])): - a[b] = a[b] + ',' + a[b + 1] - ab = a[b].strip() - if b < len(a) - 2: - a = a[:b + 1] + a[b + 2:] - else: - a = a[:b + 1] - b += 1 - else: - al = list(a[1:-1]) - a = [] - openarr = 0 - j = 0 - for i in _range(len(al)): - if al[i] == '[': - openarr += 1 - elif al[i] == ']': - openarr -= 1 - elif al[i] == ',' and not openarr: - a.append(''.join(al[j:i])) - j = i + 1 - a.append(''.join(al[j:])) - for i in _range(len(a)): - a[i] = a[i].strip() - if a[i] != '': - nval, ntype = _load_value(a[i], _dict) - if atype: - if ntype != atype: - raise ValueError("Not a homogeneous array") - else: - atype = ntype - retval.append(nval) - return retval - - -def dump(o, f): - """Writes out dict as toml to a file - - Args: - o: Object to dump into toml - f: File descriptor where the toml should be stored - - Returns: - String containing the toml corresponding to dictionary - - Raises: - TypeError: When anything other than file descriptor is passed - """ - - if not f.write: - raise TypeError("You can only dump an object to a file descriptor") - d = dumps(o) - f.write(d) - return d - - -def dumps(o, preserve=False): - """Stringifies input dict as toml - - Args: - o: Object to dump into toml - - preserve: Boolean parameter. If true, preserve inline tables. - - Returns: - String containing the toml corresponding to dict - """ - - retval = "" - addtoretval, sections = _dump_sections(o, "") - retval += addtoretval - while sections != {}: - newsections = {} - for section in sections: - addtoretval, addtosections = _dump_sections(sections[section], - section, preserve) - if addtoretval or (not addtoretval and not addtosections): - if retval and retval[-2:] != "\n\n": - retval += "\n" - retval += "[" + section + "]\n" - if addtoretval: - retval += addtoretval - for s in addtosections: - newsections[section + "." + s] = addtosections[s] - sections = newsections - return retval - - -def _dump_sections(o, sup, preserve=False): - retstr = "" - if sup != "" and sup[-1] != ".": - sup += '.' - retdict = o.__class__() - arraystr = "" - for section in o: - section = unicode(section) - qsection = section - if not re.match(r'^[A-Za-z0-9_-]+$', section): - if '"' in section: - qsection = "'" + section + "'" - else: - qsection = '"' + section + '"' - if not isinstance(o[section], dict): - arrayoftables = False - if isinstance(o[section], list): - for a in o[section]: - if isinstance(a, dict): - arrayoftables = True - if arrayoftables: - for a in o[section]: - arraytabstr = "\n" - arraystr += "[[" + sup + qsection + "]]\n" - s, d = _dump_sections(a, sup + qsection) - if s: - if s[0] == "[": - arraytabstr += s - else: - arraystr += s - while d != {}: - newd = {} - for dsec in d: - s1, d1 = _dump_sections(d[dsec], sup + qsection + - "." + dsec) - if s1: - arraytabstr += ("[" + sup + qsection + "." + - dsec + "]\n") - arraytabstr += s1 - for s1 in d1: - newd[dsec + "." + s1] = d1[s1] - d = newd - arraystr += arraytabstr - else: - if o[section] is not None: - retstr += (qsection + " = " + - unicode(_dump_value(o[section])) + '\n') - elif preserve and isinstance(o[section], InlineTableDict): - retstr += (qsection + " = " + _dump_inline_table(o[section])) - else: - retdict[qsection] = o[section] - retstr += arraystr - return (retstr, retdict) - - -def _dump_inline_table(section): - """Preserve inline table in its compact syntax instead of expanding - into subsection. - - https://github.com/toml-lang/toml#user-content-inline-table - """ - retval = "" - if isinstance(section, dict): - val_list = [] - for k, v in section.items(): - val = _dump_inline_table(v) - val_list.append(k + " = " + val) - retval += "{ " + ", ".join(val_list) + " }\n" - return retval - else: - return unicode(_dump_value(section)) - - -def _dump_value(v): - dump_funcs = { - str: _dump_str, - unicode: _dump_str, - list: _dump_list, - int: lambda v: v, - bool: lambda v: unicode(v).lower(), - float: _dump_float, - datetime.datetime: lambda v: v.isoformat().replace('+00:00', 'Z'), - } - # Lookup function corresponding to v's type - dump_fn = dump_funcs.get(type(v)) - if dump_fn is None and hasattr(v, '__iter__'): - dump_fn = dump_funcs[list] - # Evaluate function (if it exists) else return v - return dump_fn(v) if dump_fn is not None else dump_funcs[str](v) - - -def _dump_str(v): - if sys.version_info < (3,) and hasattr(v, 'decode') and isinstance(v, str): - v = v.decode('utf-8') - v = "%r" % v - if v[0] == 'u': - v = v[1:] - singlequote = v.startswith("'") - if singlequote or v.startswith('"'): - v = v[1:-1] - if singlequote: - v = v.replace("\\'", "'") - v = v.replace('"', '\\"') - v = v.split("\\x") - while len(v) > 1: - i = -1 - if not v[0]: - v = v[1:] - v[0] = v[0].replace("\\\\", "\\") - # No, I don't know why != works and == breaks - joinx = v[0][i] != "\\" - while v[0][:i] and v[0][i] == "\\": - joinx = not joinx - i -= 1 - if joinx: - joiner = "x" - else: - joiner = "u00" - v = [v[0] + joiner + v[1]] + v[2:] - return unicode('"' + v[0] + '"') - - -def _dump_list(v): - retval = "[" - for u in v: - retval += " " + unicode(_dump_value(u)) + "," - retval += "]" - return retval - - -def _dump_float(v): - return "{0:.16}".format(v).replace("e+0", "e+").replace("e-0", "e-") diff --git a/pipenv/vendor/tomlkit/__init__.py b/pipenv/vendor/tomlkit/__init__.py index 23d4ef74..89e4cf59 100644 --- a/pipenv/vendor/tomlkit/__init__.py +++ b/pipenv/vendor/tomlkit/__init__.py @@ -22,4 +22,4 @@ from .api import value from .api import ws -__version__ = "0.4.4" +__version__ = "0.4.6" diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index bb3696d9..37014921 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -4,7 +4,6 @@ from ._compat import decode from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey from .items import AoT -from .items import Bool from .items import Comment from .items import Item from .items import Key @@ -525,3 +524,21 @@ class Container(dict): return NotImplemented return self.value == other + + def _getstate(self, protocol): + return (self._parsed,) + + def __reduce__(self): + return self.__reduce_ex__(2) + + def __reduce_ex__(self, protocol): + return ( + self.__class__, + self._getstate(protocol), + (self._map, self._body, self._parsed), + ) + + def __setstate__(self, state): + self._map = state[0] + self._body = state[1] + self._parsed = state[2] diff --git a/pipenv/vendor/tomlkit/exceptions.py b/pipenv/vendor/tomlkit/exceptions.py index d889a924..46ee938b 100644 --- a/pipenv/vendor/tomlkit/exceptions.py +++ b/pipenv/vendor/tomlkit/exceptions.py @@ -85,7 +85,7 @@ class InvalidCharInStringError(ParseError): """ def __init__(self, line, col, char): # type: (int, int, str) -> None - message = "Invalid character '{}' in string".format(char) + message = "Invalid character {} in string".format(repr(char)) super(InvalidCharInStringError, self).__init__(line, col, message=message) diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 26f24701..781e2e98 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -17,6 +17,11 @@ from ._compat import decode from ._compat import unicode from ._utils import escape_string +if PY2: + from pipenv.vendor.backports.functools_lru_cache import lru_cache +else: + from functools import lru_cache + def item(value, _parent=None): from .container import Container @@ -75,18 +80,45 @@ def item(value, _parent=None): class StringType(Enum): - + # Single Line Basic SLB = '"' + # Multi Line Basic MLB = '"""' + # Single Line Literal SLL = "'" + # Multi Line Literal MLL = "'''" + @property + @lru_cache(maxsize=None) + def unit(self): # type: () -> str + return self.value[0] + + @lru_cache(maxsize=None) + def is_basic(self): # type: () -> bool + return self in {StringType.SLB, StringType.MLB} + + @lru_cache(maxsize=None) def is_literal(self): # type: () -> bool return self in {StringType.SLL, StringType.MLL} + @lru_cache(maxsize=None) + def is_singleline(self): # type: () -> bool + return self in {StringType.SLB, StringType.SLL} + + @lru_cache(maxsize=None) def is_multiline(self): # type: () -> bool return self in {StringType.MLB, StringType.MLL} + @lru_cache(maxsize=None) + def toggle(self): # type: () -> StringType + return { + StringType.SLB: StringType.MLB, + StringType.MLB: StringType.SLB, + StringType.SLL: StringType.MLL, + StringType.MLL: StringType.SLL, + }[self] + class Trivia: """ @@ -158,7 +190,10 @@ class Key: return hash(self.key) def __eq__(self, other): # type: (Key) -> bool - return self.key == other.key + if isinstance(other, Key): + return self.key == other.key + + return self.key == other def __str__(self): # type: () -> str return self.as_string() @@ -205,6 +240,15 @@ class Item(object): return self + def _getstate(self, protocol=3): + return (self._trivia,) + + def __reduce__(self): + return self.__reduce_ex__(2) + + def __reduce_ex__(self, protocol): + return self.__class__, self._getstate(protocol) + class Whitespace(Item): """ @@ -240,6 +284,9 @@ class Whitespace(Item): def __repr__(self): # type: () -> str return "<{} {}>".format(self.__class__.__name__, repr(self._s)) + def _getstate(self, protocol=3): + return self._s, self._fixed + class Comment(Item): """ @@ -273,7 +320,7 @@ class Integer(int, Item): self._raw = raw self._sign = False - if re.match("^[+\-]\d+$", raw): + if re.match(r"^[+\-]\d+$", raw): self._sign = True @property @@ -322,6 +369,9 @@ class Integer(int, Item): return Integer(result, self._trivia, raw) + def _getstate(self, protocol=3): + return int(self), self._trivia, self._raw + class Float(float, Item): """ @@ -337,7 +387,7 @@ class Float(float, Item): self._raw = raw self._sign = False - if re.match("^[+\-].+$", raw): + if re.match(r"^[+\-].+$", raw): self._sign = True @property @@ -386,6 +436,9 @@ class Float(float, Item): return Float(result, self._trivia, raw) + def _getstate(self, protocol=3): + return float(self), self._trivia, self._raw + class Bool(Item): """ @@ -408,13 +461,16 @@ class Bool(Item): def as_string(self): # type: () -> str return str(self._value).lower() + def _getstate(self, protocol=3): + return self._value, self._trivia -class DateTime(datetime, Item): + +class DateTime(Item, datetime): """ A datetime literal. """ - def __new__(cls, value, *_): # type: (datetime, ...) -> datetime + def __new__(cls, value, *_): # type: (..., datetime, ...) -> datetime return datetime.__new__( cls, value.year, @@ -458,13 +514,29 @@ class DateTime(datetime, Item): return DateTime(result, self._trivia, raw) + def _getstate(self, protocol=3): + return ( + datetime( + self.year, + self.month, + self.day, + self.hour, + self.minute, + self.second, + self.microsecond, + self.tzinfo, + ), + self._trivia, + self._raw, + ) -class Date(date, Item): + +class Date(Item, date): """ A date literal. """ - def __new__(cls, value, *_): # type: (date, ...) -> date + def __new__(cls, value, *_): # type: (..., date, ...) -> date return date.__new__(cls, value.year, value.month, value.day) def __init__(self, _, trivia, raw): # type: (date, Trivia, str) -> None @@ -498,8 +570,11 @@ class Date(date, Item): return Date(result, self._trivia, raw) + def _getstate(self, protocol=3): + return (datetime(self.year, self.month, self.day), self._trivia, self._raw) -class Time(time, Item): + +class Time(Item, time): """ A time literal. """ @@ -525,6 +600,13 @@ class Time(time, Item): def as_string(self): # type: () -> str return self._raw + def _getstate(self, protocol=3): + return ( + time(self.hour, self.minute, self.second, self.microsecond, self.tzinfo), + self._trivia, + self._raw, + ) + class Array(Item, list): """ @@ -623,6 +705,9 @@ class Array(Item, list): def __repr__(self): return str(self) + def _getstate(self, protocol=3): + return self._value, self._trivia + class Table(Item, dict): """ @@ -637,7 +722,7 @@ class Table(Item, dict): is_super_table=False, name=None, display_name=None, - ): # type: (tomlkit.container.Container, Trivia, bool) -> None + ): # type: (tomlkit.container.Container, Trivia, bool, ...) -> None super(Table, self).__init__(trivia) self.name = name @@ -790,6 +875,16 @@ class Table(Item, dict): def __repr__(self): return super(Table, self).__repr__() + def _getstate(self, protocol=3): + return ( + self._value, + self._trivia, + self._is_aot_element, + self._is_super_table, + self.name, + self.display_name, + ) + class InlineTable(Item, dict): """ @@ -924,6 +1019,9 @@ class InlineTable(Item, dict): def __repr__(self): return super(InlineTable, self).__repr__() + def _getstate(self, protocol=3): + return (self._value, self._trivia) + class String(unicode, Item): """ @@ -965,6 +1063,9 @@ class String(unicode, Item): def _new(self, result): return String(self._t, result, result, self._trivia) + def _getstate(self, protocol=3): + return self._t, unicode(self), self._original, self._trivia + class AoT(Item, list): """ @@ -974,7 +1075,7 @@ class AoT(Item, list): def __init__( self, body, name=None, parsed=False ): # type: (List[Table], Optional[str]) -> None - self.name = None + self.name = name self._body = [] self._parsed = parsed @@ -1025,6 +1126,9 @@ class AoT(Item, list): def __repr__(self): # type: () -> str return "".format(self.value) + def _getstate(self, protocol=3): + return self._body, self.name, self._parsed + class Null(Item): """ @@ -1044,3 +1148,6 @@ class Null(Item): def as_string(self): # type: () -> str return "" + + def _getstate(self, protocol=3): + return tuple() diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py index 45c8ee8c..7971d9a2 100644 --- a/pipenv/vendor/tomlkit/parser.py +++ b/pipenv/vendor/tomlkit/parser.py @@ -75,7 +75,7 @@ class Parser: else: return self._src[self._marker : self._idx] - def inc(self): # type: () -> bool + def inc(self, exception=None): # type: () -> bool """ Increments the parser if the end of the input has not been reached. Returns whether or not it was able to advance. @@ -88,15 +88,17 @@ class Parser: self._idx = len(self._src) self._current = TOMLChar("\0") - return False + if not exception: + return False + raise exception - def inc_n(self, n): # type: (int) -> bool + def inc_n(self, n, exception=None): # type: (int) -> bool """ Increments the parser by n characters if the end of the input has not been reached. """ for _ in range(n): - if not self.inc(): + if not self.inc(exception=exception): return False return True @@ -336,7 +338,7 @@ class Parser: self.mark() break - elif c in " \t\r,": + elif c in " \t\r": self.inc() else: raise self.parse_error(UnexpectedCharError, (c)) @@ -669,140 +671,143 @@ class Parser: return def _parse_literal_string(self): # type: () -> Item - return self._parse_string("'") + return self._parse_string(StringType.SLL) def _parse_basic_string(self): # type: () -> Item - return self._parse_string('"') + return self._parse_string(StringType.SLB) + + def _parse_escaped_char(self, multiline): + if multiline and self._current.is_ws(): + # When the last non-whitespace character on a line is + # a \, it will be trimmed along with all whitespace + # (including newlines) up to the next non-whitespace + # character or closing delimiter. + # """\ + # hello \ + # world""" + tmp = "" + while self._current.is_ws(): + tmp += self._current + # consume the whitespace, EOF here is an issue + # (middle of string) + self.inc(exception=UnexpectedEofError) + continue + + # the escape followed by whitespace must have a newline + # before any other chars + if "\n" not in tmp: + raise self.parse_error(InvalidCharInStringError, (self._current,)) + + return "" + + if self._current in _escaped: + c = _escaped[self._current] + + # consume this char, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) + + return c + + if self._current in {"u", "U"}: + # this needs to be a unicode + u, ue = self._peek_unicode(self._current == "U") + if u is not None: + # consume the U char and the unicode value + self.inc_n(len(ue) + 1) + + return u + + raise self.parse_error(InvalidCharInStringError, (self._current,)) def _parse_string(self, delim): # type: (str) -> Item - multiline = False + delim = StringType(delim) + assert delim.is_singleline() + + # only keep parsing for string if the current character matches the delim + if self._current != delim.unit: + raise ValueError("Expecting a {!r} character".format(delim)) + + # consume the opening/first delim, EOF here is an issue + # (middle of string or middle of delim) + self.inc(exception=UnexpectedEofError) + + if self._current == delim.unit: + # consume the closing/second delim, we do not care if EOF occurs as + # that would simply imply an empty single line string + if not self.inc() or self._current != delim.unit: + # Empty string + return String(delim, "", "", Trivia()) + + # consume the third delim, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) + + delim = delim.toggle() # convert delim to multi delim + + self.mark() # to extract the original string with whitespace and all value = "" - if delim == "'": - str_type = StringType.SLL - else: - str_type = StringType.SLB + # A newline immediately following the opening delimiter will be trimmed. + if delim.is_multiline() and self._current == "\n": + # consume the newline, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) - # Skip opening delim - if not self.inc(): - return self.parse_error(UnexpectedEofError) - - if self._current == delim: - self.inc() - - if self._current == delim: - multiline = True - if delim == "'": - str_type = StringType.MLL - else: - str_type = StringType.MLB - - if not self.inc(): - return self.parse_error(UnexpectedEofError) - else: - # Empty string - return String(str_type, "", "", Trivia()) - - self.mark() - if self._current == "\n": - # The first new line should be discarded - self.inc() - - previous = None - escaped = False + escaped = False # whether the previous key was ESCAPE while True: - if ( - previous != "\\" - or previous == "\\" - and (escaped or str_type.is_literal()) - ) and self._current == delim: - val = self.extract() + if delim.is_singleline() and self._current.is_nl(): + # single line cannot have actual newline characters + raise self.parse_error(InvalidCharInStringError, (self._current,)) + elif not escaped and self._current == delim.unit: + # try to process current as a closing delim + original = self.extract() - if multiline: - stop = True - for _ in range(3): - if self._current != delim: + close = "" + if delim.is_multiline(): + # try consuming three delims as this would mean the end of + # the string + for last in [False, False, True]: + if self._current != delim.unit: # Not a triple quote, leave in result as-is. - stop = False - - # Adding back the quote character - value += delim + # Adding back the characters we already consumed + value += close + close = "" # clear the close break - self.inc() # TODO: Handle EOF + close += delim.unit - if not stop: + # consume this delim, EOF here is only an issue if this + # is not the third (last) delim character + self.inc(exception=UnexpectedEofError if not last else None) + + if not close: # if there is no close characters, keep parsing continue else: + close = delim.unit + + # consume the closing delim, we do not care if EOF occurs as + # that would simply imply the end of self._src self.inc() - return String(str_type, value, val, Trivia()) + return String(delim, value, original, Trivia()) + elif delim.is_basic() and escaped: + # attempt to parse the current char as an escaped value, an exception + # is raised if this fails + value += self._parse_escaped_char(delim.is_multiline()) + + # no longer escaped + escaped = False + elif delim.is_basic() and self._current == "\\": + # the next char is being escaped + escaped = True + + # consume this char, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) else: - if previous == "\\" and self._current.is_ws() and multiline: - while self._current.is_ws(): - previous = self._current + # this is either a literal string where we keep everything as is, + # or this is not a special escaped char in a basic string + value += self._current - self.inc() - continue - - if self._current == delim: - continue - - if previous == "\\": - if self._current == "\\" and not escaped: - if not str_type.is_literal(): - escaped = True - else: - value += self._current - - previous = self._current - - if not self.inc(): - raise self.parse_error(UnexpectedEofError) - - continue - elif self._current in _escaped and not escaped: - if not str_type.is_literal(): - value = value[:-1] - value += _escaped[self._current] - else: - value += self._current - elif self._current in {"u", "U"} and not escaped: - # Maybe unicode - u, ue = self._peek_unicode(self._current == "U") - if u is not None: - value = value[:-1] - value += u - self.inc_n(len(ue)) - else: - if not escaped and not str_type.is_literal(): - raise self.parse_error( - InvalidCharInStringError, (self._current,) - ) - - value += self._current - else: - if not escaped and not str_type.is_literal(): - raise self.parse_error( - InvalidCharInStringError, (self._current,) - ) - - value += self._current - - if self._current.is_ws() and multiline and not escaped: - continue - else: - value += self._current - - if escaped: - escaped = False - - previous = self._current - if not self.inc(): - raise self.parse_error(UnexpectedEofError) - - if previous == "\\" and self._current.is_ws() and multiline: - value = value[:-1] + # consume this char, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) def _parse_table( self, parent_name=None diff --git a/pipenv/vendor/tomlkit/toml_char.py b/pipenv/vendor/tomlkit/toml_char.py index 8a3bf9e1..5164ea8b 100644 --- a/pipenv/vendor/tomlkit/toml_char.py +++ b/pipenv/vendor/tomlkit/toml_char.py @@ -1,7 +1,13 @@ import string +from ._compat import PY2 from ._compat import unicode +if PY2: + from pipenv.vendor.backports.functools_lru_cache import lru_cache +else: + from functools import lru_cache + class TOMLChar(unicode): def __init__(self, c): @@ -10,36 +16,42 @@ class TOMLChar(unicode): if len(self) > 1: raise ValueError("A TOML character must be of length 1") + @lru_cache(maxsize=None) def is_bare_key_char(self): # type: () -> bool """ Whether the character is a valid bare key name or not. """ return self in string.ascii_letters + string.digits + "-" + "_" + @lru_cache(maxsize=None) def is_kv_sep(self): # type: () -> bool """ Whether the character is a valid key/value separator ot not. """ return self in "= \t" + @lru_cache(maxsize=None) def is_int_float_char(self): # type: () -> bool """ Whether the character if a valid integer or float value character or not. """ return self in string.digits + "+" + "-" + "_" + "." + "e" + @lru_cache(maxsize=None) def is_ws(self): # type: () -> bool """ Whether the character is a whitespace character or not. """ return self in " \t\r\n" + @lru_cache(maxsize=None) def is_nl(self): # type: () -> bool """ Whether the character is a new line character or not. """ return self in "\n\r" + @lru_cache(maxsize=None) def is_spaces(self): # type: () -> bool """ Whether the character is a space or not diff --git a/pipenv/vendor/urllib3/__init__.py b/pipenv/vendor/urllib3/__init__.py old mode 100755 new mode 100644 index 4bd533b5..75725167 --- a/pipenv/vendor/urllib3/__init__.py +++ b/pipenv/vendor/urllib3/__init__.py @@ -23,16 +23,11 @@ from .util.retry import Retry # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.23' +__version__ = '1.24' __all__ = ( 'HTTPConnectionPool', diff --git a/pipenv/vendor/urllib3/_collections.py b/pipenv/vendor/urllib3/_collections.py old mode 100755 new mode 100644 index 6e36b84e..34f23811 --- a/pipenv/vendor/urllib3/_collections.py +++ b/pipenv/vendor/urllib3/_collections.py @@ -14,10 +14,7 @@ except ImportError: # Platform-specific: No threads available pass -try: # Python 2.7+ - from collections import OrderedDict -except ImportError: - from .packages.ordered_dict import OrderedDict +from collections import OrderedDict from .exceptions import InvalidHeader from .packages.six import iterkeys, itervalues, PY3 diff --git a/pipenv/vendor/urllib3/connection.py b/pipenv/vendor/urllib3/connection.py old mode 100755 new mode 100644 index a03b573f..02b36654 --- a/pipenv/vendor/urllib3/connection.py +++ b/pipenv/vendor/urllib3/connection.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import datetime import logging import os -import sys import socket from socket import error as SocketError, timeout as SocketTimeout import warnings @@ -78,9 +77,6 @@ class HTTPConnection(_HTTPConnection, object): - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool` - ``source_address``: Set the source address for the current connection. - - .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x - - ``socket_options``: Set specific options on the underlying socket. If not specified, then defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy. @@ -108,21 +104,13 @@ class HTTPConnection(_HTTPConnection, object): if six.PY3: # Python 3 kw.pop('strict', None) - # Pre-set source_address in case we have an older Python like 2.6. + # Pre-set source_address. self.source_address = kw.get('source_address') - if sys.version_info < (2, 7): # Python 2.6 - # _HTTPConnection on Python 2.6 will balk at this keyword arg, but - # not newer versions. We can still use it when creating a - # connection though, so we pop it *after* we have saved it as - # self.source_address. - kw.pop('source_address', None) - #: The socket options provided by the user. If no options are #: provided, we use the default options. self.socket_options = kw.pop('socket_options', self.default_socket_options) - # Superclass also sets self.source_address in Python 2.7+. _HTTPConnection.__init__(self, *args, **kw) @property @@ -183,10 +171,7 @@ class HTTPConnection(_HTTPConnection, object): def _prepare_conn(self, conn): self.sock = conn - # the _tunnel_host attribute was added in python 2.6.3 (via - # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do - # not have them. - if getattr(self, '_tunnel_host', None): + if self._tunnel_host: # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() # Mark this connection as not reusable @@ -217,13 +202,13 @@ class HTTPConnection(_HTTPConnection, object): self.endheaders() if body is not None: - stringish_types = six.string_types + (six.binary_type,) + stringish_types = six.string_types + (bytes,) if isinstance(body, stringish_types): body = (body,) for chunk in body: if not chunk: continue - if not isinstance(chunk, six.binary_type): + if not isinstance(chunk, bytes): chunk = chunk.encode('utf8') len_str = hex(len(chunk))[2:] self.send(len_str.encode('utf-8')) @@ -242,7 +227,7 @@ class HTTPSConnection(HTTPConnection): def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - ssl_context=None, **kw): + ssl_context=None, server_hostname=None, **kw): HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) @@ -250,6 +235,7 @@ class HTTPSConnection(HTTPConnection): self.key_file = key_file self.cert_file = cert_file self.ssl_context = ssl_context + self.server_hostname = server_hostname # Required property for Google AppEngine 1.9.0 which otherwise causes # HTTPS requests to go out as HTTP. (See Issue #356) @@ -270,6 +256,7 @@ class HTTPSConnection(HTTPConnection): keyfile=self.key_file, certfile=self.cert_file, ssl_context=self.ssl_context, + server_hostname=self.server_hostname ) @@ -312,12 +299,9 @@ class VerifiedHTTPSConnection(HTTPSConnection): def connect(self): # Add certificate verification conn = self._new_conn() - hostname = self.host - if getattr(self, '_tunnel_host', None): - # _tunnel_host was added in Python 2.6.3 - # (See: http://hg.python.org/cpython/rev/0f57b30a152f) + if self._tunnel_host: self.sock = conn # Calls self._set_hostport(), so self.host is # self._tunnel_host below. @@ -328,6 +312,10 @@ class VerifiedHTTPSConnection(HTTPSConnection): # Override the host with the one we're requesting data from. hostname = self._tunnel_host + server_hostname = hostname + if self.server_hostname is not None: + server_hostname = self.server_hostname + is_time_off = datetime.date.today() < RECENT_DATE if is_time_off: warnings.warn(( @@ -352,7 +340,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): certfile=self.cert_file, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, - server_hostname=hostname, + server_hostname=server_hostname, ssl_context=context) if self.assert_fingerprint: @@ -373,7 +361,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): 'for details.)'.format(hostname)), SubjectAltNameWarning ) - _match_hostname(cert, self.assert_hostname or hostname) + _match_hostname(cert, self.assert_hostname or server_hostname) self.is_verified = ( context.verify_mode == ssl.CERT_REQUIRED or diff --git a/pipenv/vendor/urllib3/connectionpool.py b/pipenv/vendor/urllib3/connectionpool.py old mode 100755 new mode 100644 index 8fcb0bce..f7a8f193 --- a/pipenv/vendor/urllib3/connectionpool.py +++ b/pipenv/vendor/urllib3/connectionpool.py @@ -89,7 +89,7 @@ class ConnectionPool(object): # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 -_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK]) +_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK} class HTTPConnectionPool(ConnectionPool, RequestMethods): @@ -313,7 +313,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Catch possible read timeouts thrown as SSL errors. If not the # case, rethrow the original. We need to do this because of: # http://bugs.python.org/issue10272 - if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 + if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python < 2.7.4 raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) def _make_request(self, conn, method, url, timeout=_Default, chunked=False, @@ -375,7 +375,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): try: try: # Python 2.7, use buffering of HTTP responses httplib_response = conn.getresponse(buffering=True) - except TypeError: # Python 2.6 and older, Python 3 + except TypeError: # Python 3 try: httplib_response = conn.getresponse() except Exception as e: @@ -801,17 +801,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): Establish tunnel connection early, because otherwise httplib would improperly set Host: header to proxy's IP:port. """ - # Python 2.7+ - try: - set_tunnel = conn.set_tunnel - except AttributeError: # Platform-specific: Python 2.6 - set_tunnel = conn._set_tunnel - - if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older - set_tunnel(self._proxy_host, self.port) - else: - set_tunnel(self._proxy_host, self.port, self.proxy_headers) - + conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) conn.connect() def _new_conn(self): diff --git a/pipenv/vendor/urllib3/contrib/__init__.py b/pipenv/vendor/urllib3/contrib/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/_appengine_environ.py b/pipenv/vendor/urllib3/contrib/_appengine_environ.py new file mode 100644 index 00000000..f3e00942 --- /dev/null +++ b/pipenv/vendor/urllib3/contrib/_appengine_environ.py @@ -0,0 +1,30 @@ +""" +This module provides means to detect the App Engine environment. +""" + +import os + + +def is_appengine(): + return (is_local_appengine() or + is_prod_appengine() or + is_prod_appengine_mvms()) + + +def is_appengine_sandbox(): + return is_appengine() and not is_prod_appengine_mvms() + + +def is_local_appengine(): + return ('APPENGINE_RUNTIME' in os.environ and + 'Development/' in os.environ['SERVER_SOFTWARE']) + + +def is_prod_appengine(): + return ('APPENGINE_RUNTIME' in os.environ and + 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and + not is_prod_appengine_mvms()) + + +def is_prod_appengine_mvms(): + return os.environ.get('GAE_VM', False) == 'true' diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/__init__.py b/pipenv/vendor/urllib3/contrib/_securetransport/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py b/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/low_level.py b/pipenv/vendor/urllib3/contrib/_securetransport/low_level.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/appengine.py b/pipenv/vendor/urllib3/contrib/appengine.py old mode 100755 new mode 100644 index 66922e06..2952f114 --- a/pipenv/vendor/urllib3/contrib/appengine.py +++ b/pipenv/vendor/urllib3/contrib/appengine.py @@ -39,8 +39,8 @@ urllib3 on Google App Engine: """ from __future__ import absolute_import +import io import logging -import os import warnings from ..packages.six.moves.urllib.parse import urljoin @@ -53,11 +53,11 @@ from ..exceptions import ( SSLError ) -from ..packages.six import BytesIO from ..request import RequestMethods from ..response import HTTPResponse from ..util.timeout import Timeout from ..util.retry import Retry +from . import _appengine_environ try: from google.appengine.api import urlfetch @@ -239,7 +239,7 @@ class AppEngineManager(RequestMethods): original_response = HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), msg=urlfetch_resp.header_msg, headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, @@ -247,7 +247,7 @@ class AppEngineManager(RequestMethods): ) return HTTPResponse( - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, original_response=original_response, @@ -280,26 +280,10 @@ class AppEngineManager(RequestMethods): return retries -def is_appengine(): - return (is_local_appengine() or - is_prod_appengine() or - is_prod_appengine_mvms()) +# Alias methods from _appengine_environ to maintain public API interface. - -def is_appengine_sandbox(): - return is_appengine() and not is_prod_appengine_mvms() - - -def is_local_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Development/' in os.environ['SERVER_SOFTWARE']) - - -def is_prod_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and - not is_prod_appengine_mvms()) - - -def is_prod_appengine_mvms(): - return os.environ.get('GAE_VM', False) == 'true' +is_appengine = _appengine_environ.is_appengine +is_appengine_sandbox = _appengine_environ.is_appengine_sandbox +is_local_appengine = _appengine_environ.is_local_appengine +is_prod_appengine = _appengine_environ.is_prod_appengine +is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms diff --git a/pipenv/vendor/urllib3/contrib/ntlmpool.py b/pipenv/vendor/urllib3/contrib/ntlmpool.py old mode 100755 new mode 100644 index 642e99ed..8ea127c5 --- a/pipenv/vendor/urllib3/contrib/ntlmpool.py +++ b/pipenv/vendor/urllib3/contrib/ntlmpool.py @@ -43,8 +43,7 @@ class NTLMConnectionPool(HTTPSConnectionPool): log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', self.num_connections, self.host, self.authurl) - headers = {} - headers['Connection'] = 'Keep-Alive' + headers = {'Connection': 'Keep-Alive'} req_header = 'Authorization' resp_header = 'www-authenticate' diff --git a/pipenv/vendor/urllib3/contrib/pyopenssl.py b/pipenv/vendor/urllib3/contrib/pyopenssl.py old mode 100755 new mode 100644 index 4d4b1aff..7c0e9465 --- a/pipenv/vendor/urllib3/contrib/pyopenssl.py +++ b/pipenv/vendor/urllib3/contrib/pyopenssl.py @@ -163,6 +163,9 @@ def _dnsname_to_stdlib(name): from ASCII bytes. We need to idna-encode that string to get it back, and then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8). + + If the name cannot be idna-encoded then we return None signalling that + the name given should be skipped. """ def idna_encode(name): """ @@ -172,14 +175,19 @@ def _dnsname_to_stdlib(name): """ import idna - for prefix in [u'*.', u'.']: - if name.startswith(prefix): - name = name[len(prefix):] - return prefix.encode('ascii') + idna.encode(name) - return idna.encode(name) + try: + for prefix in [u'*.', u'.']: + if name.startswith(prefix): + name = name[len(prefix):] + return prefix.encode('ascii') + idna.encode(name) + return idna.encode(name) + except idna.core.IDNAError: + return None name = idna_encode(name) - if sys.version_info >= (3, 0): + if name is None: + return None + elif sys.version_info >= (3, 0): name = name.decode('utf-8') return name @@ -223,9 +231,10 @@ def get_subj_alt_name(peer_cert): # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8 # decoded. This is pretty frustrating, but that's what the standard library # does with certificates, and so we need to attempt to do the same. + # We also want to skip over names which cannot be idna encoded. names = [ - ('DNS', _dnsname_to_stdlib(name)) - for name in ext.get_values_for_type(x509.DNSName) + ('DNS', name) for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) + if name is not None ] names.extend( ('IP Address', str(name)) diff --git a/pipenv/vendor/urllib3/contrib/securetransport.py b/pipenv/vendor/urllib3/contrib/securetransport.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/socks.py b/pipenv/vendor/urllib3/contrib/socks.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/exceptions.py b/pipenv/vendor/urllib3/exceptions.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/fields.py b/pipenv/vendor/urllib3/fields.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/filepost.py b/pipenv/vendor/urllib3/filepost.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/__init__.py b/pipenv/vendor/urllib3/packages/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/backports/__init__.py b/pipenv/vendor/urllib3/packages/backports/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/backports/makefile.py b/pipenv/vendor/urllib3/packages/backports/makefile.py old mode 100755 new mode 100644 index 75b80dcf..740db377 --- a/pipenv/vendor/urllib3/packages/backports/makefile.py +++ b/pipenv/vendor/urllib3/packages/backports/makefile.py @@ -16,7 +16,7 @@ def backport_makefile(self, mode="r", buffering=None, encoding=None, """ Backport of ``socket.makefile`` from Python 3.5. """ - if not set(mode) <= set(["r", "w", "b"]): + if not set(mode) <= {"r", "w", "b"}: raise ValueError( "invalid mode %r (only r, w, b allowed)" % (mode,) ) diff --git a/pipenv/vendor/urllib3/packages/ordered_dict.py b/pipenv/vendor/urllib3/packages/ordered_dict.py deleted file mode 100755 index 4479363c..00000000 --- a/pipenv/vendor/urllib3/packages/ordered_dict.py +++ /dev/null @@ -1,259 +0,0 @@ -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. -# Copyright 2009 Raymond Hettinger, released under the MIT License. -# http://code.activestate.com/recipes/576693/ -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) diff --git a/pipenv/vendor/urllib3/packages/six.py b/pipenv/vendor/urllib3/packages/six.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py b/pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py b/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py old mode 100755 new mode 100644 index 1fd42f38..d6e66c01 --- a/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py +++ b/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py @@ -9,8 +9,7 @@ import sys # ipaddress has been backported to 2.6+ in pypi. If it is installed on the # system, use it to handle IPAddress ServerAltnames (this was added in # python-3.5) otherwise only do DNS matching. This allows -# backports.ssl_match_hostname to continue to be used all the way back to -# python-2.4. +# backports.ssl_match_hostname to continue to be used in Python 2.7. try: import ipaddress except ImportError: diff --git a/pipenv/vendor/urllib3/poolmanager.py b/pipenv/vendor/urllib3/poolmanager.py old mode 100755 new mode 100644 index 506a3c9b..fe5491cf --- a/pipenv/vendor/urllib3/poolmanager.py +++ b/pipenv/vendor/urllib3/poolmanager.py @@ -47,6 +47,7 @@ _key_fields = ( 'key__socks_options', # dict 'key_assert_hostname', # bool or string 'key_assert_fingerprint', # str + 'key_server_hostname', #str ) #: The namedtuple class used to construct keys for the connection pool. diff --git a/pipenv/vendor/urllib3/request.py b/pipenv/vendor/urllib3/request.py old mode 100755 new mode 100644 index 1be33341..8f2f44bb --- a/pipenv/vendor/urllib3/request.py +++ b/pipenv/vendor/urllib3/request.py @@ -36,7 +36,7 @@ class RequestMethods(object): explicitly. """ - _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) + _encode_url_methods = {'DELETE', 'GET', 'HEAD', 'OPTIONS'} def __init__(self, headers=None): self.headers = headers or {} diff --git a/pipenv/vendor/urllib3/response.py b/pipenv/vendor/urllib3/response.py old mode 100755 new mode 100644 index 9873cb94..f0cfbb54 --- a/pipenv/vendor/urllib3/response.py +++ b/pipenv/vendor/urllib3/response.py @@ -11,7 +11,7 @@ from .exceptions import ( BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked, IncompleteRead, InvalidHeader ) -from .packages.six import string_types as basestring, binary_type, PY3 +from .packages.six import string_types as basestring, PY3 from .packages.six.moves import http_client as httplib from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed, is_response_to_head @@ -23,7 +23,7 @@ class DeflateDecoder(object): def __init__(self): self._first_try = True - self._data = binary_type() + self._data = b'' self._obj = zlib.decompressobj() def __getattr__(self, name): @@ -69,7 +69,7 @@ class GzipDecoder(object): return getattr(self._obj, name) def decompress(self, data): - ret = binary_type() + ret = b'' if self._state == GzipDecoderState.SWALLOW_DATA or not data: return ret while True: @@ -90,7 +90,31 @@ class GzipDecoder(object): self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) +class MultiDecoder(object): + """ + From RFC7231: + If one or more encodings have been applied to a representation, the + sender that applied the encodings MUST generate a Content-Encoding + header field that lists the content codings in the order in which + they were applied. + """ + + def __init__(self, modes): + self._decoders = [_get_decoder(m.strip()) for m in modes.split(',')] + + def flush(self): + return self._decoders[0].flush() + + def decompress(self, data): + for d in reversed(self._decoders): + data = d.decompress(data) + return data + + def _get_decoder(mode): + if ',' in mode: + return MultiDecoder(mode) + if mode == 'gzip': return GzipDecoder() @@ -159,7 +183,7 @@ class HTTPResponse(io.IOBase): self.msg = msg self._request_url = request_url - if body and isinstance(body, (basestring, binary_type)): + if body and isinstance(body, (basestring, bytes)): self._body = body self._pool = pool @@ -283,8 +307,13 @@ class HTTPResponse(io.IOBase): # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None and content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + if self._decoder is None: + if content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + elif ',' in content_encoding: + encodings = [e.strip() for e in content_encoding.split(',') if e.strip() in self.CONTENT_DECODERS] + if len(encodings): + self._decoder = _get_decoder(content_encoding) def _decode(self, data, decode_content, flush_decoder): """ diff --git a/pipenv/vendor/urllib3/util/__init__.py b/pipenv/vendor/urllib3/util/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/connection.py b/pipenv/vendor/urllib3/util/connection.py old mode 100755 new mode 100644 index 5cf488f4..5ad70b2f --- a/pipenv/vendor/urllib3/util/connection.py +++ b/pipenv/vendor/urllib3/util/connection.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import socket from .wait import NoWayToWaitForSocketError, wait_for_read +from ..contrib import _appengine_environ def is_connection_dropped(conn): # Platform-specific @@ -105,6 +106,13 @@ def _has_ipv6(host): sock = None has_ipv6 = False + # App Engine doesn't support IPV6 sockets and actually has a quota on the + # number of sockets that can be used, so just early out here instead of + # creating a socket needlessly. + # See https://github.com/urllib3/urllib3/issues/1446 + if _appengine_environ.is_appengine_sandbox(): + return False + if socket.has_ipv6: # has_ipv6 returns true if cPython was compiled with IPv6 support. # It does not tell us if the system has IPv6 support enabled. To diff --git a/pipenv/vendor/urllib3/util/queue.py b/pipenv/vendor/urllib3/util/queue.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/request.py b/pipenv/vendor/urllib3/util/request.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/response.py b/pipenv/vendor/urllib3/util/response.py old mode 100755 new mode 100644 index 67cf730a..3d548648 --- a/pipenv/vendor/urllib3/util/response.py +++ b/pipenv/vendor/urllib3/util/response.py @@ -59,8 +59,14 @@ def assert_header_parsing(headers): get_payload = getattr(headers, 'get_payload', None) unparsed_data = None - if get_payload: # Platform-specific: Python 3. - unparsed_data = get_payload() + if get_payload: + # get_payload is actually email.message.Message.get_payload; + # we're only interested in the result if it's not a multipart message + if not headers.is_multipart(): + payload = get_payload() + + if isinstance(payload, (bytes, str)): + unparsed_data = payload if defects or unparsed_data: raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data) diff --git a/pipenv/vendor/urllib3/util/retry.py b/pipenv/vendor/urllib3/util/retry.py old mode 100755 new mode 100644 index 7ad3dc66..e7d0abd6 --- a/pipenv/vendor/urllib3/util/retry.py +++ b/pipenv/vendor/urllib3/util/retry.py @@ -115,7 +115,7 @@ class Retry(object): (most errors are resolved immediately by a second try without a delay). urllib3 will sleep for:: - {backoff factor} * (2 ^ ({number of total retries} - 1)) + {backoff factor} * (2 ** ({number of total retries} - 1)) seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer diff --git a/pipenv/vendor/urllib3/util/ssl_.py b/pipenv/vendor/urllib3/util/ssl_.py old mode 100755 new mode 100644 index 2893752a..24ee26d6 --- a/pipenv/vendor/urllib3/util/ssl_.py +++ b/pipenv/vendor/urllib3/util/ssl_.py @@ -56,9 +56,8 @@ except ImportError: OP_NO_COMPRESSION = 0x20000 -# Python 2.7 and earlier didn't have inet_pton on non-Linux -# so we fallback on inet_aton in those cases. This means that -# we can only detect IPv4 addresses in this case. +# Python 2.7 doesn't have inet_pton on non-Linux so we fallback on inet_aton in +# those cases. This means that we can only detect IPv4 addresses in this case. if hasattr(socket, 'inet_pton'): inet_pton = socket.inet_pton else: @@ -67,7 +66,7 @@ else: import ipaddress def inet_pton(_, host): - if isinstance(host, six.binary_type): + if isinstance(host, bytes): host = host.decode('ascii') return ipaddress.ip_address(host) @@ -115,10 +114,7 @@ try: except ImportError: import sys - class SSLContext(object): # Platform-specific: Python 2 & 3.1 - supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or - (3, 2) <= sys.version_info) - + class SSLContext(object): # Platform-specific: Python 2 def __init__(self, protocol_version): self.protocol = protocol_version # Use default values from a real SSLContext @@ -141,12 +137,6 @@ except ImportError: raise SSLError("CA directories not supported in older Pythons") def set_ciphers(self, cipher_suite): - if not self.supports_set_ciphers: - raise TypeError( - 'Your version of Python does not support setting ' - 'a custom cipher suite. Please upgrade to Python ' - '2.7, 3.2, or later if you need this functionality.' - ) self.ciphers = cipher_suite def wrap_socket(self, socket, server_hostname=None, server_side=False): @@ -167,10 +157,7 @@ except ImportError: 'ssl_version': self.protocol, 'server_side': server_side, } - if self.supports_set_ciphers: # Platform-specific: Python 2.7+ - return wrap_socket(socket, ciphers=self.ciphers, **kwargs) - else: # Platform-specific: Python 2.6 - return wrap_socket(socket, **kwargs) + return wrap_socket(socket, ciphers=self.ciphers, **kwargs) def assert_fingerprint(cert, fingerprint): @@ -291,9 +278,6 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, context.options |= options - if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6 - context.set_ciphers(ciphers or DEFAULT_CIPHERS) - context.verify_mode = cert_reqs if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 # We do our own verification, including fingerprints and alternative @@ -316,8 +300,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, A pre-made :class:`SSLContext` object. If none is provided, one will be created using :func:`create_urllib3_context`. :param ciphers: - A string of ciphers we wish the client to support. This is not - supported on Python 2.6 as the ssl module does not support it. + A string of ciphers we wish the client to support. :param ca_cert_dir: A directory containing CA certificates in multiple separate files, as supported by OpenSSL's -CApath flag or the capath argument to @@ -334,7 +317,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if ca_certs or ca_cert_dir: try: context.load_verify_locations(ca_certs, ca_cert_dir) - except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 + except IOError as e: # Platform-specific: Python 2.7 raise SSLError(e) # Py33 raises FileNotFoundError which subclasses OSError # These are not equivalent unless we check the errno attribute @@ -378,7 +361,7 @@ def is_ipaddress(hostname): :param str hostname: Hostname to examine. :return: True if the hostname is an IP address, False otherwise. """ - if six.PY3 and isinstance(hostname, six.binary_type): + if six.PY3 and isinstance(hostname, bytes): # IDN A-label bytes are ASCII compatible. hostname = hostname.decode('ascii') diff --git a/pipenv/vendor/urllib3/util/timeout.py b/pipenv/vendor/urllib3/util/timeout.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/url.py b/pipenv/vendor/urllib3/util/url.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/wait.py b/pipenv/vendor/urllib3/util/wait.py old mode 100755 new mode 100644 index fa686eff..4db71baf --- a/pipenv/vendor/urllib3/util/wait.py +++ b/pipenv/vendor/urllib3/util/wait.py @@ -43,9 +43,6 @@ if sys.version_info >= (3, 5): else: # Old and broken Pythons. def _retry_on_intr(fn, timeout): - if timeout is not None and timeout <= 0: - return fn(timeout) - if timeout is None: deadline = float("inf") else: @@ -117,7 +114,7 @@ def _have_working_poll(): # from libraries like eventlet/greenlet. try: poll_obj = select.poll() - poll_obj.poll(0) + _retry_on_intr(poll_obj.poll, 0) except (AttributeError, OSError): return False else: diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 7a574b9f..18352c2e 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -14,37 +14,40 @@ first==2.0.1 iso8601==0.1.12 jinja2==2.10 markupsafe==1.0 -parse==1.8.4 +parse==1.9.0 pathlib2==2.3.2 scandir==1.9 pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.2 -requests==2.19.1 +pythonfinder==1.1.6 +requests==2.20.0 chardet==3.0.4 idna==2.7 - urllib3==1.23 - certifi==2018.8.24 -requirementslib==1.1.9 + urllib3==1.24 + certifi==2018.10.15 +requirementslib==1.2.2 attrs==18.2.0 distlib==0.2.8 packaging==18.0 pyparsing==2.2.2 pytoml==0.1.19 plette==0.2.2 - tomlkit==0.4.4 -shellingham==1.2.6 + tomlkit==0.4.6 +shellingham==1.2.7 six==1.11.0 semver==2.8.1 shutilwhich==1.1.0 toml==0.10.0 cached-property==1.4.3 -vistir==0.1.7 -pip-shims==0.3.1 +vistir==0.2.2 +pip-shims==0.3.2 ptyprocess==0.6.0 enum34==1.1.6 yaspin==0.14.0 cerberus==1.2 git+https://github.com/sarugaku/passa.git@master#egg=passa +cursor==1.2.0 +resolvelib==0.2.2 +backports.functools_lru_cache==1.5 diff --git a/pipenv/vendor/vendor_pip.txt b/pipenv/vendor/vendor_pip.txt index b9854e9a..9389dd94 100644 --- a/pipenv/vendor/vendor_pip.txt +++ b/pipenv/vendor/vendor_pip.txt @@ -9,14 +9,15 @@ msgpack-python==0.5.6 lockfile==0.12.2 progress==1.4 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -packaging==17.1 -pyparsing==2.2.0 -pytoml==0.1.16 +packaging==18.0 +pep517==0.2 +pyparsing==2.2.1 +pytoml==0.1.19 retrying==1.3.3 requests==2.19.1 chardet==3.0.4 idna==2.7 urllib3==1.23 - certifi==2018.4.16 -setuptools==39.2.0 + certifi==2018.8.24 +setuptools==40.4.3 webencodings==0.5.1 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index fe6f884c..c8a995fa 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -1,19 +1,37 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals -from .compat import NamedTemporaryFile, TemporaryDirectory, partialmethod +from .compat import ( + NamedTemporaryFile, + TemporaryDirectory, + partialmethod, + to_native_string, +) from .contextmanagers import ( atomic_open_for_write, cd, open_file, temp_environ, temp_path, + spinner, ) -from .misc import load_path, partialclass, run, shell_escape -from .path import mkdir_p, rmtree +from .misc import ( + load_path, + partialclass, + run, + shell_escape, + decode_for_output, + to_text, + to_bytes, + take, + chunked, + divide, +) +from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile +from .spin import VistirSpinner, create_spinner -__version__ = '0.1.7' +__version__ = "0.2.3" __all__ = [ @@ -31,4 +49,16 @@ __all__ = [ "TemporaryDirectory", "NamedTemporaryFile", "partialmethod", + "spinner", + "VistirSpinner", + "create_spinner", + "create_tracked_tempdir", + "create_tracked_tempfile", + "to_native_string", + "decode_for_output", + "to_text", + "to_bytes", + "take", + "chunked", + "divide", ] diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index 43470a6e..fb044acf 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py @@ -175,6 +175,7 @@ def NamedTemporaryFile( prefix=None, dir=None, delete=True, + wrapper_class_override=None ): """Create and return a temporary file. Arguments: @@ -193,6 +194,8 @@ def NamedTemporaryFile( flags = _bin_openflags # Setting O_TEMPORARY in the flags causes the OS to delete # the file when it is closed. This is only supported by Windows. + if not wrapper_class_override: + wrapper_class_override = _TemporaryFileWrapper if os.name == "nt" and delete: flags |= os.O_TEMPORARY if sys.version_info < (3, 5): @@ -203,7 +206,12 @@ def NamedTemporaryFile( file = io.open( fd, mode, buffering=buffering, newline=newline, encoding=encoding ) - return _TemporaryFileWrapper(file, name, delete) + if wrapper_class_override is not None: + return type( + str("_TempFileWrapper"), (wrapper_class_override, object), {} + )(file, name, delete) + else: + return _TemporaryFileWrapper(file, name, delete) except BaseException: os.unlink(name) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index 0c865fe6..d6e8578a 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import errno import os import sys import warnings @@ -16,22 +17,27 @@ __all__ = [ "finalize", "partialmethod", "JSONDecodeError", + "FileNotFoundError", "ResourceWarning", "FileNotFoundError", "fs_str", + "lru_cache", "TemporaryDirectory", "NamedTemporaryFile", + "to_native_string", ] if sys.version_info >= (3, 5): from pathlib import Path - + from functools import lru_cache else: from pathlib2 import Path + from pipenv.vendor.backports.functools_lru_cache import lru_cache +from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile if sys.version_info < (3, 3): from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size - from .backports.tempfile import NamedTemporaryFile + NamedTemporaryFile = _NamedTemporaryFile else: from tempfile import NamedTemporaryFile from shutil import get_terminal_size @@ -57,16 +63,18 @@ if six.PY2: pass class FileNotFoundError(IOError): - pass + """No such file or directory""" + + def __init__(self, *args, **kwargs): + self.errno = errno.ENOENT + super(FileNotFoundError, self).__init__(*args, **kwargs) else: from builtins import ResourceWarning, FileNotFoundError - class ResourceWarning(ResourceWarning): - pass - class FileNotFoundError(FileNotFoundError): - pass +if not sys.warnoptions: + warnings.simplefilter("default", ResourceWarning) class TemporaryDirectory(object): @@ -135,3 +143,10 @@ def fs_str(string): _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() + + +def to_native_string(string): + from .misc import to_text, to_bytes + if six.PY2: + return to_bytes(string) + return to_text(string) diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 80f1f897..59b97ca0 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import io import os import stat import sys @@ -13,7 +14,9 @@ from .compat import NamedTemporaryFile, Path from .path import is_file_url, is_valid_url, path_to_url, url_to_path -__all__ = ["temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file"] +__all__ = [ + "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner" +] # Borrowed from Pew. @@ -77,6 +80,70 @@ def cd(path): os.chdir(prev_cwd) +@contextmanager +def dummy_spinner(spin_type, text, **kwargs): + class FakeClass(object): + def __init__(self, text=""): + self.text = text + + def fail(self, exitcode=1, text=None): + if text: + print(text) + raise SystemExit(exitcode, text) + + def ok(self, text): + print(text) + return 0 + + def write(self, text): + print(text) + + myobj = FakeClass(text) + yield myobj + + +@contextmanager +def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): + """Get a spinner object or a dummy spinner to wrap a context. + + :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) + :param str start_text: Text to start off the spinner with (default: {None}) + :param dict handler_map: Handler map for signals to be handled gracefully (default: {None}) + :param bool nospin: If true, use the dummy spinner (default: {False}) + :return: A spinner object which can be manipulated while alive + :rtype: :class:`~vistir.spin.VistirSpinner` + + Raises: + RuntimeError -- Raised if the spinner extra is not installed + """ + + from .spin import create_spinner + has_yaspin = False + try: + import yaspin + except ImportError: + if not nospin: + raise RuntimeError( + "Failed to import spinner! Reinstall vistir with command:" + " pip install --upgrade vistir[spinner]" + ) + else: + spinner_name = "" + else: + has_yaspin = True + spinner_name = "" + if not start_text and nospin is False: + start_text = "Running..." + with create_spinner( + spinner_name=spinner_name, + text=start_text, + handler_map=handler_map, + nospin=nospin, + use_yaspin=has_yaspin + ) as _spinner: + yield _spinner + + @contextmanager def atomic_open_for_write(target, binary=False, newline=None, encoding=None): """Atomically open `target` for writing. @@ -168,12 +235,13 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None): @contextmanager -def open_file(link, session=None): +def open_file(link, session=None, stream=True): """ Open local or remote file for reading. :type link: pip._internal.index.Link or str :type session: requests.Session + :param bool stream: Try to stream if remote, default True :raises ValueError: If link points to a local directory. :return: a context manager to the opened file-like object """ @@ -192,8 +260,8 @@ def open_file(link, session=None): if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(link)) else: - with open(local_path, "rb") as local_file: - yield local_file + with io.open(local_path, "rb") as local_file: + yield local_file else: # Remote URL headers = {"Accept-Encoding": "identity"} @@ -201,8 +269,14 @@ def open_file(link, session=None): from requests import Session session = Session() - response = session.get(link, headers=headers, stream=True) - try: - yield response.raw - finally: - response.close() + with session.get(link, headers=headers, stream=stream) as resp: + try: + raw = getattr(resp, "raw", None) + result = raw if raw else resp + yield result + finally: + if raw: + conn = getattr(raw, "_connection") + if conn is not None: + conn.close() + result.close() diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 44607a98..7342bc97 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -2,19 +2,25 @@ from __future__ import absolute_import, unicode_literals import json +import logging import locale import os import subprocess import sys from collections import OrderedDict -from contextlib import contextmanager from functools import partial +from itertools import islice import six from .cmdparse import Script -from .compat import Path, fs_str, partialmethod +from .compat import Path, fs_str, partialmethod, to_native_string +from .contextmanagers import spinner as spinner + +if os.name != "nt": + class WindowsError(OSError): + pass __all__ = [ @@ -27,9 +33,28 @@ __all__ = [ "to_text", "to_bytes", "locale_encoding", + "chunked", + "take", + "divide" ] +def _get_logger(name=None, level="ERROR"): + if not name: + name = __name__ + if isinstance(level, six.string_types): + level = getattr(logging, level.upper()) + logger = logging.getLogger(name) + logger.setLevel(level) + formatter = logging.Formatter( + "%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s" + ) + handler = logging.StreamHandler(stream=sys.stderr) + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + + def shell_escape(cmd): """Escape strings for use in :func:`~subprocess.Popen` and :func:`run`. @@ -75,9 +100,11 @@ def dedup(iterable): return iter(OrderedDict.fromkeys(iterable)) -def _spawn_subprocess(script, env={}, block=True, cwd=None, combine_stderr=True): +def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=True): from distutils.spawn import find_executable + if not env: + env = {} command = find_executable(script.command) options = { "env": env, @@ -102,7 +129,7 @@ def _spawn_subprocess(script, env={}, block=True, cwd=None, combine_stderr=True) try: return subprocess.Popen(cmd, **options) except WindowsError as e: - if e.winerror != 193: + if getattr(e, "winerror", 9999) != 193: raise options["shell"] = True # Try shell mode to use Windows's file association for file launch. @@ -111,15 +138,18 @@ def _spawn_subprocess(script, env={}, block=True, cwd=None, combine_stderr=True) def _create_subprocess( cmd, - env={}, + env=None, block=True, return_object=False, cwd=os.curdir, verbose=False, spinner=None, combine_stderr=False, - display_limit=200 + display_limit=200, + start_text="" ): + if not env: + env = {} try: c = _spawn_subprocess(cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr) @@ -130,9 +160,11 @@ def _create_subprocess( c.stdin.close() output = [] err = [] - spinner_orig_text = "" + spinner_orig_text = None if spinner: - spinner_orig_text = spinner.text + spinner_orig_text = getattr(spinner, "text", None) + if spinner_orig_text is None: + spinner_orig_text = start_text if start_text is not None else "" streams = { "stdout": c.stdout, "stderr": c.stderr @@ -147,23 +179,39 @@ def _create_subprocess( line = to_text(stream.readline()) if not line: continue - line = line.rstrip() + line = to_text("{0}".format(line.rstrip())) if outstream == "stderr": stderr_line = line else: stdout_line = line if not (stdout_line or stderr_line): break - if stderr_line: - err.append(line) - if stdout_line: + if stderr_line is not None: + err.append(stderr_line) + err_line = fs_str("{0}".format(stderr_line)) + if verbose and err_line is not None: + if spinner: + spinner._hide_cursor() + spinner.write_err(err_line) + spinner._show_cursor() + else: + sys.stderr.write(err_line) + sys.stderr.flush() + if stdout_line is not None: output.append(stdout_line) - display_line = stdout_line + display_line = fs_str("{0}".format(stdout_line)) if len(stdout_line) > display_limit: display_line = "{0}...".format(stdout_line[:display_limit]) - if verbose: - spinner.write(display_line) - spinner.text = "{0} {1}".format(spinner_orig_text, display_line) + if verbose and display_line is not None: + if spinner: + spinner._hide_cursor() + spinner.write_err(display_line) + spinner._show_cursor() + else: + sys.stderr.write(display_line) + sys.stderr.flush() + if spinner: + spinner.text = to_native_string("{0} {1}".format(spinner_orig_text, display_line)) continue try: c.wait() @@ -174,19 +222,21 @@ def _create_subprocess( c.stderr.close() if spinner: if c.returncode > 0: - spinner.fail("Failed...cleaning up...") - spinner.text = "Complete!" - spinner.ok("✔") - c.out = "\n".join(output) + spinner.fail(to_native_string("Failed...cleaning up...")) + if not os.name == "nt": + spinner.ok(to_native_string("✔ Complete")) + else: + spinner.ok(to_native_string("Complete")) + c.out = "\n".join(output) if output else "" c.err = "\n".join(err) if err else "" else: c.out, c.err = c.communicate() + if not block: + c.wait() + c.out = to_text("{0}".format(c.out)) if c.out else fs_str("") + c.err = to_text("{0}".format(c.err)) if c.err else fs_str("") if not return_object: - if not block: - c.wait() - out = c.out if c.out else "" - err = c.err if c.err else "" - return out.strip(), err.strip() + return c.out.strip(), c.err.strip() return c @@ -198,7 +248,7 @@ def run( cwd=None, verbose=False, nospin=False, - spinner=None, + spinner_name=None, combine_stderr=True, display_limit=200 ): @@ -211,7 +261,7 @@ def run( :param str cwd: Current working directory contect to use for spawning the subprocess. :param bool verbose: Whether to print stdout in real time when non-blocking. :param bool nospin: Whether to disable the cli spinner. - :param str spinner: The name of the spinner to use if enabled, defaults to bouncingBar + :param str spinner_name: The name of the spinner to use if enabled, defaults to bouncingBar :param bool combine_stderr: Optionally merge stdout and stderr in the subprocess, false if nonblocking. :param int dispay_limit: The max width of output lines to display when using a spinner. :returns: A 2-tuple of (output, error) or a :class:`subprocess.Popen` object. @@ -221,8 +271,10 @@ def run( this functionality. """ - if not env: - env = os.environ.copy() + _env = os.environ.copy() + if env: + _env.update(env) + env = _env if six.PY2: fs_encode = partial(to_bytes, encoding=locale_encoding) _env = {fs_encode(k): fs_encode(v) for k, v in os.environ.items()} @@ -230,8 +282,8 @@ def run( _env[fs_encode(key)] = fs_encode(val) else: _env = {k: fs_str(v) for k, v in os.environ.items()} - if not spinner: - spinner = "bouncingBar" + if not spinner_name: + spinner_name = "bouncingBar" if six.PY2: if isinstance(cmd, six.string_types): cmd = cmd.encode("utf-8") @@ -241,48 +293,8 @@ def run( cmd = Script.parse(cmd) if block or not return_object: combine_stderr = False - sigmap = {} - if nospin is False: - try: - import signal - from yaspin import yaspin - from yaspin import spinners - from yaspin.signal_handlers import fancy_handler - except ImportError: - raise RuntimeError( - "Failed to import spinner! Reinstall vistir with command:" - " pip install --upgrade vistir[spinner]" - ) - else: - animation = getattr(spinners.Spinners, spinner) - sigmap = { - signal.SIGINT: fancy_handler - } - if os.name == "nt": - sigmap.update({ - signal.CTRL_C_EVENT: fancy_handler, - signal.CTRL_BREAK_EVENT: fancy_handler - }) - spinner_func = yaspin - else: - - @contextmanager - def spinner_func(spin_type, text, **kwargs): - class FakeClass(object): - def __init__(self, text=""): - self.text = text - - def ok(self, text): - return - - def write(self, text): - print(text) - - myobj = FakeClass(text) - yield myobj - - animation = None - with spinner_func(animation, sigmap=sigmap, text="Running...") as sp: + start_text = "" + with spinner(spinner_name=spinner_name, start_text=start_text, nospin=nospin) as sp: return _create_subprocess( cmd, env=_env, @@ -291,7 +303,8 @@ def run( cwd=cwd, verbose=verbose, spinner=sp, - combine_stderr=combine_stderr + combine_stderr=combine_stderr, + start_text=start_text ) @@ -426,7 +439,86 @@ def to_text(string, encoding="utf-8", errors=None): return string +def divide(n, iterable): + """ + split an iterable into n groups, per https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + :return: a list of new iterables derived from the original iterable + :rtype: list + """ + + seq = tuple(iterable) + q, r = divmod(len(seq), n) + + ret = [] + for i in range(n): + start = (i * q) + (i if i < r else r) + stop = ((i + 1) * q) + (i + 1 if i + 1 < r else r) + ret.append(iter(seq[start:stop])) + + return ret + + +def take(n, iterable): + """Take n elements from the supplied iterable without consuming it. + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + + from https://github.com/erikrose/more-itertools/blob/master/more_itertools/recipes.py + """ + + return list(islice(iterable, n)) + + +def chunked(n, iterable): + """Split an iterable into lists of length *n*. + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + + from https://github.com/erikrose/more-itertools/blob/master/more_itertools/more.py + """ + + return iter(partial(take, n, iter(iterable)), []) + + try: locale_encoding = locale.getdefaultencoding()[1] or "ascii" except Exception: locale_encoding = "ascii" + + +def getpreferredencoding(): + import locale + # Borrowed from Invoke + # (see https://github.com/pyinvoke/invoke/blob/93af29d/invoke/runners.py#L881) + _encoding = locale.getpreferredencoding(False) + if six.PY2 and not sys.platform == "win32": + _default_encoding = locale.getdefaultlocale()[1] + if _default_encoding is not None: + _encoding = _default_encoding + return _encoding + + +PREFERRED_ENCODING = getpreferredencoding() + + +def decode_for_output(output): + """Given a string, decode it for output to a terminal + + :param str output: A string to print to a terminal + :return: A re-encoded string using the preferred encoding + :rtype: str + """ + + if not isinstance(output, six.string_types): + return output + try: + output = output.encode(PREFERRED_ENCODING) + except AttributeError: + pass + output = output.decode(PREFERRED_ENCODING) + return output diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index 166282e8..ba008159 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -8,6 +8,7 @@ import os import posixpath import shutil import stat +import sys import warnings import six @@ -15,8 +16,15 @@ import six from six.moves import urllib_parse from six.moves.urllib import request as urllib_request -from .compat import Path, _fs_encoding, TemporaryDirectory -from .misc import locale_encoding, to_bytes, to_text +from .backports.tempfile import _TemporaryFileWrapper +from .compat import ( + _NamedTemporaryFile, + Path, + ResourceWarning, + TemporaryDirectory, + _fs_encoding, + finalize, +) __all__ = [ @@ -29,6 +37,7 @@ __all__ = [ "mkdir_p", "ensure_mkdir_p", "create_tracked_tempdir", + "create_tracked_tempfile", "path_to_url", "rmtree", "safe_expandvars", @@ -38,6 +47,10 @@ __all__ = [ ] +if os.name == "nt": + warnings.filterwarnings("ignore", category=DeprecationWarning, message="The Windows bytes API has been deprecated.*") + + def unicode_path(path): # Paths are supposed to be represented as unicode here if six.PY2 and not isinstance(path, six.text_type): @@ -53,9 +66,10 @@ def native_path(path): # once again thank you django... # https://github.com/django/django/blob/fc6b90b/django/utils/_os.py -if six.PY3 or os.name == 'nt': +if six.PY3 or os.name == "nt": abspathu = os.path.abspath else: + def abspathu(path): """ Version of os.path.abspath that uses the unicode representation @@ -74,6 +88,8 @@ def normalize_drive(path): identified with either upper or lower cased drive names. The case is always converted to uppercase because it seems to be preferred. """ + from .misc import to_text + if os.name != "nt" or not isinstance(path, six.string_types): return path @@ -95,6 +111,7 @@ def path_to_url(path): >>> path_to_url("/home/user/code/myrepo/myfile.zip") 'file:///home/user/code/myrepo/myfile.zip' """ + from .misc import to_text, to_bytes if not path: return path @@ -108,6 +125,8 @@ def url_to_path(url): Follows logic taken from pip's equivalent function """ + from .misc import to_bytes + assert is_file_url(url), "Only file: urls can be converted to local paths" _, netloc, path, _, _ = urllib_parse.urlsplit(url) # Netlocs are UNC paths @@ -120,14 +139,18 @@ def url_to_path(url): def is_valid_url(url): """Checks if a given string is an url""" + from .misc import to_text + if not url: return url - pieces = urllib_parse.urlparse(url) + pieces = urllib_parse.urlparse(to_text(url)) return all([pieces.scheme, pieces.netloc]) def is_file_url(url): """Returns true if the given url is a file url""" + from .misc import to_text + if not url: return False if not isinstance(url, six.string_types): @@ -144,9 +167,12 @@ def is_readonly_path(fn): Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)` """ - fn = to_bytes(fn, encoding="utf-8") + from .compat import to_native_string + + fn = to_native_string(fn) if os.path.exists(fn): - return bool(os.stat(fn).st_mode & stat.S_IREAD) and not os.access(fn, os.W_OK) + file_stat = os.stat(fn).st_mode + return not bool(file_stat & stat.S_IWRITE) or not os.access(fn, os.W_OK) return False @@ -158,7 +184,10 @@ def mkdir_p(newdir, mode=0o777): :raises: OSError if a file is encountered along the way """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ - newdir = abspathu(to_bytes(newdir, "utf-8")) + from .misc import to_text + from .compat import to_native_string + + newdir = to_native_string(newdir) if os.path.exists(newdir): if not os.path.isdir(newdir): raise OSError( @@ -166,17 +195,19 @@ def mkdir_p(newdir, mode=0o777): newdir ) ) - pass else: head, tail = os.path.split(newdir) # Make sure the tail doesn't point to the asame place as the head - tail_and_head_match = os.path.relpath(tail, start=os.path.basename(head)) == "." + curdir = to_native_string(".") + tail_and_head_match = ( + os.path.relpath(tail, start=os.path.basename(head)) == curdir + ) if tail and not tail_and_head_match and not os.path.isdir(newdir): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): raise OSError( "A file with the same name as the desired dir, '{0}', already exists.".format( - newdir + to_text(newdir, encoding="utf-8") ) ) os.makedirs(os.path.join(head, tail), mode) @@ -185,8 +216,8 @@ def mkdir_p(newdir, mode=0o777): def ensure_mkdir_p(mode=0o777): """Decorator to ensure `mkdir_p` is called to the function's return value. """ - def decorator(f): + def decorator(f): @functools.wraps(f) def decorated(*args, **kwargs): path = f(*args, **kwargs) @@ -210,12 +241,27 @@ def create_tracked_tempdir(*args, **kwargs): The return value is the path to the created directory. """ + tempdir = TemporaryDirectory(*args, **kwargs) TRACKED_TEMPORARY_DIRECTORIES.append(tempdir) atexit.register(tempdir.cleanup) + warnings.simplefilter("ignore", ResourceWarning) return tempdir.name +def create_tracked_tempfile(*args, **kwargs): + """Create a tracked temporary file. + + This uses the `NamedTemporaryFile` construct, but does not remove the file + until the interpreter exits. + + The return value is the file object. + """ + + kwargs["wrapper_class_override"] = _TrackedTempfileWrapper + return _NamedTemporaryFile(*args, **kwargs) + + def set_write_bit(fn): """Set read-write permissions for the current user on the target path. Fail silently if the path doesn't exist. @@ -223,10 +269,20 @@ def set_write_bit(fn): :param str fn: The target filename or path """ - fn = to_bytes(fn, encoding=locale_encoding) + from .compat import to_native_string + + fn = to_native_string(fn) if not os.path.exists(fn): return - os.chmod(fn, stat.S_IWRITE | stat.S_IWUSR | stat.S_IRUSR) + file_stat = os.stat(fn).st_mode + os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + if not os.path.isdir(fn): + return + for root, dirs, files in os.walk(fn, topdown=False): + for dir_ in [os.path.join(root,d) for d in dirs]: + set_write_bit(dir_) + for file_ in [os.path.join(root, f) for f in files]: + set_write_bit(file_) def rmtree(directory, ignore_errors=False): @@ -243,10 +299,18 @@ def rmtree(directory, ignore_errors=False): Setting `ignore_errors=True` may cause this to silently fail to delete the path """ - directory = to_bytes(directory, encoding=locale_encoding) - shutil.rmtree( - directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly - ) + from .compat import to_native_string + + directory = to_native_string(directory) + try: + shutil.rmtree( + directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly + ) + except (IOError, OSError, FileNotFoundError) as exc: + # Ignore removal failures where the file doesn't exist + if exc.errno == errno.ENOENT: + pass + raise def handle_remove_readonly(func, path, exc): @@ -263,35 +327,45 @@ def handle_remove_readonly(func, path, exc): :func:`set_write_bit` on the target path and try again. """ # Check for read-only attribute - from .compat import ResourceWarning + from .compat import ResourceWarning, FileNotFoundError, to_native_string + + PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT) default_warning_message = ( "Unable to remove file due to permissions restriction: {!r}" ) # split the initial exception out into its type, exception, and traceback exc_type, exc_exception, exc_tb = exc - path = to_bytes(path) + path = to_native_string(path) if is_readonly_path(path): # Apply write permission and call original function set_write_bit(path) try: func(path) - except (OSError, IOError) as e: - if e.errno in [errno.EACCES, errno.EPERM]: - warnings.warn( - default_warning_message.format( - to_text(path, encoding=locale_encoding) - ), ResourceWarning - ) + except (OSError, IOError, FileNotFoundError) as e: + if e.errno == errno.ENOENT: + return + elif e.errno in PERM_ERRORS: + warnings.warn(default_warning_message.format(path), ResourceWarning) return - if exc_exception.errno in [errno.EACCES, errno.EPERM]: - warnings.warn( - default_warning_message.format(to_text(path)), - ResourceWarning - ) - return - - raise + if exc_exception.errno in PERM_ERRORS: + set_write_bit(path) + try: + func(path) + except (OSError, IOError, FileNotFoundError) as e: + if e.errno in PERM_ERRORS: + warnings.warn(default_warning_message.format(path), ResourceWarning) + pass + elif e.errno == errno.ENOENT: # File already gone + pass + else: + raise + else: + return + elif exc_exception.errno == errno.ENOENT: + pass + else: + raise exc_exception def walk_up(bottom): @@ -356,6 +430,7 @@ def get_converted_relative_path(path, relative_to=None): >>> vistir.path.get_converted_relative_path('/home/user/code/myrepo/myfolder') '.' """ + from .misc import to_text, to_bytes # noqa if not relative_to: relative_to = os.getcwdu() if six.PY2 else os.getcwd() @@ -395,3 +470,28 @@ def safe_expandvars(value): if isinstance(value, six.string_types): return os.path.expandvars(value) return value + + +class _TrackedTempfileWrapper(_TemporaryFileWrapper, object): + def __init__(self, *args, **kwargs): + super(_TrackedTempfileWrapper, self).__init__(*args, **kwargs) + self._finalizer = finalize(self, self.cleanup) + + @classmethod + def _cleanup(cls, fileobj): + try: + fileobj.close() + finally: + os.unlink(fileobj.name) + + def cleanup(self): + if self._finalizer.detach(): + try: + self.close() + finally: + os.unlink(self.name) + else: + try: + self.close() + except OSError: + pass diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py new file mode 100644 index 00000000..20587d9d --- /dev/null +++ b/pipenv/vendor/vistir/spin.py @@ -0,0 +1,291 @@ +# -*- coding=utf-8 -*- + +import functools +import os +import signal +import sys + +import colorama +import cursor +import six + +from .compat import to_native_string +from .termcolors import COLOR_MAP, COLORS, colored +from io import StringIO + +try: + import yaspin +except ImportError: + yaspin = None + Spinners = None +else: + from yaspin.spinners import Spinners + +handler = None +if yaspin and os.name == "nt": + handler = yaspin.signal_handlers.default_handler +elif yaspin and os.name != "nt": + handler = yaspin.signal_handlers.fancy_handler + +CLEAR_LINE = chr(27) + "[K" + + +class DummySpinner(object): + def __init__(self, text="", **kwargs): + colorama.init() + from .misc import decode_for_output + self.text = to_native_string(decode_for_output(text)) + self.stdout = kwargs.get("stdout", sys.stdout) + self.stderr = kwargs.get("stderr", sys.stderr) + self.out_buff = StringIO() + + def __enter__(self): + if self.text and self.text != "None": + self.write_err(self.text) + return self + + def __exit__(self, exc_type, exc_val, traceback): + if exc_type: + import traceback + from .misc import decode_for_output + self.write_err(decode_for_output(traceback.format_exception(*sys.exc_info()))) + self._close_output_buffer() + return False + + def __getattr__(self, k): + try: + retval = super(DummySpinner, self).__getattribute__(k) + except AttributeError: + if k in COLOR_MAP.keys() or k.upper() in COLORS: + return self + raise + else: + return retval + + def _close_output_buffer(self): + if self.out_buff and not self.out_buff.closed: + try: + self.out_buff.close() + except Exception: + pass + + def fail(self, exitcode=1, text="FAIL"): + from .misc import decode_for_output + if text and text != "None": + self.write_err(decode_for_output(text)) + self._close_output_buffer() + + def ok(self, text="OK"): + if text and text != "None": + self.stderr.write(self.text) + self._close_output_buffer() + return 0 + + def write(self, text=None): + from .misc import decode_for_output + if text is None or isinstance(text, six.string_types) and text == "None": + pass + text = decode_for_output(text) + self.stdout.write(decode_for_output("\r")) + line = decode_for_output("{0}\n".format(text)) + self.stdout.write(line) + self.stdout.write(CLEAR_LINE) + + def write_err(self, text=None): + from .misc import decode_for_output + if text is None or isinstance(text, six.string_types) and text == "None": + pass + text = decode_for_output(text) + self.stderr.write(decode_for_output("\r")) + line = decode_for_output("{0}\n".format(text)) + self.stderr.write(line) + self.stderr.write(CLEAR_LINE) + + @staticmethod + def _hide_cursor(): + pass + + @staticmethod + def _show_cursor(): + pass + + +base_obj = yaspin.core.Yaspin if yaspin is not None else DummySpinner + + +class VistirSpinner(base_obj): + def __init__(self, *args, **kwargs): + """Get a spinner object or a dummy spinner to wrap a context. + + Keyword Arguments: + :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) + :param str start_text: Text to start off the spinner with (default: {None}) + :param dict handler_map: Handler map for signals to be handled gracefully (default: {None}) + :param bool nospin: If true, use the dummy spinner (default: {False}) + """ + + self.handler = handler + colorama.init() + sigmap = {} + if handler: + sigmap.update({ + signal.SIGINT: handler, + signal.SIGTERM: handler + }) + handler_map = kwargs.pop("handler_map", {}) + if os.name == "nt": + sigmap[signal.SIGBREAK] = handler + else: + sigmap[signal.SIGALRM] = handler + if handler_map: + sigmap.update(handler_map) + spinner_name = kwargs.pop("spinner_name", "bouncingBar") + start_text = kwargs.pop("start_text", None) + _text = kwargs.pop("text", "Running...") + kwargs["text"] = start_text if start_text is not None else _text + kwargs["sigmap"] = sigmap + kwargs["spinner"] = getattr(Spinners, spinner_name, "") + self.stdout = kwargs.pop("stdout", sys.stdout) + self.stderr = kwargs.pop("stderr", sys.stderr) + self.out_buff = StringIO() + super(VistirSpinner, self).__init__(*args, **kwargs) + self.is_dummy = bool(yaspin is None) + + def ok(self, text="OK"): + """Set Ok (success) finalizer to a spinner.""" + _text = text if text else "OK" + self._freeze(_text) + + def fail(self, text="FAIL"): + """Set fail finalizer to a spinner.""" + _text = text if text else "FAIL" + self._freeze(_text) + + def write(self, text): + from .misc import to_text + sys.stdout.write("\r") + self.stdout.write(CLEAR_LINE) + if text is None: + text = "" + text = to_native_string("{0}\n".format(text)) + sys.stdout.write(text) + self.out_buff.write(to_text(text)) + + def write_err(self, text): + """Write error text in the terminal without breaking the spinner.""" + from .misc import to_text + + self.stderr.write("\r") + self.stderr.write(CLEAR_LINE) + if text is None: + text = "" + text = to_native_string("{0}\n".format(text)) + self.stderr.write(text) + self.out_buff.write(to_text(text)) + + def _freeze(self, final_text): + """Stop spinner, compose last frame and 'freeze' it.""" + if not final_text: + final_text = "" + text = to_native_string(final_text) + self._last_frame = self._compose_out(text, mode="last") + + # Should be stopped here, otherwise prints after + # self._freeze call will mess up the spinner + self.stop() + self.stdout.write(self._last_frame) + + def stop(self, *args, **kwargs): + if self.stderr and self.stderr != sys.stderr: + self.stderr.close() + if self.stdout and self.stdout != sys.stdout: + self.stdout.close() + self.out_buff.close() + super(VistirSpinner, self).stop(*args, **kwargs) + + def _compose_color_func(self): + fn = functools.partial( + colored, + color=self._color, + on_color=self._on_color, + attrs=list(self._attrs), + ) + return fn + + def _compose_out(self, frame, mode=None): + # Ensure Unicode input + + frame = to_native_string(frame) + if self._text is None: + self._text = "" + text = to_native_string(self._text) + if self._color_func is not None: + frame = self._color_func(frame) + if self._side == "right": + frame, text = text, frame + # Mode + if not mode: + out = to_native_string("\r{0} {1}".format(frame, text)) + else: + out = to_native_string("{0} {1}\n".format(frame, text)) + return out + + def _register_signal_handlers(self): + # SIGKILL cannot be caught or ignored, and the receiving + # process cannot perform any clean-up upon receiving this + # signal. + try: + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) + except AttributeError: + pass + + for sig, sig_handler in self._sigmap.items(): + # A handler for a particular signal, once set, remains + # installed until it is explicitly reset. Store default + # signal handlers for subsequent reset at cleanup phase. + dfl_handler = signal.getsignal(sig) + self._dfl_sigmap[sig] = dfl_handler + + # ``signal.SIG_DFL`` and ``signal.SIG_IGN`` are also valid + # signal handlers and are not callables. + if callable(sig_handler): + # ``signal.signal`` accepts handler function which is + # called with two arguments: signal number and the + # interrupted stack frame. ``functools.partial`` solves + # the problem of passing spinner instance into the handler + # function. + sig_handler = functools.partial(sig_handler, spinner=self) + + signal.signal(sig, sig_handler) + + def _reset_signal_handlers(self): + for sig, sig_handler in self._dfl_sigmap.items(): + signal.signal(sig, sig_handler) + + @staticmethod + def _hide_cursor(): + cursor.hide() + + @staticmethod + def _show_cursor(): + cursor.show() + + @staticmethod + def _clear_err(): + sys.stderr.write(CLEAR_LINE) + + @staticmethod + def _clear_line(): + sys.stdout.write(CLEAR_LINE) + + +def create_spinner(*args, **kwargs): + nospin = kwargs.pop("nospin", False) + use_yaspin = kwargs.pop("use_yaspin", nospin) + if nospin: + return DummySpinner(*args, **kwargs) + return VistirSpinner(*args, **kwargs) diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py new file mode 100644 index 00000000..8395d97d --- /dev/null +++ b/pipenv/vendor/vistir/termcolors.py @@ -0,0 +1,146 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals +import colorama +import os +from .compat import to_native_string + + +ATTRIBUTES = dict( + list(zip([ + 'bold', + 'dark', + '', + 'underline', + 'blink', + '', + 'reverse', + 'concealed' + ], + list(range(1, 9)) + )) + ) +del ATTRIBUTES[''] + + +HIGHLIGHTS = dict( + list(zip([ + 'on_grey', + 'on_red', + 'on_green', + 'on_yellow', + 'on_blue', + 'on_magenta', + 'on_cyan', + 'on_white' + ], + list(range(40, 48)) + )) + ) + + +COLORS = dict( + list(zip([ + 'grey', + 'red', + 'green', + 'yellow', + 'blue', + 'magenta', + 'cyan', + 'white', + ], + list(range(30, 38)) + )) + ) + + +COLOR_MAP = { + # name: type + "blink": "attrs", + "bold": "attrs", + "concealed": "attrs", + "dark": "attrs", + "reverse": "attrs", + "underline": "attrs", + "blue": "color", + "cyan": "color", + "green": "color", + "magenta": "color", + "red": "color", + "white": "color", + "yellow": "color", + "on_blue": "on_color", + "on_cyan": "on_color", + "on_green": "on_color", + "on_grey": "on_color", + "on_magenta": "on_color", + "on_red": "on_color", + "on_white": "on_color", + "on_yellow": "on_color", +} +COLOR_ATTRS = COLOR_MAP.keys() + + +RESET = colorama.Style.RESET_ALL + + +def colored(text, color=None, on_color=None, attrs=None): + """Colorize text using a reimplementation of the colorizer from + https://github.com/pavdmyt/yaspin so that it works on windows. + + Available text colors: + red, green, yellow, blue, magenta, cyan, white. + + Available text highlights: + on_red, on_green, on_yellow, on_blue, on_magenta, on_cyan, on_white. + + Available attributes: + bold, dark, underline, blink, reverse, concealed. + + Example: + colored('Hello, World!', 'red', 'on_grey', ['blue', 'blink']) + colored('Hello, World!', 'green') + """ + if os.getenv('ANSI_COLORS_DISABLED') is None: + style = "NORMAL" + if 'bold' in attrs: + style = "BRIGHT" + attrs.remove('bold') + if color is not None: + color = color.upper() + text = to_native_string("%s%s%s%s%s") % ( + to_native_string(getattr(colorama.Fore, color)), + to_native_string(getattr(colorama.Style, style)), + to_native_string(text), + to_native_string(colorama.Fore.RESET), + to_native_string(colorama.Style.NORMAL), + ) + + if on_color is not None: + on_color = on_color.upper() + text = to_native_string("%s%s%s%s") % ( + to_native_string(getattr(colorama.Back, on_color)), + to_native_string(text), + to_native_string(colorama.Back.RESET), + to_native_string(colorama.Style.NORMAL), + ) + + if attrs is not None: + fmt_str = to_native_string("%s[%%dm%%s%s[9m") % ( + chr(27), + chr(27) + ) + for attr in attrs: + text = fmt_str % (ATTRIBUTES[attr], text) + + text += RESET + return text + + +def cprint(text, color=None, on_color=None, attrs=None, **kwargs): + """Print colorize text. + + It accepts arguments of print function. + """ + + print((colored(text, color, on_color, attrs)), **kwargs) diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index d01fb98e..06b8b621 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -16,6 +16,9 @@ import sys import threading import time +import colorama +import cursor + from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS @@ -23,6 +26,9 @@ from .helpers import to_unicode from .termcolor import colored +colorama.init() + + class Yaspin(object): """Implements a context manager that spawns a thread to write spinner frames into a tty (stdout) during @@ -369,11 +375,14 @@ class Yaspin(object): # SIGKILL cannot be caught or ignored, and the receiving # process cannot perform any clean-up upon receiving this # signal. - if signal.SIGKILL in self._sigmap.keys(): - raise ValueError( - "Trying to set handler for SIGKILL signal. " - "SIGKILL cannot be cought or ignored in POSIX systems." - ) + try: + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) + except AttributeError: + pass for sig, sig_handler in iteritems(self._sigmap): # A handler for a particular signal, once set, remains @@ -521,14 +530,12 @@ class Yaspin(object): @staticmethod def _hide_cursor(): - sys.stdout.write("\033[?25l") - sys.stdout.flush() + cursor.hide() @staticmethod def _show_cursor(): - sys.stdout.write("\033[?25h") - sys.stdout.flush() + cursor.show() @staticmethod def _clear_line(): - sys.stdout.write("\033[K") + sys.stdout.write(chr(27) + "[K") diff --git a/pytest.ini b/pytest.ini index 48dfab02..2bfca079 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ [pytest] addopts = -ra -n auto -testpaths = tests/ +testpaths = tests ; Add vendor and patched in addition to the default list of ignored dirs norecursedirs = .* build dist CVS _darcs {arch} *.egg vendor patched news tasks docs filterwarnings = diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 3198c2d4..ea0d038c 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -20,6 +20,7 @@ LIBRARY_DIRNAMES = { 'requirements-parser': 'requirements', 'backports.shutil_get_terminal_size': 'backports/shutil_get_terminal_size', 'backports.weakref': 'backports/weakref', + 'backports.functools_lru_cache': 'backports/functools_lru_cache', 'shutil_backports': 'backports/shutil_get_terminal_size', 'python-dotenv': 'dotenv', 'pip-tools': 'piptools', @@ -34,6 +35,7 @@ PY2_DOWNLOAD = ['enum34',] # from time to time, remove the no longer needed ones HARDCODED_LICENSE_URLS = { 'pytoml': 'https://github.com/avakar/pytoml/raw/master/LICENSE', + 'cursor': 'https://raw.githubusercontent.com/GijsTimmers/cursor/master/LICENSE', 'delegator.py': 'https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE', 'click-didyoumean': 'https://raw.githubusercontent.com/click-contrib/click-didyoumean/master/LICENSE', 'click-completion': 'https://raw.githubusercontent.com/click-contrib/click-completion/master/LICENSE', @@ -48,7 +50,9 @@ HARDCODED_LICENSE_URLS = { 'master/LICENSE', 'requirementslib': 'https://github.com/techalchemy/requirementslib/raw/master/LICENSE', 'distlib': 'https://github.com/vsajip/distlib/raw/master/LICENSE.txt', - 'pythonfinder': 'https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt' + 'pythonfinder': 'https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt', + 'pyparsing': 'https://raw.githubusercontent.com/pyparsing/pyparsing/master/LICENSE', + 'resolvelib': 'https://raw.githubusercontent.com/sarugaku/resolvelib/master/LICENSE' } FILE_WHITE_LIST = ( @@ -70,13 +74,14 @@ PATCHED_RENAMES = { LIBRARY_RENAMES = { 'pip': 'pipenv.patched.notpip', + "functools32": "pipenv.vendor.backports.functools_lru_cache", 'enum34': 'enum', } def drop_dir(path): if path.exists() and path.is_dir(): - shutil.rmtree(str(path)) + shutil.rmtree(str(path), ignore_errors=True) def remove_all(paths): @@ -110,8 +115,6 @@ def clean_vendor(ctx, vendor_dir): for item in vendor_dir.iterdir(): if item.is_dir(): shutil.rmtree(str(item)) - elif "LICENSE" in item.name or "COPYING" in item.name: - continue elif item.name not in FILE_WHITE_LIST: item.unlink() else: @@ -330,6 +333,7 @@ def post_install_cleanup(ctx, vendor_dir): # Cleanup setuptools unneeded parts drop_dir(vendor_dir / 'bin') drop_dir(vendor_dir / 'tests') + remove_all(vendor_dir.glob('toml.py')) def vendor(ctx, vendor_dir, package=None, rewrite=True): @@ -353,6 +357,10 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): log("Removing scandir library files...") remove_all(vendor_dir.glob('*.so')) + drop_dir(vendor_dir / 'setuptools') + drop_dir(vendor_dir / 'pkg_resources' / '_vendor') + drop_dir(vendor_dir / 'pkg_resources' / 'extern') + drop_dir(vendor_dir / 'bin') # Global import rewrites log('Renaming specified libs...') diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch index 654a3da9..fce6ae89 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch @@ -1,20 +1,8 @@ diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py -index c9804ab5..89c95c45 100644 +index f3b9b5b4..182c1c88 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py -@@ -10,10 +10,7 @@ import sysconfig - import warnings - from collections import OrderedDict - --try: -- import pipenv.patched.notpip._internal.utils.glibc --except ImportError: -- import pipenv.patched.notpip.utils.glibc -+import pipenv.patched.notpip._internal.utils.glibc - - logger = logging.getLogger(__name__) - -@@ -157,7 +154,7 @@ def is_manylinux1_compatible(): +@@ -159,7 +159,7 @@ def is_manylinux1_compatible(): pass # Check glibc version. CentOS 5 uses glibc 2.5. diff --git a/tasks/vendoring/patches/patched/pip18.patch b/tasks/vendoring/patches/patched/pip18.patch index 558c28e8..f4e607c1 100644 --- a/tasks/vendoring/patches/patched/pip18.patch +++ b/tasks/vendoring/patches/patched/pip18.patch @@ -1,5 +1,5 @@ diff --git a/pipenv/patched/pip/_internal/download.py b/pipenv/patched/pip/_internal/download.py -index 96f3b65c..3fb4ebef 100644 +index 96f3b65c..cc5b3d15 100644 --- a/pipenv/patched/pip/_internal/download.py +++ b/pipenv/patched/pip/_internal/download.py @@ -19,6 +19,7 @@ from pip._vendor.lockfile import LockError @@ -19,43 +19,78 @@ index 96f3b65c..3fb4ebef 100644 "python": platform.python_version(), "implementation": { "name": platform.python_implementation(), -@@ -322,7 +323,7 @@ class InsecureHTTPAdapter(HTTPAdapter): - conn.ca_certs = None +diff --git a/pipenv/patched/pip/_internal/utils/temp_dir.py b/pipenv/patched/pip/_internal/utils/temp_dir.py +index edc506bf..84d57dac 100644 +--- a/pipenv/patched/pip/_internal/utils/temp_dir.py ++++ b/pipenv/patched/pip/_internal/utils/temp_dir.py +@@ -3,8 +3,10 @@ from __future__ import absolute_import + import logging + import os.path + import tempfile ++import warnings + from pip._internal.utils.misc import rmtree ++from pipenv.vendor.vistir.compat import finalize, ResourceWarning --class PipSession(requests.Session): -+class PipSession(Session): + logger = logging.getLogger(__name__) - timeout = None - -@@ -752,7 +753,7 @@ def _copy_dist_from_dir(link_path, location): - - # build an sdist - setup_py = 'setup.py' -- sdist_args = [sys.executable] -+ sdist_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable)] - sdist_args.append('-c') - sdist_args.append(SETUPTOOLS_SHIM % setup_py) - sdist_args.append('sdist') -diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py -index 8c0ec82c..ad00ba04 100644 ---- a/pipenv/patched/pip/_internal/index.py -+++ b/pipenv/patched/pip/_internal/index.py -@@ -58,11 +58,12 @@ logger = logging.getLogger(__name__) - - class InstallationCandidate(object): - -- def __init__(self, project, version, location): -+ def __init__(self, project, version, location, requires_python=None): - self.project = project - self.version = parse_version(version) - self.location = location - self._key = (self.project, self.version, self.location) -+ self.requires_python = requires_python +@@ -45,6 +47,20 @@ class TempDirectory(object): + self.path = path + self.delete = delete + self.kind = kind ++ self._finalizer = None ++ if path: ++ self._register_finalizer() ++ ++ def _register_finalizer(self): ++ if self.delete and self.path: ++ self._finalizer = finalize( ++ self, ++ self._cleanup, ++ self.path, ++ warn_message=None ++ ) ++ else: ++ self._finalizer = None def __repr__(self): - return "".format( -@@ -168,6 +169,9 @@ class PackageFinder(object): + return "<{} {!r}>".format(self.__class__.__name__, self.path) +@@ -72,11 +88,27 @@ class TempDirectory(object): + self.path = os.path.realpath( + tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) + ) ++ self._register_finalizer() + logger.debug("Created temporary directory: {}".format(self.path)) + ++ @classmethod ++ def _cleanup(cls, name, warn_message=None): ++ try: ++ rmtree(name) ++ except OSError: ++ pass ++ else: ++ if warn_message: ++ warnings.warn(warn_message, ResourceWarning) ++ + def cleanup(self): + """Remove the temporary directory created and reset state + """ +- if self.path is not None and os.path.exists(self.path): +- rmtree(self.path) +- self.path = None ++ if getattr(self._finalizer, "detach", None) and self._finalizer.detach(): ++ if os.path.exists(self.path): ++ try: ++ rmtree(self.path) ++ except OSError: ++ pass ++ else: ++ self.path = None +diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py +index 8c2f24f1..cdd48874 100644 +--- a/pipenv/patched/pip/_internal/index.py ++++ b/pipenv/patched/pip/_internal/index.py +@@ -246,6 +246,9 @@ class PackageFinder(object): # The Session we'll use to make requests self.session = session @@ -65,7 +100,7 @@ index 8c0ec82c..ad00ba04 100644 # The valid tags to check potential found wheel candidates against self.valid_tags = get_supported( versions=versions, -@@ -220,6 +224,24 @@ class PackageFinder(object): +@@ -298,6 +301,25 @@ class PackageFinder(object): ) self.dependency_links.extend(links) @@ -86,11 +121,12 @@ index 8c0ec82c..ad00ba04 100644 + current_list.append(link) + + return extras ++ + @staticmethod def _sort_locations(locations, expand_dir=False): """ -@@ -272,7 +294,7 @@ class PackageFinder(object): +@@ -350,7 +372,7 @@ class PackageFinder(object): return files, urls @@ -99,7 +135,7 @@ index 8c0ec82c..ad00ba04 100644 """ Function used to generate link sort key for link tuples. The greater the return value, the more preferred it is. -@@ -292,14 +314,19 @@ class PackageFinder(object): +@@ -370,14 +392,19 @@ class PackageFinder(object): if candidate.location.is_wheel: # can raise InvalidWheelFilename wheel = Wheel(candidate.location.filename) @@ -121,7 +157,19 @@ index 8c0ec82c..ad00ba04 100644 if wheel.build_tag is not None: match = re.match(r'^(\d+)(.*)$', wheel.build_tag) build_tag_groups = match.groups() -@@ -484,7 +511,7 @@ class PackageFinder(object): +@@ -528,7 +555,10 @@ class PackageFinder(object): + + page_versions = [] + for page in self._get_pages(url_locations, project_name): +- logger.debug('Analyzing links from page %s', page.url) ++ try: ++ logger.debug('Analyzing links from page %s', page.url) ++ except AttributeError: ++ continue + with indent_log(): + page_versions.extend( + self._package_versions(page.iter_links(), search) +@@ -562,7 +592,7 @@ class PackageFinder(object): dependency_versions ) @@ -130,19 +178,19 @@ index 8c0ec82c..ad00ba04 100644 """Try to find a Link matching req Expects req, an InstallRequirement and upgrade, a boolean -@@ -594,8 +621,9 @@ class PackageFinder(object): +@@ -672,7 +702,10 @@ class PackageFinder(object): continue seen.add(location) - page = self._get_page(location) -- if page is None: + try: + page = self._get_page(location) -+ except requests.HTTPError as e: ++ except requests.HTTPError: ++ continue + if page is None: continue - yield page -@@ -631,7 +659,7 @@ class PackageFinder(object): +@@ -709,7 +742,7 @@ class PackageFinder(object): logger.debug('Skipping link %s; %s', link, reason) self.logged_links.add(link) @@ -151,7 +199,7 @@ index 8c0ec82c..ad00ba04 100644 """Return an InstallationCandidate or None""" version = None if link.egg_fragment: -@@ -647,12 +675,12 @@ class PackageFinder(object): +@@ -725,12 +758,12 @@ class PackageFinder(object): link, 'unsupported archive format: %s' % ext, ) return @@ -166,7 +214,7 @@ index 8c0ec82c..ad00ba04 100644 self._log_skipped_link(link, 'macosx10 one') return if ext == wheel_ext: -@@ -666,7 +694,7 @@ class PackageFinder(object): +@@ -744,7 +777,7 @@ class PackageFinder(object): link, 'wrong project name (not %s)' % search.supplied) return @@ -175,7 +223,7 @@ index 8c0ec82c..ad00ba04 100644 self._log_skipped_link( link, 'it is not compatible with this Python') return -@@ -702,14 +730,14 @@ class PackageFinder(object): +@@ -780,14 +813,14 @@ class PackageFinder(object): link.filename, link.requires_python) support_this_python = True @@ -191,13 +239,30 @@ index 8c0ec82c..ad00ba04 100644 + return InstallationCandidate(search.supplied, version, link, link.requires_python) def _get_page(self, link): - return HTMLPage.get_page(link, session=self.session) + return _get_html_page(link, session=self.session) +diff --git a/pipenv/patched/pip/_internal/models/candidate.py b/pipenv/patched/pip/_internal/models/candidate.py +index c736de6c..a78566c1 100644 +--- a/pipenv/patched/pip/_internal/models/candidate.py ++++ b/pipenv/patched/pip/_internal/models/candidate.py +@@ -7,10 +7,11 @@ class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + +- def __init__(self, project, version, location): ++ def __init__(self, project, version, location, requires_python=None): + self.project = project + self.version = parse_version(version) + self.location = location ++ self.requires_python = requires_python + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.location), diff --git a/pipenv/patched/pip/_internal/operations/prepare.py b/pipenv/patched/pip/_internal/operations/prepare.py -index 7740c284..b6e946d8 100644 +index 104bea33..ecf78b9a 100644 --- a/pipenv/patched/pip/_internal/operations/prepare.py +++ b/pipenv/patched/pip/_internal/operations/prepare.py @@ -17,7 +17,7 @@ from pip._internal.exceptions import ( - ) + from pip._internal.utils.compat import expanduser from pip._internal.utils.hashes import MissingHashes from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import display_path, normalize_path @@ -206,8 +271,8 @@ index 7740c284..b6e946d8 100644 logger = logging.getLogger(__name__) @@ -123,7 +123,11 @@ class IsSDist(DistAbstraction): - "Installing build dependencies" - ) + " and ".join(map(repr, sorted(missing))) + ) - self.req.run_egg_info() + try: @@ -237,10 +302,10 @@ index 7740c284..b6e946d8 100644 # We can't hit this spot and have populate_link return None. diff --git a/pipenv/patched/pip/_internal/pep425tags.py b/pipenv/patched/pip/_internal/pep425tags.py -index 0b5c7832..bea31585 100644 +index ab1a0298..763c0a24 100644 --- a/pipenv/patched/pip/_internal/pep425tags.py +++ b/pipenv/patched/pip/_internal/pep425tags.py -@@ -10,7 +10,10 @@ import sysconfig +@@ -10,7 +10,11 @@ import sysconfig import warnings from collections import OrderedDict @@ -249,81 +314,80 @@ index 0b5c7832..bea31585 100644 + import pip._internal.utils.glibc +except ImportError: + import pip.utils.glibc ++ + from pip._internal.utils.compat import get_extension_suffixes logger = logging.getLogger(__name__) - diff --git a/pipenv/patched/pip/_internal/req/req_install.py b/pipenv/patched/pip/_internal/req/req_install.py -index 462c80aa..d039adc8 100644 +index c2624fee..ee75acd6 100644 --- a/pipenv/patched/pip/_internal/req/req_install.py +++ b/pipenv/patched/pip/_internal/req/req_install.py -@@ -615,7 +615,7 @@ class InstallRequirement(object): +@@ -452,7 +452,8 @@ class InstallRequirement(object): with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py - base_cmd = [sys.executable, '-c', script] -+ base_cmd = [os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', script] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ base_cmd = [sys_executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] -@@ -797,7 +797,7 @@ class InstallRequirement(object): +@@ -613,10 +614,11 @@ class InstallRequirement(object): + + with indent_log(): + # FIXME: should we do --install-headers here too? ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) with self.build_env: call_subprocess( [ - sys.executable, -+ os.environ.get('PIP_PYTHON_PATH', sys.executable), ++ sys_executable, '-c', SETUPTOOLS_SHIM % self.setup_py ] + -@@ -1015,7 +1015,7 @@ class InstallRequirement(object): +@@ -834,7 +836,8 @@ class InstallRequirement(object): def get_install_args(self, global_options, record_filename, root, prefix, pycompile): - install_args = [sys.executable, "-u"] -+ install_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable), "-u"] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ install_args = [sys_executable, "-u"] install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ diff --git a/pipenv/patched/pip/_internal/req/req_set.py b/pipenv/patched/pip/_internal/req/req_set.py -index 2bc6b745..e552afc1 100644 +index b1983171..0bab231d 100644 --- a/pipenv/patched/pip/_internal/req/req_set.py +++ b/pipenv/patched/pip/_internal/req/req_set.py -@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) +@@ -12,13 +12,16 @@ logger = logging.getLogger(__name__) class RequirementSet(object): -- def __init__(self, require_hashes=False): -+ def __init__(self, require_hashes=False, ignore_compatibility=True): +- def __init__(self, require_hashes=False, check_supported_wheels=True): ++ def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): """Create a RequirementSet. """ -@@ -24,6 +24,7 @@ class RequirementSet(object): - self.unnamed_requirements = [] - self.successfully_downloaded = [] - self.reqs_to_cleanup = [] -+ self.ignore_compatibility = ignore_compatibility + self.requirements = OrderedDict() + self.require_hashes = require_hashes + self.check_supported_wheels = check_supported_wheels ++ if ignore_compatibility: ++ self.check_supported_wheels = False ++ self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False - def __str__(self): - reqs = [req for req in self.requirements.values() -@@ -65,7 +66,7 @@ class RequirementSet(object): - # environment markers. - if install_req.link and install_req.link.is_wheel: - wheel = Wheel(install_req.link.filename) -- if not wheel.supported(): -+ if not wheel.supported() and not self.ignore_compatibility: - raise InstallationError( - "%s is not a supported wheel on this platform." % - wheel.filename -@@ -151,7 +152,7 @@ class RequirementSet(object): + # Mapping of alias: real_name + self.requirement_aliases = {} +@@ -171,7 +174,7 @@ class RequirementSet(object): return self.requirements[name] if name in self.requirement_aliases: return self.requirements[self.requirement_aliases[name]] - raise KeyError("No project with the name %r" % project_name) -+ # raise KeyError("No project with the name %r" % project_name) ++ pass def cleanup_files(self): """Clean up files, remove builds.""" diff --git a/pipenv/patched/pip/_internal/resolve.py b/pipenv/patched/pip/_internal/resolve.py -index 8480e48c..ffc4aa7d 100644 +index 2d9f1c56..bedc2582 100644 --- a/pipenv/patched/pip/_internal/resolve.py +++ b/pipenv/patched/pip/_internal/resolve.py @@ -35,7 +35,7 @@ class Resolver(object): @@ -356,22 +420,12 @@ index 8480e48c..ffc4aa7d 100644 """Prepare a single requirements file. :return: A list of additional InstallRequirements to also install. -@@ -245,6 +249,9 @@ class Resolver(object): - # Tell user what we are doing for this requirement: - # obtain (editable), skipping, processing (local url), collecting - # (remote url or package name) -+ if ignore_requires_python or self.ignore_requires_python: -+ self.ignore_compatibility = True -+ - if req_to_install.constraint or req_to_install.prepared: - return [] - -@@ -260,11 +267,17 @@ class Resolver(object): +@@ -260,11 +264,17 @@ class Resolver(object): try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: - if self.ignore_requires_python: -+ if self.ignore_compatibility: ++ if self.ignore_requires_python or self.ignore_compatibility: logger.warning(err.args[0]) else: raise @@ -385,14 +439,13 @@ index 8480e48c..ffc4aa7d 100644 more_reqs = [] def add_req(subreq, extras_requested): -@@ -290,10 +303,14 @@ class Resolver(object): - # We add req_to_install before its dependencies, so that we +@@ -291,9 +301,13 @@ class Resolver(object): # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): + # 'unnamed' requirements will get added here + available_requested = sorted( + set(dist.extras) & set(req_to_install.extras) + ) - # 'unnamed' requirements will get added here req_to_install.is_direct = True requirement_set.add_requirement( req_to_install, parent_req_name=None, @@ -400,7 +453,7 @@ index 8480e48c..ffc4aa7d 100644 ) if not self.ignore_dependencies: -@@ -317,6 +334,19 @@ class Resolver(object): +@@ -317,6 +331,19 @@ class Resolver(object): for subreq in dist.requires(available_requested): add_req(subreq, extras_requested=available_requested) @@ -408,7 +461,7 @@ index 8480e48c..ffc4aa7d 100644 + for available in available_requested: + if hasattr(dist, '_DistInfoDistribution__dep_map'): + for req in dist._DistInfoDistribution__dep_map[available]: -+ req = InstallRequirement.from_req( ++ req = install_req_from_req( + str(req), + req_to_install, + isolated=self.isolated, @@ -420,24 +473,11 @@ index 8480e48c..ffc4aa7d 100644 if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took -diff --git a/pipenv/patched/pip/_internal/utils/misc.py b/pipenv/patched/pip/_internal/utils/misc.py -index 3236af63..439a831d 100644 ---- a/pipenv/patched/pip/_internal/utils/misc.py -+++ b/pipenv/patched/pip/_internal/utils/misc.py -@@ -96,7 +96,7 @@ def get_prog(): - try: - prog = os.path.basename(sys.argv[0]) - if prog in ('__main__.py', '-c'): -- return "%s -m pip" % sys.executable -+ return "%s -m pip" % os.environ.get('PIP_PYTHON_PATH', sys.executable) - else: - return prog - except (AttributeError, TypeError, IndexError): diff --git a/pipenv/patched/pip/_internal/utils/packaging.py b/pipenv/patched/pip/_internal/utils/packaging.py -index 5f9bb93d..276a9ccc 100644 +index c43142f0..f241cce0 100644 --- a/pipenv/patched/pip/_internal/utils/packaging.py +++ b/pipenv/patched/pip/_internal/utils/packaging.py -@@ -28,7 +28,7 @@ def check_requires_python(requires_python): +@@ -29,7 +29,7 @@ def check_requires_python(requires_python): requires_python_specifier = specifiers.SpecifierSet(requires_python) # We only use major.minor.micro @@ -446,33 +486,21 @@ index 5f9bb93d..276a9ccc 100644 return python_version in requires_python_specifier -@@ -40,20 +40,17 @@ def get_metadata(dist): - return dist.get_metadata('PKG-INFO') +@@ -48,9 +48,11 @@ def get_metadata(dist): + return feed_parser.close() -def check_dist_requires_python(dist): +def check_dist_requires_python(dist, absorb=True): - metadata = get_metadata(dist) - feed_parser = FeedParser() - feed_parser.feed(metadata) - pkg_info_dict = feed_parser.close() + pkg_info_dict = get_metadata(dist) requires_python = pkg_info_dict.get('Requires-Python') -+ if not absorb: ++ if absorb: + return requires_python try: if not check_requires_python(requires_python): -- raise exceptions.UnsupportedPythonVersion( -- "%s requires Python '%s' but the running Python is %s" % ( -- dist.project_name, -- requires_python, -- '.'.join(map(str, sys.version_info[:3])),) -- ) -+ return requires_python - except specifiers.InvalidSpecifier as e: - logger.warning( - "Package %s has an invalid Requires-Python entry %s - %s", + raise exceptions.UnsupportedPythonVersion( diff --git a/pipenv/patched/pip/_internal/wheel.py b/pipenv/patched/pip/_internal/wheel.py -index fcf9d3d3..d8aff848 100644 +index 5ce890eb..46c0181c 100644 --- a/pipenv/patched/pip/_internal/wheel.py +++ b/pipenv/patched/pip/_internal/wheel.py @@ -83,7 +83,7 @@ def fix_script(path): @@ -484,28 +512,24 @@ index fcf9d3d3..d8aff848 100644 firstline = b'#!' + exename + os.linesep.encode("ascii") rest = script.read() with open(path, 'wb') as script: -@@ -665,7 +665,7 @@ class WheelBuilder(object): +@@ -167,7 +167,8 @@ def message_about_scripts_not_on_PATH(scripts): + ] + # If an executable sits with sys.executable, we don't warn for it. + # This covers the case of venv invocations without activating the venv. +- not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) ++ executable_loc = os.environ.get("PIP_PYTHON_PATH", sys.executable) ++ not_warn_dirs.append(os.path.normcase(os.path.dirname(executable_loc))) + warn_for = { + parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() + if os.path.normcase(parent_dir) not in not_warn_dirs +@@ -667,8 +668,9 @@ class WheelBuilder(object): + # isolating. Currently, it breaks Python in virtualenvs, because it # relies on site.py to find parts of the standard library outside the # virtualenv. ++ executable_loc = os.environ.get('PIP_PYTHON_PATH', sys.executable) return [ - sys.executable, '-u', '-c', -+ os.environ.get('PIP_PYTHON_PATH', sys.executable), '-u', '-c', ++ executable_loc, '-u', '-c', SETUPTOOLS_SHIM % req.setup_py ] + list(self.global_options) -diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py -index 793dd1cb..426880e9 100644 ---- a/pipenv/patched/pip/_internal/index.py -+++ b/pipenv/patched/pip/_internal/index.py -@@ -477,7 +477,10 @@ class PackageFinder(object): - - page_versions = [] - for page in self._get_pages(url_locations, project_name): -- logger.debug('Analyzing links from page %s', page.url) -+ try: -+ logger.debug('Analyzing links from page %s', page.url) -+ except AttributeError: -+ continue - with indent_log(): - page_versions.extend( - self._package_versions(page.links, search) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 1db5ef44..3799ccf4 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -12,30 +12,56 @@ index 1fa3805..c0ecec8 100644 install_req_from_editable, ) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py -index 28da51f..de9b435 100644 +index 28da51f..c466ef0 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py -@@ -1,12 +1,13 @@ +@@ -1,45 +1,55 @@ # -*- coding=utf-8 -*- - import importlib +-import importlib -import pip -+from pip_shims import pip_version - import pkg_resources +-import pkg_resources ++__all__ = [ ++ "InstallRequirement", ++ "parse_requirements", ++ "RequirementSet", ++ "user_cache_dir", ++ "FAVORITE_HASH", ++ "is_file_url", ++ "url_to_path", ++ "PackageFinder", ++ "FormatControl", ++ "Wheel", ++ "Command", ++ "cmdoptions", ++ "get_installed_distributions", ++ "PyPI", ++ "SafeFileCache", ++ "InstallationError", ++ "parse_version", ++ "pip_version", ++ "install_req_from_editable", ++ "install_req_from_line", ++ "user_cache_dir" ++] -def do_import(module_path, subimport=None, old_path=None): -+def do_import(module_path, subimport=None, old_path=None, vendored_name=None): - old_path = old_path or module_path +- old_path = old_path or module_path - prefixes = ["pip._internal", "pip"] -+ prefix = vendored_name if vendored_name else "pip" -+ prefixes = ["{0}._internal".format(prefix), "{0}".format(prefix)] - paths = [module_path, old_path] - search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] - package = subimport if subimport else None -@@ -21,25 +22,28 @@ def do_import(module_path, subimport=None, old_path=None): - return getattr(imported, package) - - +- paths = [module_path, old_path] +- search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] +- package = subimport if subimport else None +- for to_import in search_order: +- if not subimport: +- to_import, _, package = to_import.rpartition(".") +- try: +- imported = importlib.import_module(to_import) +- except ImportError: +- continue +- else: +- return getattr(imported, package) +- +- -InstallRequirement = do_import('req.req_install', 'InstallRequirement') -parse_requirements = do_import('req.req_file', 'parse_requirements') -RequirementSet = do_import('req.req_set', 'RequirementSet') @@ -50,34 +76,38 @@ index 28da51f..de9b435 100644 -cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions') -get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') -PyPI = do_import('models.index', 'PyPI') -+InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name="notpip") -+parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name="notpip") -+RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name="notpip") -+user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name="notpip") -+FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name="notpip") -+is_file_url = do_import('download', 'is_file_url', vendored_name="notpip") -+url_to_path = do_import('download', 'url_to_path', vendored_name="notpip") -+PackageFinder = do_import('index', 'PackageFinder', vendored_name="notpip") -+FormatControl = do_import('index', 'FormatControl', vendored_name="notpip") -+Wheel = do_import('wheel', 'Wheel', vendored_name="notpip") -+Command = do_import('cli.base_command', 'Command', old_path='basecommand', vendored_name="notpip") -+cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions', vendored_name="notpip") -+get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils', vendored_name="notpip") -+PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') -+SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') -+InstallationError = do_import('exceptions', 'InstallationError', vendored_name='notpip') ++from pipenv.vendor.appdirs import user_cache_dir ++from pip_shims.shims import ( ++ InstallRequirement, ++ parse_requirements, ++ RequirementSet, ++ FAVORITE_HASH, ++ is_file_url, ++ url_to_path, ++ PackageFinder, ++ FormatControl, ++ Wheel, ++ Command, ++ cmdoptions, ++ get_installed_distributions, ++ PyPI, ++ SafeFileCache, ++ InstallationError, ++ parse_version, ++ pip_version, ++) # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('18.1'): -+if pkg_resources.parse_version(pip_version) < pkg_resources.parse_version('18.1'): ++if parse_version(pip_version) < parse_version('18.1'): install_req_from_line = InstallRequirement.from_line install_req_from_editable = InstallRequirement.from_editable else: - install_req_from_line = do_import('req.constructors', 'install_req_from_line') - install_req_from_editable = do_import('req.constructors', 'install_req_from_editable') -+ install_req_from_line = do_import('req.constructors', 'install_req_from_line', vendored_name="notpip") -+ install_req_from_editable = do_import('req.constructors', 'install_req_from_editable', vendored_name="notpip") -+ ++ from pip_shims.shims import ( ++ install_req_from_editable, install_req_from_line ++ ) diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py index 08dabe1..480ad1e 100644 --- a/pipenv/patched/piptools/repositories/local.py @@ -92,7 +122,7 @@ index 08dabe1..480ad1e 100644 else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index bf69803..a1a3906 100644 +index bf69803..31b85b9 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,7 +1,7 @@ @@ -104,7 +134,7 @@ index bf69803..a1a3906 100644 import hashlib import os from contextlib import contextmanager -@@ -15,13 +15,23 @@ from .._compat import ( +@@ -15,13 +15,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -113,11 +143,10 @@ index bf69803..a1a3906 100644 + InstallRequirement, + SafeFileCache ) -+os.environ["PIP_SHIMS_BASE_MODULE"] = str("notpip") ++os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") +from pip_shims.shims import do_import, VcsSupport, WheelCache +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet, Specifier -+from packaging.markers import Op, Value, Variable, Marker +InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) +from pip._internal.resolve import Resolver as PipResolver + @@ -128,11 +157,11 @@ index bf69803..a1a3906 100644 -from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, -+ make_install_requirement, clean_requires_python) ++ make_install_requirement, clean_requires_python) from .base import BaseRepository try: -@@ -31,10 +41,44 @@ except ImportError: +@@ -31,10 +40,44 @@ except ImportError: def RequirementTracker(): yield @@ -181,7 +210,7 @@ index bf69803..a1a3906 100644 class PyPIRepository(BaseRepository): -@@ -46,8 +90,9 @@ class PyPIRepository(BaseRepository): +@@ -46,8 +89,9 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -192,7 +221,7 @@ index bf69803..a1a3906 100644 self.pip_options = pip_options index_urls = [pip_options.index_url] + pip_options.extra_index_urls -@@ -73,6 +118,10 @@ class PyPIRepository(BaseRepository): +@@ -73,6 +117,10 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -203,7 +232,7 @@ index bf69803..a1a3906 100644 # Setup file paths self.freshen_build_caches() -@@ -113,10 +162,13 @@ class PyPIRepository(BaseRepository): +@@ -113,10 +161,13 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -219,7 +248,7 @@ index bf69803..a1a3906 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,25 +178,86 @@ class PyPIRepository(BaseRepository): +@@ -126,25 +177,87 @@ class PyPIRepository(BaseRepository): # Turn the candidate into a pinned InstallRequirement return make_install_requirement( @@ -236,8 +265,7 @@ index bf69803..a1a3906 100644 + def gen(ireq): + if self.DEFAULT_INDEX_URL not in self.finder.index_urls: + return - -- def resolve_reqs(self, download_dir, ireq, wheel_cache): ++ + url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) + releases = self.session.get(url).json()['releases'] + @@ -266,7 +294,8 @@ index bf69803..a1a3906 100644 + try: + if ireq not in self._json_dep_cache: + self._json_dep_cache[ireq] = [g for g in gen(ireq)] -+ + +- def resolve_reqs(self, download_dir, ireq, wheel_cache): + return set(self._json_dep_cache[ireq]) + except Exception: + return set() @@ -291,6 +320,7 @@ index bf69803..a1a3906 100644 + dist = None + ireq.isolated = False + ireq._wheel_cache = wheel_cache ++ try: from pip._internal.operations.prepare import RequirementPreparer - from pip._internal.resolve import Resolver as PipResolver @@ -330,7 +360,7 @@ index bf69803..a1a3906 100644 } resolver = None preparer = None -@@ -177,15 +291,98 @@ class PyPIRepository(BaseRepository): +@@ -177,15 +291,109 @@ class PyPIRepository(BaseRepository): resolver_kwargs['preparer'] = preparer reqset = RequirementSet() ireq.is_direct = True @@ -339,9 +369,21 @@ index bf69803..a1a3906 100644 resolver = PipResolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() +- reqset.cleanup_files() - return set(results) ++ cleanup_fn = getattr(reqset, "cleanup_files", None) ++ if cleanup_fn is not None: ++ try: ++ cleanup_fn() ++ except OSError: ++ pass ++ ++ if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): ++ if ireq.editable: ++ self._source_dir = TemporaryDirectory(fs_str("source")) ++ ireq.ensure_has_source_dir(self.source_dir) ++ + if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): + # Collect setup_requires info from local eggs. + # Do this after we call the preparer on these reqs to make sure their @@ -432,7 +474,7 @@ index bf69803..a1a3906 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -200,6 +397,7 @@ class PyPIRepository(BaseRepository): +@@ -200,6 +408,7 @@ class PyPIRepository(BaseRepository): # If a download_dir is passed, pip will unnecessarely # archive the entire source directory download_dir = None @@ -440,7 +482,7 @@ index bf69803..a1a3906 100644 elif ireq.link and not ireq.link.is_artifact: # No download_dir for VCS sources. This also works around pip # using git-checkout-index, which gets rid of the .git dir. -@@ -214,7 +412,8 @@ class PyPIRepository(BaseRepository): +@@ -214,7 +423,8 @@ class PyPIRepository(BaseRepository): wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control) prev_tracker = os.environ.get('PIP_REQ_TRACKER') try: @@ -450,7 +492,7 @@ index bf69803..a1a3906 100644 finally: if 'PIP_REQ_TRACKER' in os.environ: if prev_tracker: -@@ -236,6 +435,10 @@ class PyPIRepository(BaseRepository): +@@ -236,6 +446,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -461,7 +503,7 @@ index bf69803..a1a3906 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -243,24 +446,22 @@ class PyPIRepository(BaseRepository): +@@ -243,24 +457,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. diff --git a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch new file mode 100644 index 00000000..7e42237f --- /dev/null +++ b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch @@ -0,0 +1,88 @@ +diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py +index d15aeb97..cf6f91c8 100644 +--- a/pipenv/vendor/delegator.py ++++ b/pipenv/vendor/delegator.py +@@ -7,6 +7,8 @@ import locale + import errno + + from pexpect.popen_spawn import PopenSpawn ++import pexpect ++pexpect.EOF.__module__ = "pexpect.exceptions" + + # Include `unicode` in STR_TYPES for Python 2.X + try: +@@ -110,8 +112,11 @@ class Command(object): + if self.subprocess.before: + result += self.subprocess.before + +- if self.subprocess.after: +- result += self.subprocess.after ++ if self.subprocess.after and self.subprocess.after is not pexpect.EOF: ++ try: ++ result += self.subprocess.after ++ except (pexpect.EOF, pexpect.TIMEOUT): ++ pass + + result += self.subprocess.read() + return result +@@ -178,6 +183,7 @@ class Command(object): + # Use subprocess. + if self.blocking: + popen_kwargs = self._default_popen_kwargs.copy() ++ del popen_kwargs["stdin"] + popen_kwargs["universal_newlines"] = not binary + if cwd: + popen_kwargs["cwd"] = cwd +@@ -205,7 +211,10 @@ class Command(object): + if self.blocking: + raise RuntimeError("expect can only be used on non-blocking commands.") + +- self.subprocess.expect(pattern=pattern, timeout=timeout) ++ try: ++ self.subprocess.expect(pattern=pattern, timeout=timeout) ++ except pexpect.EOF: ++ pass + + def send(self, s, end=os.linesep, signal=False): + """Sends the given string or signal to std_in.""" +@@ -234,14 +243,25 @@ class Command(object): + """Blocks until process is complete.""" + if self._uses_subprocess: + # consume stdout and stderr +- try: +- stdout, stderr = self.subprocess.communicate() +- self.__out = stdout +- self.__err = stderr +- except ValueError: +- pass # Don't read from finished subprocesses. ++ if self.blocking: ++ try: ++ stdout, stderr = self.subprocess.communicate() ++ self.__out = stdout ++ self.__err = stderr ++ except ValueError: ++ pass # Don't read from finished subprocesses. ++ else: ++ self.subprocess.stdin.close() ++ self.std_out.close() ++ self.std_err.close() ++ self.subprocess.wait() + else: +- self.subprocess.wait() ++ self.subprocess.sendeof() ++ try: ++ self.subprocess.wait() ++ finally: ++ if self.subprocess.proc.stdout: ++ self.subprocess.proc.stdout.close() + + def pipe(self, command, timeout=None, cwd=None): + """Runs the current command and passes its output to the next +@@ -263,7 +283,6 @@ class Command(object): + c.run(block=False, cwd=cwd) + if data: + c.send(data) +- c.subprocess.sendeof() + c.block() + return c + diff --git a/tasks/vendoring/patches/vendor/passa-close-session.patch b/tasks/vendoring/patches/vendor/passa-close-session.patch new file mode 100644 index 00000000..38846bef --- /dev/null +++ b/tasks/vendoring/patches/vendor/passa-close-session.patch @@ -0,0 +1,12 @@ +diff --git a/pipenv/vendor/passa/internals/dependencies.py b/pipenv/vendor/passa/internals/dependencies.py +index 53b19b17..358cc33b 100644 +--- a/pipenv/vendor/passa/internals/dependencies.py ++++ b/pipenv/vendor/passa/internals/dependencies.py +@@ -154,6 +154,7 @@ def _get_dependencies_from_json(ireq, sources): + return dependencies + except Exception as e: + print("unable to read dependencies via {0} ({1})".format(url, e)) ++ session.close() + return + + diff --git a/tasks/vendoring/patches/vendor/vistir-imports.patch b/tasks/vendoring/patches/vendor/vistir-imports.patch index f93e7959..673efad8 100644 --- a/tasks/vendoring/patches/vendor/vistir-imports.patch +++ b/tasks/vendoring/patches/vendor/vistir-imports.patch @@ -1,25 +1,3 @@ -diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py -index 1f1b7a96..0c865fe6 100644 ---- a/pipenv/vendor/vistir/compat.py -+++ b/pipenv/vendor/vistir/compat.py -@@ -30,7 +30,7 @@ else: - from pathlib2 import Path - - if sys.version_info < (3, 3): -- from backports.shutil_get_terminal_size import get_terminal_size -+ from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size - from .backports.tempfile import NamedTemporaryFile - else: - from tempfile import NamedTemporaryFile -@@ -39,7 +39,7 @@ else: - try: - from weakref import finalize - except ImportError: -- from backports.weakref import finalize -+ from pipenv.vendor.backports.weakref import finalize - - try: - from functools import partialmethod diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index 483a479a..43470a6e 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py @@ -33,3 +11,30 @@ index 483a479a..43470a6e 100644 __all__ = ["finalize", "NamedTemporaryFile"] +diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py +index 9ae33fdc..ec3b65cb 100644 +--- a/pipenv/vendor/vistir/compat.py ++++ b/pipenv/vendor/vistir/compat.py +@@ -31,11 +31,11 @@ if sys.version_info >= (3, 5): + from functools import lru_cache + else: + from pathlib2 import Path +- from backports.functools_lru_cache import lru_cache ++ from pipenv.vendor.backports.functools_lru_cache import lru_cache + + from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile + if sys.version_info < (3, 3): +- from backports.shutil_get_terminal_size import get_terminal_size ++ from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size + NamedTemporaryFile = _NamedTemporaryFile + else: + from tempfile import NamedTemporaryFile +@@ -44,7 +44,7 @@ else: + try: + from weakref import finalize + except ImportError: +- from backports.weakref import finalize ++ from pipenv.vendor.backports.weakref import finalize + + try: + from functools import partialmethod diff --git a/tasks/vendoring/patches/vendor/vistir-spin-colorama.patch b/tasks/vendoring/patches/vendor/vistir-spin-colorama.patch new file mode 100644 index 00000000..69e4cac7 --- /dev/null +++ b/tasks/vendoring/patches/vendor/vistir-spin-colorama.patch @@ -0,0 +1,28 @@ +diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py +index 2a848922..57a90277 100644 +--- a/pipenv/vendor/vistir/spin.py ++++ b/pipenv/vendor/vistir/spin.py +@@ -5,6 +5,7 @@ import os + import signal + import sys + ++import colorama + import cursor + import six + +@@ -31,6 +32,7 @@ CLEAR_LINE = chr(27) + "[K" + + class DummySpinner(object): + def __init__(self, text="", **kwargs): ++ colorama.init() + self.text = to_native_string(text) + self.stdout = kwargs.get("stdout", sys.stdout) + self.stderr = kwargs.get("stderr", sys.stderr) +@@ -112,7 +114,6 @@ class VistirSpinner(base_obj): + """ + + self.handler = handler +- import colorama + colorama.init() + sigmap = {} + if handler: diff --git a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch new file mode 100644 index 00000000..a1d27cd3 --- /dev/null +++ b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch @@ -0,0 +1,62 @@ +diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py +index d01fb98e..06b8b621 100644 +--- a/pipenv/vendor/yaspin/core.py ++++ b/pipenv/vendor/yaspin/core.py +@@ -16,6 +16,9 @@ import sys + import threading + import time + ++import colorama ++import cursor ++ + from .base_spinner import default_spinner + from .compat import PY2, basestring, builtin_str, bytes, iteritems, str + from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS +@@ -23,6 +26,9 @@ from .helpers import to_unicode + from .termcolor import colored + + ++colorama.init() ++ ++ + class Yaspin(object): + """Implements a context manager that spawns a thread + to write spinner frames into a tty (stdout) during +@@ -369,11 +375,14 @@ class Yaspin(object): + # SIGKILL cannot be caught or ignored, and the receiving + # process cannot perform any clean-up upon receiving this + # signal. +- if signal.SIGKILL in self._sigmap.keys(): +- raise ValueError( +- "Trying to set handler for SIGKILL signal. " +- "SIGKILL cannot be cought or ignored in POSIX systems." +- ) ++ try: ++ if signal.SIGKILL in self._sigmap.keys(): ++ raise ValueError( ++ "Trying to set handler for SIGKILL signal. " ++ "SIGKILL cannot be cought or ignored in POSIX systems." ++ ) ++ except AttributeError: ++ pass + + for sig, sig_handler in iteritems(self._sigmap): + # A handler for a particular signal, once set, remains +@@ -521,14 +530,12 @@ class Yaspin(object): + + @staticmethod + def _hide_cursor(): +- sys.stdout.write("\033[?25l") +- sys.stdout.flush() ++ cursor.hide() + + @staticmethod + def _show_cursor(): +- sys.stdout.write("\033[?25h") +- sys.stdout.flush() ++ cursor.show() + + @staticmethod + def _clear_line(): +- sys.stdout.write("\033[K") ++ sys.stdout.write(chr(27) + "[K") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 12f27342..0ab0ab22 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,5 +1,6 @@ import json import os +import sys import warnings import pytest @@ -7,13 +8,13 @@ import pytest from pipenv._compat import TemporaryDirectory, Path from pipenv.vendor import delegator from pipenv.vendor import requests -from pipenv.vendor import six from pipenv.vendor import toml from pytest_pypi.app import prepare_packages as prepare_pypi_packages +from vistir.compat import ResourceWarning, fs_str +from vistir.path import mkdir_p -if six.PY2: - class ResourceWarning(Warning): - pass + +warnings.simplefilter("default", category=ResourceWarning) HAS_WARNED_GITHUB = False @@ -25,8 +26,8 @@ def check_internet(): resp = requests.get('http://httpbin.org/ip', timeout=1.0) resp.raise_for_status() except Exception: - warnings.warn('Cannot connect to HTTPBin...', ResourceWarning) - warnings.warn('Will skip tests requiring Internet', ResourceWarning) + warnings.warn('Cannot connect to HTTPBin...', RuntimeWarning) + warnings.warn('Will skip tests requiring Internet', RuntimeWarning) return False return True @@ -46,18 +47,15 @@ def check_github_ssh(): global HAS_WARNED_GITHUB if not res and not HAS_WARNED_GITHUB: warnings.warn( - 'Cannot connect to GitHub via SSH', ResourceWarning + 'Cannot connect to GitHub via SSH', RuntimeWarning ) warnings.warn( - 'Will skip tests requiring SSH access to GitHub', ResourceWarning + 'Will skip tests requiring SSH access to GitHub', RuntimeWarning ) HAS_WARNED_GITHUB = True return res -WE_HAVE_INTERNET = check_internet() -WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() - TESTS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PYPI_VENDOR_DIR = os.path.join(TESTS_ROOT, 'pypi') prepare_pypi_packages(PYPI_VENDOR_DIR) @@ -70,24 +68,71 @@ def pytest_runtest_setup(item): pytest.skip('requires github ssh') +@pytest.fixture +def pathlib_tmpdir(request, tmpdir): + yield Path(str(tmpdir)) + try: + tmpdir.remove(ignore_errors=True) + except Exception: + pass + + +# Borrowed from pip's test runner filesystem isolation +@pytest.fixture(autouse=True) +def isolate(pathlib_tmpdir): + """ + Isolate our tests so that things like global configuration files and the + like do not affect our test results. + We use an autouse function scoped fixture because we want to ensure that + every test has it's own isolated home directory. + """ + + # Create a directory to use as our home location. + home_dir = os.path.join(str(pathlib_tmpdir), "home") + os.makedirs(home_dir) + mkdir_p(os.path.join(home_dir, ".config", "git")) + with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: + fp.write( + b"[user]\n\tname = pipenv\n\temail = pipenv@pipenv.org\n" + ) + os.environ["GIT_CONFIG_NOSYSTEM"] = fs_str("1") + os.environ["GIT_AUTHOR_NAME"] = fs_str("pipenv") + os.environ["GIT_AUTHOR_EMAIL"] = fs_str("pipenv@pipenv.org") + mkdir_p(os.path.join(home_dir, ".virtualenvs")) + os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) + + +WE_HAVE_INTERNET = check_internet() +WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() + + class _PipenvInstance(object): """An instance of a Pipenv Project...""" - def __init__(self, pypi=None, pipfile=True, chdir=False): + def __init__(self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None): self.pypi = pypi self.original_umask = os.umask(0o007) self.original_dir = os.path.abspath(os.curdir) - self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') - path = Path(self._path.name) - try: - self.path = str(path.resolve()) - except OSError: - self.path = str(path.absolute()) + os.environ["PIPENV_NOSPIN"] = fs_str("1") + os.environ["CI"] = fs_str("1") + warnings.simplefilter("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") + path = os.environ.get("PIPENV_PROJECT_DIR", None) + if not path: + self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + path = Path(self._path.name) + try: + self.path = str(path.resolve()) + except OSError: + self.path = str(path.absolute()) + else: + self._path = None + self.path = path # set file creation perms self.pipfile_path = None self.chdir = chdir if self.pypi: - os.environ['PIPENV_TEST_INDEX'] = '{0}/simple'.format(self.pypi.url) + os.environ['PIPENV_TEST_INDEX'] = fs_str('{0}/simple'.format(self.pypi.url)) if pipfile: p_path = os.sep.join([self.path, 'Pipfile']) @@ -98,9 +143,10 @@ class _PipenvInstance(object): self.pipfile_path = p_path def __enter__(self): - os.environ['PIPENV_DONT_USE_PYENV'] = '1' - os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' - os.environ['PIPENV_VENV_IN_PROJECT'] = '1' + os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1') + os.environ['PIPENV_IGNORE_VIRTUALENVS'] = fs_str('1') + os.environ['PIPENV_VENV_IN_PROJECT'] = fs_str('1') + os.environ['PIPENV_NOSPIN'] = fs_str('1') if self.chdir: os.chdir(self.path) return self @@ -110,21 +156,21 @@ class _PipenvInstance(object): if self.chdir: os.chdir(self.original_dir) self.path = None - try: - self._path.cleanup() - except OSError as e: - _warn_msg = warn_msg.format(e) - warnings.warn(_warn_msg, ResourceWarning) - finally: - os.umask(self.original_umask) + if self._path: + try: + self._path.cleanup() + except OSError as e: + _warn_msg = warn_msg.format(e) + warnings.warn(_warn_msg, ResourceWarning) + os.umask(self.original_umask) def pipenv(self, cmd, block=True): if self.pipfile_path: - os.environ['PIPENV_PIPFILE'] = self.pipfile_path + os.environ['PIPENV_PIPFILE'] = fs_str(self.pipfile_path) # a bit of a hack to make sure the virtualenv is created with TemporaryDirectory(prefix='pipenv-', suffix='-cache') as tempdir: - os.environ['PIPENV_CACHE_DIR'] = tempdir.name + os.environ['PIPENV_CACHE_DIR'] = fs_str(tempdir.name) c = delegator.run('pipenv {0}'.format(cmd), block=block) if 'PIPENV_CACHE_DIR' in os.environ: del os.environ['PIPENV_CACHE_DIR'] @@ -162,18 +208,16 @@ class _PipenvInstance(object): @pytest.fixture() def PipenvInstance(): - return _PipenvInstance + yield _PipenvInstance -@pytest.fixture(scope='module') -def pip_src_dir(request): +@pytest.fixture(autouse=True) +def pip_src_dir(request, pathlib_tmpdir): old_src_dir = os.environ.get('PIP_SRC', '') - new_src_dir = TemporaryDirectory(prefix='pipenv-', suffix='-testsrc') - os.environ['PIP_SRC'] = new_src_dir.name + os.environ['PIP_SRC'] = pathlib_tmpdir.as_posix() def finalize(): - new_src_dir.cleanup() - os.environ['PIP_SRC'] = old_src_dir + os.environ['PIP_SRC'] = fs_str(old_src_dir) request.addfinalizer(finalize) return request diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 1f1719d0..7743804d 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -37,9 +37,9 @@ flask = "==0.12.2" """.strip() f.write(contents) - req_list = ("requests==2.14.0") + req_list = ("requests==2.14.0",) - dev_req_list = ("flask==0.12.2") + dev_req_list = ("flask==0.12.2",) c = p.pipenv('lock -r') d = p.pipenv('lock -r -d') diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 98c85602..40977ede 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -4,6 +4,7 @@ import pytest from mock import patch, Mock from first import first import pipenv.utils +import pythonfinder.utils # Pipfile format <-> requirements.txt format. @@ -215,13 +216,13 @@ class TestUtils: ), ], ) - @patch("delegator.run") + # @patch(".vendor.pythonfinder.utils.get_python_version") def test_python_version_output_variants( - self, mocked_delegator, version_output, version + self, monkeypatch, version_output, version ): - run_ret = Mock() - run_ret.out = version_output - mocked_delegator.return_value = run_ret + def mock_version(path): + return version_output.split()[1] + monkeypatch.setattr("pipenv.vendor.pythonfinder.utils.get_python_version", mock_version) assert pipenv.utils.python_version("some/path") == version @pytest.mark.utils